diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/_virtualenv.py b/bioagent-bench/.venv/lib/python3.14/site-packages/_virtualenv.py new file mode 100644 index 0000000000000000000000000000000000000000..6c1f22640d567f04c9880b90566436ee5ba14400 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/_virtualenv.py @@ -0,0 +1,101 @@ +"""Patches that are applied at runtime to the virtual environment.""" + +import os +import sys + +VIRTUALENV_PATCH_FILE = os.path.join(__file__) + + +def patch_dist(dist): + """ + Distutils allows user to configure some arguments via a configuration file: + https://docs.python.org/3.11/install/index.html#distutils-configuration-files. + + Some of this arguments though don't make sense in context of the virtual environment files, let's fix them up. + """ # noqa: D205 + # we cannot allow some install config as that would get packages installed outside of the virtual environment + old_parse_config_files = dist.Distribution.parse_config_files + + def parse_config_files(self, *args, **kwargs): + result = old_parse_config_files(self, *args, **kwargs) + install = self.get_option_dict("install") + + if "prefix" in install: # the prefix governs where to install the libraries + install["prefix"] = VIRTUALENV_PATCH_FILE, os.path.abspath(sys.prefix) + for base in ("purelib", "platlib", "headers", "scripts", "data"): + key = f"install_{base}" + if key in install: # do not allow global configs to hijack venv paths + install.pop(key, None) + return result + + dist.Distribution.parse_config_files = parse_config_files + + +# Import hook that patches some modules to ignore configuration values that break package installation in case +# of virtual environments. +_DISTUTILS_PATCH = "distutils.dist", "setuptools.dist" +# https://docs.python.org/3/library/importlib.html#setting-up-an-importer + + +class _Finder: + """A meta path finder that allows patching the imported distutils modules.""" + + fullname = None + + # lock[0] is threading.Lock(), but initialized lazily to avoid importing threading very early at startup, + # because there are gevent-based applications that need to be first to import threading by themselves. + # See https://github.com/pypa/virtualenv/issues/1895 for details. + lock = [] # noqa: RUF012 + + def find_spec(self, fullname, path, target=None): # noqa: ARG002 + if fullname in _DISTUTILS_PATCH and self.fullname is None: + # initialize lock[0] lazily + if len(self.lock) == 0: + import threading + + lock = threading.Lock() + # there is possibility that two threads T1 and T2 are simultaneously running into find_spec, + # observing .lock as empty, and further going into hereby initialization. However due to the GIL, + # list.append() operation is atomic and this way only one of the threads will "win" to put the lock + # - that every thread will use - into .lock[0]. + # https://docs.python.org/3/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe + self.lock.append(lock) + + from functools import partial + from importlib.util import find_spec + + with self.lock[0]: + self.fullname = fullname + try: + spec = find_spec(fullname, path) + if spec is not None: + # https://www.python.org/dev/peps/pep-0451/#how-loading-will-work + is_new_api = hasattr(spec.loader, "exec_module") + func_name = "exec_module" if is_new_api else "load_module" + old = getattr(spec.loader, func_name) + func = self.exec_module if is_new_api else self.load_module + if old is not func: + try: # noqa: SIM105 + setattr(spec.loader, func_name, partial(func, old)) + except AttributeError: + pass # C-Extension loaders are r/o such as zipimporter with <3.7 + return spec + finally: + self.fullname = None + return None + + @staticmethod + def exec_module(old, module): + old(module) + if module.__name__ in _DISTUTILS_PATCH: + patch_dist(module) + + @staticmethod + def load_module(old, name): + module = old(name) + if module.__name__ in _DISTUTILS_PATCH: + patch_dist(module) + return module + + +sys.meta_path.insert(0, _Finder()) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/INSTALLER b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/METADATA b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..3977fcdf1288a0be457288660b8184d580029d0c --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/METADATA @@ -0,0 +1,262 @@ +Metadata-Version: 2.4 +Name: aiohttp +Version: 3.13.5 +Summary: Async http client/server framework (asyncio) +Maintainer-email: aiohttp team +License: Apache-2.0 AND MIT +Project-URL: Homepage, https://github.com/aio-libs/aiohttp +Project-URL: Chat: Matrix, https://matrix.to/#/#aio-libs:matrix.org +Project-URL: Chat: Matrix Space, https://matrix.to/#/#aio-libs-space:matrix.org +Project-URL: CI: GitHub Actions, https://github.com/aio-libs/aiohttp/actions?query=workflow%3ACI +Project-URL: Coverage: codecov, https://codecov.io/github/aio-libs/aiohttp +Project-URL: Docs: Changelog, https://docs.aiohttp.org/en/stable/changes.html +Project-URL: Docs: RTD, https://docs.aiohttp.org +Project-URL: GitHub: issues, https://github.com/aio-libs/aiohttp/issues +Project-URL: GitHub: repo, https://github.com/aio-libs/aiohttp +Classifier: Development Status :: 5 - Production/Stable +Classifier: Framework :: AsyncIO +Classifier: Intended Audience :: Developers +Classifier: Operating System :: POSIX +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: Microsoft :: Windows +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Topic :: Internet :: WWW/HTTP +Requires-Python: >=3.9 +Description-Content-Type: text/x-rst +License-File: LICENSE.txt +License-File: vendor/llhttp/LICENSE +Requires-Dist: aiohappyeyeballs>=2.5.0 +Requires-Dist: aiosignal>=1.4.0 +Requires-Dist: async-timeout<6.0,>=4.0; python_version < "3.11" +Requires-Dist: attrs>=17.3.0 +Requires-Dist: frozenlist>=1.1.1 +Requires-Dist: multidict<7.0,>=4.5 +Requires-Dist: propcache>=0.2.0 +Requires-Dist: yarl<2.0,>=1.17.0 +Provides-Extra: speedups +Requires-Dist: aiodns>=3.3.0; extra == "speedups" +Requires-Dist: Brotli>=1.2; platform_python_implementation == "CPython" and extra == "speedups" +Requires-Dist: brotlicffi>=1.2; platform_python_implementation != "CPython" and extra == "speedups" +Requires-Dist: backports.zstd; (platform_python_implementation == "CPython" and python_version < "3.14") and extra == "speedups" +Dynamic: license-file + +================================== +Async http client/server framework +================================== + +.. image:: https://raw.githubusercontent.com/aio-libs/aiohttp/master/docs/aiohttp-plain.svg + :height: 64px + :width: 64px + :alt: aiohttp logo + +| + +.. image:: https://github.com/aio-libs/aiohttp/workflows/CI/badge.svg + :target: https://github.com/aio-libs/aiohttp/actions?query=workflow%3ACI + :alt: GitHub Actions status for master branch + +.. image:: https://codecov.io/gh/aio-libs/aiohttp/branch/master/graph/badge.svg + :target: https://codecov.io/gh/aio-libs/aiohttp + :alt: codecov.io status for master branch + +.. image:: https://badge.fury.io/py/aiohttp.svg + :target: https://pypi.org/project/aiohttp + :alt: Latest PyPI package version + +.. image:: https://img.shields.io/pypi/dm/aiohttp + :target: https://pypistats.org/packages/aiohttp + :alt: Downloads count + +.. image:: https://readthedocs.org/projects/aiohttp/badge/?version=latest + :target: https://docs.aiohttp.org/ + :alt: Latest Read The Docs + +.. image:: https://img.shields.io/endpoint?url=https://codspeed.io/badge.json + :target: https://codspeed.io/aio-libs/aiohttp + :alt: Codspeed.io status for aiohttp + + +Key Features +============ + +- Supports both client and server side of HTTP protocol. +- Supports both client and server Web-Sockets out-of-the-box and avoids + Callback Hell. +- Provides Web-server with middleware and pluggable routing. + + +Getting started +=============== + +Client +------ + +To get something from the web: + +.. code-block:: python + + import aiohttp + import asyncio + + async def main(): + + async with aiohttp.ClientSession() as session: + async with session.get('http://python.org') as response: + + print("Status:", response.status) + print("Content-type:", response.headers['content-type']) + + html = await response.text() + print("Body:", html[:15], "...") + + asyncio.run(main()) + +This prints: + +.. code-block:: + + Status: 200 + Content-type: text/html; charset=utf-8 + Body: ... + +Coming from `requests `_ ? Read `why we need so many lines `_. + +Server +------ + +An example using a simple server: + +.. code-block:: python + + # examples/server_simple.py + from aiohttp import web + + async def handle(request): + name = request.match_info.get('name', "Anonymous") + text = "Hello, " + name + return web.Response(text=text) + + async def wshandle(request): + ws = web.WebSocketResponse() + await ws.prepare(request) + + async for msg in ws: + if msg.type == web.WSMsgType.text: + await ws.send_str("Hello, {}".format(msg.data)) + elif msg.type == web.WSMsgType.binary: + await ws.send_bytes(msg.data) + elif msg.type == web.WSMsgType.close: + break + + return ws + + + app = web.Application() + app.add_routes([web.get('/', handle), + web.get('/echo', wshandle), + web.get('/{name}', handle)]) + + if __name__ == '__main__': + web.run_app(app) + + +Documentation +============= + +https://aiohttp.readthedocs.io/ + + +Demos +===== + +https://github.com/aio-libs/aiohttp-demos + + +External links +============== + +* `Third party libraries + `_ +* `Built with aiohttp + `_ +* `Powered by aiohttp + `_ + +Feel free to make a Pull Request for adding your link to these pages! + + +Communication channels +====================== + +*aio-libs Discussions*: https://github.com/aio-libs/aiohttp/discussions + +*Matrix*: `#aio-libs:matrix.org `_ + +We support `Stack Overflow +`_. +Please add *aiohttp* tag to your question there. + +Requirements +============ + +- attrs_ +- multidict_ +- yarl_ +- frozenlist_ + +Optionally you may install the aiodns_ library (highly recommended for sake of speed). + +.. _aiodns: https://pypi.python.org/pypi/aiodns +.. _attrs: https://github.com/python-attrs/attrs +.. _multidict: https://pypi.python.org/pypi/multidict +.. _frozenlist: https://pypi.org/project/frozenlist/ +.. _yarl: https://pypi.python.org/pypi/yarl +.. _async-timeout: https://pypi.python.org/pypi/async_timeout + +License +======= + +``aiohttp`` is offered under the Apache 2 license. + + +Keepsafe +======== + +The aiohttp community would like to thank Keepsafe +(https://www.getkeepsafe.com) for its support in the early days of +the project. + + +Source code +=========== + +The latest developer version is available in a GitHub repository: +https://github.com/aio-libs/aiohttp + +Benchmarks +========== + +If you are interested in efficiency, the AsyncIO community maintains a +list of benchmarks on the official wiki: +https://github.com/python/asyncio/wiki/Benchmarks + +-------- + +.. image:: https://img.shields.io/matrix/aio-libs:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat + :target: https://matrix.to/#/%23aio-libs:matrix.org + :alt: Matrix Room — #aio-libs:matrix.org + +.. image:: https://img.shields.io/matrix/aio-libs-space:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs-space%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat + :target: https://matrix.to/#/%23aio-libs-space:matrix.org + :alt: Matrix Space — #aio-libs-space:matrix.org + +.. image:: https://insights.linuxfoundation.org/api/badge/health-score?project=aiohttp + :target: https://insights.linuxfoundation.org/project/aiohttp + :alt: LFX Health Score diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/RECORD b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..25f316db79c7a481936ec4af4890a3700831076e --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/RECORD @@ -0,0 +1,84 @@ +aiohttp-3.13.5.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +aiohttp-3.13.5.dist-info/METADATA,sha256=yVHd0XDIuuar5cgTh5EZI2mDXCREeUCWCZ-VA6gWvUQ,8145 +aiohttp-3.13.5.dist-info/RECORD,, +aiohttp-3.13.5.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +aiohttp-3.13.5.dist-info/WHEEL,sha256=a6EB0SZmvnUxj7CP1sYpBVvB6TTNx-Vzw2sye5KrYcM,190 +aiohttp-3.13.5.dist-info/licenses/LICENSE.txt,sha256=n4DQ2311WpQdtFchcsJw7L2PCCuiFd3QlZhZQu2Uqes,588 +aiohttp-3.13.5.dist-info/licenses/vendor/llhttp/LICENSE,sha256=68qFTgE0zSVtZzYnwgSZ9CV363S6zwi58ltianPJEnc,1105 +aiohttp-3.13.5.dist-info/top_level.txt,sha256=iv-JIaacmTl-hSho3QmphcKnbRRYx1st47yjz_178Ro,8 +aiohttp/.hash/_cparser.pxd.hash,sha256=pjs-sEXNw_eijXGAedwG-BHnlFp8B7sOCgUagIWaU2A,121 +aiohttp/.hash/_find_header.pxd.hash,sha256=_mbpD6vM-CVCKq3ulUvsOAz5Wdo88wrDzfpOsMQaMNA,125 +aiohttp/.hash/_http_parser.pyx.hash,sha256=T47Ily1qaTY3xOHgBgJZE5A_bKruDtCG_droC3m7BD8,125 +aiohttp/.hash/_http_writer.pyx.hash,sha256=DO9V6lTsDngY8e4y5O0ghHRc-RNGq-pVt_dLNHJy7xA,125 +aiohttp/.hash/hdrs.py.hash,sha256=v6IaKbsxjsdQxBzhb5AjP0x_9G3rUe84D7avf7AI4cs,116 +aiohttp/__init__.py,sha256=r1a3wmBz41XCNY0dqSnCcgfqrDYSDYD-_vKBN3Bfy3Y,8302 +aiohttp/_cookie_helpers.py,sha256=_p7y-B8OCAk7FLjByiuwFIpDLGuNoJn3_vixzymAFnE,13659 +aiohttp/_cparser.pxd,sha256=UnbUYCHg4NdXfgyRVYAMv2KTLWClB4P-xCrvtj_r7ew,4295 +aiohttp/_find_header.pxd,sha256=0GfwFCPN2zxEKTO1_MA5sYq2UfzsG8kcV3aTqvwlz3g,68 +aiohttp/_headers.pxi,sha256=n701k28dVPjwRnx5j6LpJhLTfj7dqu2vJt7f0O60Oyg,2007 +aiohttp/_http_parser.cpython-314-x86_64-linux-gnu.so,sha256=IUYlPnjodpfpj5PSYVsS2YCRQaoU_Fa2Vhl9KqtWxY0,2872800 +aiohttp/_http_parser.pyx,sha256=veuwNKb6YzBJoO3nbvrkOCaTzbcROS5MVksUJhDHNgg,30036 +aiohttp/_http_writer.cpython-314-x86_64-linux-gnu.so,sha256=SabbqWzAMlxn_yCyCOTi44eWa8DZ4fc-xFmjtrJoIGA,460736 +aiohttp/_http_writer.pyx,sha256=TXNOC90txk4wnPakxex_ygSbr3xq1UxaCi2T4gv2teY,4690 +aiohttp/_websocket/.hash/mask.pxd.hash,sha256=Y0zBddk_ck3pi9-BFzMcpkcvCKvwvZ4GTtZFb9u1nxQ,128 +aiohttp/_websocket/.hash/mask.pyx.hash,sha256=90owpXYM8_kIma4KUcOxhWSk-Uv4NVMBoCYeFM1B3d0,128 +aiohttp/_websocket/.hash/reader_c.pxd.hash,sha256=5xf3oobk6vx4xbJm-xtZ1_QufB8fYFtLQV2MNdqUc1w,132 +aiohttp/_websocket/__init__.py,sha256=Mar3R9_vBN_Ea4lsW7iTAVXD7OKswKPGqF5xgSyt77k,44 +aiohttp/_websocket/helpers.py,sha256=P-XLv8IUaihKzDenVUqfKU5DJbWE5HvG8uhvUZK8Ic4,5038 +aiohttp/_websocket/mask.cpython-314-x86_64-linux-gnu.so,sha256=88tnUQHEMXouHomgg4mRztaJvg4OxXrmHc5enx9xRn0,242640 +aiohttp/_websocket/mask.pxd,sha256=sBmZ1Amym9kW4Ge8lj1fLZ7mPPya4LzLdpkQExQXv5M,112 +aiohttp/_websocket/mask.pyx,sha256=BHjOtV0O0w7xp9p0LNADRJvGmgfPn9sGeJvSs0fL__4,1397 +aiohttp/_websocket/models.py,sha256=XAzjs_8JYszWXIgZ6R3ZRrF-tX9Q_6LiD49WRYojopM,2121 +aiohttp/_websocket/reader.py,sha256=eC4qS0c5sOeQ2ebAHLaBpIaTVFaSKX79pY2xvh3Pqyw,1030 +aiohttp/_websocket/reader_c.cpython-314-x86_64-linux-gnu.so,sha256=_yZPMlMBz0T9mTaoMvmRV9pYjcdpscnDTy70JhJ-f1M,1783832 +aiohttp/_websocket/reader_c.pxd,sha256=nl_njtDrzlQU0rjgGGjZDB-swguE0tX_bCPobkShVa4,2625 +aiohttp/_websocket/reader_c.py,sha256=V5YtZ2gj2BjE2Q-W9sR_MdAl1VAm1pB7ZjozVJcOpbg,18868 +aiohttp/_websocket/reader_py.py,sha256=V5YtZ2gj2BjE2Q-W9sR_MdAl1VAm1pB7ZjozVJcOpbg,18868 +aiohttp/_websocket/writer.py,sha256=2OvSktPmNh_g20h1cXJt2Xu8u6IvswnPjdur7OwBbJk,11261 +aiohttp/abc.py,sha256=M66F4S6m00bIEn7y4ha_XLTMDmVQ9dPihfOVB0pGfOo,7149 +aiohttp/base_protocol.py,sha256=Tp8cxUPQvv9kUPk3w6lAzk6d2MAzV3scwI_3Go3C47c,3025 +aiohttp/client.py,sha256=JkHGrrJq1pSI7QEJw0jZKx5TLLzM87of8Yx-62m4x6g,58795 +aiohttp/client_exceptions.py,sha256=t6vDg0ZhHDvij6E89Nv473zLpNu3CyMPAp0LqLHOc9I,11744 +aiohttp/client_middleware_digest_auth.py,sha256=rInzAPEmFsaF42oVORmYSx56og1B7jZUSmmIdFD0MQ8,17641 +aiohttp/client_middlewares.py,sha256=kP5N9CMzQPMGPIEydeVUiLUTLsw8Vl8Gr4qAWYdu3vM,1918 +aiohttp/client_proto.py,sha256=eimrsOVNMt_y62gmXZ4JoBI5VZ0kGaW6WnqMkB1cnNQ,12179 +aiohttp/client_reqrep.py,sha256=eEREDrZ0M8ZFTt1wjHduR-P8_sm40K65gNz-iMGYask,53391 +aiohttp/client_ws.py,sha256=1CIjIXwyzOMIYw6AjUES4-qUwbyVHW1seJKQfg_Rta8,15109 +aiohttp/compression_utils.py,sha256=hJ2LXhN2OWukFHm5b78TJFGKcAiL2kthi9Sf5PRYO-U,11738 +aiohttp/connector.py,sha256=0AFVMAx7NwkIbovQ0qecQ3JO2bpEZpxMG_OnGZ3tP3E,69451 +aiohttp/cookiejar.py,sha256=e28ZMQwJ5P0vbPX1OX4Se7-k3zeGvocFEqzGhwpG53k,18922 +aiohttp/formdata.py,sha256=0cxe-naBSFp_NHB1l-TUSbhesU4dpNvwJMzhsLYoSns,6625 +aiohttp/hdrs.py,sha256=2rj5MyA-6yRdYPhW5UKkW4iNWhEAlGIOSBH5D4FmKNE,5111 +aiohttp/helpers.py,sha256=Q1307PCEnWz4RP8crUw8dk58c0YF2Ei3JywkKfRxz5E,30629 +aiohttp/http.py,sha256=8o8j8xH70OWjnfTWA9V44NR785QPxEPrUtzMXiAVpwc,1842 +aiohttp/http_exceptions.py,sha256=Z4IiKYxyfn1Pe947OfqW6NjXoVd12bDI4eKBZKC3Htw,3085 +aiohttp/http_parser.py,sha256=TIvopmjIH3TSTrVTbbqxpBWBpkEuOaNjkzES0-fm4-4,40472 +aiohttp/http_websocket.py,sha256=8VXFKw6KQUEmPg48GtRMB37v0gTK7A0inoxXuDxMZEc,842 +aiohttp/http_writer.py,sha256=4sbiFD_RkE0x_xlAPXwGrL0iLS53L72Qw13hBPaQLls,12508 +aiohttp/log.py,sha256=BbNKx9e3VMIm0xYjZI0IcBBoS7wjdeIeSaiJE7-qK2g,325 +aiohttp/multipart.py,sha256=qMDEHLf64qQcPQLZv4RdwKUDB2L1CC_vN-5Z-P8wDzw,42805 +aiohttp/payload.py,sha256=O6nsYNULL7AeM2cyJ6TYX73ncVnL5xJwt5AegxwMKqw,40874 +aiohttp/payload_streamer.py,sha256=ZzEYyfzcjGWkVkK3XR2pBthSCSIykYvY3Wr5cGQ2eTc,2211 +aiohttp/py.typed,sha256=sow9soTwP9T_gEAQSVh7Gb8855h04Nwmhs2We-JRgZM,7 +aiohttp/pytest_plugin.py,sha256=z4XwqmsKdyJCKxbGiA5kFf90zcedvomqk4RqjZbhKNk,12901 +aiohttp/resolver.py,sha256=gsrfUpFf8iHlcHfJvY-1fiBHW3PRvRVNb5lNZBg3zlY,10031 +aiohttp/streams.py,sha256=i32xpXEjGL2c6ifepnBxBto5y9wXobzzYAIsi8kAkKI,23920 +aiohttp/tcp_helpers.py,sha256=BSadqVWaBpMFDRWnhaaR941N9MiDZ7bdTrxgCb0CW-M,961 +aiohttp/test_utils.py,sha256=7hMcJnOUSkZ54Axgdvveeth5y6R_B2T-0H2xX7mKWYc,23130 +aiohttp/tracing.py,sha256=-6aaW6l0J9uJD45LzR4cijYH0j62pt0U_nn_aVzFku4,14558 +aiohttp/typedefs.py,sha256=wUlqwe9Mw9W8jT3HsYJcYk00qP3EMPz3nTkYXmeNN48,1657 +aiohttp/web.py,sha256=JzSNmejg5G6YeFAnkIgZfytqbU86sNu844yYKmoUpqs,17852 +aiohttp/web_app.py,sha256=lGU_aAMN-h3wy-LTTHi6SeKH8ydt1G51BXcCspgD5ZA,19452 +aiohttp/web_exceptions.py,sha256=aXW8MStpRS7JOFxYDPgPlWojMJP9RhxyAqvq2g0jOCE,10495 +aiohttp/web_fileresponse.py,sha256=arimioSXf0rZQ46H9ejCip4hVN6ri_faFWfuP_XL_us,16374 +aiohttp/web_log.py,sha256=1hhC05PJizS2POVfNxncorNM_fjYHMx2TbcpP6isxkQ,8624 +aiohttp/web_middlewares.py,sha256=sFI0AgeNjdyAjuz92QtMIpngmJSOxrqe2Jfbs4BNUu0,4165 +aiohttp/web_protocol.py,sha256=ZEwjomG6Y2olq2b967UXDezwYnuLJJHDNl48SgHpKJs,27210 +aiohttp/web_request.py,sha256=xNCrsUQSqTB64G1_9QiC9xfQmdgkJwYkGfPBwnfIYQo,30406 +aiohttp/web_response.py,sha256=DofOPOU4be_upiqouWjxfhMLyubtRFvOGnJZbIamH6s,29366 +aiohttp/web_routedef.py,sha256=VT1GAx6BrawoDh5RwBwBu5wSABSqgWwAe74AUCyZAEo,6110 +aiohttp/web_runner.py,sha256=v1G1nKiOOQgFnTSR4IMc6I9ReEFDMaHtMLvO_roDM-A,11786 +aiohttp/web_server.py,sha256=-9WDKUAiR9ll-rSdwXSqG6YjaoW79d1R4y0BGSqgUMA,2888 +aiohttp/web_urldispatcher.py,sha256=9pJ6b3LuujtZuAURNYh0tdhgCJn2ZFrakeEySNwNqhU,44512 +aiohttp/web_ws.py,sha256=lItgmyatkXh0M6EY7JoZnSZkUl6R0wv8B88X4ILqQbU,22739 +aiohttp/worker.py,sha256=o6BL8dbCGYysB76i_yGGvYagc9gyy2PyW1UisiOznjA,8345 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/REQUESTED b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/WHEEL b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..ebc70a69c0555f6054ce8510274071383de66de8 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/WHEEL @@ -0,0 +1,7 @@ +Wheel-Version: 1.0 +Generator: setuptools (82.0.1) +Root-Is-Purelib: false +Tag: cp314-cp314-manylinux_2_17_x86_64 +Tag: cp314-cp314-manylinux2014_x86_64 +Tag: cp314-cp314-manylinux_2_28_x86_64 + diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/licenses/LICENSE.txt b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/licenses/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..e497a322f2091d022983b9c5c043082ab61d1a8c --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/licenses/LICENSE.txt @@ -0,0 +1,13 @@ + Copyright aio-libs contributors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/licenses/vendor/llhttp/LICENSE b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/licenses/vendor/llhttp/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..6c1512dd6bcd6de314673f8c2c7ff3fd6139f28e --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/licenses/vendor/llhttp/LICENSE @@ -0,0 +1,22 @@ +This software is licensed under the MIT License. + +Copyright Fedor Indutny, 2018. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/top_level.txt b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..ee4ba4f3d739e094878215c84eb41ba85c80e4a8 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiohttp-3.13.5.dist-info/top_level.txt @@ -0,0 +1 @@ +aiohttp diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/INSTALLER b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/METADATA b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..03a6f0f7ff91628f60c4a95c4f3acbfa8d654ea8 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/METADATA @@ -0,0 +1,112 @@ +Metadata-Version: 2.4 +Name: aiosignal +Version: 1.4.0 +Summary: aiosignal: a list of registered asynchronous callbacks +Home-page: https://github.com/aio-libs/aiosignal +Maintainer: aiohttp team +Maintainer-email: team@aiohttp.org +License: Apache 2.0 +Project-URL: Chat: Gitter, https://gitter.im/aio-libs/Lobby +Project-URL: CI: GitHub Actions, https://github.com/aio-libs/aiosignal/actions +Project-URL: Coverage: codecov, https://codecov.io/github/aio-libs/aiosignal +Project-URL: Docs: RTD, https://docs.aiosignal.org +Project-URL: GitHub: issues, https://github.com/aio-libs/aiosignal/issues +Project-URL: GitHub: repo, https://github.com/aio-libs/aiosignal +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Development Status :: 5 - Production/Stable +Classifier: Operating System :: POSIX +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: Microsoft :: Windows +Classifier: Framework :: AsyncIO +Requires-Python: >=3.9 +Description-Content-Type: text/x-rst +License-File: LICENSE +Requires-Dist: frozenlist>=1.1.0 +Requires-Dist: typing-extensions>=4.2; python_version < "3.13" +Dynamic: license-file + +========= +aiosignal +========= + +.. image:: https://github.com/aio-libs/aiosignal/workflows/CI/badge.svg + :target: https://github.com/aio-libs/aiosignal/actions?query=workflow%3ACI + :alt: GitHub status for master branch + +.. image:: https://codecov.io/gh/aio-libs/aiosignal/branch/master/graph/badge.svg?flag=pytest + :target: https://codecov.io/gh/aio-libs/aiosignal?flags[0]=pytest + :alt: codecov.io status for master branch + +.. image:: https://badge.fury.io/py/aiosignal.svg + :target: https://pypi.org/project/aiosignal + :alt: Latest PyPI package version + +.. image:: https://readthedocs.org/projects/aiosignal/badge/?version=latest + :target: https://aiosignal.readthedocs.io/ + :alt: Latest Read The Docs + +.. image:: https://img.shields.io/discourse/topics?server=https%3A%2F%2Faio-libs.discourse.group%2F + :target: https://aio-libs.discourse.group/ + :alt: Discourse group for io-libs + +.. image:: https://badges.gitter.im/Join%20Chat.svg + :target: https://gitter.im/aio-libs/Lobby + :alt: Chat on Gitter + +Introduction +============ + +A project to manage callbacks in `asyncio` projects. + +``Signal`` is a list of registered asynchronous callbacks. + +The signal's life-cycle has two stages: after creation its content +could be filled by using standard list operations: ``sig.append()`` +etc. + +After you call ``sig.freeze()`` the signal is *frozen*: adding, removing +and dropping callbacks is forbidden. + +The only available operation is calling the previously registered +callbacks by using ``await sig.send(data)``. + +For concrete usage examples see the `Signals + +section of the `Web Server Advanced +` chapter of the `aiohttp +documentation`_. + + +Installation +------------ + +:: + + $ pip install aiosignal + + +Documentation +============= + +https://aiosignal.readthedocs.io/ + +License +======= + +``aiosignal`` is offered under the Apache 2 license. + +Source code +=========== + +The project is hosted on GitHub_ + +Please file an issue in the `bug tracker +`_ if you have found a bug +or have some suggestions to improve the library. + +.. _GitHub: https://github.com/aio-libs/aiosignal +.. _aiohttp documentation: https://docs.aiohttp.org/ diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/RECORD b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..efc3848a24dc757901735a58eba259a70c89d605 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/RECORD @@ -0,0 +1,9 @@ +aiosignal-1.4.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +aiosignal-1.4.0.dist-info/METADATA,sha256=CSR-8dqLxpZyjUcTDnAuQwf299EB1sSFv_nzpxznAI0,3662 +aiosignal-1.4.0.dist-info/RECORD,, +aiosignal-1.4.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +aiosignal-1.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91 +aiosignal-1.4.0.dist-info/licenses/LICENSE,sha256=b9UkPpLdf5jsacesN3co50kFcJ_1J6W_mNbQJjwE9bY,11332 +aiosignal-1.4.0.dist-info/top_level.txt,sha256=z45aNOKGDdrI1roqZY3BGXQ22kJFPHBmVdwtLYLtXC0,10 +aiosignal/__init__.py,sha256=TIkmUG9HTBt4dfq2nISYBiZiRB2xwvFtEZydLP0HPL4,1537 +aiosignal/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/REQUESTED b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/WHEEL b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..e7fa31b6f3f78deb1022c1f7927f07d4d16da822 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.9.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/licenses/LICENSE b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..7082a2d5b9047bfc09589f387053e24ea490bc54 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/licenses/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2013-2019 Nikolay Kim and Andrew Svetlov + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/top_level.txt b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..ac6df3afe74a5fd43afc7ab7f8393571a495fdc5 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/aiosignal-1.4.0.dist-info/top_level.txt @@ -0,0 +1 @@ +aiosignal diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/INSTALLER b/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/METADATA b/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..9bf7a9e800778c5a8c3f1357450ab0849d13d953 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/METADATA @@ -0,0 +1,145 @@ +Metadata-Version: 2.4 +Name: annotated-doc +Version: 0.0.4 +Summary: Document parameters, class attributes, return types, and variables inline, with Annotated. +Author-Email: =?utf-8?q?Sebasti=C3=A1n_Ram=C3=ADrez?= +License-Expression: MIT +License-File: LICENSE +Classifier: Intended Audience :: Information Technology +Classifier: Intended Audience :: System Administrators +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python +Classifier: Topic :: Internet +Classifier: Topic :: Software Development :: Libraries :: Application Frameworks +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Software Development +Classifier: Typing :: Typed +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Project-URL: Homepage, https://github.com/fastapi/annotated-doc +Project-URL: Documentation, https://github.com/fastapi/annotated-doc +Project-URL: Repository, https://github.com/fastapi/annotated-doc +Project-URL: Issues, https://github.com/fastapi/annotated-doc/issues +Project-URL: Changelog, https://github.com/fastapi/annotated-doc/release-notes.md +Requires-Python: >=3.8 +Description-Content-Type: text/markdown + +# Annotated Doc + +Document parameters, class attributes, return types, and variables inline, with `Annotated`. + + + Test + + + Coverage + + + Package version + + + Supported Python versions + + +## Installation + +```bash +pip install annotated-doc +``` + +Or with `uv`: + +```Python +uv add annotated-doc +``` + +## Usage + +Import `Doc` and pass a single literal string with the documentation for the specific parameter, class attribute, return type, or variable. + +For example, to document a parameter `name` in a function `hi` you could do: + +```Python +from typing import Annotated + +from annotated_doc import Doc + +def hi(name: Annotated[str, Doc("Who to say hi to")]) -> None: + print(f"Hi, {name}!") +``` + +You can also use it to document class attributes: + +```Python +from typing import Annotated + +from annotated_doc import Doc + +class User: + name: Annotated[str, Doc("The user's name")] + age: Annotated[int, Doc("The user's age")] +``` + +The same way, you could document return types and variables, or anything that could have a type annotation with `Annotated`. + +## Who Uses This + +`annotated-doc` was made for: + +* [FastAPI](https://fastapi.tiangolo.com/) +* [Typer](https://typer.tiangolo.com/) +* [SQLModel](https://sqlmodel.tiangolo.com/) +* [Asyncer](https://asyncer.tiangolo.com/) + +`annotated-doc` is supported by [griffe-typingdoc](https://github.com/mkdocstrings/griffe-typingdoc), which powers reference documentation like the one in the [FastAPI Reference](https://fastapi.tiangolo.com/reference/). + +## Reasons not to use `annotated-doc` + +You are already comfortable with one of the existing docstring formats, like: + +* Sphinx +* numpydoc +* Google +* Keras + +Your team is already comfortable using them. + +You prefer having the documentation about parameters all together in a docstring, separated from the code defining them. + +You care about a specific set of users, using one specific editor, and that editor already has support for the specific docstring format you use. + +## Reasons to use `annotated-doc` + +* No micro-syntax to learn for newcomers, it’s **just Python** syntax. +* **Editing** would be already fully supported by default by any editor (current or future) supporting Python syntax, including syntax errors, syntax highlighting, etc. +* **Rendering** would be relatively straightforward to implement by static tools (tools that don't need runtime execution), as the information can be extracted from the AST they normally already create. +* **Deduplication of information**: the name of a parameter would be defined in a single place, not duplicated inside of a docstring. +* **Elimination** of the possibility of having **inconsistencies** when removing a parameter or class variable and **forgetting to remove** its documentation. +* **Minimization** of the probability of adding a new parameter or class variable and **forgetting to add its documentation**. +* **Elimination** of the possibility of having **inconsistencies** between the **name** of a parameter in the **signature** and the name in the docstring when it is renamed. +* **Access** to the documentation string for each symbol at **runtime**, including existing (older) Python versions. +* A more formalized way to document other symbols, like type aliases, that could use Annotated. +* **Support** for apps using FastAPI, Typer and others. +* **AI Accessibility**: AI tools will have an easier way understanding each parameter as the distance from documentation to parameter is much closer. + +## History + +I ([@tiangolo](https://github.com/tiangolo)) originally wanted for this to be part of the Python standard library (in [PEP 727](https://peps.python.org/pep-0727/)), but the proposal was withdrawn as there was a fair amount of negative feedback and opposition. + +The conclusion was that this was better done as an external effort, in a third-party library. + +So, here it is, with a simpler approach, as a third-party library, in a way that can be used by others, starting with FastAPI and friends. + +## License + +This project is licensed under the terms of the MIT license. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/WHEEL b/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..045c8acdea31cbca5be986e915f784c1aafc720f --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: pdm-backend (2.4.5) +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/entry_points.txt b/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..c3ad4726d437022e5c606a4206ffb6007347a008 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/entry_points.txt @@ -0,0 +1,4 @@ +[console_scripts] + +[gui_scripts] + diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/licenses/LICENSE b/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..7a254464cc78ccea32b3ded00513c44c4e4da412 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/annotated_doc-0.0.4.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2025 Sebastián Ramírez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5c6e0650bc4bf53806420d7ef5f881ecd2bd77ea --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/__init__.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: MIT + +""" +Classes Without Boilerplate +""" + +from functools import partial +from typing import Callable, Literal, Protocol + +from . import converters, exceptions, filters, setters, validators +from ._cmp import cmp_using +from ._config import get_run_validators, set_run_validators +from ._funcs import asdict, assoc, astuple, has, resolve_types +from ._make import ( + NOTHING, + Attribute, + Converter, + Factory, + _Nothing, + attrib, + attrs, + evolve, + fields, + fields_dict, + make_class, + validate, +) +from ._next_gen import define, field, frozen, mutable +from ._version_info import VersionInfo + + +s = attributes = attrs +ib = attr = attrib +dataclass = partial(attrs, auto_attribs=True) # happy Easter ;) + + +class AttrsInstance(Protocol): + pass + + +NothingType = Literal[_Nothing.NOTHING] + +__all__ = [ + "NOTHING", + "Attribute", + "AttrsInstance", + "Converter", + "Factory", + "NothingType", + "asdict", + "assoc", + "astuple", + "attr", + "attrib", + "attributes", + "attrs", + "cmp_using", + "converters", + "define", + "evolve", + "exceptions", + "field", + "fields", + "fields_dict", + "filters", + "frozen", + "get_run_validators", + "has", + "ib", + "make_class", + "mutable", + "resolve_types", + "s", + "set_run_validators", + "setters", + "validate", + "validators", +] + + +def _make_getattr(mod_name: str) -> Callable: + """ + Create a metadata proxy for packaging information that uses *mod_name* in + its warnings and errors. + """ + + def __getattr__(name: str) -> str: + if name not in ("__version__", "__version_info__"): + msg = f"module {mod_name} has no attribute {name}" + raise AttributeError(msg) + + from importlib.metadata import metadata + + meta = metadata("attrs") + + if name == "__version_info__": + return VersionInfo._from_version_string(meta["version"]) + + return meta["version"] + + return __getattr__ + + +__getattr__ = _make_getattr(__name__) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/__init__.pyi b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..758decf8ffcb55833a7136219db2b99542bfc7a4 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/__init__.pyi @@ -0,0 +1,389 @@ +import enum +import sys + +from typing import ( + Any, + Callable, + Generic, + Literal, + Mapping, + Protocol, + Sequence, + TypeVar, + overload, +) + +# `import X as X` is required to make these public +from . import converters as converters +from . import exceptions as exceptions +from . import filters as filters +from . import setters as setters +from . import validators as validators +from ._cmp import cmp_using as cmp_using +from ._typing_compat import AttrsInstance_ +from ._version_info import VersionInfo +from attrs import ( + define as define, + field as field, + mutable as mutable, + frozen as frozen, + _EqOrderType, + _ValidatorType, + _ConverterType, + _ReprArgType, + _OnSetAttrType, + _OnSetAttrArgType, + _FieldTransformer, + _ValidatorArgType, +) + +if sys.version_info >= (3, 10): + from typing import TypeGuard, TypeAlias +else: + from typing_extensions import TypeGuard, TypeAlias + +if sys.version_info >= (3, 11): + from typing import dataclass_transform +else: + from typing_extensions import dataclass_transform + +__version__: str +__version_info__: VersionInfo +__title__: str +__description__: str +__url__: str +__uri__: str +__author__: str +__email__: str +__license__: str +__copyright__: str + +_T = TypeVar("_T") +_C = TypeVar("_C", bound=type) + +_FilterType = Callable[["Attribute[_T]", _T], bool] + +# We subclass this here to keep the protocol's qualified name clean. +class AttrsInstance(AttrsInstance_, Protocol): + pass + +_A = TypeVar("_A", bound=type[AttrsInstance]) + +class _Nothing(enum.Enum): + NOTHING = enum.auto() + +NOTHING = _Nothing.NOTHING +NothingType: TypeAlias = Literal[_Nothing.NOTHING] + +# NOTE: Factory lies about its return type to make this possible: +# `x: List[int] # = Factory(list)` +# Work around mypy issue #4554 in the common case by using an overload. + +@overload +def Factory(factory: Callable[[], _T]) -> _T: ... +@overload +def Factory( + factory: Callable[[Any], _T], + takes_self: Literal[True], +) -> _T: ... +@overload +def Factory( + factory: Callable[[], _T], + takes_self: Literal[False], +) -> _T: ... + +In = TypeVar("In") +Out = TypeVar("Out") + +class Converter(Generic[In, Out]): + @overload + def __init__(self, converter: Callable[[In], Out]) -> None: ... + @overload + def __init__( + self, + converter: Callable[[In, AttrsInstance, Attribute], Out], + *, + takes_self: Literal[True], + takes_field: Literal[True], + ) -> None: ... + @overload + def __init__( + self, + converter: Callable[[In, Attribute], Out], + *, + takes_field: Literal[True], + ) -> None: ... + @overload + def __init__( + self, + converter: Callable[[In, AttrsInstance], Out], + *, + takes_self: Literal[True], + ) -> None: ... + +class Attribute(Generic[_T]): + name: str + default: _T | None + validator: _ValidatorType[_T] | None + repr: _ReprArgType + cmp: _EqOrderType + eq: _EqOrderType + order: _EqOrderType + hash: bool | None + init: bool + converter: Converter | None + metadata: dict[Any, Any] + type: type[_T] | None + kw_only: bool + on_setattr: _OnSetAttrType + alias: str | None + + def evolve(self, **changes: Any) -> "Attribute[Any]": ... + +# NOTE: We had several choices for the annotation to use for type arg: +# 1) Type[_T] +# - Pros: Handles simple cases correctly +# - Cons: Might produce less informative errors in the case of conflicting +# TypeVars e.g. `attr.ib(default='bad', type=int)` +# 2) Callable[..., _T] +# - Pros: Better error messages than #1 for conflicting TypeVars +# - Cons: Terrible error messages for validator checks. +# e.g. attr.ib(type=int, validator=validate_str) +# -> error: Cannot infer function type argument +# 3) type (and do all of the work in the mypy plugin) +# - Pros: Simple here, and we could customize the plugin with our own errors. +# - Cons: Would need to write mypy plugin code to handle all the cases. +# We chose option #1. + +# `attr` lies about its return type to make the following possible: +# attr() -> Any +# attr(8) -> int +# attr(validator=) -> Whatever the callable expects. +# This makes this type of assignments possible: +# x: int = attr(8) +# +# This form catches explicit None or no default but with no other arguments +# returns Any. +@overload +def attrib( + default: None = ..., + validator: None = ..., + repr: _ReprArgType = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + type: None = ..., + converter: None = ..., + factory: None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., +) -> Any: ... + +# This form catches an explicit None or no default and infers the type from the +# other arguments. +@overload +def attrib( + default: None = ..., + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + type: type[_T] | None = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., +) -> _T: ... + +# This form catches an explicit default argument. +@overload +def attrib( + default: _T, + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + type: type[_T] | None = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., +) -> _T: ... + +# This form covers type=non-Type: e.g. forward references (str), Any +@overload +def attrib( + default: _T | None = ..., + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + type: object = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., +) -> Any: ... +@overload +@dataclass_transform(order_default=True, field_specifiers=(attrib, field)) +def attrs( + maybe_cls: _C, + these: dict[str, Any] | None = ..., + repr_ns: str | None = ..., + repr: bool = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + auto_detect: bool = ..., + collect_by_mro: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., + unsafe_hash: bool | None = ..., +) -> _C: ... +@overload +@dataclass_transform(order_default=True, field_specifiers=(attrib, field)) +def attrs( + maybe_cls: None = ..., + these: dict[str, Any] | None = ..., + repr_ns: str | None = ..., + repr: bool = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + auto_detect: bool = ..., + collect_by_mro: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., + unsafe_hash: bool | None = ..., +) -> Callable[[_C], _C]: ... +def fields(cls: type[AttrsInstance] | AttrsInstance) -> Any: ... +def fields_dict(cls: type[AttrsInstance]) -> dict[str, Attribute[Any]]: ... +def validate(inst: AttrsInstance) -> None: ... +def resolve_types( + cls: _A, + globalns: dict[str, Any] | None = ..., + localns: dict[str, Any] | None = ..., + attribs: list[Attribute[Any]] | None = ..., + include_extras: bool = ..., +) -> _A: ... + +# TODO: add support for returning a proper attrs class from the mypy plugin +# we use Any instead of _CountingAttr so that e.g. `make_class('Foo', +# [attr.ib()])` is valid +def make_class( + name: str, + attrs: list[str] | tuple[str, ...] | dict[str, Any], + bases: tuple[type, ...] = ..., + class_body: dict[str, Any] | None = ..., + repr_ns: str | None = ..., + repr: bool = ..., + cmp: _EqOrderType | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + collect_by_mro: bool = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., +) -> type: ... + +# _funcs -- + +# TODO: add support for returning TypedDict from the mypy plugin +# FIXME: asdict/astuple do not honor their factory args. Waiting on one of +# these: +# https://github.com/python/mypy/issues/4236 +# https://github.com/python/typing/issues/253 +# XXX: remember to fix attrs.asdict/astuple too! +def asdict( + inst: AttrsInstance, + recurse: bool = ..., + filter: _FilterType[Any] | None = ..., + dict_factory: type[Mapping[Any, Any]] = ..., + retain_collection_types: bool = ..., + value_serializer: Callable[[type, Attribute[Any], Any], Any] | None = ..., + tuple_keys: bool | None = ..., +) -> dict[str, Any]: ... + +# TODO: add support for returning NamedTuple from the mypy plugin +def astuple( + inst: AttrsInstance, + recurse: bool = ..., + filter: _FilterType[Any] | None = ..., + tuple_factory: type[Sequence[Any]] = ..., + retain_collection_types: bool = ..., +) -> tuple[Any, ...]: ... +def has(cls: type) -> TypeGuard[type[AttrsInstance]]: ... +def assoc(inst: _T, **changes: Any) -> _T: ... +def evolve(inst: _T, **changes: Any) -> _T: ... + +# _config -- + +def set_run_validators(run: bool) -> None: ... +def get_run_validators() -> bool: ... + +# aliases -- + +s = attributes = attrs +ib = attr = attrib +dataclass = attrs # Technically, partial(attrs, auto_attribs=True) ;) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_cmp.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_cmp.py new file mode 100644 index 0000000000000000000000000000000000000000..09bab491f83ef4d15129f34b5f5a9e69bb34d63c --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_cmp.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: MIT + + +import functools +import types + +from ._make import __ne__ + + +_operation_names = {"eq": "==", "lt": "<", "le": "<=", "gt": ">", "ge": ">="} + + +def cmp_using( + eq=None, + lt=None, + le=None, + gt=None, + ge=None, + require_same_type=True, + class_name="Comparable", +): + """ + Create a class that can be passed into `attrs.field`'s ``eq``, ``order``, + and ``cmp`` arguments to customize field comparison. + + The resulting class will have a full set of ordering methods if at least + one of ``{lt, le, gt, ge}`` and ``eq`` are provided. + + Args: + eq (typing.Callable | None): + Callable used to evaluate equality of two objects. + + lt (typing.Callable | None): + Callable used to evaluate whether one object is less than another + object. + + le (typing.Callable | None): + Callable used to evaluate whether one object is less than or equal + to another object. + + gt (typing.Callable | None): + Callable used to evaluate whether one object is greater than + another object. + + ge (typing.Callable | None): + Callable used to evaluate whether one object is greater than or + equal to another object. + + require_same_type (bool): + When `True`, equality and ordering methods will return + `NotImplemented` if objects are not of the same type. + + class_name (str | None): Name of class. Defaults to "Comparable". + + See `comparison` for more details. + + .. versionadded:: 21.1.0 + """ + + body = { + "__slots__": ["value"], + "__init__": _make_init(), + "_requirements": [], + "_is_comparable_to": _is_comparable_to, + } + + # Add operations. + num_order_functions = 0 + has_eq_function = False + + if eq is not None: + has_eq_function = True + body["__eq__"] = _make_operator("eq", eq) + body["__ne__"] = __ne__ + + if lt is not None: + num_order_functions += 1 + body["__lt__"] = _make_operator("lt", lt) + + if le is not None: + num_order_functions += 1 + body["__le__"] = _make_operator("le", le) + + if gt is not None: + num_order_functions += 1 + body["__gt__"] = _make_operator("gt", gt) + + if ge is not None: + num_order_functions += 1 + body["__ge__"] = _make_operator("ge", ge) + + type_ = types.new_class( + class_name, (object,), {}, lambda ns: ns.update(body) + ) + + # Add same type requirement. + if require_same_type: + type_._requirements.append(_check_same_type) + + # Add total ordering if at least one operation was defined. + if 0 < num_order_functions < 4: + if not has_eq_function: + # functools.total_ordering requires __eq__ to be defined, + # so raise early error here to keep a nice stack. + msg = "eq must be define is order to complete ordering from lt, le, gt, ge." + raise ValueError(msg) + type_ = functools.total_ordering(type_) + + return type_ + + +def _make_init(): + """ + Create __init__ method. + """ + + def __init__(self, value): + """ + Initialize object with *value*. + """ + self.value = value + + return __init__ + + +def _make_operator(name, func): + """ + Create operator method. + """ + + def method(self, other): + if not self._is_comparable_to(other): + return NotImplemented + + result = func(self.value, other.value) + if result is NotImplemented: + return NotImplemented + + return result + + method.__name__ = f"__{name}__" + method.__doc__ = ( + f"Return a {_operation_names[name]} b. Computed by attrs." + ) + + return method + + +def _is_comparable_to(self, other): + """ + Check whether `other` is comparable to `self`. + """ + return all(func(self, other) for func in self._requirements) + + +def _check_same_type(self, other): + """ + Return True if *self* and *other* are of the same type, False otherwise. + """ + return other.value.__class__ is self.value.__class__ diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_cmp.pyi b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_cmp.pyi new file mode 100644 index 0000000000000000000000000000000000000000..cc7893b04520afa719b1412c7646c3c1b39bf94b --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_cmp.pyi @@ -0,0 +1,13 @@ +from typing import Any, Callable + +_CompareWithType = Callable[[Any, Any], bool] + +def cmp_using( + eq: _CompareWithType | None = ..., + lt: _CompareWithType | None = ..., + le: _CompareWithType | None = ..., + gt: _CompareWithType | None = ..., + ge: _CompareWithType | None = ..., + require_same_type: bool = ..., + class_name: str = ..., +) -> type: ... diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_compat.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..bc68ed9eaf9853bcc6065e7691209a787bf2eccc --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_compat.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: MIT + +import inspect +import platform +import sys +import threading + +from collections.abc import Mapping, Sequence # noqa: F401 +from typing import _GenericAlias + + +PYPY = platform.python_implementation() == "PyPy" +PY_3_10_PLUS = sys.version_info[:2] >= (3, 10) +PY_3_11_PLUS = sys.version_info[:2] >= (3, 11) +PY_3_12_PLUS = sys.version_info[:2] >= (3, 12) +PY_3_13_PLUS = sys.version_info[:2] >= (3, 13) +PY_3_14_PLUS = sys.version_info[:2] >= (3, 14) + + +if PY_3_14_PLUS: + import annotationlib + + # We request forward-ref annotations to not break in the presence of + # forward references. + + def _get_annotations(cls): + return annotationlib.get_annotations( + cls, format=annotationlib.Format.FORWARDREF + ) + +else: + + def _get_annotations(cls): + """ + Get annotations for *cls*. + """ + return cls.__dict__.get("__annotations__", {}) + + +class _AnnotationExtractor: + """ + Extract type annotations from a callable, returning None whenever there + is none. + """ + + __slots__ = ["sig"] + + def __init__(self, callable): + try: + self.sig = inspect.signature(callable) + except (ValueError, TypeError): # inspect failed + self.sig = None + + def get_first_param_type(self): + """ + Return the type annotation of the first argument if it's not empty. + """ + if not self.sig: + return None + + params = list(self.sig.parameters.values()) + if params and params[0].annotation is not inspect.Parameter.empty: + return params[0].annotation + + return None + + def get_return_type(self): + """ + Return the return type if it's not empty. + """ + if ( + self.sig + and self.sig.return_annotation is not inspect.Signature.empty + ): + return self.sig.return_annotation + + return None + + +# Thread-local global to track attrs instances which are already being repr'd. +# This is needed because there is no other (thread-safe) way to pass info +# about the instances that are already being repr'd through the call stack +# in order to ensure we don't perform infinite recursion. +# +# For instance, if an instance contains a dict which contains that instance, +# we need to know that we're already repr'ing the outside instance from within +# the dict's repr() call. +# +# This lives here rather than in _make.py so that the functions in _make.py +# don't have a direct reference to the thread-local in their globals dict. +# If they have such a reference, it breaks cloudpickle. +repr_context = threading.local() + + +def get_generic_base(cl): + """If this is a generic class (A[str]), return the generic base for it.""" + if cl.__class__ is _GenericAlias: + return cl.__origin__ + return None diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_config.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_config.py new file mode 100644 index 0000000000000000000000000000000000000000..4b257726fb1e8b95583ecc3eee8d153336dc4089 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_config.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: MIT + +__all__ = ["get_run_validators", "set_run_validators"] + +_run_validators = True + + +def set_run_validators(run): + """ + Set whether or not validators are run. By default, they are run. + + .. deprecated:: 21.3.0 It will not be removed, but it also will not be + moved to new ``attrs`` namespace. Use `attrs.validators.set_disabled()` + instead. + """ + if not isinstance(run, bool): + msg = "'run' must be bool." + raise TypeError(msg) + global _run_validators + _run_validators = run + + +def get_run_validators(): + """ + Return whether or not validators are run. + + .. deprecated:: 21.3.0 It will not be removed, but it also will not be + moved to new ``attrs`` namespace. Use `attrs.validators.get_disabled()` + instead. + """ + return _run_validators diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_funcs.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_funcs.py new file mode 100644 index 0000000000000000000000000000000000000000..1adb50021373d9c09fcb9db0641bbc03248d54a3 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_funcs.py @@ -0,0 +1,497 @@ +# SPDX-License-Identifier: MIT + + +import copy + +from ._compat import get_generic_base +from ._make import _OBJ_SETATTR, NOTHING, fields +from .exceptions import AttrsAttributeNotFoundError + + +_ATOMIC_TYPES = frozenset( + { + type(None), + bool, + int, + float, + str, + complex, + bytes, + type(...), + type, + range, + property, + } +) + + +def asdict( + inst, + recurse=True, + filter=None, + dict_factory=dict, + retain_collection_types=False, + value_serializer=None, +): + """ + Return the *attrs* attribute values of *inst* as a dict. + + Optionally recurse into other *attrs*-decorated classes. + + Args: + inst: Instance of an *attrs*-decorated class. + + recurse (bool): Recurse into classes that are also *attrs*-decorated. + + filter (~typing.Callable): + A callable whose return code determines whether an attribute or + element is included (`True`) or dropped (`False`). Is called with + the `attrs.Attribute` as the first argument and the value as the + second argument. + + dict_factory (~typing.Callable): + A callable to produce dictionaries from. For example, to produce + ordered dictionaries instead of normal Python dictionaries, pass in + ``collections.OrderedDict``. + + retain_collection_types (bool): + Do not convert to `list` when encountering an attribute whose type + is `tuple` or `set`. Only meaningful if *recurse* is `True`. + + value_serializer (typing.Callable | None): + A hook that is called for every attribute or dict key/value. It + receives the current instance, field and value and must return the + (updated) value. The hook is run *after* the optional *filter* has + been applied. + + Returns: + Return type of *dict_factory*. + + Raises: + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + .. versionadded:: 16.0.0 *dict_factory* + .. versionadded:: 16.1.0 *retain_collection_types* + .. versionadded:: 20.3.0 *value_serializer* + .. versionadded:: 21.3.0 + If a dict has a collection for a key, it is serialized as a tuple. + """ + attrs = fields(inst.__class__) + rv = dict_factory() + for a in attrs: + v = getattr(inst, a.name) + if filter is not None and not filter(a, v): + continue + + if value_serializer is not None: + v = value_serializer(inst, a, v) + + if recurse is True: + value_type = type(v) + if value_type in _ATOMIC_TYPES: + rv[a.name] = v + elif has(value_type): + rv[a.name] = asdict( + v, + recurse=True, + filter=filter, + dict_factory=dict_factory, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ) + elif issubclass(value_type, (tuple, list, set, frozenset)): + cf = value_type if retain_collection_types is True else list + items = [ + _asdict_anything( + i, + is_key=False, + filter=filter, + dict_factory=dict_factory, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ) + for i in v + ] + try: + rv[a.name] = cf(items) + except TypeError: + if not issubclass(cf, tuple): + raise + # Workaround for TypeError: cf.__new__() missing 1 required + # positional argument (which appears, for a namedturle) + rv[a.name] = cf(*items) + elif issubclass(value_type, dict): + df = dict_factory + rv[a.name] = df( + ( + _asdict_anything( + kk, + is_key=True, + filter=filter, + dict_factory=df, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ), + _asdict_anything( + vv, + is_key=False, + filter=filter, + dict_factory=df, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ), + ) + for kk, vv in v.items() + ) + else: + rv[a.name] = v + else: + rv[a.name] = v + return rv + + +def _asdict_anything( + val, + is_key, + filter, + dict_factory, + retain_collection_types, + value_serializer, +): + """ + ``asdict`` only works on attrs instances, this works on anything. + """ + val_type = type(val) + if val_type in _ATOMIC_TYPES: + rv = val + if value_serializer is not None: + rv = value_serializer(None, None, rv) + elif getattr(val_type, "__attrs_attrs__", None) is not None: + # Attrs class. + rv = asdict( + val, + recurse=True, + filter=filter, + dict_factory=dict_factory, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ) + elif issubclass(val_type, (tuple, list, set, frozenset)): + if retain_collection_types is True: + cf = val.__class__ + elif is_key: + cf = tuple + else: + cf = list + + rv = cf( + [ + _asdict_anything( + i, + is_key=False, + filter=filter, + dict_factory=dict_factory, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ) + for i in val + ] + ) + elif issubclass(val_type, dict): + df = dict_factory + rv = df( + ( + _asdict_anything( + kk, + is_key=True, + filter=filter, + dict_factory=df, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ), + _asdict_anything( + vv, + is_key=False, + filter=filter, + dict_factory=df, + retain_collection_types=retain_collection_types, + value_serializer=value_serializer, + ), + ) + for kk, vv in val.items() + ) + else: + rv = val + if value_serializer is not None: + rv = value_serializer(None, None, rv) + + return rv + + +def astuple( + inst, + recurse=True, + filter=None, + tuple_factory=tuple, + retain_collection_types=False, +): + """ + Return the *attrs* attribute values of *inst* as a tuple. + + Optionally recurse into other *attrs*-decorated classes. + + Args: + inst: Instance of an *attrs*-decorated class. + + recurse (bool): + Recurse into classes that are also *attrs*-decorated. + + filter (~typing.Callable): + A callable whose return code determines whether an attribute or + element is included (`True`) or dropped (`False`). Is called with + the `attrs.Attribute` as the first argument and the value as the + second argument. + + tuple_factory (~typing.Callable): + A callable to produce tuples from. For example, to produce lists + instead of tuples. + + retain_collection_types (bool): + Do not convert to `list` or `dict` when encountering an attribute + which type is `tuple`, `dict` or `set`. Only meaningful if + *recurse* is `True`. + + Returns: + Return type of *tuple_factory* + + Raises: + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + .. versionadded:: 16.2.0 + """ + attrs = fields(inst.__class__) + rv = [] + retain = retain_collection_types # Very long. :/ + for a in attrs: + v = getattr(inst, a.name) + if filter is not None and not filter(a, v): + continue + value_type = type(v) + if recurse is True: + if value_type in _ATOMIC_TYPES: + rv.append(v) + elif has(value_type): + rv.append( + astuple( + v, + recurse=True, + filter=filter, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + ) + elif issubclass(value_type, (tuple, list, set, frozenset)): + cf = v.__class__ if retain is True else list + items = [ + ( + astuple( + j, + recurse=True, + filter=filter, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + if has(j.__class__) + else j + ) + for j in v + ] + try: + rv.append(cf(items)) + except TypeError: + if not issubclass(cf, tuple): + raise + # Workaround for TypeError: cf.__new__() missing 1 required + # positional argument (which appears, for a namedturle) + rv.append(cf(*items)) + elif issubclass(value_type, dict): + df = value_type if retain is True else dict + rv.append( + df( + ( + ( + astuple( + kk, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + if has(kk.__class__) + else kk + ), + ( + astuple( + vv, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + if has(vv.__class__) + else vv + ), + ) + for kk, vv in v.items() + ) + ) + else: + rv.append(v) + else: + rv.append(v) + + return rv if tuple_factory is list else tuple_factory(rv) + + +def has(cls): + """ + Check whether *cls* is a class with *attrs* attributes. + + Args: + cls (type): Class to introspect. + + Raises: + TypeError: If *cls* is not a class. + + Returns: + bool: + """ + attrs = getattr(cls, "__attrs_attrs__", None) + if attrs is not None: + return True + + # No attrs, maybe it's a specialized generic (A[str])? + generic_base = get_generic_base(cls) + if generic_base is not None: + generic_attrs = getattr(generic_base, "__attrs_attrs__", None) + if generic_attrs is not None: + # Stick it on here for speed next time. + cls.__attrs_attrs__ = generic_attrs + return generic_attrs is not None + return False + + +def assoc(inst, **changes): + """ + Copy *inst* and apply *changes*. + + This is different from `evolve` that applies the changes to the arguments + that create the new instance. + + `evolve`'s behavior is preferable, but there are `edge cases`_ where it + doesn't work. Therefore `assoc` is deprecated, but will not be removed. + + .. _`edge cases`: https://github.com/python-attrs/attrs/issues/251 + + Args: + inst: Instance of a class with *attrs* attributes. + + changes: Keyword changes in the new copy. + + Returns: + A copy of inst with *changes* incorporated. + + Raises: + attrs.exceptions.AttrsAttributeNotFoundError: + If *attr_name* couldn't be found on *cls*. + + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + .. deprecated:: 17.1.0 + Use `attrs.evolve` instead if you can. This function will not be + removed du to the slightly different approach compared to + `attrs.evolve`, though. + """ + new = copy.copy(inst) + attrs = fields(inst.__class__) + for k, v in changes.items(): + a = getattr(attrs, k, NOTHING) + if a is NOTHING: + msg = f"{k} is not an attrs attribute on {new.__class__}." + raise AttrsAttributeNotFoundError(msg) + _OBJ_SETATTR(new, k, v) + return new + + +def resolve_types( + cls, globalns=None, localns=None, attribs=None, include_extras=True +): + """ + Resolve any strings and forward annotations in type annotations. + + This is only required if you need concrete types in :class:`Attribute`'s + *type* field. In other words, you don't need to resolve your types if you + only use them for static type checking. + + With no arguments, names will be looked up in the module in which the class + was created. If this is not what you want, for example, if the name only + exists inside a method, you may pass *globalns* or *localns* to specify + other dictionaries in which to look up these names. See the docs of + `typing.get_type_hints` for more details. + + Args: + cls (type): Class to resolve. + + globalns (dict | None): Dictionary containing global variables. + + localns (dict | None): Dictionary containing local variables. + + attribs (list | None): + List of attribs for the given class. This is necessary when calling + from inside a ``field_transformer`` since *cls* is not an *attrs* + class yet. + + include_extras (bool): + Resolve more accurately, if possible. Pass ``include_extras`` to + ``typing.get_hints``, if supported by the typing module. On + supported Python versions (3.9+), this resolves the types more + accurately. + + Raises: + TypeError: If *cls* is not a class. + + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class and you didn't pass any attribs. + + NameError: If types cannot be resolved because of missing variables. + + Returns: + *cls* so you can use this function also as a class decorator. Please + note that you have to apply it **after** `attrs.define`. That means the + decorator has to come in the line **before** `attrs.define`. + + .. versionadded:: 20.1.0 + .. versionadded:: 21.1.0 *attribs* + .. versionadded:: 23.1.0 *include_extras* + """ + # Since calling get_type_hints is expensive we cache whether we've + # done it already. + if getattr(cls, "__attrs_types_resolved__", None) != cls: + import typing + + kwargs = { + "globalns": globalns, + "localns": localns, + "include_extras": include_extras, + } + + hints = typing.get_type_hints(cls, **kwargs) + for field in fields(cls) if attribs is None else attribs: + if field.name in hints: + # Since fields have been frozen we must work around it. + _OBJ_SETATTR(field, "type", hints[field.name]) + # We store the class we resolved so that subclasses know they haven't + # been resolved. + cls.__attrs_types_resolved__ = cls + + # Return the class so you can use it as a decorator too. + return cls diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_make.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_make.py new file mode 100644 index 0000000000000000000000000000000000000000..4b32d6a71b0d91f3c4eb9ae615771aa46cae00eb --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_make.py @@ -0,0 +1,3406 @@ +# SPDX-License-Identifier: MIT + +from __future__ import annotations + +import abc +import contextlib +import copy +import enum +import inspect +import itertools +import linecache +import sys +import types +import unicodedata +import weakref + +from collections.abc import Callable, Mapping +from functools import cached_property +from typing import Any, NamedTuple, TypeVar + +# We need to import _compat itself in addition to the _compat members to avoid +# having the thread-local in the globals here. +from . import _compat, _config, setters +from ._compat import ( + PY_3_10_PLUS, + PY_3_11_PLUS, + PY_3_13_PLUS, + _AnnotationExtractor, + _get_annotations, + get_generic_base, +) +from .exceptions import ( + DefaultAlreadySetError, + FrozenInstanceError, + NotAnAttrsClassError, + UnannotatedAttributeError, +) + + +# This is used at least twice, so cache it here. +_OBJ_SETATTR = object.__setattr__ +_INIT_FACTORY_PAT = "__attr_factory_%s" +_CLASSVAR_PREFIXES = ( + "typing.ClassVar", + "t.ClassVar", + "ClassVar", + "typing_extensions.ClassVar", +) +# we don't use a double-underscore prefix because that triggers +# name mangling when trying to create a slot for the field +# (when slots=True) +_HASH_CACHE_FIELD = "_attrs_cached_hash" + +_EMPTY_METADATA_SINGLETON = types.MappingProxyType({}) + +# Unique object for unequivocal getattr() defaults. +_SENTINEL = object() + +_DEFAULT_ON_SETATTR = setters.pipe(setters.convert, setters.validate) + + +class _Nothing(enum.Enum): + """ + Sentinel to indicate the lack of a value when `None` is ambiguous. + + If extending attrs, you can use ``typing.Literal[NOTHING]`` to show + that a value may be ``NOTHING``. + + .. versionchanged:: 21.1.0 ``bool(NOTHING)`` is now False. + .. versionchanged:: 22.2.0 ``NOTHING`` is now an ``enum.Enum`` variant. + """ + + NOTHING = enum.auto() + + def __repr__(self): + return "NOTHING" + + def __bool__(self): + return False + + +NOTHING = _Nothing.NOTHING +""" +Sentinel to indicate the lack of a value when `None` is ambiguous. + +When using in 3rd party code, use `attrs.NothingType` for type annotations. +""" + + +class _CacheHashWrapper(int): + """ + An integer subclass that pickles / copies as None + + This is used for non-slots classes with ``cache_hash=True``, to avoid + serializing a potentially (even likely) invalid hash value. Since `None` + is the default value for uncalculated hashes, whenever this is copied, + the copy's value for the hash should automatically reset. + + See GH #613 for more details. + """ + + def __reduce__(self, _none_constructor=type(None), _args=()): # noqa: B008 + return _none_constructor, _args + + +def attrib( + default=NOTHING, + validator=None, + repr=True, + cmp=None, + hash=None, + init=True, + metadata=None, + type=None, + converter=None, + factory=None, + kw_only=None, + eq=None, + order=None, + on_setattr=None, + alias=None, +): + """ + Create a new field / attribute on a class. + + Identical to `attrs.field`, except it's not keyword-only. + + Consider using `attrs.field` in new code (``attr.ib`` will *never* go away, + though). + + .. warning:: + + Does **nothing** unless the class is also decorated with + `attr.s` (or similar)! + + + .. versionadded:: 15.2.0 *convert* + .. versionadded:: 16.3.0 *metadata* + .. versionchanged:: 17.1.0 *validator* can be a ``list`` now. + .. versionchanged:: 17.1.0 + *hash* is `None` and therefore mirrors *eq* by default. + .. versionadded:: 17.3.0 *type* + .. deprecated:: 17.4.0 *convert* + .. versionadded:: 17.4.0 + *converter* as a replacement for the deprecated *convert* to achieve + consistency with other noun-based arguments. + .. versionadded:: 18.1.0 + ``factory=f`` is syntactic sugar for ``default=attr.Factory(f)``. + .. versionadded:: 18.2.0 *kw_only* + .. versionchanged:: 19.2.0 *convert* keyword argument removed. + .. versionchanged:: 19.2.0 *repr* also accepts a custom callable. + .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01. + .. versionadded:: 19.2.0 *eq* and *order* + .. versionadded:: 20.1.0 *on_setattr* + .. versionchanged:: 20.3.0 *kw_only* backported to Python 2 + .. versionchanged:: 21.1.0 + *eq*, *order*, and *cmp* also accept a custom callable + .. versionchanged:: 21.1.0 *cmp* undeprecated + .. versionadded:: 22.2.0 *alias* + .. versionchanged:: 25.4.0 + *kw_only* can now be None, and its default is also changed from False to + None. + """ + eq, eq_key, order, order_key = _determine_attrib_eq_order( + cmp, eq, order, True + ) + + if hash is not None and hash is not True and hash is not False: + msg = "Invalid value for hash. Must be True, False, or None." + raise TypeError(msg) + + if factory is not None: + if default is not NOTHING: + msg = ( + "The `default` and `factory` arguments are mutually exclusive." + ) + raise ValueError(msg) + if not callable(factory): + msg = "The `factory` argument must be a callable." + raise ValueError(msg) + default = Factory(factory) + + if metadata is None: + metadata = {} + + # Apply syntactic sugar by auto-wrapping. + if isinstance(on_setattr, (list, tuple)): + on_setattr = setters.pipe(*on_setattr) + + if validator and isinstance(validator, (list, tuple)): + validator = and_(*validator) + + if converter and isinstance(converter, (list, tuple)): + converter = pipe(*converter) + + return _CountingAttr( + default=default, + validator=validator, + repr=repr, + cmp=None, + hash=hash, + init=init, + converter=converter, + metadata=metadata, + type=type, + kw_only=kw_only, + eq=eq, + eq_key=eq_key, + order=order, + order_key=order_key, + on_setattr=on_setattr, + alias=alias, + ) + + +def _compile_and_eval( + script: str, + globs: dict[str, Any] | None, + locs: Mapping[str, object] | None = None, + filename: str = "", +) -> None: + """ + Evaluate the script with the given global (globs) and local (locs) + variables. + """ + bytecode = compile(script, filename, "exec") + eval(bytecode, globs, locs) + + +def _linecache_and_compile( + script: str, + filename: str, + globs: dict[str, Any] | None, + locals: Mapping[str, object] | None = None, +) -> dict[str, Any]: + """ + Cache the script with _linecache_, compile it and return the _locals_. + """ + + locs = {} if locals is None else locals + + # In order of debuggers like PDB being able to step through the code, + # we add a fake linecache entry. + count = 1 + base_filename = filename + while True: + linecache_tuple = ( + len(script), + None, + script.splitlines(True), + filename, + ) + old_val = linecache.cache.setdefault(filename, linecache_tuple) + if old_val == linecache_tuple: + break + + filename = f"{base_filename[:-1]}-{count}>" + count += 1 + + _compile_and_eval(script, globs, locs, filename) + + return locs + + +def _make_attr_tuple_class(cls_name: str, attr_names: list[str]) -> type: + """ + Create a tuple subclass to hold `Attribute`s for an `attrs` class. + + The subclass is a bare tuple with properties for names. + + class MyClassAttributes(tuple): + __slots__ = () + x = property(itemgetter(0)) + """ + attr_class_name = f"{cls_name}Attributes" + body = {} + for i, attr_name in enumerate(attr_names): + + def getter(self, i=i): + return self[i] + + body[attr_name] = property(getter) + return type(attr_class_name, (tuple,), body) + + +# Tuple class for extracted attributes from a class definition. +# `base_attrs` is a subset of `attrs`. +class _Attributes(NamedTuple): + attrs: type + base_attrs: list[Attribute] + base_attrs_map: dict[str, type] + + +def _is_class_var(annot): + """ + Check whether *annot* is a typing.ClassVar. + + The string comparison hack is used to avoid evaluating all string + annotations which would put attrs-based classes at a performance + disadvantage compared to plain old classes. + """ + annot = str(annot) + + # Annotation can be quoted. + if annot.startswith(("'", '"')) and annot.endswith(("'", '"')): + annot = annot[1:-1] + + return annot.startswith(_CLASSVAR_PREFIXES) + + +def _has_own_attribute(cls, attrib_name): + """ + Check whether *cls* defines *attrib_name* (and doesn't just inherit it). + """ + return attrib_name in cls.__dict__ + + +def _collect_base_attrs( + cls, taken_attr_names +) -> tuple[list[Attribute], dict[str, type]]: + """ + Collect attr.ibs from base classes of *cls*, except *taken_attr_names*. + """ + base_attrs = [] + base_attr_map = {} # A dictionary of base attrs to their classes. + + # Traverse the MRO and collect attributes. + for base_cls in reversed(cls.__mro__[1:-1]): + for a in getattr(base_cls, "__attrs_attrs__", []): + if a.inherited or a.name in taken_attr_names: + continue + + a = a.evolve(inherited=True) # noqa: PLW2901 + base_attrs.append(a) + base_attr_map[a.name] = base_cls + + # For each name, only keep the freshest definition i.e. the furthest at the + # back. base_attr_map is fine because it gets overwritten with every new + # instance. + filtered = [] + seen = set() + for a in reversed(base_attrs): + if a.name in seen: + continue + filtered.insert(0, a) + seen.add(a.name) + + return filtered, base_attr_map + + +def _collect_base_attrs_broken(cls, taken_attr_names): + """ + Collect attr.ibs from base classes of *cls*, except *taken_attr_names*. + + N.B. *taken_attr_names* will be mutated. + + Adhere to the old incorrect behavior. + + Notably it collects from the front and considers inherited attributes which + leads to the buggy behavior reported in #428. + """ + base_attrs = [] + base_attr_map = {} # A dictionary of base attrs to their classes. + + # Traverse the MRO and collect attributes. + for base_cls in cls.__mro__[1:-1]: + for a in getattr(base_cls, "__attrs_attrs__", []): + if a.name in taken_attr_names: + continue + + a = a.evolve(inherited=True) # noqa: PLW2901 + taken_attr_names.add(a.name) + base_attrs.append(a) + base_attr_map[a.name] = base_cls + + return base_attrs, base_attr_map + + +def _transform_attrs( + cls, + these, + auto_attribs, + kw_only, + collect_by_mro, + field_transformer, +) -> _Attributes: + """ + Transform all `_CountingAttr`s on a class into `Attribute`s. + + If *these* is passed, use that and don't look for them on the class. + + If *collect_by_mro* is True, collect them in the correct MRO order, + otherwise use the old -- incorrect -- order. See #428. + + Return an `_Attributes`. + """ + cd = cls.__dict__ + anns = _get_annotations(cls) + + if these is not None: + ca_list = list(these.items()) + elif auto_attribs is True: + ca_names = { + name + for name, attr in cd.items() + if attr.__class__ is _CountingAttr + } + ca_list = [] + annot_names = set() + for attr_name, type in anns.items(): + if _is_class_var(type): + continue + annot_names.add(attr_name) + a = cd.get(attr_name, NOTHING) + + if a.__class__ is not _CountingAttr: + a = attrib(a) + ca_list.append((attr_name, a)) + + unannotated = ca_names - annot_names + if unannotated: + raise UnannotatedAttributeError( + "The following `attr.ib`s lack a type annotation: " + + ", ".join( + sorted(unannotated, key=lambda n: cd.get(n).counter) + ) + + "." + ) + else: + ca_list = sorted( + ( + (name, attr) + for name, attr in cd.items() + if attr.__class__ is _CountingAttr + ), + key=lambda e: e[1].counter, + ) + + fca = Attribute.from_counting_attr + no = ClassProps.KeywordOnly.NO + own_attrs = [ + fca( + attr_name, + ca, + kw_only is not no, + anns.get(attr_name), + ) + for attr_name, ca in ca_list + ] + + if collect_by_mro: + base_attrs, base_attr_map = _collect_base_attrs( + cls, {a.name for a in own_attrs} + ) + else: + base_attrs, base_attr_map = _collect_base_attrs_broken( + cls, {a.name for a in own_attrs} + ) + + if kw_only is ClassProps.KeywordOnly.FORCE: + own_attrs = [a.evolve(kw_only=True) for a in own_attrs] + base_attrs = [a.evolve(kw_only=True) for a in base_attrs] + + attrs = base_attrs + own_attrs + + # Resolve default field alias before executing field_transformer, so that + # the transformer receives fully populated Attribute objects with usable + # alias values. + for a in attrs: + if not a.alias: + # Evolve is very slow, so we hold our nose and do it dirty. + _OBJ_SETATTR.__get__(a)("alias", _default_init_alias_for(a.name)) + _OBJ_SETATTR.__get__(a)("alias_is_default", True) + + if field_transformer is not None: + attrs = tuple(field_transformer(cls, attrs)) + + # Check attr order after executing the field_transformer. + # Mandatory vs non-mandatory attr order only matters when they are part of + # the __init__ signature and when they aren't kw_only (which are moved to + # the end and can be mandatory or non-mandatory in any order, as they will + # be specified as keyword args anyway). Check the order of those attrs: + had_default = False + for a in (a for a in attrs if a.init is not False and a.kw_only is False): + if had_default is True and a.default is NOTHING: + msg = f"No mandatory attributes allowed after an attribute with a default value or factory. Attribute in question: {a!r}" + raise ValueError(msg) + + if had_default is False and a.default is not NOTHING: + had_default = True + + # Resolve default field alias for any new attributes that the + # field_transformer may have added without setting an alias. + for a in attrs: + if not a.alias: + _OBJ_SETATTR.__get__(a)("alias", _default_init_alias_for(a.name)) + _OBJ_SETATTR.__get__(a)("alias_is_default", True) + + # Create AttrsClass *after* applying the field_transformer since it may + # add or remove attributes! + attr_names = [a.name for a in attrs] + AttrsClass = _make_attr_tuple_class(cls.__name__, attr_names) + + return _Attributes(AttrsClass(attrs), base_attrs, base_attr_map) + + +def _make_cached_property_getattr(cached_properties, original_getattr, cls): + lines = [ + # Wrapped to get `__class__` into closure cell for super() + # (It will be replaced with the newly constructed class after construction). + "def wrapper(_cls):", + " __class__ = _cls", + " def __getattr__(self, item, cached_properties=cached_properties, original_getattr=original_getattr, _cached_setattr_get=_cached_setattr_get):", + " func = cached_properties.get(item)", + " if func is not None:", + " result = func(self)", + " _setter = _cached_setattr_get(self)", + " _setter(item, result)", + " return result", + ] + if original_getattr is not None: + lines.append( + " return original_getattr(self, item)", + ) + else: + lines.extend( + [ + " try:", + " return super().__getattribute__(item)", + " except AttributeError:", + " if not hasattr(super(), '__getattr__'):", + " raise", + " return super().__getattr__(item)", + " original_error = f\"'{self.__class__.__name__}' object has no attribute '{item}'\"", + " raise AttributeError(original_error)", + ] + ) + + lines.extend( + [ + " return __getattr__", + "__getattr__ = wrapper(_cls)", + ] + ) + + unique_filename = _generate_unique_filename(cls, "getattr") + + glob = { + "cached_properties": cached_properties, + "_cached_setattr_get": _OBJ_SETATTR.__get__, + "original_getattr": original_getattr, + } + + return _linecache_and_compile( + "\n".join(lines), unique_filename, glob, locals={"_cls": cls} + )["__getattr__"] + + +def _frozen_setattrs(self, name, value): + """ + Attached to frozen classes as __setattr__. + """ + if isinstance(self, BaseException) and name in ( + "__cause__", + "__context__", + "__traceback__", + "__suppress_context__", + "__notes__", + ): + BaseException.__setattr__(self, name, value) + return + + raise FrozenInstanceError + + +def _frozen_delattrs(self, name): + """ + Attached to frozen classes as __delattr__. + """ + if isinstance(self, BaseException) and name == "__notes__": + BaseException.__delattr__(self, name) + return + + raise FrozenInstanceError + + +def evolve(*args, **changes): + """ + Create a new instance, based on the first positional argument with + *changes* applied. + + .. tip:: + + On Python 3.13 and later, you can also use `copy.replace` instead. + + Args: + + inst: + Instance of a class with *attrs* attributes. *inst* must be passed + as a positional argument. + + changes: + Keyword changes in the new copy. + + Returns: + A copy of inst with *changes* incorporated. + + Raises: + TypeError: + If *attr_name* couldn't be found in the class ``__init__``. + + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + .. versionadded:: 17.1.0 + .. deprecated:: 23.1.0 + It is now deprecated to pass the instance using the keyword argument + *inst*. It will raise a warning until at least April 2024, after which + it will become an error. Always pass the instance as a positional + argument. + .. versionchanged:: 24.1.0 + *inst* can't be passed as a keyword argument anymore. + """ + try: + (inst,) = args + except ValueError: + msg = ( + f"evolve() takes 1 positional argument, but {len(args)} were given" + ) + raise TypeError(msg) from None + + cls = inst.__class__ + attrs = fields(cls) + for a in attrs: + if not a.init: + continue + attr_name = a.name # To deal with private attributes. + init_name = a.alias + if init_name not in changes: + changes[init_name] = getattr(inst, attr_name) + + return cls(**changes) + + +class _ClassBuilder: + """ + Iteratively build *one* class. + """ + + __slots__ = ( + "_add_method_dunders", + "_attr_names", + "_attrs", + "_base_attr_map", + "_base_names", + "_cache_hash", + "_cls", + "_cls_dict", + "_delete_attribs", + "_frozen", + "_has_custom_setattr", + "_has_post_init", + "_has_pre_init", + "_is_exc", + "_on_setattr", + "_pre_init_has_args", + "_repr_added", + "_script_snippets", + "_slots", + "_weakref_slot", + "_wrote_own_setattr", + ) + + def __init__( + self, + cls: type, + these, + auto_attribs: bool, + props: ClassProps, + has_custom_setattr: bool, + ): + attrs, base_attrs, base_map = _transform_attrs( + cls, + these, + auto_attribs, + props.kw_only, + props.collected_fields_by_mro, + props.field_transformer, + ) + + self._cls = cls + self._cls_dict = dict(cls.__dict__) if props.is_slotted else {} + self._attrs = attrs + self._base_names = {a.name for a in base_attrs} + self._base_attr_map = base_map + self._attr_names = tuple(a.name for a in attrs) + self._slots = props.is_slotted + self._frozen = props.is_frozen + self._weakref_slot = props.has_weakref_slot + self._cache_hash = ( + props.hashability is ClassProps.Hashability.HASHABLE_CACHED + ) + self._has_pre_init = bool(getattr(cls, "__attrs_pre_init__", False)) + self._pre_init_has_args = False + if self._has_pre_init: + # Check if the pre init method has more arguments than just `self` + # We want to pass arguments if pre init expects arguments + pre_init_func = cls.__attrs_pre_init__ + pre_init_signature = inspect.signature(pre_init_func) + self._pre_init_has_args = len(pre_init_signature.parameters) > 1 + self._has_post_init = bool(getattr(cls, "__attrs_post_init__", False)) + self._delete_attribs = not bool(these) + self._is_exc = props.is_exception + self._on_setattr = props.on_setattr_hook + + self._has_custom_setattr = has_custom_setattr + self._wrote_own_setattr = False + + self._cls_dict["__attrs_attrs__"] = self._attrs + self._cls_dict["__attrs_props__"] = props + + if props.is_frozen: + self._cls_dict["__setattr__"] = _frozen_setattrs + self._cls_dict["__delattr__"] = _frozen_delattrs + + self._wrote_own_setattr = True + elif self._on_setattr in ( + _DEFAULT_ON_SETATTR, + setters.validate, + setters.convert, + ): + has_validator = has_converter = False + for a in attrs: + if a.validator is not None: + has_validator = True + if a.converter is not None: + has_converter = True + + if has_validator and has_converter: + break + if ( + ( + self._on_setattr == _DEFAULT_ON_SETATTR + and not (has_validator or has_converter) + ) + or (self._on_setattr == setters.validate and not has_validator) + or (self._on_setattr == setters.convert and not has_converter) + ): + # If class-level on_setattr is set to convert + validate, but + # there's no field to convert or validate, pretend like there's + # no on_setattr. + self._on_setattr = None + + if props.added_pickling: + ( + self._cls_dict["__getstate__"], + self._cls_dict["__setstate__"], + ) = self._make_getstate_setstate() + + # tuples of script, globs, hook + self._script_snippets: list[ + tuple[str, dict, Callable[[dict, dict], Any]] + ] = [] + self._repr_added = False + + # We want to only do this check once; in 99.9% of cases these + # exist. + if not hasattr(self._cls, "__module__") or not hasattr( + self._cls, "__qualname__" + ): + self._add_method_dunders = self._add_method_dunders_safe + else: + self._add_method_dunders = self._add_method_dunders_unsafe + + def __repr__(self): + return f"<_ClassBuilder(cls={self._cls.__name__})>" + + def _eval_snippets(self) -> None: + """ + Evaluate any registered snippets in one go. + """ + script = "\n".join([snippet[0] for snippet in self._script_snippets]) + globs = {} + for _, snippet_globs, _ in self._script_snippets: + globs.update(snippet_globs) + + locs = _linecache_and_compile( + script, + _generate_unique_filename(self._cls, "methods"), + globs, + ) + + for _, _, hook in self._script_snippets: + hook(self._cls_dict, locs) + + def build_class(self): + """ + Finalize class based on the accumulated configuration. + + Builder cannot be used after calling this method. + """ + self._eval_snippets() + if self._slots is True: + cls = self._create_slots_class() + self._cls.__attrs_base_of_slotted__ = weakref.ref(cls) + else: + cls = self._patch_original_class() + if PY_3_10_PLUS: + cls = abc.update_abstractmethods(cls) + + # The method gets only called if it's not inherited from a base class. + # _has_own_attribute does NOT work properly for classmethods. + if ( + getattr(cls, "__attrs_init_subclass__", None) + and "__attrs_init_subclass__" not in cls.__dict__ + ): + cls.__attrs_init_subclass__() + + return cls + + def _patch_original_class(self): + """ + Apply accumulated methods and return the class. + """ + cls = self._cls + base_names = self._base_names + + # Clean class of attribute definitions (`attr.ib()`s). + if self._delete_attribs: + for name in self._attr_names: + if ( + name not in base_names + and getattr(cls, name, _SENTINEL) is not _SENTINEL + ): + # An AttributeError can happen if a base class defines a + # class variable and we want to set an attribute with the + # same name by using only a type annotation. + with contextlib.suppress(AttributeError): + delattr(cls, name) + + # Attach our dunder methods. + for name, value in self._cls_dict.items(): + setattr(cls, name, value) + + # If we've inherited an attrs __setattr__ and don't write our own, + # reset it to object's. + if not self._wrote_own_setattr and getattr( + cls, "__attrs_own_setattr__", False + ): + cls.__attrs_own_setattr__ = False + + if not self._has_custom_setattr: + cls.__setattr__ = _OBJ_SETATTR + + return cls + + def _create_slots_class(self): + """ + Build and return a new class with a `__slots__` attribute. + """ + cd = { + k: v + for k, v in self._cls_dict.items() + if k not in (*tuple(self._attr_names), "__dict__", "__weakref__") + } + + # 3.14.0rc2+ + if hasattr(sys, "_clear_type_descriptors"): + sys._clear_type_descriptors(self._cls) + + # If our class doesn't have its own implementation of __setattr__ + # (either from the user or by us), check the bases, if one of them has + # an attrs-made __setattr__, that needs to be reset. We don't walk the + # MRO because we only care about our immediate base classes. + # XXX: This can be confused by subclassing a slotted attrs class with + # XXX: a non-attrs class and subclass the resulting class with an attrs + # XXX: class. See `test_slotted_confused` for details. For now that's + # XXX: OK with us. + if not self._wrote_own_setattr: + cd["__attrs_own_setattr__"] = False + + if not self._has_custom_setattr: + for base_cls in self._cls.__bases__: + if base_cls.__dict__.get("__attrs_own_setattr__", False): + cd["__setattr__"] = _OBJ_SETATTR + break + + # Traverse the MRO to collect existing slots + # and check for an existing __weakref__. + existing_slots = {} + weakref_inherited = False + for base_cls in self._cls.__mro__[1:-1]: + if base_cls.__dict__.get("__weakref__", None) is not None: + weakref_inherited = True + existing_slots.update( + { + name: getattr(base_cls, name) + for name in getattr(base_cls, "__slots__", []) + } + ) + + base_names = set(self._base_names) + + names = self._attr_names + if ( + self._weakref_slot + and "__weakref__" not in getattr(self._cls, "__slots__", ()) + and "__weakref__" not in names + and not weakref_inherited + ): + names += ("__weakref__",) + + cached_properties = { + name: cached_prop.func + for name, cached_prop in cd.items() + if isinstance(cached_prop, cached_property) + } + + # Collect methods with a `__class__` reference that are shadowed in the new class. + # To know to update them. + additional_closure_functions_to_update = [] + if cached_properties: + class_annotations = _get_annotations(self._cls) + for name, func in cached_properties.items(): + # Add cached properties to names for slotting. + names += (name,) + # Clear out function from class to avoid clashing. + del cd[name] + additional_closure_functions_to_update.append(func) + annotation = inspect.signature(func).return_annotation + if annotation is not inspect.Parameter.empty: + class_annotations[name] = annotation + + original_getattr = cd.get("__getattr__") + if original_getattr is not None: + additional_closure_functions_to_update.append(original_getattr) + + cd["__getattr__"] = _make_cached_property_getattr( + cached_properties, original_getattr, self._cls + ) + + # We only add the names of attributes that aren't inherited. + # Setting __slots__ to inherited attributes wastes memory. + slot_names = [name for name in names if name not in base_names] + + # There are slots for attributes from current class + # that are defined in parent classes. + # As their descriptors may be overridden by a child class, + # we collect them here and update the class dict + reused_slots = { + slot: slot_descriptor + for slot, slot_descriptor in existing_slots.items() + if slot in slot_names + } + slot_names = [name for name in slot_names if name not in reused_slots] + cd.update(reused_slots) + if self._cache_hash: + slot_names.append(_HASH_CACHE_FIELD) + + cd["__slots__"] = tuple(slot_names) + + cd["__qualname__"] = self._cls.__qualname__ + + # Create new class based on old class and our methods. + cls = type(self._cls)(self._cls.__name__, self._cls.__bases__, cd) + + # The following is a fix for + # . + # If a method mentions `__class__` or uses the no-arg super(), the + # compiler will bake a reference to the class in the method itself + # as `method.__closure__`. Since we replace the class with a + # clone, we rewrite these references so it keeps working. + for item in itertools.chain( + cls.__dict__.values(), additional_closure_functions_to_update + ): + if isinstance(item, (classmethod, staticmethod)): + # Class- and staticmethods hide their functions inside. + # These might need to be rewritten as well. + closure_cells = getattr(item.__func__, "__closure__", None) + elif isinstance(item, property): + # Workaround for property `super()` shortcut (PY3-only). + # There is no universal way for other descriptors. + closure_cells = getattr(item.fget, "__closure__", None) + else: + closure_cells = getattr(item, "__closure__", None) + + if not closure_cells: # Catch None or the empty list. + continue + for cell in closure_cells: + try: + match = cell.cell_contents is self._cls + except ValueError: # noqa: PERF203 + # ValueError: Cell is empty + pass + else: + if match: + cell.cell_contents = cls + return cls + + def add_repr(self, ns): + script, globs = _make_repr_script(self._attrs, ns) + + def _attach_repr(cls_dict, globs): + cls_dict["__repr__"] = self._add_method_dunders(globs["__repr__"]) + + self._script_snippets.append((script, globs, _attach_repr)) + self._repr_added = True + return self + + def add_str(self): + if not self._repr_added: + msg = "__str__ can only be generated if a __repr__ exists." + raise ValueError(msg) + + def __str__(self): + return self.__repr__() + + self._cls_dict["__str__"] = self._add_method_dunders(__str__) + return self + + def _make_getstate_setstate(self): + """ + Create custom __setstate__ and __getstate__ methods. + """ + # __weakref__ is not writable. + state_attr_names = tuple( + an for an in self._attr_names if an != "__weakref__" + ) + + def slots_getstate(self): + """ + Automatically created by attrs. + """ + return {name: getattr(self, name) for name in state_attr_names} + + hash_caching_enabled = self._cache_hash + + def slots_setstate(self, state): + """ + Automatically created by attrs. + """ + __bound_setattr = _OBJ_SETATTR.__get__(self) + if isinstance(state, tuple): + # Backward compatibility with attrs instances pickled with + # attrs versions before v22.2.0 which stored tuples. + for name, value in zip(state_attr_names, state): + __bound_setattr(name, value) + else: + for name in state_attr_names: + if name in state: + __bound_setattr(name, state[name]) + + # The hash code cache is not included when the object is + # serialized, but it still needs to be initialized to None to + # indicate that the first call to __hash__ should be a cache + # miss. + if hash_caching_enabled: + __bound_setattr(_HASH_CACHE_FIELD, None) + + return slots_getstate, slots_setstate + + def make_unhashable(self): + self._cls_dict["__hash__"] = None + return self + + def add_hash(self): + script, globs = _make_hash_script( + self._cls, + self._attrs, + frozen=self._frozen, + cache_hash=self._cache_hash, + ) + + def attach_hash(cls_dict: dict, locs: dict) -> None: + cls_dict["__hash__"] = self._add_method_dunders(locs["__hash__"]) + + self._script_snippets.append((script, globs, attach_hash)) + + return self + + def add_init(self): + script, globs, annotations = _make_init_script( + self._cls, + self._attrs, + self._has_pre_init, + self._pre_init_has_args, + self._has_post_init, + self._frozen, + self._slots, + self._cache_hash, + self._base_attr_map, + self._is_exc, + self._on_setattr, + attrs_init=False, + ) + + def _attach_init(cls_dict, globs): + init = globs["__init__"] + init.__annotations__ = annotations + cls_dict["__init__"] = self._add_method_dunders(init) + + self._script_snippets.append((script, globs, _attach_init)) + + return self + + def add_replace(self): + self._cls_dict["__replace__"] = self._add_method_dunders(evolve) + return self + + def add_match_args(self): + self._cls_dict["__match_args__"] = tuple( + field.name + for field in self._attrs + if field.init and not field.kw_only + ) + + def add_attrs_init(self): + script, globs, annotations = _make_init_script( + self._cls, + self._attrs, + self._has_pre_init, + self._pre_init_has_args, + self._has_post_init, + self._frozen, + self._slots, + self._cache_hash, + self._base_attr_map, + self._is_exc, + self._on_setattr, + attrs_init=True, + ) + + def _attach_attrs_init(cls_dict, globs): + init = globs["__attrs_init__"] + init.__annotations__ = annotations + cls_dict["__attrs_init__"] = self._add_method_dunders(init) + + self._script_snippets.append((script, globs, _attach_attrs_init)) + + return self + + def add_eq(self): + cd = self._cls_dict + + script, globs = _make_eq_script(self._attrs) + + def _attach_eq(cls_dict, globs): + cls_dict["__eq__"] = self._add_method_dunders(globs["__eq__"]) + + self._script_snippets.append((script, globs, _attach_eq)) + + cd["__ne__"] = __ne__ + + return self + + def add_order(self): + cd = self._cls_dict + + cd["__lt__"], cd["__le__"], cd["__gt__"], cd["__ge__"] = ( + self._add_method_dunders(meth) + for meth in _make_order(self._cls, self._attrs) + ) + + return self + + def add_setattr(self): + sa_attrs = {} + for a in self._attrs: + on_setattr = a.on_setattr or self._on_setattr + if on_setattr and on_setattr is not setters.NO_OP: + sa_attrs[a.name] = a, on_setattr + + if not sa_attrs: + return self + + if self._has_custom_setattr: + # We need to write a __setattr__ but there already is one! + msg = "Can't combine custom __setattr__ with on_setattr hooks." + raise ValueError(msg) + + # docstring comes from _add_method_dunders + def __setattr__(self, name, val): + try: + a, hook = sa_attrs[name] + except KeyError: + nval = val + else: + nval = hook(self, a, val) + + _OBJ_SETATTR(self, name, nval) + + self._cls_dict["__attrs_own_setattr__"] = True + self._cls_dict["__setattr__"] = self._add_method_dunders(__setattr__) + self._wrote_own_setattr = True + + return self + + def _add_method_dunders_unsafe(self, method: Callable) -> Callable: + """ + Add __module__ and __qualname__ to a *method*. + """ + method.__module__ = self._cls.__module__ + + method.__qualname__ = f"{self._cls.__qualname__}.{method.__name__}" + + method.__doc__ = ( + f"Method generated by attrs for class {self._cls.__qualname__}." + ) + + return method + + def _add_method_dunders_safe(self, method: Callable) -> Callable: + """ + Add __module__ and __qualname__ to a *method* if possible. + """ + with contextlib.suppress(AttributeError): + method.__module__ = self._cls.__module__ + + with contextlib.suppress(AttributeError): + method.__qualname__ = f"{self._cls.__qualname__}.{method.__name__}" + + with contextlib.suppress(AttributeError): + method.__doc__ = f"Method generated by attrs for class {self._cls.__qualname__}." + + return method + + +def _determine_attrs_eq_order(cmp, eq, order, default_eq): + """ + Validate the combination of *cmp*, *eq*, and *order*. Derive the effective + values of eq and order. If *eq* is None, set it to *default_eq*. + """ + if cmp is not None and any((eq is not None, order is not None)): + msg = "Don't mix `cmp` with `eq' and `order`." + raise ValueError(msg) + + # cmp takes precedence due to bw-compatibility. + if cmp is not None: + return cmp, cmp + + # If left None, equality is set to the specified default and ordering + # mirrors equality. + if eq is None: + eq = default_eq + + if order is None: + order = eq + + if eq is False and order is True: + msg = "`order` can only be True if `eq` is True too." + raise ValueError(msg) + + return eq, order + + +def _determine_attrib_eq_order(cmp, eq, order, default_eq): + """ + Validate the combination of *cmp*, *eq*, and *order*. Derive the effective + values of eq and order. If *eq* is None, set it to *default_eq*. + """ + if cmp is not None and any((eq is not None, order is not None)): + msg = "Don't mix `cmp` with `eq' and `order`." + raise ValueError(msg) + + def decide_callable_or_boolean(value): + """ + Decide whether a key function is used. + """ + if callable(value): + value, key = True, value + else: + key = None + return value, key + + # cmp takes precedence due to bw-compatibility. + if cmp is not None: + cmp, cmp_key = decide_callable_or_boolean(cmp) + return cmp, cmp_key, cmp, cmp_key + + # If left None, equality is set to the specified default and ordering + # mirrors equality. + if eq is None: + eq, eq_key = default_eq, None + else: + eq, eq_key = decide_callable_or_boolean(eq) + + if order is None: + order, order_key = eq, eq_key + else: + order, order_key = decide_callable_or_boolean(order) + + if eq is False and order is True: + msg = "`order` can only be True if `eq` is True too." + raise ValueError(msg) + + return eq, eq_key, order, order_key + + +def _determine_whether_to_implement( + cls, flag, auto_detect, dunders, default=True +): + """ + Check whether we should implement a set of methods for *cls*. + + *flag* is the argument passed into @attr.s like 'init', *auto_detect* the + same as passed into @attr.s and *dunders* is a tuple of attribute names + whose presence signal that the user has implemented it themselves. + + Return *default* if no reason for either for or against is found. + """ + if flag is True or flag is False: + return flag + + if flag is None and auto_detect is False: + return default + + # Logically, flag is None and auto_detect is True here. + for dunder in dunders: + if _has_own_attribute(cls, dunder): + return False + + return default + + +def attrs( + maybe_cls=None, + these=None, + repr_ns=None, + repr=None, + cmp=None, + hash=None, + init=None, + slots=False, + frozen=False, + weakref_slot=True, + str=False, + auto_attribs=False, + kw_only=False, + cache_hash=False, + auto_exc=False, + eq=None, + order=None, + auto_detect=False, + collect_by_mro=False, + getstate_setstate=None, + on_setattr=None, + field_transformer=None, + match_args=True, + unsafe_hash=None, + force_kw_only=True, +): + r""" + A class decorator that adds :term:`dunder methods` according to the + specified attributes using `attr.ib` or the *these* argument. + + Consider using `attrs.define` / `attrs.frozen` in new code (``attr.s`` will + *never* go away, though). + + Args: + repr_ns (str): + When using nested classes, there was no way in Python 2 to + automatically detect that. This argument allows to set a custom + name for a more meaningful ``repr`` output. This argument is + pointless in Python 3 and is therefore deprecated. + + .. caution:: + Refer to `attrs.define` for the rest of the parameters, but note that they + can have different defaults. + + Notably, leaving *on_setattr* as `None` will **not** add any hooks. + + .. versionadded:: 16.0.0 *slots* + .. versionadded:: 16.1.0 *frozen* + .. versionadded:: 16.3.0 *str* + .. versionadded:: 16.3.0 Support for ``__attrs_post_init__``. + .. versionchanged:: 17.1.0 + *hash* supports `None` as value which is also the default now. + .. versionadded:: 17.3.0 *auto_attribs* + .. versionchanged:: 18.1.0 + If *these* is passed, no attributes are deleted from the class body. + .. versionchanged:: 18.1.0 If *these* is ordered, the order is retained. + .. versionadded:: 18.2.0 *weakref_slot* + .. deprecated:: 18.2.0 + ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now raise a + `DeprecationWarning` if the classes compared are subclasses of + each other. ``__eq`` and ``__ne__`` never tried to compared subclasses + to each other. + .. versionchanged:: 19.2.0 + ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now do not consider + subclasses comparable anymore. + .. versionadded:: 18.2.0 *kw_only* + .. versionadded:: 18.2.0 *cache_hash* + .. versionadded:: 19.1.0 *auto_exc* + .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01. + .. versionadded:: 19.2.0 *eq* and *order* + .. versionadded:: 20.1.0 *auto_detect* + .. versionadded:: 20.1.0 *collect_by_mro* + .. versionadded:: 20.1.0 *getstate_setstate* + .. versionadded:: 20.1.0 *on_setattr* + .. versionadded:: 20.3.0 *field_transformer* + .. versionchanged:: 21.1.0 + ``init=False`` injects ``__attrs_init__`` + .. versionchanged:: 21.1.0 Support for ``__attrs_pre_init__`` + .. versionchanged:: 21.1.0 *cmp* undeprecated + .. versionadded:: 21.3.0 *match_args* + .. versionadded:: 22.2.0 + *unsafe_hash* as an alias for *hash* (for :pep:`681` compliance). + .. deprecated:: 24.1.0 *repr_ns* + .. versionchanged:: 24.1.0 + Instances are not compared as tuples of attributes anymore, but using a + big ``and`` condition. This is faster and has more correct behavior for + uncomparable values like `math.nan`. + .. versionadded:: 24.1.0 + If a class has an *inherited* classmethod called + ``__attrs_init_subclass__``, it is executed after the class is created. + .. deprecated:: 24.1.0 *hash* is deprecated in favor of *unsafe_hash*. + .. versionchanged:: 25.4.0 + *kw_only* now only applies to attributes defined in the current class, + and respects attribute-level ``kw_only=False`` settings. + .. versionadded:: 25.4.0 *force_kw_only* + """ + if repr_ns is not None: + import warnings + + warnings.warn( + DeprecationWarning( + "The `repr_ns` argument is deprecated and will be removed in or after August 2025." + ), + stacklevel=2, + ) + + eq_, order_ = _determine_attrs_eq_order(cmp, eq, order, None) + + # unsafe_hash takes precedence due to PEP 681. + if unsafe_hash is not None: + hash = unsafe_hash + + if isinstance(on_setattr, (list, tuple)): + on_setattr = setters.pipe(*on_setattr) + + def wrap(cls): + nonlocal hash + is_frozen = frozen or _has_frozen_base_class(cls) + is_exc = auto_exc is True and issubclass(cls, BaseException) + has_own_setattr = auto_detect and _has_own_attribute( + cls, "__setattr__" + ) + + if has_own_setattr and is_frozen: + msg = "Can't freeze a class with a custom __setattr__." + raise ValueError(msg) + + eq = not is_exc and _determine_whether_to_implement( + cls, eq_, auto_detect, ("__eq__", "__ne__") + ) + + Hashability = ClassProps.Hashability + + if is_exc: + hashability = Hashability.LEAVE_ALONE + elif hash is True: + hashability = ( + Hashability.HASHABLE_CACHED + if cache_hash + else Hashability.HASHABLE + ) + elif hash is False: + hashability = Hashability.LEAVE_ALONE + elif hash is None: + if auto_detect is True and _has_own_attribute(cls, "__hash__"): + hashability = Hashability.LEAVE_ALONE + elif eq is True and is_frozen is True: + hashability = ( + Hashability.HASHABLE_CACHED + if cache_hash + else Hashability.HASHABLE + ) + elif eq is False: + hashability = Hashability.LEAVE_ALONE + else: + hashability = Hashability.UNHASHABLE + else: + msg = "Invalid value for hash. Must be True, False, or None." + raise TypeError(msg) + + KeywordOnly = ClassProps.KeywordOnly + if kw_only: + kwo = KeywordOnly.FORCE if force_kw_only else KeywordOnly.YES + else: + kwo = KeywordOnly.NO + + props = ClassProps( + is_exception=is_exc, + is_frozen=is_frozen, + is_slotted=slots, + collected_fields_by_mro=collect_by_mro, + added_init=_determine_whether_to_implement( + cls, init, auto_detect, ("__init__",) + ), + added_repr=_determine_whether_to_implement( + cls, repr, auto_detect, ("__repr__",) + ), + added_eq=eq, + added_ordering=not is_exc + and _determine_whether_to_implement( + cls, + order_, + auto_detect, + ("__lt__", "__le__", "__gt__", "__ge__"), + ), + hashability=hashability, + added_match_args=match_args, + kw_only=kwo, + has_weakref_slot=weakref_slot, + added_str=str, + added_pickling=_determine_whether_to_implement( + cls, + getstate_setstate, + auto_detect, + ("__getstate__", "__setstate__"), + default=slots, + ), + on_setattr_hook=on_setattr, + field_transformer=field_transformer, + ) + + if not props.is_hashable and cache_hash: + msg = "Invalid value for cache_hash. To use hash caching, hashing must be either explicitly or implicitly enabled." + raise TypeError(msg) + + builder = _ClassBuilder( + cls, + these, + auto_attribs=auto_attribs, + props=props, + has_custom_setattr=has_own_setattr, + ) + + if props.added_repr: + builder.add_repr(repr_ns) + + if props.added_str: + builder.add_str() + + if props.added_eq: + builder.add_eq() + if props.added_ordering: + builder.add_order() + + if not frozen: + builder.add_setattr() + + if props.is_hashable: + builder.add_hash() + elif props.hashability is Hashability.UNHASHABLE: + builder.make_unhashable() + + if props.added_init: + builder.add_init() + else: + builder.add_attrs_init() + if cache_hash: + msg = "Invalid value for cache_hash. To use hash caching, init must be True." + raise TypeError(msg) + + if PY_3_13_PLUS and not _has_own_attribute(cls, "__replace__"): + builder.add_replace() + + if ( + PY_3_10_PLUS + and match_args + and not _has_own_attribute(cls, "__match_args__") + ): + builder.add_match_args() + + return builder.build_class() + + # maybe_cls's type depends on the usage of the decorator. It's a class + # if it's used as `@attrs` but `None` if used as `@attrs()`. + if maybe_cls is None: + return wrap + + return wrap(maybe_cls) + + +_attrs = attrs +""" +Internal alias so we can use it in functions that take an argument called +*attrs*. +""" + + +def _has_frozen_base_class(cls): + """ + Check whether *cls* has a frozen ancestor by looking at its + __setattr__. + """ + return cls.__setattr__ is _frozen_setattrs + + +def _generate_unique_filename(cls: type, func_name: str) -> str: + """ + Create a "filename" suitable for a function being generated. + """ + return ( + f"" + ) + + +def _make_hash_script( + cls: type, attrs: list[Attribute], frozen: bool, cache_hash: bool +) -> tuple[str, dict]: + attrs = tuple( + a for a in attrs if a.hash is True or (a.hash is None and a.eq is True) + ) + + tab = " " + + type_hash = hash(_generate_unique_filename(cls, "hash")) + # If eq is custom generated, we need to include the functions in globs + globs = {} + + hash_def = "def __hash__(self" + hash_func = "hash((" + closing_braces = "))" + if not cache_hash: + hash_def += "):" + else: + hash_def += ", *" + + hash_def += ", _cache_wrapper=__import__('attr._make')._make._CacheHashWrapper):" + hash_func = "_cache_wrapper(" + hash_func + closing_braces += ")" + + method_lines = [hash_def] + + def append_hash_computation_lines(prefix, indent): + """ + Generate the code for actually computing the hash code. + Below this will either be returned directly or used to compute + a value which is then cached, depending on the value of cache_hash + """ + + method_lines.extend( + [ + indent + prefix + hash_func, + indent + f" {type_hash},", + ] + ) + + for a in attrs: + if a.eq_key: + cmp_name = f"_{a.name}_key" + globs[cmp_name] = a.eq_key + method_lines.append( + indent + f" {cmp_name}(self.{a.name})," + ) + else: + method_lines.append(indent + f" self.{a.name},") + + method_lines.append(indent + " " + closing_braces) + + if cache_hash: + method_lines.append(tab + f"if self.{_HASH_CACHE_FIELD} is None:") + if frozen: + append_hash_computation_lines( + f"object.__setattr__(self, '{_HASH_CACHE_FIELD}', ", tab * 2 + ) + method_lines.append(tab * 2 + ")") # close __setattr__ + else: + append_hash_computation_lines( + f"self.{_HASH_CACHE_FIELD} = ", tab * 2 + ) + method_lines.append(tab + f"return self.{_HASH_CACHE_FIELD}") + else: + append_hash_computation_lines("return ", tab) + + script = "\n".join(method_lines) + return script, globs + + +def _add_hash(cls: type, attrs: list[Attribute]): + """ + Add a hash method to *cls*. + """ + script, globs = _make_hash_script( + cls, attrs, frozen=False, cache_hash=False + ) + _compile_and_eval( + script, globs, filename=_generate_unique_filename(cls, "__hash__") + ) + cls.__hash__ = globs["__hash__"] + return cls + + +def __ne__(self, other): + """ + Check equality and either forward a NotImplemented or + return the result negated. + """ + result = self.__eq__(other) + if result is NotImplemented: + return NotImplemented + + return not result + + +def _make_eq_script(attrs: list) -> tuple[str, dict]: + """ + Create __eq__ method for *cls* with *attrs*. + """ + attrs = [a for a in attrs if a.eq] + + lines = [ + "def __eq__(self, other):", + " if other.__class__ is not self.__class__:", + " return NotImplemented", + ] + + globs = {} + if attrs: + lines.append(" return (") + for a in attrs: + if a.eq_key: + cmp_name = f"_{a.name}_key" + # Add the key function to the global namespace + # of the evaluated function. + globs[cmp_name] = a.eq_key + lines.append( + f" {cmp_name}(self.{a.name}) == {cmp_name}(other.{a.name})" + ) + else: + lines.append(f" self.{a.name} == other.{a.name}") + if a is not attrs[-1]: + lines[-1] = f"{lines[-1]} and" + lines.append(" )") + else: + lines.append(" return True") + + script = "\n".join(lines) + + return script, globs + + +def _make_order(cls, attrs): + """ + Create ordering methods for *cls* with *attrs*. + """ + attrs = [a for a in attrs if a.order] + + def attrs_to_tuple(obj): + """ + Save us some typing. + """ + return tuple( + key(value) if key else value + for value, key in ( + (getattr(obj, a.name), a.order_key) for a in attrs + ) + ) + + def __lt__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) < attrs_to_tuple(other) + + return NotImplemented + + def __le__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) <= attrs_to_tuple(other) + + return NotImplemented + + def __gt__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) > attrs_to_tuple(other) + + return NotImplemented + + def __ge__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) >= attrs_to_tuple(other) + + return NotImplemented + + return __lt__, __le__, __gt__, __ge__ + + +def _add_eq(cls, attrs=None): + """ + Add equality methods to *cls* with *attrs*. + """ + if attrs is None: + attrs = cls.__attrs_attrs__ + + script, globs = _make_eq_script(attrs) + _compile_and_eval( + script, globs, filename=_generate_unique_filename(cls, "__eq__") + ) + cls.__eq__ = globs["__eq__"] + cls.__ne__ = __ne__ + + return cls + + +def _make_repr_script(attrs, ns) -> tuple[str, dict]: + """ + Create the source and globs for a __repr__ and return it. + """ + # Figure out which attributes to include, and which function to use to + # format them. The a.repr value can be either bool or a custom + # callable. + attr_names_with_reprs = tuple( + (a.name, (repr if a.repr is True else a.repr), a.init) + for a in attrs + if a.repr is not False + ) + globs = { + name + "_repr": r for name, r, _ in attr_names_with_reprs if r != repr + } + globs["_compat"] = _compat + globs["AttributeError"] = AttributeError + globs["NOTHING"] = NOTHING + attribute_fragments = [] + for name, r, i in attr_names_with_reprs: + accessor = ( + "self." + name if i else 'getattr(self, "' + name + '", NOTHING)' + ) + fragment = ( + "%s={%s!r}" % (name, accessor) + if r == repr + else "%s={%s_repr(%s)}" % (name, name, accessor) + ) + attribute_fragments.append(fragment) + repr_fragment = ", ".join(attribute_fragments) + + if ns is None: + cls_name_fragment = '{self.__class__.__qualname__.rsplit(">.", 1)[-1]}' + else: + cls_name_fragment = ns + ".{self.__class__.__name__}" + + lines = [ + "def __repr__(self):", + " try:", + " already_repring = _compat.repr_context.already_repring", + " except AttributeError:", + " already_repring = {id(self),}", + " _compat.repr_context.already_repring = already_repring", + " else:", + " if id(self) in already_repring:", + " return '...'", + " else:", + " already_repring.add(id(self))", + " try:", + f" return f'{cls_name_fragment}({repr_fragment})'", + " finally:", + " already_repring.remove(id(self))", + ] + + return "\n".join(lines), globs + + +def _add_repr(cls, ns=None, attrs=None): + """ + Add a repr method to *cls*. + """ + if attrs is None: + attrs = cls.__attrs_attrs__ + + script, globs = _make_repr_script(attrs, ns) + _compile_and_eval( + script, globs, filename=_generate_unique_filename(cls, "__repr__") + ) + cls.__repr__ = globs["__repr__"] + return cls + + +def fields(cls): + """ + Return the tuple of *attrs* attributes for a class or instance. + + The tuple also allows accessing the fields by their names (see below for + examples). + + Args: + cls (type): Class or instance to introspect. + + Raises: + TypeError: If *cls* is neither a class nor an *attrs* instance. + + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + Returns: + tuple (with name accessors) of `attrs.Attribute` + + .. versionchanged:: 16.2.0 Returned tuple allows accessing the fields + by name. + .. versionchanged:: 23.1.0 Add support for generic classes. + .. versionchanged:: 26.1.0 Add support for instances. + """ + generic_base = get_generic_base(cls) + + if generic_base is None and not isinstance(cls, type): + type_ = type(cls) + if getattr(type_, "__attrs_attrs__", None) is None: + msg = "Passed object must be a class or attrs instance." + raise TypeError(msg) + + return fields(type_) + + attrs = getattr(cls, "__attrs_attrs__", None) + + if attrs is None: + if generic_base is not None: + attrs = getattr(generic_base, "__attrs_attrs__", None) + if attrs is not None: + # Even though this is global state, stick it on here to speed + # it up. We rely on `cls` being cached for this to be + # efficient. + cls.__attrs_attrs__ = attrs + return attrs + msg = f"{cls!r} is not an attrs-decorated class." + raise NotAnAttrsClassError(msg) + + return attrs + + +def fields_dict(cls): + """ + Return an ordered dictionary of *attrs* attributes for a class, whose keys + are the attribute names. + + Args: + cls (type): Class to introspect. + + Raises: + TypeError: If *cls* is not a class. + + attrs.exceptions.NotAnAttrsClassError: + If *cls* is not an *attrs* class. + + Returns: + dict[str, attrs.Attribute]: Dict of attribute name to definition + + .. versionadded:: 18.1.0 + """ + if not isinstance(cls, type): + msg = "Passed object must be a class." + raise TypeError(msg) + attrs = getattr(cls, "__attrs_attrs__", None) + if attrs is None: + msg = f"{cls!r} is not an attrs-decorated class." + raise NotAnAttrsClassError(msg) + return {a.name: a for a in attrs} + + +def validate(inst): + """ + Validate all attributes on *inst* that have a validator. + + Leaves all exceptions through. + + Args: + inst: Instance of a class with *attrs* attributes. + """ + if _config._run_validators is False: + return + + for a in fields(inst.__class__): + v = a.validator + if v is not None: + v(inst, a, getattr(inst, a.name)) + + +def _is_slot_attr(a_name, base_attr_map): + """ + Check if the attribute name comes from a slot class. + """ + cls = base_attr_map.get(a_name) + return cls and "__slots__" in cls.__dict__ + + +def _make_init_script( + cls, + attrs, + pre_init, + pre_init_has_args, + post_init, + frozen, + slots, + cache_hash, + base_attr_map, + is_exc, + cls_on_setattr, + attrs_init, +) -> tuple[str, dict, dict]: + has_cls_on_setattr = ( + cls_on_setattr is not None and cls_on_setattr is not setters.NO_OP + ) + + if frozen and has_cls_on_setattr: + msg = "Frozen classes can't use on_setattr." + raise ValueError(msg) + + needs_cached_setattr = cache_hash or frozen + filtered_attrs = [] + attr_dict = {} + for a in attrs: + if not a.init and a.default is NOTHING: + continue + + filtered_attrs.append(a) + attr_dict[a.name] = a + + if a.on_setattr is not None: + if frozen is True and a.on_setattr is not setters.NO_OP: + msg = "Frozen classes can't use on_setattr." + raise ValueError(msg) + + needs_cached_setattr = True + elif has_cls_on_setattr and a.on_setattr is not setters.NO_OP: + needs_cached_setattr = True + + script, globs, annotations = _attrs_to_init_script( + filtered_attrs, + frozen, + slots, + pre_init, + pre_init_has_args, + post_init, + cache_hash, + base_attr_map, + is_exc, + needs_cached_setattr, + has_cls_on_setattr, + "__attrs_init__" if attrs_init else "__init__", + ) + if cls.__module__ in sys.modules: + # This makes typing.get_type_hints(CLS.__init__) resolve string types. + globs.update(sys.modules[cls.__module__].__dict__) + + globs.update({"NOTHING": NOTHING, "attr_dict": attr_dict}) + + if needs_cached_setattr: + # Save the lookup overhead in __init__ if we need to circumvent + # setattr hooks. + globs["_cached_setattr_get"] = _OBJ_SETATTR.__get__ + + return script, globs, annotations + + +def _setattr(attr_name: str, value_var: str, has_on_setattr: bool) -> str: + """ + Use the cached object.setattr to set *attr_name* to *value_var*. + """ + return f"_setattr('{attr_name}', {value_var})" + + +def _setattr_with_converter( + attr_name: str, value_var: str, has_on_setattr: bool, converter: Converter +) -> str: + """ + Use the cached object.setattr to set *attr_name* to *value_var*, but run + its converter first. + """ + return f"_setattr('{attr_name}', {converter._fmt_converter_call(attr_name, value_var)})" + + +def _assign(attr_name: str, value: str, has_on_setattr: bool) -> str: + """ + Unless *attr_name* has an on_setattr hook, use normal assignment. Otherwise + relegate to _setattr. + """ + if has_on_setattr: + return _setattr(attr_name, value, True) + + return f"self.{attr_name} = {value}" + + +def _assign_with_converter( + attr_name: str, value_var: str, has_on_setattr: bool, converter: Converter +) -> str: + """ + Unless *attr_name* has an on_setattr hook, use normal assignment after + conversion. Otherwise relegate to _setattr_with_converter. + """ + if has_on_setattr: + return _setattr_with_converter(attr_name, value_var, True, converter) + + return f"self.{attr_name} = {converter._fmt_converter_call(attr_name, value_var)}" + + +def _determine_setters( + frozen: bool, slots: bool, base_attr_map: dict[str, type] +): + """ + Determine the correct setter functions based on whether a class is frozen + and/or slotted. + """ + if frozen is True: + if slots is True: + return (), _setattr, _setattr_with_converter + + # Dict frozen classes assign directly to __dict__. + # But only if the attribute doesn't come from an ancestor slot + # class. + # Note _inst_dict will be used again below if cache_hash is True + + def fmt_setter( + attr_name: str, value_var: str, has_on_setattr: bool + ) -> str: + if _is_slot_attr(attr_name, base_attr_map): + return _setattr(attr_name, value_var, has_on_setattr) + + return f"_inst_dict['{attr_name}'] = {value_var}" + + def fmt_setter_with_converter( + attr_name: str, + value_var: str, + has_on_setattr: bool, + converter: Converter, + ) -> str: + if has_on_setattr or _is_slot_attr(attr_name, base_attr_map): + return _setattr_with_converter( + attr_name, value_var, has_on_setattr, converter + ) + + return f"_inst_dict['{attr_name}'] = {converter._fmt_converter_call(attr_name, value_var)}" + + return ( + ("_inst_dict = self.__dict__",), + fmt_setter, + fmt_setter_with_converter, + ) + + # Not frozen -- we can just assign directly. + return (), _assign, _assign_with_converter + + +def _attrs_to_init_script( + attrs: list[Attribute], + is_frozen: bool, + is_slotted: bool, + call_pre_init: bool, + pre_init_has_args: bool, + call_post_init: bool, + does_cache_hash: bool, + base_attr_map: dict[str, type], + is_exc: bool, + needs_cached_setattr: bool, + has_cls_on_setattr: bool, + method_name: str, +) -> tuple[str, dict, dict]: + """ + Return a script of an initializer for *attrs*, a dict of globals, and + annotations for the initializer. + + The globals are required by the generated script. + """ + lines = ["self.__attrs_pre_init__()"] if call_pre_init else [] + + if needs_cached_setattr: + lines.append( + # Circumvent the __setattr__ descriptor to save one lookup per + # assignment. Note _setattr will be used again below if + # does_cache_hash is True. + "_setattr = _cached_setattr_get(self)" + ) + + extra_lines, fmt_setter, fmt_setter_with_converter = _determine_setters( + is_frozen, is_slotted, base_attr_map + ) + lines.extend(extra_lines) + + args = [] # Parameters in the definition of __init__ + pre_init_args = [] # Parameters in the call to __attrs_pre_init__ + kw_only_args = [] # Used for both 'args' and 'pre_init_args' above + attrs_to_validate = [] + + # This is a dictionary of names to validator and converter callables. + # Injecting this into __init__ globals lets us avoid lookups. + names_for_globals = {} + annotations = {"return": None} + + for a in attrs: + if a.validator: + attrs_to_validate.append(a) + + attr_name = a.name + has_on_setattr = a.on_setattr is not None or ( + a.on_setattr is not setters.NO_OP and has_cls_on_setattr + ) + # a.alias is set to maybe-mangled attr_name in _ClassBuilder if not + # explicitly provided + arg_name = a.alias + + has_factory = isinstance(a.default, Factory) + maybe_self = "self" if has_factory and a.default.takes_self else "" + + if a.converter is not None and not isinstance(a.converter, Converter): + converter = Converter(a.converter) + else: + converter = a.converter + + if a.init is False: + if has_factory: + init_factory_name = _INIT_FACTORY_PAT % (a.name,) + if converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, + init_factory_name + f"({maybe_self})", + has_on_setattr, + converter, + ) + ) + names_for_globals[converter._get_global_name(a.name)] = ( + converter.converter + ) + else: + lines.append( + fmt_setter( + attr_name, + init_factory_name + f"({maybe_self})", + has_on_setattr, + ) + ) + names_for_globals[init_factory_name] = a.default.factory + elif converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, + f"attr_dict['{attr_name}'].default", + has_on_setattr, + converter, + ) + ) + names_for_globals[converter._get_global_name(a.name)] = ( + converter.converter + ) + else: + lines.append( + fmt_setter( + attr_name, + f"attr_dict['{attr_name}'].default", + has_on_setattr, + ) + ) + elif a.default is not NOTHING and not has_factory: + arg = f"{arg_name}=attr_dict['{attr_name}'].default" + if a.kw_only: + kw_only_args.append(arg) + else: + args.append(arg) + pre_init_args.append(arg_name) + + if converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, arg_name, has_on_setattr, converter + ) + ) + names_for_globals[converter._get_global_name(a.name)] = ( + converter.converter + ) + else: + lines.append(fmt_setter(attr_name, arg_name, has_on_setattr)) + + elif has_factory: + arg = f"{arg_name}=NOTHING" + if a.kw_only: + kw_only_args.append(arg) + else: + args.append(arg) + pre_init_args.append(arg_name) + lines.append(f"if {arg_name} is not NOTHING:") + + init_factory_name = _INIT_FACTORY_PAT % (a.name,) + if converter is not None: + lines.append( + " " + + fmt_setter_with_converter( + attr_name, arg_name, has_on_setattr, converter + ) + ) + lines.append("else:") + lines.append( + " " + + fmt_setter_with_converter( + attr_name, + init_factory_name + "(" + maybe_self + ")", + has_on_setattr, + converter, + ) + ) + names_for_globals[converter._get_global_name(a.name)] = ( + converter.converter + ) + else: + lines.append( + " " + fmt_setter(attr_name, arg_name, has_on_setattr) + ) + lines.append("else:") + lines.append( + " " + + fmt_setter( + attr_name, + init_factory_name + "(" + maybe_self + ")", + has_on_setattr, + ) + ) + names_for_globals[init_factory_name] = a.default.factory + else: + if a.kw_only: + kw_only_args.append(arg_name) + else: + args.append(arg_name) + pre_init_args.append(arg_name) + + if converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, arg_name, has_on_setattr, converter + ) + ) + names_for_globals[converter._get_global_name(a.name)] = ( + converter.converter + ) + else: + lines.append(fmt_setter(attr_name, arg_name, has_on_setattr)) + + if a.init is True: + if a.type is not None and converter is None: + annotations[arg_name] = a.type + elif converter is not None and converter._first_param_type: + # Use the type from the converter if present. + annotations[arg_name] = converter._first_param_type + + if attrs_to_validate: # we can skip this if there are no validators. + names_for_globals["_config"] = _config + lines.append("if _config._run_validators is True:") + for a in attrs_to_validate: + val_name = "__attr_validator_" + a.name + attr_name = "__attr_" + a.name + lines.append(f" {val_name}(self, {attr_name}, self.{a.name})") + names_for_globals[val_name] = a.validator + names_for_globals[attr_name] = a + + if call_post_init: + lines.append("self.__attrs_post_init__()") + + # Because this is set only after __attrs_post_init__ is called, a crash + # will result if post-init tries to access the hash code. This seemed + # preferable to setting this beforehand, in which case alteration to field + # values during post-init combined with post-init accessing the hash code + # would result in silent bugs. + if does_cache_hash: + if is_frozen: + if is_slotted: + init_hash_cache = f"_setattr('{_HASH_CACHE_FIELD}', None)" + else: + init_hash_cache = f"_inst_dict['{_HASH_CACHE_FIELD}'] = None" + else: + init_hash_cache = f"self.{_HASH_CACHE_FIELD} = None" + lines.append(init_hash_cache) + + # For exceptions we rely on BaseException.__init__ for proper + # initialization. + if is_exc: + vals = ",".join(f"self.{a.name}" for a in attrs if a.init) + + lines.append(f"BaseException.__init__(self, {vals})") + + args = ", ".join(args) + pre_init_args = ", ".join(pre_init_args) + if kw_only_args: + # leading comma & kw_only args + args += f"{', ' if args else ''}*, {', '.join(kw_only_args)}" + pre_init_kw_only_args = ", ".join( + [ + f"{kw_arg_name}={kw_arg_name}" + # We need to remove the defaults from the kw_only_args. + for kw_arg_name in (kwa.split("=")[0] for kwa in kw_only_args) + ] + ) + pre_init_args += ", " if pre_init_args else "" + pre_init_args += pre_init_kw_only_args + + if call_pre_init and pre_init_has_args: + # If pre init method has arguments, pass the values given to __init__. + lines[0] = f"self.__attrs_pre_init__({pre_init_args})" + + # Python <3.12 doesn't allow backslashes in f-strings. + NL = "\n " + return ( + f"""def {method_name}(self, {args}): + {NL.join(lines) if lines else "pass"} +""", + names_for_globals, + annotations, + ) + + +def _default_init_alias_for(name: str) -> str: + """ + The default __init__ parameter name for a field. + + This performs private-name adjustment via leading-unscore stripping, + and is the default value of Attribute.alias if not provided. + """ + + return name.lstrip("_") + + +class Attribute: + """ + *Read-only* representation of an attribute. + + .. warning:: + + You should never instantiate this class yourself. + + The class has *all* arguments of `attr.ib` (except for ``factory`` which is + only syntactic sugar for ``default=Factory(...)`` plus the following: + + - ``name`` (`str`): The name of the attribute. + - ``alias`` (`str`): The __init__ parameter name of the attribute, after + any explicit overrides and default private-attribute-name handling. + - ``alias_is_default`` (`bool`): Whether the ``alias`` was automatically + generated (``True``) or explicitly provided by the user (``False``). + - ``inherited`` (`bool`): Whether or not that attribute has been inherited + from a base class. + - ``eq_key`` and ``order_key`` (`typing.Callable` or `None`): The + callables that are used for comparing and ordering objects by this + attribute, respectively. These are set by passing a callable to + `attr.ib`'s ``eq``, ``order``, or ``cmp`` arguments. See also + :ref:`comparison customization `. + + Instances of this class are frequently used for introspection purposes + like: + + - `fields` returns a tuple of them. + - Validators get them passed as the first argument. + - The :ref:`field transformer ` hook receives a list of + them. + - The ``alias`` property exposes the __init__ parameter name of the field, + with any overrides and default private-attribute handling applied. + + + .. versionadded:: 20.1.0 *inherited* + .. versionadded:: 20.1.0 *on_setattr* + .. versionchanged:: 20.2.0 *inherited* is not taken into account for + equality checks and hashing anymore. + .. versionadded:: 21.1.0 *eq_key* and *order_key* + .. versionadded:: 22.2.0 *alias* + .. versionadded:: 26.1.0 *alias_is_default* + + For the full version history of the fields, see `attr.ib`. + """ + + # These slots must NOT be reordered because we use them later for + # instantiation. + __slots__ = ( # noqa: RUF023 + "name", + "default", + "validator", + "repr", + "eq", + "eq_key", + "order", + "order_key", + "hash", + "init", + "metadata", + "type", + "converter", + "kw_only", + "inherited", + "on_setattr", + "alias", + "alias_is_default", + ) + + def __init__( + self, + name, + default, + validator, + repr, + cmp, # XXX: unused, remove along with other cmp code. + hash, + init, + inherited, + metadata=None, + type=None, + converter=None, + kw_only=False, + eq=None, + eq_key=None, + order=None, + order_key=None, + on_setattr=None, + alias=None, + alias_is_default=None, + ): + eq, eq_key, order, order_key = _determine_attrib_eq_order( + cmp, eq_key or eq, order_key or order, True + ) + + # Cache this descriptor here to speed things up later. + bound_setattr = _OBJ_SETATTR.__get__(self) + + # Despite the big red warning, people *do* instantiate `Attribute` + # themselves. + bound_setattr("name", name) + bound_setattr("default", default) + bound_setattr("validator", validator) + bound_setattr("repr", repr) + bound_setattr("eq", eq) + bound_setattr("eq_key", eq_key) + bound_setattr("order", order) + bound_setattr("order_key", order_key) + bound_setattr("hash", hash) + bound_setattr("init", init) + bound_setattr("converter", converter) + bound_setattr( + "metadata", + ( + types.MappingProxyType(dict(metadata)) # Shallow copy + if metadata + else _EMPTY_METADATA_SINGLETON + ), + ) + bound_setattr("type", type) + bound_setattr("kw_only", kw_only) + bound_setattr("inherited", inherited) + bound_setattr("on_setattr", on_setattr) + bound_setattr("alias", alias) + bound_setattr( + "alias_is_default", + alias is None if alias_is_default is None else alias_is_default, + ) + + def __setattr__(self, name, value): + raise FrozenInstanceError + + @classmethod + def from_counting_attr( + cls, name: str, ca: _CountingAttr, kw_only: bool, type=None + ): + # The 'kw_only' argument is the class-level setting, and is used if the + # attribute itself does not explicitly set 'kw_only'. + # type holds the annotated value. deal with conflicts: + if type is None: + type = ca.type + elif ca.type is not None: + msg = f"Type annotation and type argument cannot both be present for '{name}'." + raise ValueError(msg) + return cls( + name, + ca._default, + ca._validator, + ca.repr, + None, + ca.hash, + ca.init, + False, + ca.metadata, + type, + ca.converter, + kw_only if ca.kw_only is None else ca.kw_only, + ca.eq, + ca.eq_key, + ca.order, + ca.order_key, + ca.on_setattr, + ca.alias, + ca.alias is None, + ) + + # Don't use attrs.evolve since fields(Attribute) doesn't work + def evolve(self, **changes): + """ + Copy *self* and apply *changes*. + + This works similarly to `attrs.evolve` but that function does not work + with :class:`attrs.Attribute`. + + It is mainly meant to be used for `transform-fields`. + + .. versionadded:: 20.3.0 + """ + new = copy.copy(self) + + new._setattrs(changes.items()) + + if "alias" in changes and "alias_is_default" not in changes: + # Explicit alias provided -- no longer the default. + _OBJ_SETATTR.__get__(new)("alias_is_default", False) + elif ( + "name" in changes + and "alias" not in changes + # Don't auto-generate alias if the user picked picked the old one. + and self.alias_is_default + ): + # Name changed, alias was auto-generated -- update it. + _OBJ_SETATTR.__get__(new)( + "alias", _default_init_alias_for(new.name) + ) + + return new + + # Don't use _add_pickle since fields(Attribute) doesn't work + def __getstate__(self): + """ + Play nice with pickle. + """ + return tuple( + getattr(self, name) if name != "metadata" else dict(self.metadata) + for name in self.__slots__ + ) + + def __setstate__(self, state): + """ + Play nice with pickle. + """ + if len(state) < len(self.__slots__): + # Pre-26.1.0 pickle without alias_is_default -- infer it + # heuristically. + state_dict = dict(zip(self.__slots__, state)) + alias_is_default = state_dict.get( + "alias" + ) is None or state_dict.get("alias") == _default_init_alias_for( + state_dict["name"] + ) + state = (*state, alias_is_default) + + self._setattrs(zip(self.__slots__, state)) + + def _setattrs(self, name_values_pairs): + bound_setattr = _OBJ_SETATTR.__get__(self) + for name, value in name_values_pairs: + if name != "metadata": + bound_setattr(name, value) + else: + bound_setattr( + name, + ( + types.MappingProxyType(dict(value)) + if value + else _EMPTY_METADATA_SINGLETON + ), + ) + + +_a = [ + Attribute( + name=name, + default=NOTHING, + validator=None, + repr=(name != "alias_is_default"), + cmp=None, + eq=True, + order=False, + hash=(name != "metadata"), + init=True, + inherited=False, + alias=_default_init_alias_for(name), + ) + for name in Attribute.__slots__ +] + +Attribute = _add_hash( + _add_eq( + _add_repr(Attribute, attrs=_a), + attrs=[a for a in _a if a.name != "inherited"], + ), + attrs=[a for a in _a if a.hash and a.name != "inherited"], +) + + +class _CountingAttr: + """ + Intermediate representation of attributes that uses a counter to preserve + the order in which the attributes have been defined. + + *Internal* data structure of the attrs library. Running into is most + likely the result of a bug like a forgotten `@attr.s` decorator. + """ + + __slots__ = ( + "_default", + "_validator", + "alias", + "converter", + "counter", + "eq", + "eq_key", + "hash", + "init", + "kw_only", + "metadata", + "on_setattr", + "order", + "order_key", + "repr", + "type", + ) + __attrs_attrs__ = ( + *tuple( + Attribute( + name=name, + alias=_default_init_alias_for(name), + default=NOTHING, + validator=None, + repr=True, + cmp=None, + hash=True, + init=True, + kw_only=False, + eq=True, + eq_key=None, + order=False, + order_key=None, + inherited=False, + on_setattr=None, + ) + for name in ( + "counter", + "_default", + "repr", + "eq", + "order", + "hash", + "init", + "on_setattr", + "alias", + ) + ), + Attribute( + name="metadata", + alias="metadata", + default=None, + validator=None, + repr=True, + cmp=None, + hash=False, + init=True, + kw_only=False, + eq=True, + eq_key=None, + order=False, + order_key=None, + inherited=False, + on_setattr=None, + ), + ) + cls_counter = 0 + + def __init__( + self, + default, + validator, + repr, + cmp, + hash, + init, + converter, + metadata, + type, + kw_only, + eq, + eq_key, + order, + order_key, + on_setattr, + alias, + ): + _CountingAttr.cls_counter += 1 + self.counter = _CountingAttr.cls_counter + self._default = default + self._validator = validator + self.converter = converter + self.repr = repr + self.eq = eq + self.eq_key = eq_key + self.order = order + self.order_key = order_key + self.hash = hash + self.init = init + self.metadata = metadata + self.type = type + self.kw_only = kw_only + self.on_setattr = on_setattr + self.alias = alias + + def validator(self, meth): + """ + Decorator that adds *meth* to the list of validators. + + Returns *meth* unchanged. + + .. versionadded:: 17.1.0 + """ + if self._validator is None: + self._validator = meth + else: + self._validator = and_(self._validator, meth) + return meth + + def default(self, meth): + """ + Decorator that allows to set the default for an attribute. + + Returns *meth* unchanged. + + Raises: + DefaultAlreadySetError: If default has been set before. + + .. versionadded:: 17.1.0 + """ + if self._default is not NOTHING: + raise DefaultAlreadySetError + + self._default = Factory(meth, takes_self=True) + + return meth + + +_CountingAttr = _add_eq(_add_repr(_CountingAttr)) + + +class ClassProps: + """ + Effective class properties as derived from parameters to `attr.s()` or + `define()` decorators. + + This is the same data structure that *attrs* uses internally to decide how + to construct the final class. + + Warning: + + This feature is currently **experimental** and is not covered by our + strict backwards-compatibility guarantees. + + + Attributes: + is_exception (bool): + Whether the class is treated as an exception class. + + is_slotted (bool): + Whether the class is `slotted `. + + has_weakref_slot (bool): + Whether the class has a slot for weak references. + + is_frozen (bool): + Whether the class is frozen. + + kw_only (KeywordOnly): + Whether / how the class enforces keyword-only arguments on the + ``__init__`` method. + + collected_fields_by_mro (bool): + Whether the class fields were collected by method resolution order. + That is, correctly but unlike `dataclasses`. + + added_init (bool): + Whether the class has an *attrs*-generated ``__init__`` method. + + added_repr (bool): + Whether the class has an *attrs*-generated ``__repr__`` method. + + added_eq (bool): + Whether the class has *attrs*-generated equality methods. + + added_ordering (bool): + Whether the class has *attrs*-generated ordering methods. + + hashability (Hashability): How `hashable ` the class is. + + added_match_args (bool): + Whether the class supports positional `match ` over its + fields. + + added_str (bool): + Whether the class has an *attrs*-generated ``__str__`` method. + + added_pickling (bool): + Whether the class has *attrs*-generated ``__getstate__`` and + ``__setstate__`` methods for `pickle`. + + on_setattr_hook (Callable[[Any, Attribute[Any], Any], Any] | None): + The class's ``__setattr__`` hook. + + field_transformer (Callable[[Attribute[Any]], Attribute[Any]] | None): + The class's `field transformers `. + + .. versionadded:: 25.4.0 + """ + + class Hashability(enum.Enum): + """ + The hashability of a class. + + .. versionadded:: 25.4.0 + """ + + HASHABLE = "hashable" + """Write a ``__hash__``.""" + HASHABLE_CACHED = "hashable_cache" + """Write a ``__hash__`` and cache the hash.""" + UNHASHABLE = "unhashable" + """Set ``__hash__`` to ``None``.""" + LEAVE_ALONE = "leave_alone" + """Don't touch ``__hash__``.""" + + class KeywordOnly(enum.Enum): + """ + How attributes should be treated regarding keyword-only parameters. + + .. versionadded:: 25.4.0 + """ + + NO = "no" + """Attributes are not keyword-only.""" + YES = "yes" + """Attributes in current class without kw_only=False are keyword-only.""" + FORCE = "force" + """All attributes are keyword-only.""" + + __slots__ = ( # noqa: RUF023 -- order matters for __init__ + "is_exception", + "is_slotted", + "has_weakref_slot", + "is_frozen", + "kw_only", + "collected_fields_by_mro", + "added_init", + "added_repr", + "added_eq", + "added_ordering", + "hashability", + "added_match_args", + "added_str", + "added_pickling", + "on_setattr_hook", + "field_transformer", + ) + + def __init__( + self, + is_exception, + is_slotted, + has_weakref_slot, + is_frozen, + kw_only, + collected_fields_by_mro, + added_init, + added_repr, + added_eq, + added_ordering, + hashability, + added_match_args, + added_str, + added_pickling, + on_setattr_hook, + field_transformer, + ): + self.is_exception = is_exception + self.is_slotted = is_slotted + self.has_weakref_slot = has_weakref_slot + self.is_frozen = is_frozen + self.kw_only = kw_only + self.collected_fields_by_mro = collected_fields_by_mro + self.added_init = added_init + self.added_repr = added_repr + self.added_eq = added_eq + self.added_ordering = added_ordering + self.hashability = hashability + self.added_match_args = added_match_args + self.added_str = added_str + self.added_pickling = added_pickling + self.on_setattr_hook = on_setattr_hook + self.field_transformer = field_transformer + + @property + def is_hashable(self): + return ( + self.hashability is ClassProps.Hashability.HASHABLE + or self.hashability is ClassProps.Hashability.HASHABLE_CACHED + ) + + +_cas = [ + Attribute( + name=name, + default=NOTHING, + validator=None, + repr=True, + cmp=None, + eq=True, + order=False, + hash=True, + init=True, + inherited=False, + alias=_default_init_alias_for(name), + ) + for name in ClassProps.__slots__ +] + +ClassProps = _add_eq(_add_repr(ClassProps, attrs=_cas), attrs=_cas) + + +class Factory: + """ + Stores a factory callable. + + If passed as the default value to `attrs.field`, the factory is used to + generate a new value. + + Args: + factory (typing.Callable): + A callable that takes either none or exactly one mandatory + positional argument depending on *takes_self*. + + takes_self (bool): + Pass the partially initialized instance that is being initialized + as a positional argument. + + .. versionadded:: 17.1.0 *takes_self* + """ + + __slots__ = ("factory", "takes_self") + + def __init__(self, factory, takes_self=False): + self.factory = factory + self.takes_self = takes_self + + def __getstate__(self): + """ + Play nice with pickle. + """ + return tuple(getattr(self, name) for name in self.__slots__) + + def __setstate__(self, state): + """ + Play nice with pickle. + """ + for name, value in zip(self.__slots__, state): + setattr(self, name, value) + + +_f = [ + Attribute( + name=name, + default=NOTHING, + validator=None, + repr=True, + cmp=None, + eq=True, + order=False, + hash=True, + init=True, + inherited=False, + ) + for name in Factory.__slots__ +] + +Factory = _add_hash(_add_eq(_add_repr(Factory, attrs=_f), attrs=_f), attrs=_f) + + +class Converter: + """ + Stores a converter callable. + + Allows for the wrapped converter to take additional arguments. The + arguments are passed in the order they are documented. + + Args: + converter (Callable): A callable that converts the passed value. + + takes_self (bool): + Pass the partially initialized instance that is being initialized + as a positional argument. (default: `False`) + + takes_field (bool): + Pass the field definition (an :class:`Attribute`) into the + converter as a positional argument. (default: `False`) + + .. versionadded:: 24.1.0 + """ + + __slots__ = ( + "__call__", + "_first_param_type", + "_global_name", + "converter", + "takes_field", + "takes_self", + ) + + def __init__(self, converter, *, takes_self=False, takes_field=False): + self.converter = converter + self.takes_self = takes_self + self.takes_field = takes_field + + ex = _AnnotationExtractor(converter) + self._first_param_type = ex.get_first_param_type() + + if not (self.takes_self or self.takes_field): + self.__call__ = lambda value, _, __: self.converter(value) + elif self.takes_self and not self.takes_field: + self.__call__ = lambda value, instance, __: self.converter( + value, instance + ) + elif not self.takes_self and self.takes_field: + self.__call__ = lambda value, __, field: self.converter( + value, field + ) + else: + self.__call__ = self.converter + + rt = ex.get_return_type() + if rt is not None: + self.__call__.__annotations__["return"] = rt + + @staticmethod + def _get_global_name(attr_name: str) -> str: + """ + Return the name that a converter for an attribute name *attr_name* + would have. + """ + return f"__attr_converter_{attr_name}" + + def _fmt_converter_call(self, attr_name: str, value_var: str) -> str: + """ + Return a string that calls the converter for an attribute name + *attr_name* and the value in variable named *value_var* according to + `self.takes_self` and `self.takes_field`. + """ + if not (self.takes_self or self.takes_field): + return f"{self._get_global_name(attr_name)}({value_var})" + + if self.takes_self and self.takes_field: + return f"{self._get_global_name(attr_name)}({value_var}, self, attr_dict['{attr_name}'])" + + if self.takes_self: + return f"{self._get_global_name(attr_name)}({value_var}, self)" + + return f"{self._get_global_name(attr_name)}({value_var}, attr_dict['{attr_name}'])" + + def __getstate__(self): + """ + Return a dict containing only converter and takes_self -- the rest gets + computed when loading. + """ + return { + "converter": self.converter, + "takes_self": self.takes_self, + "takes_field": self.takes_field, + } + + def __setstate__(self, state): + """ + Load instance from state. + """ + self.__init__(**state) + + +_f = [ + Attribute( + name=name, + default=NOTHING, + validator=None, + repr=True, + cmp=None, + eq=True, + order=False, + hash=True, + init=True, + inherited=False, + ) + for name in ("converter", "takes_self", "takes_field") +] + +Converter = _add_hash( + _add_eq(_add_repr(Converter, attrs=_f), attrs=_f), attrs=_f +) + + +def make_class( + name, attrs, bases=(object,), class_body=None, **attributes_arguments +): + r""" + A quick way to create a new class called *name* with *attrs*. + + .. note:: + + ``make_class()`` is a thin wrapper around `attr.s`, not `attrs.define` + which means that it doesn't come with some of the improved defaults. + + For example, if you want the same ``on_setattr`` behavior as in + `attrs.define`, you have to pass the hooks yourself: ``make_class(..., + on_setattr=setters.pipe(setters.convert, setters.validate)`` + + .. warning:: + + It is *your* duty to ensure that the class name and the attribute names + are valid identifiers. ``make_class()`` will *not* validate them for + you. + + Args: + name (str): The name for the new class. + + attrs (list | dict): + A list of names or a dictionary of mappings of names to `attr.ib`\ + s / `attrs.field`\ s. + + The order is deduced from the order of the names or attributes + inside *attrs*. Otherwise the order of the definition of the + attributes is used. + + bases (tuple[type, ...]): Classes that the new class will subclass. + + class_body (dict): + An optional dictionary of class attributes for the new class. + + attributes_arguments: Passed unmodified to `attr.s`. + + Returns: + type: A new class with *attrs*. + + .. versionadded:: 17.1.0 *bases* + .. versionchanged:: 18.1.0 If *attrs* is ordered, the order is retained. + .. versionchanged:: 23.2.0 *class_body* + .. versionchanged:: 25.2.0 Class names can now be unicode. + """ + # Class identifiers are converted into the normal form NFKC while parsing + name = unicodedata.normalize("NFKC", name) + + if isinstance(attrs, dict): + cls_dict = attrs + elif isinstance(attrs, (list, tuple)): + cls_dict = {a: attrib() for a in attrs} + else: + msg = "attrs argument must be a dict or a list." + raise TypeError(msg) + + pre_init = cls_dict.pop("__attrs_pre_init__", None) + post_init = cls_dict.pop("__attrs_post_init__", None) + user_init = cls_dict.pop("__init__", None) + + body = {} + if class_body is not None: + body.update(class_body) + if pre_init is not None: + body["__attrs_pre_init__"] = pre_init + if post_init is not None: + body["__attrs_post_init__"] = post_init + if user_init is not None: + body["__init__"] = user_init + + type_ = types.new_class(name, bases, {}, lambda ns: ns.update(body)) + + # For pickling to work, the __module__ variable needs to be set to the + # frame where the class is created. Bypass this step in environments where + # sys._getframe is not defined (Jython for example) or sys._getframe is not + # defined for arguments greater than 0 (IronPython). + with contextlib.suppress(AttributeError, ValueError): + type_.__module__ = sys._getframe(1).f_globals.get( + "__name__", "__main__" + ) + + # We do it here for proper warnings with meaningful stacklevel. + cmp = attributes_arguments.pop("cmp", None) + ( + attributes_arguments["eq"], + attributes_arguments["order"], + ) = _determine_attrs_eq_order( + cmp, + attributes_arguments.get("eq"), + attributes_arguments.get("order"), + True, + ) + + cls = _attrs(these=cls_dict, **attributes_arguments)(type_) + # Only add type annotations now or "_attrs()" will complain: + cls.__annotations__ = { + k: v.type for k, v in cls_dict.items() if v.type is not None + } + return cls + + +# These are required by within this module so we define them here and merely +# import into .validators / .converters. + + +@attrs(slots=True, unsafe_hash=True) +class _AndValidator: + """ + Compose many validators to a single one. + """ + + _validators = attrib() + + def __call__(self, inst, attr, value): + for v in self._validators: + v(inst, attr, value) + + +def and_(*validators): + """ + A validator that composes multiple validators into one. + + When called on a value, it runs all wrapped validators. + + Args: + validators (~collections.abc.Iterable[typing.Callable]): + Arbitrary number of validators. + + .. versionadded:: 17.1.0 + """ + vals = [] + for validator in validators: + vals.extend( + validator._validators + if isinstance(validator, _AndValidator) + else [validator] + ) + + return _AndValidator(tuple(vals)) + + +def pipe(*converters): + """ + A converter that composes multiple converters into one. + + When called on a value, it runs all wrapped converters, returning the + *last* value. + + Type annotations will be inferred from the wrapped converters', if they + have any. + + converters (~collections.abc.Iterable[typing.Callable]): + Arbitrary number of converters. + + .. versionadded:: 20.1.0 + """ + + return_instance = any(isinstance(c, Converter) for c in converters) + + if return_instance: + + def pipe_converter(val, inst, field): + for c in converters: + val = ( + c(val, inst, field) if isinstance(c, Converter) else c(val) + ) + + return val + + else: + + def pipe_converter(val): + for c in converters: + val = c(val) + + return val + + if not converters: + # If the converter list is empty, pipe_converter is the identity. + A = TypeVar("A") + pipe_converter.__annotations__.update({"val": A, "return": A}) + else: + # Get parameter type from first converter. + t = _AnnotationExtractor(converters[0]).get_first_param_type() + if t: + pipe_converter.__annotations__["val"] = t + + last = converters[-1] + if not PY_3_11_PLUS and isinstance(last, Converter): + last = last.__call__ + + # Get return type from last converter. + rt = _AnnotationExtractor(last).get_return_type() + if rt: + pipe_converter.__annotations__["return"] = rt + + if return_instance: + return Converter(pipe_converter, takes_self=True, takes_field=True) + return pipe_converter diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_next_gen.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_next_gen.py new file mode 100644 index 0000000000000000000000000000000000000000..4ccd0da2446dc126ce936b054581a527e247cabc --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_next_gen.py @@ -0,0 +1,674 @@ +# SPDX-License-Identifier: MIT + +""" +These are keyword-only APIs that call `attr.s` and `attr.ib` with different +default values. +""" + +from functools import partial + +from . import setters +from ._funcs import asdict as _asdict +from ._funcs import astuple as _astuple +from ._make import ( + _DEFAULT_ON_SETATTR, + NOTHING, + _frozen_setattrs, + attrib, + attrs, +) +from .exceptions import NotAnAttrsClassError, UnannotatedAttributeError + + +def define( + maybe_cls=None, + *, + these=None, + repr=None, + unsafe_hash=None, + hash=None, + init=None, + slots=True, + frozen=False, + weakref_slot=True, + str=False, + auto_attribs=None, + kw_only=False, + cache_hash=False, + auto_exc=True, + eq=None, + order=False, + auto_detect=True, + getstate_setstate=None, + on_setattr=None, + field_transformer=None, + match_args=True, + force_kw_only=False, +): + r""" + A class decorator that adds :term:`dunder methods` according to + :term:`fields ` specified using :doc:`type annotations `, + `field()` calls, or the *these* argument. + + Since *attrs* patches or replaces an existing class, you cannot use + `object.__init_subclass__` with *attrs* classes, because it runs too early. + As a replacement, you can define ``__attrs_init_subclass__`` on your class. + It will be called by *attrs* classes that subclass it after they're + created. See also :ref:`init-subclass`. + + Args: + slots (bool): + Create a :term:`slotted class ` that's more + memory-efficient. Slotted classes are generally superior to the + default dict classes, but have some gotchas you should know about, + so we encourage you to read the :term:`glossary entry `. + + auto_detect (bool): + Instead of setting the *init*, *repr*, *eq*, and *hash* arguments + explicitly, assume they are set to True **unless any** of the + involved methods for one of the arguments is implemented in the + *current* class (meaning, it is *not* inherited from some base + class). + + So, for example by implementing ``__eq__`` on a class yourself, + *attrs* will deduce ``eq=False`` and will create *neither* + ``__eq__`` *nor* ``__ne__`` (but Python classes come with a + sensible ``__ne__`` by default, so it *should* be enough to only + implement ``__eq__`` in most cases). + + Passing :data:`True` or :data:`False` to *init*, *repr*, *eq*, or *hash* + overrides whatever *auto_detect* would determine. + + auto_exc (bool): + If the class subclasses `BaseException` (which implicitly includes + any subclass of any exception), the following happens to behave + like a well-behaved Python exception class: + + - the values for *eq*, *order*, and *hash* are ignored and the + instances compare and hash by the instance's ids [#]_ , + - all attributes that are either passed into ``__init__`` or have a + default value are additionally available as a tuple in the + ``args`` attribute, + - the value of *str* is ignored leaving ``__str__`` to base + classes. + + .. [#] + Note that *attrs* will *not* remove existing implementations of + ``__hash__`` or the equality methods. It just won't add own + ones. + + on_setattr (~typing.Callable | list[~typing.Callable] | None | ~typing.Literal[attrs.setters.NO_OP]): + A callable that is run whenever the user attempts to set an + attribute (either by assignment like ``i.x = 42`` or by using + `setattr` like ``setattr(i, "x", 42)``). It receives the same + arguments as validators: the instance, the attribute that is being + modified, and the new value. + + If no exception is raised, the attribute is set to the return value + of the callable. + + If a list of callables is passed, they're automatically wrapped in + an `attrs.setters.pipe`. + + If left None, the default behavior is to run converters and + validators whenever an attribute is set. + + init (bool): + Create a ``__init__`` method that initializes the *attrs* + attributes. Leading underscores are stripped for the argument name, + unless an alias is set on the attribute. + + .. seealso:: + `init` shows advanced ways to customize the generated + ``__init__`` method, including executing code before and after. + + repr(bool): + Create a ``__repr__`` method with a human readable representation + of *attrs* attributes. + + str (bool): + Create a ``__str__`` method that is identical to ``__repr__``. This + is usually not necessary except for `Exception`\ s. + + eq (bool | None): + If True or None (default), add ``__eq__`` and ``__ne__`` methods + that check two instances for equality. + + .. seealso:: + `comparison` describes how to customize the comparison behavior + going as far comparing NumPy arrays. + + order (bool | None): + If True, add ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` + methods that behave like *eq* above and allow instances to be + ordered. + + They compare the instances as if they were tuples of their *attrs* + attributes if and only if the types of both classes are + *identical*. + + If `None` mirror value of *eq*. + + .. seealso:: `comparison` + + unsafe_hash (bool | None): + If None (default), the ``__hash__`` method is generated according + how *eq* and *frozen* are set. + + 1. If *both* are True, *attrs* will generate a ``__hash__`` for + you. + 2. If *eq* is True and *frozen* is False, ``__hash__`` will be set + to None, marking it unhashable (which it is). + 3. If *eq* is False, ``__hash__`` will be left untouched meaning + the ``__hash__`` method of the base class will be used. If the + base class is `object`, this means it will fall back to id-based + hashing. + + Although not recommended, you can decide for yourself and force + *attrs* to create one (for example, if the class is immutable even + though you didn't freeze it programmatically) by passing True or + not. Both of these cases are rather special and should be used + carefully. + + .. seealso:: + + - Our documentation on `hashing`, + - Python's documentation on `object.__hash__`, + - and the `GitHub issue that led to the default \ behavior + `_ for more + details. + + hash (bool | None): + Deprecated alias for *unsafe_hash*. *unsafe_hash* takes precedence. + + cache_hash (bool): + Ensure that the object's hash code is computed only once and stored + on the object. If this is set to True, hashing must be either + explicitly or implicitly enabled for this class. If the hash code + is cached, avoid any reassignments of fields involved in hash code + computation or mutations of the objects those fields point to after + object creation. If such changes occur, the behavior of the + object's hash code is undefined. + + frozen (bool): + Make instances immutable after initialization. If someone attempts + to modify a frozen instance, `attrs.exceptions.FrozenInstanceError` + is raised. + + .. note:: + + 1. This is achieved by installing a custom ``__setattr__`` + method on your class, so you can't implement your own. + + 2. True immutability is impossible in Python. + + 3. This *does* have a minor a runtime performance `impact + ` when initializing new instances. In other + words: ``__init__`` is slightly slower with ``frozen=True``. + + 4. If a class is frozen, you cannot modify ``self`` in + ``__attrs_post_init__`` or a self-written ``__init__``. You + can circumvent that limitation by using + ``object.__setattr__(self, "attribute_name", value)``. + + 5. Subclasses of a frozen class are frozen too. + + kw_only (bool): + Make attributes keyword-only in the generated ``__init__`` (if + *init* is False, this parameter is ignored). Attributes that + explicitly set ``kw_only=False`` are not affected; base class + attributes are also not affected. + + Also see *force_kw_only*. + + weakref_slot (bool): + Make instances weak-referenceable. This has no effect unless + *slots* is True. + + field_transformer (~typing.Callable | None): + A function that is called with the original class object and all + fields right before *attrs* finalizes the class. You can use this, + for example, to automatically add converters or validators to + fields based on their types. + + .. seealso:: `transform-fields` + + match_args (bool): + If True (default), set ``__match_args__`` on the class to support + :pep:`634` (*Structural Pattern Matching*). It is a tuple of all + non-keyword-only ``__init__`` parameter names on Python 3.10 and + later. Ignored on older Python versions. + + collect_by_mro (bool): + If True, *attrs* collects attributes from base classes correctly + according to the `method resolution order + `_. If False, *attrs* + will mimic the (wrong) behavior of `dataclasses` and :pep:`681`. + + See also `issue #428 + `_. + + force_kw_only (bool): + A back-compat flag for restoring pre-25.4.0 behavior. If True and + ``kw_only=True``, all attributes are made keyword-only, including + base class attributes, and those set to ``kw_only=False`` at the + attribute level. Defaults to False. + + See also `issue #980 + `_. + + getstate_setstate (bool | None): + .. note:: + + This is usually only interesting for slotted classes and you + should probably just set *auto_detect* to True. + + If True, ``__getstate__`` and ``__setstate__`` are generated and + attached to the class. This is necessary for slotted classes to be + pickleable. If left None, it's True by default for slotted classes + and False for dict classes. + + If *auto_detect* is True, and *getstate_setstate* is left None, and + **either** ``__getstate__`` or ``__setstate__`` is detected + directly on the class (meaning: not inherited), it is set to False + (this is usually what you want). + + auto_attribs (bool | None): + If True, look at type annotations to determine which attributes to + use, like `dataclasses`. If False, it will only look for explicit + :func:`field` class attributes, like classic *attrs*. + + If left None, it will guess: + + 1. If any attributes are annotated and no unannotated + `attrs.field`\ s are found, it assumes *auto_attribs=True*. + 2. Otherwise it assumes *auto_attribs=False* and tries to collect + `attrs.field`\ s. + + If *attrs* decides to look at type annotations, **all** fields + **must** be annotated. If *attrs* encounters a field that is set to + a :func:`field` / `attr.ib` but lacks a type annotation, an + `attrs.exceptions.UnannotatedAttributeError` is raised. Use + ``field_name: typing.Any = field(...)`` if you don't want to set a + type. + + .. warning:: + + For features that use the attribute name to create decorators + (for example, :ref:`validators `), you still *must* + assign :func:`field` / `attr.ib` to them. Otherwise Python will + either not find the name or try to use the default value to + call, for example, ``validator`` on it. + + Attributes annotated as `typing.ClassVar`, and attributes that are + neither annotated nor set to an `field()` are **ignored**. + + these (dict[str, object]): + A dictionary of name to the (private) return value of `field()` + mappings. This is useful to avoid the definition of your attributes + within the class body because you can't (for example, if you want + to add ``__repr__`` methods to Django models) or don't want to. + + If *these* is not `None`, *attrs* will *not* search the class body + for attributes and will *not* remove any attributes from it. + + The order is deduced from the order of the attributes inside + *these*. + + Arguably, this is a rather obscure feature. + + .. versionadded:: 20.1.0 + .. versionchanged:: 21.3.0 Converters are also run ``on_setattr``. + .. versionadded:: 22.2.0 + *unsafe_hash* as an alias for *hash* (for :pep:`681` compliance). + .. versionchanged:: 24.1.0 + Instances are not compared as tuples of attributes anymore, but using a + big ``and`` condition. This is faster and has more correct behavior for + uncomparable values like `math.nan`. + .. versionadded:: 24.1.0 + If a class has an *inherited* classmethod called + ``__attrs_init_subclass__``, it is executed after the class is created. + .. deprecated:: 24.1.0 *hash* is deprecated in favor of *unsafe_hash*. + .. versionadded:: 24.3.0 + Unless already present, a ``__replace__`` method is automatically + created for `copy.replace` (Python 3.13+ only). + .. versionchanged:: 25.4.0 + *kw_only* now only applies to attributes defined in the current class, + and respects attribute-level ``kw_only=False`` settings. + .. versionadded:: 25.4.0 + Added *force_kw_only* to go back to the previous *kw_only* behavior. + + .. note:: + + The main differences to the classic `attr.s` are: + + - Automatically detect whether or not *auto_attribs* should be `True` + (c.f. *auto_attribs* parameter). + - Converters and validators run when attributes are set by default -- + if *frozen* is `False`. + - *slots=True* + + Usually, this has only upsides and few visible effects in everyday + programming. But it *can* lead to some surprising behaviors, so + please make sure to read :term:`slotted classes`. + + - *auto_exc=True* + - *auto_detect=True* + - *order=False* + - *force_kw_only=False* + - Some options that were only relevant on Python 2 or were kept around + for backwards-compatibility have been removed. + + """ + + def do_it(cls, auto_attribs): + return attrs( + maybe_cls=cls, + these=these, + repr=repr, + hash=hash, + unsafe_hash=unsafe_hash, + init=init, + slots=slots, + frozen=frozen, + weakref_slot=weakref_slot, + str=str, + auto_attribs=auto_attribs, + kw_only=kw_only, + cache_hash=cache_hash, + auto_exc=auto_exc, + eq=eq, + order=order, + auto_detect=auto_detect, + collect_by_mro=True, + getstate_setstate=getstate_setstate, + on_setattr=on_setattr, + field_transformer=field_transformer, + match_args=match_args, + force_kw_only=force_kw_only, + ) + + def wrap(cls): + """ + Making this a wrapper ensures this code runs during class creation. + + We also ensure that frozen-ness of classes is inherited. + """ + nonlocal frozen, on_setattr + + had_on_setattr = on_setattr not in (None, setters.NO_OP) + + # By default, mutable classes convert & validate on setattr. + if frozen is False and on_setattr is None: + on_setattr = _DEFAULT_ON_SETATTR + + # However, if we subclass a frozen class, we inherit the immutability + # and disable on_setattr. + for base_cls in cls.__bases__: + if base_cls.__setattr__ is _frozen_setattrs: + if had_on_setattr: + msg = "Frozen classes can't use on_setattr (frozen-ness was inherited)." + raise ValueError(msg) + + on_setattr = setters.NO_OP + break + + if auto_attribs is not None: + return do_it(cls, auto_attribs) + + try: + return do_it(cls, True) + except UnannotatedAttributeError: + return do_it(cls, False) + + # maybe_cls's type depends on the usage of the decorator. It's a class + # if it's used as `@attrs` but `None` if used as `@attrs()`. + if maybe_cls is None: + return wrap + + return wrap(maybe_cls) + + +mutable = define +frozen = partial(define, frozen=True, on_setattr=None) + + +def field( + *, + default=NOTHING, + validator=None, + repr=True, + hash=None, + init=True, + metadata=None, + type=None, + converter=None, + factory=None, + kw_only=None, + eq=None, + order=None, + on_setattr=None, + alias=None, +): + """ + Create a new :term:`field` / :term:`attribute` on a class. + + .. warning:: + + Does **nothing** unless the class is also decorated with + `attrs.define` (or similar)! + + Args: + default: + A value that is used if an *attrs*-generated ``__init__`` is used + and no value is passed while instantiating or the attribute is + excluded using ``init=False``. + + If the value is an instance of `attrs.Factory`, its callable will + be used to construct a new value (useful for mutable data types + like lists or dicts). + + If a default is not set (or set manually to `attrs.NOTHING`), a + value *must* be supplied when instantiating; otherwise a + `TypeError` will be raised. + + .. seealso:: `defaults` + + factory (~typing.Callable): + Syntactic sugar for ``default=attr.Factory(factory)``. + + validator (~typing.Callable | list[~typing.Callable]): + Callable that is called by *attrs*-generated ``__init__`` methods + after the instance has been initialized. They receive the + initialized instance, the :func:`~attrs.Attribute`, and the passed + value. + + The return value is *not* inspected so the validator has to throw + an exception itself. + + If a `list` is passed, its items are treated as validators and must + all pass. + + Validators can be globally disabled and re-enabled using + `attrs.validators.get_disabled` / `attrs.validators.set_disabled`. + + The validator can also be set using decorator notation as shown + below. + + .. seealso:: :ref:`validators` + + repr (bool | ~typing.Callable): + Include this attribute in the generated ``__repr__`` method. If + True, include the attribute; if False, omit it. By default, the + built-in ``repr()`` function is used. To override how the attribute + value is formatted, pass a ``callable`` that takes a single value + and returns a string. Note that the resulting string is used as-is, + which means it will be used directly *instead* of calling + ``repr()`` (the default). + + eq (bool | ~typing.Callable): + If True (default), include this attribute in the generated + ``__eq__`` and ``__ne__`` methods that check two instances for + equality. To override how the attribute value is compared, pass a + callable that takes a single value and returns the value to be + compared. + + .. seealso:: `comparison` + + order (bool | ~typing.Callable): + If True (default), include this attributes in the generated + ``__lt__``, ``__le__``, ``__gt__`` and ``__ge__`` methods. To + override how the attribute value is ordered, pass a callable that + takes a single value and returns the value to be ordered. + + .. seealso:: `comparison` + + hash (bool | None): + Include this attribute in the generated ``__hash__`` method. If + None (default), mirror *eq*'s value. This is the correct behavior + according the Python spec. Setting this value to anything else + than None is *discouraged*. + + .. seealso:: `hashing` + + init (bool): + Include this attribute in the generated ``__init__`` method. + + It is possible to set this to False and set a default value. In + that case this attributed is unconditionally initialized with the + specified default value or factory. + + .. seealso:: `init` + + converter (typing.Callable | Converter): + A callable that is called by *attrs*-generated ``__init__`` methods + to convert attribute's value to the desired format. + + If a vanilla callable is passed, it is given the passed-in value as + the only positional argument. It is possible to receive additional + arguments by wrapping the callable in a `Converter`. + + Either way, the returned value will be used as the new value of the + attribute. The value is converted before being passed to the + validator, if any. + + .. seealso:: :ref:`converters` + + metadata (dict | None): + An arbitrary mapping, to be used by third-party code. + + .. seealso:: `extending-metadata`. + + type (type): + The type of the attribute. Nowadays, the preferred method to + specify the type is using a variable annotation (see :pep:`526`). + This argument is provided for backwards-compatibility and for usage + with `make_class`. Regardless of the approach used, the type will + be stored on ``Attribute.type``. + + Please note that *attrs* doesn't do anything with this metadata by + itself. You can use it as part of your own code or for `static type + checking `. + + kw_only (bool | None): + Make this attribute keyword-only in the generated ``__init__`` (if + *init* is False, this parameter is ignored). If None (default), + mirror the setting from `attrs.define`. + + on_setattr (~typing.Callable | list[~typing.Callable] | None | ~typing.Literal[attrs.setters.NO_OP]): + Allows to overwrite the *on_setattr* setting from `attr.s`. If left + None, the *on_setattr* value from `attr.s` is used. Set to + `attrs.setters.NO_OP` to run **no** `setattr` hooks for this + attribute -- regardless of the setting in `define()`. + + alias (str | None): + Override this attribute's parameter name in the generated + ``__init__`` method. If left None, default to ``name`` stripped + of leading underscores. See `private-attributes`. + + .. versionadded:: 20.1.0 + .. versionchanged:: 21.1.0 + *eq*, *order*, and *cmp* also accept a custom callable + .. versionadded:: 22.2.0 *alias* + .. versionadded:: 23.1.0 + The *type* parameter has been re-added; mostly for `attrs.make_class`. + Please note that type checkers ignore this metadata. + .. versionchanged:: 25.4.0 + *kw_only* can now be None, and its default is also changed from False to + None. + + .. seealso:: + + `attr.ib` + """ + return attrib( + default=default, + validator=validator, + repr=repr, + hash=hash, + init=init, + metadata=metadata, + type=type, + converter=converter, + factory=factory, + kw_only=kw_only, + eq=eq, + order=order, + on_setattr=on_setattr, + alias=alias, + ) + + +def asdict(inst, *, recurse=True, filter=None, value_serializer=None): + """ + Same as `attr.asdict`, except that collections types are always retained + and dict is always used as *dict_factory*. + + .. versionadded:: 21.3.0 + """ + return _asdict( + inst=inst, + recurse=recurse, + filter=filter, + value_serializer=value_serializer, + retain_collection_types=True, + ) + + +def astuple(inst, *, recurse=True, filter=None): + """ + Same as `attr.astuple`, except that collections types are always retained + and `tuple` is always used as the *tuple_factory*. + + .. versionadded:: 21.3.0 + """ + return _astuple( + inst=inst, recurse=recurse, filter=filter, retain_collection_types=True + ) + + +def inspect(cls): + """ + Inspect the class and return its effective build parameters. + + Warning: + This feature is currently **experimental** and is not covered by our + strict backwards-compatibility guarantees. + + Args: + cls: The *attrs*-decorated class to inspect. + + Returns: + The effective build parameters of the class. + + Raises: + NotAnAttrsClassError: If the class is not an *attrs*-decorated class. + + .. versionadded:: 25.4.0 + """ + try: + return cls.__dict__["__attrs_props__"] + except KeyError: + msg = f"{cls!r} is not an attrs-decorated class." + raise NotAnAttrsClassError(msg) from None diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_typing_compat.pyi b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_typing_compat.pyi new file mode 100644 index 0000000000000000000000000000000000000000..ca7b71e906a28f88726bbd342fdfe636af0281e7 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_typing_compat.pyi @@ -0,0 +1,15 @@ +from typing import Any, ClassVar, Protocol + +# MYPY is a special constant in mypy which works the same way as `TYPE_CHECKING`. +MYPY = False + +if MYPY: + # A protocol to be able to statically accept an attrs class. + class AttrsInstance_(Protocol): + __attrs_attrs__: ClassVar[Any] + +else: + # For type checkers without plug-in support use an empty protocol that + # will (hopefully) be combined into a union. + class AttrsInstance_(Protocol): + pass diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_version_info.pyi b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_version_info.pyi new file mode 100644 index 0000000000000000000000000000000000000000..45ced086337783c4b73b26cd17d2c1c260e24029 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/_version_info.pyi @@ -0,0 +1,9 @@ +class VersionInfo: + @property + def year(self) -> int: ... + @property + def minor(self) -> int: ... + @property + def micro(self) -> int: ... + @property + def releaselevel(self) -> str: ... diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/converters.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/converters.py new file mode 100644 index 0000000000000000000000000000000000000000..0a79deef04282fb33a42f6aca59563d49e70d4cb --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/converters.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: MIT + +""" +Commonly useful converters. +""" + +import typing + +from ._compat import _AnnotationExtractor +from ._make import NOTHING, Converter, Factory, pipe + + +__all__ = [ + "default_if_none", + "optional", + "pipe", + "to_bool", +] + + +def optional(converter): + """ + A converter that allows an attribute to be optional. An optional attribute + is one which can be set to `None`. + + Type annotations will be inferred from the wrapped converter's, if it has + any. + + Args: + converter (typing.Callable): + the converter that is used for non-`None` values. + + .. versionadded:: 17.1.0 + """ + + if isinstance(converter, Converter): + + def optional_converter(val, inst, field): + if val is None: + return None + return converter(val, inst, field) + + else: + + def optional_converter(val): + if val is None: + return None + return converter(val) + + xtr = _AnnotationExtractor(converter) + + t = xtr.get_first_param_type() + if t: + optional_converter.__annotations__["val"] = typing.Optional[t] + + rt = xtr.get_return_type() + if rt: + optional_converter.__annotations__["return"] = typing.Optional[rt] + + if isinstance(converter, Converter): + return Converter(optional_converter, takes_self=True, takes_field=True) + + return optional_converter + + +def default_if_none(default=NOTHING, factory=None): + """ + A converter that allows to replace `None` values by *default* or the result + of *factory*. + + Args: + default: + Value to be used if `None` is passed. Passing an instance of + `attrs.Factory` is supported, however the ``takes_self`` option is + *not*. + + factory (typing.Callable): + A callable that takes no parameters whose result is used if `None` + is passed. + + Raises: + TypeError: If **neither** *default* or *factory* is passed. + + TypeError: If **both** *default* and *factory* are passed. + + ValueError: + If an instance of `attrs.Factory` is passed with + ``takes_self=True``. + + .. versionadded:: 18.2.0 + """ + if default is NOTHING and factory is None: + msg = "Must pass either `default` or `factory`." + raise TypeError(msg) + + if default is not NOTHING and factory is not None: + msg = "Must pass either `default` or `factory` but not both." + raise TypeError(msg) + + if factory is not None: + default = Factory(factory) + + if isinstance(default, Factory): + if default.takes_self: + msg = "`takes_self` is not supported by default_if_none." + raise ValueError(msg) + + def default_if_none_converter(val): + if val is not None: + return val + + return default.factory() + + else: + + def default_if_none_converter(val): + if val is not None: + return val + + return default + + return default_if_none_converter + + +def to_bool(val): + """ + Convert "boolean" strings (for example, from environment variables) to real + booleans. + + Values mapping to `True`: + + - ``True`` + - ``"true"`` / ``"t"`` + - ``"yes"`` / ``"y"`` + - ``"on"`` + - ``"1"`` + - ``1`` + + Values mapping to `False`: + + - ``False`` + - ``"false"`` / ``"f"`` + - ``"no"`` / ``"n"`` + - ``"off"`` + - ``"0"`` + - ``0`` + + Raises: + ValueError: For any other value. + + .. versionadded:: 21.3.0 + """ + if isinstance(val, str): + val = val.lower() + + if val in (True, "true", "t", "yes", "y", "on", "1", 1): + return True + if val in (False, "false", "f", "no", "n", "off", "0", 0): + return False + + msg = f"Cannot convert value to bool: {val!r}" + raise ValueError(msg) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/converters.pyi b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/converters.pyi new file mode 100644 index 0000000000000000000000000000000000000000..12bd0c4f17bdc60fb8904598af0a3d56d5874a9e --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/converters.pyi @@ -0,0 +1,19 @@ +from typing import Callable, Any, overload + +from attrs import _ConverterType, _CallableConverterType + +@overload +def pipe(*validators: _CallableConverterType) -> _CallableConverterType: ... +@overload +def pipe(*validators: _ConverterType) -> _ConverterType: ... +@overload +def optional(converter: _CallableConverterType) -> _CallableConverterType: ... +@overload +def optional(converter: _ConverterType) -> _ConverterType: ... +@overload +def default_if_none(default: Any) -> _CallableConverterType: ... +@overload +def default_if_none( + *, factory: Callable[[], Any] +) -> _CallableConverterType: ... +def to_bool(val: str | int | bool) -> bool: ... diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/exceptions.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..a207df4028f7800f964c2917383ad26be61b665e --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/exceptions.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: MIT + +from __future__ import annotations + + +class FrozenError(AttributeError): + """ + A frozen/immutable instance or attribute have been attempted to be + modified. + + It mirrors the behavior of ``namedtuples`` by using the same error message + and subclassing `AttributeError`. + + .. versionadded:: 20.1.0 + """ + + def __init__(self): + msg = "can't set attribute" + super().__init__(msg) + self.msg = msg + + +class FrozenInstanceError(FrozenError): + """ + A frozen instance has been attempted to be modified. + + .. versionadded:: 16.1.0 + """ + + +class FrozenAttributeError(FrozenError): + """ + A frozen attribute has been attempted to be modified. + + .. versionadded:: 20.1.0 + """ + + +class AttrsAttributeNotFoundError(ValueError): + """ + An *attrs* function couldn't find an attribute that the user asked for. + + .. versionadded:: 16.2.0 + """ + + +class NotAnAttrsClassError(ValueError): + """ + A non-*attrs* class has been passed into an *attrs* function. + + .. versionadded:: 16.2.0 + """ + + +class DefaultAlreadySetError(RuntimeError): + """ + A default has been set when defining the field and is attempted to be reset + using the decorator. + + .. versionadded:: 17.1.0 + """ + + +class UnannotatedAttributeError(RuntimeError): + """ + A class with ``auto_attribs=True`` has a field without a type annotation. + + .. versionadded:: 17.3.0 + """ + + +class PythonTooOldError(RuntimeError): + """ + It was attempted to use an *attrs* feature that requires a newer Python + version. + + .. versionadded:: 18.2.0 + """ + + +class NotCallableError(TypeError): + """ + A field requiring a callable has been set with a value that is not + callable. + + .. versionadded:: 19.2.0 + """ + + def __init__(self, msg, value): + super(TypeError, self).__init__(msg, value) + self.msg = msg + self.value = value + + def __str__(self): + return str(self.msg) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/exceptions.pyi b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/exceptions.pyi new file mode 100644 index 0000000000000000000000000000000000000000..f2680118b404db8f5227d04d27e8439331341c4d --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/exceptions.pyi @@ -0,0 +1,17 @@ +from typing import Any + +class FrozenError(AttributeError): + msg: str = ... + +class FrozenInstanceError(FrozenError): ... +class FrozenAttributeError(FrozenError): ... +class AttrsAttributeNotFoundError(ValueError): ... +class NotAnAttrsClassError(ValueError): ... +class DefaultAlreadySetError(RuntimeError): ... +class UnannotatedAttributeError(RuntimeError): ... +class PythonTooOldError(RuntimeError): ... + +class NotCallableError(TypeError): + msg: str = ... + value: Any = ... + def __init__(self, msg: str, value: Any) -> None: ... diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/filters.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/filters.py new file mode 100644 index 0000000000000000000000000000000000000000..689b1705a60ff110d6077bab996f8b4588e55b82 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/filters.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: MIT + +""" +Commonly useful filters for `attrs.asdict` and `attrs.astuple`. +""" + +from ._make import Attribute + + +def _split_what(what): + """ + Returns a tuple of `frozenset`s of classes and attributes. + """ + return ( + frozenset(cls for cls in what if isinstance(cls, type)), + frozenset(cls for cls in what if isinstance(cls, str)), + frozenset(cls for cls in what if isinstance(cls, Attribute)), + ) + + +def include(*what): + """ + Create a filter that only allows *what*. + + Args: + what (list[type, str, attrs.Attribute]): + What to include. Can be a type, a name, or an attribute. + + Returns: + Callable: + A callable that can be passed to `attrs.asdict`'s and + `attrs.astuple`'s *filter* argument. + + .. versionchanged:: 23.1.0 Accept strings with field names. + """ + cls, names, attrs = _split_what(what) + + def include_(attribute, value): + return ( + value.__class__ in cls + or attribute.name in names + or attribute in attrs + ) + + return include_ + + +def exclude(*what): + """ + Create a filter that does **not** allow *what*. + + Args: + what (list[type, str, attrs.Attribute]): + What to exclude. Can be a type, a name, or an attribute. + + Returns: + Callable: + A callable that can be passed to `attrs.asdict`'s and + `attrs.astuple`'s *filter* argument. + + .. versionchanged:: 23.3.0 Accept field name string as input argument + """ + cls, names, attrs = _split_what(what) + + def exclude_(attribute, value): + return not ( + value.__class__ in cls + or attribute.name in names + or attribute in attrs + ) + + return exclude_ diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/filters.pyi b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/filters.pyi new file mode 100644 index 0000000000000000000000000000000000000000..974abdcdb51152393d9c9e460c21aa025c45880c --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/filters.pyi @@ -0,0 +1,6 @@ +from typing import Any + +from . import Attribute, _FilterType + +def include(*what: type | str | Attribute[Any]) -> _FilterType[Any]: ... +def exclude(*what: type | str | Attribute[Any]) -> _FilterType[Any]: ... diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/py.typed b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/setters.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/setters.py new file mode 100644 index 0000000000000000000000000000000000000000..78b08398a6713fc5fa827c2dc853e0d05de743c4 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/setters.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: MIT + +""" +Commonly used hooks for on_setattr. +""" + +from . import _config +from .exceptions import FrozenAttributeError + + +def pipe(*setters): + """ + Run all *setters* and return the return value of the last one. + + .. versionadded:: 20.1.0 + """ + + def wrapped_pipe(instance, attrib, new_value): + rv = new_value + + for setter in setters: + rv = setter(instance, attrib, rv) + + return rv + + return wrapped_pipe + + +def frozen(_, __, ___): + """ + Prevent an attribute to be modified. + + .. versionadded:: 20.1.0 + """ + raise FrozenAttributeError + + +def validate(instance, attrib, new_value): + """ + Run *attrib*'s validator on *new_value* if it has one. + + .. versionadded:: 20.1.0 + """ + if _config._run_validators is False: + return new_value + + v = attrib.validator + if not v: + return new_value + + v(instance, attrib, new_value) + + return new_value + + +def convert(instance, attrib, new_value): + """ + Run *attrib*'s converter -- if it has one -- on *new_value* and return the + result. + + .. versionadded:: 20.1.0 + """ + c = attrib.converter + if c: + # This can be removed once we drop 3.8 and use attrs.Converter instead. + from ._make import Converter + + if not isinstance(c, Converter): + return c(new_value) + + return c(new_value, instance, attrib) + + return new_value + + +# Sentinel for disabling class-wide *on_setattr* hooks for certain attributes. +# Sphinx's autodata stopped working, so the docstring is inlined in the API +# docs. +NO_OP = object() diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/setters.pyi b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/setters.pyi new file mode 100644 index 0000000000000000000000000000000000000000..73abf36e7d5b0f5f56e7fddeee716824c1c60d58 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/setters.pyi @@ -0,0 +1,20 @@ +from typing import Any, NewType, NoReturn, TypeVar + +from . import Attribute +from attrs import _OnSetAttrType + +_T = TypeVar("_T") + +def frozen( + instance: Any, attribute: Attribute[Any], new_value: Any +) -> NoReturn: ... +def pipe(*setters: _OnSetAttrType) -> _OnSetAttrType: ... +def validate(instance: Any, attribute: Attribute[_T], new_value: _T) -> _T: ... + +# convert is allowed to return Any, because they can be chained using pipe. +def convert( + instance: Any, attribute: Attribute[Any], new_value: Any +) -> Any: ... + +_NoOpType = NewType("_NoOpType", object) +NO_OP: _NoOpType diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/validators.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/validators.py new file mode 100644 index 0000000000000000000000000000000000000000..0b1a294432d294c4f154be2d9439d825c3ec0781 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/validators.py @@ -0,0 +1,750 @@ +# SPDX-License-Identifier: MIT + +""" +Commonly useful validators. +""" + +import operator +import re + +from contextlib import contextmanager +from re import Pattern + +from ._config import get_run_validators, set_run_validators +from ._make import _AndValidator, and_, attrib, attrs +from .converters import default_if_none +from .exceptions import NotCallableError + + +__all__ = [ + "and_", + "deep_iterable", + "deep_mapping", + "disabled", + "ge", + "get_disabled", + "gt", + "in_", + "instance_of", + "is_callable", + "le", + "lt", + "matches_re", + "max_len", + "min_len", + "not_", + "optional", + "or_", + "set_disabled", +] + + +def set_disabled(disabled): + """ + Globally disable or enable running validators. + + By default, they are run. + + Args: + disabled (bool): If `True`, disable running all validators. + + .. warning:: + + This function is not thread-safe! + + .. versionadded:: 21.3.0 + """ + set_run_validators(not disabled) + + +def get_disabled(): + """ + Return a bool indicating whether validators are currently disabled or not. + + Returns: + bool:`True` if validators are currently disabled. + + .. versionadded:: 21.3.0 + """ + return not get_run_validators() + + +@contextmanager +def disabled(): + """ + Context manager that disables running validators within its context. + + .. warning:: + + This context manager is not thread-safe! + + .. versionadded:: 21.3.0 + .. versionchanged:: 26.1.0 The contextmanager is nestable. + """ + prev = get_run_validators() + set_run_validators(False) + try: + yield + finally: + set_run_validators(prev) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _InstanceOfValidator: + type = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not isinstance(value, self.type): + msg = f"'{attr.name}' must be {self.type!r} (got {value!r} that is a {value.__class__!r})." + raise TypeError( + msg, + attr, + self.type, + value, + ) + + def __repr__(self): + return f"" + + +def instance_of(type): + """ + A validator that raises a `TypeError` if the initializer is called with a + wrong type for this particular attribute (checks are performed using + `isinstance` therefore it's also valid to pass a tuple of types). + + Args: + type (type | tuple[type]): The type to check for. + + Raises: + TypeError: + With a human readable error message, the attribute (of type + `attrs.Attribute`), the expected type, and the value it got. + """ + return _InstanceOfValidator(type) + + +@attrs(repr=False, frozen=True, slots=True) +class _MatchesReValidator: + pattern = attrib() + match_func = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not self.match_func(value): + msg = f"'{attr.name}' must match regex {self.pattern.pattern!r} ({value!r} doesn't)" + raise ValueError( + msg, + attr, + self.pattern, + value, + ) + + def __repr__(self): + return f"" + + +def matches_re(regex, flags=0, func=None): + r""" + A validator that raises `ValueError` if the initializer is called with a + string that doesn't match *regex*. + + Args: + regex (str, re.Pattern): + A regex string or precompiled pattern to match against + + flags (int): + Flags that will be passed to the underlying re function (default 0) + + func (typing.Callable): + Which underlying `re` function to call. Valid options are + `re.fullmatch`, `re.search`, and `re.match`; the default `None` + means `re.fullmatch`. For performance reasons, the pattern is + always precompiled using `re.compile`. + + .. versionadded:: 19.2.0 + .. versionchanged:: 21.3.0 *regex* can be a pre-compiled pattern. + """ + valid_funcs = (re.fullmatch, None, re.search, re.match) + if func not in valid_funcs: + msg = "'func' must be one of {}.".format( + ", ".join( + sorted((e and e.__name__) or "None" for e in set(valid_funcs)) + ) + ) + raise ValueError(msg) + + if isinstance(regex, Pattern): + if flags: + msg = "'flags' can only be used with a string pattern; pass flags to re.compile() instead" + raise TypeError(msg) + pattern = regex + else: + pattern = re.compile(regex, flags) + + if func is re.match: + match_func = pattern.match + elif func is re.search: + match_func = pattern.search + else: + match_func = pattern.fullmatch + + return _MatchesReValidator(pattern, match_func) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _OptionalValidator: + validator = attrib() + + def __call__(self, inst, attr, value): + if value is None: + return + + self.validator(inst, attr, value) + + def __repr__(self): + return f"" + + +def optional(validator): + """ + A validator that makes an attribute optional. An optional attribute is one + which can be set to `None` in addition to satisfying the requirements of + the sub-validator. + + Args: + validator + (typing.Callable | tuple[typing.Callable] | list[typing.Callable]): + A validator (or validators) that is used for non-`None` values. + + .. versionadded:: 15.1.0 + .. versionchanged:: 17.1.0 *validator* can be a list of validators. + .. versionchanged:: 23.1.0 *validator* can also be a tuple of validators. + """ + if isinstance(validator, (list, tuple)): + return _OptionalValidator(_AndValidator(validator)) + + return _OptionalValidator(validator) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _InValidator: + options = attrib() + _original_options = attrib(hash=False) + + def __call__(self, inst, attr, value): + try: + in_options = value in self.options + except TypeError: # e.g. `1 in "abc"` + in_options = False + + if not in_options: + msg = f"'{attr.name}' must be in {self._original_options!r} (got {value!r})" + raise ValueError( + msg, + attr, + self._original_options, + value, + ) + + def __repr__(self): + return f"" + + +def in_(options): + """ + A validator that raises a `ValueError` if the initializer is called with a + value that does not belong in the *options* provided. + + The check is performed using ``value in options``, so *options* has to + support that operation. + + To keep the validator hashable, dicts, lists, and sets are transparently + transformed into a `tuple`. + + Args: + options: Allowed options. + + Raises: + ValueError: + With a human readable error message, the attribute (of type + `attrs.Attribute`), the expected options, and the value it got. + + .. versionadded:: 17.1.0 + .. versionchanged:: 22.1.0 + The ValueError was incomplete until now and only contained the human + readable error message. Now it contains all the information that has + been promised since 17.1.0. + .. versionchanged:: 24.1.0 + *options* that are a list, dict, or a set are now transformed into a + tuple to keep the validator hashable. + """ + repr_options = options + if isinstance(options, (list, dict, set)): + options = tuple(options) + + return _InValidator(options, repr_options) + + +@attrs(repr=False, slots=False, unsafe_hash=True) +class _IsCallableValidator: + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not callable(value): + message = ( + "'{name}' must be callable " + "(got {value!r} that is a {actual!r})." + ) + raise NotCallableError( + msg=message.format( + name=attr.name, value=value, actual=value.__class__ + ), + value=value, + ) + + def __repr__(self): + return "" + + +def is_callable(): + """ + A validator that raises a `attrs.exceptions.NotCallableError` if the + initializer is called with a value for this particular attribute that is + not callable. + + .. versionadded:: 19.1.0 + + Raises: + attrs.exceptions.NotCallableError: + With a human readable error message containing the attribute + (`attrs.Attribute`) name, and the value it got. + """ + return _IsCallableValidator() + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _DeepIterable: + member_validator = attrib(validator=is_callable()) + iterable_validator = attrib( + default=None, validator=optional(is_callable()) + ) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.iterable_validator is not None: + self.iterable_validator(inst, attr, value) + + for member in value: + self.member_validator(inst, attr, member) + + def __repr__(self): + iterable_identifier = ( + "" + if self.iterable_validator is None + else f" {self.iterable_validator!r}" + ) + return ( + f"" + ) + + +def deep_iterable(member_validator, iterable_validator=None): + """ + A validator that performs deep validation of an iterable. + + Args: + member_validator: Validator(s) to apply to iterable members. + + iterable_validator: + Validator(s) to apply to iterable itself (optional). + + Raises + TypeError: if any sub-validators fail + + .. versionadded:: 19.1.0 + + .. versionchanged:: 25.4.0 + *member_validator* and *iterable_validator* can now be a list or tuple + of validators. + """ + if isinstance(member_validator, (list, tuple)): + member_validator = and_(*member_validator) + if isinstance(iterable_validator, (list, tuple)): + iterable_validator = and_(*iterable_validator) + return _DeepIterable(member_validator, iterable_validator) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _DeepMapping: + key_validator = attrib(validator=optional(is_callable())) + value_validator = attrib(validator=optional(is_callable())) + mapping_validator = attrib(validator=optional(is_callable())) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.mapping_validator is not None: + self.mapping_validator(inst, attr, value) + + for key in value: + if self.key_validator is not None: + self.key_validator(inst, attr, key) + if self.value_validator is not None: + self.value_validator(inst, attr, value[key]) + + def __repr__(self): + return f"" + + +def deep_mapping( + key_validator=None, value_validator=None, mapping_validator=None +): + """ + A validator that performs deep validation of a dictionary. + + All validators are optional, but at least one of *key_validator* or + *value_validator* must be provided. + + Args: + key_validator: Validator(s) to apply to dictionary keys. + + value_validator: Validator(s) to apply to dictionary values. + + mapping_validator: + Validator(s) to apply to top-level mapping attribute. + + .. versionadded:: 19.1.0 + + .. versionchanged:: 25.4.0 + *key_validator* and *value_validator* are now optional, but at least one + of them must be provided. + + .. versionchanged:: 25.4.0 + *key_validator*, *value_validator*, and *mapping_validator* can now be a + list or tuple of validators. + + Raises: + TypeError: If any sub-validator fails on validation. + + ValueError: + If neither *key_validator* nor *value_validator* is provided on + instantiation. + """ + if key_validator is None and value_validator is None: + msg = ( + "At least one of key_validator or value_validator must be provided" + ) + raise ValueError(msg) + + if isinstance(key_validator, (list, tuple)): + key_validator = and_(*key_validator) + if isinstance(value_validator, (list, tuple)): + value_validator = and_(*value_validator) + if isinstance(mapping_validator, (list, tuple)): + mapping_validator = and_(*mapping_validator) + + return _DeepMapping(key_validator, value_validator, mapping_validator) + + +@attrs(repr=False, frozen=True, slots=True) +class _NumberValidator: + bound = attrib() + compare_op = attrib() + compare_func = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not self.compare_func(value, self.bound): + msg = f"'{attr.name}' must be {self.compare_op} {self.bound}: {value}" + raise ValueError(msg) + + def __repr__(self): + return f"" + + +def lt(val): + """ + A validator that raises `ValueError` if the initializer is called with a + number larger or equal to *val*. + + The validator uses `operator.lt` to compare the values. + + Args: + val: Exclusive upper bound for values. + + .. versionadded:: 21.3.0 + """ + return _NumberValidator(val, "<", operator.lt) + + +def le(val): + """ + A validator that raises `ValueError` if the initializer is called with a + number greater than *val*. + + The validator uses `operator.le` to compare the values. + + Args: + val: Inclusive upper bound for values. + + .. versionadded:: 21.3.0 + """ + return _NumberValidator(val, "<=", operator.le) + + +def ge(val): + """ + A validator that raises `ValueError` if the initializer is called with a + number smaller than *val*. + + The validator uses `operator.ge` to compare the values. + + Args: + val: Inclusive lower bound for values + + .. versionadded:: 21.3.0 + """ + return _NumberValidator(val, ">=", operator.ge) + + +def gt(val): + """ + A validator that raises `ValueError` if the initializer is called with a + number smaller or equal to *val*. + + The validator uses `operator.gt` to compare the values. + + Args: + val: Exclusive lower bound for values + + .. versionadded:: 21.3.0 + """ + return _NumberValidator(val, ">", operator.gt) + + +@attrs(repr=False, frozen=True, slots=True) +class _MaxLengthValidator: + max_length = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if len(value) > self.max_length: + msg = f"Length of '{attr.name}' must be <= {self.max_length}: {len(value)}" + raise ValueError(msg) + + def __repr__(self): + return f"" + + +def max_len(length): + """ + A validator that raises `ValueError` if the initializer is called + with a string or iterable that is longer than *length*. + + Args: + length (int): Maximum length of the string or iterable + + .. versionadded:: 21.3.0 + """ + return _MaxLengthValidator(length) + + +@attrs(repr=False, frozen=True, slots=True) +class _MinLengthValidator: + min_length = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if len(value) < self.min_length: + msg = f"Length of '{attr.name}' must be >= {self.min_length}: {len(value)}" + raise ValueError(msg) + + def __repr__(self): + return f"" + + +def min_len(length): + """ + A validator that raises `ValueError` if the initializer is called + with a string or iterable that is shorter than *length*. + + Args: + length (int): Minimum length of the string or iterable + + .. versionadded:: 22.1.0 + """ + return _MinLengthValidator(length) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _SubclassOfValidator: + type = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not issubclass(value, self.type): + msg = f"'{attr.name}' must be a subclass of {self.type!r} (got {value!r})." + raise TypeError( + msg, + attr, + self.type, + value, + ) + + def __repr__(self): + return f"" + + +def _subclass_of(type): + """ + A validator that raises a `TypeError` if the initializer is called with a + wrong type for this particular attribute (checks are performed using + `issubclass` therefore it's also valid to pass a tuple of types). + + Args: + type (type | tuple[type, ...]): The type(s) to check for. + + Raises: + TypeError: + With a human readable error message, the attribute (of type + `attrs.Attribute`), the expected type, and the value it got. + """ + return _SubclassOfValidator(type) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _NotValidator: + validator = attrib() + msg = attrib( + converter=default_if_none( + "not_ validator child '{validator!r}' " + "did not raise a captured error" + ) + ) + exc_types = attrib( + validator=deep_iterable( + member_validator=_subclass_of(Exception), + iterable_validator=instance_of(tuple), + ), + ) + + def __call__(self, inst, attr, value): + try: + self.validator(inst, attr, value) + except self.exc_types: + pass # suppress error to invert validity + else: + raise ValueError( + self.msg.format( + validator=self.validator, + exc_types=self.exc_types, + ), + attr, + self.validator, + value, + self.exc_types, + ) + + def __repr__(self): + return f"" + + +def not_(validator, *, msg=None, exc_types=(ValueError, TypeError)): + """ + A validator that wraps and logically 'inverts' the validator passed to it. + It will raise a `ValueError` if the provided validator *doesn't* raise a + `ValueError` or `TypeError` (by default), and will suppress the exception + if the provided validator *does*. + + Intended to be used with existing validators to compose logic without + needing to create inverted variants, for example, ``not_(in_(...))``. + + Args: + validator: A validator to be logically inverted. + + msg (str): + Message to raise if validator fails. Formatted with keys + ``exc_types`` and ``validator``. + + exc_types (tuple[type, ...]): + Exception type(s) to capture. Other types raised by child + validators will not be intercepted and pass through. + + Raises: + ValueError: + With a human readable error message, the attribute (of type + `attrs.Attribute`), the validator that failed to raise an + exception, the value it got, and the expected exception types. + + .. versionadded:: 22.2.0 + """ + try: + exc_types = tuple(exc_types) + except TypeError: + exc_types = (exc_types,) + return _NotValidator(validator, msg, exc_types) + + +@attrs(repr=False, slots=True, unsafe_hash=True) +class _OrValidator: + validators = attrib() + + def __call__(self, inst, attr, value): + for v in self.validators: + try: + v(inst, attr, value) + except Exception: # noqa: BLE001, PERF203, S112 + continue + else: + return + + msg = f"None of {self.validators!r} satisfied for value {value!r}" + raise ValueError(msg) + + def __repr__(self): + return f"" + + +def or_(*validators): + """ + A validator that composes multiple validators into one. + + When called on a value, it runs all wrapped validators until one of them is + satisfied. + + Args: + validators (~collections.abc.Iterable[typing.Callable]): + Arbitrary number of validators. + + Raises: + ValueError: + If no validator is satisfied. Raised with a human-readable error + message listing all the wrapped validators and the value that + failed all of them. + + .. versionadded:: 24.1.0 + """ + vals = [] + for v in validators: + vals.extend(v.validators if isinstance(v, _OrValidator) else [v]) + + return _OrValidator(tuple(vals)) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attr/validators.pyi b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/validators.pyi new file mode 100644 index 0000000000000000000000000000000000000000..18fb112c89562c2b9625645e0babe1207a02675f --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attr/validators.pyi @@ -0,0 +1,140 @@ +from types import UnionType +from typing import ( + Any, + AnyStr, + Callable, + Container, + ContextManager, + Iterable, + Mapping, + Match, + Pattern, + TypeVar, + overload, +) + +from attrs import _ValidatorType +from attrs import _ValidatorArgType + +_T = TypeVar("_T") +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_T3 = TypeVar("_T3") +_T4 = TypeVar("_T4") +_T5 = TypeVar("_T5") +_T6 = TypeVar("_T6") +_I = TypeVar("_I", bound=Iterable) +_K = TypeVar("_K") +_V = TypeVar("_V") +_M = TypeVar("_M", bound=Mapping) + +def set_disabled(run: bool) -> None: ... +def get_disabled() -> bool: ... +def disabled() -> ContextManager[None]: ... + +# To be more precise on instance_of use some overloads. +# If there are more than 3 items in the tuple then we fall back to Any +@overload +def instance_of(type: type[_T]) -> _ValidatorType[_T]: ... +@overload +def instance_of(type: tuple[type[_T]]) -> _ValidatorType[_T]: ... +@overload +def instance_of( + type: tuple[type[_T1], type[_T2]], +) -> _ValidatorType[_T1 | _T2]: ... +@overload +def instance_of( + type: tuple[type[_T1], type[_T2], type[_T3]], +) -> _ValidatorType[_T1 | _T2 | _T3]: ... +@overload +def instance_of(type: tuple[type, ...]) -> _ValidatorType[Any]: ... +@overload +def instance_of(type: UnionType) -> _ValidatorType[Any]: ... +def optional( + validator: ( + _ValidatorType[_T] + | list[_ValidatorType[_T]] + | tuple[_ValidatorType[_T], ...] + ), +) -> _ValidatorType[_T | None]: ... +def in_(options: Container[_T]) -> _ValidatorType[_T]: ... +def and_(*validators: _ValidatorType[_T]) -> _ValidatorType[_T]: ... +def matches_re( + regex: Pattern[AnyStr] | AnyStr, + flags: int = ..., + func: Callable[[AnyStr, AnyStr, int], Match[AnyStr] | None] | None = ..., +) -> _ValidatorType[AnyStr]: ... +def deep_iterable( + member_validator: _ValidatorArgType[_T], + iterable_validator: _ValidatorArgType[_I] | None = ..., +) -> _ValidatorType[_I]: ... +@overload +def deep_mapping( + key_validator: _ValidatorArgType[_K], + value_validator: _ValidatorArgType[_V] | None = ..., + mapping_validator: _ValidatorArgType[_M] | None = ..., +) -> _ValidatorType[_M]: ... +@overload +def deep_mapping( + key_validator: _ValidatorArgType[_K] | None = ..., + value_validator: _ValidatorArgType[_V] = ..., + mapping_validator: _ValidatorArgType[_M] | None = ..., +) -> _ValidatorType[_M]: ... +def is_callable() -> _ValidatorType[_T]: ... +def lt(val: _T) -> _ValidatorType[_T]: ... +def le(val: _T) -> _ValidatorType[_T]: ... +def ge(val: _T) -> _ValidatorType[_T]: ... +def gt(val: _T) -> _ValidatorType[_T]: ... +def max_len(length: int) -> _ValidatorType[_T]: ... +def min_len(length: int) -> _ValidatorType[_T]: ... +def not_( + validator: _ValidatorType[_T], + *, + msg: str | None = None, + exc_types: type[Exception] | Iterable[type[Exception]] = ..., +) -> _ValidatorType[_T]: ... +@overload +def or_( + __v1: _ValidatorType[_T1], + __v2: _ValidatorType[_T2], +) -> _ValidatorType[_T1 | _T2]: ... +@overload +def or_( + __v1: _ValidatorType[_T1], + __v2: _ValidatorType[_T2], + __v3: _ValidatorType[_T3], +) -> _ValidatorType[_T1 | _T2 | _T3]: ... +@overload +def or_( + __v1: _ValidatorType[_T1], + __v2: _ValidatorType[_T2], + __v3: _ValidatorType[_T3], + __v4: _ValidatorType[_T4], +) -> _ValidatorType[_T1 | _T2 | _T3 | _T4]: ... +@overload +def or_( + __v1: _ValidatorType[_T1], + __v2: _ValidatorType[_T2], + __v3: _ValidatorType[_T3], + __v4: _ValidatorType[_T4], + __v5: _ValidatorType[_T5], +) -> _ValidatorType[_T1 | _T2 | _T3 | _T4 | _T5]: ... +@overload +def or_( + __v1: _ValidatorType[_T1], + __v2: _ValidatorType[_T2], + __v3: _ValidatorType[_T3], + __v4: _ValidatorType[_T4], + __v5: _ValidatorType[_T5], + __v6: _ValidatorType[_T6], +) -> _ValidatorType[_T1 | _T2 | _T3 | _T4 | _T5 | _T6]: ... +@overload +def or_( + __v1: _ValidatorType[Any], + __v2: _ValidatorType[Any], + __v3: _ValidatorType[Any], + __v4: _ValidatorType[Any], + __v5: _ValidatorType[Any], + __v6: _ValidatorType[Any], + *validators: _ValidatorType[Any], +) -> _ValidatorType[Any]: ... diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/INSTALLER b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/METADATA b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..5cf16a040efe125c907111f521c576edde40109f --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/METADATA @@ -0,0 +1,199 @@ +Metadata-Version: 2.4 +Name: attrs +Version: 26.1.0 +Summary: Classes Without Boilerplate +Project-URL: Documentation, https://www.attrs.org/ +Project-URL: Changelog, https://www.attrs.org/en/stable/changelog.html +Project-URL: GitHub, https://github.com/python-attrs/attrs +Project-URL: Funding, https://github.com/sponsors/hynek +Project-URL: Tidelift, https://tidelift.com/subscription/pkg/pypi-attrs?utm_source=pypi-attrs&utm_medium=pypi +Author-email: Hynek Schlawack +License-Expression: MIT +License-File: LICENSE +Keywords: attribute,boilerplate,class +Classifier: Development Status :: 5 - Production/Stable +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Typing :: Typed +Requires-Python: >=3.9 +Description-Content-Type: text/markdown + +

+ + attrs + +

+ + +*attrs* is the Python package that will bring back the **joy** of **writing classes** by relieving you from the drudgery of implementing object protocols (aka [dunder methods](https://www.attrs.org/en/latest/glossary.html#term-dunder-methods)). +Trusted by NASA for [Mars missions since 2020](https://github.com/readme/featured/nasa-ingenuity-helicopter)! + +Its main goal is to help you to write **concise** and **correct** software without slowing down your code. + + +## Sponsors + +*attrs* would not be possible without our [amazing sponsors](https://github.com/sponsors/hynek). +Especially those generously supporting us at the *The Organization* tier and higher: + + + +

+ + + + + + + + + + + +

+ + + +

+ Please consider joining them to help make attrs’s maintenance more sustainable! +

+ + + +## Example + +*attrs* gives you a class decorator and a way to declaratively define the attributes on that class: + + + +```pycon +>>> from attrs import asdict, define, make_class, Factory + +>>> @define +... class SomeClass: +... a_number: int = 42 +... list_of_numbers: list[int] = Factory(list) +... +... def hard_math(self, another_number): +... return self.a_number + sum(self.list_of_numbers) * another_number + + +>>> sc = SomeClass(1, [1, 2, 3]) +>>> sc +SomeClass(a_number=1, list_of_numbers=[1, 2, 3]) + +>>> sc.hard_math(3) +19 +>>> sc == SomeClass(1, [1, 2, 3]) +True +>>> sc != SomeClass(2, [3, 2, 1]) +True + +>>> asdict(sc) +{'a_number': 1, 'list_of_numbers': [1, 2, 3]} + +>>> SomeClass() +SomeClass(a_number=42, list_of_numbers=[]) + +>>> C = make_class("C", ["a", "b"]) +>>> C("foo", "bar") +C(a='foo', b='bar') +``` + +After *declaring* your attributes, *attrs* gives you: + +- a concise and explicit overview of the class's attributes, +- a nice human-readable `__repr__`, +- equality-checking methods, +- an initializer, +- and much more, + +*without* writing dull boilerplate code again and again and *without* runtime performance penalties. + +--- + +This example uses *attrs*'s modern APIs that have been introduced in version 20.1.0, and the *attrs* package import name that has been added in version 21.3.0. +The classic APIs (`@attr.s`, `attr.ib`, plus their serious-business aliases) and the `attr` package import name will remain **indefinitely**. + +Check out [*On The Core API Names*](https://www.attrs.org/en/latest/names.html) for an in-depth explanation! + + +### Hate Type Annotations!? + +No problem! +Types are entirely **optional** with *attrs*. +Simply assign `attrs.field()` to the attributes instead of annotating them with types: + +```python +from attrs import define, field + +@define +class SomeClass: + a_number = field(default=42) + list_of_numbers = field(factory=list) +``` + + +## Data Classes + +On the tin, *attrs* might remind you of `dataclasses` (and indeed, `dataclasses` [are a descendant](https://hynek.me/articles/import-attrs/) of *attrs*). +In practice it does a lot more and is more flexible. +For instance, it allows you to define [special handling of NumPy arrays for equality checks](https://www.attrs.org/en/stable/comparison.html#customization), allows more ways to [plug into the initialization process](https://www.attrs.org/en/stable/init.html#hooking-yourself-into-initialization), has a replacement for `__init_subclass__`, and allows for stepping through the generated methods using a debugger. + +For more details, please refer to our [comparison page](https://www.attrs.org/en/stable/why.html#data-classes), but generally speaking, we are more likely to commit crimes against nature to make things work that one would expect to work, but that are quite complicated in practice. + + +## Project Information + +- [**Changelog**](https://www.attrs.org/en/stable/changelog.html) +- [**Documentation**](https://www.attrs.org/) +- [**PyPI**](https://pypi.org/project/attrs/) +- [**Source Code**](https://github.com/python-attrs/attrs) +- [**Contributing**](https://github.com/python-attrs/attrs/blob/main/.github/CONTRIBUTING.md) +- [**Third-party Extensions**](https://github.com/python-attrs/attrs/wiki/Extensions-to-attrs) +- **Get Help**: use the `python-attrs` tag on [Stack Overflow](https://stackoverflow.com/questions/tagged/python-attrs) + + +### *attrs* for Enterprise + +Available as part of the [Tidelift Subscription](https://tidelift.com/?utm_source=lifter&utm_medium=referral&utm_campaign=hynek). + +The maintainers of *attrs* and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source packages you use to build your applications. +Save time, reduce risk, and improve code health, while paying the maintainers of the exact packages you use. + +## Release Information + +### Backwards-incompatible Changes + +- Field aliases are now resolved *before* calling `field_transformer`, so transformers receive fully populated `Attribute` objects with usable `alias` values instead of `None`. + The new `Attribute.alias_is_default` flag indicates whether the alias was auto-generated (`True`) or explicitly set by the user (`False`). + [#1509](https://github.com/python-attrs/attrs/issues/1509) + + +### Changes + +- Fix type annotations for `attrs.validators.optional()`, so it no longer rejects tuples with more than one validator. + [#1496](https://github.com/python-attrs/attrs/issues/1496) +- The `attrs.validators.disabled()` contextmanager can now be nested. + [#1513](https://github.com/python-attrs/attrs/issues/1513) +- Frozen classes can set `on_setattr=attrs.setters.NO_OP` in addition to `None`. + [#1515](https://github.com/python-attrs/attrs/issues/1515) +- It's now possible to pass *attrs* **instances** in addition to *attrs* **classes** to `attrs.fields()`. + [#1529](https://github.com/python-attrs/attrs/issues/1529) + + + +--- + +[Full changelog →](https://www.attrs.org/en/stable/changelog.html) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/RECORD b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..684f95b2360832d59182a7212426ff7ac8a242d1 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/RECORD @@ -0,0 +1,37 @@ +attr/__init__.py,sha256=fOYIvt1eGSqQre4uCS3sJWKZ0mwAuC8UD6qba5OS9_U,2057 +attr/__init__.pyi,sha256=pVGImAUVovq2_TYl_r_HIYnGlyOaoCuEhxo-EvsnnSc,11325 +attr/_cmp.py,sha256=3Nn1TjxllUYiX_nJoVnEkXoDk0hM1DYKj5DE7GZe4i0,4117 +attr/_cmp.pyi,sha256=U-_RU_UZOyPUEQzXE6RMYQQcjkZRY25wTH99sN0s7MM,368 +attr/_compat.py,sha256=x0g7iEUOnBVJC72zyFCgb1eKqyxS-7f2LGnNyZ_r95s,2829 +attr/_config.py,sha256=dGq3xR6fgZEF6UBt_L0T-eUHIB4i43kRmH0P28sJVw8,843 +attr/_funcs.py,sha256=Ix5IETTfz5F01F-12MF_CSFomIn2h8b67EVVz2gCtBE,16479 +attr/_make.py,sha256=H7OH2eWS5CnBzLUjNFE1WymfPrmF1r8fv2RPdt9MuYA,106129 +attr/_next_gen.py,sha256=BQtCUlzwg2gWHTYXBQvrEYBnzBUrDvO57u0Py6UCPhc,26274 +attr/_typing_compat.pyi,sha256=XDP54TUn-ZKhD62TOQebmzrwFyomhUCoGRpclb6alRA,469 +attr/_version_info.py,sha256=w4R-FYC3NK_kMkGUWJlYP4cVAlH9HRaC-um3fcjYkHM,2222 +attr/_version_info.pyi,sha256=x_M3L3WuB7r_ULXAWjx959udKQ4HLB8l-hsc1FDGNvk,209 +attr/converters.py,sha256=GlDeOzPeTFgeBBLbj9G57Ez5lAk68uhSALRYJ_exe84,3861 +attr/converters.pyi,sha256=orU2bff-VjQa2kMDyvnMQV73oJT2WRyQuw4ZR1ym1bE,643 +attr/exceptions.py,sha256=b4vMbnoQ3VpwWZhqrYi_ssXVCK8o2c4HQSS09cSUM9o,1990 +attr/exceptions.pyi,sha256=zZq8bCUnKAy9mDtBEw42ZhPhAUIHoTKedDQInJD883M,539 +attr/filters.py,sha256=ZBiKWLp3R0LfCZsq7X11pn9WX8NslS2wXM4jsnLOGc8,1795 +attr/filters.pyi,sha256=3J5BG-dTxltBk1_-RuNRUHrv2qu1v8v4aDNAQ7_mifA,208 +attr/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +attr/setters.py,sha256=5-dcT63GQK35ONEzSgfXCkbB7pPkaR-qv15mm4PVSzQ,1617 +attr/setters.pyi,sha256=NnVkaFU1BB4JB8E4JuXyrzTUgvtMpj8p3wBdJY7uix4,584 +attr/validators.py,sha256=m3QRzZTANr4f2C4eVdUoFg11NgXWak8Wat4qQTGhvcs,21553 +attr/validators.pyi,sha256=gM1ZmHaBckyYWI2EirpRNzqm3B19cw5Iq6B4Kno9YCM,4087 +attrs-26.1.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +attrs-26.1.0.dist-info/METADATA,sha256=TNQOaQ8jvzfLytNO_WdY4GLfHfB8hoM_fjzpW_H6OMw,8754 +attrs-26.1.0.dist-info/RECORD,, +attrs-26.1.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +attrs-26.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87 +attrs-26.1.0.dist-info/licenses/LICENSE,sha256=iCEVyV38KvHutnFPjsbVy8q_Znyv-HKfQkINpj9xTp8,1109 +attrs/__init__.py,sha256=RxaAZNwYiEh-fcvHLZNpQ_DWKni73M_jxEPEftiq1Zc,1183 +attrs/__init__.pyi,sha256=2gV79g9UxJppGSM48hAZJ6h_MHb70dZoJL31ZNJeZYI,9416 +attrs/converters.py,sha256=8kQljrVwfSTRu8INwEk8SI0eGrzmWftsT7rM0EqyohM,76 +attrs/exceptions.py,sha256=ACCCmg19-vDFaDPY9vFl199SPXCQMN_bENs4DALjzms,76 +attrs/filters.py,sha256=VOUMZug9uEU6dUuA0dF1jInUK0PL3fLgP0VBS5d-CDE,73 +attrs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +attrs/setters.py,sha256=eL1YidYQV3T2h9_SYIZSZR1FAcHGb1TuCTy0E0Lv2SU,73 +attrs/validators.py,sha256=xcy6wD5TtTkdCG1f4XWbocPSO0faBjk5IfVJfP6SUj0,76 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/REQUESTED b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/WHEEL b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..b1b94fd58e7e9ed0ef3449473bc48de68afcc3fe --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.29.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/licenses/LICENSE b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..2bd6453d255e19b973f19b128596a8b6dd65b2c3 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs-26.1.0.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Hynek Schlawack and the attrs contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dc1ce4b974ea1b3971ced44e3e112e897cf29b0a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/__init__.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: MIT + +from attr import ( + NOTHING, + Attribute, + AttrsInstance, + Converter, + Factory, + NothingType, + _make_getattr, + assoc, + cmp_using, + define, + evolve, + field, + fields, + fields_dict, + frozen, + has, + make_class, + mutable, + resolve_types, + validate, +) +from attr._make import ClassProps +from attr._next_gen import asdict, astuple, inspect + +from . import converters, exceptions, filters, setters, validators + + +__all__ = [ + "NOTHING", + "Attribute", + "AttrsInstance", + "ClassProps", + "Converter", + "Factory", + "NothingType", + "__author__", + "__copyright__", + "__description__", + "__doc__", + "__email__", + "__license__", + "__title__", + "__url__", + "__version__", + "__version_info__", + "asdict", + "assoc", + "astuple", + "cmp_using", + "converters", + "define", + "evolve", + "exceptions", + "field", + "fields", + "fields_dict", + "filters", + "frozen", + "has", + "inspect", + "make_class", + "mutable", + "resolve_types", + "setters", + "validate", + "validators", +] + +__getattr__ = _make_getattr(__name__) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/__init__.pyi b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..6364bac4ea29f860934803e5f3ba741812c6dfd1 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/__init__.pyi @@ -0,0 +1,314 @@ +import sys + +from typing import ( + Any, + Callable, + Mapping, + Sequence, + overload, + TypeVar, +) + +# Because we need to type our own stuff, we have to make everything from +# attr explicitly public too. +from attr import __author__ as __author__ +from attr import __copyright__ as __copyright__ +from attr import __description__ as __description__ +from attr import __email__ as __email__ +from attr import __license__ as __license__ +from attr import __title__ as __title__ +from attr import __url__ as __url__ +from attr import __version__ as __version__ +from attr import __version_info__ as __version_info__ +from attr import assoc as assoc +from attr import Attribute as Attribute +from attr import AttrsInstance as AttrsInstance +from attr import cmp_using as cmp_using +from attr import converters as converters +from attr import Converter as Converter +from attr import evolve as evolve +from attr import exceptions as exceptions +from attr import Factory as Factory +from attr import fields as fields +from attr import fields_dict as fields_dict +from attr import filters as filters +from attr import has as has +from attr import make_class as make_class +from attr import NOTHING as NOTHING +from attr import resolve_types as resolve_types +from attr import setters as setters +from attr import validate as validate +from attr import validators as validators +from attr import attrib, asdict as asdict, astuple as astuple +from attr import NothingType as NothingType + +if sys.version_info >= (3, 11): + from typing import dataclass_transform +else: + from typing_extensions import dataclass_transform + +_T = TypeVar("_T") +_C = TypeVar("_C", bound=type) + +_EqOrderType = bool | Callable[[Any], Any] +_ValidatorType = Callable[[Any, "Attribute[_T]", _T], Any] +_CallableConverterType = Callable[[Any], Any] +_ConverterType = _CallableConverterType | Converter[Any, Any] +_ReprType = Callable[[Any], str] +_ReprArgType = bool | _ReprType +_OnSetAttrType = Callable[[Any, "Attribute[Any]", Any], Any] +_OnSetAttrArgType = _OnSetAttrType | list[_OnSetAttrType] | setters._NoOpType +_FieldTransformer = Callable[ + [type, list["Attribute[Any]"]], list["Attribute[Any]"] +] +# FIXME: in reality, if multiple validators are passed they must be in a list +# or tuple, but those are invariant and so would prevent subtypes of +# _ValidatorType from working when passed in a list or tuple. +_ValidatorArgType = _ValidatorType[_T] | Sequence[_ValidatorType[_T]] + +@overload +def field( + *, + default: None = ..., + validator: None = ..., + repr: _ReprArgType = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + converter: None = ..., + factory: None = ..., + kw_only: bool | None = ..., + eq: bool | None = ..., + order: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., + type: type | None = ..., +) -> Any: ... + +# This form catches an explicit None or no default and infers the type from the +# other arguments. +@overload +def field( + *, + default: None = ..., + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType, ...] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., + type: type | None = ..., +) -> _T: ... + +# This form catches an explicit default argument. +@overload +def field( + *, + default: _T, + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType, ...] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., + type: type | None = ..., +) -> _T: ... + +# This form covers type=non-Type: e.g. forward references (str), Any +@overload +def field( + *, + default: _T | None = ..., + validator: _ValidatorArgType[_T] | None = ..., + repr: _ReprArgType = ..., + hash: bool | None = ..., + init: bool = ..., + metadata: Mapping[Any, Any] | None = ..., + converter: _ConverterType + | list[_ConverterType] + | tuple[_ConverterType, ...] + | None = ..., + factory: Callable[[], _T] | None = ..., + kw_only: bool | None = ..., + eq: _EqOrderType | None = ..., + order: _EqOrderType | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + alias: str | None = ..., + type: type | None = ..., +) -> Any: ... +@overload +@dataclass_transform(field_specifiers=(attrib, field)) +def define( + maybe_cls: _C, + *, + these: dict[str, Any] | None = ..., + repr: bool = ..., + unsafe_hash: bool | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: bool | None = ..., + order: bool | None = ..., + auto_detect: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., +) -> _C: ... +@overload +@dataclass_transform(field_specifiers=(attrib, field)) +def define( + maybe_cls: None = ..., + *, + these: dict[str, Any] | None = ..., + repr: bool = ..., + unsafe_hash: bool | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: bool | None = ..., + order: bool | None = ..., + auto_detect: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., +) -> Callable[[_C], _C]: ... + +mutable = define + +@overload +@dataclass_transform(frozen_default=True, field_specifiers=(attrib, field)) +def frozen( + maybe_cls: _C, + *, + these: dict[str, Any] | None = ..., + repr: bool = ..., + unsafe_hash: bool | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: bool | None = ..., + order: bool | None = ..., + auto_detect: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., +) -> _C: ... +@overload +@dataclass_transform(frozen_default=True, field_specifiers=(attrib, field)) +def frozen( + maybe_cls: None = ..., + *, + these: dict[str, Any] | None = ..., + repr: bool = ..., + unsafe_hash: bool | None = ..., + hash: bool | None = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: bool | None = ..., + order: bool | None = ..., + auto_detect: bool = ..., + getstate_setstate: bool | None = ..., + on_setattr: _OnSetAttrArgType | None = ..., + field_transformer: _FieldTransformer | None = ..., + match_args: bool = ..., +) -> Callable[[_C], _C]: ... + +class ClassProps: + # XXX: somehow when defining/using enums Mypy starts looking at our own + # (untyped) code and causes tons of errors. + Hashability: Any + KeywordOnly: Any + + is_exception: bool + is_slotted: bool + has_weakref_slot: bool + is_frozen: bool + # kw_only: ClassProps.KeywordOnly + kw_only: Any + collected_fields_by_mro: bool + added_init: bool + added_repr: bool + added_eq: bool + added_ordering: bool + # hashability: ClassProps.Hashability + hashability: Any + added_match_args: bool + added_str: bool + added_pickling: bool + on_setattr_hook: _OnSetAttrType | None + field_transformer: Callable[[Attribute[Any]], Attribute[Any]] | None + + def __init__( + self, + is_exception: bool, + is_slotted: bool, + has_weakref_slot: bool, + is_frozen: bool, + # kw_only: ClassProps.KeywordOnly + kw_only: Any, + collected_fields_by_mro: bool, + added_init: bool, + added_repr: bool, + added_eq: bool, + added_ordering: bool, + # hashability: ClassProps.Hashability + hashability: Any, + added_match_args: bool, + added_str: bool, + added_pickling: bool, + on_setattr_hook: _OnSetAttrType, + field_transformer: Callable[[Attribute[Any]], Attribute[Any]], + ) -> None: ... + @property + def is_hashable(self) -> bool: ... + +def inspect(cls: type) -> ClassProps: ... diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/converters.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/converters.py new file mode 100644 index 0000000000000000000000000000000000000000..7821f6c02cca81277d1ecc87b6bdafad886d8b70 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/converters.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: MIT + +from attr.converters import * # noqa: F403 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/exceptions.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..3323f9d2112c54b203763d45b455bd5abbe020f6 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/exceptions.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: MIT + +from attr.exceptions import * # noqa: F403 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/filters.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/filters.py new file mode 100644 index 0000000000000000000000000000000000000000..3080f48398e5ed8d3428ca3efeb7500633b0cb0f --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/filters.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: MIT + +from attr.filters import * # noqa: F403 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/py.typed b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/setters.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/setters.py new file mode 100644 index 0000000000000000000000000000000000000000..f3d73bb793dd49c138950961f41943bb26c57fde --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/setters.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: MIT + +from attr.setters import * # noqa: F403 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/validators.py b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/validators.py new file mode 100644 index 0000000000000000000000000000000000000000..037e124f29f32d37c1642d159bf828de44f7c349 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/attrs/validators.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: MIT + +from attr.validators import * # noqa: F403 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9d711f415d663685acccb5d9dedb2ac144246cc1 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__init__.py @@ -0,0 +1,4 @@ +from .core import contents, where + +__all__ = ["contents", "where"] +__version__ = "2026.04.22" diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__main__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..8945b5da857f4a7dec2b84f1225f012f6098418c --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__main__.py @@ -0,0 +1,12 @@ +import argparse + +from certifi import contents, where + +parser = argparse.ArgumentParser() +parser.add_argument("-c", "--contents", action="store_true") +args = parser.parse_args() + +if args.contents: + print(contents()) +else: + print(where()) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__pycache__/__init__.cpython-314.pyc b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2ddb35af111037678887957071d91dbb7f9e9c76 Binary files /dev/null and b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__pycache__/__init__.cpython-314.pyc differ diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__pycache__/core.cpython-314.pyc b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__pycache__/core.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..06b2f053df4c2fd5aee7030b0871d12886f70760 Binary files /dev/null and b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/__pycache__/core.cpython-314.pyc differ diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/cacert.pem b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/cacert.pem new file mode 100644 index 0000000000000000000000000000000000000000..27132f5a1ddaba07285b22daa40836c17aec7b63 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/cacert.pem @@ -0,0 +1,3933 @@ + +# Issuer: CN=COMODO ECC Certification Authority O=COMODO CA Limited +# Subject: CN=COMODO ECC Certification Authority O=COMODO CA Limited +# Label: "COMODO ECC Certification Authority" +# Serial: 41578283867086692638256921589707938090 +# MD5 Fingerprint: 7c:62:ff:74:9d:31:53:5e:68:4a:d5:78:aa:1e:bf:23 +# SHA1 Fingerprint: 9f:74:4e:9f:2b:4d:ba:ec:0f:31:2c:50:b6:56:3b:8e:2d:93:c3:11 +# SHA256 Fingerprint: 17:93:92:7a:06:14:54:97:89:ad:ce:2f:8f:34:f7:f0:b6:6d:0f:3a:e3:a3:b8:4d:21:ec:15:db:ba:4f:ad:c7 +-----BEGIN CERTIFICATE----- +MIICiTCCAg+gAwIBAgIQH0evqmIAcFBUTAGem2OZKjAKBggqhkjOPQQDAzCBhTEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMT +IkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwMzA2MDAw +MDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdy +ZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09N +T0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlv +biBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQDR3svdcmCFYX7deSR +FtSrYpn1PlILBs5BAH+X4QokPB0BBO490o0JlwzgdeT6+3eKKvUDYEs2ixYjFq0J +cfRK9ChQtP6IHG4/bC8vCVlbpVsLM5niwz2J+Wos77LTBumjQjBAMB0GA1UdDgQW +BBR1cacZSBm8nZ3qQUfflMRId5nTeTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/ +BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjEA7wNbeqy3eApyt4jf/7VGFAkK+qDm +fQjGGoe9GKhzvSbKYAydzpmfz1wPMOG+FDHqAjAU9JM8SaczepBGR7NjfRObTrdv +GDeAU/7dIOA1mjbRxwG55tzd8/8dLDoWV9mSOdY= +-----END CERTIFICATE----- + +# Issuer: O=Chunghwa Telecom Co., Ltd. OU=ePKI Root Certification Authority +# Subject: O=Chunghwa Telecom Co., Ltd. OU=ePKI Root Certification Authority +# Label: "ePKI Root Certification Authority" +# Serial: 28956088682735189655030529057352760477 +# MD5 Fingerprint: 1b:2e:00:ca:26:06:90:3d:ad:fe:6f:15:68:d3:6b:b3 +# SHA1 Fingerprint: 67:65:0d:f1:7e:8e:7e:5b:82:40:a4:f4:56:4b:cf:e2:3d:69:c6:f0 +# SHA256 Fingerprint: c0:a6:f4:dc:63:a2:4b:fd:cf:54:ef:2a:6a:08:2a:0a:72:de:35:80:3e:2f:f5:ff:52:7a:e5:d8:72:06:df:d5 +-----BEGIN CERTIFICATE----- +MIIFsDCCA5igAwIBAgIQFci9ZUdcr7iXAF7kBtK8nTANBgkqhkiG9w0BAQUFADBe +MQswCQYDVQQGEwJUVzEjMCEGA1UECgwaQ2h1bmdod2EgVGVsZWNvbSBDby4sIEx0 +ZC4xKjAoBgNVBAsMIWVQS0kgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAe +Fw0wNDEyMjAwMjMxMjdaFw0zNDEyMjAwMjMxMjdaMF4xCzAJBgNVBAYTAlRXMSMw +IQYDVQQKDBpDaHVuZ2h3YSBUZWxlY29tIENvLiwgTHRkLjEqMCgGA1UECwwhZVBL +SSBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEA4SUP7o3biDN1Z82tH306Tm2d0y8U82N0ywEhajfqhFAH +SyZbCUNsIZ5qyNUD9WBpj8zwIuQf5/dqIjG3LBXy4P4AakP/h2XGtRrBp0xtInAh +ijHyl3SJCRImHJ7K2RKilTza6We/CKBk49ZCt0Xvl/T29de1ShUCWH2YWEtgvM3X +DZoTM1PRYfl61dd4s5oz9wCGzh1NlDivqOx4UXCKXBCDUSH3ET00hl7lSM2XgYI1 +TBnsZfZrxQWh7kcT1rMhJ5QQCtkkO7q+RBNGMD+XPNjX12ruOzjjK9SXDrkb5wdJ +fzcq+Xd4z1TtW0ado4AOkUPB1ltfFLqfpo0kR0BZv3I4sjZsN/+Z0V0OWQqraffA +sgRFelQArr5T9rXn4fg8ozHSqf4hUmTFpmfwdQcGlBSBVcYn5AGPF8Fqcde+S/uU +WH1+ETOxQvdibBjWzwloPn9s9h6PYq2lY9sJpx8iQkEeb5mKPtf5P0B6ebClAZLS +nT0IFaUQAS2zMnaolQ2zepr7BxB4EW/hj8e6DyUadCrlHJhBmd8hh+iVBmoKs2pH +dmX2Os+PYhcZewoozRrSgx4hxyy/vv9haLdnG7t4TY3OZ+XkwY63I2binZB1NJip +NiuKmpS5nezMirH4JYlcWrYvjB9teSSnUmjDhDXiZo1jDiVN1Rmy5nk3pyKdVDEC +AwEAAaNqMGgwHQYDVR0OBBYEFB4M97Zn8uGSJglFwFU5Lnc/QkqiMAwGA1UdEwQF +MAMBAf8wOQYEZyoHAAQxMC8wLQIBADAJBgUrDgMCGgUAMAcGBWcqAwAABBRFsMLH +ClZ87lt4DJX5GFPBphzYEDANBgkqhkiG9w0BAQUFAAOCAgEACbODU1kBPpVJufGB +uvl2ICO1J2B01GqZNF5sAFPZn/KmsSQHRGoqxqWOeBLoR9lYGxMqXnmbnwoqZ6Yl +PwZpVnPDimZI+ymBV3QGypzqKOg4ZyYr8dW1P2WT+DZdjo2NQCCHGervJ8A9tDkP +JXtoUHRVnAxZfVo9QZQlUgjgRywVMRnVvwdVxrsStZf0X4OFunHB2WyBEXYKCrC/ +gpf36j36+uwtqSiUO1bd0lEursC9CBWMd1I0ltabrNMdjmEPNXubrjlpC2JgQCA2 +j6/7Nu4tCEoduL+bXPjqpRugc6bY+G7gMwRfaKonh+3ZwZCc7b3jajWvY9+rGNm6 +5ulK6lCKD2GTHuItGeIwlDWSXQ62B68ZgI9HkFFLLk3dheLSClIKF5r8GrBQAuUB +o2M3IUxExJtRmREOc5wGj1QupyheRDmHVi03vYVElOEMSyycw5KFNGHLD7ibSkNS +/jQ6fbjpKdx2qcgw+BRxgMYeNkh0IkFch4LoGHGLQYlE535YW6i4jRPpp2zDR+2z +Gp1iro2C6pSe3VkQw63d4k3jMdXH7OjysP6SHhYKGvzZ8/gntsm+HbRsZJB/9OTE +W9c3rkIO3aQab3yIVMUWbuF6aC74Or8NpDyJO3inTmODBCEIZ43ygknQW/2xzQ+D +hNQ+IIX3Sj0rnP0qCglN6oH4EZw= +-----END CERTIFICATE----- + +# Issuer: CN=NetLock Arany (Class Gold) F\u0151tan\xfas\xedtv\xe1ny O=NetLock Kft. OU=Tan\xfas\xedtv\xe1nykiad\xf3k (Certification Services) +# Subject: CN=NetLock Arany (Class Gold) F\u0151tan\xfas\xedtv\xe1ny O=NetLock Kft. OU=Tan\xfas\xedtv\xe1nykiad\xf3k (Certification Services) +# Label: "NetLock Arany (Class Gold) F\u0151tan\xfas\xedtv\xe1ny" +# Serial: 80544274841616 +# MD5 Fingerprint: c5:a1:b7:ff:73:dd:d6:d7:34:32:18:df:fc:3c:ad:88 +# SHA1 Fingerprint: 06:08:3f:59:3f:15:a1:04:a0:69:a4:6b:a9:03:d0:06:b7:97:09:91 +# SHA256 Fingerprint: 6c:61:da:c3:a2:de:f0:31:50:6b:e0:36:d2:a6:fe:40:19:94:fb:d1:3d:f9:c8:d4:66:59:92:74:c4:46:ec:98 +-----BEGIN CERTIFICATE----- +MIIEFTCCAv2gAwIBAgIGSUEs5AAQMA0GCSqGSIb3DQEBCwUAMIGnMQswCQYDVQQG +EwJIVTERMA8GA1UEBwwIQnVkYXBlc3QxFTATBgNVBAoMDE5ldExvY2sgS2Z0LjE3 +MDUGA1UECwwuVGFuw7pzw610dsOhbnlraWFkw7NrIChDZXJ0aWZpY2F0aW9uIFNl +cnZpY2VzKTE1MDMGA1UEAwwsTmV0TG9jayBBcmFueSAoQ2xhc3MgR29sZCkgRsWR +dGFuw7pzw610dsOhbnkwHhcNMDgxMjExMTUwODIxWhcNMjgxMjA2MTUwODIxWjCB +pzELMAkGA1UEBhMCSFUxETAPBgNVBAcMCEJ1ZGFwZXN0MRUwEwYDVQQKDAxOZXRM +b2NrIEtmdC4xNzA1BgNVBAsMLlRhbsO6c8OtdHbDoW55a2lhZMOzayAoQ2VydGlm +aWNhdGlvbiBTZXJ2aWNlcykxNTAzBgNVBAMMLE5ldExvY2sgQXJhbnkgKENsYXNz +IEdvbGQpIEbFkXRhbsO6c8OtdHbDoW55MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAxCRec75LbRTDofTjl5Bu0jBFHjzuZ9lk4BqKf8owyoPjIMHj9DrT +lF8afFttvzBPhCf2nx9JvMaZCpDyD/V/Q4Q3Y1GLeqVw/HpYzY6b7cNGbIRwXdrz +AZAj/E4wqX7hJ2Pn7WQ8oLjJM2P+FpD/sLj916jAwJRDC7bVWaaeVtAkH3B5r9s5 +VA1lddkVQZQBr17s9o3x/61k/iCa11zr/qYfCGSji3ZVrR47KGAuhyXoqq8fxmRG +ILdwfzzeSNuWU7c5d+Qa4scWhHaXWy+7GRWF+GmF9ZmnqfI0p6m2pgP8b4Y9VHx2 +BJtr+UBdADTHLpl1neWIA6pN+APSQnbAGwIDAKiLo0UwQzASBgNVHRMBAf8ECDAG +AQH/AgEEMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUzPpnk/C2uNClwB7zU/2M +U9+D15YwDQYJKoZIhvcNAQELBQADggEBAKt/7hwWqZw8UQCgwBEIBaeZ5m8BiFRh +bvG5GK1Krf6BQCOUL/t1fC8oS2IkgYIL9WHxHG64YTjrgfpioTtaYtOUZcTh5m2C ++C8lcLIhJsFyUR+MLMOEkMNaj7rP9KdlpeuY0fsFskZ1FSNqb4VjMIDw1Z4fKRzC +bLBQWV2QWzuoDTDPv31/zvGdg73JRm4gpvlhUbohL3u+pRVjodSVh/GeufOJ8z2F +uLjbvrW5KfnaNwUASZQDhETnv0Mxz3WLJdH0pmT1kvarBes96aULNmLazAZfNou2 +XjG4Kvte9nHfRCaexOYNkbQudZWAUWpLMKawYqGT8ZvYzsRjdT9ZR7E= +-----END CERTIFICATE----- + +# Issuer: CN=Microsec e-Szigno Root CA 2009 O=Microsec Ltd. +# Subject: CN=Microsec e-Szigno Root CA 2009 O=Microsec Ltd. +# Label: "Microsec e-Szigno Root CA 2009" +# Serial: 14014712776195784473 +# MD5 Fingerprint: f8:49:f4:03:bc:44:2d:83:be:48:69:7d:29:64:fc:b1 +# SHA1 Fingerprint: 89:df:74:fe:5c:f4:0f:4a:80:f9:e3:37:7d:54:da:91:e1:01:31:8e +# SHA256 Fingerprint: 3c:5f:81:fe:a5:fa:b8:2c:64:bf:a2:ea:ec:af:cd:e8:e0:77:fc:86:20:a7:ca:e5:37:16:3d:f3:6e:db:f3:78 +-----BEGIN CERTIFICATE----- +MIIECjCCAvKgAwIBAgIJAMJ+QwRORz8ZMA0GCSqGSIb3DQEBCwUAMIGCMQswCQYD +VQQGEwJIVTERMA8GA1UEBwwIQnVkYXBlc3QxFjAUBgNVBAoMDU1pY3Jvc2VjIEx0 +ZC4xJzAlBgNVBAMMHk1pY3Jvc2VjIGUtU3ppZ25vIFJvb3QgQ0EgMjAwOTEfMB0G +CSqGSIb3DQEJARYQaW5mb0BlLXN6aWduby5odTAeFw0wOTA2MTYxMTMwMThaFw0y +OTEyMzAxMTMwMThaMIGCMQswCQYDVQQGEwJIVTERMA8GA1UEBwwIQnVkYXBlc3Qx +FjAUBgNVBAoMDU1pY3Jvc2VjIEx0ZC4xJzAlBgNVBAMMHk1pY3Jvc2VjIGUtU3pp +Z25vIFJvb3QgQ0EgMjAwOTEfMB0GCSqGSIb3DQEJARYQaW5mb0BlLXN6aWduby5o +dTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOn4j/NjrdqG2KfgQvvP +kd6mJviZpWNwrZuuyjNAfW2WbqEORO7hE52UQlKavXWFdCyoDh2Tthi3jCyoz/tc +cbna7P7ofo/kLx2yqHWH2Leh5TvPmUpG0IMZfcChEhyVbUr02MelTTMuhTlAdX4U +fIASmFDHQWe4oIBhVKZsTh/gnQ4H6cm6M+f+wFUoLAKApxn1ntxVUwOXewdI/5n7 +N4okxFnMUBBjjqqpGrCEGob5X7uxUG6k0QrM1XF+H6cbfPVTbiJfyyvm1HxdrtbC +xkzlBQHZ7Vf8wSN5/PrIJIOV87VqUQHQd9bpEqH5GoP7ghu5sJf0dgYzQ0mg/wu1 ++rUCAwEAAaOBgDB+MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0G +A1UdDgQWBBTLD8bfQkPMPcu1SCOhGnqmKrs0aDAfBgNVHSMEGDAWgBTLD8bfQkPM +Pcu1SCOhGnqmKrs0aDAbBgNVHREEFDASgRBpbmZvQGUtc3ppZ25vLmh1MA0GCSqG +SIb3DQEBCwUAA4IBAQDJ0Q5eLtXMs3w+y/w9/w0olZMEyL/azXm4Q5DwpL7v8u8h +mLzU1F0G9u5C7DBsoKqpyvGvivo/C3NqPuouQH4frlRheesuCDfXI/OMn74dseGk +ddug4lQUsbocKaQY9hK6ohQU4zE1yED/t+AFdlfBHFny+L/k7SViXITwfn4fs775 +tyERzAMBVnCnEJIeGzSBHq2cGsMEPO0CYdYeBvNfOofyK/FFh+U9rNHHV4S9a67c +2Pm2G2JwCz02yULyMtd6YebS2z3PyKnJm9zbWETXbzivf3jTo60adbocwTZ8jx5t +HMN1Rq41Bab2XD0h7lbwyYIiLXpUq3DDfSJlgnCW +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R3 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R3 +# Label: "GlobalSign Root CA - R3" +# Serial: 4835703278459759426209954 +# MD5 Fingerprint: c5:df:b8:49:ca:05:13:55:ee:2d:ba:1a:c3:3e:b0:28 +# SHA1 Fingerprint: d6:9b:56:11:48:f0:1c:77:c5:45:78:c1:09:26:df:5b:85:69:76:ad +# SHA256 Fingerprint: cb:b5:22:d7:b7:f1:27:ad:6a:01:13:86:5b:df:1c:d4:10:2e:7d:07:59:af:63:5a:7c:f4:72:0d:c9:63:c5:3b +-----BEGIN CERTIFICATE----- +MIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G +A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp +Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4 +MTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG +A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8 +RgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT +gHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm +KPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd +QQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ +XriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw +DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o +LkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU +RUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp +jjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK +6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX +mcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs +Mx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH +WD9f +-----END CERTIFICATE----- + +# Issuer: CN=Izenpe.com O=IZENPE S.A. +# Subject: CN=Izenpe.com O=IZENPE S.A. +# Label: "Izenpe.com" +# Serial: 917563065490389241595536686991402621 +# MD5 Fingerprint: a6:b0:cd:85:80:da:5c:50:34:a3:39:90:2f:55:67:73 +# SHA1 Fingerprint: 2f:78:3d:25:52:18:a7:4a:65:39:71:b5:2c:a2:9c:45:15:6f:e9:19 +# SHA256 Fingerprint: 25:30:cc:8e:98:32:15:02:ba:d9:6f:9b:1f:ba:1b:09:9e:2d:29:9e:0f:45:48:bb:91:4f:36:3b:c0:d4:53:1f +-----BEGIN CERTIFICATE----- +MIIF8TCCA9mgAwIBAgIQALC3WhZIX7/hy/WL1xnmfTANBgkqhkiG9w0BAQsFADA4 +MQswCQYDVQQGEwJFUzEUMBIGA1UECgwLSVpFTlBFIFMuQS4xEzARBgNVBAMMCkl6 +ZW5wZS5jb20wHhcNMDcxMjEzMTMwODI4WhcNMzcxMjEzMDgyNzI1WjA4MQswCQYD +VQQGEwJFUzEUMBIGA1UECgwLSVpFTlBFIFMuQS4xEzARBgNVBAMMCkl6ZW5wZS5j +b20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ03rKDx6sp4boFmVq +scIbRTJxldn+EFvMr+eleQGPicPK8lVx93e+d5TzcqQsRNiekpsUOqHnJJAKClaO +xdgmlOHZSOEtPtoKct2jmRXagaKH9HtuJneJWK3W6wyyQXpzbm3benhB6QiIEn6H +LmYRY2xU+zydcsC8Lv/Ct90NduM61/e0aL6i9eOBbsFGb12N4E3GVFWJGjMxCrFX +uaOKmMPsOzTFlUFpfnXCPCDFYbpRR6AgkJOhkEvzTnyFRVSa0QUmQbC1TR0zvsQD +yCV8wXDbO/QJLVQnSKwv4cSsPsjLkkxTOTcj7NMB+eAJRE1NZMDhDVqHIrytG6P+ +JrUV86f8hBnp7KGItERphIPzidF0BqnMC9bC3ieFUCbKF7jJeodWLBoBHmy+E60Q +rLUk9TiRodZL2vG70t5HtfG8gfZZa88ZU+mNFctKy6lvROUbQc/hhqfK0GqfvEyN +BjNaooXlkDWgYlwWTvDjovoDGrQscbNYLN57C9saD+veIR8GdwYDsMnvmfzAuU8L +hij+0rnq49qlw0dpEuDb8PYZi+17cNcC1u2HGCgsBCRMd+RIihrGO5rUD8r6ddIB +QFqNeb+Lz0vPqhbBleStTIo+F5HUsWLlguWABKQDfo2/2n+iD5dPDNMN+9fR5XJ+ +HMh3/1uaD7euBUbl8agW7EekFwIDAQABo4H2MIHzMIGwBgNVHREEgagwgaWBD2lu +Zm9AaXplbnBlLmNvbaSBkTCBjjFHMEUGA1UECgw+SVpFTlBFIFMuQS4gLSBDSUYg +QTAxMzM3MjYwLVJNZXJjLlZpdG9yaWEtR2FzdGVpeiBUMTA1NSBGNjIgUzgxQzBB +BgNVBAkMOkF2ZGEgZGVsIE1lZGl0ZXJyYW5lbyBFdG9yYmlkZWEgMTQgLSAwMTAx +MCBWaXRvcmlhLUdhc3RlaXowDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC +AQYwHQYDVR0OBBYEFB0cZQ6o8iV7tJHP5LGx5r1VdGwFMA0GCSqGSIb3DQEBCwUA +A4ICAQB4pgwWSp9MiDrAyw6lFn2fuUhfGI8NYjb2zRlrrKvV9pF9rnHzP7MOeIWb +laQnIUdCSnxIOvVFfLMMjlF4rJUT3sb9fbgakEyrkgPH7UIBzg/YsfqikuFgba56 +awmqxinuaElnMIAkejEWOVt+8Rwu3WwJrfIxwYJOubv5vr8qhT/AQKM6WfxZSzwo +JNu0FXWuDYi6LnPAvViH5ULy617uHjAimcs30cQhbIHsvm0m5hzkQiCeR7Csg1lw +LDXWrzY0tM07+DKo7+N4ifuNRSzanLh+QBxh5z6ikixL8s36mLYp//Pye6kfLqCT +VyvehQP5aTfLnnhqBbTFMXiJ7HqnheG5ezzevh55hM6fcA5ZwjUukCox2eRFekGk +LhObNA5me0mrZJfQRsN5nXJQY6aYWwa9SG3YOYNw6DXwBdGqvOPbyALqfP2C2sJb +UjWumDqtujWTI6cfSN01RpiyEGjkpTHCClguGYEQyVB1/OpaFs4R1+7vUIgtYf8/ +QnMFlEPVjjxOAToZpR9GTnfQXeWBIiGH/pR9hNiTrdZoQ0iy2+tzJOeRf1SktoA+ +naM8THLCV8Sg1Mw4J87VBp6iSNnpn86CcDaTmjvfliHjWbcM2pE38P1ZWrOZyGls +QyYBNWNgVYkDOnXYukrZVP/u3oDYLdE41V4tC5h9Pmzb/CaIxw== +-----END CERTIFICATE----- + +# Issuer: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc. +# Subject: CN=Go Daddy Root Certificate Authority - G2 O=GoDaddy.com, Inc. +# Label: "Go Daddy Root Certificate Authority - G2" +# Serial: 0 +# MD5 Fingerprint: 80:3a:bc:22:c1:e6:fb:8d:9b:3b:27:4a:32:1b:9a:01 +# SHA1 Fingerprint: 47:be:ab:c9:22:ea:e8:0e:78:78:34:62:a7:9f:45:c2:54:fd:e6:8b +# SHA256 Fingerprint: 45:14:0b:32:47:eb:9c:c8:c5:b4:f0:d7:b5:30:91:f7:32:92:08:9e:6e:5a:63:e2:74:9d:d3:ac:a9:19:8e:da +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT +EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp +ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAwMFoXDTM3MTIzMTIz +NTk1OVowgYMxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH +EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjExMC8GA1UE +AxMoR28gRGFkZHkgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9xYgjx+lk09xvJGKP3gElY6SKD +E6bFIEMBO4Tx5oVJnyfq9oQbTqC023CYxzIBsQU+B07u9PpPL1kwIuerGVZr4oAH +/PMWdYA5UXvl+TW2dE6pjYIT5LY/qQOD+qK+ihVqf94Lw7YZFAXK6sOoBJQ7Rnwy +DfMAZiLIjWltNowRGLfTshxgtDj6AozO091GB94KPutdfMh8+7ArU6SSYmlRJQVh +GkSBjCypQ5Yj36w6gZoOKcUcqeldHraenjAKOc7xiID7S13MMuyFYkMlNAJWJwGR +tDtwKj9useiciAF9n9T521NtYJ2/LOdYq7hfRvzOxBsDPAnrSTFcaUaz4EcCAwEA +AaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FDqahQcQZyi27/a9BUFuIMGU2g/eMA0GCSqGSIb3DQEBCwUAA4IBAQCZ21151fmX +WWcDYfF+OwYxdS2hII5PZYe096acvNjpL9DbWu7PdIxztDhC2gV7+AJ1uP2lsdeu +9tfeE8tTEH6KRtGX+rcuKxGrkLAngPnon1rpN5+r5N9ss4UXnT3ZJE95kTXWXwTr +gIOrmgIttRD02JDHBHNA7XIloKmf7J6raBKZV8aPEjoJpL1E/QYVN8Gb5DKj7Tjo +2GTzLH4U/ALqn83/B2gX2yKQOC16jdFU8WnjXzPKej17CuPKf1855eJ1usV2GDPO +LPAvTK33sefOT6jEm0pUBsV/fdUID+Ic/n4XuKxe9tQWskMJDE32p2u0mYRlynqI +4uJEvlz36hz1 +-----END CERTIFICATE----- + +# Issuer: CN=Starfield Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Subject: CN=Starfield Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Label: "Starfield Root Certificate Authority - G2" +# Serial: 0 +# MD5 Fingerprint: d6:39:81:c6:52:7e:96:69:fc:fc:ca:66:ed:05:f2:96 +# SHA1 Fingerprint: b5:1c:06:7c:ee:2b:0c:3d:f8:55:ab:2d:92:f4:fe:39:d4:e7:0f:0e +# SHA256 Fingerprint: 2c:e1:cb:0b:f9:d2:f9:e1:02:99:3f:be:21:51:52:c3:b2:dd:0c:ab:de:1c:68:e5:31:9b:83:91:54:db:b7:f5 +-----BEGIN CERTIFICATE----- +MIID3TCCAsWgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBjzELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT +HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAMTKVN0YXJmaWVs +ZCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAw +MFoXDTM3MTIzMTIzNTk1OVowgY8xCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6 +b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFyZmllbGQgVGVj +aG5vbG9naWVzLCBJbmMuMTIwMAYDVQQDEylTdGFyZmllbGQgUm9vdCBDZXJ0aWZp +Y2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAL3twQP89o/8ArFvW59I2Z154qK3A2FWGMNHttfKPTUuiUP3oWmb3ooa/RMg +nLRJdzIpVv257IzdIvpy3Cdhl+72WoTsbhm5iSzchFvVdPtrX8WJpRBSiUZV9Lh1 +HOZ/5FSuS/hVclcCGfgXcVnrHigHdMWdSL5stPSksPNkN3mSwOxGXn/hbVNMYq/N +Hwtjuzqd+/x5AJhhdM8mgkBj87JyahkNmcrUDnXMN/uLicFZ8WJ/X7NfZTD4p7dN +dloedl40wOiWVpmKs/B/pM293DIxfJHP4F8R+GuqSVzRmZTRouNjWwl2tVZi4Ut0 +HZbUJtQIBFnQmA4O5t78w+wfkPECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO +BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFHwMMh+n2TB/xH1oo2Kooc6rB1snMA0G +CSqGSIb3DQEBCwUAA4IBAQARWfolTwNvlJk7mh+ChTnUdgWUXuEok21iXQnCoKjU +sHU48TRqneSfioYmUeYs0cYtbpUgSpIB7LiKZ3sx4mcujJUDJi5DnUox9g61DLu3 +4jd/IroAow57UvtruzvE03lRTs2Q9GcHGcg8RnoNAX3FWOdt5oUwF5okxBDgBPfg +8n/Uqgr/Qh037ZTlZFkSIHc40zI+OIF1lnP6aI+xy84fxez6nH7PfrHxBy22/L/K +pL/QlwVKvOoYKAKQvVR4CSFx09F9HdkWsKlhPdAKACL8x3vLCWRFCztAgfd9fDL1 +mMpYjn0q7pBZc2T5NnReJaH1ZgUufzkVqSr7UIuOhWn0 +-----END CERTIFICATE----- + +# Issuer: CN=Starfield Services Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Subject: CN=Starfield Services Root Certificate Authority - G2 O=Starfield Technologies, Inc. +# Label: "Starfield Services Root Certificate Authority - G2" +# Serial: 0 +# MD5 Fingerprint: 17:35:74:af:7b:61:1c:eb:f4:f9:3c:e2:ee:40:f9:a2 +# SHA1 Fingerprint: 92:5a:8f:8d:2c:6d:04:e0:66:5f:59:6a:ff:22:d8:63:e8:25:6f:3f +# SHA256 Fingerprint: 56:8d:69:05:a2:c8:87:08:a4:b3:02:51:90:ed:cf:ed:b1:97:4a:60:6a:13:c6:e5:29:0f:cb:2a:e6:3e:da:b5 +-----BEGIN CERTIFICATE----- +MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT +HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs +ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5 +MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD +VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy +ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy +dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p +OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2 +8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K +Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe +hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk +6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q +AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI +bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB +ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z +qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd +iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn +0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN +sSi6 +-----END CERTIFICATE----- + +# Issuer: CN=Certum Trusted Network CA O=Unizeto Technologies S.A. OU=Certum Certification Authority +# Subject: CN=Certum Trusted Network CA O=Unizeto Technologies S.A. OU=Certum Certification Authority +# Label: "Certum Trusted Network CA" +# Serial: 279744 +# MD5 Fingerprint: d5:e9:81:40:c5:18:69:fc:46:2c:89:75:62:0f:aa:78 +# SHA1 Fingerprint: 07:e0:32:e0:20:b7:2c:3f:19:2f:06:28:a2:59:3a:19:a7:0f:06:9e +# SHA256 Fingerprint: 5c:58:46:8d:55:f5:8e:49:7e:74:39:82:d2:b5:00:10:b6:d1:65:37:4a:cf:83:a7:d4:a3:2d:b7:68:c4:40:8e +-----BEGIN CERTIFICATE----- +MIIDuzCCAqOgAwIBAgIDBETAMA0GCSqGSIb3DQEBBQUAMH4xCzAJBgNVBAYTAlBM +MSIwIAYDVQQKExlVbml6ZXRvIFRlY2hub2xvZ2llcyBTLkEuMScwJQYDVQQLEx5D +ZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxIjAgBgNVBAMTGUNlcnR1bSBU +cnVzdGVkIE5ldHdvcmsgQ0EwHhcNMDgxMDIyMTIwNzM3WhcNMjkxMjMxMTIwNzM3 +WjB+MQswCQYDVQQGEwJQTDEiMCAGA1UEChMZVW5pemV0byBUZWNobm9sb2dpZXMg +Uy5BLjEnMCUGA1UECxMeQ2VydHVtIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MSIw +IAYDVQQDExlDZXJ0dW0gVHJ1c3RlZCBOZXR3b3JrIENBMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA4/t9o3K6wvDJFIf1awFO4W5AB7ptJ11/91sts1rH +UV+rpDKmYYe2bg+G0jACl/jXaVehGDldamR5xgFZrDwxSjh80gTSSyjoIF87B6LM +TXPb865Px1bVWqeWifrzq2jUI4ZZJ88JJ7ysbnKDHDBy3+Ci6dLhdHUZvSqeexVU +BBvXQzmtVSjF4hq79MDkrjhJM8x2hZ85RdKknvISjFH4fOQtf/WsX+sWn7Et0brM +kUJ3TCXJkDhv2/DM+44el1k+1WBO5gUo7Ul5E0u6SNsv+XLTOcr+H9g0cvW0QM8x +AcPs3hEtF10fuFDRXhmnad4HMyjKUJX5p1TLVIZQRan5SQIDAQABo0IwQDAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBQIds3LB/8k9sXN7buQvOKEN0Z19zAOBgNV +HQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQEFBQADggEBAKaorSLOAT2mo/9i0Eidi15y +sHhE49wcrwn9I0j6vSrEuVUEtRCjjSfeC4Jj0O7eDDd5QVsisrCaQVymcODU0HfL +I9MA4GxWL+FpDQ3Zqr8hgVDZBqWo/5U30Kr+4rP1mS1FhIrlQgnXdAIv94nYmem8 +J9RHjboNRhx3zxSkHLmkMcScKHQDNP8zGSal6Q10tz6XxnboJ5ajZt3hrvJBW8qY +VoNzcOSGGtIxQbovvi0TWnZvTuhOgQ4/WwMioBK+ZlgRSssDxLQqKi2WF+A5VLxI +03YnnZotBqbJ7DnSq9ufmgsnAjUpsUCV5/nonFWIGUbWtzT1fs45mtk48VH3Tyw= +-----END CERTIFICATE----- + +# Issuer: CN=TWCA Root Certification Authority O=TAIWAN-CA OU=Root CA +# Subject: CN=TWCA Root Certification Authority O=TAIWAN-CA OU=Root CA +# Label: "TWCA Root Certification Authority" +# Serial: 1 +# MD5 Fingerprint: aa:08:8f:f6:f9:7b:b7:f2:b1:a7:1e:9b:ea:ea:bd:79 +# SHA1 Fingerprint: cf:9e:87:6d:d3:eb:fc:42:26:97:a3:b5:a3:7a:a0:76:a9:06:23:48 +# SHA256 Fingerprint: bf:d8:8f:e1:10:1c:41:ae:3e:80:1b:f8:be:56:35:0e:e9:ba:d1:a6:b9:bd:51:5e:dc:5c:6d:5b:87:11:ac:44 +-----BEGIN CERTIFICATE----- +MIIDezCCAmOgAwIBAgIBATANBgkqhkiG9w0BAQUFADBfMQswCQYDVQQGEwJUVzES +MBAGA1UECgwJVEFJV0FOLUNBMRAwDgYDVQQLDAdSb290IENBMSowKAYDVQQDDCFU +V0NBIFJvb3QgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwODI4MDcyNDMz +WhcNMzAxMjMxMTU1OTU5WjBfMQswCQYDVQQGEwJUVzESMBAGA1UECgwJVEFJV0FO +LUNBMRAwDgYDVQQLDAdSb290IENBMSowKAYDVQQDDCFUV0NBIFJvb3QgQ2VydGlm +aWNhdGlvbiBBdXRob3JpdHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQCwfnK4pAOU5qfeCTiRShFAh6d8WWQUe7UREN3+v9XAu1bihSX0NXIP+FPQQeFE +AcK0HMMxQhZHhTMidrIKbw/lJVBPhYa+v5guEGcevhEFhgWQxFnQfHgQsIBct+HH +K3XLfJ+utdGdIzdjp9xCoi2SBBtQwXu4PhvJVgSLL1KbralW6cH/ralYhzC2gfeX +RfwZVzsrb+RH9JlF/h3x+JejiB03HFyP4HYlmlD4oFT/RJB2I9IyxsOrBr/8+7/z +rX2SYgJbKdM1o5OaQ2RgXbL6Mv87BK9NQGr5x+PvI/1ry+UPizgN7gr8/g+YnzAx +3WxSZfmLgb4i4RxYA7qRG4kHAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqOFsmjd6LWvJPelSDGRjjCDWmujANBgkq +hkiG9w0BAQUFAAOCAQEAPNV3PdrfibqHDAhUaiBQkr6wQT25JmSDCi/oQMCXKCeC +MErJk/9q56YAf4lCmtYR5VPOL8zy2gXE/uJQxDqGfczafhAJO5I1KlOy/usrBdls +XebQ79NqZp4VKIV66IIArB6nCWlWQtNoURi+VJq/REG6Sb4gumlc7rh3zc5sH62D +lhh9DrUUOYTxKOkto557HnpyWoOzeW/vtPzQCqVYT0bf+215WfKEIlKuD8z7fDvn +aspHYcN6+NOSBB+4IIThNlQWx0DeO4pz3N/GCUzf7Nr/1FNCocnyYh0igzyXxfkZ +YiesZSLX0zzG5Y6yU8xJzrww/nsOM5D77dIUkR8Hrw== +-----END CERTIFICATE----- + +# Issuer: O=SECOM Trust Systems CO.,LTD. OU=Security Communication RootCA2 +# Subject: O=SECOM Trust Systems CO.,LTD. OU=Security Communication RootCA2 +# Label: "Security Communication RootCA2" +# Serial: 0 +# MD5 Fingerprint: 6c:39:7d:a4:0e:55:59:b2:3f:d6:41:b1:12:50:de:43 +# SHA1 Fingerprint: 5f:3b:8c:f2:f8:10:b3:7d:78:b4:ce:ec:19:19:c3:73:34:b9:c7:74 +# SHA256 Fingerprint: 51:3b:2c:ec:b8:10:d4:cd:e5:dd:85:39:1a:df:c6:c2:dd:60:d8:7b:b7:36:d2:b5:21:48:4a:a4:7a:0e:be:f6 +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIBADANBgkqhkiG9w0BAQsFADBdMQswCQYDVQQGEwJKUDEl +MCMGA1UEChMcU0VDT00gVHJ1c3QgU3lzdGVtcyBDTy4sTFRELjEnMCUGA1UECxMe +U2VjdXJpdHkgQ29tbXVuaWNhdGlvbiBSb290Q0EyMB4XDTA5MDUyOTA1MDAzOVoX +DTI5MDUyOTA1MDAzOVowXTELMAkGA1UEBhMCSlAxJTAjBgNVBAoTHFNFQ09NIFRy +dXN0IFN5c3RlbXMgQ08uLExURC4xJzAlBgNVBAsTHlNlY3VyaXR5IENvbW11bmlj +YXRpb24gUm9vdENBMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANAV +OVKxUrO6xVmCxF1SrjpDZYBLx/KWvNs2l9amZIyoXvDjChz335c9S672XewhtUGr +zbl+dp+++T42NKA7wfYxEUV0kz1XgMX5iZnK5atq1LXaQZAQwdbWQonCv/Q4EpVM +VAX3NuRFg3sUZdbcDE3R3n4MqzvEFb46VqZab3ZpUql6ucjrappdUtAtCms1FgkQ +hNBqyjoGADdH5H5XTz+L62e4iKrFvlNVspHEfbmwhRkGeC7bYRr6hfVKkaHnFtWO +ojnflLhwHyg/i/xAXmODPIMqGplrz95Zajv8bxbXH/1KEOtOghY6rCcMU/Gt1SSw +awNQwS08Ft1ENCcadfsCAwEAAaNCMEAwHQYDVR0OBBYEFAqFqXdlBZh8QIH4D5cs +OPEK7DzPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQBMOqNErLlFsceTfsgLCkLfZOoc7llsCLqJX2rKSpWeeo8HxdpF +coJxDjrSzG+ntKEju/Ykn8sX/oymzsLS28yN/HH8AynBbF0zX2S2ZTuJbxh2ePXc +okgfGT+Ok+vx+hfuzU7jBBJV1uXk3fs+BXziHV7Gp7yXT2g69ekuCkO2r1dcYmh8 +t/2jioSgrGK+KwmHNPBqAbubKVY8/gA3zyNs8U6qtnRGEmyR7jTV7JqR50S+kDFy +1UkC9gLl9B/rfNmWVan/7Ir5mUf/NVoCqgTLiluHcSmRvaS0eg29mvVXIwAHIRc/ +SjnRBUkLp7Y3gaVdjKozXoEofKd9J+sAro03 +-----END CERTIFICATE----- + +# Issuer: CN=Actalis Authentication Root CA O=Actalis S.p.A./03358520967 +# Subject: CN=Actalis Authentication Root CA O=Actalis S.p.A./03358520967 +# Label: "Actalis Authentication Root CA" +# Serial: 6271844772424770508 +# MD5 Fingerprint: 69:c1:0d:4f:07:a3:1b:c3:fe:56:3d:04:bc:11:f6:a6 +# SHA1 Fingerprint: f3:73:b3:87:06:5a:28:84:8a:f2:f3:4a:ce:19:2b:dd:c7:8e:9c:ac +# SHA256 Fingerprint: 55:92:60:84:ec:96:3a:64:b9:6e:2a:be:01:ce:0b:a8:6a:64:fb:fe:bc:c7:aa:b5:af:c1:55:b3:7f:d7:60:66 +-----BEGIN CERTIFICATE----- +MIIFuzCCA6OgAwIBAgIIVwoRl0LE48wwDQYJKoZIhvcNAQELBQAwazELMAkGA1UE +BhMCSVQxDjAMBgNVBAcMBU1pbGFuMSMwIQYDVQQKDBpBY3RhbGlzIFMucC5BLi8w +MzM1ODUyMDk2NzEnMCUGA1UEAwweQWN0YWxpcyBBdXRoZW50aWNhdGlvbiBSb290 +IENBMB4XDTExMDkyMjExMjIwMloXDTMwMDkyMjExMjIwMlowazELMAkGA1UEBhMC +SVQxDjAMBgNVBAcMBU1pbGFuMSMwIQYDVQQKDBpBY3RhbGlzIFMucC5BLi8wMzM1 +ODUyMDk2NzEnMCUGA1UEAwweQWN0YWxpcyBBdXRoZW50aWNhdGlvbiBSb290IENB +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAp8bEpSmkLO/lGMWwUKNv +UTufClrJwkg4CsIcoBh/kbWHuUA/3R1oHwiD1S0eiKD4j1aPbZkCkpAW1V8IbInX +4ay8IMKx4INRimlNAJZaby/ARH6jDuSRzVju3PvHHkVH3Se5CAGfpiEd9UEtL0z9 +KK3giq0itFZljoZUj5NDKd45RnijMCO6zfB9E1fAXdKDa0hMxKufgFpbOr3JpyI/ +gCczWw63igxdBzcIy2zSekciRDXFzMwujt0q7bd9Zg1fYVEiVRvjRuPjPdA1Yprb +rxTIW6HMiRvhMCb8oJsfgadHHwTrozmSBp+Z07/T6k9QnBn+locePGX2oxgkg4YQ +51Q+qDp2JE+BIcXjDwL4k5RHILv+1A7TaLndxHqEguNTVHnd25zS8gebLra8Pu2F +be8lEfKXGkJh90qX6IuxEAf6ZYGyojnP9zz/GPvG8VqLWeICrHuS0E4UT1lF9gxe +KF+w6D9Fz8+vm2/7hNN3WpVvrJSEnu68wEqPSpP4RCHiMUVhUE4Q2OM1fEwZtN4F +v6MGn8i1zeQf1xcGDXqVdFUNaBr8EBtiZJ1t4JWgw5QHVw0U5r0F+7if5t+L4sbn +fpb2U8WANFAoWPASUHEXMLrmeGO89LKtmyuy/uE5jF66CyCU3nuDuP/jVo23Eek7 +jPKxwV2dpAtMK9myGPW1n0sCAwEAAaNjMGEwHQYDVR0OBBYEFFLYiDrIn3hm7Ynz +ezhwlMkCAjbQMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUUtiIOsifeGbt +ifN7OHCUyQICNtAwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3DQEBCwUAA4ICAQAL +e3KHwGCmSUyIWOYdiPcUZEim2FgKDk8TNd81HdTtBjHIgT5q1d07GjLukD0R0i70 +jsNjLiNmsGe+b7bAEzlgqqI0JZN1Ut6nna0Oh4lScWoWPBkdg/iaKWW+9D+a2fDz +WochcYBNy+A4mz+7+uAwTc+G02UQGRjRlwKxK3JCaKygvU5a2hi/a5iB0P2avl4V +SM0RFbnAKVy06Ij3Pjaut2L9HmLecHgQHEhb2rykOLpn7VU+Xlff1ANATIGk0k9j +pwlCCRT8AKnCgHNPLsBA2RF7SOp6AsDT6ygBJlh0wcBzIm2Tlf05fbsq4/aC4yyX +X04fkZT6/iyj2HYauE2yOE+b+h1IYHkm4vP9qdCa6HCPSXrW5b0KDtst842/6+Ok +fcvHlXHo2qN8xcL4dJIEG4aspCJTQLas/kx2z/uUMsA1n3Y/buWQbqCmJqK4LL7R +K4X9p2jIugErsWx0Hbhzlefut8cl8ABMALJ+tguLHPPAUJ4lueAI3jZm/zel0btU +ZCzJJ7VLkn5l/9Mt4blOvH+kQSGQQXemOR/qnuOf0GZvBeyqdn6/axag67XH/JJU +LysRJyU3eExRarDzzFhdFPFqSBX/wge2sY0PjlxQRrM9vwGYT7JZVEc+NHt4bVaT +LnPqZih4zR0Uv6CPLy64Lo7yFIrM6bV8+2ydDKXhlg== +-----END CERTIFICATE----- + +# Issuer: CN=Buypass Class 2 Root CA O=Buypass AS-983163327 +# Subject: CN=Buypass Class 2 Root CA O=Buypass AS-983163327 +# Label: "Buypass Class 2 Root CA" +# Serial: 2 +# MD5 Fingerprint: 46:a7:d2:fe:45:fb:64:5a:a8:59:90:9b:78:44:9b:29 +# SHA1 Fingerprint: 49:0a:75:74:de:87:0a:47:fe:58:ee:f6:c7:6b:eb:c6:0b:12:40:99 +# SHA256 Fingerprint: 9a:11:40:25:19:7c:5b:b9:5d:94:e6:3d:55:cd:43:79:08:47:b6:46:b2:3c:df:11:ad:a4:a0:0e:ff:15:fb:48 +-----BEGIN CERTIFICATE----- +MIIFWTCCA0GgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBOMQswCQYDVQQGEwJOTzEd +MBsGA1UECgwUQnV5cGFzcyBBUy05ODMxNjMzMjcxIDAeBgNVBAMMF0J1eXBhc3Mg +Q2xhc3MgMiBSb290IENBMB4XDTEwMTAyNjA4MzgwM1oXDTQwMTAyNjA4MzgwM1ow +TjELMAkGA1UEBhMCTk8xHTAbBgNVBAoMFEJ1eXBhc3MgQVMtOTgzMTYzMzI3MSAw +HgYDVQQDDBdCdXlwYXNzIENsYXNzIDIgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEB +BQADggIPADCCAgoCggIBANfHXvfBB9R3+0Mh9PT1aeTuMgHbo4Yf5FkNuud1g1Lr +6hxhFUi7HQfKjK6w3Jad6sNgkoaCKHOcVgb/S2TwDCo3SbXlzwx87vFKu3MwZfPV +L4O2fuPn9Z6rYPnT8Z2SdIrkHJasW4DptfQxh6NR/Md+oW+OU3fUl8FVM5I+GC91 +1K2GScuVr1QGbNgGE41b/+EmGVnAJLqBcXmQRFBoJJRfuLMR8SlBYaNByyM21cHx +MlAQTn/0hpPshNOOvEu/XAFOBz3cFIqUCqTqc/sLUegTBxj6DvEr0VQVfTzh97QZ +QmdiXnfgolXsttlpF9U6r0TtSsWe5HonfOV116rLJeffawrbD02TTqigzXsu8lkB +arcNuAeBfos4GzjmCleZPe4h6KP1DBbdi+w0jpwqHAAVF41og9JwnxgIzRFo1clr +Us3ERo/ctfPYV3Me6ZQ5BL/T3jjetFPsaRyifsSP5BtwrfKi+fv3FmRmaZ9JUaLi +FRhnBkp/1Wy1TbMz4GHrXb7pmA8y1x1LPC5aAVKRCfLf6o3YBkBjqhHk/sM3nhRS +P/TizPJhk9H9Z2vXUq6/aKtAQ6BXNVN48FP4YUIHZMbXb5tMOA1jrGKvNouicwoN +9SG9dKpN6nIDSdvHXx1iY8f93ZHsM+71bbRuMGjeyNYmsHVee7QHIJihdjK4TWxP +AgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMmAd+BikoL1Rpzz +uvdMw964o605MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAU18h +9bqwOlI5LJKwbADJ784g7wbylp7ppHR/ehb8t/W2+xUbP6umwHJdELFx7rxP462s +A20ucS6vxOOto70MEae0/0qyexAQH6dXQbLArvQsWdZHEIjzIVEpMMpghq9Gqx3t +OluwlN5E40EIosHsHdb9T7bWR9AUC8rmyrV7d35BH16Dx7aMOZawP5aBQW9gkOLo ++fsicdl9sz1Gv7SEr5AcD48Saq/v7h56rgJKihcrdv6sVIkkLE8/trKnToyokZf7 +KcZ7XC25y2a2t6hbElGFtQl+Ynhw/qlqYLYdDnkM/crqJIByw5c/8nerQyIKx+u2 +DISCLIBrQYoIwOula9+ZEsuK1V6ADJHgJgg2SMX6OBE1/yWDLfJ6v9r9jv6ly0Us +H8SIU653DtmadsWOLB2jutXsMq7Aqqz30XpN69QH4kj3Io6wpJ9qzo6ysmD0oyLQ +I+uUWnpp3Q+/QFesa1lQ2aOZ4W7+jQF5JyMV3pKdewlNWudLSDBaGOYKbeaP4NK7 +5t98biGCwWg5TbSYWGZizEqQXsP6JwSxeRV0mcy+rSDeJmAc61ZRpqPq5KM/p/9h +3PFaTWwyI0PurKju7koSCTxdccK+efrCh2gdC/1cacwG0Jp9VJkqyTkaGa9LKkPz +Y11aWOIv4x3kqdbQCtCev9eBCfHJxyYNrJgWVqA= +-----END CERTIFICATE----- + +# Issuer: CN=Buypass Class 3 Root CA O=Buypass AS-983163327 +# Subject: CN=Buypass Class 3 Root CA O=Buypass AS-983163327 +# Label: "Buypass Class 3 Root CA" +# Serial: 2 +# MD5 Fingerprint: 3d:3b:18:9e:2c:64:5a:e8:d5:88:ce:0e:f9:37:c2:ec +# SHA1 Fingerprint: da:fa:f7:fa:66:84:ec:06:8f:14:50:bd:c7:c2:81:a5:bc:a9:64:57 +# SHA256 Fingerprint: ed:f7:eb:bc:a2:7a:2a:38:4d:38:7b:7d:40:10:c6:66:e2:ed:b4:84:3e:4c:29:b4:ae:1d:5b:93:32:e6:b2:4d +-----BEGIN CERTIFICATE----- +MIIFWTCCA0GgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBOMQswCQYDVQQGEwJOTzEd +MBsGA1UECgwUQnV5cGFzcyBBUy05ODMxNjMzMjcxIDAeBgNVBAMMF0J1eXBhc3Mg +Q2xhc3MgMyBSb290IENBMB4XDTEwMTAyNjA4Mjg1OFoXDTQwMTAyNjA4Mjg1OFow +TjELMAkGA1UEBhMCTk8xHTAbBgNVBAoMFEJ1eXBhc3MgQVMtOTgzMTYzMzI3MSAw +HgYDVQQDDBdCdXlwYXNzIENsYXNzIDMgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEB +BQADggIPADCCAgoCggIBAKXaCpUWUOOV8l6ddjEGMnqb8RB2uACatVI2zSRHsJ8Y +ZLya9vrVediQYkwiL944PdbgqOkcLNt4EemOaFEVcsfzM4fkoF0LXOBXByow9c3E +N3coTRiR5r/VUv1xLXA+58bEiuPwKAv0dpihi4dVsjoT/Lc+JzeOIuOoTyrvYLs9 +tznDDgFHmV0ST9tD+leh7fmdvhFHJlsTmKtdFoqwNxxXnUX/iJY2v7vKB3tvh2PX +0DJq1l1sDPGzbjniazEuOQAnFN44wOwZZoYS6J1yFhNkUsepNxz9gjDthBgd9K5c +/3ATAOux9TN6S9ZV+AWNS2mw9bMoNlwUxFFzTWsL8TQH2xc519woe2v1n/MuwU8X +KhDzzMro6/1rqy6any2CbgTUUgGTLT2G/H783+9CHaZr77kgxve9oKeV/afmiSTY +zIw0bOIjL9kSGiG5VZFvC5F5GQytQIgLcOJ60g7YaEi7ghM5EFjp2CoHxhLbWNvS +O1UQRwUVZ2J+GGOmRj8JDlQyXr8NYnon74Do29lLBlo3WiXQCBJ31G8JUJc9yB3D +34xFMFbG02SrZvPAXpacw8Tvw3xrizp5f7NJzz3iiZ+gMEuFuZyUJHmPfWupRWgP +K9Dx2hzLabjKSWJtyNBjYt1gD1iqj6G8BaVmos8bdrKEZLFMOVLAMLrwjEsCsLa3 +AgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFEe4zf/lb+74suwv +Tg75JbCOPGvDMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAACAj +QTUEkMJAYmDv4jVM1z+s4jSQuKFvdvoWFqRINyzpkMLyPPgKn9iB5btb2iUspKdV +cSQy9sgL8rxq+JOssgfCX5/bzMiKqr5qb+FJEMwx14C7u8jYog5kV+qi9cKpMRXS +IGrs/CIBKM+GuIAeqcwRpTzyFrNHnfzSgCHEy9BHcEGhyoMZCCxt8l13nIoUE9Q2 +HJLw5QY33KbmkJs4j1xrG0aGQ0JfPgEHU1RdZX33inOhmlRaHylDFCfChQ+1iHsa +O5S3HWCntZznKWlXWpuTekMwGwPXYshApqr8ZORK15FTAaggiG6cX0S5y2CBNOxv +033aSF/rtJC8LakcC6wc1aJoIIAE1vyxjy+7SjENSoYc6+I2KSb12tjE8nVhz36u +dmNKekBlk4f4HoCMhuWG1o8O/FMsYOgWYRqiPkN7zTlgVGr18okmAWiDSKIz6MkE +kbIRNBE+6tBDGR8Dk5AM/1E9V/RBbuHLoL7ryWPNbczk+DaqaJ3tvV2XcEQNtg41 +3OEMXbugUZTLfhbrES+jkkXITHHZvMmZUldGL1DPvTVp9D0VzgalLA8+9oG6lLvD +u79leNKGef9JOxqDDPDeeOzI8k1MGt6CKfjBWtrt7uYnXuhF0J0cUahoq0Tj0Itq +4/g7u9xN12TyUb7mqqta6THuBrxzvxNiCp/HuZc= +-----END CERTIFICATE----- + +# Issuer: CN=T-TeleSec GlobalRoot Class 3 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center +# Subject: CN=T-TeleSec GlobalRoot Class 3 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center +# Label: "T-TeleSec GlobalRoot Class 3" +# Serial: 1 +# MD5 Fingerprint: ca:fb:40:a8:4e:39:92:8a:1d:fe:8e:2f:c4:27:ea:ef +# SHA1 Fingerprint: 55:a6:72:3e:cb:f2:ec:cd:c3:23:74:70:19:9d:2a:be:11:e3:81:d1 +# SHA256 Fingerprint: fd:73:da:d3:1c:64:4f:f1:b4:3b:ef:0c:cd:da:96:71:0b:9c:d9:87:5e:ca:7e:31:70:7a:f3:e9:6d:52:2b:bd +-----BEGIN CERTIFICATE----- +MIIDwzCCAqugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBgjELMAkGA1UEBhMCREUx +KzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnByaXNlIFNlcnZpY2VzIEdtYkgxHzAd +BgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50ZXIxJTAjBgNVBAMMHFQtVGVsZVNl +YyBHbG9iYWxSb290IENsYXNzIDMwHhcNMDgxMDAxMTAyOTU2WhcNMzMxMDAxMjM1 +OTU5WjCBgjELMAkGA1UEBhMCREUxKzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnBy +aXNlIFNlcnZpY2VzIEdtYkgxHzAdBgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50 +ZXIxJTAjBgNVBAMMHFQtVGVsZVNlYyBHbG9iYWxSb290IENsYXNzIDMwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC9dZPwYiJvJK7genasfb3ZJNW4t/zN +8ELg63iIVl6bmlQdTQyK9tPPcPRStdiTBONGhnFBSivwKixVA9ZIw+A5OO3yXDw/ +RLyTPWGrTs0NvvAgJ1gORH8EGoel15YUNpDQSXuhdfsaa3Ox+M6pCSzyU9XDFES4 +hqX2iys52qMzVNn6chr3IhUciJFrf2blw2qAsCTz34ZFiP0Zf3WHHx+xGwpzJFu5 +ZeAsVMhg02YXP+HMVDNzkQI6pn97djmiH5a2OK61yJN0HZ65tOVgnS9W0eDrXltM +EnAMbEQgqxHY9Bn20pxSN+f6tsIxO0rUFJmtxxr1XV/6B7h8DR/Wgx6zAgMBAAGj +QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBS1 +A/d2O2GCahKqGFPrAyGUv/7OyjANBgkqhkiG9w0BAQsFAAOCAQEAVj3vlNW92nOy +WL6ukK2YJ5f+AbGwUgC4TeQbIXQbfsDuXmkqJa9c1h3a0nnJ85cp4IaH3gRZD/FZ +1GSFS5mvJQQeyUapl96Cshtwn5z2r3Ex3XsFpSzTucpH9sry9uetuUg/vBa3wW30 +6gmv7PO15wWeph6KU1HWk4HMdJP2udqmJQV0eVp+QD6CSyYRMG7hP0HHRwA11fXT +91Q+gT3aSWqas+8QPebrb9HIIkfLzM8BMZLZGOMivgkeGj5asuRrDFR6fUNOuIml +e9eiPZaGzPImNC1qkp2aGtAw4l1OBLBfiyB+d8E9lYLRRpo7PHi4b6HQDWSieB4p +TpPDpFQUWw== +-----END CERTIFICATE----- + +# Issuer: CN=D-TRUST Root Class 3 CA 2 2009 O=D-Trust GmbH +# Subject: CN=D-TRUST Root Class 3 CA 2 2009 O=D-Trust GmbH +# Label: "D-TRUST Root Class 3 CA 2 2009" +# Serial: 623603 +# MD5 Fingerprint: cd:e0:25:69:8d:47:ac:9c:89:35:90:f7:fd:51:3d:2f +# SHA1 Fingerprint: 58:e8:ab:b0:36:15:33:fb:80:f7:9b:1b:6d:29:d3:ff:8d:5f:00:f0 +# SHA256 Fingerprint: 49:e7:a4:42:ac:f0:ea:62:87:05:00:54:b5:25:64:b6:50:e4:f4:9e:42:e3:48:d6:aa:38:e0:39:e9:57:b1:c1 +-----BEGIN CERTIFICATE----- +MIIEMzCCAxugAwIBAgIDCYPzMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNVBAYTAkRF +MRUwEwYDVQQKDAxELVRydXN0IEdtYkgxJzAlBgNVBAMMHkQtVFJVU1QgUm9vdCBD +bGFzcyAzIENBIDIgMjAwOTAeFw0wOTExMDUwODM1NThaFw0yOTExMDUwODM1NTha +ME0xCzAJBgNVBAYTAkRFMRUwEwYDVQQKDAxELVRydXN0IEdtYkgxJzAlBgNVBAMM +HkQtVFJVU1QgUm9vdCBDbGFzcyAzIENBIDIgMjAwOTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBANOySs96R+91myP6Oi/WUEWJNTrGa9v+2wBoqOADER03 +UAifTUpolDWzU9GUY6cgVq/eUXjsKj3zSEhQPgrfRlWLJ23DEE0NkVJD2IfgXU42 +tSHKXzlABF9bfsyjxiupQB7ZNoTWSPOSHjRGICTBpFGOShrvUD9pXRl/RcPHAY9R +ySPocq60vFYJfxLLHLGvKZAKyVXMD9O0Gu1HNVpK7ZxzBCHQqr0ME7UAyiZsxGsM +lFqVlNpQmvH/pStmMaTJOKDfHR+4CS7zp+hnUquVH+BGPtikw8paxTGA6Eian5Rp +/hnd2HN8gcqW3o7tszIFZYQ05ub9VxC1X3a/L7AQDcUCAwEAAaOCARowggEWMA8G +A1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFP3aFMSfMN4hvR5COfyrYyNJ4PGEMA4G +A1UdDwEB/wQEAwIBBjCB0wYDVR0fBIHLMIHIMIGAoH6gfIZ6bGRhcDovL2RpcmVj +dG9yeS5kLXRydXN0Lm5ldC9DTj1ELVRSVVNUJTIwUm9vdCUyMENsYXNzJTIwMyUy +MENBJTIwMiUyMDIwMDksTz1ELVRydXN0JTIwR21iSCxDPURFP2NlcnRpZmljYXRl +cmV2b2NhdGlvbmxpc3QwQ6BBoD+GPWh0dHA6Ly93d3cuZC10cnVzdC5uZXQvY3Js +L2QtdHJ1c3Rfcm9vdF9jbGFzc18zX2NhXzJfMjAwOS5jcmwwDQYJKoZIhvcNAQEL +BQADggEBAH+X2zDI36ScfSF6gHDOFBJpiBSVYEQBrLLpME+bUMJm2H6NMLVwMeni +acfzcNsgFYbQDfC+rAF1hM5+n02/t2A7nPPKHeJeaNijnZflQGDSNiH+0LS4F9p0 +o3/U37CYAqxva2ssJSRyoWXuJVrl5jLn8t+rSfrzkGkj2wTZ51xY/GXUl77M/C4K +zCUqNQT4YJEVdT1B/yMfGchs64JTBKbkTCJNjYy6zltz7GRUUG3RnFX7acM2w4y8 +PIWmawomDeCTmGCufsYkl4phX5GOZpIJhzbNi5stPvZR1FDUWSi9g/LMKHtThm3Y +Johw1+qRzT65ysCQblrGXnRl11z+o+I= +-----END CERTIFICATE----- + +# Issuer: CN=D-TRUST Root Class 3 CA 2 EV 2009 O=D-Trust GmbH +# Subject: CN=D-TRUST Root Class 3 CA 2 EV 2009 O=D-Trust GmbH +# Label: "D-TRUST Root Class 3 CA 2 EV 2009" +# Serial: 623604 +# MD5 Fingerprint: aa:c6:43:2c:5e:2d:cd:c4:34:c0:50:4f:11:02:4f:b6 +# SHA1 Fingerprint: 96:c9:1b:0b:95:b4:10:98:42:fa:d0:d8:22:79:fe:60:fa:b9:16:83 +# SHA256 Fingerprint: ee:c5:49:6b:98:8c:e9:86:25:b9:34:09:2e:ec:29:08:be:d0:b0:f3:16:c2:d4:73:0c:84:ea:f1:f3:d3:48:81 +-----BEGIN CERTIFICATE----- +MIIEQzCCAyugAwIBAgIDCYP0MA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNVBAYTAkRF +MRUwEwYDVQQKDAxELVRydXN0IEdtYkgxKjAoBgNVBAMMIUQtVFJVU1QgUm9vdCBD +bGFzcyAzIENBIDIgRVYgMjAwOTAeFw0wOTExMDUwODUwNDZaFw0yOTExMDUwODUw +NDZaMFAxCzAJBgNVBAYTAkRFMRUwEwYDVQQKDAxELVRydXN0IEdtYkgxKjAoBgNV +BAMMIUQtVFJVU1QgUm9vdCBDbGFzcyAzIENBIDIgRVYgMjAwOTCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAJnxhDRwui+3MKCOvXwEz75ivJn9gpfSegpn +ljgJ9hBOlSJzmY3aFS3nBfwZcyK3jpgAvDw9rKFs+9Z5JUut8Mxk2og+KbgPCdM0 +3TP1YtHhzRnp7hhPTFiu4h7WDFsVWtg6uMQYZB7jM7K1iXdODL/ZlGsTl28So/6Z +qQTMFexgaDbtCHu39b+T7WYxg4zGcTSHThfqr4uRjRxWQa4iN1438h3Z0S0NL2lR +p75mpoo6Kr3HGrHhFPC+Oh25z1uxav60sUYgovseO3Dvk5h9jHOW8sXvhXCtKSb8 +HgQ+HKDYD8tSg2J87otTlZCpV6LqYQXY+U3EJ/pure3511H3a6UCAwEAAaOCASQw +ggEgMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNOUikxiEyoZLsyvcop9Ntea +HNxnMA4GA1UdDwEB/wQEAwIBBjCB3QYDVR0fBIHVMIHSMIGHoIGEoIGBhn9sZGFw +Oi8vZGlyZWN0b3J5LmQtdHJ1c3QubmV0L0NOPUQtVFJVU1QlMjBSb290JTIwQ2xh +c3MlMjAzJTIwQ0ElMjAyJTIwRVYlMjAyMDA5LE89RC1UcnVzdCUyMEdtYkgsQz1E +RT9jZXJ0aWZpY2F0ZXJldm9jYXRpb25saXN0MEagRKBChkBodHRwOi8vd3d3LmQt +dHJ1c3QubmV0L2NybC9kLXRydXN0X3Jvb3RfY2xhc3NfM19jYV8yX2V2XzIwMDku +Y3JsMA0GCSqGSIb3DQEBCwUAA4IBAQA07XtaPKSUiO8aEXUHL7P+PPoeUSbrh/Yp +3uDx1MYkCenBz1UbtDDZzhr+BlGmFaQt77JLvyAoJUnRpjZ3NOhk31KxEcdzes05 +nsKtjHEh8lprr988TlWvsoRlFIm5d8sqMb7Po23Pb0iUMkZv53GMoKaEGTcH8gNF +CSuGdXzfX2lXANtu2KZyIktQ1HWYVt+3GP9DQ1CuekR78HlR10M9p9OB0/DJT7na +xpeG0ILD5EJt/rDiZE4OJudANCa1CInXCGNjOCd1HjPqbqjdn5lPdE2BiYBL3ZqX +KVwvvoFBuYz/6n1gBp7N1z3TLqMVvKjmJuVvw9y4AyHqnxbxLFS1 +-----END CERTIFICATE----- + +# Issuer: CN=CA Disig Root R2 O=Disig a.s. +# Subject: CN=CA Disig Root R2 O=Disig a.s. +# Label: "CA Disig Root R2" +# Serial: 10572350602393338211 +# MD5 Fingerprint: 26:01:fb:d8:27:a7:17:9a:45:54:38:1a:43:01:3b:03 +# SHA1 Fingerprint: b5:61:eb:ea:a4:de:e4:25:4b:69:1a:98:a5:57:47:c2:34:c7:d9:71 +# SHA256 Fingerprint: e2:3d:4a:03:6d:7b:70:e9:f5:95:b1:42:20:79:d2:b9:1e:df:bb:1f:b6:51:a0:63:3e:aa:8a:9d:c5:f8:07:03 +-----BEGIN CERTIFICATE----- +MIIFaTCCA1GgAwIBAgIJAJK4iNuwisFjMA0GCSqGSIb3DQEBCwUAMFIxCzAJBgNV +BAYTAlNLMRMwEQYDVQQHEwpCcmF0aXNsYXZhMRMwEQYDVQQKEwpEaXNpZyBhLnMu +MRkwFwYDVQQDExBDQSBEaXNpZyBSb290IFIyMB4XDTEyMDcxOTA5MTUzMFoXDTQy +MDcxOTA5MTUzMFowUjELMAkGA1UEBhMCU0sxEzARBgNVBAcTCkJyYXRpc2xhdmEx +EzARBgNVBAoTCkRpc2lnIGEucy4xGTAXBgNVBAMTEENBIERpc2lnIFJvb3QgUjIw +ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCio8QACdaFXS1tFPbCw3Oe +NcJxVX6B+6tGUODBfEl45qt5WDza/3wcn9iXAng+a0EE6UG9vgMsRfYvZNSrXaNH +PWSb6WiaxswbP7q+sos0Ai6YVRn8jG+qX9pMzk0DIaPY0jSTVpbLTAwAFjxfGs3I +x2ymrdMxp7zo5eFm1tL7A7RBZckQrg4FY8aAamkw/dLukO8NJ9+flXP04SXabBbe +QTg06ov80egEFGEtQX6sx3dOy1FU+16SGBsEWmjGycT6txOgmLcRK7fWV8x8nhfR +yyX+hk4kLlYMeE2eARKmK6cBZW58Yh2EhN/qwGu1pSqVg8NTEQxzHQuyRpDRQjrO +QG6Vrf/GlK1ul4SOfW+eioANSW1z4nuSHsPzwfPrLgVv2RvPN3YEyLRa5Beny912 +H9AZdugsBbPWnDTYltxhh5EF5EQIM8HauQhl1K6yNg3ruji6DOWbnuuNZt2Zz9aJ +QfYEkoopKW1rOhzndX0CcQ7zwOe9yxndnWCywmZgtrEE7snmhrmaZkCo5xHtgUUD +i/ZnWejBBhG93c+AAk9lQHhcR1DIm+YfgXvkRKhbhZri3lrVx/k6RGZL5DJUfORs +nLMOPReisjQS1n6yqEm70XooQL6iFh/f5DcfEXP7kAplQ6INfPgGAVUzfbANuPT1 +rqVCV3w2EYx7XsQDnYx5nQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1Ud +DwEB/wQEAwIBBjAdBgNVHQ4EFgQUtZn4r7CU9eMg1gqtzk5WpC5uQu0wDQYJKoZI +hvcNAQELBQADggIBACYGXnDnZTPIgm7ZnBc6G3pmsgH2eDtpXi/q/075KMOYKmFM +tCQSin1tERT3nLXK5ryeJ45MGcipvXrA1zYObYVybqjGom32+nNjf7xueQgcnYqf +GopTpti72TVVsRHFqQOzVju5hJMiXn7B9hJSi+osZ7z+Nkz1uM/Rs0mSO9MpDpkb +lvdhuDvEK7Z4bLQjb/D907JedR+Zlais9trhxTF7+9FGs9K8Z7RiVLoJ92Owk6Ka ++elSLotgEqv89WBW7xBci8QaQtyDW2QOy7W81k/BfDxujRNt+3vrMNDcTa/F1bal +TFtxyegxvug4BkihGuLq0t4SOVga/4AOgnXmt8kHbA7v/zjxmHHEt38OFdAlab0i +nSvtBfZGR6ztwPDUO+Ls7pZbkBNOHlY667DvlruWIxG68kOGdGSVyCh13x01utI3 +gzhTODY7z2zp+WsO0PsE6E9312UBeIYMej4hYvF/Y3EMyZ9E26gnonW+boE+18Dr +G5gPcFw0sorMwIUY6256s/daoQe/qUKS82Ail+QUoQebTnbAjn39pCXHR+3/H3Os +zMOl6W8KjptlwlCFtaOgUxLMVYdh84GuEEZhvUQhuMI9dM9+JDX6HAcOmz0iyu8x +L4ysEr3vQCj8KWefshNPZiTEUxnpHikV7+ZtsH8tZ/3zbBt1RqPlShfppNcL +-----END CERTIFICATE----- + +# Issuer: CN=ACCVRAIZ1 O=ACCV OU=PKIACCV +# Subject: CN=ACCVRAIZ1 O=ACCV OU=PKIACCV +# Label: "ACCVRAIZ1" +# Serial: 6828503384748696800 +# MD5 Fingerprint: d0:a0:5a:ee:05:b6:09:94:21:a1:7d:f1:b2:29:82:02 +# SHA1 Fingerprint: 93:05:7a:88:15:c6:4f:ce:88:2f:fa:91:16:52:28:78:bc:53:64:17 +# SHA256 Fingerprint: 9a:6e:c0:12:e1:a7:da:9d:be:34:19:4d:47:8a:d7:c0:db:18:22:fb:07:1d:f1:29:81:49:6e:d1:04:38:41:13 +-----BEGIN CERTIFICATE----- +MIIH0zCCBbugAwIBAgIIXsO3pkN/pOAwDQYJKoZIhvcNAQEFBQAwQjESMBAGA1UE +AwwJQUNDVlJBSVoxMRAwDgYDVQQLDAdQS0lBQ0NWMQ0wCwYDVQQKDARBQ0NWMQsw +CQYDVQQGEwJFUzAeFw0xMTA1MDUwOTM3MzdaFw0zMDEyMzEwOTM3MzdaMEIxEjAQ +BgNVBAMMCUFDQ1ZSQUlaMTEQMA4GA1UECwwHUEtJQUNDVjENMAsGA1UECgwEQUND +VjELMAkGA1UEBhMCRVMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCb +qau/YUqXry+XZpp0X9DZlv3P4uRm7x8fRzPCRKPfmt4ftVTdFXxpNRFvu8gMjmoY +HtiP2Ra8EEg2XPBjs5BaXCQ316PWywlxufEBcoSwfdtNgM3802/J+Nq2DoLSRYWo +G2ioPej0RGy9ocLLA76MPhMAhN9KSMDjIgro6TenGEyxCQ0jVn8ETdkXhBilyNpA +lHPrzg5XPAOBOp0KoVdDaaxXbXmQeOW1tDvYvEyNKKGno6e6Ak4l0Squ7a4DIrhr +IA8wKFSVf+DuzgpmndFALW4ir50awQUZ0m/A8p/4e7MCQvtQqR0tkw8jq8bBD5L/ +0KIV9VMJcRz/RROE5iZe+OCIHAr8Fraocwa48GOEAqDGWuzndN9wrqODJerWx5eH +k6fGioozl2A3ED6XPm4pFdahD9GILBKfb6qkxkLrQaLjlUPTAYVtjrs78yM2x/47 +4KElB0iryYl0/wiPgL/AlmXz7uxLaL2diMMxs0Dx6M/2OLuc5NF/1OVYm3z61PMO +m3WR5LpSLhl+0fXNWhn8ugb2+1KoS5kE3fj5tItQo05iifCHJPqDQsGH+tUtKSpa +cXpkatcnYGMN285J9Y0fkIkyF/hzQ7jSWpOGYdbhdQrqeWZ2iE9x6wQl1gpaepPl +uUsXQA+xtrn13k/c4LOsOxFwYIRKQ26ZIMApcQrAZQIDAQABo4ICyzCCAscwfQYI +KwYBBQUHAQEEcTBvMEwGCCsGAQUFBzAChkBodHRwOi8vd3d3LmFjY3YuZXMvZmls +ZWFkbWluL0FyY2hpdm9zL2NlcnRpZmljYWRvcy9yYWl6YWNjdjEuY3J0MB8GCCsG +AQUFBzABhhNodHRwOi8vb2NzcC5hY2N2LmVzMB0GA1UdDgQWBBTSh7Tj3zcnk1X2 +VuqB5TbMjB4/vTAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNKHtOPfNyeT +VfZW6oHlNsyMHj+9MIIBcwYDVR0gBIIBajCCAWYwggFiBgRVHSAAMIIBWDCCASIG +CCsGAQUFBwICMIIBFB6CARAAQQB1AHQAbwByAGkAZABhAGQAIABkAGUAIABDAGUA +cgB0AGkAZgBpAGMAYQBjAGkA8wBuACAAUgBhAO0AegAgAGQAZQAgAGwAYQAgAEEA +QwBDAFYAIAAoAEEAZwBlAG4AYwBpAGEAIABkAGUAIABUAGUAYwBuAG8AbABvAGcA +7QBhACAAeQAgAEMAZQByAHQAaQBmAGkAYwBhAGMAaQDzAG4AIABFAGwAZQBjAHQA +cgDzAG4AaQBjAGEALAAgAEMASQBGACAAUQA0ADYAMAAxADEANQA2AEUAKQAuACAA +QwBQAFMAIABlAG4AIABoAHQAdABwADoALwAvAHcAdwB3AC4AYQBjAGMAdgAuAGUA +czAwBggrBgEFBQcCARYkaHR0cDovL3d3dy5hY2N2LmVzL2xlZ2lzbGFjaW9uX2Mu +aHRtMFUGA1UdHwROMEwwSqBIoEaGRGh0dHA6Ly93d3cuYWNjdi5lcy9maWxlYWRt +aW4vQXJjaGl2b3MvY2VydGlmaWNhZG9zL3JhaXphY2N2MV9kZXIuY3JsMA4GA1Ud +DwEB/wQEAwIBBjAXBgNVHREEEDAOgQxhY2N2QGFjY3YuZXMwDQYJKoZIhvcNAQEF +BQADggIBAJcxAp/n/UNnSEQU5CmH7UwoZtCPNdpNYbdKl02125DgBS4OxnnQ8pdp +D70ER9m+27Up2pvZrqmZ1dM8MJP1jaGo/AaNRPTKFpV8M9xii6g3+CfYCS0b78gU +JyCpZET/LtZ1qmxNYEAZSUNUY9rizLpm5U9EelvZaoErQNV/+QEnWCzI7UiRfD+m +AM/EKXMRNt6GGT6d7hmKG9Ww7Y49nCrADdg9ZuM8Db3VlFzi4qc1GwQA9j9ajepD +vV+JHanBsMyZ4k0ACtrJJ1vnE5Bc5PUzolVt3OAJTS+xJlsndQAJxGJ3KQhfnlms +tn6tn1QwIgPBHnFk/vk4CpYY3QIUrCPLBhwepH2NDd4nQeit2hW3sCPdK6jT2iWH +7ehVRE2I9DZ+hJp4rPcOVkkO1jMl1oRQQmwgEh0q1b688nCBpHBgvgW1m54ERL5h +I6zppSSMEYCUWqKiuUnSwdzRp+0xESyeGabu4VXhwOrPDYTkF7eifKXeVSUG7szA +h1xA2syVP1XgNce4hL60Xc16gwFy7ofmXx2utYXGJt/mwZrpHgJHnyqobalbz+xF +d3+YJ5oyXSrjhO7FmGYvliAd3djDJ9ew+f7Zfc3Qn48LFFhRny+Lwzgt3uiP1o2H +pPVWQxaZLPSkVrQ0uGE3ycJYgBugl6H8WY3pEfbRD0tVNEYqi4Y7 +-----END CERTIFICATE----- + +# Issuer: CN=TWCA Global Root CA O=TAIWAN-CA OU=Root CA +# Subject: CN=TWCA Global Root CA O=TAIWAN-CA OU=Root CA +# Label: "TWCA Global Root CA" +# Serial: 3262 +# MD5 Fingerprint: f9:03:7e:cf:e6:9e:3c:73:7a:2a:90:07:69:ff:2b:96 +# SHA1 Fingerprint: 9c:bb:48:53:f6:a4:f6:d3:52:a4:e8:32:52:55:60:13:f5:ad:af:65 +# SHA256 Fingerprint: 59:76:90:07:f7:68:5d:0f:cd:50:87:2f:9f:95:d5:75:5a:5b:2b:45:7d:81:f3:69:2b:61:0a:98:67:2f:0e:1b +-----BEGIN CERTIFICATE----- +MIIFQTCCAymgAwIBAgICDL4wDQYJKoZIhvcNAQELBQAwUTELMAkGA1UEBhMCVFcx +EjAQBgNVBAoTCVRBSVdBTi1DQTEQMA4GA1UECxMHUm9vdCBDQTEcMBoGA1UEAxMT +VFdDQSBHbG9iYWwgUm9vdCBDQTAeFw0xMjA2MjcwNjI4MzNaFw0zMDEyMzExNTU5 +NTlaMFExCzAJBgNVBAYTAlRXMRIwEAYDVQQKEwlUQUlXQU4tQ0ExEDAOBgNVBAsT +B1Jvb3QgQ0ExHDAaBgNVBAMTE1RXQ0EgR2xvYmFsIFJvb3QgQ0EwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQCwBdvI64zEbooh745NnHEKH1Jw7W2CnJfF +10xORUnLQEK1EjRsGcJ0pDFfhQKX7EMzClPSnIyOt7h52yvVavKOZsTuKwEHktSz +0ALfUPZVr2YOy+BHYC8rMjk1Ujoog/h7FsYYuGLWRyWRzvAZEk2tY/XTP3VfKfCh +MBwqoJimFb3u/Rk28OKRQ4/6ytYQJ0lM793B8YVwm8rqqFpD/G2Gb3PpN0Wp8DbH +zIh1HrtsBv+baz4X7GGqcXzGHaL3SekVtTzWoWH1EfcFbx39Eb7QMAfCKbAJTibc +46KokWofwpFFiFzlmLhxpRUZyXx1EcxwdE8tmx2RRP1WKKD+u4ZqyPpcC1jcxkt2 +yKsi2XMPpfRaAok/T54igu6idFMqPVMnaR1sjjIsZAAmY2E2TqNGtz99sy2sbZCi +laLOz9qC5wc0GZbpuCGqKX6mOL6OKUohZnkfs8O1CWfe1tQHRvMq2uYiN2DLgbYP +oA/pyJV/v1WRBXrPPRXAb94JlAGD1zQbzECl8LibZ9WYkTunhHiVJqRaCPgrdLQA +BDzfuBSO6N+pjWxnkjMdwLfS7JLIvgm/LCkFbwJrnu+8vyq8W8BQj0FwcYeyTbcE +qYSjMq+u7msXi7Kx/mzhkIyIqJdIzshNy/MGz19qCkKxHh53L46g5pIOBvwFItIm +4TFRfTLcDwIDAQABoyMwITAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB +/zANBgkqhkiG9w0BAQsFAAOCAgEAXzSBdu+WHdXltdkCY4QWwa6gcFGn90xHNcgL +1yg9iXHZqjNB6hQbbCEAwGxCGX6faVsgQt+i0trEfJdLjbDorMjupWkEmQqSpqsn +LhpNgb+E1HAerUf+/UqdM+DyucRFCCEK2mlpc3INvjT+lIutwx4116KD7+U4x6WF +H6vPNOw/KP4M8VeGTslV9xzU2KV9Bnpv1d8Q34FOIWWxtuEXeZVFBs5fzNxGiWNo +RI2T9GRwoD2dKAXDOXC4Ynsg/eTb6QihuJ49CcdP+yz4k3ZB3lLg4VfSnQO8d57+ +nile98FRYB/e2guyLXW3Q0iT5/Z5xoRdgFlglPx4mI88k1HtQJAH32RjJMtOcQWh +15QaiDLxInQirqWm2BJpTGCjAu4r7NRjkgtevi92a6O2JryPA9gK8kxkRr05YuWW +6zRjESjMlfGt7+/cgFhI6Uu46mWs6fyAtbXIRfmswZ/ZuepiiI7E8UuDEq3mi4TW +nsLrgxifarsbJGAzcMzs9zLzXNl5fe+epP7JI8Mk7hWSsT2RTyaGvWZzJBPqpK5j +wa19hAM8EHiGG3njxPPyBJUgriOCxLM6AGK/5jYk4Ve6xx6QddVfP5VhK8E7zeWz +aGHQRiapIVJpLesux+t3zqY6tQMzT3bR51xUAV3LePTJDL/PEo4XLSNolOer/qmy +KwbQBM0= +-----END CERTIFICATE----- + +# Issuer: CN=T-TeleSec GlobalRoot Class 2 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center +# Subject: CN=T-TeleSec GlobalRoot Class 2 O=T-Systems Enterprise Services GmbH OU=T-Systems Trust Center +# Label: "T-TeleSec GlobalRoot Class 2" +# Serial: 1 +# MD5 Fingerprint: 2b:9b:9e:e4:7b:6c:1f:00:72:1a:cc:c1:77:79:df:6a +# SHA1 Fingerprint: 59:0d:2d:7d:88:4f:40:2e:61:7e:a5:62:32:17:65:cf:17:d8:94:e9 +# SHA256 Fingerprint: 91:e2:f5:78:8d:58:10:eb:a7:ba:58:73:7d:e1:54:8a:8e:ca:cd:01:45:98:bc:0b:14:3e:04:1b:17:05:25:52 +-----BEGIN CERTIFICATE----- +MIIDwzCCAqugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBgjELMAkGA1UEBhMCREUx +KzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnByaXNlIFNlcnZpY2VzIEdtYkgxHzAd +BgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50ZXIxJTAjBgNVBAMMHFQtVGVsZVNl +YyBHbG9iYWxSb290IENsYXNzIDIwHhcNMDgxMDAxMTA0MDE0WhcNMzMxMDAxMjM1 +OTU5WjCBgjELMAkGA1UEBhMCREUxKzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnBy +aXNlIFNlcnZpY2VzIEdtYkgxHzAdBgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50 +ZXIxJTAjBgNVBAMMHFQtVGVsZVNlYyBHbG9iYWxSb290IENsYXNzIDIwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCqX9obX+hzkeXaXPSi5kfl82hVYAUd +AqSzm1nzHoqvNK38DcLZSBnuaY/JIPwhqgcZ7bBcrGXHX+0CfHt8LRvWurmAwhiC +FoT6ZrAIxlQjgeTNuUk/9k9uN0goOA/FvudocP05l03Sx5iRUKrERLMjfTlH6VJi +1hKTXrcxlkIF+3anHqP1wvzpesVsqXFP6st4vGCvx9702cu+fjOlbpSD8DT6Iavq +jnKgP6TeMFvvhk1qlVtDRKgQFRzlAVfFmPHmBiiRqiDFt1MmUUOyCxGVWOHAD3bZ +wI18gfNycJ5v/hqO2V81xrJvNHy+SE/iWjnX2J14np+GPgNeGYtEotXHAgMBAAGj +QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBS/ +WSA2AHmgoCJrjNXyYdK4LMuCSjANBgkqhkiG9w0BAQsFAAOCAQEAMQOiYQsfdOhy +NsZt+U2e+iKo4YFWz827n+qrkRk4r6p8FU3ztqONpfSO9kSpp+ghla0+AGIWiPAC +uvxhI+YzmzB6azZie60EI4RYZeLbK4rnJVM3YlNfvNoBYimipidx5joifsFvHZVw +IEoHNN/q/xWA5brXethbdXwFeilHfkCoMRN3zUA7tFFHei4R40cR3p1m0IvVVGb6 +g1XqfMIpiRvpb7PO4gWEyS8+eIVibslfwXhjdFjASBgMmTnrpMwatXlajRWc2BQN +9noHV8cigwUtPJslJj0Ys6lDfMjIq2SPDqO/nBudMNva0Bkuqjzx+zOAduTNrRlP +BSeOE6Fuwg== +-----END CERTIFICATE----- + +# Issuer: CN=Atos TrustedRoot 2011 O=Atos +# Subject: CN=Atos TrustedRoot 2011 O=Atos +# Label: "Atos TrustedRoot 2011" +# Serial: 6643877497813316402 +# MD5 Fingerprint: ae:b9:c4:32:4b:ac:7f:5d:66:cc:77:94:bb:2a:77:56 +# SHA1 Fingerprint: 2b:b1:f5:3e:55:0c:1d:c5:f1:d4:e6:b7:6a:46:4b:55:06:02:ac:21 +# SHA256 Fingerprint: f3:56:be:a2:44:b7:a9:1e:b3:5d:53:ca:9a:d7:86:4a:ce:01:8e:2d:35:d5:f8:f9:6d:df:68:a6:f4:1a:a4:74 +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- + +# Issuer: CN=QuoVadis Root CA 1 G3 O=QuoVadis Limited +# Subject: CN=QuoVadis Root CA 1 G3 O=QuoVadis Limited +# Label: "QuoVadis Root CA 1 G3" +# Serial: 687049649626669250736271037606554624078720034195 +# MD5 Fingerprint: a4:bc:5b:3f:fe:37:9a:fa:64:f0:e2:fa:05:3d:0b:ab +# SHA1 Fingerprint: 1b:8e:ea:57:96:29:1a:c9:39:ea:b8:0a:81:1a:73:73:c0:93:79:67 +# SHA256 Fingerprint: 8a:86:6f:d1:b2:76:b5:7e:57:8e:92:1c:65:82:8a:2b:ed:58:e9:f2:f2:88:05:41:34:b7:f1:f4:bf:c9:cc:74 +-----BEGIN CERTIFICATE----- +MIIFYDCCA0igAwIBAgIUeFhfLq0sGUvjNwc1NBMotZbUZZMwDQYJKoZIhvcNAQEL +BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc +BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMSBHMzAeFw0xMjAxMTIxNzI3NDRaFw00 +MjAxMTIxNzI3NDRaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM +aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDEgRzMwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQCgvlAQjunybEC0BJyFuTHK3C3kEakEPBtV +wedYMB0ktMPvhd6MLOHBPd+C5k+tR4ds7FtJwUrVu4/sh6x/gpqG7D0DmVIB0jWe +rNrwU8lmPNSsAgHaJNM7qAJGr6Qc4/hzWHa39g6QDbXwz8z6+cZM5cOGMAqNF341 +68Xfuw6cwI2H44g4hWf6Pser4BOcBRiYz5P1sZK0/CPTz9XEJ0ngnjybCKOLXSoh +4Pw5qlPafX7PGglTvF0FBM+hSo+LdoINofjSxxR3W5A2B4GbPgb6Ul5jxaYA/qXp +UhtStZI5cgMJYr2wYBZupt0lwgNm3fME0UDiTouG9G/lg6AnhF4EwfWQvTA9xO+o +abw4m6SkltFi2mnAAZauy8RRNOoMqv8hjlmPSlzkYZqn0ukqeI1RPToV7qJZjqlc +3sX5kCLliEVx3ZGZbHqfPT2YfF72vhZooF6uCyP8Wg+qInYtyaEQHeTTRCOQiJ/G +KubX9ZqzWB4vMIkIG1SitZgj7Ah3HJVdYdHLiZxfokqRmu8hqkkWCKi9YSgxyXSt +hfbZxbGL0eUQMk1fiyA6PEkfM4VZDdvLCXVDaXP7a3F98N/ETH3Goy7IlXnLc6KO +Tk0k+17kBL5yG6YnLUlamXrXXAkgt3+UuU/xDRxeiEIbEbfnkduebPRq34wGmAOt +zCjvpUfzUwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +BjAdBgNVHQ4EFgQUo5fW816iEOGrRZ88F2Q87gFwnMwwDQYJKoZIhvcNAQELBQAD +ggIBABj6W3X8PnrHX3fHyt/PX8MSxEBd1DKquGrX1RUVRpgjpeaQWxiZTOOtQqOC +MTaIzen7xASWSIsBx40Bz1szBpZGZnQdT+3Btrm0DWHMY37XLneMlhwqI2hrhVd2 +cDMT/uFPpiN3GPoajOi9ZcnPP/TJF9zrx7zABC4tRi9pZsMbj/7sPtPKlL92CiUN +qXsCHKnQO18LwIE6PWThv6ctTr1NxNgpxiIY0MWscgKCP6o6ojoilzHdCGPDdRS5 +YCgtW2jgFqlmgiNR9etT2DGbe+m3nUvriBbP+V04ikkwj+3x6xn0dxoxGE1nVGwv +b2X52z3sIexe9PSLymBlVNFxZPT5pqOBMzYzcfCkeF9OrYMh3jRJjehZrJ3ydlo2 +8hP0r+AJx2EqbPfgna67hkooby7utHnNkDPDs3b69fBsnQGQ+p6Q9pxyz0fawx/k +NSBT8lTR32GDpgLiJTjehTItXnOQUl1CxM49S+H5GYQd1aJQzEH7QRTDvdbJWqNj +ZgKAvQU6O0ec7AAmTPWIUb+oI38YB7AL7YsmoWTTYUrrXJ/es69nA7Mf3W1daWhp +q1467HxpvMc7hU6eFbm0FU/DlXpY18ls6Wy58yljXrQs8C097Vpl4KlbQMJImYFt +nh8GKjwStIsPm6Ik8KaN1nrgS7ZklmOVhMJKzRwuJIczYOXD +-----END CERTIFICATE----- + +# Issuer: CN=QuoVadis Root CA 2 G3 O=QuoVadis Limited +# Subject: CN=QuoVadis Root CA 2 G3 O=QuoVadis Limited +# Label: "QuoVadis Root CA 2 G3" +# Serial: 390156079458959257446133169266079962026824725800 +# MD5 Fingerprint: af:0c:86:6e:bf:40:2d:7f:0b:3e:12:50:ba:12:3d:06 +# SHA1 Fingerprint: 09:3c:61:f3:8b:8b:dc:7d:55:df:75:38:02:05:00:e1:25:f5:c8:36 +# SHA256 Fingerprint: 8f:e4:fb:0a:f9:3a:4d:0d:67:db:0b:eb:b2:3e:37:c7:1b:f3:25:dc:bc:dd:24:0e:a0:4d:af:58:b4:7e:18:40 +-----BEGIN CERTIFICATE----- +MIIFYDCCA0igAwIBAgIURFc0JFuBiZs18s64KztbpybwdSgwDQYJKoZIhvcNAQEL +BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc +BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMiBHMzAeFw0xMjAxMTIxODU5MzJaFw00 +MjAxMTIxODU5MzJaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM +aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDIgRzMwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQChriWyARjcV4g/Ruv5r+LrI3HimtFhZiFf +qq8nUeVuGxbULX1QsFN3vXg6YOJkApt8hpvWGo6t/x8Vf9WVHhLL5hSEBMHfNrMW +n4rjyduYNM7YMxcoRvynyfDStNVNCXJJ+fKH46nafaF9a7I6JaltUkSs+L5u+9ym +c5GQYaYDFCDy54ejiK2toIz/pgslUiXnFgHVy7g1gQyjO/Dh4fxaXc6AcW34Sas+ +O7q414AB+6XrW7PFXmAqMaCvN+ggOp+oMiwMzAkd056OXbxMmO7FGmh77FOm6RQ1 +o9/NgJ8MSPsc9PG/Srj61YxxSscfrf5BmrODXfKEVu+lV0POKa2Mq1W/xPtbAd0j +IaFYAI7D0GoT7RPjEiuA3GfmlbLNHiJuKvhB1PLKFAeNilUSxmn1uIZoL1NesNKq +IcGY5jDjZ1XHm26sGahVpkUG0CM62+tlXSoREfA7T8pt9DTEceT/AFr2XK4jYIVz +8eQQsSWu1ZK7E8EM4DnatDlXtas1qnIhO4M15zHfeiFuuDIIfR0ykRVKYnLP43eh +vNURG3YBZwjgQQvD6xVu+KQZ2aKrr+InUlYrAoosFCT5v0ICvybIxo/gbjh9Uy3l +7ZizlWNof/k19N+IxWA1ksB8aRxhlRbQ694Lrz4EEEVlWFA4r0jyWbYW8jwNkALG +cC4BrTwV1wIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +BjAdBgNVHQ4EFgQU7edvdlq/YOxJW8ald7tyFnGbxD0wDQYJKoZIhvcNAQELBQAD +ggIBAJHfgD9DCX5xwvfrs4iP4VGyvD11+ShdyLyZm3tdquXK4Qr36LLTn91nMX66 +AarHakE7kNQIXLJgapDwyM4DYvmL7ftuKtwGTTwpD4kWilhMSA/ohGHqPHKmd+RC +roijQ1h5fq7KpVMNqT1wvSAZYaRsOPxDMuHBR//47PERIjKWnML2W2mWeyAMQ0Ga +W/ZZGYjeVYg3UQt4XAoeo0L9x52ID8DyeAIkVJOviYeIyUqAHerQbj5hLja7NQ4n +lv1mNDthcnPxFlxHBlRJAHpYErAK74X9sbgzdWqTHBLmYF5vHX/JHyPLhGGfHoJE ++V+tYlUkmlKY7VHnoX6XOuYvHxHaU4AshZ6rNRDbIl9qxV6XU/IyAgkwo1jwDQHV +csaxfGl7w/U2Rcxhbl5MlMVerugOXou/983g7aEOGzPuVBj+D77vfoRrQ+NwmNtd +dbINWQeFFSM51vHfqSYP1kjHs6Yi9TM3WpVHn3u6GBVv/9YUZINJ0gpnIdsPNWNg +KCLjsZWDzYWm3S8P52dSbrsvhXz1SnPnxT7AvSESBT/8twNJAlvIJebiVDj1eYeM +HVOyToV7BjjHLPj4sHKNJeV3UvQDHEimUF+IIDBu8oJDqz2XhOdT+yHBTw8imoa4 +WSr2Rz0ZiC3oheGe7IUIarFsNMkd7EgrO3jtZsSOeWmD3n+M +-----END CERTIFICATE----- + +# Issuer: CN=QuoVadis Root CA 3 G3 O=QuoVadis Limited +# Subject: CN=QuoVadis Root CA 3 G3 O=QuoVadis Limited +# Label: "QuoVadis Root CA 3 G3" +# Serial: 268090761170461462463995952157327242137089239581 +# MD5 Fingerprint: df:7d:b9:ad:54:6f:68:a1:df:89:57:03:97:43:b0:d7 +# SHA1 Fingerprint: 48:12:bd:92:3c:a8:c4:39:06:e7:30:6d:27:96:e6:a4:cf:22:2e:7d +# SHA256 Fingerprint: 88:ef:81:de:20:2e:b0:18:45:2e:43:f8:64:72:5c:ea:5f:bd:1f:c2:d9:d2:05:73:07:09:c5:d8:b8:69:0f:46 +-----BEGIN CERTIFICATE----- +MIIFYDCCA0igAwIBAgIULvWbAiin23r/1aOp7r0DoM8Sah0wDQYJKoZIhvcNAQEL +BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc +BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMyBHMzAeFw0xMjAxMTIyMDI2MzJaFw00 +MjAxMTIyMDI2MzJaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM +aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDMgRzMwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQCzyw4QZ47qFJenMioKVjZ/aEzHs286IxSR +/xl/pcqs7rN2nXrpixurazHb+gtTTK/FpRp5PIpM/6zfJd5O2YIyC0TeytuMrKNu +FoM7pmRLMon7FhY4futD4tN0SsJiCnMK3UmzV9KwCoWdcTzeo8vAMvMBOSBDGzXR +U7Ox7sWTaYI+FrUoRqHe6okJ7UO4BUaKhvVZR74bbwEhELn9qdIoyhA5CcoTNs+c +ra1AdHkrAj80//ogaX3T7mH1urPnMNA3I4ZyYUUpSFlob3emLoG+B01vr87ERROR +FHAGjx+f+IdpsQ7vw4kZ6+ocYfx6bIrc1gMLnia6Et3UVDmrJqMz6nWB2i3ND0/k +A9HvFZcba5DFApCTZgIhsUfei5pKgLlVj7WiL8DWM2fafsSntARE60f75li59wzw +eyuxwHApw0BiLTtIadwjPEjrewl5qW3aqDCYz4ByA4imW0aucnl8CAMhZa634Ryl +sSqiMd5mBPfAdOhx3v89WcyWJhKLhZVXGqtrdQtEPREoPHtht+KPZ0/l7DxMYIBp +VzgeAVuNVejH38DMdyM0SXV89pgR6y3e7UEuFAUCf+D+IOs15xGsIs5XPd7JMG0Q +A4XN8f+MFrXBsj6IbGB/kE+V9/YtrQE5BwT6dYB9v0lQ7e/JxHwc64B+27bQ3RP+ +ydOc17KXqQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +BjAdBgNVHQ4EFgQUxhfQvKjqAkPyGwaZXSuQILnXnOQwDQYJKoZIhvcNAQELBQAD +ggIBADRh2Va1EodVTd2jNTFGu6QHcrxfYWLopfsLN7E8trP6KZ1/AvWkyaiTt3px +KGmPc+FSkNrVvjrlt3ZqVoAh313m6Tqe5T72omnHKgqwGEfcIHB9UqM+WXzBusnI +FUBhynLWcKzSt/Ac5IYp8M7vaGPQtSCKFWGafoaYtMnCdvvMujAWzKNhxnQT5Wvv +oxXqA/4Ti2Tk08HS6IT7SdEQTXlm66r99I0xHnAUrdzeZxNMgRVhvLfZkXdxGYFg +u/BYpbWcC/ePIlUnwEsBbTuZDdQdm2NnL9DuDcpmvJRPpq3t/O5jrFc/ZSXPsoaP +0Aj/uHYUbt7lJ+yreLVTubY/6CD50qi+YUbKh4yE8/nxoGibIh6BJpsQBJFxwAYf +3KDTuVan45gtf4Od34wrnDKOMpTwATwiKp9Dwi7DmDkHOHv8XgBCH/MyJnmDhPbl +8MFREsALHgQjDFSlTC9JxUrRtm5gDWv8a4uFJGS3iQ6rJUdbPM9+Sb3H6QrG2vd+ +DhcI00iX0HGS8A85PjRqHH3Y8iKuu2n0M7SmSFXRDw4m6Oy2Cy2nhTXN/VnIn9HN +PlopNLk9hM6xZdRZkZFWdSHBd575euFgndOtBBj0fOtek49TSiIp+EgrPk2GrFt/ +ywaZWWDYWGWVjUTR939+J399roD1B0y2PpxxVJkES/1Y+Zj0 +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Assured ID Root G2 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Assured ID Root G2 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Assured ID Root G2" +# Serial: 15385348160840213938643033620894905419 +# MD5 Fingerprint: 92:38:b9:f8:63:24:82:65:2c:57:33:e6:fe:81:8f:9d +# SHA1 Fingerprint: a1:4b:48:d9:43:ee:0a:0e:40:90:4f:3c:e0:a4:c0:91:93:51:5d:3f +# SHA256 Fingerprint: 7d:05:eb:b6:82:33:9f:8c:94:51:ee:09:4e:eb:fe:fa:79:53:a1:14:ed:b2:f4:49:49:45:2f:ab:7d:2f:c1:85 +-----BEGIN CERTIFICATE----- +MIIDljCCAn6gAwIBAgIQC5McOtY5Z+pnI7/Dr5r0SzANBgkqhkiG9w0BAQsFADBl +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv +b3QgRzIwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQG +EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl +cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDZ5ygvUj82ckmIkzTz+GoeMVSA +n61UQbVH35ao1K+ALbkKz3X9iaV9JPrjIgwrvJUXCzO/GU1BBpAAvQxNEP4Htecc +biJVMWWXvdMX0h5i89vqbFCMP4QMls+3ywPgym2hFEwbid3tALBSfK+RbLE4E9Hp +EgjAALAcKxHad3A2m67OeYfcgnDmCXRwVWmvo2ifv922ebPynXApVfSr/5Vh88lA +bx3RvpO704gqu52/clpWcTs/1PPRCv4o76Pu2ZmvA9OPYLfykqGxvYmJHzDNw6Yu +YjOuFgJ3RFrngQo8p0Quebg/BLxcoIfhG69Rjs3sLPr4/m3wOnyqi+RnlTGNAgMB +AAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQW +BBTOw0q5mVXyuNtgv6l+vVa1lzan1jANBgkqhkiG9w0BAQsFAAOCAQEAyqVVjOPI +QW5pJ6d1Ee88hjZv0p3GeDgdaZaikmkuOGybfQTUiaWxMTeKySHMq2zNixya1r9I +0jJmwYrA8y8678Dj1JGG0VDjA9tzd29KOVPt3ibHtX2vK0LRdWLjSisCx1BL4Gni +lmwORGYQRI+tBev4eaymG+g3NJ1TyWGqolKvSnAWhsI6yLETcDbYz+70CjTVW0z9 +B5yiutkBclzzTcHdDrEcDcRjvq30FPuJ7KJBDkzMyFdA0G4Dqs0MjomZmWzwPDCv +ON9vvKO+KSAnq3T/EyJ43pdSVR6DtVQgA+6uwE9W3jfMw3+qBCe703e4YtsXfJwo +IhNzbM8m9Yop5w== +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Assured ID Root G3 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Assured ID Root G3 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Assured ID Root G3" +# Serial: 15459312981008553731928384953135426796 +# MD5 Fingerprint: 7c:7f:65:31:0c:81:df:8d:ba:3e:99:e2:5c:ad:6e:fb +# SHA1 Fingerprint: f5:17:a2:4f:9a:48:c6:c9:f8:a2:00:26:9f:dc:0f:48:2c:ab:30:89 +# SHA256 Fingerprint: 7e:37:cb:8b:4c:47:09:0c:ab:36:55:1b:a6:f4:5d:b8:40:68:0f:ba:16:6a:95:2d:b1:00:71:7f:43:05:3f:c2 +-----BEGIN CERTIFICATE----- +MIICRjCCAc2gAwIBAgIQC6Fa+h3foLVJRK/NJKBs7DAKBggqhkjOPQQDAzBlMQsw +CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu +ZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3Qg +RzMwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQGEwJV +UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu +Y29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzMwdjAQBgcq +hkjOPQIBBgUrgQQAIgNiAAQZ57ysRGXtzbg/WPuNsVepRC0FFfLvC/8QdJ+1YlJf +Zn4f5dwbRXkLzMZTCp2NXQLZqVneAlr2lSoOjThKiknGvMYDOAdfVdp+CW7if17Q +RSAPWXYQ1qAk8C3eNvJsKTmjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ +BAQDAgGGMB0GA1UdDgQWBBTL0L2p4ZgFUaFNN6KDec6NHSrkhDAKBggqhkjOPQQD +AwNnADBkAjAlpIFFAmsSS3V0T8gj43DydXLefInwz5FyYZ5eEJJZVrmDxxDnOOlY +JjZ91eQ0hjkCMHw2U/Aw5WJjOpnitqM7mzT6HtoQknFekROn3aRukswy1vUhZscv +6pZjamVFkpUBtA== +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Global Root G2 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Global Root G2 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Global Root G2" +# Serial: 4293743540046975378534879503202253541 +# MD5 Fingerprint: e4:a6:8a:c8:54:ac:52:42:46:0a:fd:72:48:1b:2a:44 +# SHA1 Fingerprint: df:3c:24:f9:bf:d6:66:76:1b:26:80:73:fe:06:d1:cc:8d:4f:82:a4 +# SHA256 Fingerprint: cb:3c:cb:b7:60:31:e5:e0:13:8f:8d:d3:9a:23:f9:de:47:ff:c3:5e:43:c1:14:4c:ea:27:d4:6a:5a:b1:cb:5f +-----BEGIN CERTIFICATE----- +MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH +MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT +MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j +b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI +2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx +1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ +q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz +tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ +vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP +BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV +5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY +1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4 +NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG +Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91 +8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe +pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl +MrY= +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Global Root G3 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Global Root G3 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Global Root G3" +# Serial: 7089244469030293291760083333884364146 +# MD5 Fingerprint: f5:5d:a4:50:a5:fb:28:7e:1e:0f:0d:cc:96:57:56:ca +# SHA1 Fingerprint: 7e:04:de:89:6a:3e:66:6d:00:e6:87:d3:3f:fa:d9:3b:e8:3d:34:9e +# SHA256 Fingerprint: 31:ad:66:48:f8:10:41:38:c7:38:f3:9e:a4:32:01:33:39:3e:3a:18:cc:02:29:6e:f9:7c:2a:c9:ef:67:31:d0 +-----BEGIN CERTIFICATE----- +MIICPzCCAcWgAwIBAgIQBVVWvPJepDU1w6QP1atFcjAKBggqhkjOPQQDAzBhMQsw +CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu +ZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBHMzAe +Fw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVTMRUw +EwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20x +IDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEczMHYwEAYHKoZIzj0CAQYF +K4EEACIDYgAE3afZu4q4C/sLfyHS8L6+c/MzXRq8NOrexpu80JX28MzQC7phW1FG +fp4tn+6OYwwX7Adw9c+ELkCDnOg/QW07rdOkFFk2eJ0DQ+4QE2xy3q6Ip6FrtUPO +Z9wj/wMco+I+o0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAd +BgNVHQ4EFgQUs9tIpPmhxdiuNkHMEWNpYim8S8YwCgYIKoZIzj0EAwMDaAAwZQIx +AK288mw/EkrRLTnDCgmXc/SINoyIJ7vmiI1Qhadj+Z4y3maTD/HMsQmP3Wyr+mt/ +oAIwOWZbwmSNuJ5Q3KjVSaLtx9zRSX8XAbjIho9OjIgrqJqpisXRAL34VOKa5Vt8 +sycX +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert Trusted Root G4 O=DigiCert Inc OU=www.digicert.com +# Subject: CN=DigiCert Trusted Root G4 O=DigiCert Inc OU=www.digicert.com +# Label: "DigiCert Trusted Root G4" +# Serial: 7451500558977370777930084869016614236 +# MD5 Fingerprint: 78:f2:fc:aa:60:1f:2f:b4:eb:c9:37:ba:53:2e:75:49 +# SHA1 Fingerprint: dd:fb:16:cd:49:31:c9:73:a2:03:7d:3f:c8:3a:4d:7d:77:5d:05:e4 +# SHA256 Fingerprint: 55:2f:7b:dc:f1:a7:af:9e:6c:e6:72:01:7f:4f:12:ab:f7:72:40:c7:8e:76:1a:c2:03:d1:d9:d2:0a:c8:99:88 +-----BEGIN CERTIFICATE----- +MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBi +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg +RzQwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBiMQswCQYDVQQGEwJV +UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu +Y29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQC/5pBzaN675F1KPDAiMGkz7MKnJS7JIT3y +ithZwuEppz1Yq3aaza57G4QNxDAf8xukOBbrVsaXbR2rsnnyyhHS5F/WBTxSD1If +xp4VpX6+n6lXFllVcq9ok3DCsrp1mWpzMpTREEQQLt+C8weE5nQ7bXHiLQwb7iDV +ySAdYyktzuxeTsiT+CFhmzTrBcZe7FsavOvJz82sNEBfsXpm7nfISKhmV1efVFiO +DCu3T6cw2Vbuyntd463JT17lNecxy9qTXtyOj4DatpGYQJB5w3jHtrHEtWoYOAMQ +jdjUN6QuBX2I9YI+EJFwq1WCQTLX2wRzKm6RAXwhTNS8rhsDdV14Ztk6MUSaM0C/ +CNdaSaTC5qmgZ92kJ7yhTzm1EVgX9yRcRo9k98FpiHaYdj1ZXUJ2h4mXaXpI8OCi +EhtmmnTK3kse5w5jrubU75KSOp493ADkRSWJtppEGSt+wJS00mFt6zPZxd9LBADM +fRyVw4/3IbKyEbe7f/LVjHAsQWCqsWMYRJUadmJ+9oCw++hkpjPRiQfhvbfmQ6QY +uKZ3AeEPlAwhHbJUKSWJbOUOUlFHdL4mrLZBdd56rF+NP8m800ERElvlEFDrMcXK +chYiCd98THU/Y+whX8QgUWtvsauGi0/C1kVfnSD8oR7FwI+isX4KJpn15GkvmB0t +9dmpsh3lGwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB +hjAdBgNVHQ4EFgQU7NfjgtJxXWRM3y5nP+e6mK4cD08wDQYJKoZIhvcNAQEMBQAD +ggIBALth2X2pbL4XxJEbw6GiAI3jZGgPVs93rnD5/ZpKmbnJeFwMDF/k5hQpVgs2 +SV1EY+CtnJYYZhsjDT156W1r1lT40jzBQ0CuHVD1UvyQO7uYmWlrx8GnqGikJ9yd ++SeuMIW59mdNOj6PWTkiU0TryF0Dyu1Qen1iIQqAyHNm0aAFYF/opbSnr6j3bTWc +fFqK1qI4mfN4i/RN0iAL3gTujJtHgXINwBQy7zBZLq7gcfJW5GqXb5JQbZaNaHqa +sjYUegbyJLkJEVDXCLG4iXqEI2FCKeWjzaIgQdfRnGTZ6iahixTXTBmyUEFxPT9N +cCOGDErcgdLMMpSEDQgJlxxPwO5rIHQw0uA5NBCFIRUBCOhVMt5xSdkoF1BN5r5N +0XWs0Mr7QbhDparTwwVETyw2m+L64kW4I1NsBm9nVX9GtUw/bihaeSbSpKhil9Ie +4u1Ki7wb/UdKDd9nZn6yW0HQO+T0O/QEY+nvwlQAUaCKKsnOeMzV6ocEGLPOr0mI +r/OSmbaz5mEP0oUA51Aa5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1 +/YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tKG48BtieVU+i2iW1bvGjUI+iLUaJW+fCm +gKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP82Z+ +-----END CERTIFICATE----- + +# Issuer: CN=COMODO RSA Certification Authority O=COMODO CA Limited +# Subject: CN=COMODO RSA Certification Authority O=COMODO CA Limited +# Label: "COMODO RSA Certification Authority" +# Serial: 101909084537582093308941363524873193117 +# MD5 Fingerprint: 1b:31:b0:71:40:36:cc:14:36:91:ad:c4:3e:fd:ec:18 +# SHA1 Fingerprint: af:e5:d2:44:a8:d1:19:42:30:ff:47:9f:e2:f8:97:bb:cd:7a:8c:b4 +# SHA256 Fingerprint: 52:f0:e1:c4:e5:8e:c6:29:29:1b:60:31:7f:07:46:71:b8:5d:7e:a8:0d:5b:07:27:34:63:53:4b:32:b4:02:34 +-----BEGIN CERTIFICATE----- +MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB +hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV +BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5 +MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT +EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR +Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh +dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR +6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X +pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC +9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV +/erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf +Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z ++pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w +qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah +SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC +u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf +Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq +crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E +FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl +wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM +4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV +2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna +FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ +CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK +boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke +jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL +S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb +QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl +0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB +NVOFBkpdn627G190 +-----END CERTIFICATE----- + +# Issuer: CN=USERTrust RSA Certification Authority O=The USERTRUST Network +# Subject: CN=USERTrust RSA Certification Authority O=The USERTRUST Network +# Label: "USERTrust RSA Certification Authority" +# Serial: 2645093764781058787591871645665788717 +# MD5 Fingerprint: 1b:fe:69:d1:91:b7:19:33:a3:72:a8:0f:e1:55:e5:b5 +# SHA1 Fingerprint: 2b:8f:1b:57:33:0d:bb:a2:d0:7a:6c:51:f7:0e:e9:0d:da:b9:ad:8e +# SHA256 Fingerprint: e7:93:c9:b0:2f:d8:aa:13:e2:1c:31:22:8a:cc:b0:81:19:64:3b:74:9c:89:89:64:b1:74:6d:46:c3:d4:cb:d2 +-----BEGIN CERTIFICATE----- +MIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB +iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl +cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV +BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw +MjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU +aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy +dGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B +3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY +tJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/ +Fp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2 +VN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT +79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6 +c0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT +Yo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l +c6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee +UB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE +Hg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd +BgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G +A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF +Up/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO +VWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3 +ATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs +8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR +iQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze +Sf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ +XHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/ +qS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB +VXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB +L6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG +jjxDah2nGN59PRbxYvnKkKj9 +-----END CERTIFICATE----- + +# Issuer: CN=USERTrust ECC Certification Authority O=The USERTRUST Network +# Subject: CN=USERTrust ECC Certification Authority O=The USERTRUST Network +# Label: "USERTrust ECC Certification Authority" +# Serial: 123013823720199481456569720443997572134 +# MD5 Fingerprint: fa:68:bc:d9:b5:7f:ad:fd:c9:1d:06:83:28:cc:24:c1 +# SHA1 Fingerprint: d1:cb:ca:5d:b2:d5:2a:7f:69:3b:67:4d:e5:f0:5a:1d:0c:95:7d:f0 +# SHA256 Fingerprint: 4f:f4:60:d5:4b:9c:86:da:bf:bc:fc:57:12:e0:40:0d:2b:ed:3f:bc:4d:4f:bd:aa:86:e0:6a:dc:d2:a9:ad:7a +-----BEGIN CERTIFICATE----- +MIICjzCCAhWgAwIBAgIQXIuZxVqUxdJxVt7NiYDMJjAKBggqhkjOPQQDAzCBiDEL +MAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNl +eSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMT +JVVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMjAx +MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT +Ck5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVUaGUg +VVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBFQ0MgQ2VydGlm +aWNhdGlvbiBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQarFRaqflo +I+d61SRvU8Za2EurxtW20eZzca7dnNYMYf3boIkDuAUU7FfO7l0/4iGzzvfUinng +o4N+LZfQYcTxmdwlkWOrfzCjtHDix6EznPO/LlxTsV+zfTJ/ijTjeXmjQjBAMB0G +A1UdDgQWBBQ64QmG1M8ZwpZ2dEl23OA1xmNjmjAOBgNVHQ8BAf8EBAMCAQYwDwYD +VR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjA2Z6EWCNzklwBBHU6+4WMB +zzuqQhFkoJ2UOQIReVx7Hfpkue4WQrO/isIJxOzksU0CMQDpKmFHjFJKS04YcPbW +RNZu9YO6bVi9JNlWSOrvxKJGgYhqOkbRqZtNyWHa0V1Xahg= +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R5 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R5 +# Label: "GlobalSign ECC Root CA - R5" +# Serial: 32785792099990507226680698011560947931244 +# MD5 Fingerprint: 9f:ad:3b:1c:02:1e:8a:ba:17:74:38:81:0c:a2:bc:08 +# SHA1 Fingerprint: 1f:24:c6:30:cd:a4:18:ef:20:69:ff:ad:4f:dd:5f:46:3a:1b:69:aa +# SHA256 Fingerprint: 17:9f:bc:14:8a:3d:d0:0f:d2:4e:a1:34:58:cc:43:bf:a7:f5:9c:81:82:d7:83:a5:13:f6:eb:ec:10:0c:89:24 +-----BEGIN CERTIFICATE----- +MIICHjCCAaSgAwIBAgIRYFlJ4CYuu1X5CneKcflK2GwwCgYIKoZIzj0EAwMwUDEk +MCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI1MRMwEQYDVQQKEwpH +bG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoX +DTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBD +QSAtIFI1MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu +MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAER0UOlvt9Xb/pOdEh+J8LttV7HpI6SFkc +8GIxLcB6KP4ap1yztsyX50XUWPrRd21DosCHZTQKH3rd6zwzocWdTaRvQZU4f8ke +hOvRnkmSh5SHDDqFSmafnVmTTZdhBoZKo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYD +VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUPeYpSJvqB8ohREom3m7e0oPQn1kwCgYI +KoZIzj0EAwMDaAAwZQIxAOVpEslu28YxuglB4Zf4+/2a4n0Sye18ZNPLBSWLVtmg +515dTguDnFt2KaAJJiFqYgIwcdK1j1zqO+F4CYWodZI7yFz9SO8NdCKoCOJuxUnO +xwy8p2Fp8fc74SrL+SvzZpA3 +-----END CERTIFICATE----- + +# Issuer: CN=IdenTrust Commercial Root CA 1 O=IdenTrust +# Subject: CN=IdenTrust Commercial Root CA 1 O=IdenTrust +# Label: "IdenTrust Commercial Root CA 1" +# Serial: 13298821034946342390520003877796839426 +# MD5 Fingerprint: b3:3e:77:73:75:ee:a0:d3:e3:7e:49:63:49:59:bb:c7 +# SHA1 Fingerprint: df:71:7e:aa:4a:d9:4e:c9:55:84:99:60:2d:48:de:5f:bc:f0:3a:25 +# SHA256 Fingerprint: 5d:56:49:9b:e4:d2:e0:8b:cf:ca:d0:8a:3e:38:72:3d:50:50:3b:de:70:69:48:e4:2f:55:60:30:19:e5:28:ae +-----BEGIN CERTIFICATE----- +MIIFYDCCA0igAwIBAgIQCgFCgAAAAUUjyES1AAAAAjANBgkqhkiG9w0BAQsFADBK +MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScwJQYDVQQDEx5JZGVu +VHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwHhcNMTQwMTE2MTgxMjIzWhcNMzQw +MTE2MTgxMjIzWjBKMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScw +JQYDVQQDEx5JZGVuVHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQCnUBneP5k91DNG8W9RYYKyqU+PZ4ldhNlT +3Qwo2dfw/66VQ3KZ+bVdfIrBQuExUHTRgQ18zZshq0PirK1ehm7zCYofWjK9ouuU ++ehcCuz/mNKvcbO0U59Oh++SvL3sTzIwiEsXXlfEU8L2ApeN2WIrvyQfYo3fw7gp +S0l4PJNgiCL8mdo2yMKi1CxUAGc1bnO/AljwpN3lsKImesrgNqUZFvX9t++uP0D1 +bVoE/c40yiTcdCMbXTMTEl3EASX2MN0CXZ/g1Ue9tOsbobtJSdifWwLziuQkkORi +T0/Br4sOdBeo0XKIanoBScy0RnnGF7HamB4HWfp1IYVl3ZBWzvurpWCdxJ35UrCL +vYf5jysjCiN2O/cz4ckA82n5S6LgTrx+kzmEB/dEcH7+B1rlsazRGMzyNeVJSQjK +Vsk9+w8YfYs7wRPCTY/JTw436R+hDmrfYi7LNQZReSzIJTj0+kuniVyc0uMNOYZK +dHzVWYfCP04MXFL0PfdSgvHqo6z9STQaKPNBiDoT7uje/5kdX7rL6B7yuVBgwDHT +c+XvvqDtMwt0viAgxGds8AgDelWAf0ZOlqf0Hj7h9tgJ4TNkK2PXMl6f+cB7D3hv +l7yTmvmcEpB4eoCHFddydJxVdHixuuFucAS6T6C6aMN7/zHwcz09lCqxC0EOoP5N +iGVreTO01wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB +/zAdBgNVHQ4EFgQU7UQZwNPwBovupHu+QucmVMiONnYwDQYJKoZIhvcNAQELBQAD +ggIBAA2ukDL2pkt8RHYZYR4nKM1eVO8lvOMIkPkp165oCOGUAFjvLi5+U1KMtlwH +6oi6mYtQlNeCgN9hCQCTrQ0U5s7B8jeUeLBfnLOic7iPBZM4zY0+sLj7wM+x8uwt +LRvM7Kqas6pgghstO8OEPVeKlh6cdbjTMM1gCIOQ045U8U1mwF10A0Cj7oV+wh93 +nAbowacYXVKV7cndJZ5t+qntozo00Fl72u1Q8zW/7esUTTHHYPTa8Yec4kjixsU3 ++wYQ+nVZZjFHKdp2mhzpgq7vmrlR94gjmmmVYjzlVYA211QC//G5Xc7UI2/YRYRK +W2XviQzdFKcgyxilJbQN+QHwotL0AMh0jqEqSI5l2xPE4iUXfeu+h1sXIFRRk0pT +AwvsXcoz7WL9RccvW9xYoIA55vrX/hMUpu09lEpCdNTDd1lzzY9GvlU47/rokTLq +l1gEIt44w8y8bckzOmoKaT+gyOpyj4xjhiO9bTyWnpXgSUyqorkqG5w2gXjtw+hG +4iZZRHUe2XWJUc0QhJ1hYMtd+ZciTY6Y5uN/9lu7rs3KSoFrXgvzUeF0K+l+J6fZ +mUlO+KWA2yUPHGNiiskzZ2s8EIPGrd6ozRaOjfAHN3Gf8qv8QfXBi+wAN10J5U6A +7/qxXDgGpRtK4dw4LTzcqx+QGtVKnO7RcGzM7vRX+Bi6hG6H +-----END CERTIFICATE----- + +# Issuer: CN=IdenTrust Public Sector Root CA 1 O=IdenTrust +# Subject: CN=IdenTrust Public Sector Root CA 1 O=IdenTrust +# Label: "IdenTrust Public Sector Root CA 1" +# Serial: 13298821034946342390521976156843933698 +# MD5 Fingerprint: 37:06:a5:b0:fc:89:9d:ba:f4:6b:8c:1a:64:cd:d5:ba +# SHA1 Fingerprint: ba:29:41:60:77:98:3f:f4:f3:ef:f2:31:05:3b:2e:ea:6d:4d:45:fd +# SHA256 Fingerprint: 30:d0:89:5a:9a:44:8a:26:20:91:63:55:22:d1:f5:20:10:b5:86:7a:ca:e1:2c:78:ef:95:8f:d4:f4:38:9f:2f +-----BEGIN CERTIFICATE----- +MIIFZjCCA06gAwIBAgIQCgFCgAAAAUUjz0Z8AAAAAjANBgkqhkiG9w0BAQsFADBN +MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MSowKAYDVQQDEyFJZGVu +VHJ1c3QgUHVibGljIFNlY3RvciBSb290IENBIDEwHhcNMTQwMTE2MTc1MzMyWhcN +MzQwMTE2MTc1MzMyWjBNMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0 +MSowKAYDVQQDEyFJZGVuVHJ1c3QgUHVibGljIFNlY3RvciBSb290IENBIDEwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC2IpT8pEiv6EdrCvsnduTyP4o7 +ekosMSqMjbCpwzFrqHd2hCa2rIFCDQjrVVi7evi8ZX3yoG2LqEfpYnYeEe4IFNGy +RBb06tD6Hi9e28tzQa68ALBKK0CyrOE7S8ItneShm+waOh7wCLPQ5CQ1B5+ctMlS +bdsHyo+1W/CD80/HLaXIrcuVIKQxKFdYWuSNG5qrng0M8gozOSI5Cpcu81N3uURF +/YTLNiCBWS2ab21ISGHKTN9T0a9SvESfqy9rg3LvdYDaBjMbXcjaY8ZNzaxmMc3R +3j6HEDbhuaR672BQssvKplbgN6+rNBM5Jeg5ZuSYeqoSmJxZZoY+rfGwyj4GD3vw +EUs3oERte8uojHH01bWRNszwFcYr3lEXsZdMUD2xlVl8BX0tIdUAvwFnol57plzy +9yLxkA2T26pEUWbMfXYD62qoKjgZl3YNa4ph+bz27nb9cCvdKTz4Ch5bQhyLVi9V +GxyhLrXHFub4qjySjmm2AcG1hp2JDws4lFTo6tyePSW8Uybt1as5qsVATFSrsrTZ +2fjXctscvG29ZV/viDUqZi/u9rNl8DONfJhBaUYPQxxp+pu10GFqzcpL2UyQRqsV +WaFHVCkugyhfHMKiq3IXAAaOReyL4jM9f9oZRORicsPfIsbyVtTdX5Vy7W1f90gD +W/3FKqD2cyOEEBsB5wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/ +BAUwAwEB/zAdBgNVHQ4EFgQU43HgntinQtnbcZFrlJPrw6PRFKMwDQYJKoZIhvcN +AQELBQADggIBAEf63QqwEZE4rU1d9+UOl1QZgkiHVIyqZJnYWv6IAcVYpZmxI1Qj +t2odIFflAWJBF9MJ23XLblSQdf4an4EKwt3X9wnQW3IV5B4Jaj0z8yGa5hV+rVHV +DRDtfULAj+7AmgjVQdZcDiFpboBhDhXAuM/FSRJSzL46zNQuOAXeNf0fb7iAaJg9 +TaDKQGXSc3z1i9kKlT/YPyNtGtEqJBnZhbMX73huqVjRI9PHE+1yJX9dsXNw0H8G +lwmEKYBhHfpe/3OsoOOJuBxxFcbeMX8S3OFtm6/n6J91eEyrRjuazr8FGF1NFTwW +mhlQBJqymm9li1JfPFgEKCXAZmExfrngdbkaqIHWchezxQMxNRF4eKLg6TCMf4Df +WN88uieW4oA0beOY02QnrEh+KHdcxiVhJfiFDGX6xDIvpZgF5PgLZxYWxoK4Mhn5 ++bl53B/N66+rDt0b20XkeucC4pVd/GnwU2lhlXV5C15V5jgclKlZM57IcXR5f1GJ +tshquDDIajjDbp7hNxbqBWJMWxJH7ae0s1hWx0nzfxJoCTFx8G34Tkf71oXuxVhA +GaQdp/lLQzfcaFpPz+vCZHTetBXZ9FRUGi8c15dxVJCO2SCdUyt/q4/i6jC8UDfv +8Ue1fXwsBOxonbRJRBD0ckscZOf85muQ3Wl9af0AVqW3rLatt8o+Ae+c +-----END CERTIFICATE----- + +# Issuer: CN=CFCA EV ROOT O=China Financial Certification Authority +# Subject: CN=CFCA EV ROOT O=China Financial Certification Authority +# Label: "CFCA EV ROOT" +# Serial: 407555286 +# MD5 Fingerprint: 74:e1:b6:ed:26:7a:7a:44:30:33:94:ab:7b:27:81:30 +# SHA1 Fingerprint: e2:b8:29:4b:55:84:ab:6b:58:c2:90:46:6c:ac:3f:b8:39:8f:84:83 +# SHA256 Fingerprint: 5c:c3:d7:8e:4e:1d:5e:45:54:7a:04:e6:87:3e:64:f9:0c:f9:53:6d:1c:cc:2e:f8:00:f3:55:c4:c5:fd:70:fd +-----BEGIN CERTIFICATE----- +MIIFjTCCA3WgAwIBAgIEGErM1jANBgkqhkiG9w0BAQsFADBWMQswCQYDVQQGEwJD +TjEwMC4GA1UECgwnQ2hpbmEgRmluYW5jaWFsIENlcnRpZmljYXRpb24gQXV0aG9y +aXR5MRUwEwYDVQQDDAxDRkNBIEVWIFJPT1QwHhcNMTIwODA4MDMwNzAxWhcNMjkx +MjMxMDMwNzAxWjBWMQswCQYDVQQGEwJDTjEwMC4GA1UECgwnQ2hpbmEgRmluYW5j +aWFsIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MRUwEwYDVQQDDAxDRkNBIEVWIFJP +T1QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDXXWvNED8fBVnVBU03 +sQ7smCuOFR36k0sXgiFxEFLXUWRwFsJVaU2OFW2fvwwbwuCjZ9YMrM8irq93VCpL +TIpTUnrD7i7es3ElweldPe6hL6P3KjzJIx1qqx2hp/Hz7KDVRM8Vz3IvHWOX6Jn5 +/ZOkVIBMUtRSqy5J35DNuF++P96hyk0g1CXohClTt7GIH//62pCfCqktQT+x8Rgp +7hZZLDRJGqgG16iI0gNyejLi6mhNbiyWZXvKWfry4t3uMCz7zEasxGPrb382KzRz +EpR/38wmnvFyXVBlWY9ps4deMm/DGIq1lY+wejfeWkU7xzbh72fROdOXW3NiGUgt +hxwG+3SYIElz8AXSG7Ggo7cbcNOIabla1jj0Ytwli3i/+Oh+uFzJlU9fpy25IGvP +a931DfSCt/SyZi4QKPaXWnuWFo8BGS1sbn85WAZkgwGDg8NNkt0yxoekN+kWzqot +aK8KgWU6cMGbrU1tVMoqLUuFG7OA5nBFDWteNfB/O7ic5ARwiRIlk9oKmSJgamNg +TnYGmE69g60dWIolhdLHZR4tjsbftsbhf4oEIRUpdPA+nJCdDC7xij5aqgwJHsfV +PKPtl8MeNPo4+QgO48BdK4PRVmrJtqhUUy54Mmc9gn900PvhtgVguXDbjgv5E1hv +cWAQUhC5wUEJ73IfZzF4/5YFjQIDAQABo2MwYTAfBgNVHSMEGDAWgBTj/i39KNAL +tbq2osS/BqoFjJP7LzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAd +BgNVHQ4EFgQU4/4t/SjQC7W6tqLEvwaqBYyT+y8wDQYJKoZIhvcNAQELBQADggIB +ACXGumvrh8vegjmWPfBEp2uEcwPenStPuiB/vHiyz5ewG5zz13ku9Ui20vsXiObT +ej/tUxPQ4i9qecsAIyjmHjdXNYmEwnZPNDatZ8POQQaIxffu2Bq41gt/UP+TqhdL +jOztUmCypAbqTuv0axn96/Ua4CUqmtzHQTb3yHQFhDmVOdYLO6Qn+gjYXB74BGBS +ESgoA//vU2YApUo0FmZ8/Qmkrp5nGm9BC2sGE5uPhnEFtC+NiWYzKXZUmhH4J/qy +P5Hgzg0b8zAarb8iXRvTvyUFTeGSGn+ZnzxEk8rUQElsgIfXBDrDMlI1Dlb4pd19 +xIsNER9Tyx6yF7Zod1rg1MvIB671Oi6ON7fQAUtDKXeMOZePglr4UeWJoBjnaH9d +Ci77o0cOPaYjesYBx4/IXr9tgFa+iiS6M+qf4TIRnvHST4D2G0CvOJ4RUHlzEhLN +5mydLIhyPDCBBpEi6lmt2hkuIsKNuYyH4Ga8cyNfIWRjgEj1oDwYPZTISEEdQLpe +/v5WOaHIz16eGWRGENoXkbcFgKyLmZJ956LYBws2J+dIeWCKw9cTXPhyQN9Ky8+Z +AAoACxGV2lZFA4gKn2fQ1XmxqI1AbQ3CekD6819kR5LLU7m7Wc5P/dAVUwHY3+vZ +5nbv0CO7O6l5s9UCKc2Jo5YPSjXnTkLAdc0Hz+Ys63su +-----END CERTIFICATE----- + +# Issuer: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed +# Subject: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed +# Label: "OISTE WISeKey Global Root GB CA" +# Serial: 157768595616588414422159278966750757568 +# MD5 Fingerprint: a4:eb:b9:61:28:2e:b7:2f:98:b0:35:26:90:99:51:1d +# SHA1 Fingerprint: 0f:f9:40:76:18:d3:d7:6a:4b:98:f0:a8:35:9e:0c:fd:27:ac:cc:ed +# SHA256 Fingerprint: 6b:9c:08:e8:6e:b0:f7:67:cf:ad:65:cd:98:b6:21:49:e5:49:4a:67:f5:84:5e:7b:d1:ed:01:9f:27:b8:6b:d6 +-----BEGIN CERTIFICATE----- +MIIDtTCCAp2gAwIBAgIQdrEgUnTwhYdGs/gjGvbCwDANBgkqhkiG9w0BAQsFADBt +MQswCQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUg +Rm91bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9i +YWwgUm9vdCBHQiBDQTAeFw0xNDEyMDExNTAwMzJaFw0zOTEyMDExNTEwMzFaMG0x +CzAJBgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBG +b3VuZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2Jh +bCBSb290IEdCIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2Be3 +HEokKtaXscriHvt9OO+Y9bI5mE4nuBFde9IllIiCFSZqGzG7qFshISvYD06fWvGx +WuR51jIjK+FTzJlFXHtPrby/h0oLS5daqPZI7H17Dc0hBt+eFf1Biki3IPShehtX +1F1Q/7pn2COZH8g/497/b1t3sWtuuMlk9+HKQUYOKXHQuSP8yYFfTvdv37+ErXNk +u7dCjmn21HYdfp2nuFeKUWdy19SouJVUQHMD9ur06/4oQnc/nSMbsrY9gBQHTC5P +99UKFg29ZkM3fiNDecNAhvVMKdqOmq0NpQSHiB6F4+lT1ZvIiwNjeOvgGUpuuy9r +M2RYk61pv48b74JIxwIDAQABo1EwTzALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUw +AwEB/zAdBgNVHQ4EFgQUNQ/INmNe4qPs+TtmFc5RUuORmj0wEAYJKwYBBAGCNxUB +BAMCAQAwDQYJKoZIhvcNAQELBQADggEBAEBM+4eymYGQfp3FsLAmzYh7KzKNbrgh +cViXfa43FK8+5/ea4n32cZiZBKpDdHij40lhPnOMTZTg+XHEthYOU3gf1qKHLwI5 +gSk8rxWYITD+KJAAjNHhy/peyP34EEY7onhCkRd0VQreUGdNZtGn//3ZwLWoo4rO +ZvUPQ82nK1d7Y0Zqqi5S2PTt4W2tKZB4SLrhI6qjiey1q5bAtEuiHZeeevJuQHHf +aPFlTc58Bd9TZaml8LGXBHAVRgOY1NK/VLSgWH1Sb9pWJmLU2NuJMW8c8CLC02Ic +Nc1MaRVUGpCY3useX8p3x8uOPUNpnJpY0CQ73xtAln41rYHHTnG6iBM= +-----END CERTIFICATE----- + +# Issuer: CN=SZAFIR ROOT CA2 O=Krajowa Izba Rozliczeniowa S.A. +# Subject: CN=SZAFIR ROOT CA2 O=Krajowa Izba Rozliczeniowa S.A. +# Label: "SZAFIR ROOT CA2" +# Serial: 357043034767186914217277344587386743377558296292 +# MD5 Fingerprint: 11:64:c1:89:b0:24:b1:8c:b1:07:7e:89:9e:51:9e:99 +# SHA1 Fingerprint: e2:52:fa:95:3f:ed:db:24:60:bd:6e:28:f3:9c:cc:cf:5e:b3:3f:de +# SHA256 Fingerprint: a1:33:9d:33:28:1a:0b:56:e5:57:d3:d3:2b:1c:e7:f9:36:7e:b0:94:bd:5f:a7:2a:7e:50:04:c8:de:d7:ca:fe +-----BEGIN CERTIFICATE----- +MIIDcjCCAlqgAwIBAgIUPopdB+xV0jLVt+O2XwHrLdzk1uQwDQYJKoZIhvcNAQEL +BQAwUTELMAkGA1UEBhMCUEwxKDAmBgNVBAoMH0tyYWpvd2EgSXpiYSBSb3psaWN6 +ZW5pb3dhIFMuQS4xGDAWBgNVBAMMD1NaQUZJUiBST09UIENBMjAeFw0xNTEwMTkw +NzQzMzBaFw0zNTEwMTkwNzQzMzBaMFExCzAJBgNVBAYTAlBMMSgwJgYDVQQKDB9L +cmFqb3dhIEl6YmEgUm96bGljemVuaW93YSBTLkEuMRgwFgYDVQQDDA9TWkFGSVIg +Uk9PVCBDQTIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3vD5QqEvN +QLXOYeeWyrSh2gwisPq1e3YAd4wLz32ohswmUeQgPYUM1ljj5/QqGJ3a0a4m7utT +3PSQ1hNKDJA8w/Ta0o4NkjrcsbH/ON7Dui1fgLkCvUqdGw+0w8LBZwPd3BucPbOw +3gAeqDRHu5rr/gsUvTaE2g0gv/pby6kWIK05YO4vdbbnl5z5Pv1+TW9NL++IDWr6 +3fE9biCloBK0TXC5ztdyO4mTp4CEHCdJckm1/zuVnsHMyAHs6A6KCpbns6aH5db5 +BSsNl0BwPLqsdVqc1U2dAgrSS5tmS0YHF2Wtn2yIANwiieDhZNRnvDF5YTy7ykHN +XGoAyDw4jlivAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD +AgEGMB0GA1UdDgQWBBQuFqlKGLXLzPVvUPMjX/hd56zwyDANBgkqhkiG9w0BAQsF +AAOCAQEAtXP4A9xZWx126aMqe5Aosk3AM0+qmrHUuOQn/6mWmc5G4G18TKI4pAZw +8PRBEew/R40/cof5O/2kbytTAOD/OblqBw7rHRz2onKQy4I9EYKL0rufKq8h5mOG +nXkZ7/e7DDWQw4rtTw/1zBLZpD67oPwglV9PJi8RI4NOdQcPv5vRtB3pEAT+ymCP +oky4rc/hkA/NrgrHXXu3UNLUYfrVFdvXn4dRVOul4+vJhaAlIDf7js4MNIThPIGy +d05DpYhfhmehPea0XGG2Ptv+tyjFogeutcrKjSoS75ftwjCkySp6+/NNIxuZMzSg +LvWpCz/UXeHPhJ/iGcJfitYgHuNztw== +-----END CERTIFICATE----- + +# Issuer: CN=Certum Trusted Network CA 2 O=Unizeto Technologies S.A. OU=Certum Certification Authority +# Subject: CN=Certum Trusted Network CA 2 O=Unizeto Technologies S.A. OU=Certum Certification Authority +# Label: "Certum Trusted Network CA 2" +# Serial: 44979900017204383099463764357512596969 +# MD5 Fingerprint: 6d:46:9e:d9:25:6d:08:23:5b:5e:74:7d:1e:27:db:f2 +# SHA1 Fingerprint: d3:dd:48:3e:2b:bf:4c:05:e8:af:10:f5:fa:76:26:cf:d3:dc:30:92 +# SHA256 Fingerprint: b6:76:f2:ed:da:e8:77:5c:d3:6c:b0:f6:3c:d1:d4:60:39:61:f4:9e:62:65:ba:01:3a:2f:03:07:b6:d0:b8:04 +-----BEGIN CERTIFICATE----- +MIIF0jCCA7qgAwIBAgIQIdbQSk8lD8kyN/yqXhKN6TANBgkqhkiG9w0BAQ0FADCB +gDELMAkGA1UEBhMCUEwxIjAgBgNVBAoTGVVuaXpldG8gVGVjaG5vbG9naWVzIFMu +QS4xJzAlBgNVBAsTHkNlcnR1bSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTEkMCIG +A1UEAxMbQ2VydHVtIFRydXN0ZWQgTmV0d29yayBDQSAyMCIYDzIwMTExMDA2MDgz +OTU2WhgPMjA0NjEwMDYwODM5NTZaMIGAMQswCQYDVQQGEwJQTDEiMCAGA1UEChMZ +VW5pemV0byBUZWNobm9sb2dpZXMgUy5BLjEnMCUGA1UECxMeQ2VydHVtIENlcnRp +ZmljYXRpb24gQXV0aG9yaXR5MSQwIgYDVQQDExtDZXJ0dW0gVHJ1c3RlZCBOZXR3 +b3JrIENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9+Xj45tWA +DGSdhhuWZGc/IjoedQF97/tcZ4zJzFxrqZHmuULlIEub2pt7uZld2ZuAS9eEQCsn +0+i6MLs+CRqnSZXvK0AkwpfHp+6bJe+oCgCXhVqqndwpyeI1B+twTUrWwbNWuKFB +OJvR+zF/j+Bf4bE/D44WSWDXBo0Y+aomEKsq09DRZ40bRr5HMNUuctHFY9rnY3lE +fktjJImGLjQ/KUxSiyqnwOKRKIm5wFv5HdnnJ63/mgKXwcZQkpsCLL2puTRZCr+E +Sv/f/rOf69me4Jgj7KZrdxYq28ytOxykh9xGc14ZYmhFV+SQgkK7QtbwYeDBoz1m +o130GO6IyY0XRSmZMnUCMe4pJshrAua1YkV/NxVaI2iJ1D7eTiew8EAMvE0Xy02i +sx7QBlrd9pPPV3WZ9fqGGmd4s7+W/jTcvedSVuWz5XV710GRBdxdaeOVDUO5/IOW +OZV7bIBaTxNyxtd9KXpEulKkKtVBRgkg/iKgtlswjbyJDNXXcPiHUv3a76xRLgez +Tv7QCdpw75j6VuZt27VXS9zlLCUVyJ4ueE742pyehizKV/Ma5ciSixqClnrDvFAS +adgOWkaLOusm+iPJtrCBvkIApPjW/jAux9JG9uWOdf3yzLnQh1vMBhBgu4M1t15n +3kfsmUjxpKEV/q2MYo45VU85FrmxY53/twIDAQABo0IwQDAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBS2oVQ5AsOgP46KvPrU+Bym0ToO/TAOBgNVHQ8BAf8EBAMC +AQYwDQYJKoZIhvcNAQENBQADggIBAHGlDs7k6b8/ONWJWsQCYftMxRQXLYtPU2sQ +F/xlhMcQSZDe28cmk4gmb3DWAl45oPePq5a1pRNcgRRtDoGCERuKTsZPpd1iHkTf +CVn0W3cLN+mLIMb4Ck4uWBzrM9DPhmDJ2vuAL55MYIR4PSFk1vtBHxgP58l1cb29 +XN40hz5BsA72udY/CROWFC/emh1auVbONTqwX3BNXuMp8SMoclm2q8KMZiYcdywm +djWLKKdpoPk79SPdhRB0yZADVpHnr7pH1BKXESLjokmUbOe3lEu6LaTaM4tMpkT/ +WjzGHWTYtTHkpjx6qFcL2+1hGsvxznN3Y6SHb0xRONbkX8eftoEq5IVIeVheO/jb +AoJnwTnbw3RLPTYe+SmTiGhbqEQZIfCn6IENLOiTNrQ3ssqwGyZ6miUfmpqAnksq +P/ujmv5zMnHCnsZy4YpoJ/HkD7TETKVhk/iXEAcqMCWpuchxuO9ozC1+9eB+D4Ko +b7a6bINDd82Kkhehnlt4Fj1F4jNy3eFmypnTycUm/Q1oBEauttmbjL4ZvrHG8hnj +XALKLNhvSgfZyTXaQHXyxKcZb55CEJh15pWLYLztxRLXis7VmFxWlgPF7ncGNf/P +5O4/E2Hu29othfDNrp2yGAlFw5Khchf8R7agCyzxxN5DaAhqXzvwdmP7zAYspsbi +DrW5viSP +-----END CERTIFICATE----- + +# Issuer: CN=Hellenic Academic and Research Institutions RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority +# Subject: CN=Hellenic Academic and Research Institutions RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority +# Label: "Hellenic Academic and Research Institutions RootCA 2015" +# Serial: 0 +# MD5 Fingerprint: ca:ff:e2:db:03:d9:cb:4b:e9:0f:ad:84:fd:7b:18:ce +# SHA1 Fingerprint: 01:0c:06:95:a6:98:19:14:ff:bf:5f:c6:b0:b6:95:ea:29:e9:12:a6 +# SHA256 Fingerprint: a0:40:92:9a:02:ce:53:b4:ac:f4:f2:ff:c6:98:1c:e4:49:6f:75:5e:6d:45:fe:0b:2a:69:2b:cd:52:52:3f:36 +-----BEGIN CERTIFICATE----- +MIIGCzCCA/OgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBpjELMAkGA1UEBhMCR1Ix +DzANBgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5k +IFJlc2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkxQDA+BgNVBAMT +N0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgUm9v +dENBIDIwMTUwHhcNMTUwNzA3MTAxMTIxWhcNNDAwNjMwMTAxMTIxWjCBpjELMAkG +A1UEBhMCR1IxDzANBgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNh +ZGVtaWMgYW5kIFJlc2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkx +QDA+BgNVBAMTN0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1 +dGlvbnMgUm9vdENBIDIwMTUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC +AQDC+Kk/G4n8PDwEXT2QNrCROnk8ZlrvbTkBSRq0t89/TSNTt5AA4xMqKKYx8ZEA +4yjsriFBzh/a/X0SWwGDD7mwX5nh8hKDgE0GPt+sr+ehiGsxr/CL0BgzuNtFajT0 +AoAkKAoCFZVedioNmToUW/bLy1O8E00BiDeUJRtCvCLYjqOWXjrZMts+6PAQZe10 +4S+nfK8nNLspfZu2zwnI5dMK/IhlZXQK3HMcXM1AsRzUtoSMTFDPaI6oWa7CJ06C +ojXdFPQf/7J31Ycvqm59JCfnxssm5uX+Zwdj2EUN3TpZZTlYepKZcj2chF6IIbjV +9Cz82XBST3i4vTwri5WY9bPRaM8gFH5MXF/ni+X1NYEZN9cRCLdmvtNKzoNXADrD +gfgXy5I2XdGj2HUb4Ysn6npIQf1FGQatJ5lOwXBH3bWfgVMS5bGMSF0xQxfjjMZ6 +Y5ZLKTBOhE5iGV48zpeQpX8B653g+IuJ3SWYPZK2fu/Z8VFRfS0myGlZYeCsargq +NhEEelC9MoS+L9xy1dcdFkfkR2YgP/SWxa+OAXqlD3pk9Q0Yh9muiNX6hME6wGko +LfINaFGq46V3xqSQDqE3izEjR8EJCOtu93ib14L8hCCZSRm2Ekax+0VVFqmjZayc +Bw/qa9wfLgZy7IaIEuQt218FL+TwA9MmM+eAws1CoRc0CwIDAQABo0IwQDAPBgNV +HRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUcRVnyMjJvXVd +ctA4GGqd83EkVAswDQYJKoZIhvcNAQELBQADggIBAHW7bVRLqhBYRjTyYtcWNl0I +XtVsyIe9tC5G8jH4fOpCtZMWVdyhDBKg2mF+D1hYc2Ryx+hFjtyp8iY/xnmMsVMI +M4GwVhO+5lFc2JsKT0ucVlMC6U/2DWDqTUJV6HwbISHTGzrMd/K4kPFox/la/vot +9L/J9UUbzjgQKjeKeaO04wlshYaT/4mWJ3iBj2fjRnRUjtkNaeJK9E10A/+yd+2V +Z5fkscWrv2oj6NSU4kQoYsRL4vDY4ilrGnB+JGGTe08DMiUNRSQrlrRGar9KC/ea +j8GsGsVn82800vpzY4zvFrCopEYq+OsS7HK07/grfoxSwIuEVPkvPuNVqNxmsdnh +X9izjFk0WaSrT2y7HxjbdavYy5LNlDhhDgcGH0tGEPEVvo2FXDtKK4F5D7Rpn0lQ +l033DlZdwJVqwjbDG2jJ9SrcR5q+ss7FJej6A7na+RZukYT1HCjI/CbM1xyQVqdf +bzoEvM14iQuODy+jqk+iGxI9FghAD/FGTNeqewjBCvVtJ94Cj8rDtSvK6evIIVM4 +pcw72Hc3MKJP2W/R8kCtQXoXxdZKNYm3QdV8hn9VTYNKpXMgwDqvkPGaJI7ZjnHK +e7iG2rKPmT4dEw0SEe7Uq/DpFXYC5ODfqiAeW2GFZECpkJcNrVPSWh2HagCXZWK0 +vm9qp/UsQu0yrbYhnr68 +-----END CERTIFICATE----- + +# Issuer: CN=Hellenic Academic and Research Institutions ECC RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority +# Subject: CN=Hellenic Academic and Research Institutions ECC RootCA 2015 O=Hellenic Academic and Research Institutions Cert. Authority +# Label: "Hellenic Academic and Research Institutions ECC RootCA 2015" +# Serial: 0 +# MD5 Fingerprint: 81:e5:b4:17:eb:c2:f5:e1:4b:0d:41:7b:49:92:fe:ef +# SHA1 Fingerprint: 9f:f1:71:8d:92:d5:9a:f3:7d:74:97:b4:bc:6f:84:68:0b:ba:b6:66 +# SHA256 Fingerprint: 44:b5:45:aa:8a:25:e6:5a:73:ca:15:dc:27:fc:36:d2:4c:1c:b9:95:3a:06:65:39:b1:15:82:dc:48:7b:48:33 +-----BEGIN CERTIFICATE----- +MIICwzCCAkqgAwIBAgIBADAKBggqhkjOPQQDAjCBqjELMAkGA1UEBhMCR1IxDzAN +BgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJl +c2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkxRDBCBgNVBAMTO0hl +bGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgRUNDIFJv +b3RDQSAyMDE1MB4XDTE1MDcwNzEwMzcxMloXDTQwMDYzMDEwMzcxMlowgaoxCzAJ +BgNVBAYTAkdSMQ8wDQYDVQQHEwZBdGhlbnMxRDBCBgNVBAoTO0hlbGxlbmljIEFj +YWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgQ2VydC4gQXV0aG9yaXR5 +MUQwQgYDVQQDEztIZWxsZW5pYyBBY2FkZW1pYyBhbmQgUmVzZWFyY2ggSW5zdGl0 +dXRpb25zIEVDQyBSb290Q0EgMjAxNTB2MBAGByqGSM49AgEGBSuBBAAiA2IABJKg +QehLgoRc4vgxEZmGZE4JJS+dQS8KrjVPdJWyUWRrjWvmP3CV8AVER6ZyOFB2lQJa +jq4onvktTpnvLEhvTCUp6NFxW98dwXU3tNf6e3pCnGoKVlp8aQuqgAkkbH7BRqNC +MEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFLQi +C4KZJAEOnLvkDv2/+5cgk5kqMAoGCCqGSM49BAMCA2cAMGQCMGfOFmI4oqxiRaep +lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof +TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR +-----END CERTIFICATE----- + +# Issuer: CN=ISRG Root X1 O=Internet Security Research Group +# Subject: CN=ISRG Root X1 O=Internet Security Research Group +# Label: "ISRG Root X1" +# Serial: 172886928669790476064670243504169061120 +# MD5 Fingerprint: 0c:d2:f9:e0:da:17:73:e9:ed:86:4d:a5:e3:70:e7:4e +# SHA1 Fingerprint: ca:bd:2a:79:a1:07:6a:31:f2:1d:25:36:35:cb:03:9d:43:29:a5:e8 +# SHA256 Fingerprint: 96:bc:ec:06:26:49:76:f3:74:60:77:9a:cf:28:c5:a7:cf:e8:a3:c0:aa:e1:1a:8f:fc:ee:05:c0:bd:df:08:c6 +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw +TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh +cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 +WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu +ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY +MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc +h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ +0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U +A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW +T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH +B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC +B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv +KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn +OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn +jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw +qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI +rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV +HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq +hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL +ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ +3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK +NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 +ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur +TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC +jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc +oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq +4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA +mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d +emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= +-----END CERTIFICATE----- + +# Issuer: O=FNMT-RCM OU=AC RAIZ FNMT-RCM +# Subject: O=FNMT-RCM OU=AC RAIZ FNMT-RCM +# Label: "AC RAIZ FNMT-RCM" +# Serial: 485876308206448804701554682760554759 +# MD5 Fingerprint: e2:09:04:b4:d3:bd:d1:a0:14:fd:1a:d2:47:c4:57:1d +# SHA1 Fingerprint: ec:50:35:07:b2:15:c4:95:62:19:e2:a8:9a:5b:42:99:2c:4c:2c:20 +# SHA256 Fingerprint: eb:c5:57:0c:29:01:8c:4d:67:b1:aa:12:7b:af:12:f7:03:b4:61:1e:bc:17:b7:da:b5:57:38:94:17:9b:93:fa +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIPXZONMGc2yAYdGsdUhGkHMA0GCSqGSIb3DQEBCwUAMDsx +CzAJBgNVBAYTAkVTMREwDwYDVQQKDAhGTk1ULVJDTTEZMBcGA1UECwwQQUMgUkFJ +WiBGTk1ULVJDTTAeFw0wODEwMjkxNTU5NTZaFw0zMDAxMDEwMDAwMDBaMDsxCzAJ +BgNVBAYTAkVTMREwDwYDVQQKDAhGTk1ULVJDTTEZMBcGA1UECwwQQUMgUkFJWiBG +Tk1ULVJDTTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALpxgHpMhm5/ +yBNtwMZ9HACXjywMI7sQmkCpGreHiPibVmr75nuOi5KOpyVdWRHbNi63URcfqQgf +BBckWKo3Shjf5TnUV/3XwSyRAZHiItQDwFj8d0fsjz50Q7qsNI1NOHZnjrDIbzAz +WHFctPVrbtQBULgTfmxKo0nRIBnuvMApGGWn3v7v3QqQIecaZ5JCEJhfTzC8PhxF +tBDXaEAUwED653cXeuYLj2VbPNmaUtu1vZ5Gzz3rkQUCwJaydkxNEJY7kvqcfw+Z +374jNUUeAlz+taibmSXaXvMiwzn15Cou08YfxGyqxRxqAQVKL9LFwag0Jl1mpdIC +IfkYtwb1TplvqKtMUejPUBjFd8g5CSxJkjKZqLsXF3mwWsXmo8RZZUc1g16p6DUL +mbvkzSDGm0oGObVo/CK67lWMK07q87Hj/LaZmtVC+nFNCM+HHmpxffnTtOmlcYF7 +wk5HlqX2doWjKI/pgG6BU6VtX7hI+cL5NqYuSf+4lsKMB7ObiFj86xsc3i1w4peS +MKGJ47xVqCfWS+2QrYv6YyVZLag13cqXM7zlzced0ezvXg5KkAYmY6252TUtB7p2 +ZSysV4999AeU14ECll2jB0nVetBX+RvnU0Z1qrB5QstocQjpYL05ac70r8NWQMet +UqIJ5G+GR4of6ygnXYMgrwTJbFaai0b1AgMBAAGjgYMwgYAwDwYDVR0TAQH/BAUw +AwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFPd9xf3E6Jobd2Sn9R2gzL+H +YJptMD4GA1UdIAQ3MDUwMwYEVR0gADArMCkGCCsGAQUFBwIBFh1odHRwOi8vd3d3 +LmNlcnQuZm5tdC5lcy9kcGNzLzANBgkqhkiG9w0BAQsFAAOCAgEAB5BK3/MjTvDD +nFFlm5wioooMhfNzKWtN/gHiqQxjAb8EZ6WdmF/9ARP67Jpi6Yb+tmLSbkyU+8B1 +RXxlDPiyN8+sD8+Nb/kZ94/sHvJwnvDKuO+3/3Y3dlv2bojzr2IyIpMNOmqOFGYM +LVN0V2Ue1bLdI4E7pWYjJ2cJj+F3qkPNZVEI7VFY/uY5+ctHhKQV8Xa7pO6kO8Rf +77IzlhEYt8llvhjho6Tc+hj507wTmzl6NLrTQfv6MooqtyuGC2mDOL7Nii4LcK2N +JpLuHvUBKwrZ1pebbuCoGRw6IYsMHkCtA+fdZn71uSANA+iW+YJF1DngoABd15jm +fZ5nc8OaKveri6E6FO80vFIOiZiaBECEHX5FaZNXzuvO+FB8TxxuBEOb+dY7Ixjp +6o7RTUaN8Tvkasq6+yO3m/qZASlaWFot4/nUbQ4mrcFuNLwy+AwF+mWj2zs3gyLp +1txyM/1d8iC9djwj2ij3+RvrWWTV3F9yfiD8zYm1kGdNYno/Tq0dwzn+evQoFt9B +9kiABdcPUXmsEKvU7ANm5mqwujGSQkBqvjrTcuFqN1W8rB2Vt2lh8kORdOag0wok +RqEIr9baRRmW1FMdW4R58MD3R++Lj8UGrp1MYp3/RgT408m2ECVAdf4WqslKYIYv +uu8wd+RU4riEmViAqhOLUTpPSPaLtrM= +-----END CERTIFICATE----- + +# Issuer: CN=Amazon Root CA 1 O=Amazon +# Subject: CN=Amazon Root CA 1 O=Amazon +# Label: "Amazon Root CA 1" +# Serial: 143266978916655856878034712317230054538369994 +# MD5 Fingerprint: 43:c6:bf:ae:ec:fe:ad:2f:18:c6:88:68:30:fc:c8:e6 +# SHA1 Fingerprint: 8d:a7:f9:65:ec:5e:fc:37:91:0f:1c:6e:59:fd:c1:cc:6a:6e:de:16 +# SHA256 Fingerprint: 8e:cd:e6:88:4f:3d:87:b1:12:5b:a3:1a:c3:fc:b1:3d:70:16:de:7f:57:cc:90:4f:e1:cb:97:c6:ae:98:19:6e +-----BEGIN CERTIFICATE----- +MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj +ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM +9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw +IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 +VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L +93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm +jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC +AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA +A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI +U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs +N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv +o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU +5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy +rqXRfboQnoZsG4q5WTP468SQvvG5 +-----END CERTIFICATE----- + +# Issuer: CN=Amazon Root CA 2 O=Amazon +# Subject: CN=Amazon Root CA 2 O=Amazon +# Label: "Amazon Root CA 2" +# Serial: 143266982885963551818349160658925006970653239 +# MD5 Fingerprint: c8:e5:8d:ce:a8:42:e2:7a:c0:2a:5c:7c:9e:26:bf:66 +# SHA1 Fingerprint: 5a:8c:ef:45:d7:a6:98:59:76:7a:8c:8b:44:96:b5:78:cf:47:4b:1a +# SHA256 Fingerprint: 1b:a5:b2:aa:8c:65:40:1a:82:96:01:18:f8:0b:ec:4f:62:30:4d:83:ce:c4:71:3a:19:c3:9c:01:1e:a4:6d:b4 +-----BEGIN CERTIFICATE----- +MIIFQTCCAymgAwIBAgITBmyf0pY1hp8KD+WGePhbJruKNzANBgkqhkiG9w0BAQwF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAyMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK2Wny2cSkxK +gXlRmeyKy2tgURO8TW0G/LAIjd0ZEGrHJgw12MBvIITplLGbhQPDW9tK6Mj4kHbZ +W0/jTOgGNk3Mmqw9DJArktQGGWCsN0R5hYGCrVo34A3MnaZMUnbqQ523BNFQ9lXg +1dKmSYXpN+nKfq5clU1Imj+uIFptiJXZNLhSGkOQsL9sBbm2eLfq0OQ6PBJTYv9K +8nu+NQWpEjTj82R0Yiw9AElaKP4yRLuH3WUnAnE72kr3H9rN9yFVkE8P7K6C4Z9r +2UXTu/Bfh+08LDmG2j/e7HJV63mjrdvdfLC6HM783k81ds8P+HgfajZRRidhW+me +z/CiVX18JYpvL7TFz4QuK/0NURBs+18bvBt+xa47mAExkv8LV/SasrlX6avvDXbR +8O70zoan4G7ptGmh32n2M8ZpLpcTnqWHsFcQgTfJU7O7f/aS0ZzQGPSSbtqDT6Zj +mUyl+17vIWR6IF9sZIUVyzfpYgwLKhbcAS4y2j5L9Z469hdAlO+ekQiG+r5jqFoz +7Mt0Q5X5bGlSNscpb/xVA1wf+5+9R+vnSUeVC06JIglJ4PVhHvG/LopyboBZ/1c6 ++XUyo05f7O0oYtlNc/LMgRdg7c3r3NunysV+Ar3yVAhU/bQtCSwXVEqY0VThUWcI +0u1ufm8/0i2BWSlmy5A5lREedCf+3euvAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMB +Af8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSwDPBMMPQFWAJI/TPlUq9LhONm +UjANBgkqhkiG9w0BAQwFAAOCAgEAqqiAjw54o+Ci1M3m9Zh6O+oAA7CXDpO8Wqj2 +LIxyh6mx/H9z/WNxeKWHWc8w4Q0QshNabYL1auaAn6AFC2jkR2vHat+2/XcycuUY ++gn0oJMsXdKMdYV2ZZAMA3m3MSNjrXiDCYZohMr/+c8mmpJ5581LxedhpxfL86kS +k5Nrp+gvU5LEYFiwzAJRGFuFjWJZY7attN6a+yb3ACfAXVU3dJnJUH/jWS5E4ywl +7uxMMne0nxrpS10gxdr9HIcWxkPo1LsmmkVwXqkLN1PiRnsn/eBG8om3zEK2yygm +btmlyTrIQRNg91CMFa6ybRoVGld45pIq2WWQgj9sAq+uEjonljYE1x2igGOpm/Hl +urR8FLBOybEfdF849lHqm/osohHUqS0nGkWxr7JOcQ3AWEbWaQbLU8uz/mtBzUF+ +fUwPfHJ5elnNXkoOrJupmHN5fLT0zLm4BwyydFy4x2+IoZCn9Kr5v2c69BoVYh63 +n749sSmvZ6ES8lgQGVMDMBu4Gon2nL2XA46jCfMdiyHxtN/kHNGfZQIG6lzWE7OE +76KlXIx3KadowGuuQNKotOrN8I1LOJwZmhsoVLiJkO/KdYE+HvJkJMcYr07/R54H +9jVlpNMKVv/1F2Rs76giJUmTtt8AF9pYfl3uxRuw0dFfIRDH+fO6AgonB8Xx1sfT +4PsJYGw= +-----END CERTIFICATE----- + +# Issuer: CN=Amazon Root CA 3 O=Amazon +# Subject: CN=Amazon Root CA 3 O=Amazon +# Label: "Amazon Root CA 3" +# Serial: 143266986699090766294700635381230934788665930 +# MD5 Fingerprint: a0:d4:ef:0b:f7:b5:d8:49:95:2a:ec:f5:c4:fc:81:87 +# SHA1 Fingerprint: 0d:44:dd:8c:3c:8c:1a:1a:58:75:64:81:e9:0f:2e:2a:ff:b3:d2:6e +# SHA256 Fingerprint: 18:ce:6c:fe:7b:f1:4e:60:b2:e3:47:b8:df:e8:68:cb:31:d0:2e:bb:3a:da:27:15:69:f5:03:43:b4:6d:b3:a4 +-----BEGIN CERTIFICATE----- +MIIBtjCCAVugAwIBAgITBmyf1XSXNmY/Owua2eiedgPySjAKBggqhkjOPQQDAjA5 +MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g +Um9vdCBDQSAzMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG +A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg +Q0EgMzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCmXp8ZBf8ANm+gBG1bG8lKl +ui2yEujSLtf6ycXYqm0fc4E7O5hrOXwzpcVOho6AF2hiRVd9RFgdszflZwjrZt6j +QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSr +ttvXBp43rDCGB5Fwx5zEGbF4wDAKBggqhkjOPQQDAgNJADBGAiEA4IWSoxe3jfkr +BqWTrBqYaGFy+uGh0PsceGCmQ5nFuMQCIQCcAu/xlJyzlvnrxir4tiz+OpAUFteM +YyRIHN8wfdVoOw== +-----END CERTIFICATE----- + +# Issuer: CN=Amazon Root CA 4 O=Amazon +# Subject: CN=Amazon Root CA 4 O=Amazon +# Label: "Amazon Root CA 4" +# Serial: 143266989758080763974105200630763877849284878 +# MD5 Fingerprint: 89:bc:27:d5:eb:17:8d:06:6a:69:d5:fd:89:47:b4:cd +# SHA1 Fingerprint: f6:10:84:07:d6:f8:bb:67:98:0c:c2:e2:44:c2:eb:ae:1c:ef:63:be +# SHA256 Fingerprint: e3:5d:28:41:9e:d0:20:25:cf:a6:90:38:cd:62:39:62:45:8d:a5:c6:95:fb:de:a3:c2:2b:0b:fb:25:89:70:92 +-----BEGIN CERTIFICATE----- +MIIB8jCCAXigAwIBAgITBmyf18G7EEwpQ+Vxe3ssyBrBDjAKBggqhkjOPQQDAzA5 +MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g +Um9vdCBDQSA0MB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG +A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg +Q0EgNDB2MBAGByqGSM49AgEGBSuBBAAiA2IABNKrijdPo1MN/sGKe0uoe0ZLY7Bi +9i0b2whxIdIA6GO9mif78DluXeo9pcmBqqNbIJhFXRbb/egQbeOc4OO9X4Ri83Bk +M6DLJC9wuoihKqB1+IGuYgbEgds5bimwHvouXKNCMEAwDwYDVR0TAQH/BAUwAwEB +/zAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFNPsxzplbszh2naaVvuc84ZtV+WB +MAoGCCqGSM49BAMDA2gAMGUCMDqLIfG9fhGt0O9Yli/W651+kI0rz2ZVwyzjKKlw +CkcO8DdZEv8tmZQoTipPNU0zWgIxAOp1AE47xDqUEpHJWEadIRNyp4iciuRMStuW +1KyLa2tJElMzrdfkviT8tQp21KW8EA== +-----END CERTIFICATE----- + +# Issuer: CN=TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1 O=Turkiye Bilimsel ve Teknolojik Arastirma Kurumu - TUBITAK OU=Kamu Sertifikasyon Merkezi - Kamu SM +# Subject: CN=TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1 O=Turkiye Bilimsel ve Teknolojik Arastirma Kurumu - TUBITAK OU=Kamu Sertifikasyon Merkezi - Kamu SM +# Label: "TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1" +# Serial: 1 +# MD5 Fingerprint: dc:00:81:dc:69:2f:3e:2f:b0:3b:f6:3d:5a:91:8e:49 +# SHA1 Fingerprint: 31:43:64:9b:ec:ce:27:ec:ed:3a:3f:0b:8f:0d:e4:e8:91:dd:ee:ca +# SHA256 Fingerprint: 46:ed:c3:68:90:46:d5:3a:45:3f:b3:10:4a:b8:0d:ca:ec:65:8b:26:60:ea:16:29:dd:7e:86:79:90:64:87:16 +-----BEGIN CERTIFICATE----- +MIIEYzCCA0ugAwIBAgIBATANBgkqhkiG9w0BAQsFADCB0jELMAkGA1UEBhMCVFIx +GDAWBgNVBAcTD0dlYnplIC0gS29jYWVsaTFCMEAGA1UEChM5VHVya2l5ZSBCaWxp +bXNlbCB2ZSBUZWtub2xvamlrIEFyYXN0aXJtYSBLdXJ1bXUgLSBUVUJJVEFLMS0w +KwYDVQQLEyRLYW11IFNlcnRpZmlrYXN5b24gTWVya2V6aSAtIEthbXUgU00xNjA0 +BgNVBAMTLVRVQklUQUsgS2FtdSBTTSBTU0wgS29rIFNlcnRpZmlrYXNpIC0gU3Vy +dW0gMTAeFw0xMzExMjUwODI1NTVaFw00MzEwMjUwODI1NTVaMIHSMQswCQYDVQQG +EwJUUjEYMBYGA1UEBxMPR2ViemUgLSBLb2NhZWxpMUIwQAYDVQQKEzlUdXJraXll +IEJpbGltc2VsIHZlIFRla25vbG9qaWsgQXJhc3Rpcm1hIEt1cnVtdSAtIFRVQklU +QUsxLTArBgNVBAsTJEthbXUgU2VydGlmaWthc3lvbiBNZXJrZXppIC0gS2FtdSBT +TTE2MDQGA1UEAxMtVFVCSVRBSyBLYW11IFNNIFNTTCBLb2sgU2VydGlmaWthc2kg +LSBTdXJ1bSAxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr3UwM6q7 +a9OZLBI3hNmNe5eA027n/5tQlT6QlVZC1xl8JoSNkvoBHToP4mQ4t4y86Ij5iySr +LqP1N+RAjhgleYN1Hzv/bKjFxlb4tO2KRKOrbEz8HdDc72i9z+SqzvBV96I01INr +N3wcwv61A+xXzry0tcXtAA9TNypN9E8Mg/uGz8v+jE69h/mniyFXnHrfA2eJLJ2X +YacQuFWQfw4tJzh03+f92k4S400VIgLI4OD8D62K18lUUMw7D8oWgITQUVbDjlZ/ +iSIzL+aFCr2lqBs23tPcLG07xxO9WSMs5uWk99gL7eqQQESolbuT1dCANLZGeA4f +AJNG4e7p+exPFwIDAQABo0IwQDAdBgNVHQ4EFgQUZT/HiobGPN08VFw1+DrtUgxH +V8gwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggEBACo/4fEyjq7hmFxLXs9rHmoJ0iKpEsdeV31zVmSAhHqT5Am5EM2fKifh +AHe+SMg1qIGf5LgsyX8OsNJLN13qudULXjS99HMpw+0mFZx+CFOKWI3QSyjfwbPf +IPP54+M638yclNhOT8NrF7f3cuitZjO1JVOr4PhMqZ398g26rrnZqsZr+ZO7rqu4 +lzwDGrpDxpa5RXI4s6ehlj2Re37AIVNMh+3yC1SVUZPVIqUNivGTDj5UDrDYyU7c +8jEyVupk+eq1nRZmQnLzf9OxMUP8pI4X8W0jq5Rm+K37DwhuJi1/FwcJsoz7UMCf +lo3Ptv0AnVoUmr8CRPXBwp8iXqIPoeM= +-----END CERTIFICATE----- + +# Issuer: CN=GDCA TrustAUTH R5 ROOT O=GUANG DONG CERTIFICATE AUTHORITY CO.,LTD. +# Subject: CN=GDCA TrustAUTH R5 ROOT O=GUANG DONG CERTIFICATE AUTHORITY CO.,LTD. +# Label: "GDCA TrustAUTH R5 ROOT" +# Serial: 9009899650740120186 +# MD5 Fingerprint: 63:cc:d9:3d:34:35:5c:6f:53:a3:e2:08:70:48:1f:b4 +# SHA1 Fingerprint: 0f:36:38:5b:81:1a:25:c3:9b:31:4e:83:ca:e9:34:66:70:cc:74:b4 +# SHA256 Fingerprint: bf:ff:8f:d0:44:33:48:7d:6a:8a:a6:0c:1a:29:76:7a:9f:c2:bb:b0:5e:42:0f:71:3a:13:b9:92:89:1d:38:93 +-----BEGIN CERTIFICATE----- +MIIFiDCCA3CgAwIBAgIIfQmX/vBH6nowDQYJKoZIhvcNAQELBQAwYjELMAkGA1UE +BhMCQ04xMjAwBgNVBAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZ +IENPLixMVEQuMR8wHQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMB4XDTE0 +MTEyNjA1MTMxNVoXDTQwMTIzMTE1NTk1OVowYjELMAkGA1UEBhMCQ04xMjAwBgNV +BAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZIENPLixMVEQuMR8w +HQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEA2aMW8Mh0dHeb7zMNOwZ+Vfy1YI92hhJCfVZmPoiC7XJj +Dp6L3TQsAlFRwxn9WVSEyfFrs0yw6ehGXTjGoqcuEVe6ghWinI9tsJlKCvLriXBj +TnnEt1u9ol2x8kECK62pOqPseQrsXzrj/e+APK00mxqriCZ7VqKChh/rNYmDf1+u +KU49tm7srsHwJ5uu4/Ts765/94Y9cnrrpftZTqfrlYwiOXnhLQiPzLyRuEH3FMEj +qcOtmkVEs7LXLM3GKeJQEK5cy4KOFxg2fZfmiJqwTTQJ9Cy5WmYqsBebnh52nUpm +MUHfP/vFBu8btn4aRjb3ZGM74zkYI+dndRTVdVeSN72+ahsmUPI2JgaQxXABZG12 +ZuGR224HwGGALrIuL4xwp9E7PLOR5G62xDtw8mySlwnNR30YwPO7ng/Wi64HtloP +zgsMR6flPri9fcebNaBhlzpBdRfMK5Z3KpIhHtmVdiBnaM8Nvd/WHwlqmuLMc3Gk +L30SgLdTMEZeS1SZD2fJpcjyIMGC7J0R38IC+xo70e0gmu9lZJIQDSri3nDxGGeC +jGHeuLzRL5z7D9Ar7Rt2ueQ5Vfj4oR24qoAATILnsn8JuLwwoC8N9VKejveSswoA +HQBUlwbgsQfZxw9cZX08bVlX5O2ljelAU58VS6Bx9hoh49pwBiFYFIeFd3mqgnkC +AwEAAaNCMEAwHQYDVR0OBBYEFOLJQJ9NzuiaoXzPDj9lxSmIahlRMA8GA1UdEwEB +/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQDRSVfg +p8xoWLoBDysZzY2wYUWsEe1jUGn4H3++Fo/9nesLqjJHdtJnJO29fDMylyrHBYZm +DRd9FBUb1Ov9H5r2XpdptxolpAqzkT9fNqyL7FeoPueBihhXOYV0GkLH6VsTX4/5 +COmSdI31R9KrO9b7eGZONn356ZLpBN79SWP8bfsUcZNnL0dKt7n/HipzcEYwv1ry +L3ml4Y0M2fmyYzeMN2WFcGpcWwlyua1jPLHd+PwyvzeG5LuOmCd+uh8W4XAR8gPf +JWIyJyYYMoSf/wA6E7qaTfRPuBRwIrHKK5DOKcFw9C+df/KQHtZa37dG/OaG+svg +IHZ6uqbL9XzeYqWxi+7egmaKTjowHz+Ay60nugxe19CxVsp3cbK1daFQqUBDF8Io +2c9Si1vIY9RCPqAzekYu9wogRlR+ak8x8YF+QnQ4ZXMn7sZ8uI7XpTrXmKGcjBBV +09tL7ECQ8s1uV9JiDnxXk7Gnbc2dg7sq5+W2O3FYrf3RRbxake5TFW/TRQl1brqQ +XR4EzzffHqhmsYzmIGrv/EhOdJhCrylvLmrH+33RZjEizIYAfmaDDEL0vTSSwxrq +T8p+ck0LcIymSLumoRT2+1hEmRSuqguTaaApJUqlyyvdimYHFngVV3Eb7PVHhPOe +MTd61X8kreS8/f3MboPoDKi3QWwH3b08hpcv0g== +-----END CERTIFICATE----- + +# Issuer: CN=SSL.com Root Certification Authority RSA O=SSL Corporation +# Subject: CN=SSL.com Root Certification Authority RSA O=SSL Corporation +# Label: "SSL.com Root Certification Authority RSA" +# Serial: 8875640296558310041 +# MD5 Fingerprint: 86:69:12:c0:70:f1:ec:ac:ac:c2:d5:bc:a5:5b:a1:29 +# SHA1 Fingerprint: b7:ab:33:08:d1:ea:44:77:ba:14:80:12:5a:6f:bd:a9:36:49:0c:bb +# SHA256 Fingerprint: 85:66:6a:56:2e:e0:be:5c:e9:25:c1:d8:89:0a:6f:76:a8:7e:c1:6d:4d:7d:5f:29:ea:74:19:cf:20:12:3b:69 +-----BEGIN CERTIFICATE----- +MIIF3TCCA8WgAwIBAgIIeyyb0xaAMpkwDQYJKoZIhvcNAQELBQAwfDELMAkGA1UE +BhMCVVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQK +DA9TU0wgQ29ycG9yYXRpb24xMTAvBgNVBAMMKFNTTC5jb20gUm9vdCBDZXJ0aWZp +Y2F0aW9uIEF1dGhvcml0eSBSU0EwHhcNMTYwMjEyMTczOTM5WhcNNDEwMjEyMTcz +OTM5WjB8MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hv +dXN0b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjExMC8GA1UEAwwoU1NMLmNv +bSBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IFJTQTCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBAPkP3aMrfcvQKv7sZ4Wm5y4bunfh4/WvpOz6Sl2R +xFdHaxh3a3by/ZPkPQ/CFp4LZsNWlJ4Xg4XOVu/yFv0AYvUiCVToZRdOQbngT0aX +qhvIuG5iXmmxX9sqAn78bMrzQdjt0Oj8P2FI7bADFB0QDksZ4LtO7IZl/zbzXmcC +C52GVWH9ejjt/uIZALdvoVBidXQ8oPrIJZK0bnoix/geoeOy3ZExqysdBP+lSgQ3 +6YWkMyv94tZVNHwZpEpox7Ko07fKoZOI68GXvIz5HdkihCR0xwQ9aqkpk8zruFvh +/l8lqjRYyMEjVJ0bmBHDOJx+PYZspQ9AhnwC9FwCTyjLrnGfDzrIM/4RJTXq/LrF +YD3ZfBjVsqnTdXgDciLKOsMf7yzlLqn6niy2UUb9rwPW6mBo6oUWNmuF6R7As93E +JNyAKoFBbZQ+yODJgUEAnl6/f8UImKIYLEJAs/lvOCdLToD0PYFH4Ih86hzOtXVc +US4cK38acijnALXRdMbX5J+tB5O2UzU1/Dfkw/ZdFr4hc96SCvigY2q8lpJqPvi8 +ZVWb3vUNiSYE/CUapiVpy8JtynziWV+XrOvvLsi81xtZPCvM8hnIk2snYxnP/Okm ++Mpxm3+T/jRnhE6Z6/yzeAkzcLpmpnbtG3PrGqUNxCITIJRWCk4sbE6x/c+cCbqi +M+2HAgMBAAGjYzBhMB0GA1UdDgQWBBTdBAkHovV6fVJTEpKV7jiAJQ2mWTAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFN0ECQei9Xp9UlMSkpXuOIAlDaZZMA4G +A1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAIBgRlCn7Jp0cHh5wYfGV +cpNxJK1ok1iOMq8bs3AD/CUrdIWQPXhq9LmLpZc7tRiRux6n+UBbkflVma8eEdBc +Hadm47GUBwwyOabqG7B52B2ccETjit3E+ZUfijhDPwGFpUenPUayvOUiaPd7nNgs +PgohyC0zrL/FgZkxdMF1ccW+sfAjRfSda/wZY52jvATGGAslu1OJD7OAUN5F7kR/ +q5R4ZJjT9ijdh9hwZXT7DrkT66cPYakylszeu+1jTBi7qUD3oFRuIIhxdRjqerQ0 +cuAjJ3dctpDqhiVAq+8zD8ufgr6iIPv2tS0a5sKFsXQP+8hlAqRSAUfdSSLBv9jr +a6x+3uxjMxW3IwiPxg+NQVrdjsW5j+VFP3jbutIbQLH+cU0/4IGiul607BXgk90I +H37hVZkLId6Tngr75qNJvTYw/ud3sqB1l7UtgYgXZSD32pAAn8lSzDLKNXz1PQ/Y +K9f1JmzJBjSWFupwWRoyeXkLtoh/D1JIPb9s2KJELtFOt3JY04kTlf5Eq/jXixtu +nLwsoFvVagCvXzfh1foQC5ichucmj87w7G6KVwuA406ywKBjYZC6VWg3dGq2ktuf +oYYitmUnDuy2n0Jg5GfCtdpBC8TTi2EbvPofkSvXRAdeuims2cXp71NIWuuA8ShY +Ic2wBlX7Jz9TkHCpBB5XJ7k= +-----END CERTIFICATE----- + +# Issuer: CN=SSL.com Root Certification Authority ECC O=SSL Corporation +# Subject: CN=SSL.com Root Certification Authority ECC O=SSL Corporation +# Label: "SSL.com Root Certification Authority ECC" +# Serial: 8495723813297216424 +# MD5 Fingerprint: 2e:da:e4:39:7f:9c:8f:37:d1:70:9f:26:17:51:3a:8e +# SHA1 Fingerprint: c3:19:7c:39:24:e6:54:af:1b:c4:ab:20:95:7a:e2:c3:0e:13:02:6a +# SHA256 Fingerprint: 34:17:bb:06:cc:60:07:da:1b:96:1c:92:0b:8a:b4:ce:3f:ad:82:0e:4a:a3:0b:9a:cb:c4:a7:4e:bd:ce:bc:65 +-----BEGIN CERTIFICATE----- +MIICjTCCAhSgAwIBAgIIdebfy8FoW6gwCgYIKoZIzj0EAwIwfDELMAkGA1UEBhMC +VVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQKDA9T +U0wgQ29ycG9yYXRpb24xMTAvBgNVBAMMKFNTTC5jb20gUm9vdCBDZXJ0aWZpY2F0 +aW9uIEF1dGhvcml0eSBFQ0MwHhcNMTYwMjEyMTgxNDAzWhcNNDEwMjEyMTgxNDAz +WjB8MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hvdXN0 +b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjExMC8GA1UEAwwoU1NMLmNvbSBS +b290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IEVDQzB2MBAGByqGSM49AgEGBSuB +BAAiA2IABEVuqVDEpiM2nl8ojRfLliJkP9x6jh3MCLOicSS6jkm5BBtHllirLZXI +7Z4INcgn64mMU1jrYor+8FsPazFSY0E7ic3s7LaNGdM0B9y7xgZ/wkWV7Mt/qCPg +CemB+vNH06NjMGEwHQYDVR0OBBYEFILRhXMw5zUE044CkvvlpNHEIejNMA8GA1Ud +EwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUgtGFczDnNQTTjgKS++Wk0cQh6M0wDgYD +VR0PAQH/BAQDAgGGMAoGCCqGSM49BAMCA2cAMGQCMG/n61kRpGDPYbCWe+0F+S8T +kdzt5fxQaxFGRrMcIQBiu77D5+jNB5n5DQtdcj7EqgIwH7y6C+IwJPt8bYBVCpk+ +gA0z5Wajs6O7pdWLjwkspl1+4vAHCGht0nxpbl/f5Wpl +-----END CERTIFICATE----- + +# Issuer: CN=SSL.com EV Root Certification Authority RSA R2 O=SSL Corporation +# Subject: CN=SSL.com EV Root Certification Authority RSA R2 O=SSL Corporation +# Label: "SSL.com EV Root Certification Authority RSA R2" +# Serial: 6248227494352943350 +# MD5 Fingerprint: e1:1e:31:58:1a:ae:54:53:02:f6:17:6a:11:7b:4d:95 +# SHA1 Fingerprint: 74:3a:f0:52:9b:d0:32:a0:f4:4a:83:cd:d4:ba:a9:7b:7c:2e:c4:9a +# SHA256 Fingerprint: 2e:7b:f1:6c:c2:24:85:a7:bb:e2:aa:86:96:75:07:61:b0:ae:39:be:3b:2f:e9:d0:cc:6d:4e:f7:34:91:42:5c +-----BEGIN CERTIFICATE----- +MIIF6zCCA9OgAwIBAgIIVrYpzTS8ePYwDQYJKoZIhvcNAQELBQAwgYIxCzAJBgNV +BAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEQMA4GA1UEBwwHSG91c3RvbjEYMBYGA1UE +CgwPU1NMIENvcnBvcmF0aW9uMTcwNQYDVQQDDC5TU0wuY29tIEVWIFJvb3QgQ2Vy +dGlmaWNhdGlvbiBBdXRob3JpdHkgUlNBIFIyMB4XDTE3MDUzMTE4MTQzN1oXDTQy +MDUzMDE4MTQzN1owgYIxCzAJBgNVBAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEQMA4G +A1UEBwwHSG91c3RvbjEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMTcwNQYDVQQD +DC5TU0wuY29tIEVWIFJvb3QgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgUlNBIFIy +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAjzZlQOHWTcDXtOlG2mvq +M0fNTPl9fb69LT3w23jhhqXZuglXaO1XPqDQCEGD5yhBJB/jchXQARr7XnAjssuf +OePPxU7Gkm0mxnu7s9onnQqG6YE3Bf7wcXHswxzpY6IXFJ3vG2fThVUCAtZJycxa +4bH3bzKfydQ7iEGonL3Lq9ttewkfokxykNorCPzPPFTOZw+oz12WGQvE43LrrdF9 +HSfvkusQv1vrO6/PgN3B0pYEW3p+pKk8OHakYo6gOV7qd89dAFmPZiw+B6KjBSYR +aZfqhbcPlgtLyEDhULouisv3D5oi53+aNxPN8k0TayHRwMwi8qFG9kRpnMphNQcA +b9ZhCBHqurj26bNg5U257J8UZslXWNvNh2n4ioYSA0e/ZhN2rHd9NCSFg83XqpyQ +Gp8hLH94t2S42Oim9HizVcuE0jLEeK6jj2HdzghTreyI/BXkmg3mnxp3zkyPuBQV +PWKchjgGAGYS5Fl2WlPAApiiECtoRHuOec4zSnaqW4EWG7WK2NAAe15itAnWhmMO +pgWVSbooi4iTsjQc2KRVbrcc0N6ZVTsj9CLg+SlmJuwgUHfbSguPvuUCYHBBXtSu +UDkiFCbLsjtzdFVHB3mBOagwE0TlBIqulhMlQg+5U8Sb/M3kHN48+qvWBkofZ6aY +MBzdLNvcGJVXZsb/XItW9XcCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAfBgNV +HSMEGDAWgBT5YLvU49U09rj1BoAlp3PbRmmonjAdBgNVHQ4EFgQU+WC71OPVNPa4 +9QaAJadz20ZpqJ4wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQBW +s47LCp1Jjr+kxJG7ZhcFUZh1++VQLHqe8RT6q9OKPv+RKY9ji9i0qVQBDb6Thi/5 +Sm3HXvVX+cpVHBK+Rw82xd9qt9t1wkclf7nxY/hoLVUE0fKNsKTPvDxeH3jnpaAg +cLAExbf3cqfeIg29MyVGjGSSJuM+LmOW2puMPfgYCdcDzH2GguDKBAdRUNf/ktUM +79qGn5nX67evaOI5JpS6aLe/g9Pqemc9YmeuJeVy6OLk7K4S9ksrPJ/psEDzOFSz +/bdoyNrGj1E8svuR3Bznm53htw1yj+KkxKl4+esUrMZDBcJlOSgYAsOCsp0FvmXt +ll9ldDz7CTUue5wT/RsPXcdtgTpWD8w74a8CLyKsRspGPKAcTNZEtF4uXBVmCeEm +Kf7GUmG6sXP/wwyc5WxqlD8UykAWlYTzWamsX0xhk23RO8yilQwipmdnRC652dKK +QbNmC1r7fSOl8hqw/96bg5Qu0T/fkreRrwU7ZcegbLHNYhLDkBvjJc40vG93drEQ +w/cFGsDWr3RiSBd3kmmQYRzelYB0VI8YHMPzA9C/pEN1hlMYegouCRw2n5H9gooi +S9EOUCXdywMMF8mDAAhONU2Ki+3wApRmLER/y5UnlhetCTCstnEXbosX9hwJ1C07 +mKVx01QT2WDz9UtmT/rx7iASjbSsV7FFY6GsdqnC+w== +-----END CERTIFICATE----- + +# Issuer: CN=SSL.com EV Root Certification Authority ECC O=SSL Corporation +# Subject: CN=SSL.com EV Root Certification Authority ECC O=SSL Corporation +# Label: "SSL.com EV Root Certification Authority ECC" +# Serial: 3182246526754555285 +# MD5 Fingerprint: 59:53:22:65:83:42:01:54:c0:ce:42:b9:5a:7c:f2:90 +# SHA1 Fingerprint: 4c:dd:51:a3:d1:f5:20:32:14:b0:c6:c5:32:23:03:91:c7:46:42:6d +# SHA256 Fingerprint: 22:a2:c1:f7:bd:ed:70:4c:c1:e7:01:b5:f4:08:c3:10:88:0f:e9:56:b5:de:2a:4a:44:f9:9c:87:3a:25:a7:c8 +-----BEGIN CERTIFICATE----- +MIIClDCCAhqgAwIBAgIILCmcWxbtBZUwCgYIKoZIzj0EAwIwfzELMAkGA1UEBhMC +VVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQKDA9T +U0wgQ29ycG9yYXRpb24xNDAyBgNVBAMMK1NTTC5jb20gRVYgUm9vdCBDZXJ0aWZp +Y2F0aW9uIEF1dGhvcml0eSBFQ0MwHhcNMTYwMjEyMTgxNTIzWhcNNDEwMjEyMTgx +NTIzWjB/MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hv +dXN0b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjE0MDIGA1UEAwwrU1NMLmNv +bSBFViBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IEVDQzB2MBAGByqGSM49 +AgEGBSuBBAAiA2IABKoSR5CYG/vvw0AHgyBO8TCCogbR8pKGYfL2IWjKAMTH6kMA +VIbc/R/fALhBYlzccBYy3h+Z1MzFB8gIH2EWB1E9fVwHU+M1OIzfzZ/ZLg1Kthku +WnBaBu2+8KGwytAJKaNjMGEwHQYDVR0OBBYEFFvKXuXe0oGqzagtZFG22XKbl+ZP +MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX +5k8wDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMCA2gAMGUCMQCK5kCJN+vp1RPZ +ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg +h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg== +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Label: "GlobalSign Root CA - R6" +# Serial: 1417766617973444989252670301619537 +# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae +# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1 +# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69 +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg +MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx +MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET +MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI +xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k +ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD +aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw +LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw +1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX +k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 +SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h +bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n +WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY +rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce +MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu +bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN +nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt +Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 +55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj +vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf +cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz +oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp +nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs +pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v +JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R +8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 +5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= +-----END CERTIFICATE----- + +# Issuer: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Subject: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Label: "OISTE WISeKey Global Root GC CA" +# Serial: 44084345621038548146064804565436152554 +# MD5 Fingerprint: a9:d6:b9:2d:2f:93:64:f8:a5:69:ca:91:e9:68:07:23 +# SHA1 Fingerprint: e0:11:84:5e:34:de:be:88:81:b9:9c:f6:16:26:d1:96:1f:c3:b9:31 +# SHA256 Fingerprint: 85:60:f9:1c:36:24:da:ba:95:70:b5:fe:a0:db:e3:6f:f1:1a:83:23:be:94:86:85:4f:b3:f3:4a:55:71:19:8d +-----BEGIN CERTIFICATE----- +MIICaTCCAe+gAwIBAgIQISpWDK7aDKtARb8roi066jAKBggqhkjOPQQDAzBtMQsw +CQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUgRm91 +bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9iYWwg +Um9vdCBHQyBDQTAeFw0xNzA1MDkwOTQ4MzRaFw00MjA1MDkwOTU4MzNaMG0xCzAJ +BgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBGb3Vu +ZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2JhbCBS +b290IEdDIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAETOlQwMYPchi82PG6s4ni +eUqjFqdrVCTbUf/q9Akkwwsin8tqJ4KBDdLArzHkdIJuyiXZjHWd8dvQmqJLIX4W +p2OQ0jnUsYd4XxiWD1AbNTcPasbc2RNNpI6QN+a9WzGRo1QwUjAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUSIcUrOPDnpBgOtfKie7T +rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV +57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg +Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 +-----END CERTIFICATE----- + +# Issuer: CN=UCA Global G2 Root O=UniTrust +# Subject: CN=UCA Global G2 Root O=UniTrust +# Label: "UCA Global G2 Root" +# Serial: 124779693093741543919145257850076631279 +# MD5 Fingerprint: 80:fe:f0:c4:4a:f0:5c:62:32:9f:1c:ba:78:a9:50:f8 +# SHA1 Fingerprint: 28:f9:78:16:19:7a:ff:18:25:18:aa:44:fe:c1:a0:ce:5c:b6:4c:8a +# SHA256 Fingerprint: 9b:ea:11:c9:76:fe:01:47:64:c1:be:56:a6:f9:14:b5:a5:60:31:7a:bd:99:88:39:33:82:e5:16:1a:a0:49:3c +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIQXd+x2lqj7V2+WmUgZQOQ7zANBgkqhkiG9w0BAQsFADA9 +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxGzAZBgNVBAMMElVDQSBH +bG9iYWwgRzIgUm9vdDAeFw0xNjAzMTEwMDAwMDBaFw00MDEyMzEwMDAwMDBaMD0x +CzAJBgNVBAYTAkNOMREwDwYDVQQKDAhVbmlUcnVzdDEbMBkGA1UEAwwSVUNBIEds +b2JhbCBHMiBSb290MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeYr +b3zvJgUno4Ek2m/LAfmZmqkywiKHYUGRO8vDaBsGxUypK8FnFyIdK+35KYmToni9 +kmugow2ifsqTs6bRjDXVdfkX9s9FxeV67HeToI8jrg4aA3++1NDtLnurRiNb/yzm +VHqUwCoV8MmNsHo7JOHXaOIxPAYzRrZUEaalLyJUKlgNAQLx+hVRZ2zA+te2G3/R +VogvGjqNO7uCEeBHANBSh6v7hn4PJGtAnTRnvI3HLYZveT6OqTwXS3+wmeOwcWDc +C/Vkw85DvG1xudLeJ1uK6NjGruFZfc8oLTW4lVYa8bJYS7cSN8h8s+1LgOGN+jIj +tm+3SJUIsUROhYw6AlQgL9+/V087OpAh18EmNVQg7Mc/R+zvWr9LesGtOxdQXGLY +D0tK3Cv6brxzks3sx1DoQZbXqX5t2Okdj4q1uViSukqSKwxW/YDrCPBeKW4bHAyv +j5OJrdu9o54hyokZ7N+1wxrrFv54NkzWbtA+FxyQF2smuvt6L78RHBgOLXMDj6Dl +NaBa4kx1HXHhOThTeEDMg5PXCp6dW4+K5OXgSORIskfNTip1KnvyIvbJvgmRlld6 +iIis7nCs+dwp4wwcOxJORNanTrAmyPPZGpeRaOrvjUYG0lZFWJo8DA+DuAUlwznP +O6Q0ibd5Ei9Hxeepl2n8pndntd978XplFeRhVmUCAwEAAaNCMEAwDgYDVR0PAQH/ +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIHEjMz15DD/pQwIX4wV +ZyF0Ad/fMA0GCSqGSIb3DQEBCwUAA4ICAQATZSL1jiutROTL/7lo5sOASD0Ee/oj +L3rtNtqyzm325p7lX1iPyzcyochltq44PTUbPrw7tgTQvPlJ9Zv3hcU2tsu8+Mg5 +1eRfB70VVJd0ysrtT7q6ZHafgbiERUlMjW+i67HM0cOU2kTC5uLqGOiiHycFutfl +1qnN3e92mI0ADs0b+gO3joBYDic/UvuUospeZcnWhNq5NXHzJsBPd+aBJ9J3O5oU +b3n09tDh05S60FdRvScFDcH9yBIw7m+NESsIndTUv4BFFJqIRNow6rSn4+7vW4LV +PtateJLbXDzz2K36uGt/xDYotgIVilQsnLAXc47QN6MUPJiVAAwpBVueSUmxX8fj +y88nZY41F7dXyDDZQVu5FLbowg+UMaeUmMxq67XhJ/UQqAHojhJi6IjMtX9Gl8Cb +EGY4GjZGXyJoPd/JxhMnq1MGrKI8hgZlb7F+sSlEmqO6SWkoaY/X5V+tBIZkbxqg +DMUIYs6Ao9Dz7GjevjPHF1t/gMRMTLGmhIrDO7gJzRSBuhjjVFc2/tsvfEehOjPI ++Vg7RE+xygKJBJYoaMVLuCaJu9YzL1DV/pqJuhgyklTGW+Cd+V7lDSKb9triyCGy +YiGqhkCyLmTTX8jjfhFnRR8F/uOi77Oos/N9j/gMHyIfLXC0uAE0djAA5SN4p1bX +UB+K+wb1whnw0A== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Extended Validation Root O=UniTrust +# Subject: CN=UCA Extended Validation Root O=UniTrust +# Label: "UCA Extended Validation Root" +# Serial: 106100277556486529736699587978573607008 +# MD5 Fingerprint: a1:f3:5f:43:c6:34:9b:da:bf:8c:7e:05:53:ad:96:e2 +# SHA1 Fingerprint: a3:a1:b0:6f:24:61:23:4a:e3:36:a5:c2:37:fc:a6:ff:dd:f0:d7:3a +# SHA256 Fingerprint: d4:3a:f9:b3:54:73:75:5c:96:84:fc:06:d7:d8:cb:70:ee:5c:28:e7:73:fb:29:4e:b4:1e:e7:17:22:92:4d:24 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQT9Irj/VkyDOeTzRYZiNwYDANBgkqhkiG9w0BAQsFADBH +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNVBAMMHFVDQSBF +eHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwHhcNMTUwMzEzMDAwMDAwWhcNMzgxMjMx +MDAwMDAwWjBHMQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNV +BAMMHFVDQSBFeHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCpCQcoEwKwmeBkqh5DFnpzsZGgdT6o+uM4AHrsiWog +D4vFsJszA1qGxliG1cGFu0/GnEBNyr7uaZa4rYEwmnySBesFK5pI0Lh2PpbIILvS +sPGP2KxFRv+qZ2C0d35qHzwaUnoEPQc8hQ2E0B92CvdqFN9y4zR8V05WAT558aop +O2z6+I9tTcg1367r3CTueUWnhbYFiN6IXSV8l2RnCdm/WhUFhvMJHuxYMjMR83dk +sHYf5BA1FxvyDrFspCqjc/wJHx4yGVMR59mzLC52LqGj3n5qiAno8geK+LLNEOfi +c0CTuwjRP+H8C5SzJe98ptfRr5//lpr1kXuYC3fUfugH0mK1lTnj8/FtDw5lhIpj +VMWAtuCeS31HJqcBCF3RiJ7XwzJE+oJKCmhUfzhTA8ykADNkUVkLo4KRel7sFsLz +KuZi2irbWWIQJUoqgQtHB0MGcIfS+pMRKXpITeuUx3BNr2fVUbGAIAEBtHoIppB/ +TuDvB0GHr2qlXov7z1CymlSvw4m6WC31MJixNnI5fkkE/SmnTHnkBVfblLkWU41G +sx2VYVdWf6/wFlthWG82UBEL2KwrlRYaDh8IzTY0ZRBiZtWAXxQgXy0MoHgKaNYs +1+lvK9JKBZP8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0YIrO7o1dfdRUVjzyAfd5LQD +fwIDAQABo0IwQDAdBgNVHQ4EFgQU2XQ65DA9DfcS3H5aBZ8eNJr34RQwDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBADaN +l8xCFWQpN5smLNb7rhVpLGsaGvdftvkHTFnq88nIua7Mui563MD1sC3AO6+fcAUR +ap8lTwEpcOPlDOHqWnzcSbvBHiqB9RZLcpHIojG5qtr8nR/zXUACE/xOHAbKsxSQ +VBcZEhrxH9cMaVr2cXj0lH2RC47skFSOvG+hTKv8dGT9cZr4QQehzZHkPJrgmzI5 +c6sq1WnIeJEmMX3ixzDx/BR4dxIOE/TdFpS/S2d7cFOFyrC78zhNLJA5wA3CXWvp +4uXViI3WLL+rG761KIcSF3Ru/H38j9CHJrAb+7lsq+KePRXBOy5nAliRn+/4Qh8s +t2j1da3Ptfb/EX3C8CSlrdP6oDyp+l3cpaDvRKS+1ujl5BOWF3sGPjLtx7dCvHaj +2GU4Kzg1USEODm8uNBNA4StnDG1KQTAYI1oyVZnJF+A83vbsea0rWBmirSwiGpWO +vpaQXUJXxPkUAzUrHC1RVwinOt4/5Mi0A3PCwSaAuwtCH60NryZy2sy+s6ODWA2C +xR9GUeOcGMyNm43sSet1UNWMKFnKdDTajAshqx7qG+XH/RU+wBeq+yNuJkbL+vmx +cmtpzyKEC2IPrNkZAJSidjzULZrtBJ4tBmIQN1IchXIbJ+XMxjHsN+xjWZsLHXbM +fjKaiJUINlK73nZfdklJrX+9ZSCyycErdhh2n1ax +-----END CERTIFICATE----- + +# Issuer: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Subject: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Label: "Certigna Root CA" +# Serial: 269714418870597844693661054334862075617 +# MD5 Fingerprint: 0e:5c:30:62:27:eb:5b:bc:d7:ae:62:ba:e9:d5:df:77 +# SHA1 Fingerprint: 2d:0d:52:14:ff:9e:ad:99:24:01:74:20:47:6e:6c:85:27:27:f5:43 +# SHA256 Fingerprint: d4:8d:3d:23:ee:db:50:a4:59:e5:51:97:60:1c:27:77:4b:9d:7b:18:c9:4d:5a:05:95:11:a1:02:50:b9:31:68 +-----BEGIN CERTIFICATE----- +MIIGWzCCBEOgAwIBAgIRAMrpG4nxVQMNo+ZBbcTjpuEwDQYJKoZIhvcNAQELBQAw +WjELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCURoaW15b3RpczEcMBoGA1UECwwTMDAw +MiA0ODE0NjMwODEwMDAzNjEZMBcGA1UEAwwQQ2VydGlnbmEgUm9vdCBDQTAeFw0x +MzEwMDEwODMyMjdaFw0zMzEwMDEwODMyMjdaMFoxCzAJBgNVBAYTAkZSMRIwEAYD +VQQKDAlEaGlteW90aXMxHDAaBgNVBAsMEzAwMDIgNDgxNDYzMDgxMDAwMzYxGTAX +BgNVBAMMEENlcnRpZ25hIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQDNGDllGlmx6mQWDoyUJJV8g9PFOSbcDO8WV43X2KyjQn+Cyu3NW9sO +ty3tRQgXstmzy9YXUnIo245Onoq2C/mehJpNdt4iKVzSs9IGPjA5qXSjklYcoW9M +CiBtnyN6tMbaLOQdLNyzKNAT8kxOAkmhVECe5uUFoC2EyP+YbNDrihqECB63aCPu +I9Vwzm1RaRDuoXrC0SIxwoKF0vJVdlB8JXrJhFwLrN1CTivngqIkicuQstDuI7pm +TLtipPlTWmR7fJj6o0ieD5Wupxj0auwuA0Wv8HT4Ks16XdG+RCYyKfHx9WzMfgIh +C59vpD++nVPiz32pLHxYGpfhPTc3GGYo0kDFUYqMwy3OU4gkWGQwFsWq4NYKpkDf +ePb1BHxpE4S80dGnBs8B92jAqFe7OmGtBIyT46388NtEbVncSVmurJqZNjBBe3Yz +IoejwpKGbvlw7q6Hh5UbxHq9MfPU0uWZ/75I7HX1eBYdpnDBfzwboZL7z8g81sWT +Co/1VTp2lc5ZmIoJlXcymoO6LAQ6l73UL77XbJuiyn1tJslV1c/DeVIICZkHJC1k +JWumIWmbat10TWuXekG9qxf5kBdIjzb5LdXF2+6qhUVB+s06RbFo5jZMm5BX7CO5 +hwjCxAnxl4YqKE3idMDaxIzb3+KhF1nOJFl0Mdp//TBt2dzhauH8XwIDAQABo4IB +GjCCARYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FBiHVuBud+4kNTxOc5of1uHieX4rMB8GA1UdIwQYMBaAFBiHVuBud+4kNTxOc5of +1uHieX4rMEQGA1UdIAQ9MDswOQYEVR0gADAxMC8GCCsGAQUFBwIBFiNodHRwczov +L3d3d3cuY2VydGlnbmEuZnIvYXV0b3JpdGVzLzBtBgNVHR8EZjBkMC+gLaArhilo +dHRwOi8vY3JsLmNlcnRpZ25hLmZyL2NlcnRpZ25hcm9vdGNhLmNybDAxoC+gLYYr +aHR0cDovL2NybC5kaGlteW90aXMuY29tL2NlcnRpZ25hcm9vdGNhLmNybDANBgkq +hkiG9w0BAQsFAAOCAgEAlLieT/DjlQgi581oQfccVdV8AOItOoldaDgvUSILSo3L +6btdPrtcPbEo/uRTVRPPoZAbAh1fZkYJMyjhDSSXcNMQH+pkV5a7XdrnxIxPTGRG +HVyH41neQtGbqH6mid2PHMkwgu07nM3A6RngatgCdTer9zQoKJHyBApPNeNgJgH6 +0BGM+RFq7q89w1DTj18zeTyGqHNFkIwgtnJzFyO+B2XleJINugHA64wcZr+shncB +lA2c5uk5jR+mUYyZDDl34bSb+hxnV29qao6pK0xXeXpXIs/NX2NGjVxZOob4Mkdi +o2cNGJHc+6Zr9UhhcyNZjgKnvETq9Emd8VRY+WCv2hikLyhF3HqgiIZd8zvn/yk1 +gPxkQ5Tm4xxvvq0OKmOZK8l+hfZx6AYDlf7ej0gcWtSS6Cvu5zHbugRqh5jnxV/v +faci9wHYTfmJ0A6aBVmknpjZbyvKcL5kwlWj9Omvw5Ip3IgWJJk8jSaYtlu3zM63 +Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh +jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw +3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0= +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign Root CA - G1" +# Serial: 235931866688319308814040 +# MD5 Fingerprint: 9c:42:84:57:dd:cb:0b:a7:2e:95:ad:b6:f3:da:bc:ac +# SHA1 Fingerprint: 8a:c7:ad:8f:73:ac:4e:c1:b5:75:4d:a5:40:f4:fc:cf:7c:b5:8e:8c +# SHA256 Fingerprint: 40:f6:af:03:46:a9:9a:a1:cd:1d:55:5a:4e:9c:ce:62:c7:f9:63:46:03:ee:40:66:15:83:3d:c8:c8:d0:03:67 +-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD +VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU +ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH +MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO +MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv +Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz +f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO +8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq +d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM +tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt +Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB +o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD +AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x +PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM +wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d +GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH +6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby +RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx +iN66zB+Afko= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign ECC Root CA - G3" +# Serial: 287880440101571086945156 +# MD5 Fingerprint: ce:0b:72:d1:9f:88:8e:d0:50:03:e8:e3:b8:8b:67:40 +# SHA1 Fingerprint: 30:43:fa:4f:f2:57:dc:a0:c3:80:ee:2e:58:ea:78:b2:3f:e6:bb:c1 +# SHA256 Fingerprint: 86:a1:ec:ba:08:9c:4a:8d:3b:be:27:34:c6:12:ba:34:1d:81:3e:04:3c:f9:e8:a8:62:cd:5c:57:a3:6b:be:6b +-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG +EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo +bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g +RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ +TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s +b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw +djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 +WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS +fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB +CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD ++JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Label: "emSign Root CA - C1" +# Serial: 825510296613316004955058 +# MD5 Fingerprint: d8:e3:5d:01:21:fa:78:5a:b0:df:ba:d2:ee:2a:5f:68 +# SHA1 Fingerprint: e7:2e:f1:df:fc:b2:09:28:cf:5d:d4:d5:67:37:b1:51:cb:86:4f:01 +# SHA256 Fingerprint: 12:56:09:aa:30:1d:a0:a2:49:b9:7a:82:39:cb:6a:34:21:6f:44:dc:ac:9f:39:54:b1:42:92:f2:e8:c8:60:8f +-----BEGIN CERTIFICATE----- +MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG +A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg +SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v +dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ +BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ +HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH +3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH +GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c +xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1 +aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq +TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87 +/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4 +kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG +YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT ++xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo +WXzhriKi4gp6D/piq1JM4fHfyr6DDUI= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Label: "emSign ECC Root CA - C3" +# Serial: 582948710642506000014504 +# MD5 Fingerprint: 3e:53:b3:a3:81:ee:d7:10:f8:d3:b0:1d:17:92:f5:d5 +# SHA1 Fingerprint: b6:af:43:c2:9b:81:53:7d:f6:ef:6b:c3:1f:1f:60:15:0c:ee:48:66 +# SHA256 Fingerprint: bc:4d:80:9b:15:18:9d:78:db:3e:1d:8c:f4:f9:72:6a:79:5d:a1:64:3c:a5:f1:35:8e:1d:db:0e:dc:0d:7e:b3 +-----BEGIN CERTIFICATE----- +MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG +EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx +IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND +IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci +MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti +sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O +BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB +Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c +3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J +0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ== +-----END CERTIFICATE----- + +# Issuer: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Subject: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Label: "Hongkong Post Root CA 3" +# Serial: 46170865288971385588281144162979347873371282084 +# MD5 Fingerprint: 11:fc:9f:bd:73:30:02:8a:fd:3f:f3:58:b9:cb:20:f0 +# SHA1 Fingerprint: 58:a2:d0:ec:20:52:81:5b:c1:f3:f8:64:02:24:4e:c2:8e:02:4b:02 +# SHA256 Fingerprint: 5a:2f:c0:3f:0c:83:b0:90:bb:fa:40:60:4b:09:88:44:6c:76:36:18:3d:f9:84:6e:17:10:1a:44:7f:b8:ef:d6 +-----BEGIN CERTIFICATE----- +MIIFzzCCA7egAwIBAgIUCBZfikyl7ADJk0DfxMauI7gcWqQwDQYJKoZIhvcNAQEL +BQAwbzELMAkGA1UEBhMCSEsxEjAQBgNVBAgTCUhvbmcgS29uZzESMBAGA1UEBxMJ +SG9uZyBLb25nMRYwFAYDVQQKEw1Ib25na29uZyBQb3N0MSAwHgYDVQQDExdIb25n +a29uZyBQb3N0IFJvb3QgQ0EgMzAeFw0xNzA2MDMwMjI5NDZaFw00MjA2MDMwMjI5 +NDZaMG8xCzAJBgNVBAYTAkhLMRIwEAYDVQQIEwlIb25nIEtvbmcxEjAQBgNVBAcT +CUhvbmcgS29uZzEWMBQGA1UEChMNSG9uZ2tvbmcgUG9zdDEgMB4GA1UEAxMXSG9u +Z2tvbmcgUG9zdCBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCziNfqzg8gTr7m1gNt7ln8wlffKWihgw4+aMdoWJwcYEuJQwy51BWy7sFO +dem1p+/l6TWZ5Mwc50tfjTMwIDNT2aa71T4Tjukfh0mtUC1Qyhi+AViiE3CWu4mI +VoBc+L0sPOFMV4i707mV78vH9toxdCim5lSJ9UExyuUmGs2C4HDaOym71QP1mbpV +9WTRYA6ziUm4ii8F0oRFKHyPaFASePwLtVPLwpgchKOesL4jpNrcyCse2m5FHomY +2vkALgbpDDtw1VAliJnLzXNg99X/NWfFobxeq81KuEXryGgeDQ0URhLj0mRiikKY +vLTGCAj4/ahMZJx2Ab0vqWwzD9g/KLg8aQFChn5pwckGyuV6RmXpwtZQQS4/t+Tt +bNe/JgERohYpSms0BpDsE9K2+2p20jzt8NYt3eEV7KObLyzJPivkaTv/ciWxNoZb +x39ri1UbSsUgYT2uy1DhCDq+sI9jQVMwCFk8mB13umOResoQUGC/8Ne8lYePl8X+ +l2oBlKN8W4UdKjk60FSh0Tlxnf0h+bV78OLgAo9uliQlLKAeLKjEiafv7ZkGL7YK +TE/bosw3Gq9HhS2KX8Q0NEwA/RiTZxPRN+ZItIsGxVd7GYYKecsAyVKvQv83j+Gj +Hno9UKtjBucVtT+2RTeUN7F+8kjDf8V1/peNRY8apxpyKBpADwIDAQABo2MwYTAP +BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBQXnc0e +i9Y5K3DTXNSguB+wAPzFYTAdBgNVHQ4EFgQUF53NHovWOStw01zUoLgfsAD8xWEw +DQYJKoZIhvcNAQELBQADggIBAFbVe27mIgHSQpsY1Q7XZiNc4/6gx5LS6ZStS6LG +7BJ8dNVI0lkUmcDrudHr9EgwW62nV3OZqdPlt9EuWSRY3GguLmLYauRwCy0gUCCk +MpXRAJi70/33MvJJrsZ64Ee+bs7Lo3I6LWldy8joRTnU+kLBEUx3XZL7av9YROXr +gZ6voJmtvqkBZss4HTzfQx/0TW60uhdG/H39h4F5ag0zD/ov+BS5gLNdTaqX4fnk +GMX41TiMJjz98iji7lpJiCzfeT2OnpA8vUFKOt1b9pq0zj8lMH8yfaIDlNDceqFS +3m6TjRgm/VWsvY+b0s+v54Ysyx8Jb6NvqYTUc79NoXQbTiNg8swOqn+knEwlqLJm +Ozj/2ZQw9nKEvmhVEA/GcywWaZMH/rFF7buiVWqw2rVKAiUnhde3t4ZEFolsgCs+ +l6mc1X5VTMbeRRAc6uk7nwNT7u56AQIWeNTowr5GdogTPyK7SBIdUgC0An4hGh6c +JfTzPV4e0hz5sy229zdcxsshTrD3mUcYhcErulWuBurQB7Lcq9CClnXO0lD+mefP +L5/ndtFhKvshuzHQqp9HpLIiyhY6UFfEW0NnxWViA0kB60PZ2Pierc+xYw5F9KBa +LJstxabArahH9CdMOA0uG0k7UvToiIMrVCjU8jVStDKDYmlkDJGcn5fqdBb9HxEG +mpv0 +-----END CERTIFICATE----- + +# Issuer: CN=Microsoft ECC Root Certificate Authority 2017 O=Microsoft Corporation +# Subject: CN=Microsoft ECC Root Certificate Authority 2017 O=Microsoft Corporation +# Label: "Microsoft ECC Root Certificate Authority 2017" +# Serial: 136839042543790627607696632466672567020 +# MD5 Fingerprint: dd:a1:03:e6:4a:93:10:d1:bf:f0:19:42:cb:fe:ed:67 +# SHA1 Fingerprint: 99:9a:64:c3:7f:f4:7d:9f:ab:95:f1:47:69:89:14:60:ee:c4:c3:c5 +# SHA256 Fingerprint: 35:8d:f3:9d:76:4a:f9:e1:b7:66:e9:c9:72:df:35:2e:e1:5c:fa:c2:27:af:6a:d1:d7:0e:8e:4a:6e:dc:ba:02 +-----BEGIN CERTIFICATE----- +MIICWTCCAd+gAwIBAgIQZvI9r4fei7FK6gxXMQHC7DAKBggqhkjOPQQDAzBlMQsw +CQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYD +VQQDEy1NaWNyb3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIw +MTcwHhcNMTkxMjE4MjMwNjQ1WhcNNDIwNzE4MjMxNjA0WjBlMQswCQYDVQQGEwJV +UzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1NaWNy +b3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwdjAQBgcq +hkjOPQIBBgUrgQQAIgNiAATUvD0CQnVBEyPNgASGAlEvaqiBYgtlzPbKnR5vSmZR +ogPZnZH6thaxjG7efM3beaYvzrvOcS/lpaso7GMEZpn4+vKTEAXhgShC48Zo9OYb +hGBKia/teQ87zvH2RPUBeMCjVDBSMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8E +BTADAQH/MB0GA1UdDgQWBBTIy5lycFIM+Oa+sgRXKSrPQhDtNTAQBgkrBgEEAYI3 +FQEEAwIBADAKBggqhkjOPQQDAwNoADBlAjBY8k3qDPlfXu5gKcs68tvWMoQZP3zV +L8KxzJOuULsJMsbG7X7JNpQS5GiFBqIb0C8CMQCZ6Ra0DvpWSNSkMBaReNtUjGUB +iudQZsIxtzm6uBoiB078a1QWIP8rtedMDE2mT3M= +-----END CERTIFICATE----- + +# Issuer: CN=Microsoft RSA Root Certificate Authority 2017 O=Microsoft Corporation +# Subject: CN=Microsoft RSA Root Certificate Authority 2017 O=Microsoft Corporation +# Label: "Microsoft RSA Root Certificate Authority 2017" +# Serial: 40975477897264996090493496164228220339 +# MD5 Fingerprint: 10:ff:00:ff:cf:c9:f8:c7:7a:c0:ee:35:8e:c9:0f:47 +# SHA1 Fingerprint: 73:a5:e6:4a:3b:ff:83:16:ff:0e:dc:cc:61:8a:90:6e:4e:ae:4d:74 +# SHA256 Fingerprint: c7:41:f7:0f:4b:2a:8d:88:bf:2e:71:c1:41:22:ef:53:ef:10:eb:a0:cf:a5:e6:4c:fa:20:f4:18:85:30:73:e0 +-----BEGIN CERTIFICATE----- +MIIFqDCCA5CgAwIBAgIQHtOXCV/YtLNHcB6qvn9FszANBgkqhkiG9w0BAQwFADBl +MQswCQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYw +NAYDVQQDEy1NaWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 +IDIwMTcwHhcNMTkxMjE4MjI1MTIyWhcNNDIwNzE4MjMwMDIzWjBlMQswCQYDVQQG +EwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1N +aWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDKW76UM4wplZEWCpW9R2LBifOZ +Nt9GkMml7Xhqb0eRaPgnZ1AzHaGm++DlQ6OEAlcBXZxIQIJTELy/xztokLaCLeX0 +ZdDMbRnMlfl7rEqUrQ7eS0MdhweSE5CAg2Q1OQT85elss7YfUJQ4ZVBcF0a5toW1 +HLUX6NZFndiyJrDKxHBKrmCk3bPZ7Pw71VdyvD/IybLeS2v4I2wDwAW9lcfNcztm +gGTjGqwu+UcF8ga2m3P1eDNbx6H7JyqhtJqRjJHTOoI+dkC0zVJhUXAoP8XFWvLJ +jEm7FFtNyP9nTUwSlq31/niol4fX/V4ggNyhSyL71Imtus5Hl0dVe49FyGcohJUc +aDDv70ngNXtk55iwlNpNhTs+VcQor1fznhPbRiefHqJeRIOkpcrVE7NLP8TjwuaG +YaRSMLl6IE9vDzhTyzMMEyuP1pq9KsgtsRx9S1HKR9FIJ3Jdh+vVReZIZZ2vUpC6 +W6IYZVcSn2i51BVrlMRpIpj0M+Dt+VGOQVDJNE92kKz8OMHY4Xu54+OU4UZpyw4K +UGsTuqwPN1q3ErWQgR5WrlcihtnJ0tHXUeOrO8ZV/R4O03QK0dqq6mm4lyiPSMQH ++FJDOvTKVTUssKZqwJz58oHhEmrARdlns87/I6KJClTUFLkqqNfs+avNJVgyeY+Q +W5g5xAgGwax/Dj0ApQIDAQABo1QwUjAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/ +BAUwAwEB/zAdBgNVHQ4EFgQUCctZf4aycI8awznjwNnpv7tNsiMwEAYJKwYBBAGC +NxUBBAMCAQAwDQYJKoZIhvcNAQEMBQADggIBAKyvPl3CEZaJjqPnktaXFbgToqZC +LgLNFgVZJ8og6Lq46BrsTaiXVq5lQ7GPAJtSzVXNUzltYkyLDVt8LkS/gxCP81OC +gMNPOsduET/m4xaRhPtthH80dK2Jp86519efhGSSvpWhrQlTM93uCupKUY5vVau6 +tZRGrox/2KJQJWVggEbbMwSubLWYdFQl3JPk+ONVFT24bcMKpBLBaYVu32TxU5nh +SnUgnZUP5NbcA/FZGOhHibJXWpS2qdgXKxdJ5XbLwVaZOjex/2kskZGT4d9Mozd2 +TaGf+G0eHdP67Pv0RR0Tbc/3WeUiJ3IrhvNXuzDtJE3cfVa7o7P4NHmJweDyAmH3 +pvwPuxwXC65B2Xy9J6P9LjrRk5Sxcx0ki69bIImtt2dmefU6xqaWM/5TkshGsRGR +xpl/j8nWZjEgQRCHLQzWwa80mMpkg/sTV9HB8Dx6jKXB/ZUhoHHBk2dxEuqPiApp +GWSZI1b7rCoucL5mxAyE7+WL85MB+GqQk2dLsmijtWKP6T+MejteD+eMuMZ87zf9 +dOLITzNy4ZQ5bb0Sr74MTnB8G2+NszKTc0QWbej09+CVgI+WXTik9KveCjCHk9hN +AHFiRSdLOkKEW39lt2c0Ui2cFmuqqNh7o0JMcccMyj6D5KbvtwEwXlGjefVwaaZB +RA+GsCyRxj3qrg+E +-----END CERTIFICATE----- + +# Issuer: CN=e-Szigno Root CA 2017 O=Microsec Ltd. +# Subject: CN=e-Szigno Root CA 2017 O=Microsec Ltd. +# Label: "e-Szigno Root CA 2017" +# Serial: 411379200276854331539784714 +# MD5 Fingerprint: de:1f:f6:9e:84:ae:a7:b4:21:ce:1e:58:7d:d1:84:98 +# SHA1 Fingerprint: 89:d4:83:03:4f:9e:9a:48:80:5f:72:37:d4:a9:a6:ef:cb:7c:1f:d1 +# SHA256 Fingerprint: be:b0:0b:30:83:9b:9b:c3:2c:32:e4:44:79:05:95:06:41:f2:64:21:b1:5e:d0:89:19:8b:51:8a:e2:ea:1b:99 +-----BEGIN CERTIFICATE----- +MIICQDCCAeWgAwIBAgIMAVRI7yH9l1kN9QQKMAoGCCqGSM49BAMCMHExCzAJBgNV +BAYTAkhVMREwDwYDVQQHDAhCdWRhcGVzdDEWMBQGA1UECgwNTWljcm9zZWMgTHRk +LjEXMBUGA1UEYQwOVkFUSFUtMjM1ODQ0OTcxHjAcBgNVBAMMFWUtU3ppZ25vIFJv +b3QgQ0EgMjAxNzAeFw0xNzA4MjIxMjA3MDZaFw00MjA4MjIxMjA3MDZaMHExCzAJ +BgNVBAYTAkhVMREwDwYDVQQHDAhCdWRhcGVzdDEWMBQGA1UECgwNTWljcm9zZWMg +THRkLjEXMBUGA1UEYQwOVkFUSFUtMjM1ODQ0OTcxHjAcBgNVBAMMFWUtU3ppZ25v +IFJvb3QgQ0EgMjAxNzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABJbcPYrYsHtv +xie+RJCxs1YVe45DJH0ahFnuY2iyxl6H0BVIHqiQrb1TotreOpCmYF9oMrWGQd+H +Wyx7xf58etqjYzBhMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0G +A1UdDgQWBBSHERUI0arBeAyxr87GyZDvvzAEwDAfBgNVHSMEGDAWgBSHERUI0arB +eAyxr87GyZDvvzAEwDAKBggqhkjOPQQDAgNJADBGAiEAtVfd14pVCzbhhkT61Nlo +jbjcI4qKDdQvfepz7L9NbKgCIQDLpbQS+ue16M9+k/zzNY9vTlp8tLxOsvxyqltZ ++efcMQ== +-----END CERTIFICATE----- + +# Issuer: O=CERTSIGN SA OU=certSIGN ROOT CA G2 +# Subject: O=CERTSIGN SA OU=certSIGN ROOT CA G2 +# Label: "certSIGN Root CA G2" +# Serial: 313609486401300475190 +# MD5 Fingerprint: 8c:f1:75:8a:c6:19:cf:94:b7:f7:65:20:87:c3:97:c7 +# SHA1 Fingerprint: 26:f9:93:b4:ed:3d:28:27:b0:b9:4b:a7:e9:15:1d:a3:8d:92:e5:32 +# SHA256 Fingerprint: 65:7c:fe:2f:a7:3f:aa:38:46:25:71:f3:32:a2:36:3a:46:fc:e7:02:09:51:71:07:02:cd:fb:b6:ee:da:33:05 +-----BEGIN CERTIFICATE----- +MIIFRzCCAy+gAwIBAgIJEQA0tk7GNi02MA0GCSqGSIb3DQEBCwUAMEExCzAJBgNV +BAYTAlJPMRQwEgYDVQQKEwtDRVJUU0lHTiBTQTEcMBoGA1UECxMTY2VydFNJR04g +Uk9PVCBDQSBHMjAeFw0xNzAyMDYwOTI3MzVaFw00MjAyMDYwOTI3MzVaMEExCzAJ +BgNVBAYTAlJPMRQwEgYDVQQKEwtDRVJUU0lHTiBTQTEcMBoGA1UECxMTY2VydFNJ +R04gUk9PVCBDQSBHMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMDF +dRmRfUR0dIf+DjuW3NgBFszuY5HnC2/OOwppGnzC46+CjobXXo9X69MhWf05N0Iw +vlDqtg+piNguLWkh59E3GE59kdUWX2tbAMI5Qw02hVK5U2UPHULlj88F0+7cDBrZ +uIt4ImfkabBoxTzkbFpG583H+u/E7Eu9aqSs/cwoUe+StCmrqzWaTOTECMYmzPhp +n+Sc8CnTXPnGFiWeI8MgwT0PPzhAsP6CRDiqWhqKa2NYOLQV07YRaXseVO6MGiKs +cpc/I1mbySKEwQdPzH/iV8oScLumZfNpdWO9lfsbl83kqK/20U6o2YpxJM02PbyW +xPFsqa7lzw1uKA2wDrXKUXt4FMMgL3/7FFXhEZn91QqhngLjYl/rNUssuHLoPj1P +rCy7Lobio3aP5ZMqz6WryFyNSwb/EkaseMsUBzXgqd+L6a8VTxaJW732jcZZroiF +DsGJ6x9nxUWO/203Nit4ZoORUSs9/1F3dmKh7Gc+PoGD4FapUB8fepmrY7+EF3fx +DTvf95xhszWYijqy7DwaNz9+j5LP2RIUZNoQAhVB/0/E6xyjyfqZ90bp4RjZsbgy +LcsUDFDYg2WD7rlcz8sFWkz6GZdr1l0T08JcVLwyc6B49fFtHsufpaafItzRUZ6C +eWRgKRM+o/1Pcmqr4tTluCRVLERLiohEnMqE0yo7AgMBAAGjQjBAMA8GA1UdEwEB +/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSCIS1mxteg4BXrzkwJ +d8RgnlRuAzANBgkqhkiG9w0BAQsFAAOCAgEAYN4auOfyYILVAzOBywaK8SJJ6ejq +kX/GM15oGQOGO0MBzwdw5AgeZYWR5hEit/UCI46uuR59H35s5r0l1ZUa8gWmr4UC +b6741jH/JclKyMeKqdmfS0mbEVeZkkMR3rYzpMzXjWR91M08KCy0mpbqTfXERMQl +qiCA2ClV9+BB/AYm/7k29UMUA2Z44RGx2iBfRgB4ACGlHgAoYXhvqAEBj500mv/0 +OJD7uNGzcgbJceaBxXntC6Z58hMLnPddDnskk7RI24Zf3lCGeOdA5jGokHZwYa+c +NywRtYK3qq4kNFtyDGkNzVmf9nGvnAvRCjj5BiKDUyUM/FHE5r7iOZULJK2v0ZXk +ltd0ZGtxTgI8qoXzIKNDOXZbbFD+mpwUHmUUihW9o4JFWklWatKcsWMy5WHgUyIO +pwpJ6st+H6jiYoD2EEVSmAYY3qXNL3+q1Ok+CHLsIwMCPKaq2LxndD0UF/tUSxfj +03k9bWtJySgOLnRQvwzZRjoQhsmnP+mg7H/rpXdYaXHmgwo38oZJar55CJD2AhZk +PuXaTH4MNMn5X7azKFGnpyuqSfqNZSlO42sTp5SjLVFteAxEy9/eCG/Oo2Sr05WE +1LlSVHJ7liXMvGnjSG4N0MedJ5qq+BOS3R7fY581qRY27Iy4g/Q9iY/NtBde17MX +QRBdJ3NghVdJIgc= +-----END CERTIFICATE----- + +# Issuer: CN=NAVER Global Root Certification Authority O=NAVER BUSINESS PLATFORM Corp. +# Subject: CN=NAVER Global Root Certification Authority O=NAVER BUSINESS PLATFORM Corp. +# Label: "NAVER Global Root Certification Authority" +# Serial: 9013692873798656336226253319739695165984492813 +# MD5 Fingerprint: c8:7e:41:f6:25:3b:f5:09:b3:17:e8:46:3d:bf:d0:9b +# SHA1 Fingerprint: 8f:6b:f2:a9:27:4a:da:14:a0:c4:f4:8e:61:27:f9:c0:1e:78:5d:d1 +# SHA256 Fingerprint: 88:f4:38:dc:f8:ff:d1:fa:8f:42:91:15:ff:e5:f8:2a:e1:e0:6e:0c:70:c3:75:fa:ad:71:7b:34:a4:9e:72:65 +-----BEGIN CERTIFICATE----- +MIIFojCCA4qgAwIBAgIUAZQwHqIL3fXFMyqxQ0Rx+NZQTQ0wDQYJKoZIhvcNAQEM +BQAwaTELMAkGA1UEBhMCS1IxJjAkBgNVBAoMHU5BVkVSIEJVU0lORVNTIFBMQVRG +T1JNIENvcnAuMTIwMAYDVQQDDClOQVZFUiBHbG9iYWwgUm9vdCBDZXJ0aWZpY2F0 +aW9uIEF1dGhvcml0eTAeFw0xNzA4MTgwODU4NDJaFw0zNzA4MTgyMzU5NTlaMGkx +CzAJBgNVBAYTAktSMSYwJAYDVQQKDB1OQVZFUiBCVVNJTkVTUyBQTEFURk9STSBD +b3JwLjEyMDAGA1UEAwwpTkFWRVIgR2xvYmFsIFJvb3QgQ2VydGlmaWNhdGlvbiBB +dXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC21PGTXLVA +iQqrDZBbUGOukJR0F0Vy1ntlWilLp1agS7gvQnXp2XskWjFlqxcX0TM62RHcQDaH +38dq6SZeWYp34+hInDEW+j6RscrJo+KfziFTowI2MMtSAuXaMl3Dxeb57hHHi8lE +HoSTGEq0n+USZGnQJoViAbbJAh2+g1G7XNr4rRVqmfeSVPc0W+m/6imBEtRTkZaz +kVrd/pBzKPswRrXKCAfHcXLJZtM0l/aM9BhK4dA9WkW2aacp+yPOiNgSnABIqKYP +szuSjXEOdMWLyEz59JuOuDxp7W87UC9Y7cSw0BwbagzivESq2M0UXZR4Yb8Obtoq +vC8MC3GmsxY/nOb5zJ9TNeIDoKAYv7vxvvTWjIcNQvcGufFt7QSUqP620wbGQGHf +nZ3zVHbOUzoBppJB7ASjjw2i1QnK1sua8e9DXcCrpUHPXFNwcMmIpi3Ua2FzUCaG +YQ5fG8Ir4ozVu53BA0K6lNpfqbDKzE0K70dpAy8i+/Eozr9dUGWokG2zdLAIx6yo +0es+nPxdGoMuK8u180SdOqcXYZaicdNwlhVNt0xz7hlcxVs+Qf6sdWA7G2POAN3a +CJBitOUt7kinaxeZVL6HSuOpXgRM6xBtVNbv8ejyYhbLgGvtPe31HzClrkvJE+2K +AQHJuFFYwGY6sWZLxNUxAmLpdIQM201GLQIDAQABo0IwQDAdBgNVHQ4EFgQU0p+I +36HNLL3s9TsBAZMzJ7LrYEswDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB +Af8wDQYJKoZIhvcNAQEMBQADggIBADLKgLOdPVQG3dLSLvCkASELZ0jKbY7gyKoN +qo0hV4/GPnrK21HUUrPUloSlWGB/5QuOH/XcChWB5Tu2tyIvCZwTFrFsDDUIbatj +cu3cvuzHV+YwIHHW1xDBE1UBjCpD5EHxzzp6U5LOogMFDTjfArsQLtk70pt6wKGm ++LUx5vR1yblTmXVHIloUFcd4G7ad6Qz4G3bxhYTeodoS76TiEJd6eN4MUZeoIUCL +hr0N8F5OSza7OyAfikJW4Qsav3vQIkMsRIz75Sq0bBwcupTgE34h5prCy8VCZLQe +lHsIJchxzIdFV4XTnyliIoNRlwAYl3dqmJLJfGBs32x9SuRwTMKeuB330DTHD8z7 +p/8Dvq1wkNoL3chtl1+afwkyQf3NosxabUzyqkn+Zvjp2DXrDige7kgvOtB5CTh8 +piKCk5XQA76+AqAF3SAi428diDRgxuYKuQl1C/AH6GmWNcf7I4GOODm4RStDeKLR +LBT/DShycpWbXgnbiUSYqqFJu3FS8r/2/yehNq+4tneI3TqkbZs0kNwUXTC/t+sX +5Ie3cdCh13cV1ELX8vMxmV2b3RZtP+oGI/hGoiLtk/bdmuYqh7GYVPEi92tF4+KO +dh2ajcQGjTa3FPOdVGm3jjzVpG2Tgbet9r1ke8LJaDmgkpzNNIaRkPpkUZ3+/uul +9XXeifdy +-----END CERTIFICATE----- + +# Issuer: CN=AC RAIZ FNMT-RCM SERVIDORES SEGUROS O=FNMT-RCM OU=Ceres +# Subject: CN=AC RAIZ FNMT-RCM SERVIDORES SEGUROS O=FNMT-RCM OU=Ceres +# Label: "AC RAIZ FNMT-RCM SERVIDORES SEGUROS" +# Serial: 131542671362353147877283741781055151509 +# MD5 Fingerprint: 19:36:9c:52:03:2f:d2:d1:bb:23:cc:dd:1e:12:55:bb +# SHA1 Fingerprint: 62:ff:d9:9e:c0:65:0d:03:ce:75:93:d2:ed:3f:2d:32:c9:e3:e5:4a +# SHA256 Fingerprint: 55:41:53:b1:3d:2c:f9:dd:b7:53:bf:be:1a:4e:0a:e0:8d:0a:a4:18:70:58:fe:60:a2:b8:62:b2:e4:b8:7b:cb +-----BEGIN CERTIFICATE----- +MIICbjCCAfOgAwIBAgIQYvYybOXE42hcG2LdnC6dlTAKBggqhkjOPQQDAzB4MQsw +CQYDVQQGEwJFUzERMA8GA1UECgwIRk5NVC1SQ00xDjAMBgNVBAsMBUNlcmVzMRgw +FgYDVQRhDA9WQVRFUy1RMjgyNjAwNEoxLDAqBgNVBAMMI0FDIFJBSVogRk5NVC1S +Q00gU0VSVklET1JFUyBTRUdVUk9TMB4XDTE4MTIyMDA5MzczM1oXDTQzMTIyMDA5 +MzczM1oweDELMAkGA1UEBhMCRVMxETAPBgNVBAoMCEZOTVQtUkNNMQ4wDAYDVQQL +DAVDZXJlczEYMBYGA1UEYQwPVkFURVMtUTI4MjYwMDRKMSwwKgYDVQQDDCNBQyBS +QUlaIEZOTVQtUkNNIFNFUlZJRE9SRVMgU0VHVVJPUzB2MBAGByqGSM49AgEGBSuB +BAAiA2IABPa6V1PIyqvfNkpSIeSX0oNnnvBlUdBeh8dHsVnyV0ebAAKTRBdp20LH +sbI6GA60XYyzZl2hNPk2LEnb80b8s0RpRBNm/dfF/a82Tc4DTQdxz69qBdKiQ1oK +Um8BA06Oi6NCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD +VR0OBBYEFAG5L++/EYZg8k/QQW6rcx/n0m5JMAoGCCqGSM49BAMDA2kAMGYCMQCu +SuMrQMN0EfKVrRYj3k4MGuZdpSRea0R7/DjiT8ucRRcRTBQnJlU5dUoDzBOQn5IC +MQD6SmxgiHPz7riYYqnOK8LZiqZwMR2vsJRM60/G49HzYqc8/5MuB1xJAWdpEgJy +v+c= +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign Root R46 O=GlobalSign nv-sa +# Subject: CN=GlobalSign Root R46 O=GlobalSign nv-sa +# Label: "GlobalSign Root R46" +# Serial: 1552617688466950547958867513931858518042577 +# MD5 Fingerprint: c4:14:30:e4:fa:66:43:94:2a:6a:1b:24:5f:19:d0:ef +# SHA1 Fingerprint: 53:a2:b0:4b:ca:6b:d6:45:e6:39:8a:8e:c4:0d:d2:bf:77:c3:a2:90 +# SHA256 Fingerprint: 4f:a3:12:6d:8d:3a:11:d1:c4:85:5a:4f:80:7c:ba:d6:cf:91:9d:3a:5a:88:b0:3b:ea:2c:63:72:d9:3c:40:c9 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgISEdK7udcjGJ5AXwqdLdDfJWfRMA0GCSqGSIb3DQEBDAUA +MEYxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMRwwGgYD +VQQDExNHbG9iYWxTaWduIFJvb3QgUjQ2MB4XDTE5MDMyMDAwMDAwMFoXDTQ2MDMy +MDAwMDAwMFowRjELMAkGA1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYt +c2ExHDAaBgNVBAMTE0dsb2JhbFNpZ24gUm9vdCBSNDYwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCsrHQy6LNl5brtQyYdpokNRbopiLKkHWPd08EsCVeJ +OaFV6Wc0dwxu5FUdUiXSE2te4R2pt32JMl8Nnp8semNgQB+msLZ4j5lUlghYruQG +vGIFAha/r6gjA7aUD7xubMLL1aa7DOn2wQL7Id5m3RerdELv8HQvJfTqa1VbkNud +316HCkD7rRlr+/fKYIje2sGP1q7Vf9Q8g+7XFkyDRTNrJ9CG0Bwta/OrffGFqfUo +0q3v84RLHIf8E6M6cqJaESvWJ3En7YEtbWaBkoe0G1h6zD8K+kZPTXhc+CtI4wSE +y132tGqzZfxCnlEmIyDLPRT5ge1lFgBPGmSXZgjPjHvjK8Cd+RTyG/FWaha/LIWF +zXg4mutCagI0GIMXTpRW+LaCtfOW3T3zvn8gdz57GSNrLNRyc0NXfeD412lPFzYE ++cCQYDdF3uYM2HSNrpyibXRdQr4G9dlkbgIQrImwTDsHTUB+JMWKmIJ5jqSngiCN +I/onccnfxkF0oE32kRbcRoxfKWMxWXEM2G/CtjJ9++ZdU6Z+Ffy7dXxd7Pj2Fxzs +x2sZy/N78CsHpdlseVR2bJ0cpm4O6XkMqCNqo98bMDGfsVR7/mrLZqrcZdCinkqa +ByFrgY/bxFn63iLABJzjqls2k+g9vXqhnQt2sQvHnf3PmKgGwvgqo6GDoLclcqUC +4wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUA1yrc4GHqMywptWU4jaWSf8FmSwwDQYJKoZIhvcNAQEMBQADggIBAHx4 +7PYCLLtbfpIrXTncvtgdokIzTfnvpCo7RGkerNlFo048p9gkUbJUHJNOxO97k4Vg +JuoJSOD1u8fpaNK7ajFxzHmuEajwmf3lH7wvqMxX63bEIaZHU1VNaL8FpO7XJqti +2kM3S+LGteWygxk6x9PbTZ4IevPuzz5i+6zoYMzRx6Fcg0XERczzF2sUyQQCPtIk +pnnpHs6i58FZFZ8d4kuaPp92CC1r2LpXFNqD6v6MVenQTqnMdzGxRBF6XLE+0xRF +FRhiJBPSy03OXIPBNvIQtQ6IbbjhVp+J3pZmOUdkLG5NrmJ7v2B0GbhWrJKsFjLt +rWhV/pi60zTe9Mlhww6G9kuEYO4Ne7UyWHmRVSyBQ7N0H3qqJZ4d16GLuc1CLgSk +ZoNNiTW2bKg2SnkheCLQQrzRQDGQob4Ez8pn7fXwgNNgyYMqIgXQBztSvwyeqiv5 +u+YfjyW6hY0XHgL+XVAEV8/+LbzvXMAaq7afJMbfc2hIkCwU9D9SGuTSyxTDYWnP +4vkYxboznxSjBF25cfe1lNj2M8FawTSLfJvdkzrnE6JwYZ+vj+vYxXX4M2bUdGc6 +N3ec592kD3ZDZopD8p/7DEJ4Y9HiD2971KE9dJeFt0g5QdYg/NA6s/rob8SKunE3 +vouXsXgxT7PntgMTzlSdriVZzH81Xwj3QEUxeCp6 +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign Root E46 O=GlobalSign nv-sa +# Subject: CN=GlobalSign Root E46 O=GlobalSign nv-sa +# Label: "GlobalSign Root E46" +# Serial: 1552617690338932563915843282459653771421763 +# MD5 Fingerprint: b5:b8:66:ed:de:08:83:e3:c9:e2:01:34:06:ac:51:6f +# SHA1 Fingerprint: 39:b4:6c:d5:fe:80:06:eb:e2:2f:4a:bb:08:33:a0:af:db:b9:dd:84 +# SHA256 Fingerprint: cb:b9:c4:4d:84:b8:04:3e:10:50:ea:31:a6:9f:51:49:55:d7:bf:d2:e2:c6:b4:93:01:01:9a:d6:1d:9f:50:58 +-----BEGIN CERTIFICATE----- +MIICCzCCAZGgAwIBAgISEdK7ujNu1LzmJGjFDYQdmOhDMAoGCCqGSM49BAMDMEYx +CzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMRwwGgYDVQQD +ExNHbG9iYWxTaWduIFJvb3QgRTQ2MB4XDTE5MDMyMDAwMDAwMFoXDTQ2MDMyMDAw +MDAwMFowRjELMAkGA1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2Ex +HDAaBgNVBAMTE0dsb2JhbFNpZ24gUm9vdCBFNDYwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAAScDrHPt+ieUnd1NPqlRqetMhkytAepJ8qUuwzSChDH2omwlwxwEwkBjtjq +R+q+soArzfwoDdusvKSGN+1wCAB16pMLey5SnCNoIwZD7JIvU4Tb+0cUB+hflGdd +yXqBPCCjQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBQxCpCPtsad0kRLgLWi5h+xEk8blTAKBggqhkjOPQQDAwNoADBlAjEA31SQ +7Zvvi5QCkxeCmb6zniz2C5GMn0oUsfZkvLtoURMMA/cVi4RguYv/Uo7njLwcAjA8 ++RHUjE7AwWHCFUyqqx0LMV87HOIAl0Qx5v5zli/altP+CAezNIm8BZ/3Hobui3A= +-----END CERTIFICATE----- + +# Issuer: CN=ANF Secure Server Root CA O=ANF Autoridad de Certificacion OU=ANF CA Raiz +# Subject: CN=ANF Secure Server Root CA O=ANF Autoridad de Certificacion OU=ANF CA Raiz +# Label: "ANF Secure Server Root CA" +# Serial: 996390341000653745 +# MD5 Fingerprint: 26:a6:44:5a:d9:af:4e:2f:b2:1d:b6:65:b0:4e:e8:96 +# SHA1 Fingerprint: 5b:6e:68:d0:cc:15:b6:a0:5f:1e:c1:5f:ae:02:fc:6b:2f:5d:6f:74 +# SHA256 Fingerprint: fb:8f:ec:75:91:69:b9:10:6b:1e:51:16:44:c6:18:c5:13:04:37:3f:6c:06:43:08:8d:8b:ef:fd:1b:99:75:99 +-----BEGIN CERTIFICATE----- +MIIF7zCCA9egAwIBAgIIDdPjvGz5a7EwDQYJKoZIhvcNAQELBQAwgYQxEjAQBgNV +BAUTCUc2MzI4NzUxMDELMAkGA1UEBhMCRVMxJzAlBgNVBAoTHkFORiBBdXRvcmlk +YWQgZGUgQ2VydGlmaWNhY2lvbjEUMBIGA1UECxMLQU5GIENBIFJhaXoxIjAgBgNV +BAMTGUFORiBTZWN1cmUgU2VydmVyIFJvb3QgQ0EwHhcNMTkwOTA0MTAwMDM4WhcN +MzkwODMwMTAwMDM4WjCBhDESMBAGA1UEBRMJRzYzMjg3NTEwMQswCQYDVQQGEwJF +UzEnMCUGA1UEChMeQU5GIEF1dG9yaWRhZCBkZSBDZXJ0aWZpY2FjaW9uMRQwEgYD +VQQLEwtBTkYgQ0EgUmFpejEiMCAGA1UEAxMZQU5GIFNlY3VyZSBTZXJ2ZXIgUm9v +dCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANvrayvmZFSVgpCj +cqQZAZ2cC4Ffc0m6p6zzBE57lgvsEeBbphzOG9INgxwruJ4dfkUyYA8H6XdYfp9q +yGFOtibBTI3/TO80sh9l2Ll49a2pcbnvT1gdpd50IJeh7WhM3pIXS7yr/2WanvtH +2Vdy8wmhrnZEE26cLUQ5vPnHO6RYPUG9tMJJo8gN0pcvB2VSAKduyK9o7PQUlrZX +H1bDOZ8rbeTzPvY1ZNoMHKGESy9LS+IsJJ1tk0DrtSOOMspvRdOoiXsezx76W0OL +zc2oD2rKDF65nkeP8Nm2CgtYZRczuSPkdxl9y0oukntPLxB3sY0vaJxizOBQ+OyR +p1RMVwnVdmPF6GUe7m1qzwmd+nxPrWAI/VaZDxUse6mAq4xhj0oHdkLePfTdsiQz +W7i1o0TJrH93PB0j7IKppuLIBkwC/qxcmZkLLxCKpvR/1Yd0DVlJRfbwcVw5Kda/ +SiOL9V8BY9KHcyi1Swr1+KuCLH5zJTIdC2MKF4EA/7Z2Xue0sUDKIbvVgFHlSFJn +LNJhiQcND85Cd8BEc5xEUKDbEAotlRyBr+Qc5RQe8TZBAQIvfXOn3kLMTOmJDVb3 +n5HUA8ZsyY/b2BzgQJhdZpmYgG4t/wHFzstGH6wCxkPmrqKEPMVOHj1tyRRM4y5B +u8o5vzY8KhmqQYdOpc5LMnndkEl/AgMBAAGjYzBhMB8GA1UdIwQYMBaAFJxf0Gxj +o1+TypOYCK2Mh6UsXME3MB0GA1UdDgQWBBScX9BsY6Nfk8qTmAitjIelLFzBNzAO +BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC +AgEATh65isagmD9uw2nAalxJUqzLK114OMHVVISfk/CHGT0sZonrDUL8zPB1hT+L +9IBdeeUXZ701guLyPI59WzbLWoAAKfLOKyzxj6ptBZNscsdW699QIyjlRRA96Gej +rw5VD5AJYu9LWaL2U/HANeQvwSS9eS9OICI7/RogsKQOLHDtdD+4E5UGUcjohybK +pFtqFiGS3XNgnhAY3jyB6ugYw3yJ8otQPr0R4hUDqDZ9MwFsSBXXiJCZBMXM5gf0 +vPSQ7RPi6ovDj6MzD8EpTBNO2hVWcXNyglD2mjN8orGoGjR0ZVzO0eurU+AagNjq +OknkJjCb5RyKqKkVMoaZkgoQI1YS4PbOTOK7vtuNknMBZi9iPrJyJ0U27U1W45eZ +/zo1PqVUSlJZS2Db7v54EX9K3BR5YLZrZAPbFYPhor72I5dQ8AkzNqdxliXzuUJ9 +2zg/LFis6ELhDtjTO0wugumDLmsx2d1Hhk9tl5EuT+IocTUW0fJz/iUrB0ckYyfI ++PbZa/wSMVYIwFNCr5zQM378BvAxRAMU8Vjq8moNqRGyg77FGr8H6lnco4g175x2 +MjxNBiLOFeXdntiP2t7SxDnlF4HPOEfrf4htWRvfn0IUrn7PqLBmZdo3r5+qPeoo +tt7VMVgWglvquxl1AnMaykgaIZOQCo6ThKd9OyMYkomgjaw= +-----END CERTIFICATE----- + +# Issuer: CN=Certum EC-384 CA O=Asseco Data Systems S.A. OU=Certum Certification Authority +# Subject: CN=Certum EC-384 CA O=Asseco Data Systems S.A. OU=Certum Certification Authority +# Label: "Certum EC-384 CA" +# Serial: 160250656287871593594747141429395092468 +# MD5 Fingerprint: b6:65:b3:96:60:97:12:a1:ec:4e:e1:3d:a3:c6:c9:f1 +# SHA1 Fingerprint: f3:3e:78:3c:ac:df:f4:a2:cc:ac:67:55:69:56:d7:e5:16:3c:e1:ed +# SHA256 Fingerprint: 6b:32:80:85:62:53:18:aa:50:d1:73:c9:8d:8b:da:09:d5:7e:27:41:3d:11:4c:f7:87:a0:f5:d0:6c:03:0c:f6 +-----BEGIN CERTIFICATE----- +MIICZTCCAeugAwIBAgIQeI8nXIESUiClBNAt3bpz9DAKBggqhkjOPQQDAzB0MQsw +CQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBTLkEuMScw +JQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxGTAXBgNVBAMT +EENlcnR1bSBFQy0zODQgQ0EwHhcNMTgwMzI2MDcyNDU0WhcNNDMwMzI2MDcyNDU0 +WjB0MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBT +LkEuMScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxGTAX +BgNVBAMTEENlcnR1bSBFQy0zODQgQ0EwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAATE +KI6rGFtqvm5kN2PkzeyrOvfMobgOgknXhimfoZTy42B4mIF4Bk3y7JoOV2CDn7Tm +Fy8as10CW4kjPMIRBSqniBMY81CE1700LCeJVf/OTOffph8oxPBUw7l8t1Ot68Kj +QjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI0GZnQkdjrzife81r1HfS+8 +EF9LMA4GA1UdDwEB/wQEAwIBBjAKBggqhkjOPQQDAwNoADBlAjADVS2m5hjEfO/J +UG7BJw+ch69u1RsIGL2SKcHvlJF40jocVYli5RsJHrpka/F2tNQCMQC0QoSZ/6vn +nvuRlydd3LBbMHHOXjgaatkl5+r3YZJW+OraNsKHZZYuciUvf9/DE8k= +-----END CERTIFICATE----- + +# Issuer: CN=Certum Trusted Root CA O=Asseco Data Systems S.A. OU=Certum Certification Authority +# Subject: CN=Certum Trusted Root CA O=Asseco Data Systems S.A. OU=Certum Certification Authority +# Label: "Certum Trusted Root CA" +# Serial: 40870380103424195783807378461123655149 +# MD5 Fingerprint: 51:e1:c2:e7:fe:4c:84:af:59:0e:2f:f4:54:6f:ea:29 +# SHA1 Fingerprint: c8:83:44:c0:18:ae:9f:cc:f1:87:b7:8f:22:d1:c5:d7:45:84:ba:e5 +# SHA256 Fingerprint: fe:76:96:57:38:55:77:3e:37:a9:5e:7a:d4:d9:cc:96:c3:01:57:c1:5d:31:76:5b:a9:b1:57:04:e1:ae:78:fd +-----BEGIN CERTIFICATE----- +MIIFwDCCA6igAwIBAgIQHr9ZULjJgDdMBvfrVU+17TANBgkqhkiG9w0BAQ0FADB6 +MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBTLkEu +MScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxHzAdBgNV +BAMTFkNlcnR1bSBUcnVzdGVkIFJvb3QgQ0EwHhcNMTgwMzE2MTIxMDEzWhcNNDMw +MzE2MTIxMDEzWjB6MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEg +U3lzdGVtcyBTLkEuMScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRo +b3JpdHkxHzAdBgNVBAMTFkNlcnR1bSBUcnVzdGVkIFJvb3QgQ0EwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQDRLY67tzbqbTeRn06TpwXkKQMlzhyC93yZ +n0EGze2jusDbCSzBfN8pfktlL5On1AFrAygYo9idBcEq2EXxkd7fO9CAAozPOA/q +p1x4EaTByIVcJdPTsuclzxFUl6s1wB52HO8AU5853BSlLCIls3Jy/I2z5T4IHhQq +NwuIPMqw9MjCoa68wb4pZ1Xi/K1ZXP69VyywkI3C7Te2fJmItdUDmj0VDT06qKhF +8JVOJVkdzZhpu9PMMsmN74H+rX2Ju7pgE8pllWeg8xn2A1bUatMn4qGtg/BKEiJ3 +HAVz4hlxQsDsdUaakFjgao4rpUYwBI4Zshfjvqm6f1bxJAPXsiEodg42MEx51UGa +mqi4NboMOvJEGyCI98Ul1z3G4z5D3Yf+xOr1Uz5MZf87Sst4WmsXXw3Hw09Omiqi +7VdNIuJGmj8PkTQkfVXjjJU30xrwCSss0smNtA0Aq2cpKNgB9RkEth2+dv5yXMSF +ytKAQd8FqKPVhJBPC/PgP5sZ0jeJP/J7UhyM9uH3PAeXjA6iWYEMspA90+NZRu0P +qafegGtaqge2Gcu8V/OXIXoMsSt0Puvap2ctTMSYnjYJdmZm/Bo/6khUHL4wvYBQ +v3y1zgD2DGHZ5yQD4OMBgQ692IU0iL2yNqh7XAjlRICMb/gv1SHKHRzQ+8S1h9E6 +Tsd2tTVItQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSM+xx1 +vALTn04uSNn5YFSqxLNP+jAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQENBQAD +ggIBAEii1QALLtA/vBzVtVRJHlpr9OTy4EA34MwUe7nJ+jW1dReTagVphZzNTxl4 +WxmB82M+w85bj/UvXgF2Ez8sALnNllI5SW0ETsXpD4YN4fqzX4IS8TrOZgYkNCvo +zMrnadyHncI013nR03e4qllY/p0m+jiGPp2Kh2RX5Rc64vmNueMzeMGQ2Ljdt4NR +5MTMI9UGfOZR0800McD2RrsLrfw9EAUqO0qRJe6M1ISHgCq8CYyqOhNf6DR5UMEQ +GfnTKB7U0VEwKbOukGfWHwpjscWpxkIxYxeU72nLL/qMFH3EQxiJ2fAyQOaA4kZf +5ePBAFmo+eggvIksDkc0C+pXwlM2/KfUrzHN/gLldfq5Jwn58/U7yn2fqSLLiMmq +0Uc9NneoWWRrJ8/vJ8HjJLWG965+Mk2weWjROeiQWMODvA8s1pfrzgzhIMfatz7D +P78v3DSk+yshzWePS/Tj6tQ/50+6uaWTRRxmHyH6ZF5v4HaUMst19W7l9o/HuKTM +qJZ9ZPskWkoDbGs4xugDQ5r3V7mzKWmTOPQD8rv7gmsHINFSH5pkAnuYZttcTVoP +0ISVoDwUQwbKytu4QTbaakRnh6+v40URFWkIsr4WOZckbxJF0WddCajJFdr60qZf +E2Efv4WstK2tBZQIgx51F9NxO5NQI1mg7TyRVJ12AMXDuDjb +-----END CERTIFICATE----- + +# Issuer: CN=TunTrust Root CA O=Agence Nationale de Certification Electronique +# Subject: CN=TunTrust Root CA O=Agence Nationale de Certification Electronique +# Label: "TunTrust Root CA" +# Serial: 108534058042236574382096126452369648152337120275 +# MD5 Fingerprint: 85:13:b9:90:5b:36:5c:b6:5e:b8:5a:f8:e0:31:57:b4 +# SHA1 Fingerprint: cf:e9:70:84:0f:e0:73:0f:9d:f6:0c:7f:2c:4b:ee:20:46:34:9c:bb +# SHA256 Fingerprint: 2e:44:10:2a:b5:8c:b8:54:19:45:1c:8e:19:d9:ac:f3:66:2c:af:bc:61:4b:6a:53:96:0a:30:f7:d0:e2:eb:41 +-----BEGIN CERTIFICATE----- +MIIFszCCA5ugAwIBAgIUEwLV4kBMkkaGFmddtLu7sms+/BMwDQYJKoZIhvcNAQEL +BQAwYTELMAkGA1UEBhMCVE4xNzA1BgNVBAoMLkFnZW5jZSBOYXRpb25hbGUgZGUg +Q2VydGlmaWNhdGlvbiBFbGVjdHJvbmlxdWUxGTAXBgNVBAMMEFR1blRydXN0IFJv +b3QgQ0EwHhcNMTkwNDI2MDg1NzU2WhcNNDQwNDI2MDg1NzU2WjBhMQswCQYDVQQG +EwJUTjE3MDUGA1UECgwuQWdlbmNlIE5hdGlvbmFsZSBkZSBDZXJ0aWZpY2F0aW9u +IEVsZWN0cm9uaXF1ZTEZMBcGA1UEAwwQVHVuVHJ1c3QgUm9vdCBDQTCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMPN0/y9BFPdDCA61YguBUtB9YOCfvdZ +n56eY+hz2vYGqU8ftPkLHzmMmiDQfgbU7DTZhrx1W4eI8NLZ1KMKsmwb60ksPqxd +2JQDoOw05TDENX37Jk0bbjBU2PWARZw5rZzJJQRNmpA+TkBuimvNKWfGzC3gdOgF +VwpIUPp6Q9p+7FuaDmJ2/uqdHYVy7BG7NegfJ7/Boce7SBbdVtfMTqDhuazb1YMZ +GoXRlJfXyqNlC/M4+QKu3fZnz8k/9YosRxqZbwUN/dAdgjH8KcwAWJeRTIAAHDOF +li/LQcKLEITDCSSJH7UP2dl3RxiSlGBcx5kDPP73lad9UKGAwqmDrViWVSHbhlnU +r8a83YFuB9tgYv7sEG7aaAH0gxupPqJbI9dkxt/con3YS7qC0lH4Zr8GRuR5KiY2 +eY8fTpkdso8MDhz/yV3A/ZAQprE38806JG60hZC/gLkMjNWb1sjxVj8agIl6qeIb +MlEsPvLfe/ZdeikZjuXIvTZxi11Mwh0/rViizz1wTaZQmCXcI/m4WEEIcb9PuISg +jwBUFfyRbVinljvrS5YnzWuioYasDXxU5mZMZl+QviGaAkYt5IPCgLnPSz7ofzwB +7I9ezX/SKEIBlYrilz0QIX32nRzFNKHsLA4KUiwSVXAkPcvCFDVDXSdOvsC9qnyW +5/yeYa1E0wCXAgMBAAGjYzBhMB0GA1UdDgQWBBQGmpsfU33x9aTI04Y+oXNZtPdE +ITAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFAaamx9TffH1pMjThj6hc1m0 +90QhMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAqgVutt0Vyb+z +xiD2BkewhpMl0425yAA/l/VSJ4hxyXT968pk21vvHl26v9Hr7lxpuhbI87mP0zYu +QEkHDVneixCwSQXi/5E/S7fdAo74gShczNxtr18UnH1YeA32gAm56Q6XKRm4t+v4 +FstVEuTGfbvE7Pi1HE4+Z7/FXxttbUcoqgRYYdZ2vyJ/0Adqp2RT8JeNnYA/u8EH +22Wv5psymsNUk8QcCMNE+3tjEUPRahphanltkE8pjkcFwRJpadbGNjHh/PqAulxP +xOu3Mqz4dWEX1xAZufHSCe96Qp1bWgvUxpVOKs7/B9dPfhgGiPEZtdmYu65xxBzn +dFlY7wyJz4sfdZMaBBSSSFCp61cpABbjNhzI+L/wM9VBD8TMPN3pM0MBkRArHtG5 +Xc0yGYuPjCB31yLEQtyEFpslbei0VXF/sHyz03FJuc9SpAQ/3D2gu68zngowYI7b +nV2UqL1g52KAdoGDDIzMMEZJ4gzSqK/rYXHv5yJiqfdcZGyfFoxnNidF9Ql7v/YQ +CvGwjVRDjAS6oz/v4jXH+XTgbzRB0L9zZVcg+ZtnemZoJE6AZb0QmQZZ8mWvuMZH +u/2QeItBcy6vVR/cO5JyboTT0GFMDcx2V+IthSIVNg3rAZ3r2OvEhJn7wAzMMujj +d9qDRIueVSjAi1jTkD5OGwDxFa2DK5o= +-----END CERTIFICATE----- + +# Issuer: CN=HARICA TLS RSA Root CA 2021 O=Hellenic Academic and Research Institutions CA +# Subject: CN=HARICA TLS RSA Root CA 2021 O=Hellenic Academic and Research Institutions CA +# Label: "HARICA TLS RSA Root CA 2021" +# Serial: 76817823531813593706434026085292783742 +# MD5 Fingerprint: 65:47:9b:58:86:dd:2c:f0:fc:a2:84:1f:1e:96:c4:91 +# SHA1 Fingerprint: 02:2d:05:82:fa:88:ce:14:0c:06:79:de:7f:14:10:e9:45:d7:a5:6d +# SHA256 Fingerprint: d9:5d:0e:8e:da:79:52:5b:f9:be:b1:1b:14:d2:10:0d:32:94:98:5f:0c:62:d9:fa:bd:9c:d9:99:ec:cb:7b:1d +-----BEGIN CERTIFICATE----- +MIIFpDCCA4ygAwIBAgIQOcqTHO9D88aOk8f0ZIk4fjANBgkqhkiG9w0BAQsFADBs +MQswCQYDVQQGEwJHUjE3MDUGA1UECgwuSGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJl +c2VhcmNoIEluc3RpdHV0aW9ucyBDQTEkMCIGA1UEAwwbSEFSSUNBIFRMUyBSU0Eg +Um9vdCBDQSAyMDIxMB4XDTIxMDIxOTEwNTUzOFoXDTQ1MDIxMzEwNTUzN1owbDEL +MAkGA1UEBhMCR1IxNzA1BgNVBAoMLkhlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNl +YXJjaCBJbnN0aXR1dGlvbnMgQ0ExJDAiBgNVBAMMG0hBUklDQSBUTFMgUlNBIFJv +b3QgQ0EgMjAyMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAIvC569l +mwVnlskNJLnQDmT8zuIkGCyEf3dRywQRNrhe7Wlxp57kJQmXZ8FHws+RFjZiPTgE +4VGC/6zStGndLuwRo0Xua2s7TL+MjaQenRG56Tj5eg4MmOIjHdFOY9TnuEFE+2uv +a9of08WRiFukiZLRgeaMOVig1mlDqa2YUlhu2wr7a89o+uOkXjpFc5gH6l8Cct4M +pbOfrqkdtx2z/IpZ525yZa31MJQjB/OCFks1mJxTuy/K5FrZx40d/JiZ+yykgmvw +Kh+OC19xXFyuQnspiYHLA6OZyoieC0AJQTPb5lh6/a6ZcMBaD9YThnEvdmn8kN3b +LW7R8pv1GmuebxWMevBLKKAiOIAkbDakO/IwkfN4E8/BPzWr8R0RI7VDIp4BkrcY +AuUR0YLbFQDMYTfBKnya4dC6s1BG7oKsnTH4+yPiAwBIcKMJJnkVU2DzOFytOOqB +AGMUuTNe3QvboEUHGjMJ+E20pwKmafTCWQWIZYVWrkvL4N48fS0ayOn7H6NhStYq +E613TBoYm5EPWNgGVMWX+Ko/IIqmhaZ39qb8HOLubpQzKoNQhArlT4b4UEV4AIHr +W2jjJo3Me1xR9BQsQL4aYB16cmEdH2MtiKrOokWQCPxrvrNQKlr9qEgYRtaQQJKQ +CoReaDH46+0N0x3GfZkYVVYnZS6NRcUk7M7jAgMBAAGjQjBAMA8GA1UdEwEB/wQF +MAMBAf8wHQYDVR0OBBYEFApII6ZgpJIKM+qTW8VX6iVNvRLuMA4GA1UdDwEB/wQE +AwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAPpBIqm5iFSVmewzVjIuJndftTgfvnNAU +X15QvWiWkKQUEapobQk1OUAJ2vQJLDSle1mESSmXdMgHHkdt8s4cUCbjnj1AUz/3 +f5Z2EMVGpdAgS1D0NTsY9FVqQRtHBmg8uwkIYtlfVUKqrFOFrJVWNlar5AWMxaja +H6NpvVMPxP/cyuN+8kyIhkdGGvMA9YCRotxDQpSbIPDRzbLrLFPCU3hKTwSUQZqP +JzLB5UkZv/HywouoCjkxKLR9YjYsTewfM7Z+d21+UPCfDtcRj88YxeMn/ibvBZ3P +zzfF0HvaO7AWhAw6k9a+F9sPPg4ZeAnHqQJyIkv3N3a6dcSFA1pj1bF1BcK5vZSt +jBWZp5N99sXzqnTPBIWUmAD04vnKJGW/4GKvyMX6ssmeVkjaef2WdhW+o45WxLM0 +/L5H9MG0qPzVMIho7suuyWPEdr6sOBjhXlzPrjoiUevRi7PzKzMHVIf6tLITe7pT +BGIBnfHAT+7hOtSLIBD6Alfm78ELt5BGnBkpjNxvoEppaZS3JGWg/6w/zgH7IS79 +aPib8qXPMThcFarmlwDB31qlpzmq6YR/PFGoOtmUW4y/Twhx5duoXNTSpv4Ao8YW +xw/ogM4cKGR0GQjTQuPOAF1/sdwTsOEFy9EgqoZ0njnnkf3/W9b3raYvAwtt41dU +63ZTGI0RmLo= +-----END CERTIFICATE----- + +# Issuer: CN=HARICA TLS ECC Root CA 2021 O=Hellenic Academic and Research Institutions CA +# Subject: CN=HARICA TLS ECC Root CA 2021 O=Hellenic Academic and Research Institutions CA +# Label: "HARICA TLS ECC Root CA 2021" +# Serial: 137515985548005187474074462014555733966 +# MD5 Fingerprint: ae:f7:4c:e5:66:35:d1:b7:9b:8c:22:93:74:d3:4b:b0 +# SHA1 Fingerprint: bc:b0:c1:9d:e9:98:92:70:19:38:57:e9:8d:a7:b4:5d:6e:ee:01:48 +# SHA256 Fingerprint: 3f:99:cc:47:4a:cf:ce:4d:fe:d5:87:94:66:5e:47:8d:15:47:73:9f:2e:78:0f:1b:b4:ca:9b:13:30:97:d4:01 +-----BEGIN CERTIFICATE----- +MIICVDCCAdugAwIBAgIQZ3SdjXfYO2rbIvT/WeK/zjAKBggqhkjOPQQDAzBsMQsw +CQYDVQQGEwJHUjE3MDUGA1UECgwuSGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJlc2Vh +cmNoIEluc3RpdHV0aW9ucyBDQTEkMCIGA1UEAwwbSEFSSUNBIFRMUyBFQ0MgUm9v +dCBDQSAyMDIxMB4XDTIxMDIxOTExMDExMFoXDTQ1MDIxMzExMDEwOVowbDELMAkG +A1UEBhMCR1IxNzA1BgNVBAoMLkhlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJj +aCBJbnN0aXR1dGlvbnMgQ0ExJDAiBgNVBAMMG0hBUklDQSBUTFMgRUNDIFJvb3Qg +Q0EgMjAyMTB2MBAGByqGSM49AgEGBSuBBAAiA2IABDgI/rGgltJ6rK9JOtDA4MM7 +KKrxcm1lAEeIhPyaJmuqS7psBAqIXhfyVYf8MLA04jRYVxqEU+kw2anylnTDUR9Y +STHMmE5gEYd103KUkE+bECUqqHgtvpBBWJAVcqeht6NCMEAwDwYDVR0TAQH/BAUw +AwEB/zAdBgNVHQ4EFgQUyRtTgRL+BNUW0aq8mm+3oJUZbsowDgYDVR0PAQH/BAQD +AgGGMAoGCCqGSM49BAMDA2cAMGQCMBHervjcToiwqfAircJRQO9gcS3ujwLEXQNw +SaSS6sUUiHCm0w2wqsosQJz76YJumgIwK0eaB8bRwoF8yguWGEEbo/QwCZ61IygN +nxS2PFOiTAZpffpskcYqSUXm7LcT4Tps +-----END CERTIFICATE----- + +# Issuer: CN=Autoridad de Certificacion Firmaprofesional CIF A62634068 +# Subject: CN=Autoridad de Certificacion Firmaprofesional CIF A62634068 +# Label: "Autoridad de Certificacion Firmaprofesional CIF A62634068" +# Serial: 1977337328857672817 +# MD5 Fingerprint: 4e:6e:9b:54:4c:ca:b7:fa:48:e4:90:b1:15:4b:1c:a3 +# SHA1 Fingerprint: 0b:be:c2:27:22:49:cb:39:aa:db:35:5c:53:e3:8c:ae:78:ff:b6:fe +# SHA256 Fingerprint: 57:de:05:83:ef:d2:b2:6e:03:61:da:99:da:9d:f4:64:8d:ef:7e:e8:44:1c:3b:72:8a:fa:9b:cd:e0:f9:b2:6a +-----BEGIN CERTIFICATE----- +MIIGFDCCA/ygAwIBAgIIG3Dp0v+ubHEwDQYJKoZIhvcNAQELBQAwUTELMAkGA1UE +BhMCRVMxQjBABgNVBAMMOUF1dG9yaWRhZCBkZSBDZXJ0aWZpY2FjaW9uIEZpcm1h +cHJvZmVzaW9uYWwgQ0lGIEE2MjYzNDA2ODAeFw0xNDA5MjMxNTIyMDdaFw0zNjA1 +MDUxNTIyMDdaMFExCzAJBgNVBAYTAkVTMUIwQAYDVQQDDDlBdXRvcmlkYWQgZGUg +Q2VydGlmaWNhY2lvbiBGaXJtYXByb2Zlc2lvbmFsIENJRiBBNjI2MzQwNjgwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDKlmuO6vj78aI14H9M2uDDUtd9 +thDIAl6zQyrET2qyyhxdKJp4ERppWVevtSBC5IsP5t9bpgOSL/UR5GLXMnE42QQM +cas9UX4PB99jBVzpv5RvwSmCwLTaUbDBPLutN0pcyvFLNg4kq7/DhHf9qFD0sefG +L9ItWY16Ck6WaVICqjaY7Pz6FIMMNx/Jkjd/14Et5cS54D40/mf0PmbR0/RAz15i +NA9wBj4gGFrO93IbJWyTdBSTo3OxDqqHECNZXyAFGUftaI6SEspd/NYrspI8IM/h +X68gvqB2f3bl7BqGYTM+53u0P6APjqK5am+5hyZvQWyIplD9amML9ZMWGxmPsu2b +m8mQ9QEM3xk9Dz44I8kvjwzRAv4bVdZO0I08r0+k8/6vKtMFnXkIoctXMbScyJCy +Z/QYFpM6/EfY0XiWMR+6KwxfXZmtY4laJCB22N/9q06mIqqdXuYnin1oKaPnirja +EbsXLZmdEyRG98Xi2J+Of8ePdG1asuhy9azuJBCtLxTa/y2aRnFHvkLfuwHb9H/T +KI8xWVvTyQKmtFLKbpf7Q8UIJm+K9Lv9nyiqDdVF8xM6HdjAeI9BZzwelGSuewvF +6NkBiDkal4ZkQdU7hwxu+g/GvUgUvzlN1J5Bto+WHWOWk9mVBngxaJ43BjuAiUVh +OSPHG0SjFeUc+JIwuwIDAQABo4HvMIHsMB0GA1UdDgQWBBRlzeurNR4APn7VdMAc +tHNHDhpkLzASBgNVHRMBAf8ECDAGAQH/AgEBMIGmBgNVHSAEgZ4wgZswgZgGBFUd +IAAwgY8wLwYIKwYBBQUHAgEWI2h0dHA6Ly93d3cuZmlybWFwcm9mZXNpb25hbC5j +b20vY3BzMFwGCCsGAQUFBwICMFAeTgBQAGEAcwBlAG8AIABkAGUAIABsAGEAIABC +AG8AbgBhAG4AbwB2AGEAIAA0ADcAIABCAGEAcgBjAGUAbABvAG4AYQAgADAAOAAw +ADEANzAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQELBQADggIBAHSHKAIrdx9m +iWTtj3QuRhy7qPj4Cx2Dtjqn6EWKB7fgPiDL4QjbEwj4KKE1soCzC1HA01aajTNF +Sa9J8OA9B3pFE1r/yJfY0xgsfZb43aJlQ3CTkBW6kN/oGbDbLIpgD7dvlAceHabJ +hfa9NPhAeGIQcDq+fUs5gakQ1JZBu/hfHAsdCPKxsIl68veg4MSPi3i1O1ilI45P +Vf42O+AMt8oqMEEgtIDNrvx2ZnOorm7hfNoD6JQg5iKj0B+QXSBTFCZX2lSX3xZE +EAEeiGaPcjiT3SC3NL7X8e5jjkd5KAb881lFJWAiMxujX6i6KtoaPc1A6ozuBRWV +1aUsIC+nmCjuRfzxuIgALI9C2lHVnOUTaHFFQ4ueCyE8S1wF3BqfmI7avSKecs2t +CsvMo2ebKHTEm9caPARYpoKdrcd7b/+Alun4jWq9GJAd/0kakFI3ky88Al2CdgtR +5xbHV/g4+afNmyJU72OwFW1TZQNKXkqgsqeOSQBZONXH9IBk9W6VULgRfhVwOEqw +f9DEMnDAGf/JOC0ULGb0QkTmVXYbgBVX/8Cnp6o5qtjTcNAuuuuUavpfNIbnYrX9 +ivAwhZTJryQCL2/W3Wf+47BVTwSYT6RBVuKT0Gro1vP7ZeDOdcQxWQzugsgMYDNK +GbqEZycPvEJdvSRUDewdcAZfpLz6IHxV +-----END CERTIFICATE----- + +# Issuer: CN=vTrus ECC Root CA O=iTrusChina Co.,Ltd. +# Subject: CN=vTrus ECC Root CA O=iTrusChina Co.,Ltd. +# Label: "vTrus ECC Root CA" +# Serial: 630369271402956006249506845124680065938238527194 +# MD5 Fingerprint: de:4b:c1:f5:52:8c:9b:43:e1:3e:8f:55:54:17:8d:85 +# SHA1 Fingerprint: f6:9c:db:b0:fc:f6:02:13:b6:52:32:a6:a3:91:3f:16:70:da:c3:e1 +# SHA256 Fingerprint: 30:fb:ba:2c:32:23:8e:2a:98:54:7a:f9:79:31:e5:50:42:8b:9b:3f:1c:8e:eb:66:33:dc:fa:86:c5:b2:7d:d3 +-----BEGIN CERTIFICATE----- +MIICDzCCAZWgAwIBAgIUbmq8WapTvpg5Z6LSa6Q75m0c1towCgYIKoZIzj0EAwMw +RzELMAkGA1UEBhMCQ04xHDAaBgNVBAoTE2lUcnVzQ2hpbmEgQ28uLEx0ZC4xGjAY +BgNVBAMTEXZUcnVzIEVDQyBSb290IENBMB4XDTE4MDczMTA3MjY0NFoXDTQzMDcz +MTA3MjY0NFowRzELMAkGA1UEBhMCQ04xHDAaBgNVBAoTE2lUcnVzQ2hpbmEgQ28u +LEx0ZC4xGjAYBgNVBAMTEXZUcnVzIEVDQyBSb290IENBMHYwEAYHKoZIzj0CAQYF +K4EEACIDYgAEZVBKrox5lkqqHAjDo6LN/llWQXf9JpRCux3NCNtzslt188+cToL0 +v/hhJoVs1oVbcnDS/dtitN9Ti72xRFhiQgnH+n9bEOf+QP3A2MMrMudwpremIFUd +e4BdS49nTPEQo0IwQDAdBgNVHQ4EFgQUmDnNvtiyjPeyq+GtJK97fKHbH88wDwYD +VR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwCgYIKoZIzj0EAwMDaAAwZQIw +V53dVvHH4+m4SVBrm2nDb+zDfSXkV5UTQJtS0zvzQBm8JsctBp61ezaf9SXUY2sA +AjEA6dPGnlaaKsyh2j/IZivTWJwghfqrkYpwcBE4YGQLYgmRWAD5Tfs0aNoJrSEG +GJTO +-----END CERTIFICATE----- + +# Issuer: CN=vTrus Root CA O=iTrusChina Co.,Ltd. +# Subject: CN=vTrus Root CA O=iTrusChina Co.,Ltd. +# Label: "vTrus Root CA" +# Serial: 387574501246983434957692974888460947164905180485 +# MD5 Fingerprint: b8:c9:37:df:fa:6b:31:84:64:c5:ea:11:6a:1b:75:fc +# SHA1 Fingerprint: 84:1a:69:fb:f5:cd:1a:25:34:13:3d:e3:f8:fc:b8:99:d0:c9:14:b7 +# SHA256 Fingerprint: 8a:71:de:65:59:33:6f:42:6c:26:e5:38:80:d0:0d:88:a1:8d:a4:c6:a9:1f:0d:cb:61:94:e2:06:c5:c9:63:87 +-----BEGIN CERTIFICATE----- +MIIFVjCCAz6gAwIBAgIUQ+NxE9izWRRdt86M/TX9b7wFjUUwDQYJKoZIhvcNAQEL +BQAwQzELMAkGA1UEBhMCQ04xHDAaBgNVBAoTE2lUcnVzQ2hpbmEgQ28uLEx0ZC4x +FjAUBgNVBAMTDXZUcnVzIFJvb3QgQ0EwHhcNMTgwNzMxMDcyNDA1WhcNNDMwNzMx +MDcyNDA1WjBDMQswCQYDVQQGEwJDTjEcMBoGA1UEChMTaVRydXNDaGluYSBDby4s +THRkLjEWMBQGA1UEAxMNdlRydXMgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAL1VfGHTuB0EYgWgrmy3cLRB6ksDXhA/kFocizuwZotsSKYc +IrrVQJLuM7IjWcmOvFjai57QGfIvWcaMY1q6n6MLsLOaXLoRuBLpDLvPbmyAhykU +AyyNJJrIZIO1aqwTLDPxn9wsYTwaP3BVm60AUn/PBLn+NvqcwBauYv6WTEN+VRS+ +GrPSbcKvdmaVayqwlHeFXgQPYh1jdfdr58tbmnDsPmcF8P4HCIDPKNsFxhQnL4Z9 +8Cfe/+Z+M0jnCx5Y0ScrUw5XSmXX+6KAYPxMvDVTAWqXcoKv8R1w6Jz1717CbMdH +flqUhSZNO7rrTOiwCcJlwp2dCZtOtZcFrPUGoPc2BX70kLJrxLT5ZOrpGgrIDajt +J8nU57O5q4IikCc9Kuh8kO+8T/3iCiSn3mUkpF3qwHYw03dQ+A0Em5Q2AXPKBlim +0zvc+gRGE1WKyURHuFE5Gi7oNOJ5y1lKCn+8pu8fA2dqWSslYpPZUxlmPCdiKYZN +pGvu/9ROutW04o5IWgAZCfEF2c6Rsffr6TlP9m8EQ5pV9T4FFL2/s1m02I4zhKOQ +UqqzApVg+QxMaPnu1RcN+HFXtSXkKe5lXa/R7jwXC1pDxaWG6iSe4gUH3DRCEpHW +OXSuTEGC2/KmSNGzm/MzqvOmwMVO9fSddmPmAsYiS8GVP1BkLFTltvA8Kc9XAgMB +AAGjQjBAMB0GA1UdDgQWBBRUYnBj8XWEQ1iO0RYgscasGrz2iTAPBgNVHRMBAf8E +BTADAQH/MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAKbqSSaet +8PFww+SX8J+pJdVrnjT+5hpk9jprUrIQeBqfTNqK2uwcN1LgQkv7bHbKJAs5EhWd +nxEt/Hlk3ODg9d3gV8mlsnZwUKT+twpw1aA08XXXTUm6EdGz2OyC/+sOxL9kLX1j +bhd47F18iMjrjld22VkE+rxSH0Ws8HqA7Oxvdq6R2xCOBNyS36D25q5J08FsEhvM +Kar5CKXiNxTKsbhm7xqC5PD48acWabfbqWE8n/Uxy+QARsIvdLGx14HuqCaVvIiv +TDUHKgLKeBRtRytAVunLKmChZwOgzoy8sHJnxDHO2zTlJQNgJXtxmOTAGytfdELS +S8VZCAeHvsXDf+eW2eHcKJfWjwXj9ZtOyh1QRwVTsMo554WgicEFOwE30z9J4nfr +I8iIZjs9OXYhRvHsXyO466JmdXTBQPfYaJqT4i2pLr0cox7IdMakLXogqzu4sEb9 +b91fUlV1YvCXoHzXOP0l382gmxDPi7g4Xl7FtKYCNqEeXxzP4padKar9mK5S4fNB +UvupLnKWnyfjqnN9+BojZns7q2WwMgFLFT49ok8MKzWixtlnEjUwzXYuFrOZnk1P +Ti07NEPhmg4NpGaXutIcSkwsKouLgU9xGqndXHt7CMUADTdA43x7VF8vhV929ven +sBxXVsFy6K2ir40zSbofitzmdHxghm+Hl3s= +-----END CERTIFICATE----- + +# Issuer: CN=ISRG Root X2 O=Internet Security Research Group +# Subject: CN=ISRG Root X2 O=Internet Security Research Group +# Label: "ISRG Root X2" +# Serial: 87493402998870891108772069816698636114 +# MD5 Fingerprint: d3:9e:c4:1e:23:3c:a6:df:cf:a3:7e:6d:e0:14:e6:e5 +# SHA1 Fingerprint: bd:b1:b9:3c:d5:97:8d:45:c6:26:14:55:f8:db:95:c7:5a:d1:53:af +# SHA256 Fingerprint: 69:72:9b:8e:15:a8:6e:fc:17:7a:57:af:b7:17:1d:fc:64:ad:d2:8c:2f:ca:8c:f1:50:7e:34:45:3c:cb:14:70 +-----BEGIN CERTIFICATE----- +MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw +CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg +R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00 +MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT +ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw +EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW ++1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9 +ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T +AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI +zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW +tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1 +/q4AaOeMSQ+2b1tbFfLn +-----END CERTIFICATE----- + +# Issuer: CN=HiPKI Root CA - G1 O=Chunghwa Telecom Co., Ltd. +# Subject: CN=HiPKI Root CA - G1 O=Chunghwa Telecom Co., Ltd. +# Label: "HiPKI Root CA - G1" +# Serial: 60966262342023497858655262305426234976 +# MD5 Fingerprint: 69:45:df:16:65:4b:e8:68:9a:8f:76:5f:ff:80:9e:d3 +# SHA1 Fingerprint: 6a:92:e4:a8:ee:1b:ec:96:45:37:e3:29:57:49:cd:96:e3:e5:d2:60 +# SHA256 Fingerprint: f0:15:ce:3c:c2:39:bf:ef:06:4b:e9:f1:d2:c4:17:e1:a0:26:4a:0a:94:be:1f:0c:8d:12:18:64:eb:69:49:cc +-----BEGIN CERTIFICATE----- +MIIFajCCA1KgAwIBAgIQLd2szmKXlKFD6LDNdmpeYDANBgkqhkiG9w0BAQsFADBP +MQswCQYDVQQGEwJUVzEjMCEGA1UECgwaQ2h1bmdod2EgVGVsZWNvbSBDby4sIEx0 +ZC4xGzAZBgNVBAMMEkhpUEtJIFJvb3QgQ0EgLSBHMTAeFw0xOTAyMjIwOTQ2MDRa +Fw0zNzEyMzExNTU5NTlaME8xCzAJBgNVBAYTAlRXMSMwIQYDVQQKDBpDaHVuZ2h3 +YSBUZWxlY29tIENvLiwgTHRkLjEbMBkGA1UEAwwSSGlQS0kgUm9vdCBDQSAtIEcx +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA9B5/UnMyDHPkvRN0o9Qw +qNCuS9i233VHZvR85zkEHmpwINJaR3JnVfSl6J3VHiGh8Ge6zCFovkRTv4354twv +Vcg3Px+kwJyz5HdcoEb+d/oaoDjq7Zpy3iu9lFc6uux55199QmQ5eiY29yTw1S+6 +lZgRZq2XNdZ1AYDgr/SEYYwNHl98h5ZeQa/rh+r4XfEuiAU+TCK72h8q3VJGZDnz +Qs7ZngyzsHeXZJzA9KMuH5UHsBffMNsAGJZMoYFL3QRtU6M9/Aes1MU3guvklQgZ +KILSQjqj2FPseYlgSGDIcpJQ3AOPgz+yQlda22rpEZfdhSi8MEyr48KxRURHH+CK +FgeW0iEPU8DtqX7UTuybCeyvQqww1r/REEXgphaypcXTT3OUM3ECoWqj1jOXTyFj +HluP2cFeRXF3D4FdXyGarYPM+l7WjSNfGz1BryB1ZlpK9p/7qxj3ccC2HTHsOyDr +y+K49a6SsvfhhEvyovKTmiKe0xRvNlS9H15ZFblzqMF8b3ti6RZsR1pl8w4Rm0bZ +/W3c1pzAtH2lsN0/Vm+h+fbkEkj9Bn8SV7apI09bA8PgcSojt/ewsTu8mL3WmKgM +a/aOEmem8rJY5AIJEzypuxC00jBF8ez3ABHfZfjcK0NVvxaXxA/VLGGEqnKG/uY6 +fsI/fe78LxQ+5oXdUG+3Se0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQU8ncX+l6o/vY9cdVouslGDDjYr7AwDgYDVR0PAQH/BAQDAgGGMA0GCSqG +SIb3DQEBCwUAA4ICAQBQUfB13HAE4/+qddRxosuej6ip0691x1TPOhwEmSKsxBHi +7zNKpiMdDg1H2DfHb680f0+BazVP6XKlMeJ45/dOlBhbQH3PayFUhuaVevvGyuqc +SE5XCV0vrPSltJczWNWseanMX/mF+lLFjfiRFOs6DRfQUsJ748JzjkZ4Bjgs6Fza +ZsT0pPBWGTMpWmWSBUdGSquEwx4noR8RkpkndZMPvDY7l1ePJlsMu5wP1G4wB9Tc +XzZoZjmDlicmisjEOf6aIW/Vcobpf2Lll07QJNBAsNB1CI69aO4I1258EHBGG3zg +iLKecoaZAeO/n0kZtCW+VmWuF2PlHt/o/0elv+EmBYTksMCv5wiZqAxeJoBF1Pho +L5aPruJKHJwWDBNvOIf2u8g0X5IDUXlwpt/L9ZlNec1OvFefQ05rLisY+GpzjLrF +Ne85akEez3GoorKGB1s6yeHvP2UEgEcyRHCVTjFnanRbEEV16rCf0OY1/k6fi8wr +kkVbbiVghUbN0aqwdmaTd5a+g744tiROJgvM7XpWGuDpWsZkrUx6AEhEL7lAuxM+ +vhV4nYWBSipX3tUZQ9rbyltHhoMLP7YNdnhzeSJesYAfz77RP1YQmCuVh6EfnWQU +YDksswBVLuT1sw5XxJFBAJw/6KXf6vb/yPCtbVKoF6ubYfwSUTXkJf2vqmqGOQ== +-----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R4 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R4 +# Label: "GlobalSign ECC Root CA - R4" +# Serial: 159662223612894884239637590694 +# MD5 Fingerprint: 26:29:f8:6d:e1:88:bf:a2:65:7f:aa:c4:cd:0f:7f:fc +# SHA1 Fingerprint: 6b:a0:b0:98:e1:71:ef:5a:ad:fe:48:15:80:77:10:f4:bd:6f:0b:28 +# SHA256 Fingerprint: b0:85:d7:0b:96:4f:19:1a:73:e4:af:0d:54:ae:7a:0e:07:aa:fd:af:9b:71:dd:08:62:13:8a:b7:32:5a:24:a2 +-----BEGIN CERTIFICATE----- +MIIB3DCCAYOgAwIBAgINAgPlfvU/k/2lCSGypjAKBggqhkjOPQQDAjBQMSQwIgYD +VQQLExtHbG9iYWxTaWduIEVDQyBSb290IENBIC0gUjQxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTIxMTEzMDAwMDAwWhcNMzgw +MTE5MDMxNDA3WjBQMSQwIgYDVQQLExtHbG9iYWxTaWduIEVDQyBSb290IENBIC0g +UjQxEzARBgNVBAoTCkdsb2JhbFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wWTAT +BgcqhkjOPQIBBggqhkjOPQMBBwNCAAS4xnnTj2wlDp8uORkcA6SumuU5BwkWymOx +uYb4ilfBV85C+nOh92VC/x7BALJucw7/xyHlGKSq2XE/qNS5zowdo0IwQDAOBgNV +HQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUVLB7rUW44kB/ ++wpu+74zyTyjhNUwCgYIKoZIzj0EAwIDRwAwRAIgIk90crlgr/HmnKAWBVBfw147 +bmF0774BxL4YSFlhgjICICadVGNA3jdgUM/I2O2dgq43mLyjj0xMqTQrbO/7lZsm +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R1 O=Google Trust Services LLC +# Subject: CN=GTS Root R1 O=Google Trust Services LLC +# Label: "GTS Root R1" +# Serial: 159662320309726417404178440727 +# MD5 Fingerprint: 05:fe:d0:bf:71:a8:a3:76:63:da:01:e0:d8:52:dc:40 +# SHA1 Fingerprint: e5:8c:1c:c4:91:3b:38:63:4b:e9:10:6e:e3:ad:8e:6b:9d:d9:81:4a +# SHA256 Fingerprint: d9:47:43:2a:bd:e7:b7:fa:90:fc:2e:6b:59:10:1b:12:80:e0:e1:c7:e4:e4:0f:a3:c6:88:7f:ff:57:a7:f4:cf +-----BEGIN CERTIFICATE----- +MIIFVzCCAz+gAwIBAgINAgPlk28xsBNJiGuiFzANBgkqhkiG9w0BAQwFADBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEBAQUA +A4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaMf/vo +27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vXmX7w +Cl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7zUjw +TcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0Pfybl +qAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtcvfaH +szVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4Zor8 +Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUspzBmk +MiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOORc92 +wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYWk70p +aDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+DVrN +VjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgFlQID +AQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBAJ+qQibb +C5u+/x6Wki4+omVKapi6Ist9wTrYggoGxval3sBOh2Z5ofmmWJyq+bXmYOfg6LEe +QkEzCzc9zolwFcq1JKjPa7XSQCGYzyI0zzvFIoTgxQ6KfF2I5DUkzps+GlQebtuy +h6f88/qBVRRiClmpIgUxPoLW7ttXNLwzldMXG+gnoot7TiYaelpkttGsN/H9oPM4 +7HLwEXWdyzRSjeZ2axfG34arJ45JK3VmgRAhpuo+9K4l/3wV3s6MJT/KYnAK9y8J +ZgfIPxz88NtFMN9iiMG1D53Dn0reWVlHxYciNuaCp+0KueIHoI17eko8cdLiA6Ef +MgfdG+RCzgwARWGAtQsgWSl4vflVy2PFPEz0tv/bal8xa5meLMFrUKTX5hgUvYU/ +Z6tGn6D/Qqc6f1zLXbBwHSs09dR2CQzreExZBfMzQsNhFRAbd03OIozUhfJFfbdT +6u9AWpQKXCBfTkBdYiJ23//OYb2MI3jSNwLgjt7RETeJ9r/tSQdirpLsQBqvFAnZ +0E6yove+7u7Y/9waLd64NnHi/Hm3lCXRSHNboTXns5lndcEZOitHTtNCjv0xyBZm +2tIMPNuzjsmhDYAPexZ3FL//2wmUspO8IFgV6dtxQ/PeEMMA3KgqlbbC1j+Qa3bb +bP6MvPJwNQzcmRk13NfIRmPVNnGuV/u3gm3c +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R3 O=Google Trust Services LLC +# Subject: CN=GTS Root R3 O=Google Trust Services LLC +# Label: "GTS Root R3" +# Serial: 159662495401136852707857743206 +# MD5 Fingerprint: 3e:e7:9d:58:02:94:46:51:94:e5:e0:22:4a:8b:e7:73 +# SHA1 Fingerprint: ed:e5:71:80:2b:c8:92:b9:5b:83:3c:d2:32:68:3f:09:cd:a0:1e:46 +# SHA256 Fingerprint: 34:d8:a7:3e:e2:08:d9:bc:db:0d:95:65:20:93:4b:4e:40:e6:94:82:59:6e:8b:6f:73:c8:42:6b:01:0a:6f:48 +-----BEGIN CERTIFICATE----- +MIICCTCCAY6gAwIBAgINAgPluILrIPglJ209ZjAKBggqhkjOPQQDAzBHMQswCQYD +VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG +A1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw +WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz +IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQAIgNi +AAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout736G +jOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2ADDL2 +4CejQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW +BBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEA9uEglRR7 +VKOQFhG/hMjqb2sXnh5GmCCbn9MN2azTL818+FsuVbu/3ZL3pAzcMeGiAjEA/Jdm +ZuVDFhOD3cffL74UOO0BzrEXGhF16b0DjyZ+hOXJYKaV11RZt+cRLInUue4X +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R4 O=Google Trust Services LLC +# Subject: CN=GTS Root R4 O=Google Trust Services LLC +# Label: "GTS Root R4" +# Serial: 159662532700760215368942768210 +# MD5 Fingerprint: 43:96:83:77:19:4d:76:b3:9d:65:52:e4:1d:22:a5:e8 +# SHA1 Fingerprint: 77:d3:03:67:b5:e0:0c:15:f6:0c:38:61:df:7c:e1:3b:92:46:4d:47 +# SHA256 Fingerprint: 34:9d:fa:40:58:c5:e2:63:12:3b:39:8a:e7:95:57:3c:4e:13:13:c8:3f:e6:8f:93:55:6c:d5:e8:03:1b:3c:7d +-----BEGIN CERTIFICATE----- +MIICCTCCAY6gAwIBAgINAgPlwGjvYxqccpBQUjAKBggqhkjOPQQDAzBHMQswCQYD +VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG +A1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw +WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz +IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQAIgNi +AATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzuhXyi +QHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/lxKvR +HYqjQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW +BBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNpADBmAjEA6ED/g94D +9J+uHXqnLrmvT/aDHQ4thQEd0dlq7A/Cr8deVl5c1RxYIigL9zC2L7F8AjEA8GE8 +p/SgguMh1YQdc4acLa/KNJvxn7kjNuK8YAOdgLOaVsjh4rsUecrNIdSUtUlD +-----END CERTIFICATE----- + +# Issuer: CN=Telia Root CA v2 O=Telia Finland Oyj +# Subject: CN=Telia Root CA v2 O=Telia Finland Oyj +# Label: "Telia Root CA v2" +# Serial: 7288924052977061235122729490515358 +# MD5 Fingerprint: 0e:8f:ac:aa:82:df:85:b1:f4:dc:10:1c:fc:99:d9:48 +# SHA1 Fingerprint: b9:99:cd:d1:73:50:8a:c4:47:05:08:9c:8c:88:fb:be:a0:2b:40:cd +# SHA256 Fingerprint: 24:2b:69:74:2f:cb:1e:5b:2a:bf:98:89:8b:94:57:21:87:54:4e:5b:4d:99:11:78:65:73:62:1f:6a:74:b8:2c +-----BEGIN CERTIFICATE----- +MIIFdDCCA1ygAwIBAgIPAWdfJ9b+euPkrL4JWwWeMA0GCSqGSIb3DQEBCwUAMEQx +CzAJBgNVBAYTAkZJMRowGAYDVQQKDBFUZWxpYSBGaW5sYW5kIE95ajEZMBcGA1UE +AwwQVGVsaWEgUm9vdCBDQSB2MjAeFw0xODExMjkxMTU1NTRaFw00MzExMjkxMTU1 +NTRaMEQxCzAJBgNVBAYTAkZJMRowGAYDVQQKDBFUZWxpYSBGaW5sYW5kIE95ajEZ +MBcGA1UEAwwQVGVsaWEgUm9vdCBDQSB2MjCCAiIwDQYJKoZIhvcNAQEBBQADggIP +ADCCAgoCggIBALLQPwe84nvQa5n44ndp586dpAO8gm2h/oFlH0wnrI4AuhZ76zBq +AMCzdGh+sq/H1WKzej9Qyow2RCRj0jbpDIX2Q3bVTKFgcmfiKDOlyzG4OiIjNLh9 +vVYiQJ3q9HsDrWj8soFPmNB06o3lfc1jw6P23pLCWBnglrvFxKk9pXSW/q/5iaq9 +lRdU2HhE8Qx3FZLgmEKnpNaqIJLNwaCzlrI6hEKNfdWV5Nbb6WLEWLN5xYzTNTOD +n3WhUidhOPFZPY5Q4L15POdslv5e2QJltI5c0BE0312/UqeBAMN/mUWZFdUXyApT +7GPzmX3MaRKGwhfwAZ6/hLzRUssbkmbOpFPlob/E2wnW5olWK8jjfN7j/4nlNW4o +6GwLI1GpJQXrSPjdscr6bAhR77cYbETKJuFzxokGgeWKrLDiKca5JLNrRBH0pUPC +TEPlcDaMtjNXepUugqD0XBCzYYP2AgWGLnwtbNwDRm41k9V6lS/eINhbfpSQBGq6 +WT0EBXWdN6IOLj3rwaRSg/7Qa9RmjtzG6RJOHSpXqhC8fF6CfaamyfItufUXJ63R +DolUK5X6wK0dmBR4M0KGCqlztft0DbcbMBnEWg4cJ7faGND/isgFuvGqHKI3t+ZI +pEYslOqodmJHixBTB0hXbOKSTbauBcvcwUpej6w9GU7C7WB1K9vBykLVAgMBAAGj +YzBhMB8GA1UdIwQYMBaAFHKs5DN5qkWH9v2sHZ7Wxy+G2CQ5MB0GA1UdDgQWBBRy +rOQzeapFh/b9rB2e1scvhtgkOTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUw +AwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAoDtZpwmUPjaE0n4vOaWWl/oRrfxn83EJ +8rKJhGdEr7nv7ZbsnGTbMjBvZ5qsfl+yqwE2foH65IRe0qw24GtixX1LDoJt0nZi +0f6X+J8wfBj5tFJ3gh1229MdqfDBmgC9bXXYfef6xzijnHDoRnkDry5023X4blMM +A8iZGok1GTzTyVR8qPAs5m4HeW9q4ebqkYJpCh3DflminmtGFZhb069GHWLIzoBS +SRE/yQQSwxN8PzuKlts8oB4KtItUsiRnDe+Cy748fdHif64W1lZYudogsYMVoe+K +TTJvQS8TUoKU1xrBeKJR3Stwbbca+few4GeXVtt8YVMJAygCQMez2P2ccGrGKMOF +6eLtGpOg3kuYooQ+BXcBlj37tCAPnHICehIv1aO6UXivKitEZU61/Qrowc15h2Er +3oBXRb9n8ZuRXqWk7FlIEA04x7D6w0RtBPV4UBySllva9bguulvP5fBqnUsvWHMt +Ty3EHD70sz+rFQ47GUGKpMFXEmZxTPpT41frYpUJnlTd0cI8Vzy9OK2YZLe4A5pT +VmBds9hCG1xLEooc6+t9xnppxyd/pPiL8uSUZodL6ZQHCRJ5irLrdATczvREWeAW +ysUsWNc8e89ihmpQfTU2Zqf7N+cox9jQraVplI/owd8k+BsHMYeB2F326CjYSlKA +rBPuUBQemMc= +-----END CERTIFICATE----- + +# Issuer: CN=D-TRUST BR Root CA 1 2020 O=D-Trust GmbH +# Subject: CN=D-TRUST BR Root CA 1 2020 O=D-Trust GmbH +# Label: "D-TRUST BR Root CA 1 2020" +# Serial: 165870826978392376648679885835942448534 +# MD5 Fingerprint: b5:aa:4b:d5:ed:f7:e3:55:2e:8f:72:0a:f3:75:b8:ed +# SHA1 Fingerprint: 1f:5b:98:f0:e3:b5:f7:74:3c:ed:e6:b0:36:7d:32:cd:f4:09:41:67 +# SHA256 Fingerprint: e5:9a:aa:81:60:09:c2:2b:ff:5b:25:ba:d3:7d:f3:06:f0:49:79:7c:1f:81:d8:5a:b0:89:e6:57:bd:8f:00:44 +-----BEGIN CERTIFICATE----- +MIIC2zCCAmCgAwIBAgIQfMmPK4TX3+oPyWWa00tNljAKBggqhkjOPQQDAzBIMQsw +CQYDVQQGEwJERTEVMBMGA1UEChMMRC1UcnVzdCBHbWJIMSIwIAYDVQQDExlELVRS +VVNUIEJSIFJvb3QgQ0EgMSAyMDIwMB4XDTIwMDIxMTA5NDUwMFoXDTM1MDIxMTA5 +NDQ1OVowSDELMAkGA1UEBhMCREUxFTATBgNVBAoTDEQtVHJ1c3QgR21iSDEiMCAG +A1UEAxMZRC1UUlVTVCBCUiBSb290IENBIDEgMjAyMDB2MBAGByqGSM49AgEGBSuB +BAAiA2IABMbLxyjR+4T1mu9CFCDhQ2tuda38KwOE1HaTJddZO0Flax7mNCq7dPYS +zuht56vkPE4/RAiLzRZxy7+SmfSk1zxQVFKQhYN4lGdnoxwJGT11NIXe7WB9xwy0 +QVK5buXuQqOCAQ0wggEJMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFHOREKv/ +VbNafAkl1bK6CKBrqx9tMA4GA1UdDwEB/wQEAwIBBjCBxgYDVR0fBIG+MIG7MD6g +PKA6hjhodHRwOi8vY3JsLmQtdHJ1c3QubmV0L2NybC9kLXRydXN0X2JyX3Jvb3Rf +Y2FfMV8yMDIwLmNybDB5oHegdYZzbGRhcDovL2RpcmVjdG9yeS5kLXRydXN0Lm5l +dC9DTj1ELVRSVVNUJTIwQlIlMjBSb290JTIwQ0ElMjAxJTIwMjAyMCxPPUQtVHJ1 +c3QlMjBHbWJILEM9REU/Y2VydGlmaWNhdGVyZXZvY2F0aW9ubGlzdDAKBggqhkjO +PQQDAwNpADBmAjEAlJAtE/rhY/hhY+ithXhUkZy4kzg+GkHaQBZTQgjKL47xPoFW +wKrY7RjEsK70PvomAjEA8yjixtsrmfu3Ubgko6SUeho/5jbiA1czijDLgsfWFBHV +dWNbFJWcHwHP2NVypw87 +-----END CERTIFICATE----- + +# Issuer: CN=D-TRUST EV Root CA 1 2020 O=D-Trust GmbH +# Subject: CN=D-TRUST EV Root CA 1 2020 O=D-Trust GmbH +# Label: "D-TRUST EV Root CA 1 2020" +# Serial: 126288379621884218666039612629459926992 +# MD5 Fingerprint: 8c:2d:9d:70:9f:48:99:11:06:11:fb:e9:cb:30:c0:6e +# SHA1 Fingerprint: 61:db:8c:21:59:69:03:90:d8:7c:9c:12:86:54:cf:9d:3d:f4:dd:07 +# SHA256 Fingerprint: 08:17:0d:1a:a3:64:53:90:1a:2f:95:92:45:e3:47:db:0c:8d:37:ab:aa:bc:56:b8:1a:a1:00:dc:95:89:70:db +-----BEGIN CERTIFICATE----- +MIIC2zCCAmCgAwIBAgIQXwJB13qHfEwDo6yWjfv/0DAKBggqhkjOPQQDAzBIMQsw +CQYDVQQGEwJERTEVMBMGA1UEChMMRC1UcnVzdCBHbWJIMSIwIAYDVQQDExlELVRS +VVNUIEVWIFJvb3QgQ0EgMSAyMDIwMB4XDTIwMDIxMTEwMDAwMFoXDTM1MDIxMTA5 +NTk1OVowSDELMAkGA1UEBhMCREUxFTATBgNVBAoTDEQtVHJ1c3QgR21iSDEiMCAG +A1UEAxMZRC1UUlVTVCBFViBSb290IENBIDEgMjAyMDB2MBAGByqGSM49AgEGBSuB +BAAiA2IABPEL3YZDIBnfl4XoIkqbz52Yv7QFJsnL46bSj8WeeHsxiamJrSc8ZRCC +/N/DnU7wMyPE0jL1HLDfMxddxfCxivnvubcUyilKwg+pf3VlSSowZ/Rk99Yad9rD +wpdhQntJraOCAQ0wggEJMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFH8QARY3 +OqQo5FD4pPfsazK2/umLMA4GA1UdDwEB/wQEAwIBBjCBxgYDVR0fBIG+MIG7MD6g +PKA6hjhodHRwOi8vY3JsLmQtdHJ1c3QubmV0L2NybC9kLXRydXN0X2V2X3Jvb3Rf +Y2FfMV8yMDIwLmNybDB5oHegdYZzbGRhcDovL2RpcmVjdG9yeS5kLXRydXN0Lm5l +dC9DTj1ELVRSVVNUJTIwRVYlMjBSb290JTIwQ0ElMjAxJTIwMjAyMCxPPUQtVHJ1 +c3QlMjBHbWJILEM9REU/Y2VydGlmaWNhdGVyZXZvY2F0aW9ubGlzdDAKBggqhkjO +PQQDAwNpADBmAjEAyjzGKnXCXnViOTYAYFqLwZOZzNnbQTs7h5kXO9XMT8oi96CA +y/m0sRtW9XLS/BnRAjEAkfcwkz8QRitxpNA7RJvAKQIFskF3UfN5Wp6OFKBOQtJb +gfM0agPnIjhQW+0ZT0MW +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert TLS ECC P384 Root G5 O=DigiCert, Inc. +# Subject: CN=DigiCert TLS ECC P384 Root G5 O=DigiCert, Inc. +# Label: "DigiCert TLS ECC P384 Root G5" +# Serial: 13129116028163249804115411775095713523 +# MD5 Fingerprint: d3:71:04:6a:43:1c:db:a6:59:e1:a8:a3:aa:c5:71:ed +# SHA1 Fingerprint: 17:f3:de:5e:9f:0f:19:e9:8e:f6:1f:32:26:6e:20:c4:07:ae:30:ee +# SHA256 Fingerprint: 01:8e:13:f0:77:25:32:cf:80:9b:d1:b1:72:81:86:72:83:fc:48:c6:e1:3b:e9:c6:98:12:85:4a:49:0c:1b:05 +-----BEGIN CERTIFICATE----- +MIICGTCCAZ+gAwIBAgIQCeCTZaz32ci5PhwLBCou8zAKBggqhkjOPQQDAzBOMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJjAkBgNVBAMTHURp +Z2lDZXJ0IFRMUyBFQ0MgUDM4NCBSb290IEc1MB4XDTIxMDExNTAwMDAwMFoXDTQ2 +MDExNDIzNTk1OVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkRpZ2lDZXJ0LCBJ +bmMuMSYwJAYDVQQDEx1EaWdpQ2VydCBUTFMgRUNDIFAzODQgUm9vdCBHNTB2MBAG +ByqGSM49AgEGBSuBBAAiA2IABMFEoc8Rl1Ca3iOCNQfN0MsYndLxf3c1TzvdlHJS +7cI7+Oz6e2tYIOyZrsn8aLN1udsJ7MgT9U7GCh1mMEy7H0cKPGEQQil8pQgO4CLp +0zVozptjn4S1mU1YoI71VOeVyaNCMEAwHQYDVR0OBBYEFMFRRVBZqz7nLFr6ICIS +B4CIfBFqMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49 +BAMDA2gAMGUCMQCJao1H5+z8blUD2WdsJk6Dxv3J+ysTvLd6jLRl0mlpYxNjOyZQ +LgGheQaRnUi/wr4CMEfDFXuxoJGZSZOoPHzoRgaLLPIxAJSdYsiJvRmEFOml+wG4 +DXZDjC5Ty3zfDBeWUA== +-----END CERTIFICATE----- + +# Issuer: CN=DigiCert TLS RSA4096 Root G5 O=DigiCert, Inc. +# Subject: CN=DigiCert TLS RSA4096 Root G5 O=DigiCert, Inc. +# Label: "DigiCert TLS RSA4096 Root G5" +# Serial: 11930366277458970227240571539258396554 +# MD5 Fingerprint: ac:fe:f7:34:96:a9:f2:b3:b4:12:4b:e4:27:41:6f:e1 +# SHA1 Fingerprint: a7:88:49:dc:5d:7c:75:8c:8c:de:39:98:56:b3:aa:d0:b2:a5:71:35 +# SHA256 Fingerprint: 37:1a:00:dc:05:33:b3:72:1a:7e:eb:40:e8:41:9e:70:79:9d:2b:0a:0f:2c:1d:80:69:31:65:f7:ce:c4:ad:75 +-----BEGIN CERTIFICATE----- +MIIFZjCCA06gAwIBAgIQCPm0eKj6ftpqMzeJ3nzPijANBgkqhkiG9w0BAQwFADBN +MQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJTAjBgNVBAMT +HERpZ2lDZXJ0IFRMUyBSU0E0MDk2IFJvb3QgRzUwHhcNMjEwMTE1MDAwMDAwWhcN +NDYwMTE0MjM1OTU5WjBNMQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQs +IEluYy4xJTAjBgNVBAMTHERpZ2lDZXJ0IFRMUyBSU0E0MDk2IFJvb3QgRzUwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCz0PTJeRGd/fxmgefM1eS87IE+ +ajWOLrfn3q/5B03PMJ3qCQuZvWxX2hhKuHisOjmopkisLnLlvevxGs3npAOpPxG0 +2C+JFvuUAT27L/gTBaF4HI4o4EXgg/RZG5Wzrn4DReW+wkL+7vI8toUTmDKdFqgp +wgscONyfMXdcvyej/Cestyu9dJsXLfKB2l2w4SMXPohKEiPQ6s+d3gMXsUJKoBZM +pG2T6T867jp8nVid9E6P/DsjyG244gXazOvswzH016cpVIDPRFtMbzCe88zdH5RD +nU1/cHAN1DrRN/BsnZvAFJNY781BOHW8EwOVfH/jXOnVDdXifBBiqmvwPXbzP6Po +sMH976pXTayGpxi0KcEsDr9kvimM2AItzVwv8n/vFfQMFawKsPHTDU9qTXeXAaDx +Zre3zu/O7Oyldcqs4+Fj97ihBMi8ez9dLRYiVu1ISf6nL3kwJZu6ay0/nTvEF+cd +Lvvyz6b84xQslpghjLSR6Rlgg/IwKwZzUNWYOwbpx4oMYIwo+FKbbuH2TbsGJJvX +KyY//SovcfXWJL5/MZ4PbeiPT02jP/816t9JXkGPhvnxd3lLG7SjXi/7RgLQZhNe +XoVPzthwiHvOAbWWl9fNff2C+MIkwcoBOU+NosEUQB+cZtUMCUbW8tDRSHZWOkPL +tgoRObqME2wGtZ7P6wIDAQABo0IwQDAdBgNVHQ4EFgQUUTMc7TZArxfTJc1paPKv +TiM+s0EwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcN +AQEMBQADggIBAGCmr1tfV9qJ20tQqcQjNSH/0GEwhJG3PxDPJY7Jv0Y02cEhJhxw +GXIeo8mH/qlDZJY6yFMECrZBu8RHANmfGBg7sg7zNOok992vIGCukihfNudd5N7H +PNtQOa27PShNlnx2xlv0wdsUpasZYgcYQF+Xkdycx6u1UQ3maVNVzDl92sURVXLF +O4uJ+DQtpBflF+aZfTCIITfNMBc9uPK8qHWgQ9w+iUuQrm0D4ByjoJYJu32jtyoQ +REtGBzRj7TG5BO6jm5qu5jF49OokYTurWGT/u4cnYiWB39yhL/btp/96j1EuMPik +AdKFOV8BmZZvWltwGUb+hmA+rYAQCd05JS9Yf7vSdPD3Rh9GOUrYU9DzLjtxpdRv +/PNn5AeP3SYZ4Y1b+qOTEZvpyDrDVWiakuFSdjjo4bq9+0/V77PnSIMx8IIh47a+ +p6tv75/fTM8BuGJqIz3nCU2AG3swpMPdB380vqQmsvZB6Akd4yCYqjdP//fx4ilw +MUc/dNAUFvohigLVigmUdy7yWSiLfFCSCmZ4OIN1xLVaqBHG5cGdZlXPU8Sv13WF +qUITVuwhd4GTWgzqltlJyqEI8pc7bZsEGCREjnwB8twl2F6GmrE52/WRMmrRpnCK +ovfepEWFJqgejF0pW8hL2JpqA15w8oVPbEtoL8pU9ozaMv7Da4M/OMZ+ +-----END CERTIFICATE----- + +# Issuer: CN=Certainly Root R1 O=Certainly +# Subject: CN=Certainly Root R1 O=Certainly +# Label: "Certainly Root R1" +# Serial: 188833316161142517227353805653483829216 +# MD5 Fingerprint: 07:70:d4:3e:82:87:a0:fa:33:36:13:f4:fa:33:e7:12 +# SHA1 Fingerprint: a0:50:ee:0f:28:71:f4:27:b2:12:6d:6f:50:96:25:ba:cc:86:42:af +# SHA256 Fingerprint: 77:b8:2c:d8:64:4c:43:05:f7:ac:c5:cb:15:6b:45:67:50:04:03:3d:51:c6:0c:62:02:a8:e0:c3:34:67:d3:a0 +-----BEGIN CERTIFICATE----- +MIIFRzCCAy+gAwIBAgIRAI4P+UuQcWhlM1T01EQ5t+AwDQYJKoZIhvcNAQELBQAw +PTELMAkGA1UEBhMCVVMxEjAQBgNVBAoTCUNlcnRhaW5seTEaMBgGA1UEAxMRQ2Vy +dGFpbmx5IFJvb3QgUjEwHhcNMjEwNDAxMDAwMDAwWhcNNDYwNDAxMDAwMDAwWjA9 +MQswCQYDVQQGEwJVUzESMBAGA1UEChMJQ2VydGFpbmx5MRowGAYDVQQDExFDZXJ0 +YWlubHkgUm9vdCBSMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANA2 +1B/q3avk0bbm+yLA3RMNansiExyXPGhjZjKcA7WNpIGD2ngwEc/csiu+kr+O5MQT +vqRoTNoCaBZ0vrLdBORrKt03H2As2/X3oXyVtwxwhi7xOu9S98zTm/mLvg7fMbed +aFySpvXl8wo0tf97ouSHocavFwDvA5HtqRxOcT3Si2yJ9HiG5mpJoM610rCrm/b0 +1C7jcvk2xusVtyWMOvwlDbMicyF0yEqWYZL1LwsYpfSt4u5BvQF5+paMjRcCMLT5 +r3gajLQ2EBAHBXDQ9DGQilHFhiZ5shGIXsXwClTNSaa/ApzSRKft43jvRl5tcdF5 +cBxGX1HpyTfcX35pe0HfNEXgO4T0oYoKNp43zGJS4YkNKPl6I7ENPT2a/Z2B7yyQ +wHtETrtJ4A5KVpK8y7XdeReJkd5hiXSSqOMyhb5OhaRLWcsrxXiOcVTQAjeZjOVJ +6uBUcqQRBi8LjMFbvrWhsFNunLhgkR9Za/kt9JQKl7XsxXYDVBtlUrpMklZRNaBA +2CnbrlJ2Oy0wQJuK0EJWtLeIAaSHO1OWzaMWj/Nmqhexx2DgwUMFDO6bW2BvBlyH +Wyf5QBGenDPBt+U1VwV/J84XIIwc/PH72jEpSe31C4SnT8H2TsIonPru4K8H+zMR +eiFPCyEQtkA6qyI6BJyLm4SGcprSp6XEtHWRqSsjAgMBAAGjQjBAMA4GA1UdDwEB +/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTgqj8ljZ9EXME66C6u +d0yEPmcM9DANBgkqhkiG9w0BAQsFAAOCAgEAuVevuBLaV4OPaAszHQNTVfSVcOQr +PbA56/qJYv331hgELyE03fFo8NWWWt7CgKPBjcZq91l3rhVkz1t5BXdm6ozTaw3d +8VkswTOlMIAVRQdFGjEitpIAq5lNOo93r6kiyi9jyhXWx8bwPWz8HA2YEGGeEaIi +1wrykXprOQ4vMMM2SZ/g6Q8CRFA3lFV96p/2O7qUpUzpvD5RtOjKkjZUbVwlKNrd +rRT90+7iIgXr0PK3aBLXWopBGsaSpVo7Y0VPv+E6dyIvXL9G+VoDhRNCX8reU9di +taY1BMJH/5n9hN9czulegChB8n3nHpDYT3Y+gjwN/KUD+nsa2UUeYNrEjvn8K8l7 +lcUq/6qJ34IxD3L/DCfXCh5WAFAeDJDBlrXYFIW7pw0WwfgHJBu6haEaBQmAupVj +yTrsJZ9/nbqkRxWbRHDxakvWOF5D8xh+UG7pWijmZeZ3Gzr9Hb4DJqPb1OG7fpYn +Kx3upPvaJVQTA945xsMfTZDsjxtK0hzthZU4UHlG1sGQUDGpXJpuHfUzVounmdLy +yCwzk5Iwx06MZTMQZBf9JBeW0Y3COmor6xOLRPIh80oat3df1+2IpHLlOR+Vnb5n +wXARPbv0+Em34yaXOp/SX3z7wJl8OSngex2/DaeP0ik0biQVy96QXr8axGbqwua6 +OV+KmalBWQewLK8= +-----END CERTIFICATE----- + +# Issuer: CN=Certainly Root E1 O=Certainly +# Subject: CN=Certainly Root E1 O=Certainly +# Label: "Certainly Root E1" +# Serial: 8168531406727139161245376702891150584 +# MD5 Fingerprint: 0a:9e:ca:cd:3e:52:50:c6:36:f3:4b:a3:ed:a7:53:e9 +# SHA1 Fingerprint: f9:e1:6d:dc:01:89:cf:d5:82:45:63:3e:c5:37:7d:c2:eb:93:6f:2b +# SHA256 Fingerprint: b4:58:5f:22:e4:ac:75:6a:4e:86:12:a1:36:1c:5d:9d:03:1a:93:fd:84:fe:bb:77:8f:a3:06:8b:0f:c4:2d:c2 +-----BEGIN CERTIFICATE----- +MIIB9zCCAX2gAwIBAgIQBiUzsUcDMydc+Y2aub/M+DAKBggqhkjOPQQDAzA9MQsw +CQYDVQQGEwJVUzESMBAGA1UEChMJQ2VydGFpbmx5MRowGAYDVQQDExFDZXJ0YWlu +bHkgUm9vdCBFMTAeFw0yMTA0MDEwMDAwMDBaFw00NjA0MDEwMDAwMDBaMD0xCzAJ +BgNVBAYTAlVTMRIwEAYDVQQKEwlDZXJ0YWlubHkxGjAYBgNVBAMTEUNlcnRhaW5s +eSBSb290IEUxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE3m/4fxzf7flHh4axpMCK ++IKXgOqPyEpeKn2IaKcBYhSRJHpcnqMXfYqGITQYUBsQ3tA3SybHGWCA6TS9YBk2 +QNYphwk8kXr2vBMj3VlOBF7PyAIcGFPBMdjaIOlEjeR2o0IwQDAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU8ygYy2R17ikq6+2uI1g4 +hevIIgcwCgYIKoZIzj0EAwMDaAAwZQIxALGOWiDDshliTd6wT99u0nCK8Z9+aozm +ut6Dacpps6kFtZaSF4fC0urQe87YQVt8rgIwRt7qy12a7DLCZRawTDBcMPPaTnOG +BtjOiQRINzf43TNRnXCve1XYAS59BWQOhriR +-----END CERTIFICATE----- + +# Issuer: CN=Security Communication ECC RootCA1 O=SECOM Trust Systems CO.,LTD. +# Subject: CN=Security Communication ECC RootCA1 O=SECOM Trust Systems CO.,LTD. +# Label: "Security Communication ECC RootCA1" +# Serial: 15446673492073852651 +# MD5 Fingerprint: 7e:43:b0:92:68:ec:05:43:4c:98:ab:5d:35:2e:7e:86 +# SHA1 Fingerprint: b8:0e:26:a9:bf:d2:b2:3b:c0:ef:46:c9:ba:c7:bb:f6:1d:0d:41:41 +# SHA256 Fingerprint: e7:4f:bd:a5:5b:d5:64:c4:73:a3:6b:44:1a:a7:99:c8:a6:8e:07:74:40:e8:28:8b:9f:a1:e5:0e:4b:ba:ca:11 +-----BEGIN CERTIFICATE----- +MIICODCCAb6gAwIBAgIJANZdm7N4gS7rMAoGCCqGSM49BAMDMGExCzAJBgNVBAYT +AkpQMSUwIwYDVQQKExxTRUNPTSBUcnVzdCBTeXN0ZW1zIENPLixMVEQuMSswKQYD +VQQDEyJTZWN1cml0eSBDb21tdW5pY2F0aW9uIEVDQyBSb290Q0ExMB4XDTE2MDYx +NjA1MTUyOFoXDTM4MDExODA1MTUyOFowYTELMAkGA1UEBhMCSlAxJTAjBgNVBAoT +HFNFQ09NIFRydXN0IFN5c3RlbXMgQ08uLExURC4xKzApBgNVBAMTIlNlY3VyaXR5 +IENvbW11bmljYXRpb24gRUNDIFJvb3RDQTEwdjAQBgcqhkjOPQIBBgUrgQQAIgNi +AASkpW9gAwPDvTH00xecK4R1rOX9PVdu12O/5gSJko6BnOPpR27KkBLIE+Cnnfdl +dB9sELLo5OnvbYUymUSxXv3MdhDYW72ixvnWQuRXdtyQwjWpS4g8EkdtXP9JTxpK +ULGjQjBAMB0GA1UdDgQWBBSGHOf+LaVKiwj+KBH6vqNm+GBZLzAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjAVXUI9/Lbu +9zuxNuie9sRGKEkz0FhDKmMpzE2xtHqiuQ04pV1IKv3LsnNdo4gIxwwCMQDAqy0O +be0YottT6SXbVQjgUMzfRGEWgqtJsLKB7HOHeLRMsmIbEvoWTSVLY70eN9k= +-----END CERTIFICATE----- + +# Issuer: CN=BJCA Global Root CA1 O=BEIJING CERTIFICATE AUTHORITY +# Subject: CN=BJCA Global Root CA1 O=BEIJING CERTIFICATE AUTHORITY +# Label: "BJCA Global Root CA1" +# Serial: 113562791157148395269083148143378328608 +# MD5 Fingerprint: 42:32:99:76:43:33:36:24:35:07:82:9b:28:f9:d0:90 +# SHA1 Fingerprint: d5:ec:8d:7b:4c:ba:79:f4:e7:e8:cb:9d:6b:ae:77:83:10:03:21:6a +# SHA256 Fingerprint: f3:89:6f:88:fe:7c:0a:88:27:66:a7:fa:6a:d2:74:9f:b5:7a:7f:3e:98:fb:76:9c:1f:a7:b0:9c:2c:44:d5:ae +-----BEGIN CERTIFICATE----- +MIIFdDCCA1ygAwIBAgIQVW9l47TZkGobCdFsPsBsIDANBgkqhkiG9w0BAQsFADBU +MQswCQYDVQQGEwJDTjEmMCQGA1UECgwdQkVJSklORyBDRVJUSUZJQ0FURSBBVVRI +T1JJVFkxHTAbBgNVBAMMFEJKQ0EgR2xvYmFsIFJvb3QgQ0ExMB4XDTE5MTIxOTAz +MTYxN1oXDTQ0MTIxMjAzMTYxN1owVDELMAkGA1UEBhMCQ04xJjAkBgNVBAoMHUJF +SUpJTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZMR0wGwYDVQQDDBRCSkNBIEdsb2Jh +bCBSb290IENBMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAPFmCL3Z +xRVhy4QEQaVpN3cdwbB7+sN3SJATcmTRuHyQNZ0YeYjjlwE8R4HyDqKYDZ4/N+AZ +spDyRhySsTphzvq3Rp4Dhtczbu33RYx2N95ulpH3134rhxfVizXuhJFyV9xgw8O5 +58dnJCNPYwpj9mZ9S1WnP3hkSWkSl+BMDdMJoDIwOvqfwPKcxRIqLhy1BDPapDgR +at7GGPZHOiJBhyL8xIkoVNiMpTAK+BcWyqw3/XmnkRd4OJmtWO2y3syJfQOcs4ll +5+M7sSKGjwZteAf9kRJ/sGsciQ35uMt0WwfCyPQ10WRjeulumijWML3mG90Vr4Tq +nMfK9Q7q8l0ph49pczm+LiRvRSGsxdRpJQaDrXpIhRMsDQa4bHlW/KNnMoH1V6XK +V0Jp6VwkYe/iMBhORJhVb3rCk9gZtt58R4oRTklH2yiUAguUSiz5EtBP6DF+bHq/ +pj+bOT0CFqMYs2esWz8sgytnOYFcuX6U1WTdno9uruh8W7TXakdI136z1C2OVnZO +z2nxbkRs1CTqjSShGL+9V/6pmTW12xB3uD1IutbB5/EjPtffhZ0nPNRAvQoMvfXn +jSXWgXSHRtQpdaJCbPdzied9v3pKH9MiyRVVz99vfFXQpIsHETdfg6YmV6YBW37+ +WGgHqel62bno/1Afq8K0wM7o6v0PvY1NuLxxAgMBAAGjQjBAMB0GA1UdDgQWBBTF +7+3M2I0hxkjk49cULqcWk+WYATAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQE +AwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAUoKsITQfI/Ki2Pm4rzc2IInRNwPWaZ+4 +YRC6ojGYWUfo0Q0lHhVBDOAqVdVXUsv45Mdpox1NcQJeXyFFYEhcCY5JEMEE3Kli +awLwQ8hOnThJdMkycFRtwUf8jrQ2ntScvd0g1lPJGKm1Vrl2i5VnZu69mP6u775u ++2D2/VnGKhs/I0qUJDAnyIm860Qkmss9vk/Ves6OF8tiwdneHg56/0OGNFK8YT88 +X7vZdrRTvJez/opMEi4r89fO4aL/3Xtw+zuhTaRjAv04l5U/BXCga99igUOLtFkN +SoxUnMW7gZ/NfaXvCyUeOiDbHPwfmGcCCtRzRBPbUYQaVQNW4AB+dAb/OMRyHdOo +P2gxXdMJxy6MW2Pg6Nwe0uxhHvLe5e/2mXZgLR6UcnHGCyoyx5JO1UbXHfmpGQrI ++pXObSOYqgs4rZpWDW+N8TEAiMEXnM0ZNjX+VVOg4DwzX5Ze4jLp3zO7Bkqp2IRz +znfSxqxx4VyjHQy7Ct9f4qNx2No3WqB4K/TUfet27fJhcKVlmtOJNBir+3I+17Q9 +eVzYH6Eze9mCUAyTF6ps3MKCuwJXNq+YJyo5UOGwifUll35HaBC07HPKs5fRJNz2 +YqAo07WjuGS3iGJCz51TzZm+ZGiPTx4SSPfSKcOYKMryMguTjClPPGAyzQWWYezy +r/6zcCwupvI= +-----END CERTIFICATE----- + +# Issuer: CN=BJCA Global Root CA2 O=BEIJING CERTIFICATE AUTHORITY +# Subject: CN=BJCA Global Root CA2 O=BEIJING CERTIFICATE AUTHORITY +# Label: "BJCA Global Root CA2" +# Serial: 58605626836079930195615843123109055211 +# MD5 Fingerprint: 5e:0a:f6:47:5f:a6:14:e8:11:01:95:3f:4d:01:eb:3c +# SHA1 Fingerprint: f4:27:86:eb:6e:b8:6d:88:31:67:02:fb:ba:66:a4:53:00:aa:7a:a6 +# SHA256 Fingerprint: 57:4d:f6:93:1e:27:80:39:66:7b:72:0a:fd:c1:60:0f:c2:7e:b6:6d:d3:09:29:79:fb:73:85:64:87:21:28:82 +-----BEGIN CERTIFICATE----- +MIICJTCCAaugAwIBAgIQLBcIfWQqwP6FGFkGz7RK6zAKBggqhkjOPQQDAzBUMQsw +CQYDVQQGEwJDTjEmMCQGA1UECgwdQkVJSklORyBDRVJUSUZJQ0FURSBBVVRIT1JJ +VFkxHTAbBgNVBAMMFEJKQ0EgR2xvYmFsIFJvb3QgQ0EyMB4XDTE5MTIxOTAzMTgy +MVoXDTQ0MTIxMjAzMTgyMVowVDELMAkGA1UEBhMCQ04xJjAkBgNVBAoMHUJFSUpJ +TkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZMR0wGwYDVQQDDBRCSkNBIEdsb2JhbCBS +b290IENBMjB2MBAGByqGSM49AgEGBSuBBAAiA2IABJ3LgJGNU2e1uVCxA/jlSR9B +IgmwUVJY1is0j8USRhTFiy8shP8sbqjV8QnjAyEUxEM9fMEsxEtqSs3ph+B99iK+ ++kpRuDCK/eHeGBIK9ke35xe/J4rUQUyWPGCWwf0VHKNCMEAwHQYDVR0OBBYEFNJK +sVF/BvDRgh9Obl+rg/xI1LCRMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD +AgEGMAoGCCqGSM49BAMDA2gAMGUCMBq8W9f+qdJUDkpd0m2xQNz0Q9XSSpkZElaA +94M04TVOSG0ED1cxMDAtsaqdAzjbBgIxAMvMh1PLet8gUXOQwKhbYdDFUDn9hf7B +43j4ptZLvZuHjw/l1lOWqzzIQNph91Oj9w== +-----END CERTIFICATE----- + +# Issuer: CN=Sectigo Public Server Authentication Root E46 O=Sectigo Limited +# Subject: CN=Sectigo Public Server Authentication Root E46 O=Sectigo Limited +# Label: "Sectigo Public Server Authentication Root E46" +# Serial: 88989738453351742415770396670917916916 +# MD5 Fingerprint: 28:23:f8:b2:98:5c:37:16:3b:3e:46:13:4e:b0:b3:01 +# SHA1 Fingerprint: ec:8a:39:6c:40:f0:2e:bc:42:75:d4:9f:ab:1c:1a:5b:67:be:d2:9a +# SHA256 Fingerprint: c9:0f:26:f0:fb:1b:40:18:b2:22:27:51:9b:5c:a2:b5:3e:2c:a5:b3:be:5c:f1:8e:fe:1b:ef:47:38:0c:53:83 +-----BEGIN CERTIFICATE----- +MIICOjCCAcGgAwIBAgIQQvLM2htpN0RfFf51KBC49DAKBggqhkjOPQQDAzBfMQsw +CQYDVQQGEwJHQjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1T +ZWN0aWdvIFB1YmxpYyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBFNDYwHhcN +MjEwMzIyMDAwMDAwWhcNNDYwMzIxMjM1OTU5WjBfMQswCQYDVQQGEwJHQjEYMBYG +A1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1TZWN0aWdvIFB1YmxpYyBT +ZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBFNDYwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAAR2+pmpbiDt+dd34wc7qNs9Xzjoq1WmVk/WSOrsfy2qw7LFeeyZYX8QeccC +WvkEN/U0NSt3zn8gj1KjAIns1aeibVvjS5KToID1AZTc8GgHHs3u/iVStSBDHBv+ +6xnOQ6OjQjBAMB0GA1UdDgQWBBTRItpMWfFLXyY4qp3W7usNw/upYTAOBgNVHQ8B +Af8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNnADBkAjAn7qRa +qCG76UeXlImldCBteU/IvZNeWBj7LRoAasm4PdCkT0RHlAFWovgzJQxC36oCMB3q +4S6ILuH5px0CMk7yn2xVdOOurvulGu7t0vzCAxHrRVxgED1cf5kDW21USAGKcw== +-----END CERTIFICATE----- + +# Issuer: CN=Sectigo Public Server Authentication Root R46 O=Sectigo Limited +# Subject: CN=Sectigo Public Server Authentication Root R46 O=Sectigo Limited +# Label: "Sectigo Public Server Authentication Root R46" +# Serial: 156256931880233212765902055439220583700 +# MD5 Fingerprint: 32:10:09:52:00:d5:7e:6c:43:df:15:c0:b1:16:93:e5 +# SHA1 Fingerprint: ad:98:f9:f3:e4:7d:75:3b:65:d4:82:b3:a4:52:17:bb:6e:f5:e4:38 +# SHA256 Fingerprint: 7b:b6:47:a6:2a:ee:ac:88:bf:25:7a:a5:22:d0:1f:fe:a3:95:e0:ab:45:c7:3f:93:f6:56:54:ec:38:f2:5a:06 +-----BEGIN CERTIFICATE----- +MIIFijCCA3KgAwIBAgIQdY39i658BwD6qSWn4cetFDANBgkqhkiG9w0BAQwFADBf +MQswCQYDVQQGEwJHQjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQD +Ey1TZWN0aWdvIFB1YmxpYyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBSNDYw +HhcNMjEwMzIyMDAwMDAwWhcNNDYwMzIxMjM1OTU5WjBfMQswCQYDVQQGEwJHQjEY +MBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1TZWN0aWdvIFB1Ymxp +YyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBSNDYwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCTvtU2UnXYASOgHEdCSe5jtrch/cSV1UgrJnwUUxDa +ef0rty2k1Cz66jLdScK5vQ9IPXtamFSvnl0xdE8H/FAh3aTPaE8bEmNtJZlMKpnz +SDBh+oF8HqcIStw+KxwfGExxqjWMrfhu6DtK2eWUAtaJhBOqbchPM8xQljeSM9xf +iOefVNlI8JhD1mb9nxc4Q8UBUQvX4yMPFF1bFOdLvt30yNoDN9HWOaEhUTCDsG3X +ME6WW5HwcCSrv0WBZEMNvSE6Lzzpng3LILVCJ8zab5vuZDCQOc2TZYEhMbUjUDM3 +IuM47fgxMMxF/mL50V0yeUKH32rMVhlATc6qu/m1dkmU8Sf4kaWD5QazYw6A3OAS +VYCmO2a0OYctyPDQ0RTp5A1NDvZdV3LFOxxHVp3i1fuBYYzMTYCQNFu31xR13NgE +SJ/AwSiItOkcyqex8Va3e0lMWeUgFaiEAin6OJRpmkkGj80feRQXEgyDet4fsZfu ++Zd4KKTIRJLpfSYFplhym3kT2BFfrsU4YjRosoYwjviQYZ4ybPUHNs2iTG7sijbt +8uaZFURww3y8nDnAtOFr94MlI1fZEoDlSfB1D++N6xybVCi0ITz8fAr/73trdf+L +HaAZBav6+CuBQug4urv7qv094PPK306Xlynt8xhW6aWWrL3DkJiy4Pmi1KZHQ3xt +zwIDAQABo0IwQDAdBgNVHQ4EFgQUVnNYZJX5khqwEioEYnmhQBWIIUkwDgYDVR0P +AQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAC9c +mTz8Bl6MlC5w6tIyMY208FHVvArzZJ8HXtXBc2hkeqK5Duj5XYUtqDdFqij0lgVQ +YKlJfp/imTYpE0RHap1VIDzYm/EDMrraQKFz6oOht0SmDpkBm+S8f74TlH7Kph52 +gDY9hAaLMyZlbcp+nv4fjFg4exqDsQ+8FxG75gbMY/qB8oFM2gsQa6H61SilzwZA +Fv97fRheORKkU55+MkIQpiGRqRxOF3yEvJ+M0ejf5lG5Nkc/kLnHvALcWxxPDkjB +JYOcCj+esQMzEhonrPcibCTRAUH4WAP+JWgiH5paPHxsnnVI84HxZmduTILA7rpX +DhjvLpr3Etiga+kFpaHpaPi8TD8SHkXoUsCjvxInebnMMTzD9joiFgOgyY9mpFui +TdaBJQbpdqQACj7LzTWb4OE4y2BThihCQRxEV+ioratF4yUQvNs+ZUH7G6aXD+u5 +dHn5HrwdVw1Hr8Mvn4dGp+smWg9WY7ViYG4A++MnESLn/pmPNPW56MORcr3Ywx65 +LvKRRFHQV80MNNVIIb/bE/FmJUNS0nAiNs2fxBx1IK1jcmMGDw4nztJqDby1ORrp +0XZ60Vzk50lJLVU3aPAaOpg+VBeHVOmmJ1CJeyAvP/+/oYtKR5j/K3tJPsMpRmAY +QqszKbrAKbkTidOIijlBO8n9pu0f9GBj39ItVQGL +-----END CERTIFICATE----- + +# Issuer: CN=SSL.com TLS RSA Root CA 2022 O=SSL Corporation +# Subject: CN=SSL.com TLS RSA Root CA 2022 O=SSL Corporation +# Label: "SSL.com TLS RSA Root CA 2022" +# Serial: 148535279242832292258835760425842727825 +# MD5 Fingerprint: d8:4e:c6:59:30:d8:fe:a0:d6:7a:5a:2c:2c:69:78:da +# SHA1 Fingerprint: ec:2c:83:40:72:af:26:95:10:ff:0e:f2:03:ee:31:70:f6:78:9d:ca +# SHA256 Fingerprint: 8f:af:7d:2e:2c:b4:70:9b:b8:e0:b3:36:66:bf:75:a5:dd:45:b5:de:48:0f:8e:a8:d4:bf:e6:be:bc:17:f2:ed +-----BEGIN CERTIFICATE----- +MIIFiTCCA3GgAwIBAgIQb77arXO9CEDii02+1PdbkTANBgkqhkiG9w0BAQsFADBO +MQswCQYDVQQGEwJVUzEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMSUwIwYDVQQD +DBxTU0wuY29tIFRMUyBSU0EgUm9vdCBDQSAyMDIyMB4XDTIyMDgyNTE2MzQyMloX +DTQ2MDgxOTE2MzQyMVowTjELMAkGA1UEBhMCVVMxGDAWBgNVBAoMD1NTTCBDb3Jw +b3JhdGlvbjElMCMGA1UEAwwcU1NMLmNvbSBUTFMgUlNBIFJvb3QgQ0EgMjAyMjCC +AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANCkCXJPQIgSYT41I57u9nTP +L3tYPc48DRAokC+X94xI2KDYJbFMsBFMF3NQ0CJKY7uB0ylu1bUJPiYYf7ISf5OY +t6/wNr/y7hienDtSxUcZXXTzZGbVXcdotL8bHAajvI9AI7YexoS9UcQbOcGV0ins +S657Lb85/bRi3pZ7QcacoOAGcvvwB5cJOYF0r/c0WRFXCsJbwST0MXMwgsadugL3 +PnxEX4MN8/HdIGkWCVDi1FW24IBydm5MR7d1VVm0U3TZlMZBrViKMWYPHqIbKUBO +L9975hYsLfy/7PO0+r4Y9ptJ1O4Fbtk085zx7AGL0SDGD6C1vBdOSHtRwvzpXGk3 +R2azaPgVKPC506QVzFpPulJwoxJF3ca6TvvC0PeoUidtbnm1jPx7jMEWTO6Af77w +dr5BUxIzrlo4QqvXDz5BjXYHMtWrifZOZ9mxQnUjbvPNQrL8VfVThxc7wDNY8VLS ++YCk8OjwO4s4zKTGkH8PnP2L0aPP2oOnaclQNtVcBdIKQXTbYxE3waWglksejBYS +d66UNHsef8JmAOSqg+qKkK3ONkRN0VHpvB/zagX9wHQfJRlAUW7qglFA35u5CCoG +AtUjHBPW6dvbxrB6y3snm/vg1UYk7RBLY0ulBY+6uB0rpvqR4pJSvezrZ5dtmi2f +gTIFZzL7SAg/2SW4BCUvAgMBAAGjYzBhMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0j +BBgwFoAU+y437uOEeicuzRk1sTN8/9REQrkwHQYDVR0OBBYEFPsuN+7jhHonLs0Z +NbEzfP/UREK5MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAjYlt +hEUY8U+zoO9opMAdrDC8Z2awms22qyIZZtM7QbUQnRC6cm4pJCAcAZli05bg4vsM +QtfhWsSWTVTNj8pDU/0quOr4ZcoBwq1gaAafORpR2eCNJvkLTqVTJXojpBzOCBvf +R4iyrT7gJ4eLSYwfqUdYe5byiB0YrrPRpgqU+tvT5TgKa3kSM/tKWTcWQA673vWJ +DPFs0/dRa1419dvAJuoSc06pkZCmF8NsLzjUo3KUQyxi4U5cMj29TH0ZR6LDSeeW +P4+a0zvkEdiLA9z2tmBVGKaBUfPhqBVq6+AL8BQx1rmMRTqoENjwuSfr98t67wVy +lrXEj5ZzxOhWc5y8aVFjvO9nHEMaX3cZHxj4HCUp+UmZKbaSPaKDN7EgkaibMOlq +bLQjk2UEqxHzDh1TJElTHaE/nUiSEeJ9DU/1172iWD54nR4fK/4huxoTtrEoZP2w +AgDHbICivRZQIA9ygV/MlP+7mea6kMvq+cYMwq7FGc4zoWtcu358NFcXrfA/rs3q +r5nsLFR+jM4uElZI7xc7P0peYNLcdDa8pUNjyw9bowJWCZ4kLOGGgYz+qxcs+sji +Mho6/4UIyYOf8kpIEFR3N+2ivEC+5BB09+Rbu7nzifmPQdjH5FCQNYA+HLhNkNPU +98OwoX6EyneSMSy4kLGCenROmxMmtNVQZlR4rmA= +-----END CERTIFICATE----- + +# Issuer: CN=SSL.com TLS ECC Root CA 2022 O=SSL Corporation +# Subject: CN=SSL.com TLS ECC Root CA 2022 O=SSL Corporation +# Label: "SSL.com TLS ECC Root CA 2022" +# Serial: 26605119622390491762507526719404364228 +# MD5 Fingerprint: 99:d7:5c:f1:51:36:cc:e9:ce:d9:19:2e:77:71:56:c5 +# SHA1 Fingerprint: 9f:5f:d9:1a:54:6d:f5:0c:71:f0:ee:7a:bd:17:49:98:84:73:e2:39 +# SHA256 Fingerprint: c3:2f:fd:9f:46:f9:36:d1:6c:36:73:99:09:59:43:4b:9a:d6:0a:af:bb:9e:7c:f3:36:54:f1:44:cc:1b:a1:43 +-----BEGIN CERTIFICATE----- +MIICOjCCAcCgAwIBAgIQFAP1q/s3ixdAW+JDsqXRxDAKBggqhkjOPQQDAzBOMQsw +CQYDVQQGEwJVUzEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMSUwIwYDVQQDDBxT +U0wuY29tIFRMUyBFQ0MgUm9vdCBDQSAyMDIyMB4XDTIyMDgyNTE2MzM0OFoXDTQ2 +MDgxOTE2MzM0N1owTjELMAkGA1UEBhMCVVMxGDAWBgNVBAoMD1NTTCBDb3Jwb3Jh +dGlvbjElMCMGA1UEAwwcU1NMLmNvbSBUTFMgRUNDIFJvb3QgQ0EgMjAyMjB2MBAG +ByqGSM49AgEGBSuBBAAiA2IABEUpNXP6wrgjzhR9qLFNoFs27iosU8NgCTWyJGYm +acCzldZdkkAZDsalE3D07xJRKF3nzL35PIXBz5SQySvOkkJYWWf9lCcQZIxPBLFN +SeR7T5v15wj4A4j3p8OSSxlUgaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSME +GDAWgBSJjy+j6CugFFR781a4Jl9nOAuc0DAdBgNVHQ4EFgQUiY8vo+groBRUe/NW +uCZfZzgLnNAwDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMDA2gAMGUCMFXjIlbp +15IkWE8elDIPDAI2wv2sdDJO4fscgIijzPvX6yv/N33w7deedWo1dlJF4AIxAMeN +b0Igj762TVntd00pxCAgRWSGOlDGxK0tk/UYfXLtqc/ErFc2KAhl3zx5Zn6g6g== +-----END CERTIFICATE----- + +# Issuer: CN=Atos TrustedRoot Root CA ECC TLS 2021 O=Atos +# Subject: CN=Atos TrustedRoot Root CA ECC TLS 2021 O=Atos +# Label: "Atos TrustedRoot Root CA ECC TLS 2021" +# Serial: 81873346711060652204712539181482831616 +# MD5 Fingerprint: 16:9f:ad:f1:70:ad:79:d6:ed:29:b4:d1:c5:79:70:a8 +# SHA1 Fingerprint: 9e:bc:75:10:42:b3:02:f3:81:f4:f7:30:62:d4:8f:c3:a7:51:b2:dd +# SHA256 Fingerprint: b2:fa:e5:3e:14:cc:d7:ab:92:12:06:47:01:ae:27:9c:1d:89:88:fa:cb:77:5f:a8:a0:08:91:4e:66:39:88:a8 +-----BEGIN CERTIFICATE----- +MIICFTCCAZugAwIBAgIQPZg7pmY9kGP3fiZXOATvADAKBggqhkjOPQQDAzBMMS4w +LAYDVQQDDCVBdG9zIFRydXN0ZWRSb290IFJvb3QgQ0EgRUNDIFRMUyAyMDIxMQ0w +CwYDVQQKDARBdG9zMQswCQYDVQQGEwJERTAeFw0yMTA0MjIwOTI2MjNaFw00MTA0 +MTcwOTI2MjJaMEwxLjAsBgNVBAMMJUF0b3MgVHJ1c3RlZFJvb3QgUm9vdCBDQSBF +Q0MgVExTIDIwMjExDTALBgNVBAoMBEF0b3MxCzAJBgNVBAYTAkRFMHYwEAYHKoZI +zj0CAQYFK4EEACIDYgAEloZYKDcKZ9Cg3iQZGeHkBQcfl+3oZIK59sRxUM6KDP/X +tXa7oWyTbIOiaG6l2b4siJVBzV3dscqDY4PMwL502eCdpO5KTlbgmClBk1IQ1SQ4 +AjJn8ZQSb+/Xxd4u/RmAo0IwQDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR2 +KCXWfeBmmnoJsmo7jjPXNtNPojAOBgNVHQ8BAf8EBAMCAYYwCgYIKoZIzj0EAwMD +aAAwZQIwW5kp85wxtolrbNa9d+F851F+uDrNozZffPc8dz7kUK2o59JZDCaOMDtu +CCrCp1rIAjEAmeMM56PDr9NJLkaCI2ZdyQAUEv049OGYa3cpetskz2VAv9LcjBHo +9H1/IISpQuQo +-----END CERTIFICATE----- + +# Issuer: CN=Atos TrustedRoot Root CA RSA TLS 2021 O=Atos +# Subject: CN=Atos TrustedRoot Root CA RSA TLS 2021 O=Atos +# Label: "Atos TrustedRoot Root CA RSA TLS 2021" +# Serial: 111436099570196163832749341232207667876 +# MD5 Fingerprint: d4:d3:46:b8:9a:c0:9c:76:5d:9e:3a:c3:b9:99:31:d2 +# SHA1 Fingerprint: 18:52:3b:0d:06:37:e4:d6:3a:df:23:e4:98:fb:5b:16:fb:86:74:48 +# SHA256 Fingerprint: 81:a9:08:8e:a5:9f:b3:64:c5:48:a6:f8:55:59:09:9b:6f:04:05:ef:bf:18:e5:32:4e:c9:f4:57:ba:00:11:2f +-----BEGIN CERTIFICATE----- +MIIFZDCCA0ygAwIBAgIQU9XP5hmTC/srBRLYwiqipDANBgkqhkiG9w0BAQwFADBM +MS4wLAYDVQQDDCVBdG9zIFRydXN0ZWRSb290IFJvb3QgQ0EgUlNBIFRMUyAyMDIx +MQ0wCwYDVQQKDARBdG9zMQswCQYDVQQGEwJERTAeFw0yMTA0MjIwOTIxMTBaFw00 +MTA0MTcwOTIxMDlaMEwxLjAsBgNVBAMMJUF0b3MgVHJ1c3RlZFJvb3QgUm9vdCBD +QSBSU0EgVExTIDIwMjExDTALBgNVBAoMBEF0b3MxCzAJBgNVBAYTAkRFMIICIjAN +BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtoAOxHm9BYx9sKOdTSJNy/BBl01Z +4NH+VoyX8te9j2y3I49f1cTYQcvyAh5x5en2XssIKl4w8i1mx4QbZFc4nXUtVsYv +Ye+W/CBGvevUez8/fEc4BKkbqlLfEzfTFRVOvV98r61jx3ncCHvVoOX3W3WsgFWZ +kmGbzSoXfduP9LVq6hdKZChmFSlsAvFr1bqjM9xaZ6cF4r9lthawEO3NUDPJcFDs +GY6wx/J0W2tExn2WuZgIWWbeKQGb9Cpt0xU6kGpn8bRrZtkh68rZYnxGEFzedUln +nkL5/nWpo63/dgpnQOPF943HhZpZnmKaau1Fh5hnstVKPNe0OwANwI8f4UDErmwh +3El+fsqyjW22v5MvoVw+j8rtgI5Y4dtXz4U2OLJxpAmMkokIiEjxQGMYsluMWuPD +0xeqqxmjLBvk1cbiZnrXghmmOxYsL3GHX0WelXOTwkKBIROW1527k2gV+p2kHYzy +geBYBr3JtuP2iV2J+axEoctr+hbxx1A9JNr3w+SH1VbxT5Aw+kUJWdo0zuATHAR8 +ANSbhqRAvNncTFd+rrcztl524WWLZt+NyteYr842mIycg5kDcPOvdO3GDjbnvezB +c6eUWsuSZIKmAMFwoW4sKeFYV+xafJlrJaSQOoD0IJ2azsct+bJLKZWD6TWNp0lI +pw9MGZHQ9b8Q4HECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU +dEmZ0f+0emhFdcN+tNzMzjkz2ggwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB +DAUAA4ICAQAjQ1MkYlxt/T7Cz1UAbMVWiLkO3TriJQ2VSpfKgInuKs1l+NsW4AmS +4BjHeJi78+xCUvuppILXTdiK/ORO/auQxDh1MoSf/7OwKwIzNsAQkG8dnK/haZPs +o0UvFJ/1TCplQ3IM98P4lYsU84UgYt1UU90s3BiVaU+DR3BAM1h3Egyi61IxHkzJ +qM7F78PRreBrAwA0JrRUITWXAdxfG/F851X6LWh3e9NpzNMOa7pNdkTWwhWaJuyw +xfW70Xp0wmzNxbVe9kzmWy2B27O3Opee7c9GslA9hGCZcbUztVdF5kJHdWoOsAgM +rr3e97sPWD2PAzHoPYJQyi9eDF20l74gNAf0xBLh7tew2VktafcxBPTy+av5EzH4 +AXcOPUIjJsyacmdRIXrMPIWo6iFqO9taPKU0nprALN+AnCng33eU0aKAQv9qTFsR +0PXNor6uzFFcw9VUewyu1rkGd4Di7wcaaMxZUa1+XGdrudviB0JbuAEFWDlN5LuY +o7Ey7Nmj1m+UI/87tyll5gfp77YZ6ufCOB0yiJA8EytuzO+rdwY0d4RPcuSBhPm5 +dDTedk+SKlOxJTnbPP/lPqYO5Wue/9vsL3SD3460s6neFE3/MaNFcyT6lSnMEpcE +oji2jbDwN/zIIX8/syQbPYtuzE2wFg2WHYMfRsCbvUOZ58SWLs5fyQ== +-----END CERTIFICATE----- + +# Issuer: CN=TrustAsia Global Root CA G3 O=TrustAsia Technologies, Inc. +# Subject: CN=TrustAsia Global Root CA G3 O=TrustAsia Technologies, Inc. +# Label: "TrustAsia Global Root CA G3" +# Serial: 576386314500428537169965010905813481816650257167 +# MD5 Fingerprint: 30:42:1b:b7:bb:81:75:35:e4:16:4f:53:d2:94:de:04 +# SHA1 Fingerprint: 63:cf:b6:c1:27:2b:56:e4:88:8e:1c:23:9a:b6:2e:81:47:24:c3:c7 +# SHA256 Fingerprint: e0:d3:22:6a:eb:11:63:c2:e4:8f:f9:be:3b:50:b4:c6:43:1b:e7:bb:1e:ac:c5:c3:6b:5d:5e:c5:09:03:9a:08 +-----BEGIN CERTIFICATE----- +MIIFpTCCA42gAwIBAgIUZPYOZXdhaqs7tOqFhLuxibhxkw8wDQYJKoZIhvcNAQEM +BQAwWjELMAkGA1UEBhMCQ04xJTAjBgNVBAoMHFRydXN0QXNpYSBUZWNobm9sb2dp +ZXMsIEluYy4xJDAiBgNVBAMMG1RydXN0QXNpYSBHbG9iYWwgUm9vdCBDQSBHMzAe +Fw0yMTA1MjAwMjEwMTlaFw00NjA1MTkwMjEwMTlaMFoxCzAJBgNVBAYTAkNOMSUw +IwYDVQQKDBxUcnVzdEFzaWEgVGVjaG5vbG9naWVzLCBJbmMuMSQwIgYDVQQDDBtU +cnVzdEFzaWEgR2xvYmFsIFJvb3QgQ0EgRzMwggIiMA0GCSqGSIb3DQEBAQUAA4IC +DwAwggIKAoICAQDAMYJhkuSUGwoqZdC+BqmHO1ES6nBBruL7dOoKjbmzTNyPtxNS +T1QY4SxzlZHFZjtqz6xjbYdT8PfxObegQ2OwxANdV6nnRM7EoYNl9lA+sX4WuDqK +AtCWHwDNBSHvBm3dIZwZQ0WhxeiAysKtQGIXBsaqvPPW5vxQfmZCHzyLpnl5hkA1 +nyDvP+uLRx+PjsXUjrYsyUQE49RDdT/VP68czH5GX6zfZBCK70bwkPAPLfSIC7Ep +qq+FqklYqL9joDiR5rPmd2jE+SoZhLsO4fWvieylL1AgdB4SQXMeJNnKziyhWTXA +yB1GJ2Faj/lN03J5Zh6fFZAhLf3ti1ZwA0pJPn9pMRJpxx5cynoTi+jm9WAPzJMs +hH/x/Gr8m0ed262IPfN2dTPXS6TIi/n1Q1hPy8gDVI+lhXgEGvNz8teHHUGf59gX +zhqcD0r83ERoVGjiQTz+LISGNzzNPy+i2+f3VANfWdP3kXjHi3dqFuVJhZBFcnAv +kV34PmVACxmZySYgWmjBNb9Pp1Hx2BErW+Canig7CjoKH8GB5S7wprlppYiU5msT +f9FkPz2ccEblooV7WIQn3MSAPmeamseaMQ4w7OYXQJXZRe0Blqq/DPNL0WP3E1jA +uPP6Z92bfW1K/zJMtSU7/xxnD4UiWQWRkUF3gdCFTIcQcf+eQxuulXUtgQIDAQAB +o2MwYTAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFEDk5PIj7zjKsK5Xf/Ih +MBY027ySMB0GA1UdDgQWBBRA5OTyI+84yrCuV3/yITAWNNu8kjAOBgNVHQ8BAf8E +BAMCAQYwDQYJKoZIhvcNAQEMBQADggIBACY7UeFNOPMyGLS0XuFlXsSUT9SnYaP4 +wM8zAQLpw6o1D/GUE3d3NZ4tVlFEbuHGLige/9rsR82XRBf34EzC4Xx8MnpmyFq2 +XFNFV1pF1AWZLy4jVe5jaN/TG3inEpQGAHUNcoTpLrxaatXeL1nHo+zSh2bbt1S1 +JKv0Q3jbSwTEb93mPmY+KfJLaHEih6D4sTNjduMNhXJEIlU/HHzp/LgV6FL6qj6j +ITk1dImmasI5+njPtqzn59ZW/yOSLlALqbUHM/Q4X6RJpstlcHboCoWASzY9M/eV +VHUl2qzEc4Jl6VL1XP04lQJqaTDFHApXB64ipCz5xUG3uOyfT0gA+QEEVcys+TIx +xHWVBqB/0Y0n3bOppHKH/lmLmnp0Ft0WpWIp6zqW3IunaFnT63eROfjXy9mPX1on +AX1daBli2MjN9LdyR75bl87yraKZk62Uy5P2EgmVtqvXO9A/EcswFi55gORngS1d +7XB4tmBZrOFdRWOPyN9yaFvqHbgB8X7754qz41SgOAngPN5C8sLtLpvzHzW2Ntjj +gKGLzZlkD8Kqq7HK9W+eQ42EVJmzbsASZthwEPEGNTNDqJwuuhQxzhB/HIbjj9LV ++Hfsm6vxL2PZQl/gZ4FkkfGXL/xuJvYz+NO1+MRiqzFRJQJ6+N1rZdVtTTDIZbpo +FGWsJwt0ivKH +-----END CERTIFICATE----- + +# Issuer: CN=TrustAsia Global Root CA G4 O=TrustAsia Technologies, Inc. +# Subject: CN=TrustAsia Global Root CA G4 O=TrustAsia Technologies, Inc. +# Label: "TrustAsia Global Root CA G4" +# Serial: 451799571007117016466790293371524403291602933463 +# MD5 Fingerprint: 54:dd:b2:d7:5f:d8:3e:ed:7c:e0:0b:2e:cc:ed:eb:eb +# SHA1 Fingerprint: 57:73:a5:61:5d:80:b2:e6:ac:38:82:fc:68:07:31:ac:9f:b5:92:5a +# SHA256 Fingerprint: be:4b:56:cb:50:56:c0:13:6a:52:6d:f4:44:50:8d:aa:36:a0:b5:4f:42:e4:ac:38:f7:2a:f4:70:e4:79:65:4c +-----BEGIN CERTIFICATE----- +MIICVTCCAdygAwIBAgIUTyNkuI6XY57GU4HBdk7LKnQV1tcwCgYIKoZIzj0EAwMw +WjELMAkGA1UEBhMCQ04xJTAjBgNVBAoMHFRydXN0QXNpYSBUZWNobm9sb2dpZXMs +IEluYy4xJDAiBgNVBAMMG1RydXN0QXNpYSBHbG9iYWwgUm9vdCBDQSBHNDAeFw0y +MTA1MjAwMjEwMjJaFw00NjA1MTkwMjEwMjJaMFoxCzAJBgNVBAYTAkNOMSUwIwYD +VQQKDBxUcnVzdEFzaWEgVGVjaG5vbG9naWVzLCBJbmMuMSQwIgYDVQQDDBtUcnVz +dEFzaWEgR2xvYmFsIFJvb3QgQ0EgRzQwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAATx +s8045CVD5d4ZCbuBeaIVXxVjAd7Cq92zphtnS4CDr5nLrBfbK5bKfFJV4hrhPVbw +LxYI+hW8m7tH5j/uqOFMjPXTNvk4XatwmkcN4oFBButJ+bAp3TPsUKV/eSm4IJij +YzBhMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUpbtKl86zK3+kMd6Xg1mD +pm9xy94wHQYDVR0OBBYEFKW7SpfOsyt/pDHel4NZg6ZvccveMA4GA1UdDwEB/wQE +AwIBBjAKBggqhkjOPQQDAwNnADBkAjBe8usGzEkxn0AAbbd+NvBNEU/zy4k6LHiR +UKNbwMp1JvK/kF0LgoxgKJ/GcJpo5PECMFxYDlZ2z1jD1xCMuo6u47xkdUfFVZDj +/bpV6wfEU6s3qe4hsiFbYI89MvHVI5TWWA== +-----END CERTIFICATE----- + +# Issuer: CN=Telekom Security TLS ECC Root 2020 O=Deutsche Telekom Security GmbH +# Subject: CN=Telekom Security TLS ECC Root 2020 O=Deutsche Telekom Security GmbH +# Label: "Telekom Security TLS ECC Root 2020" +# Serial: 72082518505882327255703894282316633856 +# MD5 Fingerprint: c1:ab:fe:6a:10:2c:03:8d:bc:1c:22:32:c0:85:a7:fd +# SHA1 Fingerprint: c0:f8:96:c5:a9:3b:01:06:21:07:da:18:42:48:bc:e9:9d:88:d5:ec +# SHA256 Fingerprint: 57:8a:f4:de:d0:85:3f:4e:59:98:db:4a:ea:f9:cb:ea:8d:94:5f:60:b6:20:a3:8d:1a:3c:13:b2:bc:7b:a8:e1 +-----BEGIN CERTIFICATE----- +MIICQjCCAcmgAwIBAgIQNjqWjMlcsljN0AFdxeVXADAKBggqhkjOPQQDAzBjMQsw +CQYDVQQGEwJERTEnMCUGA1UECgweRGV1dHNjaGUgVGVsZWtvbSBTZWN1cml0eSBH +bWJIMSswKQYDVQQDDCJUZWxla29tIFNlY3VyaXR5IFRMUyBFQ0MgUm9vdCAyMDIw +MB4XDTIwMDgyNTA3NDgyMFoXDTQ1MDgyNTIzNTk1OVowYzELMAkGA1UEBhMCREUx +JzAlBgNVBAoMHkRldXRzY2hlIFRlbGVrb20gU2VjdXJpdHkgR21iSDErMCkGA1UE +AwwiVGVsZWtvbSBTZWN1cml0eSBUTFMgRUNDIFJvb3QgMjAyMDB2MBAGByqGSM49 +AgEGBSuBBAAiA2IABM6//leov9Wq9xCazbzREaK9Z0LMkOsVGJDZos0MKiXrPk/O +tdKPD/M12kOLAoC+b1EkHQ9rK8qfwm9QMuU3ILYg/4gND21Ju9sGpIeQkpT0CdDP +f8iAC8GXs7s1J8nCG6NCMEAwHQYDVR0OBBYEFONyzG6VmUex5rNhTNHLq+O6zd6f +MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cA +MGQCMHVSi7ekEE+uShCLsoRbQuHmKjYC2qBuGT8lv9pZMo7k+5Dck2TOrbRBR2Di +z6fLHgIwN0GMZt9Ba9aDAEH9L1r3ULRn0SyocddDypwnJJGDSA3PzfdUga/sf+Rn +27iQ7t0l +-----END CERTIFICATE----- + +# Issuer: CN=Telekom Security TLS RSA Root 2023 O=Deutsche Telekom Security GmbH +# Subject: CN=Telekom Security TLS RSA Root 2023 O=Deutsche Telekom Security GmbH +# Label: "Telekom Security TLS RSA Root 2023" +# Serial: 44676229530606711399881795178081572759 +# MD5 Fingerprint: bf:5b:eb:54:40:cd:48:71:c4:20:8d:7d:de:0a:42:f2 +# SHA1 Fingerprint: 54:d3:ac:b3:bd:57:56:f6:85:9d:ce:e5:c3:21:e2:d4:ad:83:d0:93 +# SHA256 Fingerprint: ef:c6:5c:ad:bb:59:ad:b6:ef:e8:4d:a2:23:11:b3:56:24:b7:1b:3b:1e:a0:da:8b:66:55:17:4e:c8:97:86:46 +-----BEGIN CERTIFICATE----- +MIIFszCCA5ugAwIBAgIQIZxULej27HF3+k7ow3BXlzANBgkqhkiG9w0BAQwFADBj +MQswCQYDVQQGEwJERTEnMCUGA1UECgweRGV1dHNjaGUgVGVsZWtvbSBTZWN1cml0 +eSBHbWJIMSswKQYDVQQDDCJUZWxla29tIFNlY3VyaXR5IFRMUyBSU0EgUm9vdCAy +MDIzMB4XDTIzMDMyODEyMTY0NVoXDTQ4MDMyNzIzNTk1OVowYzELMAkGA1UEBhMC +REUxJzAlBgNVBAoMHkRldXRzY2hlIFRlbGVrb20gU2VjdXJpdHkgR21iSDErMCkG +A1UEAwwiVGVsZWtvbSBTZWN1cml0eSBUTFMgUlNBIFJvb3QgMjAyMzCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAO01oYGA88tKaVvC+1GDrib94W7zgRJ9 +cUD/h3VCKSHtgVIs3xLBGYSJwb3FKNXVS2xE1kzbB5ZKVXrKNoIENqil/Cf2SfHV +cp6R+SPWcHu79ZvB7JPPGeplfohwoHP89v+1VmLhc2o0mD6CuKyVU/QBoCcHcqMA +U6DksquDOFczJZSfvkgdmOGjup5czQRxUX11eKvzWarE4GC+j4NSuHUaQTXtvPM6 +Y+mpFEXX5lLRbtLevOP1Czvm4MS9Q2QTps70mDdsipWol8hHD/BeEIvnHRz+sTug +BTNoBUGCwQMrAcjnj02r6LX2zWtEtefdi+zqJbQAIldNsLGyMcEWzv/9FIS3R/qy +8XDe24tsNlikfLMR0cN3f1+2JeANxdKz+bi4d9s3cXFH42AYTyS2dTd4uaNir73J +co4vzLuu2+QVUhkHM/tqty1LkCiCc/4YizWN26cEar7qwU02OxY2kTLvtkCJkUPg +8qKrBC7m8kwOFjQgrIfBLX7JZkcXFBGk8/ehJImr2BrIoVyxo/eMbcgByU/J7MT8 +rFEz0ciD0cmfHdRHNCk+y7AO+oMLKFjlKdw/fKifybYKu6boRhYPluV75Gp6SG12 +mAWl3G0eQh5C2hrgUve1g8Aae3g1LDj1H/1Joy7SWWO/gLCMk3PLNaaZlSJhZQNg ++y+TS/qanIA7AgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUtqeX +gj10hZv3PJ+TmpV5dVKMbUcwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBS2 +p5eCPXSFm/c8n5OalXl1UoxtRzANBgkqhkiG9w0BAQwFAAOCAgEAqMxhpr51nhVQ +pGv7qHBFfLp+sVr8WyP6Cnf4mHGCDG3gXkaqk/QeoMPhk9tLrbKmXauw1GLLXrtm +9S3ul0A8Yute1hTWjOKWi0FpkzXmuZlrYrShF2Y0pmtjxrlO8iLpWA1WQdH6DErw +M807u20hOq6OcrXDSvvpfeWxm4bu4uB9tPcy/SKE8YXJN3nptT+/XOR0so8RYgDd +GGah2XsjX/GO1WfoVNpbOms2b/mBsTNHM3dA+VKq3dSDz4V4mZqTuXNnQkYRIer+ +CqkbGmVps4+uFrb2S1ayLfmlyOw7YqPta9BO1UAJpB+Y1zqlklkg5LB9zVtzaL1t +xKITDmcZuI1CfmwMmm6gJC3VRRvcxAIU/oVbZZfKTpBQCHpCNfnqwmbU+AGuHrS+ +w6jv/naaoqYfRvaE7fzbzsQCzndILIyy7MMAo+wsVRjBfhnu4S/yrYObnqsZ38aK +L4x35bcF7DvB7L6Gs4a8wPfc5+pbrrLMtTWGS9DiP7bY+A4A7l3j941Y/8+LN+lj +X273CXE2whJdV/LItM3z7gLfEdxquVeEHVlNjM7IDiPCtyaaEBRx/pOyiriA8A4Q +ntOoUAw3gi/q4Iqd4Sw5/7W0cwDk90imc6y/st53BIe0o82bNSQ3+pCTE4FCxpgm +dTdmQRCsu/WU48IxK63nI1bMNSWSs1A= +-----END CERTIFICATE----- + +# Issuer: CN=TWCA CYBER Root CA O=TAIWAN-CA OU=Root CA +# Subject: CN=TWCA CYBER Root CA O=TAIWAN-CA OU=Root CA +# Label: "TWCA CYBER Root CA" +# Serial: 85076849864375384482682434040119489222 +# MD5 Fingerprint: 0b:33:a0:97:52:95:d4:a9:fd:bb:db:6e:a3:55:5b:51 +# SHA1 Fingerprint: f6:b1:1c:1a:83:38:e9:7b:db:b3:a8:c8:33:24:e0:2d:9c:7f:26:66 +# SHA256 Fingerprint: 3f:63:bb:28:14:be:17:4e:c8:b6:43:9c:f0:8d:6d:56:f0:b7:c4:05:88:3a:56:48:a3:34:42:4d:6b:3e:c5:58 +-----BEGIN CERTIFICATE----- +MIIFjTCCA3WgAwIBAgIQQAE0jMIAAAAAAAAAATzyxjANBgkqhkiG9w0BAQwFADBQ +MQswCQYDVQQGEwJUVzESMBAGA1UEChMJVEFJV0FOLUNBMRAwDgYDVQQLEwdSb290 +IENBMRswGQYDVQQDExJUV0NBIENZQkVSIFJvb3QgQ0EwHhcNMjIxMTIyMDY1NDI5 +WhcNNDcxMTIyMTU1OTU5WjBQMQswCQYDVQQGEwJUVzESMBAGA1UEChMJVEFJV0FO +LUNBMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJUV0NBIENZQkVSIFJvb3Qg +Q0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDG+Moe2Qkgfh1sTs6P +40czRJzHyWmqOlt47nDSkvgEs1JSHWdyKKHfi12VCv7qze33Kc7wb3+szT3vsxxF +avcokPFhV8UMxKNQXd7UtcsZyoC5dc4pztKFIuwCY8xEMCDa6pFbVuYdHNWdZsc/ +34bKS1PE2Y2yHer43CdTo0fhYcx9tbD47nORxc5zb87uEB8aBs/pJ2DFTxnk684i +JkXXYJndzk834H/nY62wuFm40AZoNWDTNq5xQwTxaWV4fPMf88oon1oglWa0zbfu +j3ikRRjpJi+NmykosaS3Om251Bw4ckVYsV7r8Cibt4LK/c/WMw+f+5eesRycnupf +Xtuq3VTpMCEobY5583WSjCb+3MX2w7DfRFlDo7YDKPYIMKoNM+HvnKkHIuNZW0CP +2oi3aQiotyMuRAlZN1vH4xfyIutuOVLF3lSnmMlLIJXcRolftBL5hSmO68gnFSDA +S9TMfAxsNAwmmyYxpjyn9tnQS6Jk/zuZQXLB4HCX8SS7K8R0IrGsayIyJNN4KsDA +oS/xUgXJP+92ZuJF2A09rZXIx4kmyA+upwMu+8Ff+iDhcK2wZSA3M2Cw1a/XDBzC +kHDXShi8fgGwsOsVHkQGzaRP6AzRwyAQ4VRlnrZR0Bp2a0JaWHY06rc3Ga4udfmW +5cFZ95RXKSWNOkyrTZpB0F8mAwIDAQABo2MwYTAOBgNVHQ8BAf8EBAMCAQYwDwYD +VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBSdhWEUfMFib5do5E83QOGt4A1WNzAd +BgNVHQ4EFgQUnYVhFHzBYm+XaORPN0DhreANVjcwDQYJKoZIhvcNAQEMBQADggIB +AGSPesRiDrWIzLjHhg6hShbNcAu3p4ULs3a2D6f/CIsLJc+o1IN1KriWiLb73y0t +tGlTITVX1olNc79pj3CjYcya2x6a4CD4bLubIp1dhDGaLIrdaqHXKGnK/nZVekZn +68xDiBaiA9a5F/gZbG0jAn/xX9AKKSM70aoK7akXJlQKTcKlTfjF/biBzysseKNn +TKkHmvPfXvt89YnNdJdhEGoHK4Fa0o635yDRIG4kqIQnoVesqlVYL9zZyvpoBJ7t +RCT5dEA7IzOrg1oYJkK2bVS1FmAwbLGg+LhBoF1JSdJlBTrq/p1hvIbZv97Tujqx +f36SNI7JAG7cmL3c7IAFrQI932XtCwP39xaEBDG6k5TY8hL4iuO/Qq+n1M0RFxbI +Qh0UqEL20kCGoE8jypZFVmAGzbdVAaYBlGX+bgUJurSkquLvWL69J1bY73NxW0Qz +8ppy6rBePm6pUlvscG21h483XjyMnM7k8M4MZ0HMzvaAq07MTFb1wWFZk7Q+ptq4 +NxKfKjLji7gh7MMrZQzvIt6IKTtM1/r+t+FHvpw+PoP7UV31aPcuIYXcv/Fa4nzX +xeSDwWrruoBa3lwtcHb4yOWHh8qgnaHlIhInD0Q9HWzq1MKLL295q39QpsQZp6F6 +t5b5wR9iWqJDB0BeJsas7a5wFsWqynKKTbDPAYsDP27X +-----END CERTIFICATE----- + +# Issuer: CN=SecureSign Root CA12 O=Cybertrust Japan Co., Ltd. +# Subject: CN=SecureSign Root CA12 O=Cybertrust Japan Co., Ltd. +# Label: "SecureSign Root CA12" +# Serial: 587887345431707215246142177076162061960426065942 +# MD5 Fingerprint: c6:89:ca:64:42:9b:62:08:49:0b:1e:7f:e9:07:3d:e8 +# SHA1 Fingerprint: 7a:22:1e:3d:de:1b:06:ac:9e:c8:47:70:16:8e:3c:e5:f7:6b:06:f4 +# SHA256 Fingerprint: 3f:03:4b:b5:70:4d:44:b2:d0:85:45:a0:20:57:de:93:eb:f3:90:5f:ce:72:1a:cb:c7:30:c0:6d:da:ee:90:4e +-----BEGIN CERTIFICATE----- +MIIDcjCCAlqgAwIBAgIUZvnHwa/swlG07VOX5uaCwysckBYwDQYJKoZIhvcNAQEL +BQAwUTELMAkGA1UEBhMCSlAxIzAhBgNVBAoTGkN5YmVydHJ1c3QgSmFwYW4gQ28u +LCBMdGQuMR0wGwYDVQQDExRTZWN1cmVTaWduIFJvb3QgQ0ExMjAeFw0yMDA0MDgw +NTM2NDZaFw00MDA0MDgwNTM2NDZaMFExCzAJBgNVBAYTAkpQMSMwIQYDVQQKExpD +eWJlcnRydXN0IEphcGFuIENvLiwgTHRkLjEdMBsGA1UEAxMUU2VjdXJlU2lnbiBS +b290IENBMTIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6OcE3emhF +KxS06+QT61d1I02PJC0W6K6OyX2kVzsqdiUzg2zqMoqUm048luT9Ub+ZyZN+v/mt +p7JIKwccJ/VMvHASd6SFVLX9kHrko+RRWAPNEHl57muTH2SOa2SroxPjcf59q5zd +J1M3s6oYwlkm7Fsf0uZlfO+TvdhYXAvA42VvPMfKWeP+bl+sg779XSVOKik71gur +FzJ4pOE+lEa+Ym6b3kaosRbnhW70CEBFEaCeVESE99g2zvVQR9wsMJvuwPWW0v4J +hscGWa5Pro4RmHvzC1KqYiaqId+OJTN5lxZJjfU+1UefNzFJM3IFTQy2VYzxV4+K +h9GtxRESOaCtAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD +AgEGMB0GA1UdDgQWBBRXNPN0zwRL1SXm8UC2LEzZLemgrTANBgkqhkiG9w0BAQsF +AAOCAQEAPrvbFxbS8hQBICw4g0utvsqFepq2m2um4fylOqyttCg6r9cBg0krY6Ld +mmQOmFxv3Y67ilQiLUoT865AQ9tPkbeGGuwAtEGBpE/6aouIs3YIcipJQMPTw4WJ +mBClnW8Zt7vPemVV2zfrPIpyMpcemik+rY3moxtt9XUa5rBouVui7mlHJzWhhpmA +8zNL4WukJsPvdFlseqJkth5Ew1DgDzk9qTPxpfPSvWKErI4cqc1avTc7bgoitPQV +55FYxTpE05Uo2cBl6XLK0A+9H7MV2anjpEcJnuDLN/v9vZfVvhgaaaI5gdka9at/ +yOPiZwud9AzqVN/Ssq+xIvEg37xEHA== +-----END CERTIFICATE----- + +# Issuer: CN=SecureSign Root CA14 O=Cybertrust Japan Co., Ltd. +# Subject: CN=SecureSign Root CA14 O=Cybertrust Japan Co., Ltd. +# Label: "SecureSign Root CA14" +# Serial: 575790784512929437950770173562378038616896959179 +# MD5 Fingerprint: 71:0d:72:fa:92:19:65:5e:89:04:ac:16:33:f0:bc:d5 +# SHA1 Fingerprint: dd:50:c0:f7:79:b3:64:2e:74:a2:b8:9d:9f:d3:40:dd:bb:f0:f2:4f +# SHA256 Fingerprint: 4b:00:9c:10:34:49:4f:9a:b5:6b:ba:3b:a1:d6:27:31:fc:4d:20:d8:95:5a:dc:ec:10:a9:25:60:72:61:e3:38 +-----BEGIN CERTIFICATE----- +MIIFcjCCA1qgAwIBAgIUZNtaDCBO6Ncpd8hQJ6JaJ90t8sswDQYJKoZIhvcNAQEM +BQAwUTELMAkGA1UEBhMCSlAxIzAhBgNVBAoTGkN5YmVydHJ1c3QgSmFwYW4gQ28u +LCBMdGQuMR0wGwYDVQQDExRTZWN1cmVTaWduIFJvb3QgQ0ExNDAeFw0yMDA0MDgw +NzA2MTlaFw00NTA0MDgwNzA2MTlaMFExCzAJBgNVBAYTAkpQMSMwIQYDVQQKExpD +eWJlcnRydXN0IEphcGFuIENvLiwgTHRkLjEdMBsGA1UEAxMUU2VjdXJlU2lnbiBS +b290IENBMTQwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDF0nqh1oq/ +FjHQmNE6lPxauG4iwWL3pwon71D2LrGeaBLwbCRjOfHw3xDG3rdSINVSW0KZnvOg +vlIfX8xnbacuUKLBl422+JX1sLrcneC+y9/3OPJH9aaakpUqYllQC6KxNedlsmGy +6pJxaeQp8E+BgQQ8sqVb1MWoWWd7VRxJq3qdwudzTe/NCcLEVxLbAQ4jeQkHO6Lo +/IrPj8BGJJw4J+CDnRugv3gVEOuGTgpa/d/aLIJ+7sr2KeH6caH3iGicnPCNvg9J +kdjqOvn90Ghx2+m1K06Ckm9mH+Dw3EzsytHqunQG+bOEkJTRX45zGRBdAuVwpcAQ +0BB8b8VYSbSwbprafZX1zNoCr7gsfXmPvkPx+SgojQlD+Ajda8iLLCSxjVIHvXib +y8posqTdDEx5YMaZ0ZPxMBoH064iwurO8YQJzOAUbn8/ftKChazcqRZOhaBgy/ac +18izju3Gm5h1DVXoX+WViwKkrkMpKBGk5hIwAUt1ax5mnXkvpXYvHUC0bcl9eQjs +0Wq2XSqypWa9a4X0dFbD9ed1Uigspf9mR6XU/v6eVL9lfgHWMI+lNpyiUBzuOIAB +SMbHdPTGrMNASRZhdCyvjG817XsYAFs2PJxQDcqSMxDxJklt33UkN4Ii1+iW/RVL +ApY+B3KVfqs9TC7XyvDf4Fg/LS8EmjijAQIDAQABo0IwQDAPBgNVHRMBAf8EBTAD +AQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUBpOjCl4oaTeqYR3r6/wtbyPk +86AwDQYJKoZIhvcNAQEMBQADggIBAJaAcgkGfpzMkwQWu6A6jZJOtxEaCnFxEM0E +rX+lRVAQZk5KQaID2RFPeje5S+LGjzJmdSX7684/AykmjbgWHfYfM25I5uj4V7Ib +ed87hwriZLoAymzvftAj63iP/2SbNDefNWWipAA9EiOWWF3KY4fGoweITedpdopT +zfFP7ELyk+OZpDc8h7hi2/DsHzc/N19DzFGdtfCXwreFamgLRB7lUe6TzktuhsHS +DCRZNhqfLJGP4xjblJUK7ZGqDpncllPjYYPGFrojutzdfhrGe0K22VoF3Jpf1d+4 +2kd92jjbrDnVHmtsKheMYc2xbXIBw8MgAGJoFjHVdqqGuw6qnsb58Nn4DSEC5MUo +FlkRudlpcyqSeLiSV5sI8jrlL5WwWLdrIBRtFO8KvH7YVdiI2i/6GaX7i+B/OfVy +K4XELKzvGUWSTLNhB9xNH27SgRNcmvMSZ4PPmz+Ln52kuaiWA3rF7iDeM9ovnhp6 +dB7h7sxaOgTdsxoEqBRjrLdHEoOabPXm6RUVkRqEGQ6UROcSjiVbgGcZ3GOTEAtl +Lor6CZpO2oYofaphNdgOpygau1LgePhsumywbrmHXumZNTfxPWQrqaA0k89jL9WB +365jJ6UeTo3cKXhZ+PmhIIynJkBugnLNeLLIjzwec+fBH7/PzqUqm9tEZDKgu39c +JRNItX+S +-----END CERTIFICATE----- + +# Issuer: CN=SecureSign Root CA15 O=Cybertrust Japan Co., Ltd. +# Subject: CN=SecureSign Root CA15 O=Cybertrust Japan Co., Ltd. +# Label: "SecureSign Root CA15" +# Serial: 126083514594751269499665114766174399806381178503 +# MD5 Fingerprint: 13:30:fc:c4:62:a6:a9:de:b5:c1:68:af:b5:d2:31:47 +# SHA1 Fingerprint: cb:ba:83:c8:c1:5a:5d:f1:f9:73:6f:ca:d7:ef:28:13:06:4a:07:7d +# SHA256 Fingerprint: e7:78:f0:f0:95:fe:84:37:29:cd:1a:00:82:17:9e:53:14:a9:c2:91:44:28:05:e1:fb:1d:8f:b6:b8:88:6c:3a +-----BEGIN CERTIFICATE----- +MIICIzCCAamgAwIBAgIUFhXHw9hJp75pDIqI7fBw+d23PocwCgYIKoZIzj0EAwMw +UTELMAkGA1UEBhMCSlAxIzAhBgNVBAoTGkN5YmVydHJ1c3QgSmFwYW4gQ28uLCBM +dGQuMR0wGwYDVQQDExRTZWN1cmVTaWduIFJvb3QgQ0ExNTAeFw0yMDA0MDgwODMy +NTZaFw00NTA0MDgwODMyNTZaMFExCzAJBgNVBAYTAkpQMSMwIQYDVQQKExpDeWJl +cnRydXN0IEphcGFuIENvLiwgTHRkLjEdMBsGA1UEAxMUU2VjdXJlU2lnbiBSb290 +IENBMTUwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQLUHSNZDKZmbPSYAi4Io5GdCx4 +wCtELW1fHcmuS1Iggz24FG1Th2CeX2yF2wYUleDHKP+dX+Sq8bOLbe1PL0vJSpSR +ZHX+AezB2Ot6lHhWGENfa4HL9rzatAy2KZMIaY+jQjBAMA8GA1UdEwEB/wQFMAMB +Af8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBTrQciu/NWeUUj1vYv0hyCTQSvT +9DAKBggqhkjOPQQDAwNoADBlAjEA2S6Jfl5OpBEHvVnCB96rMjhTKkZEBhd6zlHp +4P9mLQlO4E/0BdGF9jVg3PVys0Z9AjBEmEYagoUeYWmJSwdLZrWeqrqgHkHZAXQ6 +bkU6iYAZezKYVWOr62Nuk22rGwlgMU4= +-----END CERTIFICATE----- + +# Issuer: CN=D-TRUST BR Root CA 2 2023 O=D-Trust GmbH +# Subject: CN=D-TRUST BR Root CA 2 2023 O=D-Trust GmbH +# Label: "D-TRUST BR Root CA 2 2023" +# Serial: 153168538924886464690566649552453098598 +# MD5 Fingerprint: e1:09:ed:d3:60:d4:56:1b:47:1f:b7:0c:5f:1b:5f:85 +# SHA1 Fingerprint: 2d:b0:70:ee:71:94:af:69:68:17:db:79:ce:58:9f:a0:6b:96:f7:87 +# SHA256 Fingerprint: 05:52:e6:f8:3f:df:65:e8:fa:96:70:e6:66:df:28:a4:e2:13:40:b5:10:cb:e5:25:66:f9:7c:4f:b9:4b:2b:d1 +-----BEGIN CERTIFICATE----- +MIIFqTCCA5GgAwIBAgIQczswBEhb2U14LnNLyaHcZjANBgkqhkiG9w0BAQ0FADBI +MQswCQYDVQQGEwJERTEVMBMGA1UEChMMRC1UcnVzdCBHbWJIMSIwIAYDVQQDExlE +LVRSVVNUIEJSIFJvb3QgQ0EgMiAyMDIzMB4XDTIzMDUwOTA4NTYzMVoXDTM4MDUw +OTA4NTYzMFowSDELMAkGA1UEBhMCREUxFTATBgNVBAoTDEQtVHJ1c3QgR21iSDEi +MCAGA1UEAxMZRC1UUlVTVCBCUiBSb290IENBIDIgMjAyMzCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBAK7/CVmRgApKaOYkP7in5Mg6CjoWzckjYaCTcfKr +i3OPoGdlYNJUa2NRb0kz4HIHE304zQaSBylSa053bATTlfrdTIzZXcFhfUvnKLNE +gXtRr90zsWh81k5M/itoucpmacTsXld/9w3HnDY25QdgrMBM6ghs7wZ8T1soegj8 +k12b9py0i4a6Ibn08OhZWiihNIQaJZG2tY/vsvmA+vk9PBFy2OMvhnbFeSzBqZCT +Rphny4NqoFAjpzv2gTng7fC5v2Xx2Mt6++9zA84A9H3X4F07ZrjcjrqDy4d2A/wl +2ecjbwb9Z/Pg/4S8R7+1FhhGaRTMBffb00msa8yr5LULQyReS2tNZ9/WtT5PeB+U +cSTq3nD88ZP+npNa5JRal1QMNXtfbO4AHyTsA7oC9Xb0n9Sa7YUsOCIvx9gvdhFP +/Wxc6PWOJ4d/GUohR5AdeY0cW/jPSoXk7bNbjb7EZChdQcRurDhaTyN0dKkSw/bS +uREVMweR2Ds3OmMwBtHFIjYoYiMQ4EbMl6zWK11kJNXuHA7e+whadSr2Y23OC0K+ +0bpwHJwh5Q8xaRfX/Aq03u2AnMuStIv13lmiWAmlY0cL4UEyNEHZmrHZqLAbWt4N +DfTisl01gLmB1IRpkQLLddCNxbU9CZEJjxShFHR5PtbJFR2kWVki3PaKRT08EtY+ +XTIvAgMBAAGjgY4wgYswDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUZ5Dw1t61 +GNVGKX5cq/ieCLxklRAwDgYDVR0PAQH/BAQDAgEGMEkGA1UdHwRCMEAwPqA8oDqG +OGh0dHA6Ly9jcmwuZC10cnVzdC5uZXQvY3JsL2QtdHJ1c3RfYnJfcm9vdF9jYV8y +XzIwMjMuY3JsMA0GCSqGSIb3DQEBDQUAA4ICAQA097N3U9swFrktpSHxQCF16+tI +FoE9c+CeJyrrd6kTpGoKWloUMz1oH4Guaf2Mn2VsNELZLdB/eBaxOqwjMa1ef67n +riv6uvw8l5VAk1/DLQOj7aRvU9f6QA4w9QAgLABMjDu0ox+2v5Eyq6+SmNMW5tTR +VFxDWy6u71cqqLRvpO8NVhTaIasgdp4D/Ca4nj8+AybmTNudX0KEPUUDAxxZiMrc +LmEkWqTqJwtzEr5SswrPMhfiHocaFpVIbVrg0M8JkiZmkdijYQ6qgYF/6FKC0ULn +4B0Y+qSFNueG4A3rvNTJ1jxD8V1Jbn6Bm2m1iWKPiFLY1/4nwSPFyysCu7Ff/vtD +hQNGvl3GyiEm/9cCnnRK3PgTFbGBVzbLZVzRHTF36SXDw7IyN9XxmAnkbWOACKsG +koHU6XCPpz+y7YaMgmo1yEJagtFSGkUPFaUA8JR7ZSdXOUPPfH/mvTWze/EZTN46 +ls/pdu4D58JDUjxqgejBWoC9EV2Ta/vH5mQ/u2kc6d0li690yVRAysuTEwrt+2aS +Ecr1wPrYg1UDfNPFIkZ1cGt5SAYqgpq/5usWDiJFAbzdNpQ0qTUmiteXue4Icr80 +knCDgKs4qllo3UCkGJCy89UDyibK79XH4I9TjvAA46jtn/mtd+ArY0+ew+43u3gJ +hJ65bvspmZDogNOfJA== +-----END CERTIFICATE----- + +# Issuer: CN=TrustAsia TLS ECC Root CA O=TrustAsia Technologies, Inc. +# Subject: CN=TrustAsia TLS ECC Root CA O=TrustAsia Technologies, Inc. +# Label: "TrustAsia TLS ECC Root CA" +# Serial: 310892014698942880364840003424242768478804666567 +# MD5 Fingerprint: 09:48:04:77:d2:fc:65:93:71:66:b1:11:95:4f:06:8c +# SHA1 Fingerprint: b5:ec:39:f3:a1:66:37:ae:c3:05:94:57:e2:be:11:be:b7:a1:7f:36 +# SHA256 Fingerprint: c0:07:6b:9e:f0:53:1f:b1:a6:56:d6:7c:4e:be:97:cd:5d:ba:a4:1e:f4:45:98:ac:c2:48:98:78:c9:2d:87:11 +-----BEGIN CERTIFICATE----- +MIICMTCCAbegAwIBAgIUNnThTXxlE8msg1UloD5Sfi9QaMcwCgYIKoZIzj0EAwMw +WDELMAkGA1UEBhMCQ04xJTAjBgNVBAoTHFRydXN0QXNpYSBUZWNobm9sb2dpZXMs +IEluYy4xIjAgBgNVBAMTGVRydXN0QXNpYSBUTFMgRUNDIFJvb3QgQ0EwHhcNMjQw +NTE1MDU0MTU2WhcNNDQwNTE1MDU0MTU1WjBYMQswCQYDVQQGEwJDTjElMCMGA1UE +ChMcVHJ1c3RBc2lhIFRlY2hub2xvZ2llcywgSW5jLjEiMCAGA1UEAxMZVHJ1c3RB +c2lhIFRMUyBFQ0MgUm9vdCBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABLh/pVs/ +AT598IhtrimY4ZtcU5nb9wj/1WrgjstEpvDBjL1P1M7UiFPoXlfXTr4sP/MSpwDp +guMqWzJ8S5sUKZ74LYO1644xST0mYekdcouJtgq7nDM1D9rs3qlKH8kzsaNCMEAw +DwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQULIVTu7FDzTLqnqOH/qKYqKaT6RAw +DgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2gAMGUCMFRH18MtYYZI9HlaVQ01 +L18N9mdsd0AaRuf4aFtOJx24mH1/k78ITcTaRTChD15KeAIxAKORh/IRM4PDwYqR +OkwrULG9IpRdNYlzg8WbGf60oenUoWa2AaU2+dhoYSi3dOGiMQ== +-----END CERTIFICATE----- + +# Issuer: CN=TrustAsia TLS RSA Root CA O=TrustAsia Technologies, Inc. +# Subject: CN=TrustAsia TLS RSA Root CA O=TrustAsia Technologies, Inc. +# Label: "TrustAsia TLS RSA Root CA" +# Serial: 160405846464868906657516898462547310235378010780 +# MD5 Fingerprint: 3b:9e:c3:86:0f:34:3c:6b:c5:46:c4:8e:1d:e7:19:12 +# SHA1 Fingerprint: a5:46:50:c5:62:ea:95:9a:1a:a7:04:6f:17:58:c7:29:53:3d:03:fa +# SHA256 Fingerprint: 06:c0:8d:7d:af:d8:76:97:1e:b1:12:4f:e6:7f:84:7e:c0:c7:a1:58:d3:ea:53:cb:e9:40:e2:ea:97:91:f4:c3 +-----BEGIN CERTIFICATE----- +MIIFgDCCA2igAwIBAgIUHBjYz+VTPyI1RlNUJDxsR9FcSpwwDQYJKoZIhvcNAQEM +BQAwWDELMAkGA1UEBhMCQ04xJTAjBgNVBAoTHFRydXN0QXNpYSBUZWNobm9sb2dp +ZXMsIEluYy4xIjAgBgNVBAMTGVRydXN0QXNpYSBUTFMgUlNBIFJvb3QgQ0EwHhcN +MjQwNTE1MDU0MTU3WhcNNDQwNTE1MDU0MTU2WjBYMQswCQYDVQQGEwJDTjElMCMG +A1UEChMcVHJ1c3RBc2lhIFRlY2hub2xvZ2llcywgSW5jLjEiMCAGA1UEAxMZVHJ1 +c3RBc2lhIFRMUyBSU0EgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC +AgoCggIBAMMWuBtqpERz5dZO9LnPWwvB0ZqB9WOwj0PBuwhaGnrhB3YmH49pVr7+ +NmDQDIPNlOrnxS1cLwUWAp4KqC/lYCZUlviYQB2srp10Zy9U+5RjmOMmSoPGlbYJ +Q1DNDX3eRA5gEk9bNb2/mThtfWza4mhzH/kxpRkQcwUqwzIZheo0qt1CHjCNP561 +HmHVb70AcnKtEj+qpklz8oYVlQwQX1Fkzv93uMltrOXVmPGZLmzjyUT5tUMnCE32 +ft5EebuyjBza00tsLtbDeLdM1aTk2tyKjg7/D8OmYCYozza/+lcK7Fs/6TAWe8Tb +xNRkoDD75f0dcZLdKY9BWN4ArTr9PXwaqLEX8E40eFgl1oUh63kd0Nyrz2I8sMeX +i9bQn9P+PN7F4/w6g3CEIR0JwqH8uyghZVNgepBtljhb//HXeltt08lwSUq6HTrQ +UNoyIBnkiz/r1RYmNzz7dZ6wB3C4FGB33PYPXFIKvF1tjVEK2sUYyJtt3LCDs3+j +TnhMmCWr8n4uIF6CFabW2I+s5c0yhsj55NqJ4js+k8UTav/H9xj8Z7XvGCxUq0DT +bE3txci3OE9kxJRMT6DNrqXGJyV1J23G2pyOsAWZ1SgRxSHUuPzHlqtKZFlhaxP8 +S8ySpg+kUb8OWJDZgoM5pl+z+m6Ss80zDoWo8SnTq1mt1tve1CuBAgMBAAGjQjBA +MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLgHkXlcBvRG/XtZylomkadFK/hT +MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQwFAAOCAgEAIZtqBSBdGBanEqT3 +Rz/NyjuujsCCztxIJXgXbODgcMTWltnZ9r96nBO7U5WS/8+S4PPFJzVXqDuiGev4 +iqME3mmL5Dw8veWv0BIb5Ylrc5tvJQJLkIKvQMKtuppgJFqBTQUYo+IzeXoLH5Pt +7DlK9RME7I10nYEKqG/odv6LTytpEoYKNDbdgptvT+Bz3Ul/KD7JO6NXBNiT2Twp +2xIQaOHEibgGIOcberyxk2GaGUARtWqFVwHxtlotJnMnlvm5P1vQiJ3koP26TpUJ +g3933FEFlJ0gcXax7PqJtZwuhfG5WyRasQmr2soaB82G39tp27RIGAAtvKLEiUUj +pQ7hRGU+isFqMB3iYPg6qocJQrmBktwliJiJ8Xw18WLK7nn4GS/+X/jbh87qqA8M +pugLoDzga5SYnH+tBuYc6kIQX+ImFTw3OffXvO645e8D7r0i+yiGNFjEWn9hongP +XvPKnbwbPKfILfanIhHKA9jnZwqKDss1jjQ52MjqjZ9k4DewbNfFj8GQYSbbJIwe +SsCI3zWQzj8C9GRh3sfIB5XeMhg6j6JCQCTl1jNdfK7vsU1P1FeQNWrcrgSXSYk0 +ly4wBOeY99sLAZDBHwo/+ML+TvrbmnNzFrwFuHnYWa8G5z9nODmxfKuU4CkUpijy +323imttUQ/hHWKNddBWcwauwxzQ= +-----END CERTIFICATE----- + +# Issuer: CN=D-TRUST EV Root CA 2 2023 O=D-Trust GmbH +# Subject: CN=D-TRUST EV Root CA 2 2023 O=D-Trust GmbH +# Label: "D-TRUST EV Root CA 2 2023" +# Serial: 139766439402180512324132425437959641711 +# MD5 Fingerprint: 96:b4:78:09:f0:09:cb:77:eb:bb:1b:4d:6f:36:bc:b6 +# SHA1 Fingerprint: a5:5b:d8:47:6c:8f:19:f7:4c:f4:6d:6b:b6:c2:79:82:22:df:54:8b +# SHA256 Fingerprint: 8e:82:21:b2:e7:d4:00:78:36:a1:67:2f:0d:cc:29:9c:33:bc:07:d3:16:f1:32:fa:1a:20:6d:58:71:50:f1:ce +-----BEGIN CERTIFICATE----- +MIIFqTCCA5GgAwIBAgIQaSYJfoBLTKCnjHhiU19abzANBgkqhkiG9w0BAQ0FADBI +MQswCQYDVQQGEwJERTEVMBMGA1UEChMMRC1UcnVzdCBHbWJIMSIwIAYDVQQDExlE +LVRSVVNUIEVWIFJvb3QgQ0EgMiAyMDIzMB4XDTIzMDUwOTA5MTAzM1oXDTM4MDUw +OTA5MTAzMlowSDELMAkGA1UEBhMCREUxFTATBgNVBAoTDEQtVHJ1c3QgR21iSDEi +MCAGA1UEAxMZRC1UUlVTVCBFViBSb290IENBIDIgMjAyMzCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBANiOo4mAC7JXUtypU0w3uX9jFxPvp1sjW2l1sJkK +F8GLxNuo4MwxusLyzV3pt/gdr2rElYfXR8mV2IIEUD2BCP/kPbOx1sWy/YgJ25yE +7CUXFId/MHibaljJtnMoPDT3mfd/06b4HEV8rSyMlD/YZxBTfiLNTiVR8CUkNRFe +EMbsh2aJgWi6zCudR3Mfvc2RpHJqnKIbGKBv7FD0fUDCqDDPvXPIEysQEx6Lmqg6 +lHPTGGkKSv/BAQP/eX+1SH977ugpbzZMlWGG2Pmic4ruri+W7mjNPU0oQvlFKzIb +RlUWaqZLKfm7lVa/Rh3sHZMdwGWyH6FDrlaeoLGPaxK3YG14C8qKXO0elg6DpkiV +jTujIcSuWMYAsoS0I6SWhjW42J7YrDRJmGOVxcttSEfi8i4YHtAxq9107PncjLgc +jmgjutDzUNzPZY9zOjLHfP7KgiJPvo5iR2blzYfi6NUPGJ/lBHJLRjwQ8kTCZFZx +TnXonMkmdMV9WdEKWw9t/p51HBjGGjp82A0EzM23RWV6sY+4roRIPrN6TagD4uJ+ +ARZZaBhDM7DS3LAaQzXupdqpRlyuhoFBAUp0JuyfBr/CBTdkdXgpaP3F9ev+R/nk +hbDhezGdpn9yo7nELC7MmVcOIQxFAZRl62UJxmMiCzNJkkg8/M3OsD6Onov4/knF +NXJHAgMBAAGjgY4wgYswDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUqvyREBuH +kV8Wub9PS5FeAByxMoAwDgYDVR0PAQH/BAQDAgEGMEkGA1UdHwRCMEAwPqA8oDqG +OGh0dHA6Ly9jcmwuZC10cnVzdC5uZXQvY3JsL2QtdHJ1c3RfZXZfcm9vdF9jYV8y +XzIwMjMuY3JsMA0GCSqGSIb3DQEBDQUAA4ICAQCTy6UfmRHsmg1fLBWTxj++EI14 +QvBukEdHjqOSMo1wj/Zbjb6JzkcBahsgIIlbyIIQbODnmaprxiqgYzWRaoUlrRc4 +pZt+UPJ26oUFKidBK7GB0aL2QHWpDsvxVUjY7NHss+jOFKE17MJeNRqrphYBBo7q +3C+jisosketSjl8MmxfPy3MHGcRqwnNU73xDUmPBEcrCRbH0O1P1aa4846XerOhU +t7KR/aypH/KH5BfGSah82ApB9PI+53c0BFLd6IHyTS9URZ0V4U/M5d40VxDJI3IX +cI1QcB9WbMy5/zpaT2N6w25lBx2Eof+pDGOJbbJAiDnXH3dotfyc1dZnaVuodNv8 +ifYbMvekJKZ2t0dT741Jj6m2g1qllpBFYfXeA08mD6iL8AOWsKwV0HFaanuU5nCT +2vFp4LJiTZ6P/4mdm13NRemUAiKN4DV/6PEEeXFsVIP4M7kFMhtYVRFP0OUnR3Hs +7dpn1mKmS00PaaLJvOwiS5THaJQXfuKOKD62xur1NGyfN4gHONuGcfrNlUhDbqNP +gofXNJhuS5N5YHVpD/Aa1VP6IQzCP+k/HxiMkl14p3ZnGbuy6n/pcAlWVqOwDAst +Nl7F6cTVg8uGF5csbBNvh1qvSaYd2804BC5f4ko1Di1L+KIkBI3Y4WNeApI02phh +XBxvWHZks/wCuPWdCg== +-----END CERTIFICATE----- + +# Issuer: CN=SwissSign RSA TLS Root CA 2022 - 1 O=SwissSign AG +# Subject: CN=SwissSign RSA TLS Root CA 2022 - 1 O=SwissSign AG +# Label: "SwissSign RSA TLS Root CA 2022 - 1" +# Serial: 388078645722908516278762308316089881486363258315 +# MD5 Fingerprint: 16:2e:e4:19:76:81:85:ba:8e:91:58:f1:15:ef:72:39 +# SHA1 Fingerprint: 81:34:0a:be:4c:cd:ce:cc:e7:7d:cc:8a:d4:57:e2:45:a0:77:5d:ce +# SHA256 Fingerprint: 19:31:44:f4:31:e0:fd:db:74:07:17:d4:de:92:6a:57:11:33:88:4b:43:60:d3:0e:27:29:13:cb:e6:60:ce:41 +-----BEGIN CERTIFICATE----- +MIIFkzCCA3ugAwIBAgIUQ/oMX04bgBhE79G0TzUfRPSA7cswDQYJKoZIhvcNAQEL +BQAwUTELMAkGA1UEBhMCQ0gxFTATBgNVBAoTDFN3aXNzU2lnbiBBRzErMCkGA1UE +AxMiU3dpc3NTaWduIFJTQSBUTFMgUm9vdCBDQSAyMDIyIC0gMTAeFw0yMjA2MDgx +MTA4MjJaFw00NzA2MDgxMTA4MjJaMFExCzAJBgNVBAYTAkNIMRUwEwYDVQQKEwxT +d2lzc1NpZ24gQUcxKzApBgNVBAMTIlN3aXNzU2lnbiBSU0EgVExTIFJvb3QgQ0Eg +MjAyMiAtIDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDLKmjiC8NX +vDVjvHClO/OMPE5Xlm7DTjak9gLKHqquuN6orx122ro10JFwB9+zBvKK8i5VUXu7 +LCTLf5ImgKO0lPaCoaTo+nUdWfMHamFk4saMla+ju45vVs9xzF6BYQ1t8qsCLqSX +5XH8irCRIFucdFJtrhUnWXjyCcplDn/L9Ovn3KlMd/YrFgSVrpxxpT8q2kFC5zyE +EPThPYxr4iuRR1VPuFa+Rd4iUU1OKNlfGUEGjw5NBuBwQCMBauTLE5tzrE0USJIt +/m2n+IdreXXhvhCxqohAWVTXz8TQm0SzOGlkjIHRI36qOTw7D59Ke4LKa2/KIj4x +0LDQKhySio/YGZxH5D4MucLNvkEM+KRHBdvBFzA4OmnczcNpI/2aDwLOEGrOyvi5 +KaM2iYauC8BPY7kGWUleDsFpswrzd34unYyzJ5jSmY0lpx+Gs6ZUcDj8fV3oT4MM +0ZPlEuRU2j7yrTrePjxF8CgPBrnh25d7mUWe3f6VWQQvdT/TromZhqwUtKiE+shd +OxtYk8EXlFXIC+OCeYSf8wCENO7cMdWP8vpPlkwGqnj73mSiI80fPsWMvDdUDrta +clXvyFu1cvh43zcgTFeRc5JzrBh3Q4IgaezprClG5QtO+DdziZaKHG29777YtvTK +wP1H8K4LWCDFyB02rpeNUIMmJCn3nTsPBQIDAQABo2MwYTAPBgNVHRMBAf8EBTAD +AQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBRvjmKLk0Ow4UD2p8P98Q+4 +DxU4pTAdBgNVHQ4EFgQUb45ii5NDsOFA9qfD/fEPuA8VOKUwDQYJKoZIhvcNAQEL +BQADggIBAKwsKUF9+lz1GpUYvyypiqkkVHX1uECry6gkUSsYP2OprphWKwVDIqO3 +10aewCoSPY6WlkDfDDOLazeROpW7OSltwAJsipQLBwJNGD77+3v1dj2b9l4wBlgz +Hqp41eZUBDqyggmNzhYzWUUo8aWjlw5DI/0LIICQ/+Mmz7hkkeUFjxOgdg3XNwwQ +iJb0Pr6VvfHDffCjw3lHC1ySFWPtUnWK50Zpy1FVCypM9fJkT6lc/2cyjlUtMoIc +gC9qkfjLvH4YoiaoLqNTKIftV+Vlek4ASltOU8liNr3CjlvrzG4ngRhZi0Rjn9UM +ZfQpZX+RLOV/fuiJz48gy20HQhFRJjKKLjpHE7iNvUcNCfAWpO2Whi4Z2L6MOuhF +LhG6rlrnub+xzI/goP+4s9GFe3lmozm1O2bYQL7Pt2eLSMkZJVX8vY3PXtpOpvJp +zv1/THfQwUY1mFwjmwJFQ5Ra3bxHrSL+ul4vkSkphnsh3m5kt8sNjzdbowhq6/Td +Ao9QAwKxuDdollDruF/UKIqlIgyKhPBZLtU30WHlQnNYKoH3dtvi4k0NX/a3vgW0 +rk4N3hY9A4GzJl5LuEsAz/+MF7psYC0nhzck5npgL7XTgwSqT0N1osGDsieYK7EO +gLrAhV5Cud+xYJHT6xh+cHiudoO+cVrQkOPKwRYlZ0rwtnu64ZzZ +-----END CERTIFICATE----- + +# Issuer: CN=OISTE Server Root ECC G1 O=OISTE Foundation +# Subject: CN=OISTE Server Root ECC G1 O=OISTE Foundation +# Label: "OISTE Server Root ECC G1" +# Serial: 47819833811561661340092227008453318557 +# MD5 Fingerprint: 42:a7:d2:35:ae:02:92:db:19:76:08:de:2f:05:b4:d4 +# SHA1 Fingerprint: 3b:f6:8b:09:ae:2a:92:7b:ba:e3:8d:3f:11:95:d9:e6:44:0c:45:e2 +# SHA256 Fingerprint: ee:c9:97:c0:c3:0f:21:6f:7e:3b:8b:30:7d:2b:ae:42:41:2d:75:3f:c8:21:9d:af:d1:52:0b:25:72:85:0f:49 +-----BEGIN CERTIFICATE----- +MIICNTCCAbqgAwIBAgIQI/nD1jWvjyhLH/BU6n6XnTAKBggqhkjOPQQDAzBLMQsw +CQYDVQQGEwJDSDEZMBcGA1UECgwQT0lTVEUgRm91bmRhdGlvbjEhMB8GA1UEAwwY +T0lTVEUgU2VydmVyIFJvb3QgRUNDIEcxMB4XDTIzMDUzMTE0NDIyOFoXDTQ4MDUy +NDE0NDIyN1owSzELMAkGA1UEBhMCQ0gxGTAXBgNVBAoMEE9JU1RFIEZvdW5kYXRp +b24xITAfBgNVBAMMGE9JU1RFIFNlcnZlciBSb290IEVDQyBHMTB2MBAGByqGSM49 +AgEGBSuBBAAiA2IABBcv+hK8rBjzCvRE1nZCnrPoH7d5qVi2+GXROiFPqOujvqQy +cvO2Ackr/XeFblPdreqqLiWStukhEaivtUwL85Zgmjvn6hp4LrQ95SjeHIC6XG4N +2xml4z+cKrhAS93mT6NjMGEwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBQ3 +TYhlz/w9itWj8UnATgwQb0K0nDAdBgNVHQ4EFgQUN02IZc/8PYrVo/FJwE4MEG9C +tJwwDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMDA2kAMGYCMQCpKjAd0MKfkFFR +QD6VVCHNFmb3U2wIFjnQEnx/Yxvf4zgAOdktUyBFCxxgZzFDJe0CMQCSia7pXGKD +YmH5LVerVrkR3SW+ak5KGoJr3M/TvEqzPNcum9v4KGm8ay3sMaE641c= +-----END CERTIFICATE----- + +# Issuer: CN=OISTE Server Root RSA G1 O=OISTE Foundation +# Subject: CN=OISTE Server Root RSA G1 O=OISTE Foundation +# Label: "OISTE Server Root RSA G1" +# Serial: 113845518112613905024960613408179309848 +# MD5 Fingerprint: 23:a7:9e:d4:70:b8:b9:14:57:41:8a:7e:44:59:e2:68 +# SHA1 Fingerprint: f7:00:34:25:94:88:68:31:e4:34:87:3f:70:fe:86:b3:86:9f:f0:6e +# SHA256 Fingerprint: 9a:e3:62:32:a5:18:9f:fd:db:35:3d:fd:26:52:0c:01:53:95:d2:27:77:da:c5:9d:b5:7b:98:c0:89:a6:51:e6 +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIQVaXZZ5Qoxu0M+ifdWwFNGDANBgkqhkiG9w0BAQwFADBL +MQswCQYDVQQGEwJDSDEZMBcGA1UECgwQT0lTVEUgRm91bmRhdGlvbjEhMB8GA1UE +AwwYT0lTVEUgU2VydmVyIFJvb3QgUlNBIEcxMB4XDTIzMDUzMTE0MzcxNloXDTQ4 +MDUyNDE0MzcxNVowSzELMAkGA1UEBhMCQ0gxGTAXBgNVBAoMEE9JU1RFIEZvdW5k +YXRpb24xITAfBgNVBAMMGE9JU1RFIFNlcnZlciBSb290IFJTQSBHMTCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAKqu9KuCz/vlNwvn1ZatkOhLKdxVYOPM +vLO8LZK55KN68YG0nnJyQ98/qwsmtO57Gmn7KNByXEptaZnwYx4M0rH/1ow00O7b +rEi56rAUjtgHqSSY3ekJvqgiG1k50SeH3BzN+Puz6+mTeO0Pzjd8JnduodgsIUzk +ik/HEzxux9UTl7Ko2yRpg1bTacuCErudG/L4NPKYKyqOBGf244ehHa1uzjZ0Dl4z +O8vbUZeUapU8zhhabkvG/AePLhq5SvdkNCncpo1Q4Y2LS+VIG24ugBA/5J8bZT8R +tOpXaZ+0AOuFJJkk9SGdl6r7NH8CaxWQrbueWhl/pIzY+m0o/DjH40ytas7ZTpOS +jswMZ78LS5bOZmdTaMsXEY5Z96ycG7mOaES3GK/m5Q9l3JUJsJMStR8+lKXHiHUh +sd4JJCpM4rzsTGdHwimIuQq6+cF0zowYJmXa92/GjHtoXAvuY8BeS/FOzJ8vD+Ho +mnqT8eDI278n5mUpezbgMxVz8p1rhAhoKzYHKyfMeNhqhw5HdPSqoBNdZH702xSu ++zrkL8Fl47l6QGzwBrd7KJvX4V84c5Ss2XCTLdyEr0YconosP4EmQufU2MVshGYR +i3drVByjtdgQ8K4p92cIiBdcuJd5z+orKu5YM+Vt6SmqZQENghPsJQtdLEByFSnT +kCz3GkPVavBpAgMBAAGjYzBhMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU +8snBDw1jALvsRQ5KH7WxszbNDo0wHQYDVR0OBBYEFPLJwQ8NYwC77EUOSh+1sbM2 +zQ6NMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQwFAAOCAgEANGd5sjrG5T33 +I3K5Ce+SrScfoE4KsvXaFwyihdJ+klH9FWXXXGtkFu6KRcoMQzZENdl//nk6HOjG +5D1rd9QhEOP28yBOqb6J8xycqd+8MDoX0TJD0KqKchxRKEzdNsjkLWd9kYccnbz8 +qyiWXmFcuCIzGEgWUOrKL+mlSdx/PKQZvDatkuK59EvV6wit53j+F8Bdh3foZ3dP +AGav9LEDOr4SfEE15fSmG0eLy3n31r8Xbk5l8PjaV8GUgeV6Vg27Rn9vkf195hfk +gSe7BYhW3SCl95gtkRlpMV+bMPKZrXJAlszYd2abtNUOshD+FKrDgHGdPY3ofRRs +YWSGRqbXVMW215AWRqWFyp464+YTFrYVI8ypKVL9AMb2kI5Wj4kI3Zaq5tNqqYY1 +9tVFeEJKRvwDyF7YZvZFZSS0vod7VSCd9521Kvy5YhnLbDuv0204bKt7ph6N/Ome +/msVuduCmsuY33OhkKCgxeDoAaijFJzIwZqsFVAzje18KotzlUBDJvyBpCpfOZC3 +J8tRd/iWkx7P8nd9H0aTolkelUTFLXVksNb54Dxp6gS1HAviRkRNQzuXSXERvSS2 +wq1yVAb+axj5d9spLFKebXd7Yv0PTY6YMjAwcRLWJTXjn/hvnLXrahut6hDTlhZy +BiElxky8j3C7DOReIoMt0r7+hVu05L0= +-----END CERTIFICATE----- + +# Issuer: CN=e-Szigno TLS Root CA 2023 O=Microsec Ltd. +# Subject: CN=e-Szigno TLS Root CA 2023 O=Microsec Ltd. +# Label: "e-Szigno TLS Root CA 2023" +# Serial: 71934828665710877219916191754 +# MD5 Fingerprint: 6a:e9:99:74:a5:da:5e:f1:d9:2e:f2:c8:d1:86:8b:71 +# SHA1 Fingerprint: 6f:9a:d5:d5:df:e8:2c:eb:be:37:07:ee:4f:4f:52:58:29:41:d1:fe +# SHA256 Fingerprint: b4:91:41:50:2d:00:66:3d:74:0f:2e:7e:c3:40:c5:28:00:96:26:66:12:1a:36:d0:9c:f7:dd:2b:90:38:4f:b4 +-----BEGIN CERTIFICATE----- +MIICzzCCAjGgAwIBAgINAOhvGHvWOWuYSkmYCjAKBggqhkjOPQQDBDB1MQswCQYD +VQQGEwJIVTERMA8GA1UEBwwIQnVkYXBlc3QxFjAUBgNVBAoMDU1pY3Jvc2VjIEx0 +ZC4xFzAVBgNVBGEMDlZBVEhVLTIzNTg0NDk3MSIwIAYDVQQDDBllLVN6aWdubyBU +TFMgUm9vdCBDQSAyMDIzMB4XDTIzMDcxNzE0MDAwMFoXDTM4MDcxNzE0MDAwMFow +dTELMAkGA1UEBhMCSFUxETAPBgNVBAcMCEJ1ZGFwZXN0MRYwFAYDVQQKDA1NaWNy +b3NlYyBMdGQuMRcwFQYDVQRhDA5WQVRIVS0yMzU4NDQ5NzEiMCAGA1UEAwwZZS1T +emlnbm8gVExTIFJvb3QgQ0EgMjAyMzCBmzAQBgcqhkjOPQIBBgUrgQQAIwOBhgAE +AGgP36J8PKp0iGEKjcJMpQEiFNT3YHdCnAo4YKGMZz6zY+n6kbCLS+Y53wLCMAFS +AL/fjO1ZrTJlqwlZULUZwmgcAOAFX9pQJhzDrAQixTpN7+lXWDajwRlTEArRzT/v +SzUaQ49CE0y5LBqcvjC2xN7cS53kpDzLLtmt3999Cd8ukv+ho2MwYTAPBgNVHRMB +Af8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUWYQCYlpGePVd3I8K +ECgj3NXW+0UwHwYDVR0jBBgwFoAUWYQCYlpGePVd3I8KECgj3NXW+0UwCgYIKoZI +zj0EAwQDgYsAMIGHAkIBLdqu9S54tma4n7Zwf2Z0z+yOfP7AAXmazlIC58PRDHpt +y7Ve7hekm9sEdu4pKeiv+62sUvTXK9Z3hBC9xdIoaDQCQTV2WnXzkoYI9bIeCvZl +C9p2x1L/Cx6AcCIwwzPbGO2E14vs7dOoY4G1VnxHx1YwlGhza9IuqbnZLBwpvQy6 +uWWL +-----END CERTIFICATE----- diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/core.py b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/core.py new file mode 100644 index 0000000000000000000000000000000000000000..1c9661cc7c2f6917c2506a30b2710002f81ab23a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/core.py @@ -0,0 +1,83 @@ +""" +certifi.py +~~~~~~~~~~ + +This module returns the installation location of cacert.pem or its contents. +""" +import sys +import atexit + +def exit_cacert_ctx() -> None: + _CACERT_CTX.__exit__(None, None, None) # type: ignore[union-attr] + + +if sys.version_info >= (3, 11): + + from importlib.resources import as_file, files + + _CACERT_CTX = None + _CACERT_PATH = None + + def where() -> str: + # This is slightly terrible, but we want to delay extracting the file + # in cases where we're inside of a zipimport situation until someone + # actually calls where(), but we don't want to re-extract the file + # on every call of where(), so we'll do it once then store it in a + # global variable. + global _CACERT_CTX + global _CACERT_PATH + if _CACERT_PATH is None: + # This is slightly janky, the importlib.resources API wants you to + # manage the cleanup of this file, so it doesn't actually return a + # path, it returns a context manager that will give you the path + # when you enter it and will do any cleanup when you leave it. In + # the common case of not needing a temporary file, it will just + # return the file system location and the __exit__() is a no-op. + # + # We also have to hold onto the actual context manager, because + # it will do the cleanup whenever it gets garbage collected, so + # we will also store that at the global level as well. + _CACERT_CTX = as_file(files("certifi").joinpath("cacert.pem")) + _CACERT_PATH = str(_CACERT_CTX.__enter__()) + atexit.register(exit_cacert_ctx) + + return _CACERT_PATH + + def contents() -> str: + return files("certifi").joinpath("cacert.pem").read_text(encoding="ascii") + +else: + + from importlib.resources import path as get_path, read_text + + _CACERT_CTX = None + _CACERT_PATH = None + + def where() -> str: + # This is slightly terrible, but we want to delay extracting the + # file in cases where we're inside of a zipimport situation until + # someone actually calls where(), but we don't want to re-extract + # the file on every call of where(), so we'll do it once then store + # it in a global variable. + global _CACERT_CTX + global _CACERT_PATH + if _CACERT_PATH is None: + # This is slightly janky, the importlib.resources API wants you + # to manage the cleanup of this file, so it doesn't actually + # return a path, it returns a context manager that will give + # you the path when you enter it and will do any cleanup when + # you leave it. In the common case of not needing a temporary + # file, it will just return the file system location and the + # __exit__() is a no-op. + # + # We also have to hold onto the actual context manager, because + # it will do the cleanup whenever it gets garbage collected, so + # we will also store that at the global level as well. + _CACERT_CTX = get_path("certifi", "cacert.pem") + _CACERT_PATH = str(_CACERT_CTX.__enter__()) + atexit.register(exit_cacert_ctx) + + return _CACERT_PATH + + def contents() -> str: + return read_text("certifi", "cacert.pem", encoding="ascii") diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/py.typed b/bioagent-bench/.venv/lib/python3.14/site-packages/certifi/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/INSTALLER b/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/METADATA b/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..30fbba6dd73ef6f9c454ae08f409d0306eeadb9d --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/METADATA @@ -0,0 +1,84 @@ +Metadata-Version: 2.4 +Name: click +Version: 8.3.3 +Summary: Composable command line interface toolkit +Maintainer-email: Pallets +Requires-Python: >=3.10 +Description-Content-Type: text/markdown +License-Expression: BSD-3-Clause +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Typing :: Typed +License-File: LICENSE.txt +Requires-Dist: colorama; platform_system == 'Windows' +Project-URL: Changes, https://click.palletsprojects.com/page/changes/ +Project-URL: Chat, https://discord.gg/pallets +Project-URL: Documentation, https://click.palletsprojects.com/ +Project-URL: Donate, https://palletsprojects.com/donate +Project-URL: Source, https://github.com/pallets/click/ + +
+ +# Click + +Click is a Python package for creating beautiful command line interfaces +in a composable way with as little code as necessary. It's the "Command +Line Interface Creation Kit". It's highly configurable but comes with +sensible defaults out of the box. + +It aims to make the process of writing command line tools quick and fun +while also preventing any frustration caused by the inability to +implement an intended CLI API. + +Click in three points: + +- Arbitrary nesting of commands +- Automatic help page generation +- Supports lazy loading of subcommands at runtime + + +## A Simple Example + +```python +import click + +@click.command() +@click.option("--count", default=1, help="Number of greetings.") +@click.option("--name", prompt="Your name", help="The person to greet.") +def hello(count, name): + """Simple program that greets NAME for a total of COUNT times.""" + for _ in range(count): + click.echo(f"Hello, {name}!") + +if __name__ == '__main__': + hello() +``` + +``` +$ python hello.py --count=3 +Your name: Click +Hello, Click! +Hello, Click! +Hello, Click! +``` + + +## Donate + +The Pallets organization develops and supports Click and other popular +packages. In order to grow the community of contributors and users, and +allow the maintainers to devote more time to the projects, [please +donate today][]. + +[please donate today]: https://palletsprojects.com/donate + +## Contributing + +See our [detailed contributing documentation][contrib] for many ways to +contribute, including reporting issues, requesting features, asking or answering +questions, and making PRs. + +[contrib]: https://palletsprojects.com/contributing/ + diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/RECORD b/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..c684fdb0b3b6eee6baf0abb8fa14719ac8dcc631 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/RECORD @@ -0,0 +1,24 @@ +click-8.3.3.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +click-8.3.3.dist-info/METADATA,sha256=_sjhT7_ZKYkyseqbWJtQA8eAcV2CeDJdcNrX_92pSCY,2621 +click-8.3.3.dist-info/RECORD,, +click-8.3.3.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +click-8.3.3.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 +click-8.3.3.dist-info/licenses/LICENSE.txt,sha256=morRBqOU6FO_4h9C9OctWSgZoigF2ZG18ydQKSkrZY0,1475 +click/__init__.py,sha256=tq5olp0lVuUNbwOj0p5nMVdqhOyYz4aZzF8S5T0aFMU,4526 +click/_compat.py,sha256=v3xBZkFbvA1BXPRkFfBJc6-pIwPI7345m-kQEnpVAs4,18693 +click/_termui_impl.py,sha256=R3aYjPD2x1xgNXiXEqHSR-jAVhbXKkLKeqgr-4eDOkA,28066 +click/_textwrap.py,sha256=BOae0RQ6vg3FkNgSJyOoGzG1meGMxJ_ukWVZKx_v-0o,1400 +click/_utils.py,sha256=kZwtTf5gMuCilJJceS2iTCvRvCY-0aN5rJq8gKw7p8g,943 +click/_winconsole.py,sha256=_vxUuUaxwBhoR0vUWCNuHY8VUefiMdCIyU2SXPqoF-A,8465 +click/core.py,sha256=wzDpH4Wh45hBfsdw3BHqn_095457WRrb92dyaeIGkHk,134478 +click/decorators.py,sha256=5P7abhJtAQYp_KHgjUvhMv464ERwOzrv2enNknlwHyQ,18461 +click/exceptions.py,sha256=8utf8w6V5hJXMnO_ic1FNrtbwuEn1NUu1aDwV8UqnG4,9954 +click/formatting.py,sha256=RVfwwr0rwWNpgGr8NaHodPzkIr7_tUyVh_nDdanLMNc,9730 +click/globals.py,sha256=gM-Nh6A4M0HB_SgkaF5M4ncGGMDHc_flHXu9_oh4GEU,1923 +click/parser.py,sha256=Q31pH0FlQZEq-UXE_ABRzlygEfvxPTuZbWNh4xfXmzw,19010 +click/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +click/shell_completion.py,sha256=Cc4GQUFuWpfQBa9sF5qXeeYI7n3tI_1k6ZdSn4BZbT0,20994 +click/termui.py,sha256=0xjUaEZJz1_1y8eqon2kOe2OXrmHzZCJoVohFb43yTs,31434 +click/testing.py,sha256=PYLYW4ZjCAP6HRWtNhFpr3imJxQVEPFq3EW3JL0LTfA,22714 +click/types.py,sha256=ek54BNSFwPKsqtfT7jsqcc4WHui8AIFVMKM4oVZIXhc,39927 +click/utils.py,sha256=vep6qqMkv3WkzA9OCu7f6Ku-3beZHNpo8qJqL-uzUk8,20282 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/REQUESTED b/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/WHEEL b/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..d8b9936dad9ab2513fa6979f411560d3b6b57e37 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: flit 3.12.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/licenses/LICENSE.txt b/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/licenses/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..d12a849186982399c537c5b9a8fd77bf2edd5eab --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/click-8.3.3.dist-info/licenses/LICENSE.txt @@ -0,0 +1,28 @@ +Copyright 2014 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/AUTHORS b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/AUTHORS new file mode 100644 index 0000000000000000000000000000000000000000..7092efd949bb407beb485f25d6c1847804a79170 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/AUTHORS @@ -0,0 +1,8 @@ +# This is the list of HuggingFace Datasets authors for copyright purposes. +# +# This does not necessarily list everyone who has contributed code, since in +# some cases, their employer may be the copyright holder. To see the full list +# of contributors, see the revision history in source control. + +Google Inc. +HuggingFace Inc. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/INSTALLER b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/LICENSE b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/METADATA b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..0ef17d742c5241aeebe03d3d2e297b13e1c63974 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/METADATA @@ -0,0 +1,378 @@ +Metadata-Version: 2.2 +Name: datasets +Version: 4.8.5 +Summary: HuggingFace community-driven open-source library of datasets +Home-page: https://github.com/huggingface/datasets +Download-URL: https://github.com/huggingface/datasets/tags +Author: HuggingFace Inc. +Author-email: thomas@huggingface.co +License: Apache 2.0 +Keywords: datasets machine learning datasets +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Education +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence +Requires-Python: >=3.10.0 +Description-Content-Type: text/markdown +License-File: LICENSE +License-File: AUTHORS +Requires-Dist: filelock +Requires-Dist: numpy>=1.17 +Requires-Dist: pyarrow>=21.0.0 +Requires-Dist: dill<0.4.2,>=0.3.0 +Requires-Dist: pandas +Requires-Dist: requests>=2.32.2 +Requires-Dist: httpx<1.0.0 +Requires-Dist: tqdm>=4.66.3 +Requires-Dist: xxhash +Requires-Dist: multiprocess<0.70.20 +Requires-Dist: fsspec[http]<=2026.2.0,>=2023.1.0 +Requires-Dist: huggingface-hub<2.0,>=0.25.0 +Requires-Dist: packaging +Requires-Dist: pyyaml>=5.1 +Provides-Extra: audio +Requires-Dist: torchcodec>=0.6.0; extra == "audio" +Requires-Dist: torch>=2.8.0; extra == "audio" +Provides-Extra: vision +Requires-Dist: Pillow>=9.4.0; extra == "vision" +Provides-Extra: tensorflow +Requires-Dist: tensorflow>=2.6.0; extra == "tensorflow" +Provides-Extra: tensorflow-gpu +Requires-Dist: tensorflow>=2.6.0; extra == "tensorflow-gpu" +Provides-Extra: torch +Requires-Dist: torch; extra == "torch" +Provides-Extra: jax +Requires-Dist: jax>=0.3.14; extra == "jax" +Requires-Dist: jaxlib>=0.3.14; extra == "jax" +Provides-Extra: streaming +Provides-Extra: dev +Requires-Dist: numba>=0.56.4; python_version < "3.14" and extra == "dev" +Requires-Dist: absl-py; extra == "dev" +Requires-Dist: decorator; extra == "dev" +Requires-Dist: joblib<1.3.0; extra == "dev" +Requires-Dist: joblibspark; python_version < "3.14" and extra == "dev" +Requires-Dist: pytest; extra == "dev" +Requires-Dist: pytest-datadir; extra == "dev" +Requires-Dist: pytest-xdist; extra == "dev" +Requires-Dist: aiohttp; extra == "dev" +Requires-Dist: elasticsearch<8.0.0,>=7.17.12; extra == "dev" +Requires-Dist: faiss-cpu>=1.8.0.post1; extra == "dev" +Requires-Dist: h5py; extra == "dev" +Requires-Dist: pylance; extra == "dev" +Requires-Dist: jax>=0.3.14; sys_platform != "win32" and extra == "dev" +Requires-Dist: jaxlib>=0.3.14; sys_platform != "win32" and extra == "dev" +Requires-Dist: lz4; python_version < "3.14" and extra == "dev" +Requires-Dist: moto[server]; extra == "dev" +Requires-Dist: pyspark>=3.4; extra == "dev" +Requires-Dist: py7zr; extra == "dev" +Requires-Dist: rarfile>=4.0; extra == "dev" +Requires-Dist: sqlalchemy; extra == "dev" +Requires-Dist: protobuf<4.0.0; extra == "dev" +Requires-Dist: tensorflow>=2.6.0; (python_version < "3.10" and sys_platform != "win32") and extra == "dev" +Requires-Dist: tensorflow>=2.16.0; (python_version >= "3.10" and sys_platform != "win32" and python_version < "3.14") and extra == "dev" +Requires-Dist: tiktoken; extra == "dev" +Requires-Dist: torch>=2.8.0; extra == "dev" +Requires-Dist: torchdata; extra == "dev" +Requires-Dist: transformers>=4.42.0; extra == "dev" +Requires-Dist: zstandard; extra == "dev" +Requires-Dist: polars[timezone]>=0.20.0; extra == "dev" +Requires-Dist: Pillow>=9.4.0; extra == "dev" +Requires-Dist: torchcodec>=0.7.0; python_version < "3.14" and extra == "dev" +Requires-Dist: nibabel>=5.3.1; extra == "dev" +Requires-Dist: ruff>=0.3.0; extra == "dev" +Requires-Dist: transformers; extra == "dev" +Requires-Dist: torch; extra == "dev" +Requires-Dist: tensorflow>=2.6.0; extra == "dev" +Provides-Extra: tests +Requires-Dist: numba>=0.56.4; python_version < "3.14" and extra == "tests" +Requires-Dist: absl-py; extra == "tests" +Requires-Dist: decorator; extra == "tests" +Requires-Dist: joblib<1.3.0; extra == "tests" +Requires-Dist: joblibspark; python_version < "3.14" and extra == "tests" +Requires-Dist: pytest; extra == "tests" +Requires-Dist: pytest-datadir; extra == "tests" +Requires-Dist: pytest-xdist; extra == "tests" +Requires-Dist: aiohttp; extra == "tests" +Requires-Dist: elasticsearch<8.0.0,>=7.17.12; extra == "tests" +Requires-Dist: faiss-cpu>=1.8.0.post1; extra == "tests" +Requires-Dist: h5py; extra == "tests" +Requires-Dist: pylance; extra == "tests" +Requires-Dist: jax>=0.3.14; sys_platform != "win32" and extra == "tests" +Requires-Dist: jaxlib>=0.3.14; sys_platform != "win32" and extra == "tests" +Requires-Dist: lz4; python_version < "3.14" and extra == "tests" +Requires-Dist: moto[server]; extra == "tests" +Requires-Dist: pyspark>=3.4; extra == "tests" +Requires-Dist: py7zr; extra == "tests" +Requires-Dist: rarfile>=4.0; extra == "tests" +Requires-Dist: sqlalchemy; extra == "tests" +Requires-Dist: protobuf<4.0.0; extra == "tests" +Requires-Dist: tensorflow>=2.6.0; (python_version < "3.10" and sys_platform != "win32") and extra == "tests" +Requires-Dist: tensorflow>=2.16.0; (python_version >= "3.10" and sys_platform != "win32" and python_version < "3.14") and extra == "tests" +Requires-Dist: tiktoken; extra == "tests" +Requires-Dist: torch>=2.8.0; extra == "tests" +Requires-Dist: torchdata; extra == "tests" +Requires-Dist: transformers>=4.42.0; extra == "tests" +Requires-Dist: zstandard; extra == "tests" +Requires-Dist: polars[timezone]>=0.20.0; extra == "tests" +Requires-Dist: Pillow>=9.4.0; extra == "tests" +Requires-Dist: torchcodec>=0.7.0; python_version < "3.14" and extra == "tests" +Requires-Dist: nibabel>=5.3.1; extra == "tests" +Provides-Extra: tests-numpy2 +Requires-Dist: numba>=0.56.4; python_version < "3.14" and extra == "tests-numpy2" +Requires-Dist: absl-py; extra == "tests-numpy2" +Requires-Dist: decorator; extra == "tests-numpy2" +Requires-Dist: joblib<1.3.0; extra == "tests-numpy2" +Requires-Dist: joblibspark; python_version < "3.14" and extra == "tests-numpy2" +Requires-Dist: pytest; extra == "tests-numpy2" +Requires-Dist: pytest-datadir; extra == "tests-numpy2" +Requires-Dist: pytest-xdist; extra == "tests-numpy2" +Requires-Dist: aiohttp; extra == "tests-numpy2" +Requires-Dist: elasticsearch<8.0.0,>=7.17.12; extra == "tests-numpy2" +Requires-Dist: h5py; extra == "tests-numpy2" +Requires-Dist: pylance; extra == "tests-numpy2" +Requires-Dist: jax>=0.3.14; sys_platform != "win32" and extra == "tests-numpy2" +Requires-Dist: jaxlib>=0.3.14; sys_platform != "win32" and extra == "tests-numpy2" +Requires-Dist: lz4; python_version < "3.14" and extra == "tests-numpy2" +Requires-Dist: moto[server]; extra == "tests-numpy2" +Requires-Dist: pyspark>=3.4; extra == "tests-numpy2" +Requires-Dist: py7zr; extra == "tests-numpy2" +Requires-Dist: rarfile>=4.0; extra == "tests-numpy2" +Requires-Dist: sqlalchemy; extra == "tests-numpy2" +Requires-Dist: protobuf<4.0.0; extra == "tests-numpy2" +Requires-Dist: tiktoken; extra == "tests-numpy2" +Requires-Dist: torch>=2.8.0; extra == "tests-numpy2" +Requires-Dist: torchdata; extra == "tests-numpy2" +Requires-Dist: transformers>=4.42.0; extra == "tests-numpy2" +Requires-Dist: zstandard; extra == "tests-numpy2" +Requires-Dist: polars[timezone]>=0.20.0; extra == "tests-numpy2" +Requires-Dist: Pillow>=9.4.0; extra == "tests-numpy2" +Requires-Dist: torchcodec>=0.7.0; python_version < "3.14" and extra == "tests-numpy2" +Requires-Dist: nibabel>=5.3.1; extra == "tests-numpy2" +Provides-Extra: quality +Requires-Dist: ruff>=0.3.0; extra == "quality" +Provides-Extra: benchmarks +Requires-Dist: tensorflow==2.12.0; extra == "benchmarks" +Requires-Dist: torch==2.0.1; extra == "benchmarks" +Requires-Dist: transformers==4.30.1; extra == "benchmarks" +Provides-Extra: docs +Requires-Dist: transformers; extra == "docs" +Requires-Dist: torch; extra == "docs" +Requires-Dist: tensorflow>=2.6.0; extra == "docs" +Provides-Extra: pdfs +Requires-Dist: pdfplumber>=0.11.4; extra == "pdfs" +Provides-Extra: nibabel +Requires-Dist: nibabel>=5.3.2; extra == "nibabel" +Requires-Dist: ipyniivue==2.4.2; extra == "nibabel" +Dynamic: author +Dynamic: author-email +Dynamic: classifier +Dynamic: description +Dynamic: description-content-type +Dynamic: download-url +Dynamic: home-page +Dynamic: keywords +Dynamic: license +Dynamic: provides-extra +Dynamic: requires-dist +Dynamic: requires-python +Dynamic: summary + +

+ + + + Hugging Face Datasets Library + +
+
+

+ +

+ Build + GitHub + Documentation + GitHub release + Number of datasets + Contributor Covenant + DOI +

+ +🤗 Datasets is a lightweight library providing **two** main features: + +- **one-line dataloaders for many public datasets**: one-liners to download and pre-process any of the ![number of datasets](https://img.shields.io/endpoint?url=https://huggingface.co/api/shields/datasets&color=brightgreen) major public datasets (image datasets, audio datasets, text datasets in 467 languages and dialects, etc.) provided on the [HuggingFace Datasets Hub](https://huggingface.co/datasets). With a simple command like `squad_dataset = load_dataset("rajpurkar/squad")`, get any of these datasets ready to use in a dataloader for training/evaluating a ML model (Numpy/Pandas/PyTorch/TensorFlow/JAX), +- **efficient data pre-processing**: simple, fast and reproducible data pre-processing for the public datasets as well as your own local datasets in CSV, JSON, text, PNG, JPEG, WAV, MP3, Parquet, HDF5, etc. With simple commands like `processed_dataset = dataset.map(process_example)`, efficiently prepare the dataset for inspection and ML model evaluation and training. + +[🎓 **Documentation**](https://huggingface.co/docs/datasets/) [🔎 **Find a dataset in the Hub**](https://huggingface.co/datasets) [🌟 **Share a dataset on the Hub**](https://huggingface.co/docs/datasets/share) + +

+ +

+ +🤗 Datasets is designed to let the community easily add and share new datasets. + +🤗 Datasets has many additional interesting features: + +- Thrive on large datasets: 🤗 Datasets naturally frees the user from RAM memory limitation, all datasets are memory-mapped using an efficient zero-serialization cost backend (Apache Arrow). +- Smart caching: never wait for your data to process several times. +- Lightweight and fast with a transparent and pythonic API (multi-processing/caching/memory-mapping). +- Built-in interoperability with NumPy, PyTorch, TensorFlow 2, JAX, Pandas, Polars and more. +- Native support for audio, image and video data. +- Enable streaming mode to save disk space and start iterating over the dataset immediately. + +🤗 Datasets originated from a fork of the awesome [TensorFlow Datasets](https://github.com/tensorflow/datasets) and the HuggingFace team want to deeply thank the TensorFlow Datasets team for building this amazing library. + +# Installation + +## With pip + +🤗 Datasets can be installed from PyPi and has to be installed in a virtual environment (venv or conda for instance) + +```bash +pip install datasets +``` + +## With conda + +🤗 Datasets can be installed using conda as follows: + +```bash +conda install -c huggingface -c conda-forge datasets +``` + +Follow the installation pages of TensorFlow and PyTorch to see how to install them with conda. + +For more details on installation, check the installation page in the documentation: https://huggingface.co/docs/datasets/installation + +## Installation to use with Machine Learning & Data frameworks frameworks + +If you plan to use 🤗 Datasets with PyTorch (2.0+), TensorFlow (2.6+) or JAX (0.4+) you should also install PyTorch, TensorFlow or JAX. +🤗 Datasets is also well integrated with data frameworks like PyArrow, Pandas, Polars and Spark, which should be installed separately. + +For more details on using the library with these frameworks, check the quick start page in the documentation: https://huggingface.co/docs/datasets/quickstart + +# Usage + +🤗 Datasets is made to be very simple to use - the API is centered around a single function, `datasets.load_dataset(dataset_name, **kwargs)`, that instantiates a dataset. + +This library can be used for text/image/audio/etc. datasets. Here is an example to load a text dataset: + +Here is a quick example: + +```python +from datasets import load_dataset + +# Print all the available datasets +from huggingface_hub import list_datasets +print([dataset.id for dataset in list_datasets(limit=20)]) + +# Load a dataset and print the first example in the training set +squad_dataset = load_dataset('rajpurkar/squad') +print(squad_dataset['train'][0]) + +# Process the dataset - add a column with the length of the context texts +dataset_with_length = squad_dataset.map(lambda x: {"length": len(x["context"])}) + +# Process the dataset - tokenize the context texts (using a tokenizer from the 🤗 Transformers library) +from transformers import AutoTokenizer +tokenizer = AutoTokenizer.from_pretrained('bert-base-cased') + +tokenized_dataset = squad_dataset.map(lambda x: tokenizer(x['context']), batched=True) +``` + +If your dataset is bigger than your disk or if you don't want to wait to download the data, you can use streaming: + +```python +# If you want to use the dataset immediately and efficiently stream the data as you iterate over the dataset +image_dataset = load_dataset('timm/imagenet-1k-wds', streaming=True) +for example in image_dataset["train"]: + break +``` + +For more details on using the library, check the quick start page in the documentation: https://huggingface.co/docs/datasets/quickstart and the specific pages on: + +- Loading a dataset: https://huggingface.co/docs/datasets/loading +- What's in a Dataset: https://huggingface.co/docs/datasets/access +- Processing data with 🤗 Datasets: https://huggingface.co/docs/datasets/process + - Processing audio data: https://huggingface.co/docs/datasets/audio_process + - Processing image data: https://huggingface.co/docs/datasets/image_process + - Processing text data: https://huggingface.co/docs/datasets/nlp_process +- Streaming a dataset: https://huggingface.co/docs/datasets/stream +- etc. + +# Add a new dataset to the Hub + +We have a very detailed step-by-step guide to add a new dataset to the ![number of datasets](https://img.shields.io/endpoint?url=https://huggingface.co/api/shields/datasets&color=brightgreen) datasets already provided on the [HuggingFace Datasets Hub](https://huggingface.co/datasets). + +You can find: +- [how to upload a dataset to the Hub using your web browser or Python](https://huggingface.co/docs/datasets/upload_dataset) and also +- [how to upload it using Git](https://huggingface.co/docs/datasets/share). + +# Disclaimers + +You can use 🤗 Datasets to load datasets based on versioned git repositories maintained by the dataset authors. For reproducibility reasons, we ask users to pin the `revision` of the repositories they use. + +If you're a dataset owner and wish to update any part of it (description, citation, license, etc.), or do not want your dataset to be included in the Hugging Face Hub, please get in touch by opening a discussion or a pull request in the Community tab of the dataset page. Thanks for your contribution to the ML community! + +## BibTeX + +If you want to cite our 🤗 Datasets library, you can use our [paper](https://huggingface.co/papers/2109.02846): + +```bibtex +@inproceedings{lhoest-etal-2021-datasets, + title = "Datasets: A Community Library for Natural Language Processing", + author = "Lhoest, Quentin and + Villanova del Moral, Albert and + Jernite, Yacine and + Thakur, Abhishek and + von Platen, Patrick and + Patil, Suraj and + Chaumond, Julien and + Drame, Mariama and + Plu, Julien and + Tunstall, Lewis and + Davison, Joe and + {\v{S}}a{\v{s}}ko, Mario and + Chhablani, Gunjan and + Malik, Bhavitvya and + Brandeis, Simon and + Le Scao, Teven and + Sanh, Victor and + Xu, Canwen and + Patry, Nicolas and + McMillan-Major, Angelina and + Schmid, Philipp and + Gugger, Sylvain and + Delangue, Cl{\'e}ment and + Matussi{\`e}re, Th{\'e}o and + Debut, Lysandre and + Bekman, Stas and + Cistac, Pierric and + Goehringer, Thibault and + Mustar, Victor and + Lagunas, Fran{\c{c}}ois and + Rush, Alexander and + Wolf, Thomas", + booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing: System Demonstrations", + month = nov, + year = "2021", + address = "Online and Punta Cana, Dominican Republic", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2021.emnlp-demo.21", + pages = "175--184", + abstract = "The scale, variety, and quantity of publicly-available NLP datasets has grown rapidly as researchers propose new tasks, larger models, and novel benchmarks. Datasets is a community library for contemporary NLP designed to support this ecosystem. Datasets aims to standardize end-user interfaces, versioning, and documentation, while providing a lightweight front-end that behaves similarly for small datasets as for internet-scale corpora. The design of the library incorporates a distributed, community-driven approach to adding datasets and documenting usage. After a year of development, the library now includes more than 650 unique datasets, has more than 250 contributors, and has helped support a variety of novel cross-dataset research projects and shared tasks. The library is available at https://github.com/huggingface/datasets.", + eprint={2109.02846}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, +} +``` + +If you need to cite a specific version of our 🤗 Datasets library for reproducibility, you can use the corresponding version Zenodo DOI from this [list](https://zenodo.org/search?q=conceptrecid:%224817768%22&sort=-version&all_versions=True). diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/RECORD b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..a55a3eec4466d1a3722c89a586b2a64d6d781452 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/RECORD @@ -0,0 +1,144 @@ +../../../bin/datasets-cli,sha256=GVKzRggab4ghPFkZQI7HU43v8tr1G7_GliLThUMU53Y,346 +datasets-4.8.5.dist-info/AUTHORS,sha256=L0FBY23tCNHLmvsOKAbumHn8WZZIK98sH53JYxhAchU,327 +datasets-4.8.5.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +datasets-4.8.5.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358 +datasets-4.8.5.dist-info/METADATA,sha256=wCGYNTardA0tY9YMU1Nx0jrU8qLghkvZqAnBPYismnI,19909 +datasets-4.8.5.dist-info/RECORD,, +datasets-4.8.5.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets-4.8.5.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91 +datasets-4.8.5.dist-info/entry_points.txt,sha256=iM-h4A7OQCrZqr3L2mwiyMtPeFj8w4HAHzmI45y3tg0,69 +datasets-4.8.5.dist-info/top_level.txt,sha256=9A857YvCQm_Dg3UjeKkWPz9sDBos0t3zN2pf5krTemQ,9 +datasets/__init__.py,sha256=cd7P4m0d97ju1Mewvm2rPqWKxMNwtLdwQq0Drn0uVVQ,1630 +datasets/arrow_dataset.py,sha256=x0NnkMkWp1OMQfpuX72sJfTch3DZCeGJAaiSi1fRXm4,340442 +datasets/arrow_reader.py,sha256=54yLKdxCtZsuLgcZ3EiUvun_oors0zS3TTtQn7OPNa4,25215 +datasets/arrow_writer.py,sha256=30Ipiwwb-kiQskaYqh3rAUzG4VouzOW77AQ4Zc2Shy0,38706 +datasets/builder.py,sha256=p6kvyAJ6k16eZ__vRqNH9cqDMEpEeG6P1By3xPjAXx4,86014 +datasets/combine.py,sha256=MAqgg95h_DW1bb00gSZdD_a7DNnF9_kis8xwS4WFRKw,11761 +datasets/commands/__init__.py,sha256=rujbQtxJbwHhF9WQqp2DD9tfVTghDMJdl0v6H551Pcs,312 +datasets/commands/datasets_cli.py,sha256=CRy2H60h2sxJDpfa4LZ4dxipdGIZlTL47pEXLV6tfwQ,1175 +datasets/commands/delete_from_hub.py,sha256=o0wdolb1r1Jnl6F0KdqKn3u0l8VR2od6KzbRoqrSNPM,1396 +datasets/commands/env.py,sha256=8qg-hpXSXXsHvtYFvJkn5rn9IncqPsjjx3nR8no4a2I,1239 +datasets/commands/test.py,sha256=ybafVFdQED7hc4JyY_U8BG9_CkkhL0Gp5dOOAuzaHUw,7824 +datasets/config.py,sha256=0p88mChuZ61dJ9mhknkDmmLoXJMwoxkBXcYGdGCX6Hg,10358 +datasets/data_files.py,sha256=lsI_vKh_1BWJABYE5wRF3YUOlJQ4Fv77oDiYbyyvcuE,32812 +datasets/dataset_dict.py,sha256=CTqH8LY5cCiI38lXySyVrYgpBlUjVyvcZjpd2RfTxj8,128001 +datasets/distributed.py,sha256=YUEhqqx7uxCAQac9dZOHfJIMAVLPW5vDQkTqFZUbXRs,1815 +datasets/download/__init__.py,sha256=lbFOtITDaR7PHrhzJ8VfRnpaOT6NYozSxUcLv_GVfTg,281 +datasets/download/download_config.py,sha256=ODHFej2H-JIqFllklUFkiF4ILJ62l8clAGk13b8Ru3U,3796 +datasets/download/download_manager.py,sha256=Jq0BfYhQ7k5sT2tjbTh-zsj-b3MGS_IIP_bcjRUPfeo,12780 +datasets/download/streaming_download_manager.py,sha256=-eRyxNnKj-dH-3bzdKu-rFzHjhh57XKNwJc-087BNw0,7553 +datasets/exceptions.py,sha256=B93GwElhEvlhHPU9GBSY8if27jhRwu875-gL6B2CL6c,4185 +datasets/features/__init__.py,sha256=dm-1YEPtAwzaljZIVyHWl8HlYCESM1mxeOD52zVR3a8,603 +datasets/features/_torchcodec.py,sha256=Ws7JMYlUlPa7NHh1ZgxWQNrJV0c14kfYYCdrLjCkmjA,627 +datasets/features/audio.py,sha256=eQM6i6kO1057pe1Sob-0tY0GMF9Y8wULAXqsChABOzc,16110 +datasets/features/features.py,sha256=1ScT8Cyej5SwhaDnyincBtZtEQ4lAIBqVUGokVGQSbw,98124 +datasets/features/image.py,sha256=0FMvaNyZjdGWCYf2QtY_kYUNqMEk8bTZMrhXcqmhLOc,18403 +datasets/features/nifti.py,sha256=Zu1fGqZPUK1qj-s5yG19cl0YwXM5QZQ_NDvHy61cFDA,14313 +datasets/features/pdf.py,sha256=Bjex_wBN3SWVSIf5E0ry2mHQL2yeLz5RyOrT_Ro0gVA,12207 +datasets/features/translation.py,sha256=aIJfNMXTTQLamEk4L8mfTDDdyzscZUnhSPAor8RjE_8,4490 +datasets/features/video.py,sha256=KWa61swxqNNtKLq9EC_KSP2VcMpGqCqMJjJHHHgWEYw,17261 +datasets/filesystems/__init__.py,sha256=jBDUQosQqEFIXUDLZwRWaTgNomwL6Fq2qiYPvvxuae0,1523 +datasets/filesystems/compression.py,sha256=J16G_P0F9SxVfsLy9XLesyQJgvp4kpaO9MAihokEUvY,4528 +datasets/fingerprint.py,sha256=hnzIVs-jCCKJYqi6LvE8UWNmLNOJ97bvWz6uA1EAKiI,21922 +datasets/formatting/__init__.py,sha256=-pM10fSzw4MVj_L3NFWEv2sUyBh4mbnvCkfXgfS6WII,5412 +datasets/formatting/formatting.py,sha256=8zMv7dfTyyEDvzNjKDzT7Eb2_fn2n6OknfGmYK_0-MM,26626 +datasets/formatting/jax_formatter.py,sha256=Hd2Lgl94bkka4LNz9_Q45VIKSV2KWkSLp3DRXy9laiY,7494 +datasets/formatting/np_formatter.py,sha256=TlXk6IlA_sxzeESg9jUUlhPX_npO6ImVj56A4Hp58Ms,5184 +datasets/formatting/polars_formatter.py,sha256=oTm4l30SgGha-Oku42C0dA91Y8f2oifF9aWvi3QITDk,4744 +datasets/formatting/tf_formatter.py,sha256=KRtG0yMzbVQ5TzA2WYa3WlZKblLojkqHzbNMKXY5geo,5318 +datasets/formatting/torch_formatter.py,sha256=N-10rmMTD8XVUaS6-LQ9xqVxfyAMVeSJEKS6cojy74k,5393 +datasets/hub.py,sha256=6qnJVVTdEIShaUGlHrRjFJwvnJiNQuzsU01C8ID-8lA,4822 +datasets/info.py,sha256=R4AV3Of94kWMmFfZ7H66fusd7v2PWnRsCXpJaGRQFa4,19984 +datasets/inspect.py,sha256=dryCltaOJpfKyDxV2i-PWNE2P_VR_NPDBFRbLmeeQDQ,15498 +datasets/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/io/abc.py,sha256=LwDMXYs6YkhZuz1JiMK4PDIqgNjv7I8xH3UMUELW2ys,1672 +datasets/io/csv.py,sha256=aNVSwbZVPyVvi3a5ZsJr11FMEOMKcXKZvoB1OcV0a2U,5198 +datasets/io/generator.py,sha256=WtUWhml4llyZd_6gPuOPcC_YiKc5xkTXAjTN-z4ivsY,2098 +datasets/io/json.py,sha256=pAKp3fhtGYOs2-6fqlTyqdKH_ehZztVnfvvdv1G_IaM,7010 +datasets/io/parquet.py,sha256=QzEK2oKCMopP3talCc5gCvP0xb2e34XTsDD9kF39s9c,5868 +datasets/io/spark.py,sha256=VUIODLHgIbiK0CI0UvthQ_gUO0MQDtHUozvw7Dfs8FI,1797 +datasets/io/sql.py,sha256=KZhCE2-al9QYz-e8rPctYvWQ07uRk11oS08OI1AtA-M,4175 +datasets/io/text.py,sha256=9BdYqlv-XXERc_WJwlD9GugwlAOv53iMVdrBoxKL2UI,1908 +datasets/iterable_dataset.py,sha256=C1EQNrShaJ_b74IwXYDNWI9EmKFf77RjlGo3QDShqHU,226915 +datasets/load.py,sha256=ExlnM0-H4upMj7-k3s63c1MX4QlsmvFiDiZZOIjFFtI,79529 +datasets/naming.py,sha256=WzNdyHVGeAk_YPx3ZF09F-0g0WHVWil6UOQiKZFEFa0,3028 +datasets/packaged_modules/__init__.py,sha256=KdpPFE884RmN4nznKjdVQ3pWx4r5JcQ3uUTxIWyaaqc,6578 +datasets/packaged_modules/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/arrow/arrow.py,sha256=L2v-iUt583lAGJ9c_bzPm3gVqX0fV_LtoqSS73sRXUQ,3213 +datasets/packaged_modules/audiofolder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/audiofolder/audiofolder.py,sha256=N4mOZypp8oTI-9FBSeEFE-oQ23U6ZmqPlFcqbUkviA8,1744 +datasets/packaged_modules/cache/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/cache/cache.py,sha256=qfhDDDuLNTl870-uRFwc6VTxJf9Icg5hYzpsIVrOf_M,8293 +datasets/packaged_modules/csv/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/csv/csv.py,sha256=IY7nnFlOZIBmpcbdJ6wPPUVlhjd4K8rbw17h5YJU3Yw,8977 +datasets/packaged_modules/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/eval/eval.py,sha256=GRFbIzWPzDCmWxWBefQ-kN-jjvxM0-2TdEZgVw-Qo0c,3226 +datasets/packaged_modules/folder_based_builder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/folder_based_builder/folder_based_builder.py,sha256=YaQaj9fCEutUfrZINhuit0j6Szqaj6fOoUm6G9Wio9I,22939 +datasets/packaged_modules/generator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/generator/generator.py,sha256=HTpaY75-NypiL5g0mCgDcnvTaWvzdTPLIKnsFlLlpuI,1380 +datasets/packaged_modules/hdf5/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/hdf5/hdf5.py,sha256=CchNe8i3boLosSTddnmkn_AxATEEv-sbkDqhS1qwYaw,12592 +datasets/packaged_modules/imagefolder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/imagefolder/imagefolder.py,sha256=UpMVe8TUyayzHsVSfKN5wiXcc94QdamMvxauI4oFdw4,1956 +datasets/packaged_modules/json/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/json/json.py,sha256=qQb7Nx8oEWGjWouTDl-EPqjb-T308WPEsSMIMq-GUeA,19186 +datasets/packaged_modules/lance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/lance/lance.py,sha256=WASv1nlupD87eYrMk1ssmFMPpyOb5gWkoUL67murEmw,9833 +datasets/packaged_modules/niftifolder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/niftifolder/niftifolder.py,sha256=VPmkFLF590iCsG6b4vS3cONleyYM26cwUSi7kzj1ebg,575 +datasets/packaged_modules/pandas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/pandas/pandas.py,sha256=qLtTReWnf2am9BZUa76sJwKags3Zg3qQsLoszZZq81E,1954 +datasets/packaged_modules/parquet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/parquet/parquet.py,sha256=AaIIQBDiinH03ZTPUBvuRA6nsjosVFAfJUgG7mRweaw,10175 +datasets/packaged_modules/pdffolder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/pdffolder/pdffolder.py,sha256=bPYBh9-XOr2C-gg_Fl8h-UKhsVQ7VXjBL2FfW8abiGU,565 +datasets/packaged_modules/spark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/spark/spark.py,sha256=T9LDFok-hAWDgcjyW0SIMfId5BXF3CAbvl9zFx8Ut6w,14685 +datasets/packaged_modules/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/sql/sql.py,sha256=DFGr-BdMyO6xE0MS2CCp-TtRZ-PUNKivUXSwF5R4K4g,4536 +datasets/packaged_modules/text/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/text/text.py,sha256=TwrXqmv9-CsceZncffycwKi2BrdvzfLb4MKRQtFTuUM,6862 +datasets/packaged_modules/videofolder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/videofolder/videofolder.py,sha256=HLTMldDZ3WfK8OAbI2wssBuNCP6ucRBpNLpCoJVDL10,807 +datasets/packaged_modules/webdataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/webdataset/_tenbin.py,sha256=oovYsgR2R3eXSn1xSCLG3oTly1szKDP4UOiRp4ORdIk,8533 +datasets/packaged_modules/webdataset/webdataset.py,sha256=ANRARDeBugOAq5_3WXasI2IZ-276bum7AH6yqnWTO2I,10496 +datasets/packaged_modules/xml/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/packaged_modules/xml/xml.py,sha256=eMbwRIB1N7QSsvlIt1mPwseE_HYVuiXhyabxy-LILQk,2844 +datasets/parallel/__init__.py,sha256=wiRFK4x67ez2vvmjwM2Sb9R1yFdf38laSarU9y0Bido,76 +datasets/parallel/parallel.py,sha256=E-oOQ6zwKrkLFPwZ-3EOcr_aANJDhE-d6QTq7Mp7WvA,4738 +datasets/search.py,sha256=PxveDc1hy2Yux7Gtvat1_Qe_0D6aIKQBWI5KbV0gS6I,35592 +datasets/splits.py,sha256=10xtCAtS3XFHawm4AqZrfpXecfNcAcUnOJ9eVtwTkgY,23810 +datasets/streaming.py,sha256=7MIamZ2NmReUmJ_2pxgdSopIf7Oh5nFFAbb7WHLuW7E,5772 +datasets/table.py,sha256=h3JDG5XCUYKidrOVNq8T2LsxtGxNfR0B_3Pqf-DHDz8,95611 +datasets/utils/__init__.py,sha256=PuZtB9YTbRyvdwubnsx-JGdHuMA7p0I0Rmh0E_uxYF0,999 +datasets/utils/_dataset_viewer.py,sha256=SrE1N18S5yCoCx0rAhwaHNDVS9uhxjspA84iNT4TFRw,4397 +datasets/utils/_dill.py,sha256=RTXoz1e5nS-eHOsPvPhuKjIweEAl0cP8rFaKZkgmMQY,19409 +datasets/utils/_filelock.py,sha256=iXW3bxsIr5JWNemhKtF_-q_0ysajkUTItzMm8LY9LBY,2355 +datasets/utils/deprecation_utils.py,sha256=hTHwlzRs92NfNVudH71LMpW70sjbsP5amebrIgi3A-U,3452 +datasets/utils/doc_utils.py,sha256=HoSm0TFaQaCYGfDgNhpBJ4Xc2WQZuOD6dTxLd9D87fs,407 +datasets/utils/experimental.py,sha256=JgOjaEY3RWZ--3u0-ry82gLCDUpudfBfl-hWZ46SyS4,1097 +datasets/utils/extract.py,sha256=7rEeGuKxVrLlRwEkAeVFk-zMxq29XrIRyOvQihuEeHs,18318 +datasets/utils/file_utils.py,sha256=sOKdSorD7t0OhGOqb-QjOZpKrWJt-ZjSb1xsmPSpqig,55582 +datasets/utils/filelock.py,sha256=H6C5dQGFCzVKyeDRRY8fZ4YGTEvvNd-MTjpL_sWYb5k,352 +datasets/utils/hub.py,sha256=V2JGolL5VjFT0YiEhI0sxJED_9tGdvma7lH22d64S9I,130 +datasets/utils/info_utils.py,sha256=SN9r8OSFSyGCNtpcpmCzbvSNU-_7LqTJsaRv07DeSYI,4217 +datasets/utils/json.py,sha256=fmD3dS5Tc7t3rxZLNyx4LaVdmUK5tWZG4GraoT0MvOI,5269 +datasets/utils/logging.py,sha256=c2g1gl3IV4C7A2-ky0yfrHZuYi4P41HRTdS3XCmdMew,5381 +datasets/utils/metadata.py,sha256=Hrmn8xUoEzwpJKG3Y6tfJt5t7nW1OCxNjfLTlEaxsrI,9367 +datasets/utils/patching.py,sha256=yrDATN-xaGgoM747brbx4ZDMnvp-teahfOaW53RNVyk,4967 +datasets/utils/py_utils.py,sha256=KGsYmXmOXD3QHRMd3aFRYXpj47biTSRwtzUU-d0lm8U,23246 +datasets/utils/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +datasets/utils/resources/creators.json,sha256=XtIpMZefgBOdTevRrQTkFiufbgCbp_iyxseyphYQkn0,257 +datasets/utils/resources/languages.json,sha256=Z0rQNPsfje8zMi8KdvvwxF4APwwqcskJFUvhNiLAgPM,199138 +datasets/utils/resources/multilingualities.json,sha256=02Uc8RtRzfl13l98Y_alZm5HuMYwPzL78B0S5a1X-8c,205 +datasets/utils/resources/readme_structure.yaml,sha256=hNf9msoBZw5jfakQrDb0Af8T325TXdcaHsAO2MUcZvY,3877 +datasets/utils/resources/size_categories.json,sha256=_5nAP7z8R6t7_GfER81QudFO6Y1tqYu4AWrr4Aot8S8,171 +datasets/utils/sharding.py,sha256=VBQ4bRJQijMNDQTgFb1_ddlQ28wAcA0aQp4e-1jFIAk,4215 +datasets/utils/stratify.py,sha256=-MVaLmijYhGyKDpnZS9A8SiHekaIyVm84HVyIIQOmfg,4085 +datasets/utils/tf_utils.py,sha256=T3OysLGbkO7y-J-o9OVGyn9l-l-A3ruj-24JM_UULm8,24448 +datasets/utils/tqdm.py,sha256=IKCB-lz6BZwRokw1QHaQsWFGWL4ztWtRd5kCGzwp_Oc,4520 +datasets/utils/track.py,sha256=M81CGLn3MyJzHm98CQkbF3_1DG7evQsw-V52_Bp2paI,1838 +datasets/utils/typing.py,sha256=G11ytWmwjqVia2IdziRDIWvQ4mLJee-sKzgJfHqU16E,205 +datasets/utils/version.py,sha256=Z82cHpjTbQVJyWgnwSU8DsW2G0y-sSbSoOVeQrAds9k,3281 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/REQUESTED b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/WHEEL b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..505164bc02d63fe6b0b3299f849a77c5f1beeb41 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (75.8.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/entry_points.txt b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..a93c17922e3b5496fe8e1a43f3defa58727f5220 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +datasets-cli = datasets.commands.datasets_cli:main diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/top_level.txt b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..aee11b288aa3e6803c53bde002f7594c44497f5b --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets-4.8.5.dist-info/top_level.txt @@ -0,0 +1 @@ +datasets diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..103c3ca6eeec9a83fce17a7a904db1922737295f --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/__init__.py @@ -0,0 +1,47 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "4.8.5" + +from .arrow_dataset import Column, Dataset +from .arrow_reader import ReadInstruction +from .builder import ArrowBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder +from .combine import concatenate_datasets, interleave_datasets +from .dataset_dict import DatasetDict, IterableDatasetDict +from .download import * +from .features import * +from .fingerprint import disable_caching, enable_caching, is_caching_enabled +from .info import DatasetInfo +from .inspect import ( + get_dataset_config_info, + get_dataset_config_names, + get_dataset_default_config_name, + get_dataset_infos, + get_dataset_split_names, +) +from .iterable_dataset import IterableColumn, IterableDataset +from .load import load_dataset, load_dataset_builder, load_from_disk +from .splits import ( + NamedSplit, + NamedSplitAll, + Split, + SplitBase, + SplitDict, + SplitGenerator, + SplitInfo, + SubSplitInfo, + percent, +) +from .utils import * +from .utils import logging diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/arrow_dataset.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/arrow_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..3e3f2007952a6db1eead0060d304f3efbfe6b23d --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/arrow_dataset.py @@ -0,0 +1,7370 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Simple Dataset wrapping an Arrow Table.""" + +import asyncio +import contextlib +import copy +import glob +import inspect +import itertools +import json +import math +import os +import posixpath +import random +import re +import shutil +import string +import sys +import tempfile +import time +import warnings +import weakref +from collections import Counter, defaultdict +from collections.abc import Iterable, Iterator, Mapping +from collections.abc import Sequence as Sequence_ +from copy import deepcopy +from functools import partial, wraps +from math import ceil, floor +from pathlib import Path +from random import sample +from typing import ( + TYPE_CHECKING, + Any, + BinaryIO, + Callable, + Optional, + Union, + overload, +) + +import fsspec +import multiprocess as mp +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.dataset as pds +from fsspec.core import url_to_fs +from fsspec.implementations.dirfs import DirFileSystem +from huggingface_hub import ( + CommitInfo, + CommitOperationAdd, + CommitOperationDelete, + DatasetCard, + DatasetCardData, + HfApi, + HfFileSystem, + HfFileSystemResolvedPath, +) +from huggingface_hub.utils import EntryNotFoundError, HfHubHTTPError, RepositoryNotFoundError +from packaging import version +from tqdm.contrib.concurrent import thread_map + +from . import config +from .arrow_reader import ArrowReader +from .arrow_writer import ArrowWriter, OptimizedTypedSequence +from .data_files import sanitize_patterns +from .download.streaming_download_manager import xgetsize +from .features import Audio, ClassLabel, Features, Image, List, Value, Video +from .features.features import ( + FeatureType, + _align_features, + _check_if_features_can_be_aligned, + _fix_for_backward_compatible_features, + generate_from_arrow_type, + pandas_types_mapper, + require_decoding, +) +from .filesystems import is_remote_filesystem +from .fingerprint import ( + fingerprint_transform, + format_kwargs_for_fingerprint, + format_transform_for_fingerprint, + generate_fingerprint, + generate_random_fingerprint, + get_temporary_cache_files_directory, + is_caching_enabled, + maybe_register_dataset_for_temp_dir_deletion, + update_fingerprint, + validate_fingerprint, +) +from .formatting import format_table, get_format_type_from_alias, get_formatter, query_table +from .formatting.formatting import LazyDict, _is_range_contiguous +from .info import DatasetInfo, DatasetInfosDict +from .naming import _split_re +from .search import IndexableMixin +from .splits import NamedSplit, Split, SplitDict, SplitInfo +from .table import ( + InMemoryTable, + MemoryMappedTable, + Table, + _batch_arrow_table, + _memory_mapped_record_batch_reader_from_file, + cast_array_to_feature, + concat_tables, + embed_table_storage, + list_table_cache_files, + table_cast, + table_iter, + table_visitor, +) +from .utils import logging +from .utils import tqdm as hf_tqdm +from .utils.file_utils import estimate_dataset_size +from .utils.info_utils import is_small_dataset +from .utils.metadata import MetadataConfigs +from .utils.py_utils import ( + Literal, + asdict, + convert_file_size_to_int, + glob_pattern_to_regex, + iflatmap_unordered, + string_to_dict, +) +from .utils.stratify import stratified_shuffle_split_generate_indices +from .utils.tf_utils import dataset_to_tf, minimal_tf_collate_fn, multiprocess_dataset_to_tf +from .utils.typing import ListLike, PathLike + + +if config.HF_HUB_VERSION >= version.parse("1.6.0"): + from huggingface_hub.errors import BucketNotFoundError + from huggingface_hub.hf_file_system import HfFileSystemResolvedBucketPath, HfFileSystemResolvedRepositoryPath + +else: + BucketNotFoundError = None + HfFileSystemResolvedBucketPath = None + HfFileSystemResolvedRepositoryPath = HfFileSystemResolvedPath + + +if TYPE_CHECKING: + import sqlite3 + + import polars as pl + import pyspark + import sqlalchemy + + from .dataset_dict import DatasetDict + from .iterable_dataset import IterableDataset + +logger = logging.get_logger(__name__) + +PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED = ( + "data/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.parquet" +) + + +class DatasetInfoMixin: + """This base class exposes some attributes of DatasetInfo + at the base level of the Dataset for easy access. + """ + + def __init__(self, info: DatasetInfo, split: Optional[NamedSplit]): + self._info = info + self._split = split + + @property + def info(self): + """[`~datasets.DatasetInfo`] object containing all the metadata in the dataset.""" + return self._info + + @property + def split(self): + """[`~datasets.NamedSplit`] object corresponding to a named dataset split.""" + return self._split + + @property + def builder_name(self) -> str: + return self._info.builder_name + + @property + def citation(self) -> str: + return self._info.citation + + @property + def config_name(self) -> str: + return self._info.config_name + + @property + def dataset_size(self) -> Optional[int]: + return self._info.dataset_size + + @property + def description(self) -> str: + return self._info.description + + @property + def download_checksums(self) -> Optional[dict]: + return self._info.download_checksums + + @property + def download_size(self) -> Optional[int]: + return self._info.download_size + + @property + def features(self) -> Optional[Features]: + return self._info.features.copy() if self._info.features is not None else None + + @property + def homepage(self) -> Optional[str]: + return self._info.homepage + + @property + def license(self) -> Optional[str]: + return self._info.license + + @property + def size_in_bytes(self) -> Optional[int]: + return self._info.size_in_bytes + + @property + def supervised_keys(self): + return self._info.supervised_keys + + @property + def version(self): + return self._info.version + + +class TensorflowDatasetMixin: + _TF_DATASET_REFS = set() + + @staticmethod + def _get_output_signature( + dataset: "Dataset", + collate_fn: Callable, + collate_fn_args: dict, + cols_to_retain: Optional[list[str]] = None, + batch_size: Optional[int] = None, + num_test_batches: int = 20, + ): + """Private method used by `to_tf_dataset()` to find the shapes and dtypes of samples from this dataset + after being passed through the collate_fn. Tensorflow needs an exact signature for tf.numpy_function, so + the only way to do this is to run test batches - the collator may add or rename columns, so we can't figure + it out just by inspecting the dataset. + + Args: + dataset (`Dataset`): Dataset to load samples from. + collate_fn(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for + validation/evaluation. + collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate + lists of samples into a batch. + collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the + `collate_fn`. + batch_size (`int`, optional): The size of batches loaded from the dataset. Used for shape inference. + Can be None, which indicates that batch sizes can be variable. + num_test_batches (`int`): The number of batches to load from the dataset for shape inference. + + Returns: + `dict`: Dict mapping column names to tf.Tensorspec objects + `dict`: Dict mapping column names to np.dtype objects + """ + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + if len(dataset) == 0: + raise ValueError("Unable to get the output signature because the dataset is empty.") + if batch_size is not None: + batch_size = min(len(dataset), batch_size) + test_batch_size = 1 + + if cols_to_retain is not None: + cols_to_retain = list(set(cols_to_retain + ["label_ids", "label", "labels"])) + + test_batches = [] + for _ in range(num_test_batches): + indices = sample(range(len(dataset)), test_batch_size) + test_batch = dataset[indices] + if cols_to_retain is not None: + test_batch = {key: value for key, value in test_batch.items() if key in cols_to_retain} + test_batch = [{key: value[i] for key, value in test_batch.items()} for i in range(test_batch_size)] + test_batch = collate_fn(test_batch, **collate_fn_args) + test_batches.append(test_batch) + + tf_columns_to_signatures = {} + np_columns_to_dtypes = {} + for column in test_batches[0].keys(): + raw_arrays = [batch[column] for batch in test_batches] + # In case the collate_fn returns something strange + np_arrays = [] + for array in raw_arrays: + if isinstance(array, np.ndarray): + np_arrays.append(array) + elif isinstance(array, tf.Tensor): + np_arrays.append(array.numpy()) + else: + np_arrays.append(np.array(array)) + + if np.issubdtype(np_arrays[0].dtype, np.integer) or np_arrays[0].dtype == bool: + tf_dtype = tf.int64 + np_dtype = np.int64 + elif np.issubdtype(np_arrays[0].dtype, np.number): + tf_dtype = tf.float32 + np_dtype = np.float32 + elif np_arrays[0].dtype.kind == "U": # Unicode strings + np_dtype = np.str_ + tf_dtype = tf.string + else: + raise RuntimeError( + f"Unrecognized array dtype {np_arrays[0].dtype}. \n" + "Nested types and image/audio types are not supported yet." + ) + shapes = [array.shape for array in np_arrays] + static_shape = [] + for dim in range(len(shapes[0])): + sizes = {shape[dim] for shape in shapes} + if dim == 0: + static_shape.append(batch_size) + continue + if len(sizes) == 1: # This dimension looks constant + static_shape.append(sizes.pop()) + else: # Use None for variable dimensions + static_shape.append(None) + tf_columns_to_signatures[column] = tf.TensorSpec(shape=static_shape, dtype=tf_dtype) + np_columns_to_dtypes[column] = np_dtype + + return tf_columns_to_signatures, np_columns_to_dtypes + + def to_tf_dataset( + self, + batch_size: Optional[int] = None, + columns: Optional[Union[str, list[str]]] = None, + shuffle: bool = False, + collate_fn: Optional[Callable] = None, + drop_remainder: bool = False, + collate_fn_args: Optional[dict[str, Any]] = None, + label_cols: Optional[Union[str, list[str]]] = None, + prefetch: bool = True, + num_workers: int = 0, + num_test_batches: int = 20, + ): + """Create a `tf.data.Dataset` from the underlying Dataset. This `tf.data.Dataset` will load and collate batches from + the Dataset, and is suitable for passing to methods like `model.fit()` or `model.predict()`. The dataset will yield + `dicts` for both inputs and labels unless the `dict` would contain only a single key, in which case a raw + `tf.Tensor` is yielded instead. + + Args: + batch_size (`int`, *optional*): + Size of batches to load from the dataset. Defaults to `None`, which implies that the dataset won't be + batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`. + columns (`List[str]` or `str`, *optional*): + Dataset column(s) to load in the `tf.data.Dataset`. + Column names that are created by the `collate_fn` and that do not exist in the original dataset can be used. + shuffle(`bool`, defaults to `False`): + Shuffle the dataset order when loading. Recommended `True` for training, `False` for + validation/evaluation. + drop_remainder(`bool`, defaults to `False`): + Drop the last incomplete batch when loading. Ensures + that all batches yielded by the dataset will have the same length on the batch dimension. + collate_fn(`Callable`, *optional*): + A function or callable object (such as a `DataCollator`) that will collate + lists of samples into a batch. + collate_fn_args (`Dict`, *optional*): + An optional `dict` of keyword arguments to be passed to the + `collate_fn`. + label_cols (`List[str]` or `str`, defaults to `None`): + Dataset column(s) to load as labels. + Note that many models compute loss internally rather than letting Keras do it, in which case + passing the labels here is optional, as long as they're in the input `columns`. + prefetch (`bool`, defaults to `True`): + Whether to run the dataloader in a separate thread and maintain + a small buffer of batches for training. Improves performance by allowing data to be loaded in the + background while the model is training. + num_workers (`int`, defaults to `0`): + Number of workers to use for loading the dataset. + num_test_batches (`int`, defaults to `20`): + Number of batches to use to infer the output signature of the dataset. + The higher this number, the more accurate the signature will be, but the longer it will take to + create the dataset. + + Returns: + `tf.data.Dataset` + + Example: + + ```py + >>> ds_train = ds["train"].to_tf_dataset( + ... columns=['input_ids', 'token_type_ids', 'attention_mask', 'label'], + ... shuffle=True, + ... batch_size=16, + ... collate_fn=data_collator, + ... ) + ``` + """ + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + if (isinstance(columns, list) and len(columns) == 1) or ( + isinstance(label_cols, list) and len(label_cols) == 1 + ): + warnings.warn( + "The output of `to_tf_dataset` will change when a passing single element list for `labels` or " + "`columns` in the next datasets version. To return a tuple structure rather than dict, pass a " + "single string.\n" + "Old behaviour: columns=['a'], labels=['labels'] -> (tf.Tensor, tf.Tensor) \n" + " : columns='a', labels='labels' -> (tf.Tensor, tf.Tensor) \n" + "New behaviour: columns=['a'],labels=['labels'] -> ({'a': tf.Tensor}, {'labels': tf.Tensor}) \n" + " : columns='a', labels='labels' -> (tf.Tensor, tf.Tensor) ", + FutureWarning, + ) + + if isinstance(tf.distribute.get_strategy(), tf.distribute.TPUStrategy): + logger.warning( + "Note that to_tf_dataset() loads the data with a generator rather than a full tf.data " + "pipeline and is not compatible with remote TPU connections. If you encounter errors, please " + "try using a TPU VM or, if your data can fit in memory, loading it into memory as a dict of " + "Tensors instead of streaming with to_tf_dataset()." + ) + + if collate_fn is None: + # Set a very simple default collator that just stacks things together + collate_fn = minimal_tf_collate_fn + if collate_fn_args is None: + collate_fn_args = {} + if label_cols and not columns: + raise ValueError("Cannot specify label_cols without specifying columns!") + if label_cols is None: + label_cols = [] + elif isinstance(label_cols, str): + label_cols = [label_cols] + if len(set(label_cols)) < len(label_cols): + raise ValueError("List of label_cols contains duplicates.") + if columns: + if isinstance(columns, str): + columns = [columns] + if len(set(columns)) < len(columns): + raise ValueError("List of columns contains duplicates.") + cols_to_retain = list(set(columns + label_cols)) + else: + cols_to_retain = None # Indicates keeping all valid columns + columns = [] + + if self.format["type"] not in ["custom", "numpy"]: + dataset = self.with_format("numpy") + else: + dataset = self + + # TODO(Matt, QL): deprecate the retention of label_ids and label + + output_signature, columns_to_np_types = dataset._get_output_signature( + dataset, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + cols_to_retain=cols_to_retain, + batch_size=batch_size if drop_remainder else None, + num_test_batches=num_test_batches, + ) + + if "labels" in output_signature: + if ("label_ids" in columns or "label" in columns) and "labels" not in columns: + columns = [col for col in columns if col not in ["label_ids", "label"]] + ["labels"] + if ("label_ids" in label_cols or "label" in label_cols) and "labels" not in label_cols: + label_cols = [col for col in label_cols if col not in ["label_ids", "label"]] + ["labels"] + + for col in columns: + if col not in output_signature: + raise ValueError(f"Column {col} not found in dataset!") + + for col in label_cols: + if col not in output_signature: + raise ValueError(f"Label column {col} not found in dataset!") + + if num_workers == 0: + tf_dataset = dataset_to_tf( + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + output_signature=output_signature, + shuffle=shuffle, + batch_size=batch_size, + drop_remainder=drop_remainder, + ) + elif num_workers > 0: + if batch_size is None: + raise NotImplementedError( + "`batch_size` must be specified when using multiple workers, as unbatched multiprocessing " + "is not supported yet. Please provide a `batch_size` if `num_workers` is greater than 0." + ) + tf_dataset = multiprocess_dataset_to_tf( + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + output_signature=output_signature, + shuffle=shuffle, + batch_size=batch_size, + drop_remainder=drop_remainder, + num_workers=num_workers, + ) + else: + raise ValueError("num_workers must be >= 0") + + def split_features_and_labels(input_batch): + # TODO(Matt, QL): deprecate returning the dict content when there's only one key + features = {key: tensor for key, tensor in input_batch.items() if key in columns} + labels = {key: tensor for key, tensor in input_batch.items() if key in label_cols} + if len(features) == 1: + features = list(features.values())[0] + if len(labels) == 1: + labels = list(labels.values())[0] + if isinstance(labels, dict) and len(labels) == 0: + return features + else: + return features, labels + + if cols_to_retain is not None: + tf_dataset = tf_dataset.map(split_features_and_labels) + + if prefetch: + tf_dataset = tf_dataset.prefetch(tf.data.experimental.AUTOTUNE) + + # Remove a reference to the open Arrow file on delete + def cleanup_callback(ref): + dataset.__del__() + self._TF_DATASET_REFS.remove(ref) + + self._TF_DATASET_REFS.add(weakref.ref(tf_dataset, cleanup_callback)) + + return tf_dataset + + +class DatasetTransformationNotAllowedError(Exception): + pass + + +def transmit_format(func): + """Wrapper for dataset transforms that recreate a new Dataset to transmit the format of the original dataset to the new dataset""" + + @wraps(func) + def wrapper(*args, **kwargs): + if args: + self: "Dataset" = args[0] + args = args[1:] + else: + self: "Dataset" = kwargs.pop("self") + # don't use self.format since it returns a list of columns for 'columns' even if self_format_columns is None + unformatted_columns = set(self.column_names) - set(self._format_columns or []) + self_format = { + "type": self._format_type, + "format_kwargs": self._format_kwargs, + "columns": self._format_columns, + "output_all_columns": self._output_all_columns, + } + # apply actual function + out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs) + datasets: list["Dataset"] = list(out.values()) if isinstance(out, dict) else [out] + # re-apply format to the output + for dataset in datasets: + new_format = self_format.copy() + if new_format["columns"] is not None: # new formatted columns = (columns - previously unformatted columns) + # sort the columns to have a deterministic list of columns that we can compare with `out_format` + new_format["columns"] = sorted(set(dataset.column_names) - unformatted_columns) + out_format = { + "type": dataset._format_type, + "format_kwargs": dataset._format_kwargs, + "columns": sorted(dataset._format_columns) if dataset._format_columns is not None else None, + "output_all_columns": dataset._output_all_columns, + } + if out_format != new_format: + fingerprint = dataset._fingerprint + dataset.set_format(**new_format) + dataset._fingerprint = fingerprint + return out + + wrapper._decorator_name_ = "transmit_format" + return wrapper + + +def update_metadata_with_features(table: Table, features: Features): + """To be used in dataset transforms that modify the features of the dataset, in order to update the features stored in the metadata of its schema.""" + features = Features({col_name: features[col_name] for col_name in table.column_names}) + if table.schema.metadata is None or b"huggingface" not in table.schema.metadata: + pa_metadata = ArrowWriter._build_metadata(DatasetInfo(features=features)) + else: + metadata = json.loads(table.schema.metadata[b"huggingface"].decode()) + if "info" not in metadata: + metadata["info"] = asdict(DatasetInfo(features=features)) + else: + metadata["info"]["features"] = asdict(DatasetInfo(features=features))["features"] + pa_metadata = {"huggingface": json.dumps(metadata)} + table = table.replace_schema_metadata(pa_metadata) + return table + + +def _check_table(table) -> Table: + """We check the table type to make sure it's an instance of :class:`datasets.table.Table`""" + if isinstance(table, pa.Table): + # for a pyarrow table, we can just consider it as a in-memory table + # this is here for backward compatibility + return InMemoryTable(table) + elif isinstance(table, Table): + return table + else: + raise TypeError(f"Expected a pyarrow.Table or a datasets.table.Table object, but got {table}.") + + +def _check_column_names(column_names: list[str]): + """Check the column names to make sure they don't contain duplicates.""" + counter = Counter(column_names) + if not all(count == 1 for count in counter.values()): + duplicated_columns = [col for col in counter if counter[col] > 1] + raise ValueError(f"The table can't have duplicated columns but columns {duplicated_columns} are duplicated.") + + +def _check_valid_indices_value(index, size): + if (index < 0 and index + size < 0) or (index >= size): + raise IndexError(f"Index {index} out of range for dataset of size {size}.") + + +class NonExistentDatasetError(Exception): + """Used when we expect the existence of a dataset""" + + pass + + +class Column(Sequence_): + """ + An iterable for a specific column of a [`Dataset`]. + + Example: + + Iterate on the texts of the "text" column of a dataset: + + ```python + for text in dataset["text"]: + ... + ``` + + It also works with nested columns: + + ```python + for source in dataset["metadata"]["source"]: + ... + ``` + """ + + def __init__(self, source: Union["Dataset", "Column"], column_name: str): + self.source = source + self.column_name = column_name + if not isinstance(source.features, dict) or column_name not in source.features: + raise ValueError(f"Column '{column_name}' doesn't exist.") + self.features = source.features[column_name] + + def __iter__(self) -> Iterator[Any]: + if isinstance(self.source, Dataset): + if self.source._format_type == "custom": + # the formatting transform may require all columns + source = self.source + else: + source = self.source._fast_select_column(self.column_name) + else: + source = self.source + for example in source: + yield example[self.column_name] + + def __getitem__(self, key: Union[int, str, list[int]]) -> Any: + if isinstance(key, str): + return Column(self, key) + elif isinstance(self.source, Dataset): + if self.source._format_type == "custom": + # the formatting transform may require all columns + source = self.source + else: + source = self.source._fast_select_column(self.column_name) + return source[key][self.column_name] + elif isinstance(key, int): + return self.source[key][self.column_name] + else: + return [item[self.column_name] for item in self.source[key]] + + def __len__(self) -> int: + return len(self.source) + + def __repr__(self): + return "Column(" + repr(list(self[:5]))[:-1] + (", ...])" if len(self) > 5 else "])") + + def __str__(self): + return "Column(" + str(list(self[:5]))[:-1] + (", ...])" if len(self) > 5 else "])") + + def __eq__(self, value): + if isinstance(value, Column): + return list(self) == list(value) + else: + return value == list(self) + + +class Dataset(DatasetInfoMixin, IndexableMixin, TensorflowDatasetMixin): + """A Dataset backed by an Arrow table.""" + + def __init__( + self, + arrow_table: Table, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + indices_table: Optional[Table] = None, + fingerprint: Optional[str] = None, + ): + info = info.copy() if info is not None else DatasetInfo() + DatasetInfoMixin.__init__(self, info=info, split=split) + IndexableMixin.__init__(self) + + self._data: Table = _check_table(arrow_table) + self._indices: Optional[Table] = _check_table(indices_table) if indices_table is not None else None + maybe_register_dataset_for_temp_dir_deletion(self) + + self._format_type: Optional[str] = None + self._format_kwargs: dict = {} + self._format_columns: Optional[list] = None + self._output_all_columns: bool = False + self._fingerprint: str = fingerprint + + # Read metadata + + if self._data.schema.metadata is not None and b"huggingface" in self._data.schema.metadata: + metadata = json.loads(self._data.schema.metadata[b"huggingface"].decode()) + if ( + "fingerprint" in metadata and self._fingerprint is None + ): # try to load fingerprint from the arrow file metadata + self._fingerprint = metadata["fingerprint"] + + # Infer features if None + inferred_features = Features.from_arrow_schema(arrow_table.schema) + if self.info.features is None: + self.info.features = inferred_features + else: # make sure the nested columns are in the right order + try: + self.info.features = self.info.features.reorder_fields_as(inferred_features) + except ValueError as e: + raise ValueError( + f"{e}\nThe 'source' features come from dataset_info.json, and the 'target' ones are those of the dataset arrow file." + ) + + # In case there are types like pa.dictionary that we need to convert to the underlying type + + if self.data.schema != self.info.features.arrow_schema: + self._data = self.data.cast(self.info.features.arrow_schema) + + # Infer fingerprint if None + + if self._fingerprint is None: + self._fingerprint = generate_fingerprint(self) + + # Sanity checks + + if self._info.features is None: + raise ValueError("Features can't be None in a Dataset object") + if self._fingerprint is None: + raise ValueError("Fingerprint can't be None in a Dataset object") + if self.info.features.type != inferred_features.type: + raise ValueError( + f"External features info don't match the dataset:\nGot\n{self.info.features}\nwith type\n{self.info.features.type}\n\nbut expected something like\n{inferred_features}\nwith type\n{inferred_features.type}" + ) + + if self._indices is not None: + if not pa.types.is_unsigned_integer(self._indices.column(0).type): + raise ValueError( + f"indices must be an Arrow table of unsigned integers, current type is {self._indices.column(0).type}" + ) + _check_column_names(self._data.column_names) + + self._data = update_metadata_with_features(self._data, self._info.features) + + @property + def features(self) -> Features: + features = super().features + if features is None: # this is already checked in __init__ + raise ValueError("Features can't be None in a Dataset object") + return features + + @classmethod + def from_file( + cls, + filename: str, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + indices_filename: Optional[str] = None, + in_memory: bool = False, + ) -> "Dataset": + """Instantiate a Dataset backed by an Arrow table at filename. + + Args: + filename (`str`): + File name of the dataset. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + indices_filename (`str`, *optional*): + File names of the indices. + in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + + Returns: + [`Dataset`] + """ + table = ArrowReader.read_table(filename, in_memory=in_memory) + + if indices_filename is not None: + indices_pa_table = ArrowReader.read_table(indices_filename, in_memory=in_memory) + else: + indices_pa_table = None + + return cls( + arrow_table=table, + info=info, + split=split, + indices_table=indices_pa_table, + ) + + @classmethod + def from_buffer( + cls, + buffer: pa.Buffer, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + indices_buffer: Optional[pa.Buffer] = None, + ) -> "Dataset": + """Instantiate a Dataset backed by an Arrow buffer. + + Args: + buffer (`pyarrow.Buffer`): + Arrow buffer. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + indices_buffer (`pyarrow.Buffer`, *optional*): + Indices Arrow buffer. + + Returns: + [`Dataset`] + """ + table = InMemoryTable.from_buffer(buffer) + + if indices_buffer is not None: + indices_table = InMemoryTable.from_buffer(buffer) + else: + indices_table = None + + return cls(table, info=info, split=split, indices_table=indices_table) + + @classmethod + def from_pandas( + cls, + df: pd.DataFrame, + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + preserve_index: Optional[bool] = None, + ) -> "Dataset": + """ + Convert `pandas.DataFrame` to a `pyarrow.Table` to create a [`Dataset`]. + + The column types in the resulting Arrow Table are inferred from the dtypes of the `pandas.Series` in the + DataFrame. In the case of non-object Series, the NumPy dtype is translated to its Arrow equivalent. In the + case of `object`, we need to guess the datatype by looking at the Python objects in this Series. + + Be aware that Series of the `object` dtype don't carry enough information to always lead to a meaningful Arrow + type. In the case that we cannot infer a type, e.g. because the DataFrame is of length 0 or the Series only + contains `None/nan` objects, the type is set to `null`. This behavior can be avoided by constructing explicit + features and passing it to this function. + + Important: a dataset created with from_pandas() lives in memory + and therefore doesn't have an associated cache directory. + This may change in the future, but in the meantime if you + want to reduce memory usage you should write it back on disk + and reload using e.g. save_to_disk / load_from_disk. + + Args: + df (`pandas.DataFrame`): + Dataframe that contains the dataset. + features ([`Features`], *optional*): + Dataset features. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + preserve_index (`bool`, *optional*): + Whether to store the index as an additional column in the resulting Dataset. + The default of `None` will store the index as a column, except for `RangeIndex` which is stored as metadata only. + Use `preserve_index=True` to force it to be stored as a column. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> ds = Dataset.from_pandas(df) + ``` + """ + if info is not None and features is not None and info.features != features: + raise ValueError( + f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}" + ) + features = features if features is not None else info.features if info is not None else None + if features is not None: + features = _fix_for_backward_compatible_features(features) + if info is None: + info = DatasetInfo() + info.features = features + table = InMemoryTable.from_pandas( + df=df, + preserve_index=preserve_index, + ) + if features is not None: + # more expensive cast than InMemoryTable.from_pandas(..., schema=features.arrow_schema) + # needed to support the str to Audio conversion for instance + table = table.cast(features.arrow_schema) + return cls(table, info=info, split=split) + + @classmethod + def from_polars( + cls, + df: Union["pl.DataFrame", "pl.LazyFrame"], + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + ) -> "Dataset": + """ + Collect the underlying arrow arrays in an Arrow Table. + + This operation is mostly zero copy. + + Data types that do copy: + * CategoricalType + + Args: + df (`polars.DataFrame`): DataFrame to convert to Arrow Table + features (`Features`, optional): Dataset features. + info (`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (`NamedSplit`, optional): Name of the dataset split. + + Examples: + ```py + >>> ds = Dataset.from_polars(df) + ``` + """ + import polars as pl + + if info is not None and features is not None and info.features != features: + raise ValueError( + f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}" + ) + features = features if features is not None else info.features if info is not None else None + if features is not None: + features = _fix_for_backward_compatible_features(features) + if info is None: + info = DatasetInfo() + info.features = features + if isinstance(df, pl.LazyFrame): + df = df.collect() + table = InMemoryTable(df.to_arrow()) + if features is not None: + # more expensive cast than InMemoryTable.from_polars(..., schema=features.arrow_schema) + # needed to support the str to Audio conversion for instance + table = table.cast(features.arrow_schema) + return cls(table, info=info, split=split) + + @classmethod + def from_dict( + cls, + mapping: dict, + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + on_mixed_types: Optional[Literal["use_json"]] = None, + ) -> "Dataset": + """ + Convert `dict` to a `pyarrow.Table` to create a [`Dataset`]. + + Important: a dataset created with from_dict() lives in memory + and therefore doesn't have an associated cache directory. + This may change in the future, but in the meantime if you + want to reduce memory usage you should write it back on disk + and reload using e.g. save_to_disk / load_from_disk. + + Args: + mapping (`Mapping`): + Mapping of strings to Arrays or Python lists. + features ([`Features`], *optional*): + Dataset features. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`): + If "use_json", use the Json() type for mixed-types fields, + i.e. unstructured fields that contain data without a predefined schema. + In this case, a field with mixed type is set to Json(). + + This allow loading lists with a mix of strings/integers/floats + for example, or dictionaries with arbitrary value types. + + + + Returns: + [`Dataset`] + + Examples: + + Get a Dataset from a dictionary containing one list per column: + + ```py + >>> ds = Dataset.from_dict({"text": ["hello there !", "general kenobi !"]}) + ``` + + Pass features to set the column types, e.g. for an image dataset: + + ```py + >>> features = Features({"image": Image()}) + >>> ds = Dataset.from_dict({"image": ["path/to/image.png"]}, features=features) + ``` + + Datasets are based on Arrow which is a columnar format, and therefore they expect every example to have the same + type and subtypes, and dictionaries to have the same keys and values types. + Loading a dataset errors out when fields have mismatching types, and fills missing fields in dictionaries with None so all dictionaries have the same keys and value types. + + To avoid this and allow mixed-types without errors, you can use `on_mixed_types="use_json"` or specify `features=` with a [`Json`] type: + + ```py + >>> ds = Dataset.from_dict({"a": [0, "foo", {"subfield": "bar"}]}) + Traceback (most recent call last): + ... + File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status + pyarrow.lib.ArrowInvalid: Could not convert 'foo' with type str: tried to convert to int64 + + >>> ds = Dataset.from_dict({"a": [0, "foo", {"subfield": "bar"}]}, on_mixed_types="use_json") + >>> ds.features + {'a': Json()} + >>> list(ds["a"]) + [0, "foo", {"subfield": "bar"}] + + >>> features = Features({"a": Json()}) + >>> ds = Dataset.from_dict({"a": [0, "foo", {"subfield": "bar"}]}, features=features) + >>> ds.features + {'a': Json()} + >>> list(ds["a"]) + [0, "foo", {"subfield": "bar"}] + ``` + + This is also useful for lists of dictionaries with arbitrary keys and values, to avoid filling missing fields with None: + + ```py + >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}) + >>> ds.features + {'a': List({'b': Value('int64'), 'c': Value('int64')})} + >>> list(ds["a"]) + [[{'b': 0, 'c': None}, {'b': None, 'c': 0}]] # missing fields are filled with None + + >>> features = Features({"a": List(Json())}) + >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}, features=features) + >>> ds.features + {'a': List(Json())} + >>> list(ds["a"]) + [[{'b': 0}, {'c': 0}]] # OK + + >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}, on_mixed_types="use_json") + >>> ds.features + {'a': List(Json())} + >>> list(ds["a"]) + [[{'b': 0}, {'c': 0}]] # OK + ``` + + Another example with tool calling data: + + ```py + >>> messages = [ + ... {"role": "user", "content": "Turn on the living room lights and play my electronic music playlist."}, + ... {"role": "assistant", "tool_calls": [ + ... {"type": "function", "function": { + ... "name": "control_light", + ... "arguments": {"room": "living room", "state": "on"} + ... }}, + ... {"type": "function", "function": { + ... "name": "play_music", + ... "arguments": {"playlist": "electronic"} # mixed-type here since keys ["playlist"] and ["room", "state"] are different + ... }}] + ... }, + ... {"role": "tool", "name": "control_light", "content": "The lights in the living room are now on."}, + ... {"role": "tool", "name": "play_music", "content": "The music is now playing."}, + ... {"role": "assistant", "content": "Done!"} + ... ] + >>> ds = Dataset.from_dict({"messages": [messages]}, on_mixed_types="use_json") + >>> ds.features + {'messages': List({'role': Value('string'), 'content': Value('string'), 'tool_calls': List(Json()), 'name': Value('string')})} + >>> ds[0]["messages"][1]["tool_calls"][0]["function"]["arguments"] + {"room": "living room", "state": "on"} + ``` + """ + if info is not None and features is not None and info.features != features: + raise ValueError( + f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}" + ) + features = features if features is not None else info.features if info is not None else None + if features is not None: + features = _fix_for_backward_compatible_features(features) + arrow_typed_mapping = {} + for col, data in mapping.items(): + if isinstance(data, (pa.Array, pa.ChunkedArray)): + data = cast_array_to_feature(data, features[col]) if features is not None else data + else: + data = OptimizedTypedSequence( + features.encode_column(data, col) if features is not None else data, + type=features[col] if features is not None else None, + col=col, + on_mixed_types=on_mixed_types, + ) + arrow_typed_mapping[col] = data + mapping = arrow_typed_mapping + pa_table = InMemoryTable.from_pydict(mapping=mapping) + if info is None: + info = DatasetInfo() + info.features = features + if info.features is None: + info.features = Features( + { + col: generate_from_arrow_type(data.type) + if isinstance(data, (pa.Array, pa.ChunkedArray)) + else data.get_inferred_type() + for col, data in mapping.items() + } + ) + return cls(pa_table, info=info, split=split) + + @classmethod + def from_list( + cls, + mapping: list[dict], + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + on_mixed_types: Optional[Literal["use_json"]] = None, + ) -> "Dataset": + """ + Convert a list of dicts to a `pyarrow.Table` to create a [`Dataset`]`. + + Note that the keys of the first entry will be used to determine the dataset columns, + regardless of what is passed to features. + + Important: a dataset created with from_list() lives in memory + and therefore doesn't have an associated cache directory. + This may change in the future, but in the meantime if you + want to reduce memory usage you should write it back on disk + and reload using e.g. save_to_disk / load_from_disk. + + Args: + mapping (`List[dict]`): A list of mappings of strings to row values. + features (`Features`, optional): Dataset features. + info (`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (`NamedSplit`, optional): Name of the dataset split. + on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`): + If "use_json", use the Json() type for mixed-types fields, + i.e. unstructured fields that contain data without a predefined schema. + In this case, a field with mixed type is set to Json(). + + This allow loading lists with a mix of strings/integers/floats + for example, or dictionaries with arbitrary value types. + + + + Returns: + [`Dataset`] + + Examples: + + Get a Dataset from a list containing the examples: + + ```py + >>> ds = Dataset.from_list([{"text": "hello there !"}, {"text": "general kenobi !"}]}) + ``` + + Pass features to set the column types, e.g. for an image dataset: + + ```py + >>> features = Features({"image": Image()}) + >>> ds = Dataset.from_list([{"image": "path/to/image.png"}], features=features) + ``` + + Datasets are based on Arrow which is a columnar format, and therefore they expect every example to have the same + type and subtypes, and dictionaries to have the same keys and values types. + Loading a dataset errors out when fields have mismatching types, and fills missing fields in dictionaries with None so all dictionaries have the same keys and value types. + + To avoid this and allow mixed-types without errors, you can use `on_mixed_types="use_json"` or specify `features=` with a [`Json`] type: + + ```py + >>> ds = Dataset.from_list([{"a": 0}, {"a": "foo"}, {"a": {"subfield": "bar"}}]) + Traceback (most recent call last): + ... + File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status + pyarrow.lib.ArrowInvalid: Could not convert 'foo' with type str: tried to convert to int64 + + >>> ds = Dataset.from_list([{"a": 0}, {"a": "foo"}, {"a": {"subfield": "bar"}}], on_mixed_types="use_json") + >>> ds.features + {'a': Json()} + >>> list(ds["a"]) + [0, "foo", {"subfield": "bar"}] + + >>> features = Features({"a": Json()}) + >>> ds = Dataset.from_list([{"a": 0}, {"a": "foo"}, {"a": {"subfield": "bar"}}], features=features) + >>> ds.features + {'a': Json()} + >>> list(ds["a"]) + [0, "foo", {"subfield": "bar"}] + ``` + + This is also useful for lists of dictionaries with arbitrary keys and values, to avoid filling missing fields with None: + + ```py + >>> ds = Dataset.from_list([{"a": [{"b": 0}, {"c": 0}]}]) + >>> ds.features + {'a': List({'b': Value('int64'), 'c': Value('int64')})} + >>> list(ds["a"]) + [[{'b': 0, 'c': None}, {'b': None, 'c': 0}]] # missing fields are filled with None + + >>> features = Features({"a": List(Json())}) + >>> ds = Dataset.from_list([{"a": [{"b": 0}, {"c": 0}]}], features=features) + >>> ds.features + {'a': List(Json())} + >>> list(ds["a"]) + [[{'b': 0}, {'c': 0}]] # OK + + >>> ds = Dataset.from_list([{"a": [{"b": 0}, {"c": 0}]}], on_mixed_types="use_json") + >>> ds.features + {'a': List(Json())} + >>> list(ds["a"]) + [[{'b': 0}, {'c': 0}]] # OK + ``` + + Another example with tool calling data: + + ```py + >>> messages = [ + ... {"role": "user", "content": "Turn on the living room lights and play my electronic music playlist."}, + ... {"role": "assistant", "tool_calls": [ + ... {"type": "function", "function": { + ... "name": "control_light", + ... "arguments": {"room": "living room", "state": "on"} + ... }}, + ... {"type": "function", "function": { + ... "name": "play_music", + ... "arguments": {"playlist": "electronic"} # mixed-type here since keys ["playlist"] and ["room", "state"] are different + ... }}] + ... }, + ... {"role": "tool", "name": "control_light", "content": "The lights in the living room are now on."}, + ... {"role": "tool", "name": "play_music", "content": "The music is now playing."}, + ... {"role": "assistant", "content": "Done!"} + ... ] + >>> ds = Dataset.from_list([{"messages": messages}], on_mixed_types="use_json") + >>> ds.features + {'messages': List({'role': Value('string'), 'content': Value('string'), 'tool_calls': List(Json()), 'name': Value('string')})} + >>> ds[0]["messages"][1]["tool_calls"][0]["function"]["arguments"] + {"room": "living room", "state": "on"} + ``` + """ + # for simplicity and consistency wrt OptimizedTypedSequence we do not use InMemoryTable.from_pylist here + mapping = {k: [r.get(k) for r in mapping] for k in mapping[0]} if mapping else {} + return cls.from_dict(mapping, features, info, split, on_mixed_types=on_mixed_types) + + @staticmethod + def from_csv( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ) -> "Dataset": + """Create Dataset from CSV file(s). + + Read the CSV files, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk. + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the CSV file(s). + split ([`NamedSplit`], *optional*): + Split name to be assigned to the dataset. + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default. + + + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`pandas.read_csv`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> ds = Dataset.from_csv('path/to/dataset.csv') + ``` + """ + # Dynamic import to avoid circular dependency + from .io.csv import CsvDatasetReader + + return CsvDatasetReader( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + num_proc=num_proc, + **kwargs, + ).read() + + @staticmethod + def from_generator( + generator: Callable, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + gen_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + split: NamedSplit = Split.TRAIN, + fingerprint: Optional[str] = None, + **kwargs, + ) -> "Dataset": + """Create a Dataset from a generator. + + Load the data from the generator, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk. + + Args: + generator (:`Callable`): + A generator function that `yields` examples. + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + gen_kwargs(`dict`, *optional*): + Keyword arguments to be passed to the `generator` callable. + You can define a sharded dataset by passing the list of shards in `gen_kwargs` and setting `num_proc` greater than 1. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default. + If `num_proc` is greater than one, then all list values in `gen_kwargs` must be the same length. These values will be split between calls to the generator. The number of shards will be the minimum of the shortest list in `gen_kwargs` and `num_proc`. + + + split ([`NamedSplit`], defaults to `Split.TRAIN`): + Split name to be assigned to the dataset. + + + fingerprint (`str`, *optional*): + Fingerprint that will be used to generate dataset ID. + By default `fingerprint` is generated by hashing the generator function and all the args which can be slow + if it uses large objects like AI models. + + + **kwargs (additional keyword arguments): + Keyword arguments to be passed to :[`GeneratorConfig`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> def gen(): + ... yield {"text": "Good", "label": 0} + ... yield {"text": "Bad", "label": 1} + ... + >>> ds = Dataset.from_generator(gen) + ``` + + ```py + >>> def gen(shards): + ... for shard in shards: + ... with open(shard) as f: + ... for line in f: + ... yield {"line": line} + ... + >>> shards = [f"data{i}.txt" for i in range(32)] + >>> ds = Dataset.from_generator(gen, gen_kwargs={"shards": shards}) + ``` + """ + from .io.generator import GeneratorDatasetInputStream + + return GeneratorDatasetInputStream( + generator=generator, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + gen_kwargs=gen_kwargs, + num_proc=num_proc, + split=split, + fingerprint=fingerprint, + **kwargs, + ).read() + + @staticmethod + def from_json( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + field: Optional[str] = None, + num_proc: Optional[int] = None, + **kwargs, + ) -> "Dataset": + """Create Dataset from JSON or JSON Lines file(s). + + Read the JSON files, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk. + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the JSON or JSON Lines file(s). + split ([`NamedSplit`], *optional*): + Split name to be assigned to the dataset. + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + field (`str`, *optional*): + Field name of the JSON file where the dataset is contained in. + num_proc (`int`, *optional* defaults to `None`): + Number of processes when downloading and generating the dataset locally. + This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default. + + + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`JsonConfig`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> ds = Dataset.from_json('path/to/dataset.json') + ``` + """ + # Dynamic import to avoid circular dependency + from .io.json import JsonDatasetReader + + return JsonDatasetReader( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + field=field, + num_proc=num_proc, + **kwargs, + ).read() + + @staticmethod + def from_parquet( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + columns: Optional[list[str]] = None, + num_proc: Optional[int] = None, + filters: Optional[Union[pds.Expression, list[tuple], list[list[tuple]]]] = None, + fragment_scan_options: Optional[pds.ParquetFragmentScanOptions] = None, + on_bad_files: Literal["error", "warn", "skip"] = "error", + **kwargs, + ) -> "Dataset": + """Create Dataset from Parquet file(s). + + Read the Parquet files, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk. + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the Parquet file(s). + split (`NamedSplit`, *optional*): + Split name to be assigned to the dataset. + features (`Features`, *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + columns (`List[str]`, *optional*): + If not `None`, only these columns will be read from the file. + A column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e'. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default. + + + filters (`Union[pyarrow.dataset.Expression, list[tuple], list[list[tuple]]]`, *optional*): + Return only the rows matching the filter. + If possible the predicate will be pushed down to exploit the partition information + or internal metadata found in the data source, e.g. Parquet statistics. + Otherwise filters the loaded RecordBatches before yielding them. + fragment_scan_options (`pyarrow.dataset.ParquetFragmentScanOptions`, *optional*) + Scan-specific options for Parquet fragments. + This is especially useful to configure buffering and caching. + + + on_bad_files (`Literal["error", "warn", "skip"]`, *optional*, defaults to "error") + Specify what to do upon encountering a bad file (a file that can't be read). Allowed values are : + * 'error', raise an Exception when a bad file is encountered. + * 'warn', raise a warning when a bad file is encountered and skip that file. + * 'skip', skip bad files without raising or warning when they are encountered. + + + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`ParquetConfig`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> ds = Dataset.from_parquet('path/to/dataset.parquet') + ``` + + Load a subset of columns: + + ```python + >>> ds = Dataset.from_parquet('path/to/dataset.parquet', columns=["col_0", "col_1"]) + ``` + + Efficiently filter data, possibly skipping entire files or row groups: + + ```python + >>> filters = [("col_0", "==", 0)] + >>> ds = Dataset.from_parquet(parquet_files_list, filters=filters) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.parquet import ParquetDatasetReader + + return ParquetDatasetReader( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + columns=columns, + num_proc=num_proc, + filters=filters, + fragment_scan_options=fragment_scan_options, + on_bad_files=on_bad_files, + **kwargs, + ).read() + + @staticmethod + def from_text( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + num_proc: Optional[int] = None, + keep_linebreaks: bool = False, + sample_by: Literal["line", "paragraph", "document"] = "line", + **kwargs, + ) -> "Dataset": + """Create Dataset from text file(s). + + Read the text files, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk. + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the text file(s). + split (`NamedSplit`, *optional*): + Split name to be assigned to the dataset. + features (`Features`, *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + This is helpful if the dataset is made of multiple files. Multiprocessing is disabled by default. + + + keep_linebreaks: (`bool`, defaults to False): + Whether to keep line breaks. + sample_by (`Literal["line", "paragraph", "document"]`, defaults to "line"): + Whether to load data per line, praragraph or document. + By default one row in the dataset = one line. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`TextConfig`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> ds = Dataset.from_text('path/to/dataset.txt') + ``` + """ + # Dynamic import to avoid circular dependency + from .io.text import TextDatasetReader + + return TextDatasetReader( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + num_proc=num_proc, + keep_linebreaks=keep_linebreaks, + sample_by=sample_by, + **kwargs, + ).read() + + @staticmethod + def from_spark( + df: "pyspark.sql.DataFrame", + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + keep_in_memory: bool = False, + cache_dir: str = None, + working_dir: str = None, + load_from_cache_file: bool = True, + **kwargs, + ) -> "Dataset": + """Create a Dataset from Spark DataFrame. Dataset downloading is distributed over Spark workers. + + Read the Spark DataFrame, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk. + + Args: + df (`pyspark.sql.DataFrame`): + The DataFrame containing the desired data. + split (`NamedSplit`, *optional*): + Split name to be assigned to the dataset. + features (`Features`, *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. When using a multi-node Spark cluster, the cache_dir must be accessible to both + workers and the driver. + keep_in_memory (`bool`): + Whether to copy the data in-memory. + working_dir (`str`, *optional*) + Intermediate directory for each Spark worker to write data to before moving it to `cache_dir`. Setting + a non-NFS intermediate directory may improve performance. + load_from_cache_file (`bool`): + Whether to load the dataset from the cache if possible. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> df = spark.createDataFrame( + >>> data=[[1, "Elia"], [2, "Teo"], [3, "Fang"]], + >>> columns=["id", "name"], + >>> ) + >>> ds = Dataset.from_spark(df) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.spark import SparkDatasetReader + + if sys.platform == "win32": + raise OSError("Dataset.from_spark is not currently supported on Windows") + + return SparkDatasetReader( + df, + split=split, + features=features, + streaming=False, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + working_dir=working_dir, + load_from_cache_file=load_from_cache_file, + **kwargs, + ).read() + + @staticmethod + def from_sql( + sql: Union[str, "sqlalchemy.sql.Selectable"], + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + **kwargs, + ) -> "Dataset": + """Create Dataset from SQL query or database table. + + Query the SQL database, cache the data in Arrow format on disk and return the Dataset from the memory-mapped Arrow data on disk. + + Args: + sql (`str` or `sqlalchemy.sql.Selectable`): + SQL query to be executed or a table name. + con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`): + A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) used to instantiate a database connection or a SQLite3/SQLAlchemy connection object. + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`SqlConfig`]. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> # Fetch a database table + >>> ds = Dataset.from_sql("test_data", "postgres:///db_name") + >>> # Execute a SQL query on the table + >>> ds = Dataset.from_sql("SELECT sentence FROM test_data", "postgres:///db_name") + >>> # Use a Selectable object to specify the query + >>> from sqlalchemy import select, text + >>> stmt = select([text("sentence")]).select_from(text("test_data")) + >>> ds = Dataset.from_sql(stmt, "postgres:///db_name") + ``` + + > [!TIP] + > The returned dataset can only be cached if `con` is specified as URI string. + """ + from .io.sql import SqlDatasetReader + + return SqlDatasetReader( + sql, + con, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + **kwargs, + ).read() + + def __setstate__(self, state): + self.__dict__.update(state) + maybe_register_dataset_for_temp_dir_deletion(self) + return self + + def __del__(self): + if hasattr(self, "_data"): + del self._data + if hasattr(self, "_indices"): + del self._indices + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Here `del` is used to del the pyarrow tables. This properly closes the files used for memory mapped tables + self.__del__() + + def save_to_disk( + self, + dataset_path: PathLike, + max_shard_size: Optional[Union[str, int]] = None, + num_shards: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + ): + """ + Saves a dataset to a dataset directory, or in a filesystem using any implementation of `fsspec.spec.AbstractFileSystem`. + + For [`Image`], [`Audio`] and [`Video`] data: + + All the Image(), Audio() and Video() data are stored in the arrow files. + If you want to store paths or urls, please use the Value("string") type. + + Args: + dataset_path (`path-like`): + Path (e.g. `dataset/train`) or remote URI (e.g. `s3://my-bucket/dataset/train`) + of the dataset directory where the dataset will be saved to. + max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`): + The maximum size of the dataset shards to be saved to the filesystem. If expressed as a string, needs to be digits followed by a unit + (like `"50MB"`). + num_shards (`int`, *optional*): + Number of shards to write. By default the number of shards depends on `max_shard_size` and `num_proc`. + + + num_proc (`int`, *optional*): + Number of processes when downloading and generating the dataset locally. + Multiprocessing is disabled by default. + + + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Example: + + ```py + >>> ds.save_to_disk("path/to/dataset/directory") + >>> ds.save_to_disk("path/to/dataset/directory", max_shard_size="1GB") + >>> ds.save_to_disk("path/to/dataset/directory", num_shards=1024) + ``` + """ + if max_shard_size is not None and num_shards is not None: + raise ValueError( + "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both." + ) + if self.list_indexes(): + raise ValueError("please remove all the indexes using `dataset.drop_index` before saving a dataset") + + if num_shards is None: + dataset_nbytes = self._estimate_nbytes() + max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE) + num_shards = int(dataset_nbytes / max_shard_size) + 1 + num_shards = max(num_shards, num_proc or 1) + # if we have only a few large samples, we should only create as many shards as samples + num_shards = min(len(self.data), num_shards) + + fs: fsspec.AbstractFileSystem + fs, _ = url_to_fs(dataset_path, **(storage_options or {})) + + if not is_remote_filesystem(fs): + parent_cache_files_paths = { + Path(cache_filename["filename"]).resolve().parent for cache_filename in self.cache_files + } + # Check that the dataset doesn't overwrite itself. It can cause a permission error on Windows and a segfault on linux. + if Path(dataset_path).expanduser().resolve() in parent_cache_files_paths: + raise PermissionError( + f"Tried to overwrite {Path(dataset_path).expanduser().resolve()} but a dataset can't overwrite itself." + ) + + fs.makedirs(dataset_path, exist_ok=True) + + # Get json serializable state + state = { + key: self.__dict__[key] + for key in [ + "_fingerprint", + "_format_columns", + "_format_kwargs", + "_format_type", + "_output_all_columns", + ] + } + state["_split"] = str(self.split) if self.split is not None else self.split + state["_data_files"] = [ + {"filename": f"data-{shard_idx:05d}-of-{num_shards:05d}.arrow"} for shard_idx in range(num_shards) + ] + for k in state["_format_kwargs"].keys(): + try: + json.dumps(state["_format_kwargs"][k]) + except TypeError as e: + raise TypeError( + str(e) + f"\nThe format kwargs must be JSON serializable, but key '{k}' isn't." + ) from None + # Get json serializable dataset info + dataset_info = asdict(self._info) + + shards_done = 0 + pbar = hf_tqdm( + unit=" examples", + total=len(self), + desc=f"Saving the dataset ({shards_done}/{num_shards} shards)", + ) + kwargs_per_job = ( + { + "job_id": shard_idx, + "shard": self.shard(num_shards=num_shards, index=shard_idx, contiguous=True), + "fpath": posixpath.join(dataset_path, f"data-{shard_idx:05d}-of-{num_shards:05d}.arrow"), + "storage_options": storage_options, + } + for shard_idx in range(num_shards) + ) + shard_lengths = [None] * num_shards + shard_sizes = [None] * num_shards + if num_proc is not None and num_proc >= 1: + with mp.get_context("spawn").Pool(num_proc) as pool: + with pbar: + for job_id, done, content in iflatmap_unordered( + pool, Dataset._save_to_disk_single, kwargs_iterable=kwargs_per_job + ): + if done: + shards_done += 1 + pbar.set_description(f"Saving the dataset ({shards_done}/{num_shards} shards)") + logger.debug(f"Finished writing shard number {job_id} of {num_shards}.") + shard_lengths[job_id], shard_sizes[job_id] = content + else: + pbar.update(content) + else: + with pbar: + for kwargs in kwargs_per_job: + for job_id, done, content in Dataset._save_to_disk_single(**kwargs): + if done: + shards_done += 1 + pbar.set_description(f"Saving the dataset ({shards_done}/{num_shards} shards)") + logger.debug(f"Finished writing shard number {job_id} of {num_shards}.") + shard_lengths[job_id], shard_sizes[job_id] = content + else: + pbar.update(content) + with fs.open( + posixpath.join(dataset_path, config.DATASET_STATE_JSON_FILENAME), "w", encoding="utf-8" + ) as state_file: + json.dump(state, state_file, indent=2, sort_keys=True) + with fs.open( + posixpath.join(dataset_path, config.DATASET_INFO_FILENAME), "w", encoding="utf-8" + ) as dataset_info_file: + # Sort only the first level of keys, or we might shuffle fields of nested features if we use sort_keys=True + sorted_keys_dataset_info = {key: dataset_info[key] for key in sorted(dataset_info)} + json.dump(sorted_keys_dataset_info, dataset_info_file, indent=2) + + @staticmethod + def _save_to_disk_single(job_id: int, shard: "Dataset", fpath: str, storage_options: Optional[dict]): + batch_size = config.DEFAULT_MAX_BATCH_SIZE + + num_examples_progress_update = 0 + writer = ArrowWriter( + features=shard.features, + path=fpath, + storage_options=storage_options, + embed_local_files=True, + ) + try: + _time = time.time() + for pa_table in shard.with_format("arrow").iter(batch_size): + writer.write_table(pa_table) + num_examples_progress_update += len(pa_table) + if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL: + _time = time.time() + yield job_id, False, num_examples_progress_update + num_examples_progress_update = 0 + finally: + yield job_id, False, num_examples_progress_update + num_examples, num_bytes = writer.finalize() + writer.close() + + yield job_id, True, (num_examples, num_bytes) + + @staticmethod + def _build_local_temp_path(uri_or_path: str) -> Path: + """ + Builds and returns a Path concatenating a local temporary dir with the dir path (or absolute/relative + path extracted from the uri) passed. + + Args: + uri_or_path (`str`): Path (e.g. `"dataset/train"`) or remote URI (e.g. + `"s3://my-bucket/dataset/train"`) to concatenate. + + Returns: + :class:`Path`: the concatenated path (temp dir + path) + """ + src_dataset_path = Path(uri_or_path) + tmp_dir = get_temporary_cache_files_directory() + return Path(tmp_dir, src_dataset_path.relative_to(src_dataset_path.anchor)) + + @staticmethod + def load_from_disk( + dataset_path: PathLike, + keep_in_memory: Optional[bool] = None, + storage_options: Optional[dict] = None, + ) -> "Dataset": + """ + Loads a dataset that was previously saved using [`save_to_disk`] from a dataset directory, or from a + filesystem using any implementation of `fsspec.spec.AbstractFileSystem`. + + Args: + dataset_path (`path-like`): + Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3//my-bucket/dataset/train"`) + of the dataset directory where the dataset will be loaded from. + keep_in_memory (`bool`, defaults to `None`): + Whether to copy the dataset in-memory. If `None`, the + dataset will not be copied in-memory unless explicitly enabled by setting + `datasets.config.IN_MEMORY_MAX_SIZE` to nonzero. See more details in the + [improve performance](../cache#improve-performance) section. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Returns: + [`Dataset`] or [`DatasetDict`]: + - If `dataset_path` is a path of a dataset directory, the dataset requested. + - If `dataset_path` is a path of a dataset dict directory, a `datasets.DatasetDict` with each split. + + Example: + + ```py + >>> ds = load_from_disk("path/to/dataset/directory") + ``` + """ + fs: fsspec.AbstractFileSystem + fs, dataset_path = url_to_fs(dataset_path, **(storage_options or {})) + + dest_dataset_path = dataset_path + dataset_dict_json_path = posixpath.join(dest_dataset_path, config.DATASETDICT_JSON_FILENAME) + dataset_state_json_path = posixpath.join(dest_dataset_path, config.DATASET_STATE_JSON_FILENAME) + dataset_info_path = posixpath.join(dest_dataset_path, config.DATASET_INFO_FILENAME) + + dataset_dict_is_file = fs.isfile(dataset_dict_json_path) + dataset_info_is_file = fs.isfile(dataset_info_path) + dataset_state_is_file = fs.isfile(dataset_state_json_path) + if not dataset_info_is_file and not dataset_state_is_file: + if dataset_dict_is_file: + raise FileNotFoundError( + f"No such files: '{dataset_info_path}', nor '{dataset_state_json_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead." + ) + raise FileNotFoundError( + f"No such files: '{dataset_info_path}', nor '{dataset_state_json_path}' found. Expected to load a `Dataset` object but provided path is not a `Dataset`." + ) + if not dataset_info_is_file: + if dataset_dict_is_file: + raise FileNotFoundError( + f"No such file: '{dataset_info_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead." + ) + raise FileNotFoundError( + f"No such file: '{dataset_info_path}'. Expected to load a `Dataset` object but provided path is not a `Dataset`." + ) + if not dataset_state_is_file: + if dataset_dict_is_file: + raise FileNotFoundError( + f"No such file: '{dataset_state_json_path}' found. Expected to load a `Dataset` object, but got a `DatasetDict`. Please use either `datasets.load_from_disk` or `DatasetDict.load_from_disk` instead." + ) + raise FileNotFoundError( + f"No such file: '{dataset_state_json_path}'. Expected to load a `Dataset` object but provided path is not a `Dataset`." + ) + + # copies file from filesystem if it is remote filesystem to local filesystem and modifies dataset_path to temp directory containing local copies + if is_remote_filesystem(fs): + src_dataset_path = dest_dataset_path + dest_dataset_path = Dataset._build_local_temp_path(src_dataset_path) + fs.download(src_dataset_path, dest_dataset_path.as_posix(), recursive=True) + dataset_state_json_path = posixpath.join(dest_dataset_path, config.DATASET_STATE_JSON_FILENAME) + dataset_info_path = posixpath.join(dest_dataset_path, config.DATASET_INFO_FILENAME) + + with open(dataset_state_json_path, encoding="utf-8") as state_file: + state = json.load(state_file) + with open(dataset_info_path, encoding="utf-8") as dataset_info_file: + dataset_info = DatasetInfo.from_dict(json.load(dataset_info_file)) + + dataset_size = estimate_dataset_size( + Path(dest_dataset_path, data_file["filename"]) for data_file in state["_data_files"] + ) + keep_in_memory = keep_in_memory if keep_in_memory is not None else is_small_dataset(dataset_size) + table_cls = InMemoryTable if keep_in_memory else MemoryMappedTable + + arrow_table = concat_tables( + thread_map( + table_cls.from_file, + [posixpath.join(dest_dataset_path, data_file["filename"]) for data_file in state["_data_files"]], + tqdm_class=hf_tqdm, + desc="Loading dataset from disk", + # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached + disable=len(state["_data_files"]) <= 16 or None, + ) + ) + + split = state["_split"] + split = Split(split) if split is not None else split + + dataset = Dataset( + arrow_table=arrow_table, + info=dataset_info, + split=split, + fingerprint=state["_fingerprint"], + ) + + format = { + "type": state["_format_type"], + "format_kwargs": state["_format_kwargs"], + "columns": state["_format_columns"], + "output_all_columns": state["_output_all_columns"], + } + dataset = dataset.with_format(**format) + + return dataset + + @property + def data(self) -> Table: + """The Apache Arrow table backing the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.data + MemoryMappedTable + text: string + label: int64 + ---- + text: [["compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .","the soundtrack alone is worth the price of admission .","rodriguez does a splendid job of racial profiling hollywood style--casting excellent latin actors of all ages--a trend long overdue .","beneath the film's obvious determination to shock at any cost lies considerable skill and determination , backed by sheer nerve .","bielinsky is a filmmaker of impressive talent .","so beautifully acted and directed , it's clear that washington most certainly has a new career ahead of him if he so chooses .","a visual spectacle full of stunning images and effects .","a gentle and engrossing character study .","it's enough to watch huppert scheming , with her small , intelligent eyes as steady as any noir villain , and to enjoy the perfectly pitched web of tension that chabrol spins .","an engrossing portrait of uncompromising artists trying to create something original against the backdrop of a corporate music industry that only seems to care about the bottom line .",...,"ultimately , jane learns her place as a girl , softens up and loses some of the intensity that made her an interesting character to begin with .","ah-nuld's action hero days might be over .","it's clear why deuces wild , which was shot two years ago , has been gathering dust on mgm's shelf .","feels like nothing quite so much as a middle-aged moviemaker's attempt to surround himself with beautiful , half-naked women .","when the precise nature of matthew's predicament finally comes into sharp focus , the revelation fails to justify the build-up .","this picture is murder by numbers , and as easy to be bored by as your abc's , despite a few whopping shootouts .","hilarious musical comedy though stymied by accents thick as mud .","if you are into splatter movies , then you will probably have a reasonably good time with the salton sea .","a dull , simple-minded and stereotypical tale of drugs , death and mind-numbing indifference on the inner-city streets .","the feature-length stretch . . . strains the show's concept ."]] + label: [[1,1,1,1,1,1,1,1,1,1,...,0,0,0,0,0,0,0,0,0,0]] + ``` + """ + return self._data + + @property + def cache_files(self) -> list[dict]: + """The cache files containing the Apache Arrow table backing the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.cache_files + [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-validation.arrow'}] + ``` + """ + cache_files = list_table_cache_files(self._data) + if self._indices is not None: + cache_files += list_table_cache_files(self._indices) + return [{"filename": cache_filename} for cache_filename in cache_files] + + @property + def num_columns(self) -> int: + """Number of columns in the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.num_columns + 2 + ``` + """ + return self._data.num_columns + + @property + def num_rows(self) -> int: + """Number of rows in the dataset (same as [`Dataset.__len__`]). + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.num_rows + 1066 + ``` + """ + if self._indices is not None: + return self._indices.num_rows + return self._data.num_rows + + @property + def column_names(self) -> list[str]: + """Names of the columns in the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.column_names + ['text', 'label'] + ``` + """ + return self._data.column_names + + @property + def shape(self) -> tuple[int, int]: + """Shape of the dataset (number of columns, number of rows). + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.shape + (1066, 2) + ``` + """ + if self._indices is not None: + return (self._indices.num_rows, self._data.num_columns) + return self._data.shape + + def unique(self, column: str) -> list: + """Return a list of the unique elements in a column. + + This is implemented in the low-level backend and as such, very fast. + + Args: + column (`str`): + Column name (list all the column names with [`~datasets.Dataset.column_names`]). + + Returns: + `list`: List of unique elements in the given column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.unique('label') + [1, 0] + ``` + """ + if column not in self._data.column_names: + raise ValueError(f"Column ({column}) not in table columns ({self._data.column_names}).") + + if self._indices is not None and self._indices.num_rows != self._data.num_rows: + dataset = self.flatten_indices() + else: + dataset = self + + return dataset._data.column(column).unique().to_pylist() + + def class_encode_column(self, column: str, include_nulls: bool = False) -> "Dataset": + """Casts the given column as [`~datasets.features.ClassLabel`] and updates the table. + + Args: + column (`str`): + The name of the column to cast (list all the column names with [`~datasets.Dataset.column_names`]) + include_nulls (`bool`, defaults to `False`): + Whether to include null values in the class labels. If `True`, the null values will be encoded as the `"None"` class label. + + + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("google/boolq", split="validation") + >>> ds.features + {'answer': Value('bool'), + 'passage': Value('string'), + 'question': Value('string')} + >>> ds = ds.class_encode_column('answer') + >>> ds.features + {'answer': ClassLabel(num_classes=2, names=['False', 'True']), + 'passage': Value('string'), + 'question': Value('string')} + ``` + """ + # Sanity checks + if column not in self._data.column_names: + raise ValueError(f"Column ({column}) not in table columns ({self._data.column_names}).") + src_feat = self._info.features[column] + if not isinstance(src_feat, Value): + raise ValueError( + f"Class encoding is only supported for {Value.__name__} column, and column {column} is {type(src_feat).__name__}." + ) + + if src_feat.dtype != "string" or (include_nulls and None in self.unique(column)): + + def stringify_column(batch): + batch[column] = [ + str(sample) if include_nulls or sample is not None else None for sample in batch[column] + ] + return batch + + dset = self.map( + stringify_column, + batched=True, + desc="Stringifying the column", + ) + else: + dset = self + + # Create the new feature + class_names = sorted(str(sample) for sample in dset.unique(column) if include_nulls or sample is not None) + dst_feat = ClassLabel(names=class_names) + + def cast_to_class_labels(batch): + batch[column] = [ + dst_feat.str2int(str(sample)) if include_nulls or sample is not None else None + for sample in batch[column] + ] + return batch + + new_features = dset.features.copy() + new_features[column] = dst_feat + + dset = dset.map( + cast_to_class_labels, + batched=True, + features=new_features, + desc="Casting to class labels", + ) + + return dset + + @fingerprint_transform(inplace=False) + def flatten(self, new_fingerprint: Optional[str] = None, max_depth=16) -> "Dataset": + """Flatten the table. + Each column with a struct type is flattened into one column per struct field. + Other columns are left unchanged. + + Args: + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A copy of the dataset with flattened columns. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("rajpurkar/squad", split="train") + >>> ds.features + {'id': Value('string'), + 'title': Value('string'), + 'context': Value('string'), + 'question': Value('string'), + 'answers': {'text': List(Value('string')), + 'answer_start': List(Value('int32'))}} + >>> ds = ds.flatten() + >>> ds + Dataset({ + features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'], + num_rows: 87599 + }) + ``` + """ + dataset = copy.deepcopy(self) + for depth in range(1, max_depth): + if any(isinstance(field.type, pa.StructType) for field in dataset._data.schema): + dataset._data = dataset._data.flatten() + else: + break + dataset.info.features = self._info.features.flatten(max_depth=max_depth) + dataset.info.features = Features({col: dataset.info.features[col] for col in dataset.data.column_names}) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + logger.info(f"Flattened dataset from depth {depth} to depth {1 if depth + 1 < max_depth else 'unknown'}.") + dataset._fingerprint = new_fingerprint + return dataset + + def cast( + self, + features: Features, + batch_size: Optional[int] = 1000, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + num_proc: Optional[int] = None, + ) -> "Dataset": + """ + Cast the dataset to a new set of features. + + Args: + features ([`Features`]): + New features to cast the dataset to. + The name of the fields in the features must match the current column names. + The type of the data must also be convertible from one type to the other. + For non-trivial conversion, e.g. `str` <-> `ClassLabel` you should use [`~datasets.Dataset.map`] to update the Dataset. + batch_size (`int`, defaults to `1000`): + Number of examples per batch provided to cast. + If `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to cast. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + load_from_cache_file (`bool`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + cache_file_name (`str`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running [`~datasets.Dataset.map`]. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes for multiprocessing. By default it doesn't + use multiprocessing. + + Returns: + [`Dataset`]: A copy of the dataset with casted features. + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel, Value + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.features + {'label': ClassLabel(names=['neg', 'pos']), + 'text': Value('string')} + >>> new_features = ds.features.copy() + >>> new_features['label'] = ClassLabel(names=['bad', 'good']) + >>> new_features['text'] = Value('large_string') + >>> ds = ds.cast(new_features) + >>> ds.features + {'label': ClassLabel(names=['bad', 'good']), + 'text': Value('large_string')} + ``` + """ + if sorted(features) != sorted(self._data.column_names): + raise ValueError( + f"The columns in features ({list(features)}) must be identical " + f"as the columns in the dataset: {self._data.column_names}" + ) + + features = _fix_for_backward_compatible_features(features) + schema = features.arrow_schema + format = self.format + dataset = self.with_format("arrow") + # capture the PyArrow version here to make the lambda serializable on Windows + dataset = dataset.map( + partial(table_cast, schema=schema), + batched=True, + batch_size=batch_size, + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + cache_file_name=cache_file_name, + writer_batch_size=writer_batch_size, + num_proc=num_proc, + features=features, + desc="Casting the dataset", + ) + dataset = dataset.with_format(**format) + return dataset + + @fingerprint_transform(inplace=False) + def cast_column(self, column: str, feature: FeatureType, new_fingerprint: Optional[str] = None) -> "Dataset": + """Cast column to feature for decoding. + + Args: + column (`str`): + Column name. + feature (`FeatureType`): + Target feature. + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.features + {'label': ClassLabel(names=['neg', 'pos']), + 'text': Value('string')} + >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good'])) + >>> ds.features + {'label': ClassLabel(names=['bad', 'good']), + 'text': Value('string')} + ``` + """ + feature = _fix_for_backward_compatible_features(feature) + if hasattr(feature, "decode_example"): + dataset = copy.deepcopy(self) + dataset._info.features[column] = feature + dataset._fingerprint = new_fingerprint + dataset._data = dataset._data.cast(dataset.features.arrow_schema) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + return dataset + else: + features = self.features + features[column] = feature + return self.cast(features) + + @transmit_format + @fingerprint_transform(inplace=False) + def remove_columns(self, column_names: Union[str, list[str]], new_fingerprint: Optional[str] = None) -> "Dataset": + """ + Remove one or several column(s) in the dataset and the features associated to them. + + You can also remove a column using [`~datasets.Dataset.map`] with `remove_columns` but the present method + doesn't copy the data of the remaining columns and is thus faster. + + Args: + column_names (`Union[str, List[str]]`): + Name of the column(s) to remove. + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A copy of the dataset object without the columns to remove. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds = ds.remove_columns('label') + Dataset({ + features: ['text'], + num_rows: 1066 + }) + >>> ds = ds.remove_columns(column_names=ds.column_names) # Removing all the columns returns an empty dataset with the `num_rows` property set to 0 + Dataset({ + features: [], + num_rows: 0 + }) + ``` + """ + dataset = copy.deepcopy(self) + if isinstance(column_names, str): + column_names = [column_names] + + missing_columns = set(column_names) - set(self._data.column_names) + if missing_columns: + raise ValueError( + f"Column name {list(missing_columns)} not in the dataset. " + f"Current columns in the dataset: {dataset._data.column_names}" + ) + + for column_name in column_names: + del dataset._info.features[column_name] + + dataset._data = dataset._data.drop(column_names) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + dataset._fingerprint = new_fingerprint + return dataset + + @fingerprint_transform(inplace=False) + def rename_column( + self, original_column_name: str, new_column_name: str, new_fingerprint: Optional[str] = None + ) -> "Dataset": + """ + Rename a column in the dataset, and move the features associated to the original column under the new column + name. + + Args: + original_column_name (`str`): + Name of the column to rename. + new_column_name (`str`): + New name for the column. + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A copy of the dataset with a renamed column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds = ds.rename_column('label', 'label_new') + Dataset({ + features: ['text', 'label_new'], + num_rows: 1066 + }) + ``` + """ + dataset = copy.deepcopy(self) + if original_column_name not in dataset._data.column_names: + raise ValueError( + f"Original column name {original_column_name} not in the dataset. " + f"Current columns in the dataset: {dataset._data.column_names}" + ) + if new_column_name in dataset._data.column_names: + raise ValueError( + f"New column name {new_column_name} already in the dataset. " + f"Please choose a column name which is not already in the dataset. " + f"Current columns in the dataset: {dataset._data.column_names}" + ) + if not new_column_name: + raise ValueError("New column name is empty.") + + def rename(columns): + return [new_column_name if col == original_column_name else col for col in columns] + + new_column_names = rename(self._data.column_names) + if self._format_columns is not None: + dataset._format_columns = rename(self._format_columns) + + dataset._info.features = Features( + { + new_column_name if col == original_column_name else col: feature + for col, feature in self._info.features.items() + } + ) + + dataset._data = dataset._data.rename_columns(new_column_names) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + dataset._fingerprint = new_fingerprint + return dataset + + @fingerprint_transform(inplace=False) + def rename_columns(self, column_mapping: dict[str, str], new_fingerprint: Optional[str] = None) -> "Dataset": + """ + Rename several columns in the dataset, and move the features associated to the original columns under + the new column names. + + Args: + column_mapping (`Dict[str, str]`): + A mapping of columns to rename to their new names + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A copy of the dataset with renamed columns + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds = ds.rename_columns({'text': 'text_new', 'label': 'label_new'}) + Dataset({ + features: ['text_new', 'label_new'], + num_rows: 1066 + }) + ``` + """ + dataset = copy.deepcopy(self) + + extra_columns = set(column_mapping.keys()) - set(dataset.column_names) + if extra_columns: + raise ValueError( + f"Original column names {extra_columns} not in the dataset. " + f"Current columns in the dataset: {dataset._data.column_names}" + ) + + number_of_duplicates_in_new_columns = len(column_mapping.values()) - len(set(column_mapping.values())) + if number_of_duplicates_in_new_columns != 0: + raise ValueError( + "New column names must all be different, but this column mapping " + f"has {number_of_duplicates_in_new_columns} duplicates" + ) + + empty_new_columns = [new_col for new_col in column_mapping.values() if not new_col] + if empty_new_columns: + raise ValueError(f"New column names {empty_new_columns} are empty.") + + def rename(columns): + return [column_mapping[col] if col in column_mapping else col for col in columns] + + new_column_names = rename(self._data.column_names) + if self._format_columns is not None: + dataset._format_columns = rename(self._format_columns) + + dataset._info.features = Features( + { + column_mapping[col] if col in column_mapping else col: feature + for col, feature in (self._info.features or {}).items() + } + ) + + dataset._data = dataset._data.rename_columns(new_column_names) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + dataset._fingerprint = new_fingerprint + return dataset + + @transmit_format + @fingerprint_transform(inplace=False) + def select_columns(self, column_names: Union[str, list[str]], new_fingerprint: Optional[str] = None) -> "Dataset": + """Select one or several column(s) in the dataset and the features + associated to them. + + Args: + column_names (`Union[str, List[str]]`): + Name of the column(s) to keep. + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. If `None`, + the new fingerprint is computed using a hash of the previous + fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A copy of the dataset object which only consists of + selected columns. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds = ds.select_columns(['text']) + >>> ds + Dataset({ + features: ['text'], + num_rows: 1066 + }) + ``` + """ + if isinstance(column_names, str): + column_names = [column_names] + + missing_columns = set(column_names) - set(self._data.column_names) + if missing_columns: + raise ValueError( + f"Column name {list(missing_columns)} not in the " + "dataset. Current columns in the dataset: " + f"{self._data.column_names}." + ) + + dataset = copy.deepcopy(self) + dataset._data = dataset._data.select(column_names) + dataset._info.features = Features({col: self._info.features[col] for col in dataset._data.column_names}) + dataset._data = update_metadata_with_features(dataset._data, dataset.features) + dataset._fingerprint = new_fingerprint + return dataset + + @transmit_format + def _fast_select_column(self, column_name: str) -> "Dataset": + dataset = copy.copy(self) + dataset._data = dataset._data.select([column_name]) + dataset._info = DatasetInfo(features=Features({column_name: self._info.features[column_name]})) + return dataset + + def __len__(self): + """Number of rows in the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.__len__ + + ``` + """ + return self.num_rows + + def __iter__(self): + """Iterate through the examples. + + If a formatting is set with [`Dataset.set_format`] rows will be returned with the + selected format. + """ + if self._indices is None: + # Fast iteration + # Benchmark: https://gist.github.com/mariosasko/0248288a2e3a7556873969717c1fe52b (fast_iter_batch) + format_kwargs = self._format_kwargs if self._format_kwargs is not None else {} + formatter = get_formatter(self._format_type, features=self._info.features, **format_kwargs) + batch_size = config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER + for pa_subtable in table_iter(self.data, batch_size=batch_size): + for i in range(pa_subtable.num_rows): + pa_subtable_ex = pa_subtable.slice(i, 1) + formatted_output = format_table( + pa_subtable_ex, + 0, + formatter=formatter, + format_columns=self._format_columns, + output_all_columns=self._output_all_columns, + ) + yield formatted_output + else: + for i in range(self.num_rows): + yield self._getitem( + i, + ) + + def iter(self, batch_size: int, drop_last_batch: bool = False): + """Iterate through the batches of size `batch_size`. + + If a formatting is set with [`~datasets.Dataset.set_format`] rows will be returned with the + selected format. + + Args: + batch_size (:obj:`int`): size of each batch to yield. + drop_last_batch (:obj:`bool`, default `False`): Whether a last batch smaller than the batch_size should be + dropped + """ + if self._indices is None: + # Fast iteration + # Benchmark: https://gist.github.com/mariosasko/0248288a2e3a7556873969717c1fe52b (fast_iter_batch) + format_kwargs = self._format_kwargs if self._format_kwargs is not None else {} + formatter = get_formatter(self._format_type, features=self._info.features, **format_kwargs) + for pa_subtable in table_iter(self.data, batch_size=batch_size, drop_last_batch=drop_last_batch): + formatted_batch = format_table( + pa_subtable, + range(pa_subtable.num_rows), + formatter=formatter, + format_columns=self._format_columns, + output_all_columns=self._output_all_columns, + ) + yield formatted_batch + else: + num_rows = self.num_rows if not drop_last_batch else self.num_rows // batch_size * batch_size + for i in range(0, num_rows, batch_size): + yield self._getitem( + slice(i, i + batch_size), + ) + + def __repr__(self): + return f"Dataset({{\n features: {list(self._info.features.keys())},\n num_rows: {self.num_rows}\n}})" + + @property + def format(self): + return { + "type": self._format_type, + "format_kwargs": self._format_kwargs, + "columns": self.column_names if self._format_columns is None else self._format_columns, + "output_all_columns": self._output_all_columns, + } + + @contextlib.contextmanager + def formatted_as( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ): + """To be used in a `with` statement. Set `__getitem__` return format (type and columns). + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__`` returns python objects (default). + columns (`List[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + """ + old_format_type = self._format_type + old_format_kwargs = self._format_kwargs + old_format_columns = self._format_columns + old_output_all_columns = self._output_all_columns + try: + self.set_format(type, columns, output_all_columns, **format_kwargs) + yield + finally: + self.set_format(old_format_type, old_format_columns, old_output_all_columns, **old_format_kwargs) + + @fingerprint_transform(inplace=True) + def set_format( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ): + """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly. + The format `type` (for example "numpy") is used to format batches when using `__getitem__`. + It's also possible to use custom transforms for formatting using [`~datasets.Dataset.set_transform`]. + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__` returns python objects (default). + columns (`List[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + + It is possible to call [`~datasets.Dataset.map`] after calling `set_format`. Since `map` may add new columns, then the list of formatted columns + gets updated. In this case, if you apply `map` on a dataset to add a new column, then this column will be formatted as: + + ``` + new formatted columns = (all columns - previously unformatted columns) + ``` + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds.set_format(type='numpy', columns=['text', 'label']) + >>> ds.format + {'type': 'numpy', + 'format_kwargs': {}, + 'columns': ['text', 'label'], + 'output_all_columns': False} + ``` + """ + format_kwargs.update(format_kwargs.pop("format_kwargs", {})) # allow to use self.set_format(**self.format) + + # Check that the format_type and format_kwargs are valid and make it possible to have a Formatter + type = get_format_type_from_alias(type) + get_formatter(type, features=self._info.features, **format_kwargs) + + # Check filter column + if isinstance(columns, str): + columns = [columns] + if isinstance(columns, tuple): + columns = list(columns) + if columns is not None: + missing_columns = set(columns) - set(self._data.column_names) + if missing_columns: + raise ValueError( + f"Columns {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}" + ) + if columns is not None: + columns = columns.copy() # Ensures modifications made to the list after this call don't cause bugs + + self._format_type = type + self._format_kwargs = format_kwargs + self._format_columns = columns + self._output_all_columns = output_all_columns + logger.debug( + "Set __getitem__(key) output type to %s for %s columns " + " (when key is int or slice) and %s output other (un-formatted) columns.", + "python objects" if type is None else type, + "no" if columns is None else str(columns), + "do" if output_all_columns else "don't", + ) + + def reset_format(self): + """Reset `__getitem__` return format to python objects and all columns. + + Same as `self.set_format()` + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds.set_format(type='numpy', columns=['input_ids', 'token_type_ids', 'attention_mask', 'label']) + >>> ds.format + {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': 'numpy'} + >>> ds.reset_format() + >>> ds.format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': None} + ``` + """ + self.set_format() + + def set_transform( + self, + transform: Optional[Callable], + columns: Optional[list] = None, + output_all_columns: bool = False, + ): + """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called. + As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`]. + + Args: + transform (`Callable`, *optional*): + User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`]. + A formatting function is a callable that takes a batch (as a `dict`) as input and returns a batch. + This function is applied right before returning the objects in `__getitem__`. + columns (`List[str]`, *optional*): + Columns to format in the output. + If specified, then the input batch of the transform only contains those columns. + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + If set to True, then the other un-formatted columns are kept with the output of the transform. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') + >>> def encode(batch): + ... return tokenizer(batch['text'], padding=True, truncation=True, return_tensors='pt') + >>> ds.set_transform(encode) + >>> ds[0] + {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1]), + 'input_ids': tensor([ 101, 29353, 2135, 15102, 1996, 9428, 20868, 2890, 8663, 6895, + 20470, 2571, 3663, 2090, 4603, 3017, 3008, 1998, 2037, 24211, + 5637, 1998, 11690, 2336, 1012, 102]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0])} + ``` + """ + self.set_format("custom", columns=columns, output_all_columns=output_all_columns, transform=transform) + + def with_format( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ): + """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly. + The format `type` (for example "numpy") is used to format batches when using `__getitem__`. + + It's also possible to use custom transforms for formatting using [`~datasets.Dataset.with_transform`]. + + Contrary to [`~datasets.Dataset.set_format`], `with_format` returns a new [`Dataset`] object. + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__` returns python objects (default). + columns (`List[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds.format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': None} + >>> ds = ds.with_format("torch") + >>> ds.format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': 'torch'} + >>> ds[0] + {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .', + 'label': tensor(1), + 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617, + 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105, + 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])} + ``` + """ + dataset = copy.deepcopy(self) + dataset.set_format(type=type, columns=columns, output_all_columns=output_all_columns, **format_kwargs) + return dataset + + def with_transform( + self, + transform: Optional[Callable], + columns: Optional[list] = None, + output_all_columns: bool = False, + ): + """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called. + + As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`]. + + Contrary to [`~datasets.Dataset.set_transform`], `with_transform` returns a new [`Dataset`] object. + + Args: + transform (`Callable`, `optional`): + User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`]. + A formatting function is a callable that takes a batch (as a `dict`) as input and returns a batch. + This function is applied right before returning the objects in `__getitem__`. + columns (`List[str]`, `optional`): + Columns to format in the output. + If specified, then the input batch of the transform only contains those columns. + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + If set to `True`, then the other un-formatted columns are kept with the output of the transform. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> def encode(example): + ... return tokenizer(example["text"], padding=True, truncation=True, return_tensors='pt') + >>> ds = ds.with_transform(encode) + >>> ds[0] + {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1]), + 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617, + 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105, + 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0])} + ``` + """ + dataset = copy.deepcopy(self) + dataset.set_transform(transform=transform, columns=columns, output_all_columns=output_all_columns) + return dataset + + def _getitem(self, key: Union[int, slice, str, ListLike[int]], **kwargs) -> Union[dict, list]: + """ + Can be used to index columns (by string names) or rows (by integer, slice, or list-like of integer indices) + """ + if isinstance(key, bool): + raise TypeError("dataset index must be int, str, slice or collection of int, not bool") + format_type = kwargs["format_type"] if "format_type" in kwargs else self._format_type + format_columns = kwargs["format_columns"] if "format_columns" in kwargs else self._format_columns + output_all_columns = ( + kwargs["output_all_columns"] if "output_all_columns" in kwargs else self._output_all_columns + ) + format_kwargs = kwargs["format_kwargs"] if "format_kwargs" in kwargs else self._format_kwargs + format_kwargs = format_kwargs if format_kwargs is not None else {} + formatter = get_formatter(format_type, features=self._info.features, **format_kwargs) + pa_subtable = query_table(self._data, key, indices=self._indices) + formatted_output = format_table( + pa_subtable, key, formatter=formatter, format_columns=format_columns, output_all_columns=output_all_columns + ) + return formatted_output + + @overload + def __getitem__(self, key: Union[int, slice, Iterable[int]]) -> dict: # noqa: F811 + ... + + @overload + def __getitem__(self, key: str) -> list: # noqa: F811 + ... + + def __getitem__(self, key): # noqa: F811 + """Can be used to index columns (by string names) or rows (by integer index or iterable of indices or bools).""" + if isinstance(key, str): + if self._format_type is None or self._format_type not in ("arrow", "pandas", "polars"): + return Column(self, key) + return self._getitem(key) + + def __getitems__(self, keys: list) -> list: + """Can be used to get a batch using a list of integers indices.""" + batch = self.__getitem__(keys) + n_examples = len(batch[next(iter(batch))]) + return [{col: array[i] for col, array in batch.items()} for i in range(n_examples)] + + def cleanup_cache_files(self) -> int: + """Clean up all cache files in the dataset cache directory, excepted the currently used cache file if there is + one. + + Be careful when running this command that no other process is currently using other cache files. + + Returns: + `int`: Number of removed files. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.cleanup_cache_files() + 10 + ``` + """ + current_cache_files = [os.path.abspath(cache_file["filename"]) for cache_file in self.cache_files] + if not current_cache_files: + return 0 + cache_directory = os.path.dirname(current_cache_files[0]) + logger.info(f"Listing files in {cache_directory}") + files: list[str] = os.listdir(cache_directory) + files_to_remove = [] + for f_name in files: + full_name = os.path.abspath(os.path.join(cache_directory, f_name)) + if f_name.startswith("cache-") and f_name.endswith(".arrow"): + if full_name in current_cache_files: + logger.info(f"Keeping currently used cache file at {full_name}") + continue + files_to_remove.append(full_name) + for file_path in files_to_remove: + logger.info(f"Removing {file_path}") + os.remove(file_path) + return len(files_to_remove) + + def _get_cache_file_path(self, fingerprint): + if is_caching_enabled() and self.cache_files: + cache_file_name = "cache-" + fingerprint + ".arrow" + cache_directory = os.path.dirname(self.cache_files[0]["filename"]) + else: + cache_file_name = "cache-" + generate_random_fingerprint() + ".arrow" + cache_directory = get_temporary_cache_files_directory() + cache_file_path = os.path.join(cache_directory, cache_file_name) + return cache_file_path + + @transmit_format + def map( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + with_rank: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[Union[str, list[str]]] = None, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + features: Optional[Features] = None, + disable_nullable: bool = False, + fn_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + suffix_template: str = "_{rank:05d}_of_{num_proc:05d}", + new_fingerprint: Optional[str] = None, + desc: Optional[str] = None, + try_original_type: Optional[bool] = True, + on_mixed_types: Optional[Literal["use_json"]] = "use_json", + ) -> "Dataset": + """ + Apply a function to all the examples in the table (individually or in batches) and update the table. + If your function returns a column that already exists, then it overwrites it. + + You can specify whether the function should be batched or not with the `batched` parameter: + + - If batched is `False`, then the function takes 1 example in and should return 1 example. + An example is a dictionary, e.g. `{"text": "Hello there !"}`. + - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples. + A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`. + - If batched is `True` and `batch_size` is `n > 1`, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples. + Note that the last batch may have less than `n` examples. + A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`. + + If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simultaneous calls. + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`Callable`): Function with one of the following signatures: + + - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` and `with_rank=False` + - `function(example: Dict[str, Any], *extra_args) -> Dict[str, Any]` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False` and `with_rank=False` + - `function(batch: Dict[str, List], *extra_args) -> Dict[str, List]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + + For advanced usage, the function can also return a `pyarrow.Table`. + If the function is asynchronous, then `map` will run your function in parallel. + Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged. + If no function is provided, default to identity function: `lambda x: x`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the + signature of `function` should be `def function(example, idx[, rank]): ...`. + with_rank (`bool`, defaults to `False`): + Provide process rank to `function`. Note that in this case the + signature of `function` should be `def function(example[, idx], rank): ...`. + input_columns (`Optional[Union[str, List[str]]]`, defaults to `None`): + The columns to be passed into `function` + as positional arguments. If `None`, a `dict` mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True`. + If `batch_size <= 0` or `batch_size == None`, provide the full dataset as a single batch to `function`. + drop_last_batch (`bool`, defaults to `False`): + Whether a last batch smaller than the batch_size should be + dropped instead of being processed by the function. + remove_columns (`Optional[Union[str, List[str]]]`, defaults to `None`): + Remove a selection of columns while doing the mapping. + Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding + columns with names in `remove_columns`, these columns will be kept. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + cache_file_name (`str`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + features (`Optional[datasets.Features]`, defaults to `None`): + Use a specific Features to store the cache file + instead of the automatically generated one. + disable_nullable (`bool`, defaults to `False`): + Disallow null values in the table. + fn_kwargs (`Dict`, *optional*, defaults to `None`): + Keyword arguments to be passed to `function`. + num_proc (`int`, *optional*, defaults to `None`): + The number of processes to use for multiprocessing. + - If `None` or `0`, no multiprocessing is used and the operation runs in the main process. + - If greater than `1`, one or multiple worker processes are used to process data in parallel. + Note: The function passed to `map()` must be picklable for multiprocessing to work correctly + (i.e., prefer functions defined at the top level of a module, not inside another function or class). + suffix_template (`str`): + If `cache_file_name` is specified, then this suffix + will be added at the end of the base name of each. Defaults to `"_{rank:05d}_of_{num_proc:05d}"`. For example, if `cache_file_name` is "processed.arrow", then for + `rank=1` and `num_proc=4`, the resulting file would be `"processed_00001_of_00004.arrow"` for the default suffix. + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + desc (`str`, *optional*, defaults to `None`): + Meaningful description to be displayed alongside with the progress bar while mapping examples. + try_original_type (`Optional[bool]`, defaults to `True`): + Try to keep the types of the original columns (e.g. int32 -> int32). + Set to False if you want to always infer new types. + on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`): + If "use_json", use the Json() type for mixed-types fields, + i.e. unstructured fields that contain data without a predefined schema. + In this case, a field with mixed type is set to Json(). + + This allow loading lists with a mix of strings/integers/floats + for example, or dictionaries with arbitrary value types. + + + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> def add_prefix(example): + ... example["text"] = "Review: " + example["text"] + ... return example + >>> ds = ds.map(add_prefix) + >>> ds[0:3]["text"] + ['Review: compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .', + 'Review: the soundtrack alone is worth the price of admission .', + 'Review: rodriguez does a splendid job of racial profiling hollywood style--casting excellent latin actors of all ages--a trend long overdue .'] + + # process a batch of examples + >>> ds = ds.map(lambda example: tokenizer(example["text"]), batched=True) + # set number of processors + >>> ds = ds.map(add_prefix, num_proc=4) + ``` + """ + if keep_in_memory and cache_file_name is not None: + raise ValueError("Please use either `keep_in_memory` or `cache_file_name` but not both.") + + if num_proc == 0: + num_proc = None + elif num_proc is not None and num_proc < 0: + raise ValueError("num_proc must be >= 0 or None.") + + string_formatter = string.Formatter() + fields = {field_name for _, field_name, _, _ in string_formatter.parse(suffix_template) if field_name} + if fields != {"rank", "num_proc"}: + raise ValueError(f"suffix_template must contain exactly the fields 'rank' and 'num_proc', got: {fields}") + + # If the array is empty we do nothing (but we make sure to handle an empty indices mapping and remove the requested columns anyway) + if len(self) == 0: + if self._indices is not None: # empty indices mapping + self = Dataset( + self.data.slice(0, 0), + info=self.info.copy(), + split=self.split, + fingerprint=new_fingerprint, + ) + if remove_columns: + return self.remove_columns(remove_columns) + else: + return self + + if function is None: + function = lambda x: x # noqa: E731 + + if isinstance(input_columns, str): + input_columns = [input_columns] + + if input_columns is not None: + missing_columns = set(input_columns) - set(self._data.column_names) + if missing_columns: + raise ValueError( + f"Input column {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}" + ) + + if isinstance(remove_columns, str): + remove_columns = [remove_columns] + + if remove_columns is not None: + missing_columns = set(remove_columns) - set(self._data.column_names) + if missing_columns: + raise ValueError( + f"Column to remove {list(missing_columns)} not in the dataset. Current columns in the dataset: {self._data.column_names}" + ) + + load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled() + + if fn_kwargs is None: + fn_kwargs = {} + + if features is not None: + features = _fix_for_backward_compatible_features(features) + + if num_proc is not None and num_proc > len(self): + num_proc = len(self) + logger.warning( + f"num_proc must be <= {len(self)}. Reducing num_proc to {num_proc} for dataset of size {len(self)}." + ) + + dataset_kwargs = { + "shard": self, + "function": function, + "with_indices": with_indices, + "with_rank": with_rank, + "input_columns": input_columns, + "batched": batched, + "batch_size": batch_size, + "drop_last_batch": drop_last_batch, + "remove_columns": remove_columns, + "keep_in_memory": keep_in_memory, + "writer_batch_size": writer_batch_size, + "features": features, + "disable_nullable": disable_nullable, + "fn_kwargs": fn_kwargs, + "try_original_type": try_original_type, + "on_mixed_types": on_mixed_types, + } + + if new_fingerprint is None: + # we create a unique hash from the function, + # current dataset file and the mapping args + transform = format_transform_for_fingerprint(Dataset._map_single) + kwargs_for_fingerprint = format_kwargs_for_fingerprint(Dataset._map_single, (), dataset_kwargs) + kwargs_for_fingerprint["fingerprint_name"] = "new_fingerprint" + new_fingerprint = update_fingerprint(self._fingerprint, transform, kwargs_for_fingerprint) + else: + validate_fingerprint(new_fingerprint) + dataset_kwargs["new_fingerprint"] = new_fingerprint + + if self.cache_files: + if cache_file_name is None: + cache_file_name = self._get_cache_file_path(new_fingerprint) + dataset_kwargs["cache_file_name"] = cache_file_name + + if cache_file_name is not None: + cache_file_prefix, cache_file_ext = os.path.splitext(cache_file_name) + if not cache_file_ext: + raise ValueError(f"Expected cache_file_name to have an extension, but got: {cache_file_name}") + else: + cache_file_prefix = cache_file_ext = None + + def load_processed_shard_from_cache(shard_kwargs: dict[str, Any]) -> Dataset: + """Load a processed shard from cache if it exists, otherwise throw an error.""" + shard = shard_kwargs["shard"] + # Check if we've already cached this computation (indexed by a hash) + if shard_kwargs["cache_file_name"] is not None: + if os.path.exists(shard_kwargs["cache_file_name"]) and load_from_cache_file: + info = shard.info.copy() + info.features = features + return Dataset.from_file(shard_kwargs["cache_file_name"], info=info, split=shard.split) + raise NonExistentDatasetError + + existing_cache_file_map: dict[int, list[str]] = defaultdict(list) + if cache_file_name is not None: + if os.path.exists(cache_file_name): + existing_cache_file_map[1] = [cache_file_name] + + assert cache_file_prefix is not None and cache_file_ext is not None + cache_file_with_suffix_pattern = cache_file_prefix + suffix_template + cache_file_ext + + for cache_file in glob.iglob(f"{cache_file_prefix}*{cache_file_ext}"): + suffix_variable_map = string_to_dict( + Path(cache_file).as_posix(), Path(cache_file_with_suffix_pattern).as_posix() + ) + if suffix_variable_map is not None: + file_num_proc = int(suffix_variable_map["num_proc"]) + existing_cache_file_map[file_num_proc].append(cache_file) + + num_shards = num_proc or 1 + if existing_cache_file_map: + # to avoid remapping when a different num_proc is given than when originally cached, update num_shards to + # what was used originally + + def select_existing_cache_files(mapped_num_proc: int) -> tuple[float, ...]: + percent_missing = (mapped_num_proc - len(existing_cache_file_map[mapped_num_proc])) / mapped_num_proc + num_shards_diff = abs(mapped_num_proc - num_shards) + return ( + percent_missing, # choose the most complete set of existing cache files + num_shards_diff, # then choose the mapped_num_proc closest to the current num_proc + mapped_num_proc, # finally, choose whichever mapped_num_proc is lower + ) + + num_shards = min(existing_cache_file_map, key=select_existing_cache_files) + + existing_cache_files = existing_cache_file_map[num_shards] + + def format_cache_file_name( + cache_file_name: Optional[str], + rank: Union[int, Literal["*"]], # noqa: F722 + ) -> Optional[str]: + if not cache_file_name: + return cache_file_name + + assert cache_file_prefix is not None and cache_file_ext is not None + + if isinstance(rank, int): + cache_file_name = ( + cache_file_prefix + suffix_template.format(rank=rank, num_proc=num_shards) + cache_file_ext + ) + if not os.path.exists(cache_file_name): + process_name = ( + "Main process" if num_proc is None or num_proc == 1 else f"Process #{rank % num_shards + 1}" + ) + logger.info(f"{process_name} will write at {cache_file_name}") + else: + # TODO: this assumes the format_spec of rank in suffix_template + cache_file_name = ( + cache_file_prefix + + suffix_template.replace("{rank:05d}", "{rank}").format(rank=rank, num_proc=num_shards) + + cache_file_ext + ) + return cache_file_name + + def format_new_fingerprint(new_fingerprint: str, rank: int) -> str: + new_fingerprint = new_fingerprint + suffix_template.format(rank=rank, num_proc=num_shards) + validate_fingerprint(new_fingerprint) + return new_fingerprint + + if num_proc is not None and num_proc >= 1: + prev_env = deepcopy(os.environ) + # check if parallelism if off + # from https://github.com/huggingface/tokenizers/blob/bb668bc439dc34389b71dbb8ce0c597f15707b53/tokenizers/src/utils/parallelism.rs#L22 + if prev_env.get("TOKENIZERS_PARALLELISM", "false").lower() not in ( + "", + "off", + "false", + "f", + "no", + "n", + "0", + ): + logger.warning("Setting TOKENIZERS_PARALLELISM=false for forked processes.") + os.environ["TOKENIZERS_PARALLELISM"] = "false" + else: + prev_env = os.environ + + kwargs_per_job: list[Optional[dict[str, Any]]] + if num_shards == 1: + shards = [self] + kwargs_per_job = [dataset_kwargs] + else: + shards = [ + self.shard(num_shards=num_shards, index=rank, contiguous=True, keep_in_memory=keep_in_memory) + for rank in range(num_shards) + ] + kwargs_per_job = [ + { + **dataset_kwargs, + "shard": shards[rank], + "cache_file_name": format_cache_file_name(cache_file_name, rank), + "rank": rank, + "offset": sum(len(s) for s in shards[:rank]), + "new_fingerprint": format_new_fingerprint(new_fingerprint, rank), + } + for rank in range(num_shards) + ] + + transformed_shards: list[Optional[Dataset]] = [None] * num_shards + for rank in range(num_shards): + try: + job_kwargs = kwargs_per_job[rank] + assert job_kwargs is not None + transformed_shards[rank] = load_processed_shard_from_cache(job_kwargs) + kwargs_per_job[rank] = None + except NonExistentDatasetError: + pass + + if unprocessed_kwargs_per_job := [kwargs for kwargs in kwargs_per_job if kwargs is not None]: + if len(unprocessed_kwargs_per_job) != num_shards: + logger.info( + f"Reprocessing {len(unprocessed_kwargs_per_job)}/{num_shards} shards because some of them were " + "missing from the cache." + ) + + pbar_total = len(self) + pbar_initial = len(existing_cache_files) * pbar_total // num_shards + if batched and drop_last_batch: + batch_size = batch_size or 1 + pbar_initial = pbar_initial // num_shards // batch_size * num_shards * batch_size + pbar_total = pbar_total // num_shards // batch_size * num_shards * batch_size + + with hf_tqdm( + unit=" examples", + initial=pbar_initial, + total=pbar_total, + desc=(desc or "Map") + (f" (num_proc={num_proc})" if num_proc is not None and num_proc >= 1 else ""), + ) as pbar: + shards_done = 0 + + def check_if_shard_done(rank: Optional[int], done: bool, content: Union[Dataset, int]) -> None: + nonlocal shards_done + if done: + shards_done += 1 + logger.debug(f"Finished processing shard number {rank} of {num_shards}.") + assert isinstance(content, Dataset) + transformed_shards[rank or 0] = content + else: + assert isinstance(content, int) + pbar.update(content) + + if num_proc is not None and num_proc >= 1: + with mp.Pool(num_proc) as pool: + os.environ = prev_env + logger.info(f"Spawning {num_proc} processes") + + for rank, done, content in iflatmap_unordered( + pool, Dataset._map_single, kwargs_iterable=unprocessed_kwargs_per_job + ): + check_if_shard_done(rank, done, content) + + pool.close() + pool.join() + else: + for unprocessed_kwargs in unprocessed_kwargs_per_job: + for rank, done, content in Dataset._map_single(**unprocessed_kwargs): + check_if_shard_done(rank, done, content) + + # Avoids PermissionError on Windows (the error: https://github.com/huggingface/datasets/actions/runs/4026734820/jobs/6921621805) + for job_kwargs in unprocessed_kwargs_per_job: + if "shard" in job_kwargs: + del job_kwargs["shard"] + else: + logger.info(f"Loading cached processed dataset at {format_cache_file_name(cache_file_name, '*')}") + + all_transformed_shards = [shard for shard in transformed_shards if shard is not None] + if len(transformed_shards) != len(all_transformed_shards): + raise ValueError( + f"Failed to retrieve results from map: result list {transformed_shards} still contains None - " + "at least one worker failed to return its results" + ) + + if num_shards == 1: + result = all_transformed_shards[0] + else: + logger.info(f"Concatenating {num_shards} shards") + result = _concatenate_map_style_datasets(all_transformed_shards) + + # update fingerprint if the dataset changed + result._fingerprint = ( + new_fingerprint + if any( + transformed_shard._fingerprint != shard._fingerprint + for transformed_shard, shard in zip(all_transformed_shards, shards) + ) + else self._fingerprint + ) + + return result + + @staticmethod + def _map_single( + shard: "Dataset", + function: Optional[Callable] = None, + with_indices: bool = False, + with_rank: bool = False, + input_columns: Optional[list[str]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[list[str]] = None, + keep_in_memory: bool = False, + cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + features: Optional[Features] = None, + disable_nullable: bool = False, + fn_kwargs: Optional[dict] = None, + new_fingerprint: Optional[str] = None, + rank: Optional[int] = None, + offset: int = 0, + try_original_type: Optional[bool] = True, + on_mixed_types: Optional[Literal["use_json"]] = "use_json", + ) -> Iterable[tuple[Optional[int], bool, Union[int, "Dataset"]]]: + """Apply a function to all the elements in the table (individually or in batches) + and update the table (if function does update examples). + + Args: + shard (`datasets.Dataset`): Dataset to map the transform on. + function (`Callable`): with one of the following signature: + - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` and `with_rank=False` + - `function(example: Dict[str, Any], *extra_args) -> Dict[str, Any]` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False` and `with_rank=False` + - `function(batch: Dict[str, List], *extra_args) -> Dict[str, List]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + + For advanced usage, the function can also return a `pyarrow.Table`. + Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged. + If no function is provided, default to identity function: lambda x: x + with_indices (`bool`, defaults to `False`): Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`. + with_rank (`bool`, default `False`): Provide process rank to `function`. Note that in this case the signature of `function` should be `def function(example[, idx], rank): ...`. + input_columns (`Optional[List[str]]`, defaults to `None`): The columns to be passed into `function` as + positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): Provide batch of examples to `function` + batch_size (`int`, optional, defaults to `1000`): Number of examples per batch provided to `function` if `batched=True` + `batch_size <= 0` or `batch_size == None`: Provide the full dataset as a single batch to `function` + drop_last_batch (`bool`, default: `False`): Whether a last batch smaller than the batch_size should be + dropped instead of being processed by the function. + remove_columns (`Optional[List[str]]`, defaults to `None`): Remove a selection of columns while doing the mapping. + Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding + columns with names in `remove_columns`, these columns will be kept. + keep_in_memory (`bool`, defaults to `False`): Keep the dataset in memory instead of writing it to a cache file. + cache_file_name (`str`, optional, defaults to `None`): Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + writer_batch_size (`int`, default `1000`): Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `.map()`. + features (`Optional[datasets.Features]`, defaults to `None`): Use a specific Features to store the cache file + instead of the automatically generated one. + disable_nullable (`bool`, defaults to `False`): Disallow null values in the table. + fn_kwargs (`Dict`, optional, defaults to `None`): Keyword arguments to be passed to `function` + new_fingerprint (`str`, optional, defaults to `None`): the new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + rank: (`int`, optional, defaults to `None`): If specified, this is the process rank when doing multiprocessing + offset: (`int`, defaults to 0): If specified, this is an offset applied to the indices passed to `function` if `with_indices=True`. + try_original_type: (`Optional[bool]`, defaults to `True`): + Try to keep the types of the original columns (e.g. int32 -> int32). + Set to False if you want to always infer new types. + on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`): + If "use_json", use the Json() type for mixed-types fields, + i.e. unstructured fields that contain data without a predefined schema. + In this case, a field with mixed type is set to Json(). + + This allow loading lists with a mix of strings/integers/floats + for example, or dictionaries with arbitrary value types. + + + """ + if fn_kwargs is None: + fn_kwargs = {} + + # If we do batch computation but no batch size is provided, default to the full dataset + if batched and (batch_size is None or batch_size <= 0): + batch_size = shard.num_rows + + # We set this variable to True after processing the first example/batch in + # `apply_function_on_filtered_inputs` if the map function returns a dict. + # If set to False, no new arrow table will be created + + update_data = None + + format_kwargs = shard._format_kwargs.copy() + # Lazy formatting is only available for the default format (None/python) + if not input_columns and shard._format_type is None: + format_kwargs["lazy"] = True + input_formatter = get_formatter( + shard._format_type, + features=shard.features, + **format_kwargs, + ) + + check_same_num_examples = batched and len(shard.list_indexes()) > 0 + + def validate_function_output(processed_inputs): + """Validate output of the map function.""" + allowed_processed_inputs_types = (Mapping, pa.Table, pd.DataFrame) + if config.POLARS_AVAILABLE and "polars" in sys.modules: + import polars as pl + + allowed_processed_inputs_types += (pl.DataFrame,) + if processed_inputs is not None and not isinstance(processed_inputs, allowed_processed_inputs_types): + raise TypeError( + f"Provided `function` which is applied to all elements of table returns a variable of type {type(processed_inputs)}. Make sure provided `function` returns a variable of type `dict` (or a pyarrow table) to update the dataset or `None` if you are only interested in side effects." + ) + if batched and isinstance(processed_inputs, Mapping): + allowed_batch_return_types = (list, np.ndarray, pd.Series) + if config.POLARS_AVAILABLE and "polars" in sys.modules: + import polars as pl + + allowed_batch_return_types += (pl.Series, pl.DataFrame) + if config.TF_AVAILABLE and "tensorflow" in sys.modules: + import tensorflow as tf + + allowed_batch_return_types += (tf.Tensor,) + if config.TORCH_AVAILABLE and "torch" in sys.modules: + import torch + + allowed_batch_return_types += (torch.Tensor,) + if config.JAX_AVAILABLE and "jax" in sys.modules: + import jax.numpy as jnp + + allowed_batch_return_types += (jnp.ndarray,) + all_dict_values_are_lists = all( + isinstance(value, allowed_batch_return_types) for value in processed_inputs.values() + ) + if all_dict_values_are_lists is False: + raise TypeError( + f"Provided `function` which is applied to all elements of table returns a `dict` of types {[type(x) for x in processed_inputs.values()]}. When using `batched=True`, make sure provided `function` returns a `dict` of types like `{allowed_batch_return_types}`." + ) + + def prepare_inputs(pa_inputs, indices, offset=0): + """Utility to apply the function on a selection of columns.""" + inputs = format_table( + pa_inputs, + 0 if not batched else range(pa_inputs.num_rows), + format_columns=input_columns, + formatter=input_formatter, + ) + fn_args = [inputs] if input_columns is None else [inputs[col] for col in input_columns] + if offset == 0: + effective_indices = indices + else: + effective_indices = [i + offset for i in indices] if isinstance(indices, list) else indices + offset + additional_args = () + if with_indices: + additional_args += (effective_indices,) + if with_rank: + additional_args += (rank,) + return inputs, fn_args, additional_args, fn_kwargs + + def prepare_outputs(pa_inputs, inputs, processed_inputs): + nonlocal update_data + if not (update_data := (processed_inputs is not None)): + return None + if isinstance(processed_inputs, LazyDict): + processed_inputs = { + k: v for k, v in processed_inputs.data.items() if k not in processed_inputs.keys_to_format + } + returned_lazy_dict = True + else: + returned_lazy_dict = False + validate_function_output(processed_inputs) + if shard._format_type or input_columns: + # TODO(QL, MS): ideally the behavior should be the same even if the dataset is formatted (may require major release) + inputs_to_merge = dict(zip(pa_inputs.column_names, pa_inputs.itercolumns())) + elif isinstance(inputs, LazyDict): + inputs_to_merge = { + k: (v if k not in inputs.keys_to_format else pa_inputs[k]) for k, v in inputs.data.items() + } + else: + inputs_to_merge = inputs + if remove_columns is not None: + for column in remove_columns: + # `function` can modify input in-place causing column to be already removed. + if column in inputs_to_merge: + inputs_to_merge.pop(column) + if returned_lazy_dict and column in processed_inputs: + processed_inputs.pop(column) + if check_same_num_examples: + input_num_examples = len(pa_inputs) + processed_inputs_num_examples = len(processed_inputs[next(iter(processed_inputs.keys()))]) + if input_num_examples != processed_inputs_num_examples: + raise DatasetTransformationNotAllowedError( + "Using `.map` in batched mode on a dataset with attached indexes is allowed only if it doesn't create or remove existing examples. You can first run `.drop_index() to remove your index and then re-add it." + ) from None + if isinstance(inputs, Mapping) and isinstance(processed_inputs, Mapping): + # The .map() transform *updates* the dataset: + # the output dictionary contains both the input data and the output data. + # The output dictionary may contain Arrow values from `inputs_to_merge` so that we can re-write them efficiently. + return {**inputs_to_merge, **processed_inputs} + else: + return processed_inputs + + def apply_function(pa_inputs, indices, offset=0): + """Utility to apply the function on a selection of columns.""" + inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(pa_inputs, indices, offset=offset) + processed_inputs = function(*fn_args, *additional_args, **fn_kwargs) + return prepare_outputs(pa_inputs, inputs, processed_inputs) + + async def async_apply_function(pa_inputs, indices, offset=0): + """Utility to apply the function on a selection of columns. Same code but async""" + inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(pa_inputs, indices, offset=offset) + processed_inputs = await function(*fn_args, *additional_args, **fn_kwargs) + return prepare_outputs(pa_inputs, inputs, processed_inputs) + + def init_buffer_and_writer(): + # Prepare output buffer and batched writer in memory or on file if we update the table + writer_features = features + if writer_features is None: + writer_features = shard.features + update_features = True + else: + update_features = False + if keep_in_memory or cache_file_name is None: + buf_writer = pa.BufferOutputStream() + tmp_file = None + writer = ArrowWriter( + features=writer_features, + stream=buf_writer, + writer_batch_size=writer_batch_size, + update_features=update_features, + fingerprint=new_fingerprint, + disable_nullable=disable_nullable, + on_mixed_types=on_mixed_types, + ) + else: + buf_writer = None + logger.info(f"Caching processed dataset at {cache_file_name}") + cache_dir = os.path.dirname(cache_file_name) + os.makedirs(cache_dir, exist_ok=True) + tmp_file = tempfile.NamedTemporaryFile("wb", dir=cache_dir, delete=False) + writer = ArrowWriter( + features=writer_features, + path=tmp_file.name, + writer_batch_size=writer_batch_size, + update_features=update_features, + fingerprint=new_fingerprint, + disable_nullable=disable_nullable, + on_mixed_types=on_mixed_types, + ) + return buf_writer, writer, tmp_file + + tasks: list[asyncio.Task] = [] + if inspect.iscoroutinefunction(function): + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + else: + loop = None + + def iter_outputs(shard_iterable): + nonlocal tasks, loop + if inspect.iscoroutinefunction(function): + indices: Union[list[int], list[list[int]]] = [] + for i, example in shard_iterable: + indices.append(i) + tasks.append(loop.create_task(async_apply_function(example, i, offset=offset))) + # keep the total active tasks under a certain number + if len(tasks) >= config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL: + done, pending = loop.run_until_complete( + asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + ) + while tasks and len(pending) >= config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL: + done, pending = loop.run_until_complete( + asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + ) + # yield finished tasks + while tasks and tasks[0].done(): + yield indices.pop(0), tasks.pop(0).result() + while tasks: + yield indices[0], loop.run_until_complete(tasks[0]) + indices.pop(0), tasks.pop(0) + else: + for i, example in shard_iterable: + yield i, apply_function(example, i, offset=offset) + + num_examples_progress_update = 0 + # If `update_data` is True after processing the first example/batch, initialize these resources with `init_buffer_and_writer` + buf_writer, writer, tmp_file = None, None, None + + # Optionally initialize the writer as a context manager + with contextlib.ExitStack() as stack: + try: + arrow_formatted_shard = shard.with_format("arrow") + + # Loop over single examples or batches and write to buffer/file if examples are to be updated + if not batched: + shard_iterable = enumerate(arrow_formatted_shard) + else: + num_rows = len(shard) if not drop_last_batch else len(shard) // batch_size * batch_size + shard_iterable = zip( + (list(range(i, min(i + batch_size, num_rows))) for i in range(0, num_rows, batch_size)), + arrow_formatted_shard.iter(batch_size, drop_last_batch=drop_last_batch), + ) + if not batched: + _time = time.time() + for i, example in iter_outputs(shard_iterable): + if update_data: + if i == 0: + buf_writer, writer, tmp_file = init_buffer_and_writer() + stack.enter_context(writer) + if isinstance(example, pa.Table): + writer.write_row(example) + elif isinstance(example, pd.DataFrame): + writer.write_row(pa.Table.from_pandas(example)) + elif config.POLARS_AVAILABLE and "polars" in sys.modules: + import polars as pl + + if isinstance(example, pl.DataFrame): + writer.write_row(example.to_arrow()) + else: + writer.write(example) + else: + writer.write(example) + num_examples_progress_update += 1 + if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL: + _time = time.time() + yield rank, False, num_examples_progress_update + num_examples_progress_update = 0 + else: + _time = time.time() + for i, batch in iter_outputs(shard_iterable): + num_examples_in_batch = len(i) + if update_data: + if i and i[0] == 0: + buf_writer, writer, tmp_file = init_buffer_and_writer() + stack.enter_context(writer) + if isinstance(batch, pa.Table): + writer.write_table(batch) + elif isinstance(batch, pd.DataFrame): + writer.write_table(pa.Table.from_pandas(batch)) + elif config.POLARS_AVAILABLE and "polars" in sys.modules: + import polars as pl + + if isinstance(batch, pl.DataFrame): + writer.write_table(batch.to_arrow()) + else: + writer.write_batch(batch, try_original_type=try_original_type) + else: + writer.write_batch(batch, try_original_type=try_original_type) + num_examples_progress_update += num_examples_in_batch + if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL: + _time = time.time() + yield rank, False, num_examples_progress_update + num_examples_progress_update = 0 + if update_data and writer is not None: + writer.finalize() # close_stream=bool(buf_writer is None)) # We only close if we are writing in a file + except (Exception, KeyboardInterrupt): + yield rank, False, num_examples_progress_update + if update_data: + if writer is not None: + writer.finalize() + if tmp_file is not None: + tmp_file.close() + if os.path.exists(tmp_file.name): + os.remove(tmp_file.name) + if loop: + logger.debug(f"Canceling {len(tasks)} async tasks.") + for task in tasks: + task.cancel(msg="KeyboardInterrupt") + try: + loop.run_until_complete(asyncio.gather(*tasks)) + except (asyncio.CancelledError, ValueError): + logger.debug("Tasks canceled.") + raise + + yield rank, False, num_examples_progress_update + if update_data and tmp_file is not None: + tmp_file.close() + shutil.move(tmp_file.name, cache_file_name) + umask = os.umask(0o666) + os.umask(umask) + os.chmod(cache_file_name, 0o666 & ~umask) + + if update_data: + # Create new Dataset from buffer or file + info = shard.info.copy() + info.features = writer._features + if buf_writer is None: + yield rank, True, Dataset.from_file(cache_file_name, info=info, split=shard.split) + else: + yield rank, True, Dataset.from_buffer(buf_writer.getvalue(), info=info, split=shard.split) + else: + yield rank, True, shard + + @transmit_format + @fingerprint_transform(inplace=False) + def batch( + self, + batch_size: int, + drop_last_batch: bool = False, + num_proc: Optional[int] = None, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """ + Group samples from the dataset into batches. + + Args: + batch_size (`int`): + The number of samples in each batch. + drop_last_batch (`bool`, defaults to `False`): + Whether to drop the last incomplete batch. + num_proc (`int`, *optional*, defaults to `None`): + Max number of processes when generating cache. Already cached shards are loaded sequentially. + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Returns: + [`Dataset`]: A new Dataset where each item is a batch of multiple samples from the original dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> batched_ds = ds.batch(batch_size=4) + >>> batched_ds[0] + {'text': ['compassionately explores the seemingly irreconcilable situation...', ...], # 4 items + 'label': [1, 1, 1, 1]} + ``` + """ + + def batch_fn(example): + return {k: [v] for k, v in example.items()} + + if self._format_type in ("arrow", "pandas", "polars"): + return self.with_format("arrow").map( + _batch_arrow_table, + batched=True, + batch_size=batch_size, + drop_last_batch=drop_last_batch, + num_proc=num_proc, + new_fingerprint=new_fingerprint, + desc="Batching examples", + ) + + return self.map( + batch_fn, + batched=True, + batch_size=batch_size, + drop_last_batch=drop_last_batch, + num_proc=num_proc, + new_fingerprint=new_fingerprint, + desc="Batching examples", + ) + + @transmit_format + @fingerprint_transform( + inplace=False, ignore_kwargs=["load_from_cache_file", "cache_file_name", "desc"], version="2.0.1" + ) + def filter( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + with_rank: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + fn_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + suffix_template: str = "_{rank:05d}_of_{num_proc:05d}", + new_fingerprint: Optional[str] = None, + desc: Optional[str] = None, + ) -> "Dataset": + """Apply a filter function to all the elements in the table in batches + and update the table so that the dataset only includes examples according to the filter function. + + If the function is asynchronous, then `filter` will run your function in parallel, with up to one thousand simultaneous calls (configurable). + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`Callable`): Callable with one of the following signatures: + + - `function(example: Dict[str, Any]) -> bool` if `batched=False` and `with_indices=False` and `with_rank=False` + - `function(example: Dict[str, Any], *extra_args) -> bool` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + - `function(batch: Dict[str, List]) -> List[bool]` if `batched=True` and `with_indices=False` and `with_rank=False` + - `function(batch: Dict[str, List], *extra_args) -> List[bool]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + + If the function is asynchronous, then `filter` will run your function in parallel. + If no function is provided, defaults to an always `True` function: `lambda x: True`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the + signature of `function` should be `def function(example, idx[, rank]): ...`. + with_rank (`bool`, defaults to `False`): + Provide process rank to `function`. Note that in this case the + signature of `function` should be `def function(example[, idx], rank): ...`. + input_columns (`str` or `List[str]`, *optional*): + The columns to be passed into `function` as + positional arguments. If `None`, a `dict` mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if + `batched = True`. If `batched = False`, one example per batch is passed to `function`. + If `batch_size <= 0` or `batch_size == None`, provide the full dataset as a single batch to `function`. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + cache_file_name (`str`, *optional*): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + fn_kwargs (`dict`, *optional*): + Keyword arguments to be passed to `function`. + num_proc (`int`, *optional*, defaults to `None`): + The number of processes to use for multiprocessing. + - If `None` or `0`, no multiprocessing is used and the operation runs in the main process. + - If greater than `1`, one or multiple worker processes are used to process data in parallel. + Note: The function passed to `map()` must be picklable for multiprocessing to work correctly + (i.e., prefer functions defined at the top level of a module, not inside another function or class). + suffix_template (`str`): + If `cache_file_name` is specified, then this suffix will be added at the end of the base name of each. + For example, if `cache_file_name` is `"processed.arrow"`, then for `rank = 1` and `num_proc = 4`, + the resulting file would be `"processed_00001_of_00004.arrow"` for the default suffix (default + `_{rank:05d}_of_{num_proc:05d}`). + new_fingerprint (`str`, *optional*): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + desc (`str`, *optional*, defaults to `None`): + Meaningful description to be displayed alongside with the progress bar while filtering examples. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds = ds.filter(lambda x: x["label"] == 1) + >>> ds + Dataset({ + features: ['text', 'label'], + num_rows: 533 + }) + ``` + + """ + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.filter` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it.`" + ) + + if function is None: + function = lambda x: True # noqa: E731 + + if len(self) == 0: + return self + + # We generally batch the underlying map() to get faster throughput, + # but in case of async we force batch_size=1 to enable parallelism + if inspect.iscoroutinefunction(function) and not batched: + batch_size = 1 + + indices = self.map( + function=partial( + async_get_indices_from_mask_function + if inspect.iscoroutinefunction(function) + else get_indices_from_mask_function, + function, + batched, + with_indices, + with_rank, + input_columns, + self._indices, + ), + with_indices=True, + with_rank=True, + features=Features({"indices": Value("uint64")}), + batched=True, + batch_size=batch_size, + remove_columns=self.column_names, + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + cache_file_name=cache_file_name, + writer_batch_size=writer_batch_size, + fn_kwargs=fn_kwargs, + num_proc=num_proc, + suffix_template=suffix_template, + new_fingerprint=new_fingerprint, + input_columns=input_columns, + desc=desc or "Filter", + ) + new_dataset = copy.deepcopy(self) + new_dataset._indices = indices.data + new_dataset._fingerprint = new_fingerprint + return new_dataset + + @transmit_format + @fingerprint_transform(inplace=False, ignore_kwargs=["cache_file_name"]) + def flatten_indices( + self, + keep_in_memory: bool = False, + cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + features: Optional[Features] = None, + disable_nullable: bool = False, + num_proc: Optional[int] = None, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create and cache a new Dataset by flattening the indices mapping. + + Args: + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + cache_file_name (`str`, *optional*, default `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + features (`Optional[datasets.Features]`, defaults to `None`): + Use a specific [`Features`] to store the cache file + instead of the automatically generated one. + disable_nullable (`bool`, defaults to `False`): + Allow null values in the table. + num_proc (`int`, optional, default `None`): + Max number of processes when generating cache. Already cached shards are loaded sequentially + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + """ + + return self.map( + batched=True, # for speed + keep_in_memory=keep_in_memory, + cache_file_name=cache_file_name, + writer_batch_size=writer_batch_size, + features=features, + disable_nullable=disable_nullable, + new_fingerprint=new_fingerprint, + desc="Flattening the indices", + num_proc=num_proc, + ) + + def _new_dataset_with_indices( + self, + indices_cache_file_name: Optional[str] = None, + indices_buffer: Optional[pa.Buffer] = None, + fingerprint: Optional[str] = None, + ) -> "Dataset": + """Return a new Dataset obtained by adding indices (provided in indices_cache_file_name or in a buffer) to the + current Dataset. + """ + + if indices_cache_file_name is None and indices_buffer is None: + raise ValueError("At least one of indices_cache_file_name or indices_buffer must be provided.") + + if fingerprint is None: + raise ValueError("please specify a fingerprint for the dataset with indices") + + if indices_cache_file_name is not None: + indices_table = MemoryMappedTable.from_file(indices_cache_file_name) + else: + indices_table = InMemoryTable.from_buffer(indices_buffer) + + # Return new Dataset object + # don't forget to copy the objects + return Dataset( + self._data, + info=self.info.copy(), + split=self.split, + indices_table=indices_table, + fingerprint=fingerprint, + ) + + @transmit_format + @fingerprint_transform(inplace=False, ignore_kwargs=["indices_cache_file_name"]) + def select( + self, + indices: Iterable, + keep_in_memory: bool = False, + indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create a new dataset with rows selected following the list/array of indices. + + Args: + indices (`range`, `list`, `iterable`, `ndarray` or `Series`): + Range, list or 1D-array of integer indices for indexing. + If the indices correspond to a contiguous range, the Arrow table is simply sliced. + However passing a list of indices that are not contiguous creates indices mapping, which is much less efficient, + but still faster than recreating an Arrow table made of the requested rows. + keep_in_memory (`bool`, defaults to `False`): + Keep the indices mapping in memory instead of writing it to a cache file. + indices_cache_file_name (`str`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + indices mapping instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds = ds.select(range(4)) + >>> ds + Dataset({ + features: ['text', 'label'], + num_rows: 4 + }) + ``` + """ + if keep_in_memory and indices_cache_file_name is not None: + raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.") + + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + + # If the array is empty we do nothing + if len(self) == 0: + return self + + # If indices is a PyArrow array, we convert to NumPy + if isinstance(indices, (pa.Array, pa.ChunkedArray)): + indices = indices.to_numpy().astype(np.int64) + + # Convert generator objects to lists + if isinstance(indices, Iterator): + indices = list(indices) + + # If the indices are contiguous, simply slice the arrow table + if isinstance(indices, range): + if _is_range_contiguous(indices) and indices.start >= 0: + start, length = indices.start, indices.stop - indices.start + return self._select_contiguous(start, length, new_fingerprint=new_fingerprint) + else: + try: + start = next(iter(indices)) + except StopIteration: + # if `indices` is an empty iterable, we return an empty dataset + return self._select_contiguous(0, 0, new_fingerprint=new_fingerprint) + if start >= 0: + counter_from_start = itertools.count(start=start) + if all(i == j for i, j in zip(indices, counter_from_start)): + length = next(counter_from_start) - start + return self._select_contiguous(start, length, new_fingerprint=new_fingerprint) + + # If not contiguous, we need to create a new indices mapping + return self._select_with_indices_mapping( + indices, + keep_in_memory=keep_in_memory, + indices_cache_file_name=indices_cache_file_name, + writer_batch_size=writer_batch_size, + new_fingerprint=new_fingerprint, + ) + + @transmit_format + @fingerprint_transform(inplace=False) + def _select_contiguous( + self, + start: int, + length: int, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create a new dataset with rows from a contiguous slice of data. + The slice is defined by that start index and its length. + + Args: + start (`int`): start index. + length (`int`): length of the slice to select. + new_fingerprint (`str`, optional, default `None`): the new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds._select_contiguous(0, 4) + Dataset({ + features: ['text', 'label'], + num_rows: 4 + }) + ``` + """ + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + + # If the array is empty we do nothing + if len(self) == 0: + return self + + _check_valid_indices_value(start, len(self)) + _check_valid_indices_value(start + length - 1, len(self)) + if self._indices is None or length == 0: + return Dataset( + self.data.slice(start, length), + info=self.info.copy(), + split=self.split, + fingerprint=new_fingerprint, + ) + else: + return Dataset( + self.data, + info=self.info.copy(), + split=self.split, + indices_table=self._indices.slice(start, length), + fingerprint=new_fingerprint, + ) + + @transmit_format + @fingerprint_transform(inplace=False, ignore_kwargs=["indices_cache_file_name"]) + def _select_with_indices_mapping( + self, + indices: Iterable, + keep_in_memory: bool = False, + indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create a new dataset with rows selected following the list/array of indices. + The new dataset is made by creating a new indices mapping on top of the main arrow table. + + Args: + indices (sequence, iterable, range, ndarray or Series): List or 1D-array of integer indices for indexing. + keep_in_memory (`bool`, default `False`): Keep the indices mapping in memory instead of writing it to a cache file. + indices_cache_file_name (`str`, optional, default `None`): Provide the name of a path for the cache file. It is used to store the + indices mapping instead of the automatically generated cache file name. + writer_batch_size (`int`, default `1000`): Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `.map()`. + new_fingerprint (`str`, optional, default `None`): the new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds._select_with_indices_mapping(range(4)) + Dataset({ + features: ['text', 'label'], + num_rows: 4 + }) + ``` + """ + if keep_in_memory and indices_cache_file_name is not None: + raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.") + + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.select` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + + # If the array is empty we do nothing + if len(self) == 0: + return self + + # Prepare the writer for our indices arrow table + if keep_in_memory or indices_cache_file_name is None: + buf_writer = pa.BufferOutputStream() + tmp_file = None + writer = ArrowWriter( + stream=buf_writer, writer_batch_size=writer_batch_size, fingerprint=new_fingerprint, unit="indices" + ) + else: + buf_writer = None + logger.info(f"Caching indices mapping at {indices_cache_file_name}") + cache_dir = os.path.dirname(indices_cache_file_name) + os.makedirs(cache_dir, exist_ok=True) + tmp_file = tempfile.NamedTemporaryFile("wb", dir=cache_dir, delete=False) + writer = ArrowWriter( + path=tmp_file.name, writer_batch_size=writer_batch_size, fingerprint=new_fingerprint, unit="indices" + ) + + indices = indices if isinstance(indices, list) else list(indices) + + size = len(self) + if indices: + _check_valid_indices_value(int(max(indices)), size=size) + _check_valid_indices_value(int(min(indices)), size=size) + else: + return self._select_contiguous(0, 0, new_fingerprint=new_fingerprint) + + indices_array = pa.array(indices, type=pa.uint64()) + # Check if we need to convert indices + if self._indices is not None: + indices_array = self._indices.column(0).take(indices_array) + + indices_table = pa.Table.from_arrays([indices_array], names=["indices"]) + + with writer: + try: + writer.write_table(indices_table) + writer.finalize() # close_stream=bool(buf_writer is None)) We only close if we are writing in a file + except (Exception, KeyboardInterrupt): + if tmp_file is not None: + tmp_file.close() + if os.path.exists(tmp_file.name): + os.remove(tmp_file.name) + raise + + if tmp_file is not None: + tmp_file.close() + shutil.move(tmp_file.name, indices_cache_file_name) + umask = os.umask(0o666) + os.umask(umask) + os.chmod(indices_cache_file_name, 0o666 & ~umask) + + # Return new Dataset object + if buf_writer is None: + return self._new_dataset_with_indices( + indices_cache_file_name=indices_cache_file_name, fingerprint=new_fingerprint + ) + else: + return self._new_dataset_with_indices(indices_buffer=buf_writer.getvalue(), fingerprint=new_fingerprint) + + def skip(self, n: int) -> "Dataset": + """ + Create a new [`Dataset`] that skips the first `n` elements. + + Args: + n (`int`): + Number of elements to skip. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + >>> ds = ds.skip(1) + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}, + {'label': 1, + 'text': 'if you sometimes like to go to the movies to have fun , wasabi is a good place to start .'}] + ``` + """ + return self.select(range(n, len(self))) + + def repeat(self, num_times: int) -> "Dataset": + """ + Create a new [`Dataset`] that repeats the underlying dataset `num_times` times. + + Like itertools.repeat, repeating once just returns the full dataset. + + Args: + num_times (`int`): + Number of times to repeat the dataset. + + Example: + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> ds = ds.take(2).repeat(2) + >>> list(ds) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}, + {'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + ``` + """ + if num_times is None: + raise ValueError("Map style datasets do not support indefinite repetition.") + return _concatenate_map_style_datasets([self] * num_times) if num_times > 0 else self.select([]) + + def take(self, n: int) -> "Dataset": + """ + Create a new [`Dataset`] with only the first `n` elements. + + Args: + n (`int`): + Number of elements to take. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> small_ds = ds.take(2) + >>> list(small_ds) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}] + ``` + """ + return self.select(range(n)) + + @transmit_format + @fingerprint_transform(inplace=False, ignore_kwargs=["load_from_cache_file", "indices_cache_file_name"]) + def sort( + self, + column_names: Union[str, Sequence_[str]], + reverse: Union[bool, Sequence_[bool]] = False, + null_placement: str = "at_end", + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create a new dataset sorted according to a single or multiple columns. + + Args: + column_names (`Union[str, Sequence[str]]`): + Column name(s) to sort by. + reverse (`Union[bool, Sequence[bool]]`, defaults to `False`): + If `True`, sort by descending order rather than ascending. If a single bool is provided, + the value is applied to the sorting of all column names. Otherwise a list of bools with the + same length and order as column_names must be provided. + null_placement (`str`, defaults to `at_end`): + Put `None` values at the beginning if `at_start` or `first` or at the end if `at_end` or `last` + + + keep_in_memory (`bool`, defaults to `False`): + Keep the sorted indices in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the sorted indices + can be identified, use it instead of recomputing. + indices_cache_file_name (`str`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + sorted indices instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + Higher value gives smaller cache files, lower value consume less temporary memory. + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='validation') + >>> ds['label'][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + >>> sorted_ds = ds.sort('label') + >>> sorted_ds['label'][:10] + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + >>> another_sorted_ds = ds.sort(['label', 'text'], reverse=[True, False]) + >>> another_sorted_ds['label'][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + ``` + """ + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.sort` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + # If the array is empty we do nothing + if len(self) == 0: + return self + + # Check proper format of and for duplicates in column_names + if isinstance(column_names, str): + column_names = [column_names] + + # Check proper format and length of reverse + if not isinstance(reverse, bool): + if len(reverse) != len(column_names): + raise ValueError( + "Parameter 'reverse' should be either a boolean or a list of booleans with the same length as 'column_names'." + ) + else: + reverse = [reverse] * len(column_names) + + # Check whether column name(s) exist in dataset + for column in column_names: + if not isinstance(column, str) or column not in self._data.column_names: + raise ValueError( + f"Column '{column}' not found in the dataset. Please provide a column selected in: {self._data.column_names}" + ) + + # Change null_placement to conform to pyarrow's sort_indices() while ensuring backwards compatibility + if null_placement not in ["at_start", "at_end"]: + if null_placement == "first": + null_placement = "at_start" + elif null_placement == "last": + null_placement = "at_end" + else: + raise ValueError( + f"null_placement '{null_placement}' is an invalid parameter value. Must be either 'last', 'at_end', 'first' or 'at_start'." + ) + + load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled() + + # Check if we've already cached this computation (indexed by a hash) + if self.cache_files: + if indices_cache_file_name is None: + # we create a unique hash from the function, current dataset file and the mapping args + indices_cache_file_name = self._get_cache_file_path(new_fingerprint) + if os.path.exists(indices_cache_file_name) and load_from_cache_file: + logger.info(f"Loading cached sorted indices for dataset at {indices_cache_file_name}") + return self._new_dataset_with_indices( + fingerprint=new_fingerprint, indices_cache_file_name=indices_cache_file_name + ) + + sort_table = query_table( + table=self._data, + key=slice(0, len(self)), + indices=self._indices, + ) + + sort_keys = [ + (col, "ascending" if not col_reverse else "descending") for col, col_reverse in zip(column_names, reverse) + ] + + indices = pc.sort_indices(sort_table, sort_keys=sort_keys, null_placement=null_placement) + + return self.select( + indices=indices, + keep_in_memory=keep_in_memory, + indices_cache_file_name=indices_cache_file_name, + writer_batch_size=writer_batch_size, + new_fingerprint=new_fingerprint, + ) + + @transmit_format + @fingerprint_transform( + inplace=False, randomized_function=True, ignore_kwargs=["load_from_cache_file", "indices_cache_file_name"] + ) + def shuffle( + self, + seed: Optional[int] = None, + generator: Optional[np.random.Generator] = None, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + new_fingerprint: Optional[str] = None, + ) -> "Dataset": + """Create a new Dataset where the rows are shuffled. + + Currently shuffling uses numpy random generators. + You can either supply a NumPy BitGenerator to use, or a seed to initiate NumPy's default random generator (PCG64). + + Shuffling takes the list of indices `[0:len(my_dataset)]` and shuffles it to create an indices mapping. + However as soon as your [`Dataset`] has an indices mapping, the speed can become 10x slower. + This is because there is an extra step to get the row index to read using the indices mapping, and most importantly, you aren't reading contiguous chunks of data anymore. + To restore the speed, you'd need to rewrite the entire dataset on your disk again using [`Dataset.flatten_indices`], which removes the indices mapping. + This may take a lot of time depending of the size of your dataset though: + + ```python + my_dataset[0] # fast + my_dataset = my_dataset.shuffle(seed=42) + my_dataset[0] # up to 10x slower + my_dataset = my_dataset.flatten_indices() # rewrite the shuffled dataset on disk as contiguous chunks of data + my_dataset[0] # fast again + ``` + + In this case, we recommend switching to an [`IterableDataset`] and leveraging its fast approximate shuffling method [`IterableDataset.shuffle`]. + It only shuffles the shards order and adds a shuffle buffer to your dataset, which keeps the speed of your dataset optimal: + + ```python + my_iterable_dataset = my_dataset.to_iterable_dataset(num_shards=128) + for example in enumerate(my_iterable_dataset): # fast + pass + + shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=42, buffer_size=100) + + for example in enumerate(shuffled_iterable_dataset): # as fast as before + pass + ``` + + Args: + seed (`int`, *optional*): + A seed to initialize the default BitGenerator if `generator=None`. + If `None`, then fresh, unpredictable entropy will be pulled from the OS. + If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state. + generator (`numpy.random.Generator`, *optional*): + Numpy random Generator to use to compute the permutation of the dataset rows. + If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy). + keep_in_memory (`bool`, default `False`): + Keep the shuffled indices in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the shuffled indices + can be identified, use it instead of recomputing. + indices_cache_file_name (`str`, *optional*): + Provide the name of a path for the cache file. It is used to store the + shuffled indices instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds['label'][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + # set a seed + >>> shuffled_ds = ds.shuffle(seed=42) + >>> shuffled_ds['label'][:10] + [1, 0, 1, 1, 0, 0, 0, 0, 0, 0] + ``` + """ + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.shuffle` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + # If the array is empty we do nothing + if len(self) == 0: + return self + + if keep_in_memory and indices_cache_file_name is not None: + raise ValueError("Please use either `keep_in_memory` or `indices_cache_file_name` but not both.") + + if seed is not None and generator is not None: + raise ValueError("Both `seed` and `generator` were provided. Please specify just one of them.") + + if generator is not None and not isinstance(generator, np.random.Generator): + raise ValueError("The provided generator must be an instance of numpy.random.Generator") + + load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled() + + if generator is None: + if seed is None: + _, seed, pos, *_ = np.random.get_state() + seed = seed[pos] if pos < 624 else seed[0] + _ = np.random.random() # do 1 step of rng + generator = np.random.default_rng(seed) + + # Check if we've already cached this computation (indexed by a hash) + if self.cache_files: + if indices_cache_file_name is None: + # we create a unique hash from the function, current dataset file and the mapping args + indices_cache_file_name = self._get_cache_file_path(new_fingerprint) + if os.path.exists(indices_cache_file_name) and load_from_cache_file: + logger.info(f"Loading cached shuffled indices for dataset at {indices_cache_file_name}") + return self._new_dataset_with_indices( + fingerprint=new_fingerprint, indices_cache_file_name=indices_cache_file_name + ) + + permutation = generator.permutation(len(self)) + + return self.select( + indices=permutation, + keep_in_memory=keep_in_memory, + indices_cache_file_name=indices_cache_file_name if not keep_in_memory else None, + writer_batch_size=writer_batch_size, + new_fingerprint=new_fingerprint, + ) + + @transmit_format + @fingerprint_transform( + inplace=False, + randomized_function=True, + fingerprint_names=["train_new_fingerprint", "test_new_fingerprint"], + ignore_kwargs=["load_from_cache_file", "train_indices_cache_file_name", "test_indices_cache_file_name"], + ) + def train_test_split( + self, + test_size: Union[float, int, None] = None, + train_size: Union[float, int, None] = None, + shuffle: bool = True, + stratify_by_column: Optional[str] = None, + seed: Optional[int] = None, + generator: Optional[np.random.Generator] = None, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + train_indices_cache_file_name: Optional[str] = None, + test_indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + train_new_fingerprint: Optional[str] = None, + test_new_fingerprint: Optional[str] = None, + ) -> "DatasetDict": + """Return a dictionary ([`datasets.DatasetDict`]) with two random train and test subsets (`train` and `test` `Dataset` splits). + Splits are created from the dataset according to `test_size`, `train_size` and `shuffle`. + + This method is similar to scikit-learn `train_test_split`. + + Args: + test_size (`Union[float, int, None]`, *optional*): + Size of the test split + If `float`, should be between `0.0` and `1.0` and represent the proportion of the dataset to include in the test split. + If `int`, represents the absolute number of test samples. + If `None`, the value is set to the complement of the train size. + If `train_size` is also `None`, it will be set to `0.25`. + train_size (`Union[float, int, None]`, *optional*): + Size of the train split + If `float`, should be between `0.0` and `1.0` and represent the proportion of the dataset to include in the train split. + If `int`, represents the absolute number of train samples. + If `None`, the value is automatically set to the complement of the test size. + shuffle (`bool`, *optional*, defaults to `True`): + Whether or not to shuffle the data before splitting. + stratify_by_column (`str`, *optional*, defaults to `None`): + The column name of labels to be used to perform stratified split of data. + seed (`int`, *optional*): + A seed to initialize the default BitGenerator if `generator=None`. + If `None`, then fresh, unpredictable entropy will be pulled from the OS. + If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state. + generator (`numpy.random.Generator`, *optional*): + Numpy random Generator to use to compute the permutation of the dataset rows. + If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy). + keep_in_memory (`bool`, defaults to `False`): + Keep the splits indices in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the splits indices + can be identified, use it instead of recomputing. + train_cache_file_name (`str`, *optional*): + Provide the name of a path for the cache file. It is used to store the + train split indices instead of the automatically generated cache file name. + test_cache_file_name (`str`, *optional*): + Provide the name of a path for the cache file. It is used to store the + test split indices instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + train_new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the train set after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + test_new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the test set after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds = ds.train_test_split(test_size=0.2, shuffle=True) + DatasetDict({ + train: Dataset({ + features: ['text', 'label'], + num_rows: 852 + }) + test: Dataset({ + features: ['text', 'label'], + num_rows: 214 + }) + }) + + # set a seed + >>> ds = ds.train_test_split(test_size=0.2, seed=42) + + # stratified split + >>> ds = load_dataset("stanfordnlp/imdb",split="train") + Dataset({ + features: ['text', 'label'], + num_rows: 25000 + }) + >>> ds = ds.train_test_split(test_size=0.2, stratify_by_column="label") + DatasetDict({ + train: Dataset({ + features: ['text', 'label'], + num_rows: 20000 + }) + test: Dataset({ + features: ['text', 'label'], + num_rows: 5000 + }) + }) + ``` + """ + from .dataset_dict import DatasetDict # import here because of circular dependency + + if len(self.list_indexes()) > 0: + raise DatasetTransformationNotAllowedError( + "Using `.train_test_split` on a dataset with attached indexes is not allowed. You can first run `.drop_index() to remove your index and then re-add it." + ) + # If the array is empty we do nothing + if len(self) == 0: + return DatasetDict({"train": self, "test": self}) + + if test_size is None and train_size is None: + test_size = 0.25 + + # Safety checks similar to scikit-learn's ones. + # (adapted from https://github.com/scikit-learn/scikit-learn/blob/fd237278e895b42abe8d8d09105cbb82dc2cbba7/sklearn/model_selection/_split.py#L1750) + n_samples = len(self) + if ( + isinstance(test_size, int) + and (test_size >= n_samples or test_size <= 0) + or isinstance(test_size, float) + and (test_size <= 0 or test_size >= 1) + ): + raise ValueError( + f"test_size={test_size} should be either positive and smaller " + f"than the number of samples {n_samples} or a float in the (0, 1) range" + ) + + if ( + isinstance(train_size, int) + and (train_size >= n_samples or train_size <= 0) + or isinstance(train_size, float) + and (train_size <= 0 or train_size >= 1) + ): + raise ValueError( + f"train_size={train_size} should be either positive and smaller " + f"than the number of samples {n_samples} or a float in the (0, 1) range" + ) + + if train_size is not None and not isinstance(train_size, (int, float)): + raise ValueError(f"Invalid value for train_size: {train_size} of type {type(train_size)}") + if test_size is not None and not isinstance(test_size, (int, float)): + raise ValueError(f"Invalid value for test_size: {test_size} of type {type(test_size)}") + + if isinstance(train_size, float) and isinstance(test_size, float) and train_size + test_size > 1: + raise ValueError( + f"The sum of test_size and train_size = {train_size + test_size}, should be in the (0, 1)" + " range. Reduce test_size and/or train_size." + ) + + if isinstance(test_size, float): + n_test = ceil(test_size * n_samples) + elif isinstance(test_size, int): + n_test = float(test_size) + + if isinstance(train_size, float): + n_train = floor(train_size * n_samples) + elif isinstance(train_size, int): + n_train = float(train_size) + + if train_size is None: + n_train = n_samples - n_test + elif test_size is None: + n_test = n_samples - n_train + + if n_train + n_test > n_samples: + raise ValueError( + f"The sum of train_size and test_size = {n_train + n_test}, " + "should be smaller than the number of " + f"samples {n_samples}. Reduce test_size and/or " + "train_size." + ) + + n_train, n_test = int(n_train), int(n_test) + + if n_train == 0: + raise ValueError( + f"With n_samples={n_samples}, test_size={test_size} and train_size={train_size}, the " + "resulting train set will be empty. Adjust any of the " + "aforementioned parameters." + ) + + load_from_cache_file = load_from_cache_file if load_from_cache_file is not None else is_caching_enabled() + + if generator is None and shuffle is True: + if seed is None: + _, seed, pos, *_ = np.random.get_state() + seed = seed[pos] if pos < 624 else seed[0] + _ = np.random.random() # do 1 step of rng + generator = np.random.default_rng(seed) + + # Check if we've already cached this computation (indexed by a hash) + if self.cache_files: + if train_indices_cache_file_name is None or test_indices_cache_file_name is None: + # we create a unique hash from the function, current dataset file and the mapping args + + if train_indices_cache_file_name is None: + train_indices_cache_file_name = self._get_cache_file_path(train_new_fingerprint) + if test_indices_cache_file_name is None: + test_indices_cache_file_name = self._get_cache_file_path(test_new_fingerprint) + if ( + os.path.exists(train_indices_cache_file_name) + and os.path.exists(test_indices_cache_file_name) + and load_from_cache_file + ): + logger.info( + f"Loading cached split indices for dataset at {train_indices_cache_file_name} and {test_indices_cache_file_name}" + ) + return DatasetDict( + { + "train": self._new_dataset_with_indices( + fingerprint=train_new_fingerprint, indices_cache_file_name=train_indices_cache_file_name + ), + "test": self._new_dataset_with_indices( + fingerprint=test_new_fingerprint, indices_cache_file_name=test_indices_cache_file_name + ), + } + ) + if not shuffle: + if stratify_by_column is not None: + raise ValueError("Stratified train/test split is not implemented for `shuffle=False`") + train_indices = np.arange(n_train) + test_indices = np.arange(n_train, n_train + n_test) + else: + # stratified partition + if stratify_by_column is not None: + if stratify_by_column not in self._info.features.keys(): + raise ValueError(f"Key {stratify_by_column} not found in {self._info.features.keys()}") + if not isinstance(self._info.features[stratify_by_column], ClassLabel): + raise ValueError( + f"Stratifying by column is only supported for {ClassLabel.__name__} column, and column {stratify_by_column} is {type(self._info.features[stratify_by_column]).__name__}." + ) + try: + train_indices, test_indices = next( + stratified_shuffle_split_generate_indices( + np.asarray(self.with_format("numpy")[stratify_by_column]), n_train, n_test, rng=generator + ) + ) + except Exception as error: + if str(error) == "Minimum class count error": + raise ValueError( + f"The least populated class in {stratify_by_column} column has only 1" + " member, which is too few. The minimum" + " number of groups for any class cannot" + " be less than 2." + ) + else: + raise error + + # random partition + else: + permutation = generator.permutation(len(self)) + test_indices = permutation[:n_test] + train_indices = permutation[n_test : (n_test + n_train)] + + train_split = self.select( + indices=train_indices, + keep_in_memory=keep_in_memory, + indices_cache_file_name=train_indices_cache_file_name, + writer_batch_size=writer_batch_size, + new_fingerprint=train_new_fingerprint, + ) + test_split = self.select( + indices=test_indices, + keep_in_memory=keep_in_memory, + indices_cache_file_name=test_indices_cache_file_name, + writer_batch_size=writer_batch_size, + new_fingerprint=test_new_fingerprint, + ) + + return DatasetDict({"train": train_split, "test": test_split}) + + def shard( + self, + num_shards: int, + index: int, + contiguous: bool = True, + keep_in_memory: bool = False, + indices_cache_file_name: Optional[str] = None, + writer_batch_size: Optional[int] = 1000, + ) -> "Dataset": + """Return the `index`-nth shard from dataset split into `num_shards` pieces. + + This shards deterministically. `dataset.shard(n, i)` splits the dataset into contiguous chunks, + so it can be easily concatenated back together after processing. If `len(dataset) % n == l`, then the + first `l` dataset each have length `(len(dataset) // n) + 1`, and the remaining dataset have length `(len(dataset) // n)`. + `datasets.concatenate_datasets([dset.shard(n, i) for i in range(n)])` returns a dataset with the same order as the original. + + Note: n should be less or equal to the number of elements in the dataset `len(dataset)`. + + On the other hand, `dataset.shard(n, i, contiguous=False)` contains all elements of the dataset whose index mod `n = i`. + + Be sure to shard before using any randomizing operator (such as `shuffle`). + It is best if the shard operator is used early in the dataset pipeline. + + Args: + num_shards (`int`): + How many shards to split the dataset into. + index (`int`): + Which shard to select and return. + contiguous: (`bool`, defaults to `True`): + Whether to select contiguous blocks of indices for shards. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + indices_cache_file_name (`str`, *optional*): + Provide the name of a path for the cache file. It is used to store the + indices of each shard instead of the automatically generated cache file name. + writer_batch_size (`int`, defaults to `1000`): + This only concerns the indices mapping. + Number of indices per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds + Dataset({ + features: ['text', 'label'], + num_rows: 1066 + }) + >>> ds = ds.shard(num_shards=2, index=0) + >>> ds + Dataset({ + features: ['text', 'label'], + num_rows: 533 + }) + ``` + """ + if not 0 <= index < num_shards: + raise ValueError("index should be in [0, num_shards-1]") + if contiguous: + div = len(self) // num_shards + mod = len(self) % num_shards + start = div * index + min(index, mod) + end = start + div + (1 if index < mod else 0) + indices = range(start, end) + else: + indices = np.arange(index, len(self), num_shards) + + return self.select( + indices=indices, + keep_in_memory=keep_in_memory, + indices_cache_file_name=indices_cache_file_name, + writer_batch_size=writer_batch_size, + ) + + def to_csv( + self, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_csv_kwargs, + ) -> int: + """Exports the dataset to csv + + Args: + path_or_buf (`PathLike` or `FileOrBuffer`): + Either a path to a file (e.g. `file.csv`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.csv`), + or a BinaryIO, where the dataset will be saved to in the specified format. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + num_proc (`int`, *optional*): + Number of processes for multiprocessing. By default it doesn't + use multiprocessing. `batch_size` in this case defaults to + `datasets.config.DEFAULT_MAX_BATCH_SIZE` but feel free to make it 5x or 10x of the default + value if you have sufficient compute power. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + **to_csv_kwargs (additional keyword arguments): + Parameters to pass to pandas's [`pandas.DataFrame.to_csv`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html). + + + + Now, `index` defaults to `False` if not specified. + + If you would like to write the index, pass `index=True` and also set a name for the index column by + passing `index_label`. + + + + Returns: + `int`: The number of characters or bytes written. + + Example: + + ```py + >>> ds.to_csv("path/to/dataset/directory") + ``` + """ + # Dynamic import to avoid circular dependency + from .io.csv import CsvDatasetWriter + + return CsvDatasetWriter( + self, + path_or_buf, + batch_size=batch_size, + num_proc=num_proc, + storage_options=storage_options, + **to_csv_kwargs, + ).write() + + def to_dict(self, batch_size: Optional[int] = None, batched: bool = False) -> Union[dict, Iterator[dict]]: + """Returns the dataset as a Python dict. Can also return a generator for large datasets. + + Args: + batch_size (`int`, *optional*): The size (number of rows) of the batches if `batched` is `True`. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + batched (`bool`): + Set to `True` to return a generator that yields the dataset as batches + of `batch_size` rows. Defaults to `False` (returns the whole datasets once). + + Returns: + `dict` or `Iterator[dict]` + + Example: + + ```py + >>> ds.to_dict() + ``` + """ + result = query_table( + table=self._data, + key=slice(0, len(self)), + indices=self._indices, + ).to_pydict() + from .utils.json import get_json_field_paths_from_feature, json_decode_field + + for json_field_path in get_json_field_paths_from_feature(self.features): + col, *json_field_subpath = json_field_path + result[col] = [json_decode_field(row, json_field_subpath) for row in result[col]] + return result + + def to_list(self) -> list: + """Returns the dataset as a Python list. + + Returns: + `list` + + Example: + + ```py + >>> ds.to_list() + ``` + """ + result = query_table( + table=self._data, + key=slice(0, len(self)), + indices=self._indices, + ).to_pylist() + from .utils.json import get_json_field_paths_from_feature, json_decode_field + + for json_field_path in get_json_field_paths_from_feature(self.features): + col, *json_field_subpath = json_field_path + for row in result: + row[col] = json_decode_field(row[col], json_field_subpath) + return result + + def to_json( + self, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_json_kwargs, + ) -> int: + """Export the dataset to JSON Lines or JSON. + + The default output format is [JSON Lines](https://jsonlines.org/). + To export to [JSON](https://www.json.org), pass `lines=False` argument and the desired `orient`. + + Args: + path_or_buf (`PathLike` or `FileOrBuffer`): + Either a path to a file (e.g. `file.json`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.json`), + or a BinaryIO, where the dataset will be saved to in the specified format. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + num_proc (`int`, *optional*): + Number of processes for multiprocessing. By default, it doesn't + use multiprocessing. `batch_size` in this case defaults to + `datasets.config.DEFAULT_MAX_BATCH_SIZE` but feel free to make it 5x or 10x of the default + value if you have sufficient compute power. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + **to_json_kwargs (additional keyword arguments): + Parameters to pass to pandas's [`pandas.DataFrame.to_json`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_json.html). + Default arguments are `lines=True` and `orient="records". + + + + The parameter `index` defaults to `False` if `orient` is `"split"` or `"table"`. + + If you would like to write the index, pass `index=True`. + + + + Returns: + `int`: The number of characters or bytes written. + + Example: + + ```py + >>> ds.to_json("path/to/dataset/directory/filename.jsonl") + ``` + """ + # Dynamic import to avoid circular dependency + from .io.json import JsonDatasetWriter + + return JsonDatasetWriter( + self, + path_or_buf, + batch_size=batch_size, + num_proc=num_proc, + storage_options=storage_options, + **to_json_kwargs, + ).write() + + def to_pandas( + self, batch_size: Optional[int] = None, batched: bool = False + ) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]: + """Returns the dataset as a `pandas.DataFrame`. Can also return a generator for large datasets. + + Args: + batch_size (`int`, *optional*): + The size (number of rows) of the batches if `batched` is `True`. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + batched (`bool`): + Set to `True` to return a generator that yields the dataset as batches + of `batch_size` rows. Defaults to `False` (returns the whole datasets once). + + Returns: + `pandas.DataFrame` or `Iterator[pandas.DataFrame]` + + Example: + + ```py + >>> ds.to_pandas() + ``` + """ + if not batched: + return query_table( + table=self._data, + key=slice(0, len(self)), + indices=self._indices, + ).to_pandas(types_mapper=pandas_types_mapper) + else: + batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + return ( + query_table( + table=self._data, + key=slice(offset, offset + batch_size), + indices=self._indices, + ).to_pandas(types_mapper=pandas_types_mapper) + for offset in range(0, len(self), batch_size) + ) + + def to_polars( + self, + batch_size: Optional[int] = None, + batched: bool = False, + schema_overrides: Optional[dict] = None, + rechunk: bool = True, + ) -> Union["pl.DataFrame", Iterator["pl.DataFrame"]]: + """Returns the dataset as a `polars.DataFrame`. Can also return a generator for large datasets. + + Args: + batch_size (`int`, *optional*): + The size (number of rows) of the batches if `batched` is `True`. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + batched (`bool`): + Set to `True` to return a generator that yields the dataset as batches + of `batch_size` rows. Defaults to `False` (returns the whole datasets once). + schema_overrides (`dict`, *optional*): + Support type specification or override of one or more columns; note that + any dtypes inferred from the schema param will be overridden. + rechunk (`bool`): + Make sure that all data is in contiguous memory. Defaults to `True`. + Returns: + `polars.DataFrame` or `Iterator[polars.DataFrame]` + + Example: + + ```py + >>> ds.to_polars() + ``` + """ + if config.POLARS_AVAILABLE: + import polars as pl + + if not batched: + return pl.from_arrow( + query_table( + table=self._data, + key=slice(0, len(self)), + indices=self._indices if self._indices is not None else None, + ), + schema_overrides=schema_overrides, + rechunk=rechunk, + ) + else: + batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + return ( + pl.from_arrow( + query_table( + table=self._data, + key=slice(offset, offset + batch_size), + indices=self._indices if self._indices is not None else None, + ), + schema_overrides=schema_overrides, + rechunk=rechunk, + ) + for offset in range(0, len(self), batch_size) + ) + else: + raise ValueError("Polars needs to be installed to be able to return Polars dataframes.") + + def to_parquet( + self, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + storage_options: Optional[dict] = None, + **parquet_writer_kwargs, + ) -> int: + """Exports the dataset to parquet + + Args: + path_or_buf (`PathLike` or `FileOrBuffer`): + Either a path to a file (e.g. `file.parquet`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.parquet`), + or a BinaryIO, where the dataset will be saved to in the specified format. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + By default it aims for row groups with maximum uncompressed byte size of "100MB", + defined by `datasets.config.MAX_ROW_GROUP_SIZE`. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + **parquet_writer_kwargs (additional keyword arguments): + Parameters to pass to PyArrow's `pyarrow.parquet.ParquetWriter`. + + Returns: + `int`: The number of characters or bytes written. + + Example: + + ```py + >>> ds.to_parquet("path/to/dataset/directory") + ``` + """ + # Dynamic import to avoid circular dependency + from .io.parquet import ParquetDatasetWriter + + return ParquetDatasetWriter( + self, path_or_buf, batch_size=batch_size, storage_options=storage_options, **parquet_writer_kwargs + ).write() + + def to_sql( + self, + name: str, + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"], + batch_size: Optional[int] = None, + **sql_writer_kwargs, + ) -> int: + """Exports the dataset to a SQL database. + + Args: + name (`str`): + Name of SQL table. + con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`): + A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) or a SQLite3/SQLAlchemy connection object used to write to a database. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + **sql_writer_kwargs (additional keyword arguments): + Parameters to pass to pandas's [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html). + + + + Now, `index` defaults to `False` if not specified. + + If you would like to write the index, pass `index=True` and also set a name for the index column by + passing `index_label`. + + + + Returns: + `int`: The number of records written. + + Example: + + ```py + >>> # con provided as a connection URI string + >>> ds.to_sql("data", "sqlite:///my_own_db.sql") + >>> # con provided as a sqlite3 connection object + >>> import sqlite3 + >>> con = sqlite3.connect("my_own_db.sql") + >>> with con: + ... ds.to_sql("data", con) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.sql import SqlDatasetWriter + + return SqlDatasetWriter(self, name, con, batch_size=batch_size, **sql_writer_kwargs).write() + + def _estimate_nbytes(self) -> int: + dataset_nbytes = self.data.nbytes + + # Find decodable columns, because if there are any, we need to + # adjust the dataset size computation (needed for sharding) to account for possible external files + decodable_columns = [ + k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True) + ] + + if decodable_columns: + # Approximate the space needed to store the bytes from the external files by analyzing the first 1000 examples + extra_nbytes = 0 + + def extra_nbytes_visitor(array, feature): + nonlocal extra_nbytes + if isinstance(feature, (Audio, Image, Video)): + for x in array.to_pylist(): + if x is not None and x["bytes"] is None and x["path"] is not None: + size = xgetsize(x["path"]) + extra_nbytes += size + extra_nbytes -= array.field("path").nbytes + + table = self.with_format("arrow")[:1000] + table_visitor(table, extra_nbytes_visitor) + + extra_nbytes = extra_nbytes * len(self.data) // len(table) + dataset_nbytes = dataset_nbytes + extra_nbytes + + if self._indices is not None: + dataset_nbytes = dataset_nbytes * len(self._indices) // len(self.data) + return dataset_nbytes + + @staticmethod + def _generate_tables_from_shards(shards: list["Dataset"], batch_size: int): + for shard_idx, shard in enumerate(shards): + for pa_table in shard.with_format("arrow").iter(batch_size): + yield shard_idx, pa_table + + @staticmethod + def _generate_tables_from_cache_file(filename: str): + for batch_idx, batch in enumerate(_memory_mapped_record_batch_reader_from_file(filename)): + yield batch_idx, pa.Table.from_batches([batch]) + + def to_iterable_dataset(self, num_shards: Optional[int] = 1) -> "IterableDataset": + """Get an [`datasets.IterableDataset`] from a map-style [`datasets.Dataset`]. + This is equivalent to loading a dataset in streaming mode with [`datasets.load_dataset`], but much faster since the data is streamed from local files. + + Contrary to map-style datasets, iterable datasets are lazy and can only be iterated over (e.g. using a for loop). + Since they are read sequentially in training loops, iterable datasets are much faster than map-style datasets. + All the transformations applied to iterable datasets like filtering or processing are done on-the-fly when you start iterating over the dataset. + + Still, it is possible to shuffle an iterable dataset using [`datasets.IterableDataset.shuffle`]. + This is a fast approximate shuffling that works best if you have multiple shards and if you specify a buffer size that is big enough. + + To get the best speed performance, make sure your dataset doesn't have an indices mapping. + If this is the case, the data are not read contiguously, which can be slow sometimes. + You can use `ds = ds.flatten_indices()` to write your dataset in contiguous chunks of data and have optimal speed before switching to an iterable dataset. + + Args: + num_shards (`int`, default to `1`): + Number of shards to define when instantiating the iterable dataset. This is especially useful for big datasets to be able to shuffle properly, + and also to enable fast parallel loading using a PyTorch DataLoader or in distributed setups for example. + Shards are defined using [`datasets.Dataset.shard`]: it simply slices the data without writing anything on disk. + + Returns: + [`datasets.IterableDataset`] + + Example: + + Basic usage: + ```python + >>> ids = ds.to_iterable_dataset() + >>> for example in ids: + ... pass + ``` + + With lazy filtering and processing: + ```python + >>> ids = ds.to_iterable_dataset() + >>> ids = ids.filter(filter_fn).map(process_fn) # will filter and process on-the-fly when you start iterating over the iterable dataset + >>> for example in ids: + ... pass + ``` + + With sharding to enable efficient shuffling: + ```python + >>> ids = ds.to_iterable_dataset(num_shards=64) # the dataset is split into 64 shards to be iterated over + >>> ids = ids.shuffle(buffer_size=10_000) # will shuffle the shards order and use a shuffle buffer for fast approximate shuffling when you start iterating + >>> for example in ids: + ... pass + ``` + + With a PyTorch DataLoader: + ```python + >>> import torch + >>> ids = ds.to_iterable_dataset(num_shards=64) + >>> ids = ids.filter(filter_fn).map(process_fn) + >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards to each worker to load, filter and process when you start iterating + >>> for example in ids: + ... pass + ``` + + With a PyTorch DataLoader and shuffling: + ```python + >>> import torch + >>> ids = ds.to_iterable_dataset(num_shards=64) + >>> ids = ids.shuffle(buffer_size=10_000) # will shuffle the shards order and use a shuffle buffer when you start iterating + >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards from the shuffled list of shards to each worker when you start iterating + >>> for example in ids: + ... pass + ``` + + In a distributed setup like PyTorch DDP with a PyTorch DataLoader and shuffling + ```python + >>> from datasets.distributed import split_dataset_by_node + >>> ids = ds.to_iterable_dataset(num_shards=512) + >>> ids = ids.shuffle(buffer_size=10_000, seed=42) # will shuffle the shards order and use a shuffle buffer when you start iterating + >>> ids = split_dataset_by_node(ds, world_size=8, rank=0) # will keep only 512 / 8 = 64 shards from the shuffled lists of shards when you start iterating + >>> dataloader = torch.utils.data.DataLoader(ids, num_workers=4) # will assign 64 / 4 = 16 shards from this node's list of shards to each worker when you start iterating + >>> for example in ids: + ... pass + ``` + + With shuffling and multiple epochs: + ```python + >>> ids = ds.to_iterable_dataset(num_shards=64) + >>> ids = ids.shuffle(buffer_size=10_000, seed=42) # will shuffle the shards order and use a shuffle buffer when you start iterating + >>> for epoch in range(n_epochs): + ... ids.set_epoch(epoch) # will use effective_seed = seed + epoch to shuffle the shards and for the shuffle buffer when you start iterating + ... for example in ids: + ... pass + ``` + Feel free to also use [`IterableDataset.set_epoch`] when using a PyTorch DataLoader or in distributed setups. + """ + from .iterable_dataset import ArrowExamplesIterable, IterableDataset + + if self._format_type is not None: + if self._format_kwargs or ( + self._format_columns is not None and set(self._format_columns) != set(self.column_names) + ): + raise NotImplementedError( + "Converting a formatted dataset with kwargs or selected columns to a formatted iterable dataset is not implemented yet. Please run `my_dataset = my_dataset.with_format(None)` before calling to_iterable_dataset" + ) + if num_shards > len(self): + raise ValueError( + f"Unable to shard a dataset of size {len(self)} into {num_shards} shards (the number of shards exceeds the number of samples)." + ) + if self._indices is not None: + logger.info( + "Converting an Arrow dataset to iterable but it has an indices mapping that can make it slower. " + "You can use `ds = ds.flatten_indices()` to write your dataset in contiguous chunks of data and have optimal speed." + ) + shards = ( + [copy.deepcopy(self)] + if num_shards == 1 + else [ + self.shard(num_shards=num_shards, index=shard_idx, contiguous=True) for shard_idx in range(num_shards) + ] + ) + ex_iterable = ArrowExamplesIterable( + Dataset._generate_tables_from_shards, + kwargs={"shards": shards, "batch_size": config.DEFAULT_MAX_BATCH_SIZE}, + ) + ds = IterableDataset(ex_iterable, info=DatasetInfo(features=self.features)) + if self._format_type: + ds = ds.with_format(self._format_type) + return ds + + def _push_parquet_shards_to_hub_single( + self, + job_id: int, + num_jobs: int, + resolved_output_path: HfFileSystemResolvedPath, + data_dir: str, + split: str, + token: Optional[str], + create_pr: Optional[bool], + num_shards: int, + embed_external_files: bool, + writer_batch_size: int, + ): + div = num_shards // num_jobs + mod = num_shards % num_jobs + start = div * job_id + min(job_id, mod) + end = start + div + (1 if job_id < mod else 0) + + index_shards = ( + (start + i, self.shard(num_shards=end - start, index=i, contiguous=True)) for i in range(end - start) + ) + + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + + additions: list[CommitOperationAdd] = [] + new_parquet_paths: list[str] = [] + uploaded_size = 0 + for index, shard in index_shards: + if embed_external_files: + format = shard.format + shard = shard.with_format("arrow") + shard = shard.map( + embed_table_storage, + batched=True, + batch_size=writer_batch_size, + keep_in_memory=True, + ) + shard = shard.with_format(**format) + shard_path_in_repo = f"{data_dir}/{split}-{index:05d}-of-{num_shards:05d}.parquet" + tmp_file = tempfile.NamedTemporaryFile(suffix=".parquet", delete=False) + try: + shard.to_parquet(tmp_file, batch_size=writer_batch_size) + uploaded_size += tmp_file.tell() + tmp_file.close() + new_parquet_paths.append(shard_path_in_repo) + if ( + isinstance(resolved_output_path, HfFileSystemResolvedRepositoryPath) + and not resolved_output_path.path_in_repo + ): + shard_addition = CommitOperationAdd(path_in_repo=shard_path_in_repo, path_or_fileobj=tmp_file.name) + api.preupload_lfs_files( + repo_id=resolved_output_path.repo_id, + additions=[shard_addition], + repo_type=resolved_output_path.repo_type, + revision=resolved_output_path.revision, + create_pr=create_pr, + ) + additions.append(shard_addition) + elif isinstance(resolved_output_path, HfFileSystemResolvedBucketPath): + if resolved_output_path.path: + shard_path_in_repo = resolved_output_path.path + "/" + shard_path_in_repo + api.batch_bucket_files( + bucket_id=resolved_output_path.bucket_id, add=[(tmp_file.name, shard_path_in_repo)] + ) + else: + raise NotImplementedError(f"Bad HF path: {resolved_output_path}") + except (Exception, KeyboardInterrupt): + tmp_file.close() + Path(tmp_file.name).unlink() + raise + tmp_file.close() + Path(tmp_file.name).unlink() + yield job_id, False, 1 + + yield job_id, True, (additions, new_parquet_paths, uploaded_size) + + def _push_parquet_shards_to_hub( + self, + resolved_output_path: HfFileSystemResolvedPath, + data_dir: str, + split: str, + token: Optional[str], + create_pr: Optional[bool], + max_shard_size: Optional[Union[int, str]], + num_shards: Optional[int], + embed_external_files: bool, + num_proc: Optional[int], + ) -> tuple[list[CommitOperationAdd], list[str], int, SplitInfo]: + """Pushes the dataset shards as Parquet files to the hub. + + Returns: + additions (`List[CommitOperation]`): list of the `CommitOperationAdd` of the uploaded shards + new_parquet_paths (`List[str]`): list of paths of the new files uploaded to the output path, + relative to output path + features (`features`): features of the uploaded dataset + split_info (`int`): info of the uploaded split, including the approximate size in bytes of + the uploaded dataset after uncompression + uploaded_size (`int`): number of uploaded bytes to the repository or bucket + """ + from .arrow_writer import get_writer_batch_size_from_data_size, get_writer_batch_size_from_features + + dataset_nbytes = self._estimate_nbytes() + writer_batch_size = get_writer_batch_size_from_features(self.features) or get_writer_batch_size_from_data_size( + len(self), dataset_nbytes + ) + + # Find decodable columns, because if there are any, we need to: + # embed the bytes from the files in the shards + decodable_columns = ( + [k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True)] + if embed_external_files + else [] + ) + embed_external_files = embed_external_files and bool(decodable_columns) + + if num_shards is None: + max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE) + num_shards = int(dataset_nbytes / max_shard_size) + 1 + num_shards = max(num_shards, num_proc or 1) + + additions: list[CommitOperationAdd] = [] + new_parquet_paths: list[str] = [] + uploaded_size = 0 + + num_jobs = num_proc or 1 + if num_shards <= 1: + logger.warning( + f"Setting num_proc from {num_jobs} back to 1 for the {split} split to disable multiprocessing as it only contains one shard." + ) + num_jobs = 1 + elif num_shards < num_jobs: + logger.warning( + f"Setting num_proc from {num_jobs} to {num_shards} for the {split} split as it only contains {num_shards} shards." + ) + num_proc = num_shards + kwargs_iterable = [ + { + "self": self.shard(num_shards=num_jobs, index=job_id, contiguous=True), + "job_id": job_id, + "num_jobs": num_jobs, + "resolved_output_path": resolved_output_path, + "data_dir": data_dir, + "split": split, + "token": token, + "create_pr": create_pr, + "num_shards": num_shards, + "embed_external_files": embed_external_files, + "writer_batch_size": writer_batch_size, + } + for job_id in range(num_jobs) + ] + desc = "Uploading the dataset shards" + desc += f" (num_proc={num_proc})" if num_proc is not None and num_proc >= 1 else "" + pbar = hf_tqdm( + unit=" shards", + total=num_shards, + desc=desc, + ) + with ( + contextlib.nullcontext() + if num_proc is None or num_proc < 1 + else mp.get_context("spawn").Pool(num_proc) as pool + ): + update_stream = ( + Dataset._push_parquet_shards_to_hub_single(**kwargs_iterable[0]) + if pool is None + else iflatmap_unordered( + pool, + Dataset._push_parquet_shards_to_hub_single, + kwargs_iterable=kwargs_iterable, + ) + ) + for job_id, done, content in update_stream: + if not done: + pbar.update(content) + else: + job_additions, job_new_parquet_paths, job_uploaded_size = content + additions += job_additions + new_parquet_paths += job_new_parquet_paths + uploaded_size += job_uploaded_size + + split_info = SplitInfo(name=split, num_bytes=dataset_nbytes, num_examples=len(self)) + return additions, new_parquet_paths, self.features, split_info, uploaded_size + + def push_to_hub( + self, + repo_id: str, + config_name: str = "default", + set_default: Optional[bool] = None, + split: Optional[str] = None, + data_dir: Optional[str] = None, + commit_message: Optional[str] = None, + commit_description: Optional[str] = None, + private: Optional[bool] = None, + token: Optional[str] = None, + revision: Optional[str] = None, + create_pr: Optional[bool] = False, + max_shard_size: Optional[Union[int, str]] = None, + num_shards: Optional[int] = None, + embed_external_files: bool = True, + num_proc: Optional[int] = None, + ) -> CommitInfo: + """Pushes the dataset to the hub as a Parquet dataset. + The dataset is pushed using HTTP requests and does not need to have neither git or git-lfs installed. + + The resulting Parquet files are self-contained by default. If your dataset contains [`Image`], [`Audio`] or [`Video`] + data, the Parquet files will store the bytes of your images or audio files. + You can disable this by setting `embed_external_files` to `False`. + + Args: + repo_id (`str`): + The ID of the repository to push to in the following format: `/` or + `/`. Also accepts ``, which will default to the namespace + of the logged-in user. + + It could also be a location inside a bucket, e.g. `buckets///...` + config_name (`str`, defaults to "default"): + The configuration name (or subset) of a dataset. Defaults to "default". + set_default (`bool`, *optional*): + Whether to set this configuration as the default one. Otherwise, the default configuration is the one + named "default". + split (`str`, *optional*): + The name of the split that will be given to that dataset. Defaults to `self.split`. + data_dir (`str`, *optional*): + Directory name that will contain the uploaded data files. Defaults to the `config_name` if different + from "default", else "data". + + + commit_message (`str`, *optional*): + Message to commit while pushing. Will default to `"Upload dataset"`. + commit_description (`str`, *optional*): + Description of the commit that will be created. + Additionally, description of the PR if a PR is created (`create_pr` is True). + + + private (`bool`, *optional*): + Whether to make the repo private. If `None` (default), the repo will be public unless the + organization's default is private. This value is ignored if the repo already exists. + token (`str`, *optional*): + An optional authentication token for the Hugging Face Hub. If no token is passed, will default + to the token saved locally when logging in with `huggingface-cli login`. Will raise an error + if no token is passed and the user is not logged-in. + revision (`str`, *optional*): + Branch to push the uploaded files to. Defaults to the `"main"` branch. + + + create_pr (`bool`, *optional*, defaults to `False`): + Whether to create a PR with the uploaded files or directly commit. + + + max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`): + The maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed by + a unit (like `"5MB"`). + num_shards (`int`, *optional*): + Number of shards to write. By default, the number of shards depends on `max_shard_size`. + + + embed_external_files (`bool`, defaults to `True`): + Whether to embed file bytes in the shards. + In particular, this will do the following before the push for the fields of type: + + - [`Audio`] and [`Image`]: remove local path information and embed file content in the Parquet files. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when preparing and uploading the dataset. + This is helpful if the dataset is made of many samples or media files to embed. + I uses "spawn" context to work with hf_xet, the rust client for fast uploads to HF. + Multiprocessing is disabled by default. + + + + Return: + huggingface_hub.CommitInfo + + Example: + + ```python + >>> dataset.push_to_hub("/") + >>> dataset_dict.push_to_hub("/", private=True) + >>> dataset.push_to_hub("/", max_shard_size="1GB") + >>> dataset.push_to_hub("/", num_shards=1024) + ``` + + If your dataset has multiple splits (e.g. train/validation/test): + + ```python + >>> train_dataset.push_to_hub("/", split="train") + >>> val_dataset.push_to_hub("/", split="validation") + >>> # later + >>> dataset = load_dataset("/") + >>> train_dataset = dataset["train"] + >>> val_dataset = dataset["validation"] + ``` + + If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages): + + ```python + >>> english_dataset.push_to_hub("/", "en") + >>> french_dataset.push_to_hub("/", "fr") + >>> # later + >>> english_dataset = load_dataset("/", "en") + >>> french_dataset = load_dataset("/", "fr") + ``` + """ + if config_name == "data": + raise ValueError("`config_name` cannot be 'data'. Please, choose another name for configuration.") + + if max_shard_size is not None and num_shards is not None: + raise ValueError( + "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both." + ) + + if split is None: + split = str(self.split) if self.split is not None else "train" + + if not re.match(_split_re, split): + raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.") + + if not data_dir: + data_dir = config_name if config_name != "default" else "data" # for backward compatibility + + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + if repo_id.startswith("buckets/"): + if BucketNotFoundError is None: + raise ImportError("Pushing datasets to buckets requires huggingface_hub>=1.6.0") + _, _namespace, _bucket_name, *_path_segments = repo_id.split("/") + try: + bucket_id = api.bucket_info(_namespace + "/" + _bucket_name).id + except BucketNotFoundError: + bucket_url = api.create_bucket(_namespace + "/" + _bucket_name, private=private, exist_ok=True) + bucket_id = bucket_url.bucket_id + path = "/".join(s for s in _path_segments if s) + return _push_to_bucket( + self, + bucket_id=bucket_id, + path=path, + config_name=config_name, + set_default=set_default, + split=split, + data_dir=data_dir, + token=token, + max_shard_size=max_shard_size, + num_shards=num_shards, + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + else: + try: + repo_id = api.repo_info(repo_id, repo_type="dataset").id + except RepositoryNotFoundError: + repo_url = api.create_repo( + repo_id, + repo_type="dataset", + private=private, + exist_ok=True, + ) + repo_id = repo_url.repo_id + + if revision is not None and not revision.startswith("refs/pr/"): + # We do not call create_branch for a PR reference: 400 Bad Request + api.create_branch(repo_id, branch=revision, repo_type="dataset", exist_ok=True) + return _push_to_repo( + self, + repo_id=repo_id, + config_name=config_name, + set_default=set_default, + split=split, + data_dir=data_dir, + commit_message=commit_message, + commit_description=commit_description, + token=token, + revision=revision, + create_pr=create_pr, + max_shard_size=max_shard_size, + num_shards=num_shards, + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + + @transmit_format + @fingerprint_transform(inplace=False) + def add_column( + self, + name: str, + column: Union[list, np.ndarray], + new_fingerprint: Optional[str] = None, + feature: Optional[FeatureType] = None, + ): + """Add column to Dataset. + + + + Args: + name (`str`): + Column name. + column (`list` or `np.array`): + Column data to be added. + feature (`FeatureType` or `None`, defaults to `None`): + Column datatype. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> more_text = ds["text"] + >>> ds = ds.add_column(name="text_2", column=more_text) + >>> ds + Dataset({ + features: ['text', 'label', 'text_2'], + num_rows: 1066 + }) + ``` + """ + + if feature: + pyarrow_schema = Features({name: feature}).arrow_schema + else: + pyarrow_schema = None + + column_table = InMemoryTable.from_pydict({name: column}, schema=pyarrow_schema) + _check_column_names(self._data.column_names + column_table.column_names) + dataset = self.flatten_indices() if self._indices is not None else self + # Concatenate tables horizontally + table = concat_tables([dataset._data, column_table], axis=1) + # Update features + info = dataset.info.copy() + info.features.update(Features.from_arrow_schema(column_table.schema)) + table = update_metadata_with_features(table, info.features) + return Dataset(table, info=info, split=self.split, indices_table=None, fingerprint=new_fingerprint) + + def add_faiss_index( + self, + column: str, + index_name: Optional[str] = None, + device: Optional[int] = None, + string_factory: Optional[str] = None, + metric_type: Optional[int] = None, + custom_index: Optional["faiss.Index"] = None, # noqa: F821 + batch_size: int = 1000, + train_size: Optional[int] = None, + faiss_verbose: bool = False, + dtype=np.float32, + ): + """Add a dense index using Faiss for fast retrieval. + By default the index is done over the vectors of the specified column. + You can specify `device` if you want to run it on GPU (`device` must be the GPU index). + You can find more information about Faiss here: + + - For [string factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory) + + Args: + column (`str`): + The column of the vectors to add to the index. + index_name (`str`, *optional*): + The `index_name`/identifier of the index. + This is the `index_name` that is used to call [`~datasets.Dataset.get_nearest_examples`] or [`~datasets.Dataset.search`]. + By default it corresponds to `column`. + device (`Union[int, List[int]]`, *optional*): + If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs. + If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU. + string_factory (`str`, *optional*): + This is passed to the index factory of Faiss to create the index. + Default index class is `IndexFlat`. + metric_type (`int`, *optional*): + Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`. + custom_index (`faiss.Index`, *optional*): + Custom Faiss index that you already have instantiated and configured for your needs. + batch_size (`int`): + Size of the batch to use while adding vectors to the `FaissIndex`. Default value is `1000`. + + train_size (`int`, *optional*): + If the index needs a training step, specifies how many vectors will be used to train the index. + faiss_verbose (`bool`, defaults to `False`): + Enable the verbosity of the Faiss index. + dtype (`data-type`): + The dtype of the numpy arrays that are indexed. + Default is `np.float32`. + + Example: + + ```python + >>> ds = datasets.load_dataset('community-datasets/crime_and_punish', split='train') + >>> ds_with_embeddings = ds.map(lambda example: {'embeddings': embed(example['line']})) + >>> ds_with_embeddings.add_faiss_index(column='embeddings') + >>> # query + >>> scores, retrieved_examples = ds_with_embeddings.get_nearest_examples('embeddings', embed('my new query'), k=10) + >>> # save index + >>> ds_with_embeddings.save_faiss_index('embeddings', 'my_index.faiss') + + >>> ds = datasets.load_dataset('community-datasets/crime_and_punish', split='train') + >>> # load index + >>> ds.load_faiss_index('embeddings', 'my_index.faiss') + >>> # query + >>> scores, retrieved_examples = ds.get_nearest_examples('embeddings', embed('my new query'), k=10) + ``` + """ + with self.formatted_as(type="numpy", columns=[column], dtype=dtype): + super().add_faiss_index( + column=column, + index_name=index_name, + device=device, + string_factory=string_factory, + metric_type=metric_type, + custom_index=custom_index, + batch_size=batch_size, + train_size=train_size, + faiss_verbose=faiss_verbose, + ) + return self + + def add_faiss_index_from_external_arrays( + self, + external_arrays: np.array, + index_name: str, + device: Optional[int] = None, + string_factory: Optional[str] = None, + metric_type: Optional[int] = None, + custom_index: Optional["faiss.Index"] = None, # noqa: F821 + batch_size: int = 1000, + train_size: Optional[int] = None, + faiss_verbose: bool = False, + dtype=np.float32, + ): + """Add a dense index using Faiss for fast retrieval. + The index is created using the vectors of `external_arrays`. + You can specify `device` if you want to run it on GPU (`device` must be the GPU index). + You can find more information about Faiss here: + + - For [string factory](https://github.com/facebookresearch/faiss/wiki/The-index-factory) + + Args: + external_arrays (`np.array`): + If you want to use arrays from outside the lib for the index, you can set `external_arrays`. + It will use `external_arrays` to create the Faiss index instead of the arrays in the given `column`. + index_name (`str`): + The `index_name`/identifier of the index. + This is the `index_name` that is used to call [`~datasets.Dataset.get_nearest_examples`] or [`~datasets.Dataset.search`]. + device (Optional `Union[int, List[int]]`, *optional*): + If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs. + If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU. + string_factory (`str`, *optional*): + This is passed to the index factory of Faiss to create the index. + Default index class is `IndexFlat`. + metric_type (`int`, *optional*): + Type of metric. Ex: `faiss.faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`. + custom_index (`faiss.Index`, *optional*): + Custom Faiss index that you already have instantiated and configured for your needs. + batch_size (`int`, *optional*): + Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000. + + train_size (`int`, *optional*): + If the index needs a training step, specifies how many vectors will be used to train the index. + faiss_verbose (`bool`, defaults to False): + Enable the verbosity of the Faiss index. + dtype (`numpy.dtype`): + The dtype of the numpy arrays that are indexed. Default is np.float32. + """ + super().add_faiss_index_from_external_arrays( + external_arrays=external_arrays.astype(dtype), + index_name=index_name, + device=device, + string_factory=string_factory, + metric_type=metric_type, + custom_index=custom_index, + batch_size=batch_size, + train_size=train_size, + faiss_verbose=faiss_verbose, + ) + + def add_elasticsearch_index( + self, + column: str, + index_name: Optional[str] = None, + host: Optional[str] = None, + port: Optional[int] = None, + es_client: Optional["elasticsearch.Elasticsearch"] = None, # noqa: F821 + es_index_name: Optional[str] = None, + es_index_config: Optional[dict] = None, + ): + """Add a text index using ElasticSearch for fast retrieval. This is done in-place. + + Args: + column (`str`): + The column of the documents to add to the index. + index_name (`str`, *optional*): + The `index_name`/identifier of the index. + This is the index name that is used to call [`~Dataset.get_nearest_examples`] or [`~Dataset.search`]. + By default it corresponds to `column`. + host (`str`, *optional*, defaults to `localhost`): + Host of where ElasticSearch is running. + port (`str`, *optional*, defaults to `9200`): + Port of where ElasticSearch is running. + es_client (`elasticsearch.Elasticsearch`, *optional*): + The elasticsearch client used to create the index if host and port are `None`. + es_index_name (`str`, *optional*): + The elasticsearch index name used to create the index. + es_index_config (`dict`, *optional*): + The configuration of the elasticsearch index. + Default config is: + ``` + { + "settings": { + "number_of_shards": 1, + "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}}, + }, + "mappings": { + "properties": { + "text": { + "type": "text", + "analyzer": "standard", + "similarity": "BM25" + }, + } + }, + } + ``` + Example: + + ```python + >>> es_client = elasticsearch.Elasticsearch() + >>> ds = datasets.load_dataset('community-datasets/crime_and_punish', split='train') + >>> ds.add_elasticsearch_index(column='line', es_client=es_client, es_index_name="my_es_index") + >>> scores, retrieved_examples = ds.get_nearest_examples('line', 'my new query', k=10) + ``` + """ + with self.formatted_as(type=None, columns=[column]): + super().add_elasticsearch_index( + column=column, + index_name=index_name, + host=host, + port=port, + es_client=es_client, + es_index_name=es_index_name, + es_index_config=es_index_config, + ) + return self + + @transmit_format + @fingerprint_transform(inplace=False) + def add_item(self, item: dict, new_fingerprint: Optional[str] = None): + """Add item to Dataset. + + + + Args: + item (`dict`): + Item data to be added. + + Returns: + [`Dataset`] + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> new_review = {'label': 0, 'text': 'this movie is the absolute worst thing I have ever seen'} + >>> ds = ds.add_item(new_review) + >>> ds[-1] + {'label': 0, 'text': 'this movie is the absolute worst thing I have ever seen'} + ``` + """ + item_table = InMemoryTable.from_pydict({k: [v] for k, v in item.items()}) + # We don't call _check_if_features_can_be_aligned here so this cast is "unsafe" + dset_features, item_features = _align_features( + [self._info.features, Features.from_arrow_schema(item_table.schema)] + ) + # Cast to align the schemas of the tables and concatenate the tables + table = concat_tables( + [ + self._data.cast(dset_features.arrow_schema) if self._info.features != dset_features else self._data, + item_table.cast(item_features.arrow_schema), + ] + ) + if self._indices is None: + indices_table = None + else: + item_indices_array = pa.array([len(self._data)], type=pa.uint64()) + item_indices_table = InMemoryTable.from_arrays([item_indices_array], names=["indices"]) + indices_table = concat_tables([self._indices, item_indices_table]) + info = self.info.copy() + info.features.update(item_features) + table = update_metadata_with_features(table, info.features) + return Dataset( + table, + info=info, + split=self.split, + indices_table=indices_table, + fingerprint=new_fingerprint, + ) + + def align_labels_with_mapping(self, label2id: dict, label_column: str) -> "Dataset": + """Align the dataset's label ID and label name mapping to match an input `label2id` mapping. + This is useful when you want to ensure that a model's predicted labels are aligned with the dataset. + The alignment in done using the lowercase label names. + + Args: + label2id (`dict`): + The label name to ID mapping to align the dataset with. + label_column (`str`): + The column name of labels to align on. + + Example: + + ```python + >>> # dataset with mapping {'entailment': 0, 'neutral': 1, 'contradiction': 2} + >>> ds = load_dataset("nyu-mll/glue", "mnli", split="train") + >>> # mapping to align with + >>> label2id = {'CONTRADICTION': 0, 'NEUTRAL': 1, 'ENTAILMENT': 2} + >>> ds_aligned = ds.align_labels_with_mapping(label2id, "label") + ``` + + """ + # Sanity checks + if label_column not in self._data.column_names: + raise ValueError(f"Column ({label_column}) not in table columns ({self._data.column_names}).") + + label_feature = self._info.features[label_column] + if not ( + isinstance(label_feature, ClassLabel) + or (isinstance(label_feature, Sequence) and isinstance(label_feature.feature, ClassLabel)) + ): + raise ValueError( + f"Aligning labels with a mapping is only supported for {ClassLabel.__name__} column or {Sequence.__name__} column with the inner type {ClassLabel.__name__}, and column {label_feature} is of type {type(label_feature).__name__}." + ) + + # Sort input mapping by ID value to ensure the label names are aligned + label2id = dict(sorted(label2id.items(), key=lambda item: item[1])) + label_names = list(label2id.keys()) + # Some label mappings use uppercase label names so we lowercase them during alignment + label2id = {k.lower(): v for k, v in label2id.items()} + int2str_function = ( + label_feature.int2str if isinstance(label_feature, ClassLabel) else label_feature.feature.int2str + ) + + if isinstance(label_feature, ClassLabel): + + def process_label_ids(batch): + dset_label_names = [ + int2str_function(label_id).lower() if label_id is not None else None + for label_id in batch[label_column] + ] + batch[label_column] = [ + label2id[label_name] if label_name is not None else None for label_name in dset_label_names + ] + return batch + + else: + + def process_label_ids(batch): + dset_label_names = [ + [int2str_function(label_id).lower() if label_id is not None else None for label_id in seq] + for seq in batch[label_column] + ] + batch[label_column] = [ + [label2id[label_name] if label_name is not None else None for label_name in seq] + for seq in dset_label_names + ] + return batch + + features = self.features + features[label_column] = ( + ClassLabel(num_classes=len(label_names), names=label_names) + if isinstance(label_feature, ClassLabel) + else List(ClassLabel(num_classes=len(label_names), names=label_names)) + ) + return self.map(process_label_ids, features=features, batched=True, desc="Aligning the labels") + + +def _push_to_repo( + dset: Union["Dataset", "IterableDataset"], + repo_id: str, + config_name: str = "default", + set_default: Optional[bool] = None, + split: Optional[str] = None, + data_dir: Optional[str] = None, + commit_message: Optional[str] = None, + commit_description: Optional[str] = None, + token: Optional[str] = None, + revision: Optional[str] = None, + create_pr: Optional[bool] = False, + max_shard_size: Optional[Union[int, str]] = None, + num_shards: Optional[int] = None, + embed_external_files: bool = True, + num_proc: Optional[int] = None, +) -> CommitInfo: + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + resolved_output_path = HfFileSystemResolvedRepositoryPath( + repo_id=repo_id, repo_type="dataset", revision=revision or "main", path_in_repo="" + ) + + additions, new_parquet_paths, features, split_info, uploaded_size = dset._push_parquet_shards_to_hub( + resolved_output_path=resolved_output_path, + data_dir=data_dir, + split=split, + token=token, + max_shard_size=max_shard_size, + num_shards=num_shards, + create_pr=create_pr, + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + + commit_message = commit_message if commit_message is not None else "Upload dataset" + if len(additions) > config.UPLOADS_MAX_NUMBER_PER_COMMIT: + logger.info( + f"Number of files to upload is larger than {config.UPLOADS_MAX_NUMBER_PER_COMMIT}. Splitting the push into multiple commits." + ) + num_commits = math.ceil(len(additions) / config.UPLOADS_MAX_NUMBER_PER_COMMIT) + for i in range(0, num_commits): + operations = additions[ + i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT + ] + for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1): + # We need to retry if another commit happens at the same time + sleep_time *= 1 + random.random() + try: + commit_info = api.create_commit( + repo_id, + operations=operations, + commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})", + commit_description=commit_description, + repo_type="dataset", + revision=revision, + create_pr=create_pr, + ) + except HfHubHTTPError as err: + if ( + err.__context__ + and isinstance(err.__context__, HfHubHTTPError) + and err.__context__.response.status_code == 409 + ): + # 409 is Conflict (another commit is in progress) + time.sleep(sleep_time) + logger.info( + f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})" + ) + continue + else: + raise + break + logger.info( + f"Commit #{i + 1} completed" + + (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "") + + "." + ) + last_commit_additions = [] + else: + last_commit_additions = additions + + for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1): + # We need to retry if there was a commit in between in case it touched the dataset card data + sleep_time *= 1 + random.random() + + # We make sure to get info from this commit + parent_commit = api.repo_info(repo_id, repo_type="dataset", revision=revision).sha + hf_path = HfFileSystemResolvedRepositoryPath( + repo_type=resolved_output_path.repo_type, + repo_id=resolved_output_path.repo_id, + revision=parent_commit, + path_in_repo=resolved_output_path.path_in_repo, + ).unresolve() + hffs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token) + dirfs = DirFileSystem(fs=hffs, path=hf_path) + + # Check the files to delete + try: + files_to_delete = dirfs.glob(f"{data_dir}/{split}-*", detail=True) + except EntryNotFoundError: # needed for huggingface_hub<=1.7.1 + files_to_delete = {} + + # Don't delete the new files + deletions = [ + CommitOperationDelete(path_in_repo=file_to_delete) + for file_to_delete in files_to_delete + if file_to_delete not in new_parquet_paths + ] + deleted_size = sum(file_info["size"] for file_info in files_to_delete.values()) + + # Update the dataset card + new_dataset_card, new_legacy_dataset_infos = _get_updated_dataset_card( + fs=dirfs, + config_name=config_name, + splits_info=[split_info], + features=features, + data_dir=data_dir, + set_default=set_default, + uploaded_sizes=[uploaded_size], + deleted_sizes=[deleted_size], + remove_other_splits=False, + ) + dataset_card_additions = [ + CommitOperationAdd(path_in_repo=config.REPOCARD_FILENAME, path_or_fileobj=str(new_dataset_card).encode()) + ] + if new_legacy_dataset_infos: + dataset_card_additions.append( + CommitOperationAdd( + path_in_repo=config.DATASETDICT_INFOS_FILENAME, + path_or_fileobj=json.dumps(new_legacy_dataset_infos).encode("utf-8"), + ) + ) + operations = last_commit_additions + dataset_card_additions + deletions + + try: + commit_info = api.create_commit( + repo_id, + operations=operations, + commit_message=commit_message, + commit_description=commit_description, + repo_type="dataset", + revision=revision, + create_pr=create_pr, + parent_commit=parent_commit, + ) + except HfHubHTTPError as err: + if ( + err.__context__ + and isinstance(err.__context__, HfHubHTTPError) + and err.__context__.response.status_code in (412, 409) + ): + # 412 is Precondition failed (parent_commit isn't satisfied) + # 409 is Conflict (another commit is in progress) + time.sleep(sleep_time) + logger.info( + f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})" + ) + continue + else: + raise + break + + return commit_info + + +def _push_to_bucket( + dset: Union["Dataset", "IterableDataset"], + bucket_id: str, + path: str, + config_name: str = "default", + set_default: Optional[bool] = None, + split: Optional[str] = None, + data_dir: Optional[str] = None, + token: Optional[str] = None, + create_pr: Optional[bool] = False, + max_shard_size: Optional[Union[int, str]] = None, + num_shards: Optional[int] = None, + embed_external_files: bool = True, + num_proc: Optional[int] = None, +) -> None: + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + resolved_output_path = HfFileSystemResolvedBucketPath(bucket_id=bucket_id, path=path) + hf_path = resolved_output_path.unresolve() + hffs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token) + dirfs = DirFileSystem(fs=hffs, path=hf_path) + + # Check the files to delete before uploading + try: + files_to_delete = dirfs.glob(f"{data_dir}/{split}-*", detail=True) + except EntryNotFoundError: # needed for huggingface_hub<=1.7.1 + files_to_delete = {} + + # Upload the Parquet files + _, new_parquet_paths, features, split_info, uploaded_size = dset._push_parquet_shards_to_hub( + resolved_output_path=resolved_output_path, + data_dir=data_dir, + split=split, + token=token, + max_shard_size=max_shard_size, + num_shards=num_shards, + create_pr=create_pr, + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + + # Don't delete the new files + new_parquet_paths = set(new_parquet_paths) + delete = [file_to_delete for file_to_delete in files_to_delete if file_to_delete not in new_parquet_paths] + deleted_size = sum(file_info["size"] for file_info in files_to_delete.values()) + + # Update the dataset card + new_dataset_card, new_legacy_dataset_infos = _get_updated_dataset_card( + fs=dirfs, + config_name=config_name, + splits_info=[split_info], + features=features, + data_dir=data_dir, + set_default=set_default, + uploaded_sizes=[uploaded_size], + deleted_sizes=[deleted_size], + remove_other_splits=False, + ) + path_prefix = (path + "/") if path else "" + add = [(str(new_dataset_card).encode(), path_prefix + config.REPOCARD_FILENAME)] + if new_legacy_dataset_infos: + add.append( + (json.dumps(new_legacy_dataset_infos).encode("utf-8"), path_prefix + config.DATASETDICT_INFOS_FILENAME) + ) + + # Upload dataset card and delete old files + api.batch_bucket_files( + bucket_id=bucket_id, + add=add, + delete=delete, + ) + + +def _get_updated_dataset_card( + fs: DirFileSystem, + config_name: str, + splits_info: list[SplitInfo], + features: Features, + data_dir: str, + set_default: Optional[bool], + uploaded_sizes: list[int], + deleted_sizes: list[int], + remove_other_splits: bool, +) -> tuple[DatasetCard, Optional[dict]]: + """Update a dataset card in push_to_hub""" + # get the deprecated dataset_infos.json to update them + try: + legacy_dataset_info: dict = json.loads(fs.read_text(config.DATASETDICT_INFOS_FILENAME, encoding="utf-8")).get( + config_name, None + ) + repo_info = DatasetInfo.from_dict(legacy_dataset_info) if legacy_dataset_info else None + except FileNotFoundError: + legacy_dataset_info = None + repo_info = None + # get the info from the README to update them + try: + dataset_card = DatasetCard(fs.read_text(config.REPOCARD_FILENAME, newline="", encoding="utf-8")) + dataset_card_data = dataset_card.data + metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data) + dataset_infos: DatasetInfosDict = DatasetInfosDict.from_dataset_card_data(dataset_card_data) + if dataset_infos and config_name in dataset_infos: + repo_info = dataset_infos[config_name] + else: + repo_info = None + except FileNotFoundError: + dataset_card = None + dataset_card_data = DatasetCardData() + metadata_configs = MetadataConfigs() + # update the total info to dump from existing info + if repo_info is not None and not remove_other_splits: + logger.info("Updating downloaded metadata with the new split" + ("s." if len(splits_info) > 1 else ".")) + for split_info, deleted_size, uploaded_size in zip(splits_info, deleted_sizes, uploaded_sizes): + split = split_info.name + if repo_info.splits and any(s != split for s in repo_info.splits): + if features != repo_info.features: + raise ValueError( + f"Features of the new split don't match the features of the existing splits on the hub: {features} != {repo_info.features}" + ) + + if split in repo_info.splits: + repo_info.download_size -= deleted_size + repo_info.dataset_size -= repo_info.splits.get(split, SplitInfo()).num_bytes or 0 + + repo_info.download_checksums = None + repo_info.download_size = (repo_info.download_size or 0) + uploaded_size + repo_info.dataset_size = (repo_info.dataset_size or 0) + split_info.num_bytes + repo_info.size_in_bytes = repo_info.download_size + repo_info.dataset_size + repo_info.splits.pop(split, None) + repo_info.splits[split] = split_info + info_to_dump = repo_info + else: + info_to_dump = DatasetInfo( + config_name=config_name, features=features, splits=SplitDict(), download_size=0, dataset_size=0 + ) + for split_info, uploaded_size in zip(splits_info, uploaded_sizes): + info_to_dump.splits.add(split_info) + info_to_dump.download_size += uploaded_size + info_to_dump.dataset_size += split_info.num_bytes + info_to_dump.size_in_bytes = info_to_dump.download_size + info_to_dump.dataset_size + # create the metadata configs if it was uploaded with push_to_hub before metadata configs existed + repo_splits: list[str] = [] + pattern = glob_pattern_to_regex(PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED) + for file_path in fs.glob(PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED.replace("{split}", "*")): + split_pattern_fields = string_to_dict(file_path, pattern) + assert split_pattern_fields is not None + repo_split = split_pattern_fields["split"] + if repo_split not in repo_splits: + repo_splits.append(repo_split) + if not metadata_configs and repo_splits: + default_metadata_configs_to_dump = { + "data_files": [{"split": split, "path": f"data/{split}-*"} for split in repo_splits] + } + MetadataConfigs({"default": default_metadata_configs_to_dump}).to_dataset_card_data(dataset_card_data) + # update the metadata configs + if config_name in metadata_configs: + metadata_config = metadata_configs[config_name] + if "data_files" in metadata_config: + data_files_to_dump = sanitize_patterns(metadata_config["data_files"]) + else: + data_files_to_dump = {} + # add the new splits + for split_info in splits_info: + split = split_info.name + data_files_to_dump[split] = [f"{data_dir}/{split}-*"] + metadata_config_to_dump = { + "data_files": [ + { + "split": _split, + "path": _pattern[0] if len(_pattern) == 1 else _pattern, + } + for _split, _pattern in data_files_to_dump.items() + ] + } + else: + metadata_config_to_dump = { + "data_files": [ + {"split": split_info.name, "path": f"{data_dir}/{split_info.name}-*"} for split_info in splits_info + ] + } + configs_to_dump = {config_name: metadata_config_to_dump} + if set_default and config_name != "default": + if metadata_configs: + current_default_config_name = metadata_configs.get_default_config_name() + if current_default_config_name == "default": + raise ValueError( + "There exists a configuration named 'default'. To set a different configuration as default, " + "rename the 'default' one first." + ) + if current_default_config_name: + _ = metadata_configs[current_default_config_name].pop("default") + configs_to_dump[current_default_config_name] = metadata_configs[current_default_config_name] + metadata_config_to_dump["default"] = True + # push to the deprecated dataset_infos.json + if legacy_dataset_info: + legacy_dataset_infos: dict = json.loads(fs.read_text(config.DATASETDICT_INFOS_FILENAME, encoding="utf-8")) + legacy_dataset_infos[config_name] = asdict(info_to_dump) + new_legacy_dataset_infos = json.dumps(dataset_infos, indent=4) + else: + new_legacy_dataset_infos = None + # push to README + DatasetInfosDict({config_name: info_to_dump}).to_dataset_card_data(dataset_card_data) + MetadataConfigs(configs_to_dump).to_dataset_card_data(dataset_card_data) + new_dataset_card = DatasetCard(f"---\n{dataset_card_data}\n---\n") if dataset_card is None else dataset_card + return new_dataset_card, new_legacy_dataset_infos + + +def _concatenate_map_style_datasets( + dsets: list[Dataset], + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + axis: int = 0, +): + """ + Converts a list of :class:`Dataset` with the same schema into a single :class:`Dataset`. + When you concatenate on axis 0, missing data are filled with None values. + + Args: + dsets (`List[datasets.Dataset]`): List of Datasets to concatenate. + info (:class:`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (:class:`NamedSplit`, optional): Name of the dataset split. + axis (``{0, 1}``, default ``0``, meaning over rows): + Axis to concatenate over, where ``0`` means over rows (vertically) and ``1`` means over columns + (horizontally). + + *New in version 1.6.0* + + Example: + + ```py + >>> ds3 = _concatenate_map_style_datasets([ds1, ds2]) + ``` + """ + # Ignore datasets with no rows + if any(dset.num_rows > 0 for dset in dsets): + dsets = [dset for dset in dsets if dset.num_rows > 0] + else: + # Return first dataset if all datasets are empty + return dsets[0] + + # Perform checks (and a potential cast if axis=0) + if axis == 0: + _check_if_features_can_be_aligned([dset.features for dset in dsets]) + else: + if not all(dset.num_rows == dsets[0].num_rows for dset in dsets): + raise ValueError("Number of rows must match for all datasets") + _check_column_names([col_name for dset in dsets for col_name in dset._data.column_names]) + + # Find common format or reset format + format = dsets[0].format + if any(dset.format != format for dset in dsets): + format = {} + logger.info("Some of the datasets have disparate format. Resetting the format of the concatenated dataset.") + + def apply_offset_to_indices_table(table, offset): + if offset == 0: + return table + else: + array = table["indices"] + new_array = pc.add(array, pa.scalar(offset, type=pa.uint64())) + return InMemoryTable.from_arrays([new_array], names=["indices"]) + + # Concatenate indices if they exist + if any(dset._indices is not None for dset in dsets): + if axis == 0: + # Datasets with no indices tables are replaced with a dataset with an indices table in memory. + # Applying an offset to an indices table also brings the table in memory. + indices_tables = [] + for i in range(len(dsets)): + if dsets[i]._indices is None: + dsets[i] = dsets[i]._select_with_indices_mapping(range(len(dsets[i]))) + indices_tables.append(dsets[i]._indices) + + # An offset needs to be applied to the indices before concatenating + offset = 0 + for i in range(len(dsets)): + indices_tables[i] = apply_offset_to_indices_table(indices_tables[i], offset) + offset += len(dsets[i]._data) + + # Concatenate indices + indices_tables = [t for t in indices_tables if len(t) > 0] + if indices_tables: + indices_table = concat_tables(indices_tables) + else: + indices_table = InMemoryTable.from_batches([], schema=pa.schema({"indices": pa.int64()})) + else: + if len(dsets) == 1: + indices_table = dsets[0]._indices + else: + for i in range(len(dsets)): + dsets[i] = dsets[i].flatten_indices() + indices_table = None + else: + indices_table = None + + table = concat_tables([dset._data for dset in dsets], axis=axis) + if axis == 0: + features_list = _align_features([dset.features for dset in dsets]) + else: + features_list = [dset.features for dset in dsets] + table = update_metadata_with_features(table, {k: v for features in features_list for k, v in features.items()}) + + # Concatenate infos + if info is None: + info = DatasetInfo.from_merge([dset.info for dset in dsets]) + fingerprint = update_fingerprint( + "".join(dset._fingerprint for dset in dsets), _concatenate_map_style_datasets, {"info": info, "split": split} + ) + + # Make final concatenated dataset + concatenated_dataset = Dataset( + table, + info=info, + split=split, + indices_table=indices_table, + fingerprint=fingerprint, + ) + concatenated_dataset.set_format(**format) + return concatenated_dataset + + +def _interleave_map_style_datasets( + datasets: list["Dataset"], + probabilities: Optional[list[float]] = None, + seed: Optional[int] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + stopping_strategy: Literal[ + "first_exhausted", "all_exhausted", "all_exhausted_without_replacement" + ] = "first_exhausted", + **kwargs, +) -> "Dataset": + """ + Interleave several map-style datasets (sources) into a single map-style dataset. + The new dataset is constructed by alternating between the sources to get the examples. + If `probabilities = None` (default) the new dataset is constructed by cycling between each source to get the examples. + If `probabilities` is not `None, the new dataset is constructed by getting examples from a random source at a time according to the provided probabilities. + + Args: + datasets (`List[Dataset]`): list of datasets to interleave + probabilities (`List[float]`, optional, default None): If specified, the new dataset is constructed by sampling + examples from one source at a time according to these probabilities. + seed (`int`, optional, default None): The random seed used to choose a source for each example. + info (:class:`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (:class:`NamedSplit`, optional): Name of the dataset split. + stopping_strategy (`str`, defaults to `first_exhausted`): + Two strategies are proposed right now. + By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples. + If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once. + When strategy is `all_exhausted_without_replacement` we make sure that each sample in each dataset is sampled only once. + Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous: + - with no probabilities, the resulting dataset will have max_length_datasets*nb_dataset samples. + - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting. + **kwargs (additional keyword arguments): Keyword arguments to be passed to :meth:`datasets.Datasets.select` when selecting the indices used to interleave the datasets. + + Output: + :class:`datasets.Dataset` + """ + if stopping_strategy not in ["first_exhausted", "all_exhausted", "all_exhausted_without_replacement"]: + raise ValueError( + f"{stopping_strategy} stopping strategy in `interleave_datasets` is not implemented yet with a list of {type(datasets[0])}" + ) + + # To interleave the datasets, we concatenate them and then we re-order the indices + concatenated_datasets = _concatenate_map_style_datasets(datasets, info=info, split=split) + + # Let's now build the indices to pass to .select() + lengths = [len(dset) for dset in datasets] + offsets = np.cumsum([0] + lengths[:-1]) + + # if stopping_strategy is "first_exhausted", it is an undersampling situation whereas it is an oversampling situation if it is "all_exhausted" + oversampling = stopping_strategy == "all_exhausted" + + if probabilities is None and stopping_strategy == "all_exhausted_without_replacement": + # Without replacement situation with cycling between each sources + # Example: If lengths of the datasets are [3, 4, 3] + # Then the resulting indices should be [0, 3, 7, 1, 4, 8, 2, 5, 9, 6] + # We cycle through datasets until all are exhausted, but skip exhausted datasets + + # Reasoning behind the following operation: keeping the first indices of each dataset + # while offsetting in order to correspond to the right indices of the concatenated dataset + # and flattening to effectively interleave the datasets. Then we remove the exausted datasets + # and we continue with the following indices, until all datasets are exhausted + chunks_boundaries = [0] + sorted(set(lengths)) + chunks = zip(chunks_boundaries[:-1], chunks_boundaries[1:]) + indices_chunks = [] + for start, end in chunks: + indices_chunks.append((np.array(offsets).reshape(1, -1) + np.arange(start, end).reshape(-1, 1)).flatten()) + exhausted_indices = [i for i in range(len(lengths)) if lengths[i] == end] + lengths = np.delete(lengths, exhausted_indices).tolist() + offsets = np.delete(offsets, exhausted_indices) + indices = np.concatenate(indices_chunks).tolist() + + elif probabilities is None and not oversampling: + # Undersampling situation with cycling between each sources + # Example:: If lengths of the datasets are [3, 4, 5] + # Then the resulting indices should be [0, 3, 7, 1, 4, 8, 2, 5, 9] + # Note that we only have 3 examples per dataset since the first dataset ran out of examples + + # Reasoning behind the following operation: keeping the min_length first indices of each dataset + # while offsetting in order to correspond to the right indices of the concatenated dataset + # and flattening to effectively interleave the datasets + indices = (offsets.reshape(1, -1) + np.arange(min(lengths)).reshape(-1, 1)).flatten().tolist() + elif probabilities is None: + # Oversampling situation with cycling between each sources + # Then the resulting indices should be [0, 3, 7, 1, 4, 8, 2, 5, 9, 0, 6, 10, 1, 3, 11] + # Note that we have 5 examples per dataset with a rolling window since the longest dataset has 5 samples + + # Reasoning behind the following operation: for each dataset indices (i.e column) repeat the indices to have max_length indices per dataset + # For example, if the max_length is 5 and the i-th dataset has 3 samples, the i-th column will be [0,1,2,0,1] + indices = np.mod(np.arange(max(lengths)).reshape(-1, 1), np.array(lengths).reshape(1, -1)) + + # We have to keep the indices to their respective dataset offsets and to flatten to effectively interleave the datasets + indices = (indices + offsets).flatten().tolist() + + else: + # boolean array indicating if at index i if the dataset_i has been fully exhausted + is_exhausted = np.full(len(lengths), False) + + # if undersampling ("first_exhausted"), we stop as soon as one dataset is exhausted + # if oversampling ("all_exhausted"), we stop as soons as every dataset is exhausted, i.e as soon as every samples of every dataset has been visited at least once + bool_strategy_func = ( + np.all if (oversampling or stopping_strategy == "all_exhausted_without_replacement") else np.any + ) + + def iter_random_indices(): + """Get an infinite iterator that randomly samples the index of the source to pick examples from.""" + rng = np.random.default_rng(seed) + while True: + yield from (int(i) for i in rng.choice(len(datasets), size=1000, p=probabilities)) + + current_index = [0] * len(datasets) + indices = [] + for source_idx in iter_random_indices(): + # If no oversampling, we stop as soon as a dataset has ran out of examples (np.any) + # Otherwise, we stop as soon as every dataset has ran out of examples (np.all) + if bool_strategy_func(is_exhausted): + # the stopping condition was reached, let's stop + break + + # let's add the example at the current index of the `source_idx`-th dataset + # For without replacement sampling we additionally need to make sure the current source is not exhausted to not oversample. + if stopping_strategy != "all_exhausted_without_replacement" or not is_exhausted[source_idx]: + indices.append(current_index[source_idx] + offsets[source_idx]) + current_index[source_idx] += 1 + + # we've ran out of examples for the current dataset, let's update our boolean array and bring the current_index back to 0 + if current_index[source_idx] >= lengths[source_idx]: + is_exhausted[source_idx] = True + # We don't want to reset the iterator when stopping strategy is without replacement. + if stopping_strategy != "all_exhausted_without_replacement": + current_index[source_idx] = 0 + + return concatenated_datasets.select(indices, **kwargs) + + +def _split_by_node_map_style_dataset(dataset: Dataset, rank: int, world_size: int) -> Dataset: + """ + Split a dataset for the node at rank `rank` in a pool of nodes of size `world_size`. + Each node is assigned a chunk of data, e.g. rank 0 is given the first chunk of the dataset. + To maximize data loading throughput, chunks are made of contiguous data on disk if possible. + + Args: + dataset ([`Dataset`]): + The dataset to split by node. + rank (`int`): + Rank of the current node. + world_size (`int`): + Total number of nodes. + + Returns: + [`Dataset`]: The dataset to be used on the node at rank `rank`. + """ + return dataset.shard(num_shards=world_size, index=rank, contiguous=True) + + +# This is outside Dataset.filter as it needs to be picklable for multiprocessing + + +def get_indices_from_mask_function( + function: Callable, + batched: bool, + with_indices: bool, + with_rank: bool, + input_columns: Optional[Union[str, list[str]]], + indices_mapping: Optional[Table] = None, + *args, + **fn_kwargs, +): + if batched: + # we extract indices and rank from args + *inputs, indices, rank = args + additional_args = () + if with_indices: + additional_args += (indices,) + if with_rank: + additional_args += (rank,) + mask = function(*inputs, *additional_args, **fn_kwargs) + if isinstance(mask, (pa.Array, pa.ChunkedArray)): + mask = mask.to_pylist() + else: + # we get batched data (to return less data than input) but `function` only accepts one example + # therefore we need to call `function` on each example of the batch to get the mask + *inputs, indices, rank = args + mask = [] + if input_columns is None: + # inputs only contains a batch of examples + batch: dict = inputs[0] + num_examples = len(batch[next(iter(batch.keys()))]) + for i in range(num_examples): + example = {key: batch[key][i] for key in batch} + additional_args = () + if with_indices: + additional_args += (indices[i],) + if with_rank: + additional_args += (rank,) + mask.append(function(example, *additional_args, **fn_kwargs)) + else: + # inputs is a list of columns + columns: list[list] = inputs + num_examples = len(columns[0]) + for i in range(num_examples): + input = [column[i] for column in columns] + additional_args = () + if with_indices: + additional_args += (indices[i],) + if with_rank: + additional_args += (rank,) + mask.append(function(*input, *additional_args, **fn_kwargs)) + indices_array = [i for i, to_keep in zip(indices, mask) if to_keep] + if indices_mapping is not None: + indices_array = pa.array(indices_array, type=pa.uint64()) + indices_array = indices_mapping.column(0).take(indices_array) + indices_array = indices_array.to_pylist() + return {"indices": indices_array} + + +async def async_get_indices_from_mask_function( + function: Callable, + batched: bool, + with_indices: bool, + with_rank: bool, + input_columns: Optional[Union[str, list[str]]], + indices_mapping: Optional[Table] = None, + *args, + **fn_kwargs, +): + """same function but async""" + if batched: + # we extract indices and rank from args + *inputs, indices, rank = args + additional_args = () + if with_indices: + additional_args += (indices,) + if with_rank: + additional_args += (rank,) + mask = await function(*inputs, *additional_args, **fn_kwargs) + if isinstance(mask, (pa.Array, pa.ChunkedArray)): + mask = mask.to_pylist() + else: + # we get batched data (to return less data than input) but `function` only accepts one example + # therefore we need to call `function` on each example of the batch to get the mask + *inputs, indices, rank = args + mask = [] + if input_columns is None: + # inputs only contains a batch of examples + batch: dict = inputs[0] + num_examples = len(batch[next(iter(batch.keys()))]) + for i in range(num_examples): + example = {key: batch[key][i] for key in batch} + additional_args = () + if with_indices: + additional_args += (indices[i],) + if with_rank: + additional_args += (rank,) + mask.append(await function(example, *additional_args, **fn_kwargs)) + else: + # inputs is a list of columns + columns: list[list] = inputs + num_examples = len(columns[0]) + for i in range(num_examples): + input = [column[i] for column in columns] + additional_args = () + if with_indices: + additional_args += (indices[i],) + if with_rank: + additional_args += (rank,) + mask.append(await function(*input, *additional_args, **fn_kwargs)) + indices_array = [i for i, to_keep in zip(indices, mask) if to_keep] + if indices_mapping is not None: + indices_array = pa.array(indices_array, type=pa.uint64()) + indices_array = indices_mapping.column(0).take(indices_array) + indices_array = indices_array.to_pylist() + return {"indices": indices_array} diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/arrow_reader.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/arrow_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..d9cf2cf0f4bd239a8f67dd76a279a4e81c26d0d6 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/arrow_reader.py @@ -0,0 +1,620 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Arrow ArrowReader.""" + +import copy +import math +import os +import re +from dataclasses import dataclass +from functools import partial +from typing import TYPE_CHECKING, Optional, Union + +import pyarrow as pa +import pyarrow.parquet as pq +from tqdm.contrib.concurrent import thread_map + +from .download.download_config import DownloadConfig # noqa: F401 +from .naming import _split_re, filenames_for_dataset_split +from .table import InMemoryTable, MemoryMappedTable, Table, concat_tables +from .utils import logging +from .utils import tqdm as hf_tqdm + + +if TYPE_CHECKING: + from .info import DatasetInfo # noqa: F401 + from .splits import Split, SplitInfo # noqa: F401 + + +logger = logging.get_logger(__name__) + +HF_GCP_BASE_URL = "https://storage.googleapis.com/huggingface-nlp/cache/datasets" + +_SUB_SPEC_RE = re.compile( + rf""" +^ + (?P{_split_re[1:-1]}) + (\[ + ((?P-?[\d_]+) + (?P%)?)? + : + ((?P-?[\d_]+) + (?P%)?)? + \])?(\((?P[^\)]*)\))? +$ +""", # remove ^ and $ + re.X, +) + +_ADDITION_SEP_RE = re.compile(r"\s*\+\s*") + + +class DatasetNotOnHfGcsError(ConnectionError): + """When you can't get the dataset from the Hf google cloud storage""" + + pass + + +class MissingFilesOnHfGcsError(ConnectionError): + """When some files are missing on the Hf oogle cloud storage""" + + pass + + +@dataclass(frozen=True) +class FileInstructions: + """The file instructions associated with a split ReadInstruction. + + Attributes: + num_examples: `int`, The total number of examples + file_instructions: List[dict(filename, skip, take)], the files information. + The filenames contains the relative path, not absolute. + skip/take indicates which example read in the file: `ds.slice(skip, take)` + """ + + num_examples: int + file_instructions: list[dict] + + +def make_file_instructions( + name: str, + split_infos: list["SplitInfo"], + instruction: Union[str, "ReadInstruction"], + filetype_suffix: Optional[str] = None, + prefix_path: Optional[str] = None, +) -> FileInstructions: + """Returns instructions of the split dict. + + Args: + name (`str`): Name of the dataset. + split_infos (`list` of `[SplitInfo]`): Dataset splits information. + instruction ([`ReadInstruction`] or `str`): Reading instruction for a dataset. + filetype_suffix (`str`, *optional*): Suffix of dataset files, e.g. 'arrow' or 'parquet'. + prefix_path (`str`, *optional*): Prefix of dataset files, e.g. directory name. + + Returns: + [`FileInstructions`] + """ + if not isinstance(name, str): + raise TypeError(f"Expected str 'name', but got: {type(name).__name__}") + elif not name: + raise ValueError("Expected non-empty str 'name'") + name2len = {info.name: info.num_examples for info in split_infos} + name2shard_lengths = {info.name: info.shard_lengths for info in split_infos} + name2filenames = { + info.name: filenames_for_dataset_split( + path=prefix_path, + dataset_name=name, + split=info.name, + filetype_suffix=filetype_suffix, + shard_lengths=name2shard_lengths[info.name], + ) + for info in split_infos + } + if not isinstance(instruction, ReadInstruction): + instruction = ReadInstruction.from_spec(instruction) + # Create the absolute instruction (per split) + absolute_instructions = instruction.to_absolute(name2len) + + # For each split, return the files instruction (skip/take) + file_instructions = [] + num_examples = 0 + for abs_instr in absolute_instructions: + split_length = name2len[abs_instr.splitname] + filenames = name2filenames[abs_instr.splitname] + shard_lengths = name2shard_lengths[abs_instr.splitname] + from_ = 0 if abs_instr.from_ is None else abs_instr.from_ + to = split_length if abs_instr.to is None else abs_instr.to + if shard_lengths is None: # not sharded + for filename in filenames: + take = to - from_ + if take == 0: + continue + num_examples += take + file_instructions.append({"filename": filename, "skip": from_, "take": take}) + else: # sharded + index_start = 0 # Beginning (included) of moving window. + index_end = 0 # End (excluded) of moving window. + for filename, shard_length in zip(filenames, shard_lengths): + index_end += shard_length + if from_ < index_end and to > index_start: # There is something to take. + skip = from_ - index_start if from_ > index_start else 0 + take = to - index_start - skip if to < index_end else -1 + if take == 0: + continue + file_instructions.append({"filename": filename, "skip": skip, "take": take}) + num_examples += shard_length - skip if take == -1 else take + index_start += shard_length + return FileInstructions( + num_examples=num_examples, + file_instructions=file_instructions, + ) + + +class BaseReader: + """ + Build a Dataset object out of Instruction instance(s). + """ + + def __init__(self, path: str, info: Optional["DatasetInfo"]): + """Initializes ArrowReader. + + Args: + path (str): path where tfrecords are stored. + info (DatasetInfo): info about the dataset. + """ + self._path: str = path + self._info: Optional["DatasetInfo"] = info + self._filetype_suffix: Optional[str] = None + + def _get_table_from_filename(self, filename_skip_take, in_memory=False) -> Table: + """Returns a Dataset instance from given (filename, skip, take).""" + raise NotImplementedError + + def _read_files(self, files, in_memory=False) -> Table: + """Returns Dataset for given file instructions. + + Args: + files: List[dict(filename, skip, take)], the files information. + The filenames contain the absolute path, not relative. + skip/take indicates which example read in the file: `ds.slice(skip, take)` + in_memory (bool, default False): Whether to copy the data in-memory. + """ + if len(files) == 0 or not all(isinstance(f, dict) for f in files): + raise ValueError("please provide valid file informations") + files = copy.deepcopy(files) + for f in files: + f["filename"] = os.path.join(self._path, f["filename"]) + + pa_tables = thread_map( + partial(self._get_table_from_filename, in_memory=in_memory), + files, + tqdm_class=hf_tqdm, + desc="Loading dataset shards", + # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached + disable=len(files) <= 16 or None, + ) + pa_tables = [t for t in pa_tables if len(t) > 0] + if not pa_tables and (self._info is None or self._info.features is None): + raise ValueError( + "Tried to read an empty table. Please specify at least info.features to create an empty table with the right type." + ) + pa_tables = pa_tables or [InMemoryTable.from_batches([], schema=pa.schema(self._info.features.type))] + pa_table = concat_tables(pa_tables) if len(pa_tables) != 1 else pa_tables[0] + return pa_table + + def get_file_instructions(self, name, instruction, split_infos): + """Return list of dict {'filename': str, 'skip': int, 'take': int}""" + file_instructions = make_file_instructions( + name, split_infos, instruction, filetype_suffix=self._filetype_suffix, prefix_path=self._path + ) + files = file_instructions.file_instructions + return files + + def read( + self, + name, + instructions, + split_infos, + in_memory=False, + ): + """Returns Dataset instance(s). + + Args: + name (str): name of the dataset. + instructions (ReadInstruction): instructions to read. + Instruction can be string and will then be passed to the Instruction + constructor as it. + split_infos (list of SplitInfo proto): the available splits for dataset. + in_memory (bool, default False): Whether to copy the data in-memory. + + Returns: + kwargs to build a single Dataset instance. + """ + + files = self.get_file_instructions(name, instructions, split_infos) + if not files: + msg = f'Instruction "{instructions}" corresponds to no data!' + raise ValueError(msg) + return self.read_files(files=files, original_instructions=instructions, in_memory=in_memory) + + def read_files( + self, + files: list[dict], + original_instructions: Union[None, "ReadInstruction", "Split"] = None, + in_memory=False, + ): + """Returns single Dataset instance for the set of file instructions. + + Args: + files: List[dict(filename, skip, take)], the files information. + The filenames contains the relative path, not absolute. + skip/take indicates which example read in the file: `ds.skip().take()` + original_instructions: store the original instructions used to build the dataset split in the dataset. + in_memory (bool, default False): Whether to copy the data in-memory. + + Returns: + kwargs to build a Dataset instance. + """ + # Prepend path to filename + pa_table = self._read_files(files, in_memory=in_memory) + # If original_instructions is not None, convert it to a human-readable NamedSplit + if original_instructions is not None: + from .splits import Split # noqa + + split = Split(str(original_instructions)) + else: + split = None + dataset_kwargs = {"arrow_table": pa_table, "info": self._info, "split": split} + return dataset_kwargs + + +class ArrowReader(BaseReader): + """ + Build a Dataset object out of Instruction instance(s). + This Reader uses either memory mapping or file descriptors (in-memory) on arrow files. + """ + + def __init__(self, path: str, info: Optional["DatasetInfo"]): + """Initializes ArrowReader. + + Args: + path (str): path where Arrow files are stored. + info (DatasetInfo): info about the dataset. + """ + super().__init__(path, info) + self._filetype_suffix = "arrow" + + def _get_table_from_filename(self, filename_skip_take, in_memory=False) -> Table: + """Returns a Dataset instance from given (filename, skip, take).""" + filename, skip, take = ( + filename_skip_take["filename"], + filename_skip_take["skip"] if "skip" in filename_skip_take else None, + filename_skip_take["take"] if "take" in filename_skip_take else None, + ) + table = ArrowReader.read_table(filename, in_memory=in_memory) + if take == -1: + take = len(table) - skip + # here we don't want to slice an empty table, or it may segfault + if skip is not None and take is not None and not (skip == 0 and take == len(table)): + table = table.slice(skip, take) + return table + + @staticmethod + def read_table(filename, in_memory=False) -> Table: + """ + Read table from file. + + Args: + filename (str): File name of the table. + in_memory (bool, default=False): Whether to copy the data in-memory. + + Returns: + pyarrow.Table + """ + table_cls = InMemoryTable if in_memory else MemoryMappedTable + return table_cls.from_file(filename) + + +class ParquetReader(BaseReader): + """ + Build a Dataset object out of Instruction instance(s). + This Reader uses memory mapping on parquet files. + """ + + def __init__(self, path: str, info: Optional["DatasetInfo"]): + """Initializes ParquetReader. + + Args: + path (str): path where tfrecords are stored. + info (DatasetInfo): info about the dataset. + """ + super().__init__(path, info) + self._filetype_suffix = "parquet" + + def _get_table_from_filename(self, filename_skip_take, **kwargs): + """Returns a Dataset instance from given (filename, skip, take).""" + filename, skip, take = ( + filename_skip_take["filename"], + filename_skip_take["skip"] if "skip" in filename_skip_take else None, + filename_skip_take["take"] if "take" in filename_skip_take else None, + ) + # Parquet read_table always loads data in memory, independently of memory_map + pa_table = pq.read_table(filename, memory_map=True) + # here we don't want to slice an empty table, or it may segfault + if skip is not None and take is not None and not (skip == 0 and take == len(pa_table)): + pa_table = pa_table.slice(skip, take) + return pa_table + + +@dataclass(frozen=True) +class _AbsoluteInstruction: + """A machine friendly slice: defined absolute positive boundaries.""" + + splitname: str + from_: int # uint (starting index). + to: int # uint (ending index). + + +@dataclass(frozen=True) +class _RelativeInstruction: + """Represents a single parsed slicing instruction, can use % and negatives.""" + + splitname: str + from_: Optional[int] = None # int (starting index) or None if no lower boundary. + to: Optional[int] = None # int (ending index) or None if no upper boundary. + unit: Optional[str] = None + rounding: Optional[str] = None + + def __post_init__(self): + if self.unit is not None and self.unit not in ["%", "abs"]: + raise ValueError("unit must be either % or abs") + if self.rounding is not None and self.rounding not in ["closest", "pct1_dropremainder"]: + raise ValueError("rounding must be either closest or pct1_dropremainder") + if self.unit != "%" and self.rounding is not None: + raise ValueError("It is forbidden to specify rounding if not using percent slicing.") + if self.unit == "%" and self.from_ is not None and abs(self.from_) > 100: + raise ValueError("Percent slice boundaries must be > -100 and < 100.") + if self.unit == "%" and self.to is not None and abs(self.to) > 100: + raise ValueError("Percent slice boundaries must be > -100 and < 100.") + # Update via __dict__ due to instance being "frozen" + self.__dict__["rounding"] = "closest" if self.rounding is None and self.unit == "%" else self.rounding + + +def _str_to_read_instruction(spec): + """Returns ReadInstruction for given string.""" + res = _SUB_SPEC_RE.match(spec) + if not res: + raise ValueError(f"Unrecognized instruction format: {spec}") + unit = "%" if res.group("from_pct") or res.group("to_pct") else "abs" + return ReadInstruction( + split_name=res.group("split"), + rounding=res.group("rounding"), + from_=int(res.group("from")) if res.group("from") else None, + to=int(res.group("to")) if res.group("to") else None, + unit=unit, + ) + + +def _pct_to_abs_pct1(boundary, num_examples): + # Using math.trunc here, since -99.5% should give -99%, not -100%. + if num_examples < 100: + msg = ( + 'Using "pct1_dropremainder" rounding on a split with less than 100 ' + "elements is forbidden: it always results in an empty dataset." + ) + raise ValueError(msg) + return boundary * math.trunc(num_examples / 100.0) + + +def _pct_to_abs_closest(boundary, num_examples): + return int(round(boundary * num_examples / 100.0)) + + +def _rel_to_abs_instr(rel_instr, name2len): + """Returns _AbsoluteInstruction instance for given RelativeInstruction. + + Args: + rel_instr: RelativeInstruction instance. + name2len: dict {split_name: num_examples}. + """ + pct_to_abs = _pct_to_abs_closest if rel_instr.rounding == "closest" else _pct_to_abs_pct1 + split = rel_instr.splitname + if split not in name2len: + raise ValueError(f'Unknown split "{split}". Should be one of {list(name2len)}.') + num_examples = name2len[split] + from_ = rel_instr.from_ + to = rel_instr.to + if rel_instr.unit == "%": + from_ = 0 if from_ is None else pct_to_abs(from_, num_examples) + to = num_examples if to is None else pct_to_abs(to, num_examples) + else: + from_ = 0 if from_ is None else from_ + to = num_examples if to is None else to + if from_ < 0: + from_ = max(num_examples + from_, 0) + if to < 0: + to = max(num_examples + to, 0) + from_ = min(from_, num_examples) + to = min(to, num_examples) + return _AbsoluteInstruction(split, from_, to) + + +class ReadInstruction: + """Reading instruction for a dataset. + + Examples:: + + # The following lines are equivalent: + ds = datasets.load_dataset('ylecun/mnist', split='test[:33%]') + ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction.from_spec('test[:33%]')) + ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction('test', to=33, unit='%')) + ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction( + 'test', from_=0, to=33, unit='%')) + + # The following lines are equivalent: + ds = datasets.load_dataset('ylecun/mnist', split='test[:33%]+train[1:-1]') + ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction.from_spec( + 'test[:33%]+train[1:-1]')) + ds = datasets.load_dataset('ylecun/mnist', split=( + datasets.ReadInstruction('test', to=33, unit='%') + + datasets.ReadInstruction('train', from_=1, to=-1, unit='abs'))) + + # The following lines are equivalent: + ds = datasets.load_dataset('ylecun/mnist', split='test[:33%](pct1_dropremainder)') + ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction.from_spec( + 'test[:33%](pct1_dropremainder)')) + ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction( + 'test', from_=0, to=33, unit='%', rounding="pct1_dropremainder")) + + # 10-fold validation: + tests = datasets.load_dataset( + 'ylecun/mnist', + [datasets.ReadInstruction('train', from_=k, to=k+10, unit='%') + for k in range(0, 100, 10)]) + trains = datasets.load_dataset( + 'ylecun/mnist', + [datasets.ReadInstruction('train', to=k, unit='%') + datasets.ReadInstruction('train', from_=k+10, unit='%') + for k in range(0, 100, 10)]) + + """ + + def _init(self, relative_instructions): + # Private initializer. + self._relative_instructions = relative_instructions + + @classmethod + def _read_instruction_from_relative_instructions(cls, relative_instructions): + """Returns ReadInstruction obj initialized with relative_instructions.""" + # Use __new__ to bypass __init__ used by public API and not conveniant here. + result = cls.__new__(cls) + result._init(relative_instructions) # pylint: disable=protected-access + return result + + def __init__(self, split_name, rounding=None, from_=None, to=None, unit=None): + """Initialize ReadInstruction. + + Args: + split_name (str): name of the split to read. Eg: 'train'. + rounding (str, optional): The rounding behaviour to use when percent slicing is + used. Ignored when slicing with absolute indices. + Possible values: + - 'closest' (default): The specified percentages are rounded to the + closest value. Use this if you want specified percents to be as + much exact as possible. + - 'pct1_dropremainder': the specified percentages are treated as + multiple of 1%. Use this option if you want consistency. Eg: + len(5%) == 5 * len(1%). + Using this option, one might not be able to use the full set of + examples, if the number of those is not a multiple of 100. + from_ (int): + to (int): alternative way of specifying slicing boundaries. If any of + {from_, to, unit} argument is used, slicing cannot be specified as + string. + unit (str): optional, one of: + '%': to set the slicing unit as percents of the split size. + 'abs': to set the slicing unit as absolute numbers. + """ + # This constructor is not always called. See factory method + # `_read_instruction_from_relative_instructions`. Common init instructions + # MUST be placed in the _init method. + self._init([_RelativeInstruction(split_name, from_, to, unit, rounding)]) + + @classmethod + def from_spec(cls, spec): + """Creates a `ReadInstruction` instance out of a string spec. + + Args: + spec (`str`): + Split(s) + optional slice(s) to read + optional rounding + if percents are used as the slicing unit. A slice can be specified, + using absolute numbers (`int`) or percentages (`int`). + + Examples: + + ``` + test: test split. + test + validation: test split + validation split. + test[10:]: test split, minus its first 10 records. + test[:10%]: first 10% records of test split. + test[:20%](pct1_dropremainder): first 10% records, rounded with the pct1_dropremainder rounding. + test[:-5%]+train[40%:60%]: first 95% of test + middle 20% of train. + ``` + + Returns: + ReadInstruction instance. + """ + spec = str(spec) # Need to convert to str in case of NamedSplit instance. + subs = _ADDITION_SEP_RE.split(spec) + if not subs: + raise ValueError(f"No instructions could be built out of {spec}") + instruction = _str_to_read_instruction(subs[0]) + return sum((_str_to_read_instruction(sub) for sub in subs[1:]), instruction) + + def to_spec(self): + rel_instr_specs = [] + for rel_instr in self._relative_instructions: + rel_instr_spec = rel_instr.splitname + if rel_instr.from_ is not None or rel_instr.to is not None: + from_ = rel_instr.from_ + to = rel_instr.to + unit = rel_instr.unit + rounding = rel_instr.rounding + unit = unit if unit == "%" else "" + from_ = str(from_) + unit if from_ is not None else "" + to = str(to) + unit if to is not None else "" + slice_str = f"[{from_}:{to}]" + rounding_str = ( + f"({rounding})" if unit == "%" and rounding is not None and rounding != "closest" else "" + ) + rel_instr_spec += slice_str + rounding_str + rel_instr_specs.append(rel_instr_spec) + return "+".join(rel_instr_specs) + + def __add__(self, other): + """Returns a new ReadInstruction obj, result of appending other to self.""" + if not isinstance(other, ReadInstruction): + msg = "ReadInstruction can only be added to another ReadInstruction obj." + raise TypeError(msg) + self_ris = self._relative_instructions + other_ris = other._relative_instructions # pylint: disable=protected-access + if ( + self_ris[0].unit != "abs" + and other_ris[0].unit != "abs" + and self._relative_instructions[0].rounding != other_ris[0].rounding + ): + raise ValueError("It is forbidden to sum ReadInstruction instances with different rounding values.") + return self._read_instruction_from_relative_instructions(self_ris + other_ris) + + def __str__(self): + return self.to_spec() + + def __repr__(self): + return f"ReadInstruction({self._relative_instructions})" + + def to_absolute(self, name2len): + """Translate instruction into a list of absolute instructions. + + Those absolute instructions are then to be added together. + + Args: + name2len (`dict`): + Associating split names to number of examples. + + Returns: + list of _AbsoluteInstruction instances (corresponds to the + in spec). + """ + return [_rel_to_abs_instr(rel_instr, name2len) for rel_instr in self._relative_instructions] diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/arrow_writer.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/arrow_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..2bd6f9fc6130e00256dbc384cae41020ba6f1314 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/arrow_writer.py @@ -0,0 +1,828 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""To write records into Parquet files.""" + +import io +import json +import sys +from collections.abc import Iterable +from typing import Any, Literal, Optional + +import fsspec +import numpy as np +import pyarrow as pa +import pyarrow.json as paj +import pyarrow.parquet as pq +from fsspec.core import url_to_fs + +from . import config +from .features import Audio, Features, Image, Pdf, Value, Video +from .features.features import ( + FeatureType, + List, + _ArrayXDExtensionType, + _visit, + cast_to_python_objects, + generate_from_arrow_type, + get_nested_type, + list_of_np_array_to_pyarrow_listarray, + numpy_to_pyarrow_listarray, + require_storage_embed, + to_pyarrow_listarray, +) +from .filesystems import is_remote_filesystem +from .info import DatasetInfo +from .table import array_cast, cast_array_to_feature, embed_table_storage, table_cast +from .utils import logging +from .utils.json import ( + find_mixed_struct_types_field_paths, + get_json_field_path_from_pyarrow_json_error, + get_json_field_paths_from_feature, + insert_json_field_path, + json_encode_field, + json_encode_fields_in_json_lines, + set_json_types_in_feature, + ujson_dumps, +) +from .utils.py_utils import asdict, convert_file_size_to_int, first_non_null_non_empty_value + + +logger = logging.get_logger(__name__) + +type_ = type # keep python's type function + + +def get_arrow_writer_batch_size_from_features(features: Optional[Features]) -> Optional[int]: + """ + Get the writer_batch_size that defines the maximum record batch size in the arrow files based on configuration values. + The default value is 100 for image/audio datasets and 10 for videos. + This allows to avoid overflows in arrow buffers. + + Args: + features (`datasets.Features` or `None`): + Dataset Features from `datasets`. + Returns: + writer_batch_size (`Optional[int]`): + Writer batch size to pass to a dataset builder. + If `None`, then it will use the `datasets` default, i.e. `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + """ + if not features: + return None + + batch_size = np.inf + + def set_batch_size(feature: FeatureType) -> None: + nonlocal batch_size + if isinstance(feature, Image) and config.ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETS is not None: + batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETS) + elif isinstance(feature, Audio) and config.ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETS is not None: + batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETS) + elif isinstance(feature, Video) and config.ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETS is not None: + batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETS) + elif ( + isinstance(feature, Value) + and feature.dtype == "binary" + and config.ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS is not None + ): + batch_size = min(batch_size, config.ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS) + + _visit(features, set_batch_size) + + return None if batch_size is np.inf else batch_size + + +def get_writer_batch_size_from_features(features: Optional[Features]) -> Optional[int]: + """ + Get the writer_batch_size that defines the maximum row group size in the parquet files based on configuration values. + By default these are not set, but it can be helpful to hard set those values in some cases. + This allows to optimize random access to parquet file, since accessing 1 row requires + to read its entire row group. + + Args: + features (`datasets.Features` or `None`): + Dataset Features from `datasets`. + Returns: + writer_batch_size (`Optional[int]`): + Writer batch size to pass to a parquet writer. + If `None`, then it will use the `datasets` default, i.e. aiming for row groups of 100MB. + """ + if not features: + return None + + batch_size = np.inf + + def set_batch_size(feature: FeatureType) -> None: + nonlocal batch_size + if isinstance(feature, Image) and config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS is not None: + batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS) + elif isinstance(feature, Audio) and config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS is not None: + batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS) + elif isinstance(feature, Video) and config.PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS is not None: + batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS) + elif ( + isinstance(feature, Value) + and feature.dtype == "binary" + and config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS is not None + ): + batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS) + + _visit(features, set_batch_size) + + return None if batch_size is np.inf else batch_size + + +def get_writer_batch_size_from_data_size(num_rows: int, num_bytes: int) -> int: + """ + Get the writer_batch_size that defines the maximum row group size in the parquet files. + The default in `datasets` is aiming for row groups of maximum 100MB uncompressed. + This allows to optimize random access to parquet file, since accessing 1 row requires + to read its entire row group. + + This can be improved to get optimized size for querying/iterating + but at least it matches the dataset viewer expectations on HF. + + Args: + num_rows (`int`): + Number of rows in the dataset. + num_bytes (`int`): + Number of bytes in the dataset. + For dataset with external files to embed (image, audio, videos), this can also be an + estimate from `dataset._estimate_nbytes()`. + Returns: + writer_batch_size (`Optional[int]`): + Writer batch size to pass to a parquet writer. + """ + return max(1, num_rows * convert_file_size_to_int(config.MAX_ROW_GROUP_SIZE) // num_bytes) if num_bytes > 0 else 1 + + +class SchemaInferenceError(ValueError): + pass + + +class TypedSequence: + """ + This data container generalizes the typing when instantiating pyarrow arrays, tables or batches. + + More specifically it adds several features: + - Support extension types like ``datasets.features.Array2DExtensionType``: + By default pyarrow arrays don't return extension arrays. One has to call + ``pa.ExtensionArray.from_storage(type, pa.array(data, type.storage_type))`` + in order to get an extension array. + - Support for ``try_type`` parameter that can be used instead of ``type``: + When an array is transformed, we like to keep the same type as before if possible. + For example when calling :func:`datasets.Dataset.map`, we don't want to change the type + of each column by default. + - Better error message when a pyarrow array overflows. + + Example:: + + from datasets.features import Array2D, Array2DExtensionType, Value + from datasets.arrow_writer import TypedSequence + import pyarrow as pa + + arr = pa.array(TypedSequence([1, 2, 3], type=Value("int32"))) + assert arr.type == pa.int32() + + arr = pa.array(TypedSequence([1, 2, 3], try_type=Value("int32"))) + assert arr.type == pa.int32() + + arr = pa.array(TypedSequence(["foo", "bar"], try_type=Value("int32"))) + assert arr.type == pa.string() + + arr = pa.array(TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64"))) + assert arr.type == Array2DExtensionType((1, 3), "int64") + + table = pa.Table.from_pydict({ + "image": TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64")) + }) + assert table["image"].type == Array2DExtensionType((1, 3), "int64") + + """ + + def __init__( + self, + data: Iterable, + type: Optional[FeatureType] = None, + try_type: Optional[FeatureType] = None, + optimized_int_type: Optional[FeatureType] = None, + on_mixed_types: Optional[Literal["use_json"]] = None, + ): + # assert type is None or try_type is None, + if type is not None and try_type is not None: + raise ValueError("You cannot specify both type and try_type") + # set attributes + self.data = data + self.type = type + self.try_type = try_type # is ignored if it doesn't match the data + self.optimized_int_type = optimized_int_type + self.on_mixed_types = on_mixed_types + # when trying a type (is ignored if data is not compatible) + self.trying_type = self.try_type is not None + self.trying_int_optimization = optimized_int_type is not None and type is None and try_type is None + # used to get back the inferred type after __arrow_array__() is called once + self._inferred_type = None + + def get_inferred_type(self) -> FeatureType: + """Return the inferred feature type. + This is done by converting the sequence to an Arrow array, and getting the corresponding + feature type. + + Since building the Arrow array can be expensive, the value of the inferred type is cached + as soon as pa.array is called on the typed sequence. + + Returns: + FeatureType: inferred feature type of the sequence. + """ + if self._inferred_type is None: + pa.array(self) + return self._inferred_type + + @staticmethod + def _infer_custom_type_and_encode(data: Iterable) -> tuple[Iterable, Optional[FeatureType]]: + """Implement type inference for custom objects like PIL.Image.Image -> Image type. + + This function is only used for custom python objects that can't be directly passed to build + an Arrow array. In such cases is infers the feature type to use, and it encodes the data so + that they can be passed to an Arrow array. + + Args: + data (Iterable): array of data to infer the type, e.g. a list of PIL images. + + Returns: + Tuple[Iterable, Optional[FeatureType]]: a tuple with: + - the (possibly encoded) array, if the inferred feature type requires encoding + - the inferred feature type if the array is made of supported custom objects like + PIL images, else None. + """ + if config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + non_null_idx, non_null_value = first_non_null_non_empty_value(data) + if isinstance(non_null_value, PIL.Image.Image): + return [Image().encode_example(value) if value is not None else None for value in data], Image() + if isinstance(non_null_value, list) and isinstance(non_null_value[0], PIL.Image.Image): + return [ + [Image().encode_example(x) for x in value] if value is not None else None for value in data + ], List(Image()) + if config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules: + import pdfplumber + + non_null_idx, non_null_value = first_non_null_non_empty_value(data) + if isinstance(non_null_value, pdfplumber.pdf.PDF): + return [Pdf().encode_example(value) if value is not None else None for value in data], Pdf() + if isinstance(non_null_value, list) and isinstance(non_null_value[0], pdfplumber.pdf.PDF): + return [ + [Pdf().encode_example(x) for x in value] if value is not None else None for value in data + ], List(Pdf()) + return data, None + + def __arrow_array__(self, type: Optional[pa.DataType] = None): + out = self._arrow_array(type=type) + if self._inferred_type is None: + self._inferred_type = generate_from_arrow_type(out.type) + return out + + def _arrow_array(self, type: Optional[pa.DataType] = None): + """This function is called when calling pa.array(typed_sequence)""" + + if type is not None: + raise ValueError("TypedSequence is supposed to be used with pa.array(typed_sequence, type=None)") + del type # make sure we don't use it + data = self.data + # automatic type inference for custom objects + if self.type is None and self.try_type is None: + data, self._inferred_type = self._infer_custom_type_and_encode(data) + if self._inferred_type is None: + type = self.try_type if self.trying_type else self.type + else: + type = self._inferred_type + pa_type = get_nested_type(type) if type is not None else None + optimized_int_pa_type = ( + get_nested_type(self.optimized_int_type) if self.optimized_int_type is not None else None + ) + trying_cast_to_python_objects = False + json_field_paths = [] + try: + # custom pyarrow types + if isinstance(pa_type, _ArrayXDExtensionType): + storage = to_pyarrow_listarray(data, pa_type) + return pa.ExtensionArray.from_storage(pa_type, storage) + + # efficient np array to pyarrow array + if isinstance(data, np.ndarray): + out = numpy_to_pyarrow_listarray(data) + elif isinstance(data, list) and data and isinstance(first_non_null_non_empty_value(data)[1], np.ndarray): + out = list_of_np_array_to_pyarrow_listarray(data) + else: + trying_cast_to_python_objects = True + examples = data + # find fields to json-encode + if self.on_mixed_types == "use_json" and type is None: + json_field_paths = find_mixed_struct_types_field_paths(examples, allow_root=True) + elif type is not None: + json_field_paths = get_json_field_paths_from_feature(type) + # json encode if needed + if json_field_paths: + for json_field_path in json_field_paths: + examples = [json_encode_field(examples, json_field_path) for examples in examples] + # to arrow array + out = pa.array(cast_to_python_objects(examples, only_1d_for_numpy=True)) + # cast to json type if needed + if json_field_paths: + pa_table = pa.Table.from_arrays([out], names=["obj"]) + features = Features.from_arrow_schema(pa_table.schema) + feature = set_json_types_in_feature(features["obj"], json_field_paths) + pa_table = table_cast(pa_table, Features({"obj": feature}).arrow_schema) + out = pa_table[0] # get the "obj" column + # use smaller integer precisions if possible + if self.trying_int_optimization: + if pa.types.is_int64(out.type): + out = out.cast(optimized_int_pa_type) + elif pa.types.is_list(out.type): + if pa.types.is_int64(out.type.value_type): + out = array_cast(out, pa.list_(optimized_int_pa_type)) + elif pa.types.is_list(out.type.value_type) and pa.types.is_int64(out.type.value_type.value_type): + out = array_cast(out, pa.list_(pa.list_(optimized_int_pa_type))) + # otherwise we can finally use the user's type + elif type is not None: + # We use cast_array_to_feature to support casting to custom types like Audio and Image + # Also, when trying type "string", we don't want to convert integers or floats to "string". + # We only do it if trying_type is False - since this is what the user asks for. + out = cast_array_to_feature( + out, type, allow_primitive_to_str=not self.trying_type, allow_decimal_to_str=not self.trying_type + ) + return out + except ( + TypeError, + pa.lib.ArrowTypeError, + pa.lib.ArrowInvalid, + pa.lib.ArrowNotImplementedError, + ) as e: # handle type errors and overflows + # Ignore ArrowNotImplementedError caused by trying type, otherwise re-raise + if not self.trying_type and isinstance(e, pa.lib.ArrowNotImplementedError): + raise + + if self.trying_type: + try: # second chance + if isinstance(data, np.ndarray): + return numpy_to_pyarrow_listarray(data) + elif isinstance(data, list) and data and any(isinstance(value, np.ndarray) for value in data): + return list_of_np_array_to_pyarrow_listarray(data) + else: + trying_cast_to_python_objects = True + return pa.array(cast_to_python_objects(data, only_1d_for_numpy=True)) + except pa.lib.ArrowInvalid as e: + if "overflow" in str(e): + raise OverflowError( + f"There was an overflow with type {type_(data)}. Try to reduce writer_batch_size to have batches smaller than 2GB.\n({e})" + ) from None + elif self.trying_int_optimization and "not in range" in str(e): + optimized_int_pa_type_str = np.dtype(optimized_int_pa_type.to_pandas_dtype()).name + logger.info( + f"Failed to cast a sequence to {optimized_int_pa_type_str}. Falling back to int64." + ) + return out + elif trying_cast_to_python_objects and "Could not convert" in str(e): + out = pa.array( + cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False) + ) + if type is not None: + out = cast_array_to_feature( + out, type, allow_primitive_to_str=True, allow_decimal_to_str=True + ) + return out + else: + raise + elif "overflow" in str(e): + raise OverflowError( + f"There was an overflow with type {type_(data)}. Try to reduce writer_batch_size to have batches smaller than 2GB.\n({e})" + ) from None + elif self.trying_int_optimization and "not in range" in str(e): + optimized_int_pa_type_str = np.dtype(optimized_int_pa_type.to_pandas_dtype()).name + logger.info(f"Failed to cast a sequence to {optimized_int_pa_type_str}. Falling back to int64.") + return out + elif trying_cast_to_python_objects and ( + "Could not convert" in str(e) or "cannot mix struct and non-struct" in str(e) or "Expected " in str(e) + ): + try: # third chance + out = pa.array(cast_to_python_objects(data, only_1d_for_numpy=True, optimize_list_casting=False)) + except (pa.ArrowInvalid, pa.ArrowTypeError) as ee: + # in case of mixed types, we use the JSON Lines reader in pyarrow to locate them and set them to json fields + if self.on_mixed_types == "use_json" and ( + "Could not convert " in str(ee) + or "cannot mix struct and non-struct" in str(ee) + or "Expected " in str(ee) + ): + # we use "obj" to have valid JSON Lines since data may contain lists + original_batch = "\n".join([ujson_dumps({"obj": example}) for example in data]).encode() + json_field_paths = [["obj"] + json_field_path for json_field_path in json_field_paths] + batch = json_encode_fields_in_json_lines(original_batch, json_field_paths) + pa_table = None + while True: + try: # fourth chance + pa_table = paj.read_json( + io.BytesIO(batch), read_options=paj.ReadOptions(use_threads=False) + ) + break + except pa.ArrowInvalid as eee: + if "JSON parse error: Column(" in str(eee) and ") changed from" in str(eee): + json_field_path = get_json_field_path_from_pyarrow_json_error(str(eee)) + insert_json_field_path(json_field_paths, json_field_path) + batch = json_encode_fields_in_json_lines(original_batch, json_field_paths) + else: + break + if pa_table is not None: + features = Features.from_arrow_schema(pa_table.schema) + features = set_json_types_in_feature(features, json_field_paths) + pa_table = table_cast(pa_table, features.arrow_schema) + out = pa_table[0] # get the "obj" column + return out + else: + raise + else: + raise + if type is not None: + out = cast_array_to_feature(out, type, allow_primitive_to_str=True, allow_decimal_to_str=True) + return out + else: + raise + + +class OptimizedTypedSequence(TypedSequence): + def __init__( + self, + data, + type: Optional[FeatureType] = None, + try_type: Optional[FeatureType] = None, + col: Optional[str] = None, + optimized_int_type: Optional[FeatureType] = None, + on_mixed_types: Optional[Literal["use_json"]] = None, + ): + optimized_int_type_by_col = { + "attention_mask": Value("int8"), # binary tensor + "special_tokens_mask": Value("int8"), + "input_ids": Value("int32"), # typical vocab size: 0-50k (max ~500k, never > 1M) + "token_type_ids": Value( + "int8" + ), # binary mask; some (XLNetModel) use an additional token represented by a 2 + } + if type is None and try_type is None: + optimized_int_type = optimized_int_type_by_col.get(col, None) + super().__init__( + data, type=type, try_type=try_type, optimized_int_type=optimized_int_type, on_mixed_types=on_mixed_types + ) + + +class ArrowWriter: + """Shuffles and writes Examples to Arrow files.""" + + def __init__( + self, + schema: Optional[pa.Schema] = None, + features: Optional[Features] = None, + path: Optional[str] = None, + stream: Optional[pa.NativeFile] = None, + fingerprint: Optional[str] = None, + writer_batch_size: Optional[int] = None, + disable_nullable: bool = False, + update_features: bool = False, + on_mixed_types: Optional[Literal["use_json"]] = "use_json", + with_metadata: bool = True, + unit: str = "examples", + embed_local_files: bool = False, + storage_options: Optional[dict] = None, + ): + if path is None and stream is None: + raise ValueError("At least one of path and stream must be provided.") + if features is not None: + self._features = features + self._schema = None + elif schema is not None: + self._schema: pa.Schema = schema + self._features = Features.from_arrow_schema(self._schema) + else: + self._features = None + self._schema = None + + self._disable_nullable = disable_nullable + + if stream is None: + fs, path = url_to_fs(path, **(storage_options or {})) + self._fs: fsspec.AbstractFileSystem = fs + self._path = path if not is_remote_filesystem(self._fs) else self._fs.unstrip_protocol(path) + self.stream = self._fs.open(path, "wb") + self._closable_stream = True + else: + self._fs = None + self._path = None + self.stream = stream + self._closable_stream = False + + self.fingerprint = fingerprint + self.disable_nullable = disable_nullable + self.writer_batch_size = ( + writer_batch_size + or get_arrow_writer_batch_size_from_features(self._features) + or config.DEFAULT_MAX_BATCH_SIZE + ) + self.update_features = update_features + self.on_mixed_types = on_mixed_types + self.with_metadata = with_metadata + self.unit = unit + self.embed_local_files = embed_local_files + + self._num_examples = 0 + self._num_bytes = 0 + self.current_examples: list[tuple[dict[str, Any], str]] = [] + self.current_rows: list[pa.Table] = [] + self.pa_writer: Optional[pa.RecordBatchStreamWriter] = None + self.hkey_record = [] + + def __len__(self): + """Return the number of writed and staged examples""" + return self._num_examples + len(self.current_examples) + len(self.current_rows) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + # Try closing if opened; if closed: pyarrow.lib.ArrowInvalid: Invalid operation on closed file + if self.pa_writer: # it might be None + try: + self.pa_writer.close() + except Exception: # pyarrow.lib.ArrowInvalid, OSError + pass + if self._closable_stream and not self.stream.closed: + self.stream.close() # This also closes self.pa_writer if it is opened + + def _build_schema(self, inferred_schema: pa.Schema): + schema = self.schema + features = self._features + inferred_features = Features.from_arrow_schema(inferred_schema) + if self._features is not None: + if self.update_features: # keep original features it they match, or update them + fields = {field.name: field for field in self._features.type} + for inferred_field in inferred_features.type: + name = inferred_field.name + if name in fields: + if inferred_field == fields[name]: + inferred_features[name] = self._features[name] + features = inferred_features + schema: pa.Schema = inferred_schema + else: + features = inferred_features + schema: pa.Schema = inferred_features.arrow_schema + + if self.disable_nullable: + schema = pa.schema(pa.field(field.name, field.type, nullable=False) for field in schema) + if self.with_metadata: + schema = schema.with_metadata(self._build_metadata(DatasetInfo(features=features), self.fingerprint)) + else: + schema = schema.with_metadata({}) + + return schema, features + + def _build_writer(self, inferred_schema: pa.Schema): + self._schema, self._features = self._build_schema(inferred_schema) + self.pa_writer = pa.RecordBatchStreamWriter(self.stream, self._schema) + + @property + def schema(self): + _schema = ( + self._schema + if self._schema is not None + else (pa.schema(self._features.type) if self._features is not None else None) + ) + if self._disable_nullable and _schema is not None: + _schema = pa.schema(pa.field(field.name, field.type, nullable=False) for field in _schema) + return _schema if _schema is not None else [] + + @staticmethod + def _build_metadata(info: DatasetInfo, fingerprint: Optional[str] = None) -> dict[str, str]: + info_keys = ["features"] # we can add support for more DatasetInfo keys in the future + info_as_dict = asdict(info) + metadata = {} + metadata["info"] = {key: info_as_dict[key] for key in info_keys} + if fingerprint is not None: + metadata["fingerprint"] = fingerprint + return {"huggingface": json.dumps(metadata)} + + def write_examples_on_file(self): + """Write stored examples from the write-pool of examples. It makes a table out of the examples and write it.""" + if not self.current_examples: + return + # preserve the order the columns + if self.schema: + schema_cols = set(self.schema.names) + examples_cols = self.current_examples[0][0].keys() # .keys() preserves the order (unlike set) + common_cols = [col for col in self.schema.names if col in examples_cols] + extra_cols = [col for col in examples_cols if col not in schema_cols] + cols = common_cols + extra_cols + else: + cols = list(self.current_examples[0][0]) + batch_examples = {} + for col in cols: + # We use row[0][col] since current_examples contains (example, key) tuples. + # Moreover, examples could be Arrow arrays of 1 element. + # This can happen in `.map()` when we want to re-write the same Arrow data + if all(isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) for row in self.current_examples): + arrays = [row[0][col] for row in self.current_examples] + arrays = [ + chunk + for array in arrays + for chunk in (array.chunks if isinstance(array, pa.ChunkedArray) else [array]) + ] + batch_examples[col] = pa.concat_arrays(arrays) + else: + batch_examples[col] = [ + row[0][col].to_pylist()[0] if isinstance(row[0][col], (pa.Array, pa.ChunkedArray)) else row[0][col] + for row in self.current_examples + ] + self._write_batch(batch_examples=batch_examples) + self.current_examples = [] + + def write_rows_on_file(self): + """Write stored rows from the write-pool of rows. It concatenates the single-row tables and it writes the resulting table.""" + if not self.current_rows: + return + table = pa.concat_tables(self.current_rows) + self._write_table(table) + self.current_rows = [] + + def write( + self, + example: dict[str, Any], + writer_batch_size: Optional[int] = None, + ): + """Add a given (Example,Key) pair to the write-pool of examples which is written to file. + + Args: + example: the Example to add. + """ + # Store example as a tuple so as to keep the structure of `self.current_examples` uniform + self.current_examples.append((example, "")) + + if writer_batch_size is None: + writer_batch_size = self.writer_batch_size + if writer_batch_size is not None and len(self.current_examples) >= writer_batch_size: + self.write_examples_on_file() + + def write_row(self, row: pa.Table, writer_batch_size: Optional[int] = None): + """Add a given single-row Table to the write-pool of rows which is written to file. + + Args: + row: the row to add. + """ + if len(row) != 1: + raise ValueError(f"Only single-row pyarrow tables are allowed but got table with {len(row)} rows.") + self.current_rows.append(row) + if writer_batch_size is None: + writer_batch_size = self.writer_batch_size + if writer_batch_size is not None and len(self.current_rows) >= writer_batch_size: + self.write_rows_on_file() + + def write_batch( + self, + batch_examples: dict[str, list], + writer_batch_size: Optional[int] = None, + try_original_type: Optional[bool] = True, + ): + """Write a batch of Example to file. + Ignores the batch if it appears to be empty, + preventing a potential schema update of unknown types. + + Args: + batch_examples: the batch of examples to add. + try_original_type: use `try_type` when instantiating OptimizedTypedSequence if `True`, otherwise `try_type = None`. + """ + self.write_examples_on_file() # in case there are buffered examples to write first + self._write_batch(batch_examples, writer_batch_size=writer_batch_size, try_original_type=try_original_type) + + def _write_batch( + self, + batch_examples: dict[str, list], + writer_batch_size: Optional[int] = None, + try_original_type: Optional[bool] = True, + ): + if batch_examples and len(next(iter(batch_examples.values()))) == 0: + return + features = None if self.pa_writer is None and self.update_features else self._features + try_features = self._features if self.pa_writer is None and self.update_features else None + arrays = [] + inferred_features = Features() + # preserve the order the columns + if self.schema: + schema_cols = set(self.schema.names) + batch_cols = batch_examples.keys() # .keys() preserves the order (unlike set) + common_cols = [col for col in self.schema.names if col in batch_cols] + extra_cols = [col for col in batch_cols if col not in schema_cols] + cols = common_cols + extra_cols + else: + cols = list(batch_examples) + for col in cols: + col_values = batch_examples[col] + col_type = features[col] if features else None + if isinstance(col_values, (pa.Array, pa.ChunkedArray)): + array = cast_array_to_feature(col_values, col_type) if col_type is not None else col_values + arrays.append(array) + inferred_features[col] = generate_from_arrow_type(col_values.type) + else: + col_try_type = ( + try_features[col] + if try_features is not None and col in try_features and try_original_type + else None + ) + typed_sequence = OptimizedTypedSequence( + col_values, type=col_type, try_type=col_try_type, col=col, on_mixed_types=self.on_mixed_types + ) + arrays.append(pa.array(typed_sequence)) + inferred_features[col] = typed_sequence.get_inferred_type() + schema = inferred_features.arrow_schema if self.pa_writer is None else self.schema + pa_table = pa.Table.from_arrays(arrays, schema=schema) + self.write_table(pa_table, writer_batch_size) + + def write_table(self, pa_table: pa.Table, writer_batch_size: Optional[int] = None): + """Write a Table to file. + + Args: + example: the Table to add. + """ + self.write_rows_on_file() # in case there are buffered rows to write first + self._write_table(pa_table, writer_batch_size=writer_batch_size) + + def _write_table(self, pa_table: pa.Table, writer_batch_size: Optional[int] = None): + if writer_batch_size is None: + writer_batch_size = self.writer_batch_size + if self.pa_writer is None: + self._build_writer(inferred_schema=pa_table.schema) + pa_table = pa_table.combine_chunks() + pa_table = table_cast(pa_table, self._schema) + if self.embed_local_files: + pa_table = embed_table_storage(pa_table, local_files=True, remote_files=False) + self._num_bytes += pa_table.nbytes + self._num_examples += pa_table.num_rows + self.pa_writer.write_table(pa_table, writer_batch_size) + + def finalize(self, close_stream=True): + self.write_rows_on_file() + # In case current_examples < writer_batch_size, but user uses finalize() + self.write_examples_on_file() + # If schema is known, infer features even if no examples were written + if self.pa_writer is None and self.schema: + self._build_writer(self.schema) + if self.pa_writer is not None: + self.pa_writer.close() + self.pa_writer = None + if close_stream: + self.stream.close() + else: + if close_stream: + self.stream.close() + raise SchemaInferenceError("Please pass `features` or at least one example when writing data") + logger.debug( + f"Done writing {self._num_examples} {self.unit} in {self._num_bytes} bytes {self._path if self._path else ''}." + ) + return self._num_examples, self._num_bytes + + +class ParquetWriter(ArrowWriter): + def __init__(self, *args, use_content_defined_chunking=True, write_page_index=True, **kwargs): + super().__init__(*args, **kwargs) + if use_content_defined_chunking is True: + use_content_defined_chunking = config.DEFAULT_CDC_OPTIONS + self.use_content_defined_chunking = use_content_defined_chunking + self.write_page_index = write_page_index + + def _build_writer(self, inferred_schema: pa.Schema): + self._schema, self._features = self._build_schema(inferred_schema) + self.pa_writer = pq.ParquetWriter( + self.stream, + self._schema, + use_content_defined_chunking=self.use_content_defined_chunking, + write_page_index=self.write_page_index, + compression={ + col: "none" if require_storage_embed(feature) else "snappy" for col, feature in self._features.items() + }, + use_dictionary=[col for col, feature in self._features.items() if not require_storage_embed(feature)], + column_encoding={ + col: "PLAIN" for col, feature in self._features.items() if require_storage_embed(feature) + }, + ) + if self.use_content_defined_chunking is not False: + self.pa_writer.add_key_value_metadata( + {"content_defined_chunking": json.dumps(self.use_content_defined_chunking)} + ) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/builder.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..b995242f01612ca6583f1f461f6472e3aba4e2fc --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/builder.py @@ -0,0 +1,1878 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""DatasetBuilder base class.""" + +import abc +import contextlib +import copy +import inspect +import os +import posixpath +import shutil +import time +import urllib +from collections.abc import Iterator +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional, Union +from unittest.mock import patch + +import fsspec +import pyarrow as pa +from fsspec.core import url_to_fs +from multiprocess import Pool +from tqdm.contrib.concurrent import thread_map + +from . import config, utils +from .arrow_dataset import Dataset +from .arrow_reader import ( + ArrowReader, + ReadInstruction, +) +from .arrow_writer import ArrowWriter, ParquetWriter, SchemaInferenceError +from .data_files import DataFilesDict, DataFilesPatternsDict, sanitize_patterns +from .dataset_dict import DatasetDict, IterableDatasetDict +from .download.download_config import DownloadConfig +from .download.download_manager import DownloadManager, DownloadMode +from .download.streaming_download_manager import StreamingDownloadManager, xjoin +from .exceptions import DatasetGenerationCastError, DatasetGenerationError, FileFormatError +from .features import Features +from .filesystems import ( + is_remote_filesystem, + rename, +) +from .fingerprint import Hasher +from .info import DatasetInfo +from .iterable_dataset import ArrowExamplesIterable, ExamplesIterable, IterableDataset +from .naming import INVALID_WINDOWS_CHARACTERS_IN_PATH, camelcase_to_snakecase +from .splits import Split, SplitDict, SplitGenerator, SplitInfo +from .streaming import extend_dataset_builder_for_streaming +from .table import CastError +from .utils import logging +from .utils import tqdm as hf_tqdm +from .utils._filelock import FileLock +from .utils.file_utils import is_remote_url +from .utils.info_utils import VerificationMode, verify_checksums, verify_splits +from .utils.py_utils import ( + classproperty, + convert_file_size_to_int, + has_sufficient_disk_space, + iflatmap_unordered, + map_nested, + memoize, + size_str, + temporary_assignment, +) +from .utils.sharding import _number_of_shards_in_gen_kwargs, _split_gen_kwargs +from .utils.track import tracked_list + + +if TYPE_CHECKING: + from .load import DatasetModule + + +logger = logging.get_logger(__name__) + + +class InvalidConfigName(ValueError): + pass + + +@dataclass +class BuilderConfig: + """Base class for `DatasetBuilder` data configuration. + + `DatasetBuilder` subclasses with data configuration options should subclass + `BuilderConfig` and add their own properties. + + Attributes: + name (`str`, defaults to `default`): + The name of the configuration. + version (`Version` or `str`, defaults to `0.0.0`): + The version of the configuration. + data_dir (`str`, *optional*): + Path to the directory containing the source data. + data_files (`str` or `Sequence` or `Mapping`, *optional*): + Path(s) to source data file(s). + description (`str`, *optional*): + A human description of the configuration. + """ + + name: str = "default" + version: Optional[Union[utils.Version, str]] = utils.Version("0.0.0") + data_dir: Optional[str] = None + data_files: Optional[Union[DataFilesDict, DataFilesPatternsDict]] = None + description: Optional[str] = None + + def __post_init__(self): + # The config name is used to name the cache directory. + for invalid_char in INVALID_WINDOWS_CHARACTERS_IN_PATH: + if invalid_char in self.name: + raise InvalidConfigName( + f"Bad characters from black list '{INVALID_WINDOWS_CHARACTERS_IN_PATH}' found in '{self.name}'. " + f"They could create issues when creating a directory for this config on Windows filesystem." + ) + if self.data_files is not None and not isinstance(self.data_files, (DataFilesDict, DataFilesPatternsDict)): + raise ValueError(f"Expected a DataFilesDict in data_files but got {self.data_files}") + + def __eq__(self, o): + # we need to override the default dataclass __eq__ since it doesn't check for + # other attributes that the ones of the signature. + if set(self.__dict__.keys()) != set(o.__dict__.keys()): + return False + return all((k, getattr(self, k)) == (k, getattr(o, k)) for k in self.__dict__.keys()) + + def create_config_id( + self, + config_kwargs: dict, + custom_features: Optional[Features] = None, + ) -> str: + """ + The config id is used to build the cache directory. + By default it is equal to the config name. + However the name of a config is not sufficient to have a unique identifier for the dataset being generated + since it doesn't take into account: + - the config kwargs that can be used to overwrite attributes + - the custom features used to write the dataset + - the data_files for json/text/csv/pandas datasets + + Therefore the config id is just the config name with an optional suffix based on these. + """ + # Possibly add a suffix to the name to handle custom features/data_files/config_kwargs + suffix: Optional[str] = None + config_kwargs_to_add_to_suffix = config_kwargs.copy() + # name and version are already used to build the cache directory + config_kwargs_to_add_to_suffix.pop("name", None) + config_kwargs_to_add_to_suffix.pop("version", None) + # data dir handling (when specified it points to the manually downloaded data): + # it was previously ignored before the introduction of config id because we didn't want + # to change the config name. Now it's fine to take it into account for the config id. + # config_kwargs_to_add_to_suffix.pop("data_dir", None) + if "data_dir" in config_kwargs_to_add_to_suffix: + if config_kwargs_to_add_to_suffix["data_dir"] is None: + config_kwargs_to_add_to_suffix.pop("data_dir", None) + else: + # canonicalize the data dir to avoid two paths to the same location having different + # hashes + data_dir = config_kwargs_to_add_to_suffix["data_dir"] + data_dir = os.path.normpath(data_dir) + config_kwargs_to_add_to_suffix["data_dir"] = data_dir + if config_kwargs_to_add_to_suffix: + # we don't care about the order of the kwargs + config_kwargs_to_add_to_suffix = { + k: config_kwargs_to_add_to_suffix[k] for k in sorted(config_kwargs_to_add_to_suffix) + } + if all(isinstance(v, (str, bool, int, float)) for v in config_kwargs_to_add_to_suffix.values()): + suffix = ",".join( + str(k) + "=" + urllib.parse.quote_plus(str(v)) for k, v in config_kwargs_to_add_to_suffix.items() + ) + if len(suffix) > 32: # hash if too long + suffix = Hasher.hash(config_kwargs_to_add_to_suffix) + else: + suffix = Hasher.hash(config_kwargs_to_add_to_suffix) + + if custom_features is not None: + m = Hasher() + if suffix: + m.update(suffix) + m.update(custom_features) + suffix = m.hexdigest() + + if suffix: + config_id = self.name + "-" + suffix + if len(config_id) > config.MAX_DATASET_CONFIG_ID_READABLE_LENGTH: + config_id = self.name + "-" + Hasher.hash(suffix) + return config_id + else: + return self.name + + def _resolve_data_files(self, base_path: str, download_config: DownloadConfig) -> None: + if isinstance(self.data_files, DataFilesPatternsDict): + base_path = xjoin(base_path, self.data_dir) if self.data_dir else base_path + self.data_files = self.data_files.resolve(base_path, download_config) + + +class DatasetBuilder: + """Abstract base class for all datasets. + + `DatasetBuilder` has 3 key methods: + + - [`DatasetBuilder.info`]: Documents the dataset, including feature + names, types, shapes, version, splits, citation, etc. + - [`DatasetBuilder.download_and_prepare`]: Downloads the source data + and writes it to disk. + - [`DatasetBuilder.as_dataset`]: Generates a [`Dataset`]. + + Some `DatasetBuilder`s expose multiple variants of the + dataset by defining a [`BuilderConfig`] subclass and accepting a + config object (or name) on construction. Configurable datasets expose a + pre-defined set of configurations in [`DatasetBuilder.builder_configs`]. + + Args: + cache_dir (`str`, *optional*): + Directory to cache data. Defaults to `"~/.cache/huggingface/datasets"`. + dataset_name (`str`, *optional*): + Name of the dataset, if different from the builder name. Useful for packaged builders + like csv, imagefolder, audiofolder, etc. to reflect the difference between datasets + that use the same packaged builder. + config_name (`str`, *optional*): + Name of the dataset configuration. + It affects the data generated on disk. Different configurations will have their own subdirectories and + versions. + If not provided, the default configuration is used (if it exists). + + + + Parameter `name` was renamed to `config_name`. + + + hash (`str`, *optional*): + Hash specific to the dataset builder code. Used to update the caching directory when the + dataset builder code is updated (to avoid reusing old data). + The typical caching directory (defined in `self._relative_data_dir`) is `name/version/hash/`. + base_path (`str`, *optional*): + Base path for relative paths that are used to download files. + This can be a remote URL. + features ([`Features`], *optional*): + Features types to use with this dataset. + It can be used to change the [`Features`] types of a dataset, for example. + token (`str` or `bool`, *optional*): + String or boolean to use as Bearer token for remote files on the + Datasets Hub. If `True`, will get token from `"~/.huggingface"`. + repo_id (`str`, *optional*): + ID of the dataset repository. + Used to distinguish builders with the same name but not coming from the same namespace, for example "rajpurkar/squad" + and "lhoestq/squad" repo IDs. In the latter, the builder name would be "lhoestq___squad". + data_files (`str` or `Sequence` or `Mapping`, *optional*): + Path(s) to source data file(s). + For builders like "csv" or "json" that need the user to specify data files. They can be either + local or remote files. For convenience, you can use a `DataFilesDict`. + data_dir (`str`, *optional*): + Path to directory containing source data file(s). + Use only if `data_files` is not passed, in which case it is equivalent to passing + `os.path.join(data_dir, "**")` as `data_files`. + For builders that require manual download, it must be the path to the local directory containing the + manually downloaded data. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the dataset file-system backend, if any. + writer_batch_size (`int`, *optional*): + Batch size used by the ArrowWriter. + It defines the number of samples that are kept in memory before writing them + and also the length of the arrow chunks. + None means that the ArrowWriter will use its default value. + **config_kwargs (additional keyword arguments): Keyword arguments to be passed to the corresponding builder + configuration class, set on the class attribute [`DatasetBuilder.BUILDER_CONFIG_CLASS`]. The builder + configuration class is [`BuilderConfig`] or a subclass of it. + """ + + # Default version + VERSION = None # Default version set in BuilderConfig + + # Class for the builder config. + BUILDER_CONFIG_CLASS = BuilderConfig + + # Named configurations that modify the data generated by download_and_prepare. + BUILDER_CONFIGS = [] + + # Optional default config name to be used when name is None + DEFAULT_CONFIG_NAME = None + + # Default batch size used by the ArrowWriter + # It defines the number of samples that are kept in memory before writing them + # and also the length of the arrow chunks + # None means that the ArrowWriter will use its default value + DEFAULT_WRITER_BATCH_SIZE = None + + def __init__( + self, + cache_dir: Optional[str] = None, + dataset_name: Optional[str] = None, + config_name: Optional[str] = None, + hash: Optional[str] = None, + base_path: Optional[str] = None, + info: Optional[DatasetInfo] = None, + features: Optional[Features] = None, + token: Optional[Union[bool, str]] = None, + repo_id: Optional[str] = None, + data_files: Optional[Union[str, list, dict, DataFilesDict]] = None, + data_dir: Optional[str] = None, + storage_options: Optional[dict] = None, + writer_batch_size: Optional[int] = None, + config_id: Optional[str] = None, + **config_kwargs, + ): + # DatasetBuilder name + self.name: str = camelcase_to_snakecase(self.__module__.split(".")[-1]) + self.hash: Optional[str] = hash + self.base_path = base_path + self.token = token + self.repo_id = repo_id + self.storage_options = storage_options or {} + self.dataset_name = camelcase_to_snakecase(dataset_name) if dataset_name else self.name + self._writer_batch_size = writer_batch_size or self.DEFAULT_WRITER_BATCH_SIZE + + if data_files is not None and not isinstance(data_files, DataFilesDict): + data_files = DataFilesDict.from_patterns( + sanitize_patterns(data_files), + base_path=base_path, + download_config=DownloadConfig(token=token, storage_options=self.storage_options), + ) + + # Prepare config: DatasetConfig contains name, version and description but can be extended by each dataset + if "features" in inspect.signature(self.BUILDER_CONFIG_CLASS.__init__).parameters and features is not None: + config_kwargs["features"] = features + if data_files is not None: + config_kwargs["data_files"] = data_files + if data_dir is not None: + config_kwargs["data_dir"] = data_dir + self.config_kwargs = config_kwargs + self.config, self.config_id = self._create_builder_config( + config_name=config_name, + custom_features=features, + config_id=config_id, + **config_kwargs, + ) + + # prepare info: DatasetInfo are a standardized dataclass across all datasets + # Prefill datasetinfo + if info is None: + info = self._info() + info.builder_name = self.name + info.dataset_name = self.dataset_name + info.config_name = self.config.name + info.version = self.config.version + self.info = info + # update info with user specified infos + if features is not None: + self.info.features = features + + # Prepare data dirs: + # cache_dir can be a remote bucket on GCS or S3 + self._cache_dir_root = str(cache_dir or config.HF_DATASETS_CACHE) + self._cache_dir_root = ( + self._cache_dir_root if is_remote_url(self._cache_dir_root) else os.path.expanduser(self._cache_dir_root) + ) + self._cache_downloaded_dir = ( + posixpath.join(self._cache_dir_root, config.DOWNLOADED_DATASETS_DIR) + if cache_dir + else str(config.DOWNLOADED_DATASETS_PATH) + ) + self._cache_downloaded_dir = ( + self._cache_downloaded_dir + if is_remote_url(self._cache_downloaded_dir) + else os.path.expanduser(self._cache_downloaded_dir) + ) + + # In case there exists a legacy cache directory + self._legacy_relative_data_dir = None + + self._cache_dir = self._build_cache_dir() + if not is_remote_url(self._cache_dir_root): + os.makedirs(self._cache_dir_root, exist_ok=True) + lock_path = os.path.join( + self._cache_dir_root, Path(self._cache_dir).as_posix().replace("/", "_") + ".lock" + ) + with FileLock(lock_path): + if os.path.exists(self._cache_dir): # check if data exist + if len(os.listdir(self._cache_dir)) > 0: + if os.path.exists(os.path.join(self._cache_dir, config.DATASET_INFO_FILENAME)): + logger.debug("Overwrite dataset info from restored data version if exists.") + self.info = DatasetInfo.from_directory(self._cache_dir) + else: # dir exists but no data, remove the empty dir as data aren't available anymore + logger.warning( + f"Old caching folder {self._cache_dir} for dataset {self.dataset_name} exists but no data were found. Removing it. " + ) + os.rmdir(self._cache_dir) + + # Store in the cache by default unless the user specifies a custom output_dir to download_and_prepare + self._output_dir = self._cache_dir + self._fs: fsspec.AbstractFileSystem = fsspec.filesystem("file") + + # Set download manager + self.dl_manager = None + + # Set to True by "datasets-cli test" to generate file checksums for (deprecated) dataset_infos.json independently of verification_mode value. + self._record_checksums = False + + # Set in `.download_and_prepare` once the format of the generated dataset is known + self._file_format = None + + # Enable streaming (e.g. it patches "open" to work with remote files) + extend_dataset_builder_for_streaming(self) + + def __getstate__(self): + return self.__dict__ + + def __setstate__(self, d): + self.__dict__ = d + # Re-enable streaming, since patched functions are not kept when pickling + extend_dataset_builder_for_streaming(self) + + def _check_legacy_cache(self) -> Optional[str]: + """Check for the old cache directory template {cache_dir}/{namespace}___{builder_name} from 2.13""" + if ( + self.__module__.startswith("datasets.") + and not is_remote_url(self._cache_dir_root) + and self.config.name == "default" + ): + from .packaged_modules import _PACKAGED_DATASETS_MODULES + + namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None + config_name = self.repo_id.replace("/", "--") if self.repo_id is not None else self.dataset_name + config_id = config_name + self.config_id[len(self.config.name) :] + hash = _PACKAGED_DATASETS_MODULES.get(self.name, "missing")[1] + legacy_relative_data_dir = posixpath.join( + self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}", + config_id, + "0.0.0", + hash, + ) + legacy_cache_dir = posixpath.join(self._cache_dir_root, legacy_relative_data_dir) + if os.path.isdir(legacy_cache_dir): + return legacy_relative_data_dir + + def _check_legacy_cache2(self, dataset_module: "DatasetModule") -> Optional[str]: + """Check for the old cache directory template {cache_dir}/{namespace}___{dataset_name}/{config_name}-xxx from 2.14 and 2.15""" + if ( + self.__module__.startswith("datasets.") + and not is_remote_url(self._cache_dir_root) + and not (set(self.config_kwargs) - {"data_files", "data_dir"}) + ): + from .packaged_modules import _PACKAGED_DATASETS_MODULES_2_15_HASHES + from .utils._dill import Pickler + + def update_hash_with_config_parameters(hash: str, config_parameters: dict) -> str: + """ + Used to update hash of packaged modules which is used for creating unique cache directories to reflect + different config parameters which are passed in metadata from readme. + """ + params_to_exclude = {"config_name", "version", "description"} + params_to_add_to_hash = { + param: value + for param, value in sorted(config_parameters.items()) + if param not in params_to_exclude + } + m = Hasher() + m.update(hash) + m.update(params_to_add_to_hash) + return m.hexdigest() + + namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None + with patch.object(Pickler, "_legacy_no_dict_keys_sorting", True): + config_id = self.config.name + "-" + Hasher.hash({"data_files": self.config.data_files}) + hash = _PACKAGED_DATASETS_MODULES_2_15_HASHES.get(self.name, "missing") + if ( + dataset_module.builder_configs_parameters.metadata_configs + and self.config.name in dataset_module.builder_configs_parameters.metadata_configs + ): + hash = update_hash_with_config_parameters( + hash, dataset_module.builder_configs_parameters.metadata_configs[self.config.name] + ) + legacy_relative_data_dir = posixpath.join( + self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}", + config_id, + "0.0.0", + hash, + ) + legacy_cache_dir = posixpath.join(self._cache_dir_root, legacy_relative_data_dir) + if os.path.isdir(legacy_cache_dir): + return legacy_relative_data_dir + + def _create_builder_config( + self, config_name=None, custom_features=None, config_id=None, **config_kwargs + ) -> tuple[BuilderConfig, str]: + """Create and validate BuilderConfig object as well as a unique config id for this config. + Raises ValueError if there are multiple builder configs and config_name and DEFAULT_CONFIG_NAME are None. + config_kwargs override the defaults kwargs in config + """ + builder_config = None + + # try default config + if config_name is None and self.BUILDER_CONFIGS: + if self.DEFAULT_CONFIG_NAME is not None: + builder_config = self.builder_configs.get(self.DEFAULT_CONFIG_NAME) + logger.info(f"No config specified, defaulting to: {self.dataset_name}/{builder_config.name}") + else: + if len(self.BUILDER_CONFIGS) > 1: + if not config_kwargs: + example_of_usage = ( + f"load_dataset('{self.repo_id or self.dataset_name}', '{self.BUILDER_CONFIGS[0].name}')" + ) + raise ValueError( + "Config name is missing." + f"\nPlease pick one among the available configs: {list(self.builder_configs.keys())}" + + f"\nExample of usage:\n\t`{example_of_usage}`" + ) + else: + builder_config = self.BUILDER_CONFIGS[0] + logger.info( + f"No config specified, defaulting to the single config: {self.dataset_name}/{builder_config.name}" + ) + + # try to get config by name + if isinstance(config_name, str): + builder_config = self.builder_configs.get(config_name) + if builder_config is None and self.BUILDER_CONFIGS: + raise ValueError( + f"BuilderConfig '{config_name}' not found. Available: {list(self.builder_configs.keys())}" + ) + + # if not using an existing config, then create a new config on the fly + if not builder_config: + if config_name is not None: + config_kwargs["name"] = config_name + elif self.DEFAULT_CONFIG_NAME and not config_kwargs: + # Use DEFAULT_CONFIG_NAME only if no config_kwargs are passed + config_kwargs["name"] = self.DEFAULT_CONFIG_NAME + if "version" not in config_kwargs and hasattr(self, "VERSION") and self.VERSION: + config_kwargs["version"] = self.VERSION + builder_config = self.BUILDER_CONFIG_CLASS(**config_kwargs) + + # otherwise use the config_kwargs to overwrite the attributes + else: + builder_config = copy.deepcopy(builder_config) if config_kwargs else builder_config + for key, value in config_kwargs.items(): + if value is not None: + if not hasattr(builder_config, key): + raise ValueError(f"BuilderConfig {builder_config} doesn't have a '{key}' key.") + setattr(builder_config, key, value) + + if not builder_config.name: + raise ValueError(f"BuilderConfig must have a name, got {builder_config.name}") + + # resolve data files if needed + builder_config._resolve_data_files( + base_path=self.base_path, + download_config=DownloadConfig(token=self.token, storage_options=self.storage_options), + ) + + # compute the config id that is going to be used for caching + if config_id is None: + config_id = builder_config.create_config_id( + config_kwargs, + custom_features=custom_features, + ) + is_custom = (config_id not in self.builder_configs) and config_id != "default" + if is_custom: + logger.info(f"Using custom data configuration {config_id}") + else: + if ( + builder_config.name in self.builder_configs + and builder_config != self.builder_configs[builder_config.name] + ): + raise ValueError( + "Cannot name a custom BuilderConfig the same as an available " + f"BuilderConfig. Change the name. Available BuilderConfigs: {list(self.builder_configs.keys())}" + ) + if not builder_config.version: + raise ValueError(f"BuilderConfig {builder_config.name} must have a version") + + return builder_config, config_id + + @classproperty + @classmethod + @memoize() + def builder_configs(cls) -> dict[str, BuilderConfig]: + """Dictionary of pre-defined configurations for this builder class.""" + configs = {config.name: config for config in cls.BUILDER_CONFIGS} + if len(configs) != len(cls.BUILDER_CONFIGS): + names = [config.name for config in cls.BUILDER_CONFIGS] + raise ValueError(f"Names in BUILDER_CONFIGS must not be duplicated. Got {names}") + return configs + + @property + def cache_dir(self): + return self._cache_dir + + def _use_legacy_cache_dir_if_possible(self, dataset_module: "DatasetModule"): + # Check for the legacy cache directory template (datasets<3.0.0) + self._legacy_relative_data_dir = ( + self._check_legacy_cache2(dataset_module) or self._check_legacy_cache() or None + ) + self._cache_dir = self._build_cache_dir() + self._output_dir = self._cache_dir + + def _relative_data_dir(self, with_version=True, with_hash=True) -> str: + """Relative path of this dataset in cache_dir: + Will be: + self.dataset_name/self.config.version/self.hash/ + or if a repo_id with a namespace has been specified: + self.namespace___self.dataset_name/self.config.version/self.hash/ + If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped. + """ + if self._legacy_relative_data_dir is not None and with_version and with_hash: + return self._legacy_relative_data_dir + + namespace = self.repo_id.split("/")[0] if self.repo_id and self.repo_id.count("/") > 0 else None + builder_data_dir = self.dataset_name if namespace is None else f"{namespace}___{self.dataset_name}" + builder_data_dir = posixpath.join(builder_data_dir, self.config_id) + if with_version: + builder_data_dir = posixpath.join(builder_data_dir, str(self.config.version)) + if with_hash and self.hash and isinstance(self.hash, str): + builder_data_dir = posixpath.join(builder_data_dir, self.hash) + return builder_data_dir + + def _build_cache_dir(self): + """Return the data directory for the current version.""" + builder_data_dir = posixpath.join(self._cache_dir_root, self._relative_data_dir(with_version=False)) + version_data_dir = posixpath.join(self._cache_dir_root, self._relative_data_dir(with_version=True)) + + def _other_versions_on_disk(): + """Returns previous versions on disk.""" + if not os.path.exists(builder_data_dir): + return [] + + version_dirnames = [] + for dir_name in os.listdir(builder_data_dir): + try: + version_dirnames.append((utils.Version(dir_name), dir_name)) + except ValueError: # Invalid version (ex: incomplete data dir) + pass + version_dirnames.sort(reverse=True) + return version_dirnames + + # Check and warn if other versions exist + if not is_remote_url(builder_data_dir): + version_dirs = _other_versions_on_disk() + if version_dirs: + other_version = version_dirs[0][0] + if other_version != self.config.version: + warn_msg = ( + f"Found a different version {str(other_version)} of dataset {self.dataset_name} in " + f"cache_dir {self._cache_dir_root}. Using currently defined version " + f"{str(self.config.version)}." + ) + logger.warning(warn_msg) + + return version_data_dir + + @abc.abstractmethod + def _info(self) -> DatasetInfo: + """Construct the DatasetInfo object. See `DatasetInfo` for details. + + Warning: This function is only called once and the result is cached for all + following .info() calls. + + Returns: + info: (DatasetInfo) The dataset information + """ + raise NotImplementedError + + @classmethod + def get_imported_module_dir(cls): + """Return the path of the module of this class or subclass.""" + return os.path.dirname(inspect.getfile(inspect.getmodule(cls))) + + def _rename(self, src: str, dst: str): + rename(self._fs, src, dst) + + def download_and_prepare( + self, + output_dir: Optional[str] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + verification_mode: Optional[Union[VerificationMode, str]] = None, + dl_manager: Optional[DownloadManager] = None, + base_path: Optional[str] = None, + file_format: str = "arrow", + max_shard_size: Optional[Union[int, str]] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **download_and_prepare_kwargs, + ): + """Downloads and prepares dataset for reading. + + Args: + output_dir (`str`, *optional*): + Output directory for the dataset. + Default to this builder's `cache_dir`, which is inside `~/.cache/huggingface/datasets` by default. + + + download_config (`DownloadConfig`, *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, *optional*): + Select the download/generate mode, default to `REUSE_DATASET_IF_EXISTS`. + verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`): + Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...). + + + dl_manager (`DownloadManager`, *optional*): + Specific `DownloadManger` to use. + base_path (`str`, *optional*): + Base path for relative paths that are used to download files. This can be a remote url. + If not specified, the value of the `base_path` attribute (`self.base_path`) will be used instead. + file_format (`str`, *optional*): + Format of the data files in which the dataset will be written. + Supported formats: "arrow", "parquet". Default to "arrow" format. + If the format is "parquet", then image and audio data are embedded into the Parquet files instead of pointing to local files. + + + max_shard_size (`Union[str, int]`, *optional*): + Maximum number of bytes written per shard, default is "500MB". + The size is based on uncompressed data size, so in practice your shard files may be smaller than + `max_shard_size` thanks to Parquet compression for example. + + + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + Multiprocessing is disabled by default. + + + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the caching file-system backend, if any. + + + **download_and_prepare_kwargs (additional keyword arguments): Keyword arguments. + + Example: + + Download and prepare the dataset as Arrow files that can be loaded as a Dataset using `builder.as_dataset()`: + + ```py + >>> from datasets import load_dataset_builder + >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes") + >>> builder.download_and_prepare() + ``` + + Download and prepare the dataset as sharded Parquet files locally: + + ```py + >>> from datasets import load_dataset_builder + >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes") + >>> builder.download_and_prepare("./output_dir", file_format="parquet") + ``` + + Download and prepare the dataset as sharded Parquet files in a cloud storage: + + ```py + >>> from datasets import load_dataset_builder + >>> storage_options = {"key": aws_access_key_id, "secret": aws_secret_access_key} + >>> builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes") + >>> builder.download_and_prepare("s3://my-bucket/my_rotten_tomatoes", storage_options=storage_options, file_format="parquet") + ``` + """ + output_dir = output_dir if output_dir is not None else self._cache_dir + # output_dir can be a remote bucket on GCS or S3 + fs, output_dir = url_to_fs(output_dir, **(storage_options or {})) + self._fs = fs + self._output_dir = output_dir if not is_remote_filesystem(self._fs) else self._fs.unstrip_protocol(output_dir) + + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + verification_mode = VerificationMode(verification_mode or VerificationMode.BASIC_CHECKS) + base_path = base_path if base_path is not None else self.base_path + + if file_format is not None and file_format not in ["arrow", "parquet"]: + raise ValueError(f"Unsupported file_format: {file_format}. Expected 'arrow' or 'parquet'") + self._file_format = file_format + + if self._fs._strip_protocol(self._output_dir) == "": + # We don't support the root directory, because it has no dirname, + # and we need a dirname to use a .incomplete directory + # when the dataset is being written + raise RuntimeError( + f"Unable to download and prepare the dataset at the root {self._output_dir}. " + f"Please specify a subdirectory, e.g. '{self._output_dir + self.dataset_name}'" + ) + + if dl_manager is None: + if download_config is None: + download_config = DownloadConfig( + cache_dir=self._cache_downloaded_dir, + force_download=download_mode == DownloadMode.FORCE_REDOWNLOAD, + force_extract=download_mode == DownloadMode.FORCE_REDOWNLOAD, + use_etag=False, + num_proc=num_proc, + token=self.token, + storage_options=self.storage_options, + ) # We don't use etag for data files to speed up the process + + dl_manager = DownloadManager( + dataset_name=self.dataset_name, + download_config=download_config, + data_dir=self.config.data_dir, + base_path=base_path, + record_checksums=self._record_checksums, + ) + + is_local = not is_remote_filesystem(self._fs) + self.dl_manager = dl_manager + + # Prevent parallel local disk operations + if is_local: + # Create parent directory of the output_dir to put the lock file in there + Path(self._output_dir).parent.mkdir(parents=True, exist_ok=True) + lock_path = self._output_dir + "_builder.lock" + + # File locking only with local paths; no file locking on GCS or S3 + with FileLock(lock_path) if is_local else contextlib.nullcontext(): + # Check if the data already exists + data_exists = self._fs.exists(posixpath.join(self._output_dir, config.DATASET_INFO_FILENAME)) + if data_exists and download_mode == DownloadMode.REUSE_DATASET_IF_EXISTS: + logger.info(f"Found cached dataset {self.dataset_name} ({self._output_dir})") + # We need to update the info in case some splits were added in the meantime + # for example when calling load_dataset from multiple workers. + self.info = self._load_info() + return + + logger.info(f"Generating dataset {self.dataset_name} ({self._output_dir})") + if is_local: # if cache dir is local, check for available space + if not has_sufficient_disk_space( + self.info.size_in_bytes or 0, directory=Path(self._output_dir).parent + ): + raise OSError( + f"Not enough disk space. Needed: {size_str(self.info.size_in_bytes or 0)} (download: {size_str(self.info.download_size or 0)}, generated: {size_str(self.info.dataset_size or 0)}" + ) + + @contextlib.contextmanager + def incomplete_dir(dirname): + """Create temporary dir for dirname and rename on exit.""" + if not is_local: + self._fs.makedirs(dirname, exist_ok=True) + yield dirname + else: + tmp_dir = dirname + ".incomplete" + os.makedirs(tmp_dir, exist_ok=True) + try: + yield tmp_dir + if os.path.isdir(dirname): + shutil.rmtree(dirname) + # LocalFileSystem.mv does copy + rm, it is more efficient to simply rename a local directory + shutil.move(tmp_dir, dirname) + finally: + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + + # Print is intentional: we want this to always go to stdout so user has + # information needed to cancel download/preparation if needed. + # This comes right before the progress bar. + if self.info.size_in_bytes: + logger.info( + f"Downloading and preparing dataset {self.dataset_name}/{self.config.name} " + f"(download: {size_str(self.info.download_size)}, generated: {size_str(self.info.dataset_size)}, " + f"total: {size_str(self.info.size_in_bytes)}) to {self._output_dir}..." + ) + else: + _dest = self._fs._strip_protocol(self._output_dir) if is_local else self._output_dir + logger.info(f"Downloading and preparing dataset {self.dataset_name}/{self.config.name} to {_dest}...") + + # Create a tmp dir and rename to self._output_dir on successful exit. + with incomplete_dir(self._output_dir) as tmp_output_dir: + # Temporarily assign _output_dir to tmp_data_dir to avoid having to forward + # it to every sub function. + with temporary_assignment(self, "_output_dir", tmp_output_dir): + prepare_split_kwargs = {"file_format": file_format} + if max_shard_size is not None: + prepare_split_kwargs["max_shard_size"] = max_shard_size + if num_proc is not None: + prepare_split_kwargs["num_proc"] = num_proc + self._download_and_prepare( + dl_manager=dl_manager, + verification_mode=verification_mode, + **prepare_split_kwargs, + **download_and_prepare_kwargs, + ) + # Sync info + self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values()) + if dl_manager.record_checksums: + self.info.download_checksums = dl_manager.get_recorded_sizes_checksums() + if self.info.download_size is not None: + self.info.size_in_bytes = self.info.dataset_size + self.info.download_size + # Save info + self._save_info() + + logger.info( + f"Dataset {self.dataset_name} downloaded and prepared to {self._output_dir}. " + f"Subsequent calls will reuse this data." + ) + + def _download_and_prepare(self, dl_manager, verification_mode, **prepare_split_kwargs): + """Downloads and prepares dataset for reading. + + This is the internal implementation to overwrite called when user calls + `download_and_prepare`. It should download all required data and generate + the pre-processed datasets files. + + Args: + dl_manager ([`DownloadManager`]): + `DownloadManager` used to download and cache data. + verification_mode ([`VerificationMode`]): + if `ALL_CHECKS`, perform all the verifications including checksums. + if `BASIC_CHECKS`, do not perform checksums, only perform split tests. + if `NO_CHECKS`, do not perform any verification. + prepare_split_kwargs: Additional options, such as `file_format`, `max_shard_size` + """ + # Generating data for all splits + split_dict = SplitDict(dataset_name=self.dataset_name) + split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs) + split_generators: list[SplitGenerator] = self._split_generators(dl_manager, **split_generators_kwargs) + + # Checksums verification + if verification_mode == VerificationMode.ALL_CHECKS and dl_manager.record_checksums: + verify_checksums( + self.info.download_checksums, dl_manager.get_recorded_sizes_checksums(), "dataset source files" + ) + + # Build splits + for split_generator in split_generators: + if str(split_generator.split_info.name).lower() == "all": + raise ValueError( + "`all` is a special split keyword corresponding to the " + "union of all splits, so cannot be used as key in " + "._split_generator()." + ) + + logger.info(f"Generating {split_generator.split_info.name} split") + split_dict.add(split_generator.split_info) + + try: + # Prepare split will record examples associated to the split + self._prepare_split(split_generator, **prepare_split_kwargs) + except OSError as e: + raise OSError("Cannot find data file. " + "\nOriginal error:\n" + str(e)) from None + dl_manager.manage_extracted_files() + + if verification_mode == VerificationMode.BASIC_CHECKS or verification_mode == VerificationMode.ALL_CHECKS: + verify_splits(self.info.splits, split_dict) + + # Update the info object with the splits. + self.info.splits = split_dict + self.info.download_size = dl_manager.downloaded_size + + def _load_info(self) -> DatasetInfo: + return DatasetInfo.from_directory(self._output_dir, storage_options=self._fs.storage_options) + + def _save_info(self): + file_lock = ( + FileLock(self._output_dir + "_info.lock") + if not is_remote_filesystem(self._fs) + else contextlib.nullcontext() + ) + with file_lock: + self.info.write_to_directory(self._output_dir, storage_options=self._fs.storage_options) + + def _make_split_generators_kwargs(self, prepare_split_kwargs): + """Get kwargs for `self._split_generators()` from `prepare_split_kwargs`.""" + del prepare_split_kwargs + return {} + + def as_dataset( + self, + split: Optional[Union[str, Split, list[str], list[Split]]] = None, + in_memory=False, + ) -> Union[Dataset, DatasetDict]: + """Return a Dataset for the specified split. + + Args: + split (`datasets.Split`): + Which subset of the data to return. + verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`): + Verification mode determining the checks to run on the + downloaded/processed dataset information (checksums/size/splits/...). + + + in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + + Returns: + datasets.Dataset + + Example: + + ```py + >>> from datasets import load_dataset_builder + >>> builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes') + >>> builder.download_and_prepare() + >>> ds = builder.as_dataset(split='train') + >>> ds + Dataset({ + features: ['text', 'label'], + num_rows: 8530 + }) + ``` + """ + if self._file_format is not None and self._file_format != "arrow": + raise FileFormatError('Loading a dataset not written in the "arrow" format is not supported.') + if is_remote_filesystem(self._fs): + raise NotImplementedError(f"Loading a dataset cached in a {type(self._fs).__name__} is not supported.") + if not os.path.exists(self._output_dir): + raise FileNotFoundError( + f"Dataset {self.dataset_name}: could not find data in {self._output_dir}. Please make sure to call " + "builder.download_and_prepare(), or use " + "datasets.load_dataset() before trying to access the Dataset object." + ) + + logger.debug(f"Constructing Dataset for split {split or ', '.join(self.info.splits)}, from {self._output_dir}") + + # By default, return all splits + if split is None: + split = {s: s for s in self.info.splits} + + # Create a dataset for each of the given splits + datasets = map_nested( + partial( + self._build_single_dataset, + in_memory=in_memory, + ), + split, + map_tuple=True, + disable_tqdm=True, + ) + if isinstance(datasets, dict): + datasets = DatasetDict(datasets) + return datasets + + def _build_single_dataset( + self, + split: Union[str, ReadInstruction, Split], + in_memory: bool = False, + ): + """as_dataset for a single split.""" + if not isinstance(split, ReadInstruction): + split = str(split) + if split == "all": + split = "+".join(self.info.splits.keys()) + split = Split(split) + + # Build base dataset + ds = self._as_dataset( + split=split, + in_memory=in_memory, + ) + return ds + + def _as_dataset(self, split: Union[ReadInstruction, Split] = Split.TRAIN, in_memory: bool = False) -> Dataset: + """Constructs a `Dataset`. + + This is the internal implementation to overwrite called when user calls + `as_dataset`. It should read the pre-processed datasets files and generate + the `Dataset` object. + + Args: + split (`datasets.Split`): + which subset of the data to read. + in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + + Returns: + `Dataset` + """ + cache_dir = self._fs._strip_protocol(self._output_dir) + dataset_name = self.dataset_name + if self._check_legacy_cache(): + dataset_name = self.name + dataset_kwargs = ArrowReader(cache_dir, self.info).read( + name=dataset_name, + instructions=split, + split_infos=self.info.splits.values(), + in_memory=in_memory, + ) + fingerprint = self._get_dataset_fingerprint(split) + return Dataset(fingerprint=fingerprint, **dataset_kwargs) + + def _get_dataset_fingerprint(self, split: Union[ReadInstruction, Split]) -> str: + """The dataset fingerprint is the hash of the relative directory dataset_name/config_name/version/hash, as well as the split specs.""" + hasher = Hasher() + hasher.update(Path(self._relative_data_dir()).as_posix()) + hasher.update(str(split)) # for example: train, train+test, train[:10%], test[:33%](pct1_dropremainder) + fingerprint = hasher.hexdigest() + return fingerprint + + def as_streaming_dataset( + self, + split: Optional[str] = None, + base_path: Optional[str] = None, + ) -> Union[dict[str, IterableDataset], IterableDataset]: + if is_remote_filesystem(self._fs): + raise NotImplementedError( + f"Loading a streaming dataset cached in a {type(self._fs).__name__} is not supported yet." + ) + + dl_manager = StreamingDownloadManager( + base_path=base_path or self.base_path, + download_config=DownloadConfig(token=self.token, storage_options=self.storage_options), + dataset_name=self.dataset_name, + data_dir=self.config.data_dir, + ) + splits_generators = {sg.name: sg for sg in self._split_generators(dl_manager)} + # By default, return all splits + if split is None: + splits_generator = splits_generators + elif split in splits_generators: + splits_generator = splits_generators[split] + else: + raise ValueError(f"Bad split: {split}. Available splits: {list(splits_generators)}") + + # Create a dataset for each of the given splits + datasets = map_nested( + self._as_streaming_dataset_single, + splits_generator, + map_tuple=True, + ) + if isinstance(datasets, dict): + datasets = IterableDatasetDict(datasets) + return datasets + + def _as_streaming_dataset_single( + self, + splits_generator, + ) -> IterableDataset: + ex_iterable = self._get_examples_iterable_for_split(splits_generator) + # add auth to be able to access and decode audio/image files from private repositories. + token_per_repo_id = {self.repo_id: self.token} if self.repo_id else {} + return IterableDataset( + ex_iterable, info=self.info, split=splits_generator.name, token_per_repo_id=token_per_repo_id + ) + + @abc.abstractmethod + def _split_generators(self, dl_manager: Union[DownloadManager, StreamingDownloadManager]): + """Specify feature dictionary generators and dataset splits. + + This function returns a list of `SplitGenerator`s defining how to generate + data and what splits to use. + + Example: + + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + gen_kwargs={'file': 'train_data.zip'}, + ), + datasets.SplitGenerator( + name=datasets.Split.TEST, + gen_kwargs={'file': 'test_data.zip'}, + ), + ] + + The above code will first call `_generate_examples(file='train_data.zip')` + to write the train data, then `_generate_examples(file='test_data.zip')` to + write the test data. + + Datasets are typically split into different subsets to be used at various + stages of training and evaluation. + + Note that for datasets without a `VALIDATION` split, you can use a + fraction of the `TRAIN` data for evaluation as you iterate on your model + so as not to overfit to the `TEST` data. + + For downloads and extractions, use the given `download_manager`. + Note that the `DownloadManager` caches downloads, so it is fine to have each + generator attempt to download the source data. + + A good practice is to download all data in this function, and then + distribute the relevant parts to each split with the `gen_kwargs` argument + + Args: + dl_manager (`Union[DownloadManager, StreamingDownloadManager]`): + Download manager to download the data + + Returns: + `list`. + """ + raise NotImplementedError() + + @abc.abstractmethod + def _prepare_split( + self, + split_generator: SplitGenerator, + file_format: str = "arrow", + max_shard_size: Optional[Union[str, int]] = None, + num_proc: Optional[int] = None, + **kwargs, + ): + """Generate the examples and record them on disk. + + Args: + split_generator (`SplitGenerator`): + Split generator to process + file_format (`str`, *optional*): + format of the data files in which the dataset will be written. + Supported formats: "arrow", "parquet". Default to "arrow" format. + max_shard_size (`Union[str, int]`, *optional*): + Maximum number of bytes written per shard, default is "500MB". + The size is based on uncompressed data size, so in practice your shard files may be smaller than + `max_shard_size` thanks to Parquet compression for example. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + Multiprocessing is disabled by default. + + + **kwargs: Additional kwargs forwarded from _download_and_prepare + """ + raise NotImplementedError() + + def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable: + """Generate the examples on the fly. + + Args: + split_generator (`SplitGenerator`): + Split generator to process + """ + raise NotImplementedError() + + +@dataclass +class Key: + original_shard_id: int + item_or_batch_id: int + + def __str__(self): + return str((self.original_shard_id, self.item_or_batch_id)) + + +class GeneratorBasedBuilder(DatasetBuilder): + """Base class for datasets with data generation based on dict generators. + + `GeneratorBasedBuilder` is a convenience class that abstracts away much + of the data writing and reading of `DatasetBuilder`. It expects subclasses to + implement generators of feature dictionaries across the dataset splits + (`_split_generators`). See the method docstrings for details. + """ + + def _generate_shards(self, **kwargs) -> Iterator[Union[str, dict[str, Any]]]: + """Default function generating shards paths for each `SplitGenerator`. + + This function is useful to list the original shards from where the data + comes from and is either converted to Arrow or streamed to an IterableDataset. + + This is optional and only used for certain utilities, but not in Dataset + nor IterableDataset. E.g. it's used to map original shard files to Parquet + files in the Dataset Viewer after conversion. + + Args: + **kwargs (additional keyword arguments): + Arguments forwarded from the SplitGenerator.gen_kwargs + + Yields: + shard: generally a string representing the shard path, or a dict + representing the shard in case of shards spanning intra or inter-files. + """ + raise NotImplementedError() + + @abc.abstractmethod + def _generate_examples(self, **kwargs) -> Iterator[tuple[Key, dict[str, Any]]]: + """Default function generating examples for each `SplitGenerator`. + + This function preprocess the examples from the raw data to the preprocessed + dataset files. + This function is called once for each `SplitGenerator` defined in + `_split_generators`. The examples yielded here will be written on + disk. + + Args: + **kwargs (additional keyword arguments): + Arguments forwarded from the SplitGenerator.gen_kwargs + + Yields: + key: `str` or `int`, a unique deterministic example identification key. + * Unique: An error will be raised if two examples are yield with the + same key. + * Deterministic: When generating the dataset twice, the same example + should have the same key. + Good keys can be the image id, or line number if examples are extracted + from a text file. + The key will be hashed and sorted to shuffle examples deterministically, + such as generating the dataset multiple times keep examples in the + same order. + example: `dict`, a feature dictionary + ready to be encoded and written to disk. The example will be + encoded with `self.info.features.encode_example({...})`. + """ + raise NotImplementedError() + + def _prepare_split( + self, + split_generator: SplitGenerator, + file_format="arrow", + num_proc: Optional[int] = None, + max_shard_size: Optional[Union[int, str]] = None, + ): + max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE) + + if self.info.splits is not None: + split_info = self.info.splits[split_generator.name] + else: + split_info = split_generator.split_info + + SUFFIX = "-JJJJJ-SSSSS-of-NNNNN" + fname = f"{self.dataset_name}-{split_generator.name}{SUFFIX}.{file_format}" + fpath = posixpath.join(self._output_dir, fname) + + if num_proc and num_proc > 1: + num_original_shards = _number_of_shards_in_gen_kwargs(split_generator.gen_kwargs) + if num_original_shards <= 1: + logger.warning( + f"Setting num_proc from {num_proc} back to 1 for the {split_info.name} split to disable multiprocessing as it only contains one shard." + ) + num_proc = 1 + elif num_original_shards < num_proc: + logger.warning( + f"Setting num_proc from {num_proc} to {num_original_shards} for the {split_info.name} split as it only contains {num_original_shards} shards." + ) + num_proc = num_original_shards + + pbar = hf_tqdm( + unit=" examples", + total=split_info.num_examples, + desc=f"Generating {split_info.name} split", + ) + + _prepare_split_args = { + "fpath": fpath, + "file_format": file_format, + "max_shard_size": max_shard_size, + "split_info": split_info, + } + + if num_proc is None or num_proc == 1: + result = None + gen_kwargs = split_generator.gen_kwargs + job_id = 0 + with pbar: + for job_id, done, content in self._prepare_split_single( + gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args + ): + if done: + result = content + else: + pbar.update(content) + # wrapping everything into lists for consistency with the multiprocessed code path + assert result is not None, "Failed to retrieve results from prepare_split" + ( + examples_per_job, + bytes_per_job, + features_per_job, + shards_per_job, + shard_lengths_per_job, + original_shards_per_job, + original_shard_lengths_per_job, + ) = ([item] for item in result) + else: + kwargs_per_job = [ + {"gen_kwargs": gen_kwargs, "job_id": job_id, **_prepare_split_args} + for job_id, gen_kwargs in enumerate( + _split_gen_kwargs(split_generator.gen_kwargs, max_num_jobs=num_proc) + ) + ] + num_jobs = len(kwargs_per_job) + + examples_per_job = [None] * num_jobs + bytes_per_job = [None] * num_jobs + features_per_job = [None] * num_jobs + shards_per_job = [None] * num_jobs + shard_lengths_per_job = [None] * num_jobs + original_shards_per_job = [None] * num_jobs + original_shard_lengths_per_job = [None] * num_jobs + + with Pool(num_proc) as pool: + with pbar: + for job_id, done, content in iflatmap_unordered( + pool, self._prepare_split_single, kwargs_iterable=kwargs_per_job + ): + if done: + # the content is the result of the job + ( + examples_per_job[job_id], + bytes_per_job[job_id], + features_per_job[job_id], + shards_per_job[job_id], + shard_lengths_per_job[job_id], + original_shards_per_job[job_id], + original_shard_lengths_per_job[job_id], + ) = content + else: + # the content is the number of examples progress update + pbar.update(content) + + assert None not in examples_per_job, ( + f"Failed to retrieve results from prepare_split: result list {examples_per_job} still contains None - at least one worker failed to return its results" + ) + + total_shards = sum(shards_per_job) + total_original_shards = sum(original_shards_per_job) + total_num_examples = sum(examples_per_job) + total_num_bytes = sum(bytes_per_job) + features = features_per_job[0] + + split_generator.split_info.num_examples = total_num_examples + split_generator.split_info.num_bytes = total_num_bytes + + # should rename everything at the end + logger.debug(f"Renaming {total_shards} shards.") + if total_shards > 1: + # use the -SSSSS-of-NNNNN pattern + + def _rename_shard(shard_and_job: tuple[int]): + shard_id, job_id = shard_and_job + global_shard_id = sum(shards_per_job[:job_id]) + shard_id + self._rename( + fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + fpath.replace("JJJJJ-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"), + ) + + shards_and_jobs = [ + (shard_id, job_id) + for job_id, num_shards in enumerate(shards_per_job) + for shard_id in range(num_shards) + ] + thread_map(_rename_shard, shards_and_jobs, disable=True, max_workers=64) + + split_generator.split_info.shard_lengths = [ + shard_length for shard_lengths in shard_lengths_per_job for shard_length in shard_lengths + ] + else: + # don't use any pattern + shard_id, job_id = 0, 0 + self._rename( + fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + fpath.replace(SUFFIX, ""), + ) + + if total_original_shards > 1 and config.SAVE_ORIGINAL_SHARD_LENGTHS: + split_generator.split_info.original_shard_lengths = [ + original_shard_length + for original_shard_lengths in original_shard_lengths_per_job + for original_shard_length in original_shard_lengths + ] + + if self.info.features is None: + self.info.features = features + + def _prepare_split_single( + self, + gen_kwargs: dict, + fpath: str, + file_format: str, + max_shard_size: int, + split_info: SplitInfo, + job_id: int, + ) -> Iterator[tuple[int, bool, tuple[int, int, Features, int, int, int]]]: + generator = self._generate_examples(**gen_kwargs) + writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter + embed_local_files = file_format == "parquet" + shard_lengths = [] + original_shard_lengths = [] + total_num_examples, total_num_bytes = 0, 0 + + shard_id = 0 + original_shard_id = 0 + num_examples_progress_update = 0 + try: + writer = writer_class( + features=self.info.features, + path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + writer_batch_size=self._writer_batch_size, + storage_options=self._fs.storage_options, + embed_local_files=embed_local_files, + ) + try: + _time = time.time() + for key, record in generator: + if isinstance(key, Key): # old custom builders may not use Key + original_shard_id = key.original_shard_id + if max_shard_size is not None and writer._num_bytes > max_shard_size: + num_examples, num_bytes = writer.finalize() + writer.close() + shard_lengths.append(num_examples) + total_num_examples += num_examples + total_num_bytes += num_bytes + shard_id += 1 + writer = writer_class( + features=writer._features, + path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + writer_batch_size=self._writer_batch_size, + storage_options=self._fs.storage_options, + embed_local_files=embed_local_files, + ) + example = self.info.features.encode_example(record) if self.info.features is not None else record + writer.write(example) + if len(original_shard_lengths) <= original_shard_id: + original_shard_lengths.extend([0] * (1 + original_shard_id - len(original_shard_lengths))) + original_shard_lengths[original_shard_id] += 1 + num_examples_progress_update += 1 + if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL: + _time = time.time() + yield job_id, False, num_examples_progress_update + num_examples_progress_update = 0 + finally: + yield job_id, False, num_examples_progress_update + num_shards = shard_id + 1 + num_original_shards = original_shard_id + 1 + num_examples, num_bytes = writer.finalize() + writer.close() + shard_lengths.append(num_examples) + total_num_examples += num_examples + total_num_bytes += num_bytes + except Exception as e: + # Ignore the writer's error for no examples written to the file if this error was caused by the error in _generate_examples before the first example was yielded + if isinstance(e, SchemaInferenceError) and e.__context__ is not None: + e = e.__context__ + raise DatasetGenerationError("An error occurred while generating the dataset") from e + + yield ( + job_id, + True, + ( + total_num_examples, + total_num_bytes, + writer._features, + num_shards, + shard_lengths, + num_original_shards, + original_shard_lengths, + ), + ) + + def _download_and_prepare(self, dl_manager, verification_mode, **prepare_splits_kwargs): + super()._download_and_prepare( + dl_manager, + verification_mode, + **prepare_splits_kwargs, + ) + + def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable: + return ExamplesIterable( + self._generate_examples, + split_generator.gen_kwargs, + generate_more_kwargs_fn=getattr(self, "_generate_more_gen_kwargs", None), + ) + + +class ArrowBasedBuilder(DatasetBuilder): + """Base class for datasets with data generation based on Arrow loading functions (CSV/JSON/Parquet).""" + + def _generate_shards(self, **kwargs) -> Iterator[Union[str, dict[str, Any]]]: + """Default function generating shards paths for each `SplitGenerator`. + + This function is useful to list the original shards from where the data + comes from and is either converted to Arrow or streamed to an IterableDataset. + + This is optional and only used for certain utilities, but not in Dataset + nor IterableDataset. E.g. it's used to map original shard files to Parquet + files in the Dataset Viewer after conversion. + + Args: + **kwargs (additional keyword arguments): + Arguments forwarded from the SplitGenerator.gen_kwargs + + Yields: + shard: generally a string representing the shard path, or a dict + representing the shard in case of shards spanning intra or inter-files. + """ + raise NotImplementedError() + + @abc.abstractmethod + def _generate_tables(self, **kwargs) -> Iterator[tuple[Key, pa.Table]]: + """Default function generating examples for each `SplitGenerator`. + + This function preprocess the examples from the raw data to the preprocessed + dataset files. + This function is called once for each `SplitGenerator` defined in + `_split_generators`. The examples yielded here will be written on + disk. + + Args: + **kwargs (additional keyword arguments): + Arguments forwarded from the SplitGenerator.gen_kwargs + + Yields: + key: tuple[int, int] original_shard_id and table_idx within that shard + example: `pyarrow.Table`, a feature table + ready to be encoded and written to disk. + """ + raise NotImplementedError() + + def _prepare_split( + self, + split_generator: SplitGenerator, + file_format: str = "arrow", + num_proc: Optional[int] = None, + max_shard_size: Optional[Union[str, int]] = None, + ): + max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE) + + try: + split_info = self.info.splits[split_generator.name] + except Exception: + split_info = split_generator.split_info + + SUFFIX = "-JJJJJ-SSSSS-of-NNNNN" + fname = f"{self.dataset_name}-{split_generator.name}{SUFFIX}.{file_format}" + fpath = posixpath.join(self._output_dir, fname) + + if num_proc and num_proc > 1: + num_original_shards = _number_of_shards_in_gen_kwargs(split_generator.gen_kwargs) + if num_original_shards <= 1: + logger.warning( + f"Setting num_proc from {num_proc} back to 1 for the {split_info.name} split to disable multiprocessing as it only contains one shard." + ) + num_proc = 1 + elif num_original_shards < num_proc: + logger.warning( + f"Setting num_proc from {num_proc} to {num_original_shards} for the {split_info.name} split as it only contains {num_original_shards} shards." + ) + num_proc = num_original_shards + + pbar = hf_tqdm( + unit=" examples", + total=split_info.num_examples, + desc=f"Generating {split_info.name} split", + ) + + _prepare_split_args = { + "fpath": fpath, + "file_format": file_format, + "max_shard_size": max_shard_size, + } + + if num_proc is None or num_proc == 1: + result = None + gen_kwargs = split_generator.gen_kwargs + job_id = 0 + with pbar: + for job_id, done, content in self._prepare_split_single( + gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args + ): + if done: + result = content + else: + pbar.update(content) + # wrapping everything into lists for consistency with the multiprocessed code path + assert result is not None, "Failed to retrieve results from prepare_split" + ( + examples_per_job, + bytes_per_job, + features_per_job, + shards_per_job, + shard_lengths_per_job, + original_shards_per_job, + original_shard_lengths_per_job, + ) = ([item] for item in result) + else: + kwargs_per_job = [ + {"gen_kwargs": gen_kwargs, "job_id": job_id, **_prepare_split_args} + for job_id, gen_kwargs in enumerate( + _split_gen_kwargs(split_generator.gen_kwargs, max_num_jobs=num_proc) + ) + ] + num_jobs = len(kwargs_per_job) + + examples_per_job = [None] * num_jobs + bytes_per_job = [None] * num_jobs + features_per_job = [None] * num_jobs + shards_per_job = [None] * num_jobs + shard_lengths_per_job = [None] * num_jobs + original_shards_per_job = [None] * num_jobs + original_shard_lengths_per_job = [None] * num_jobs + + with Pool(num_proc) as pool: + with pbar: + for job_id, done, content in iflatmap_unordered( + pool, self._prepare_split_single, kwargs_iterable=kwargs_per_job + ): + if done: + # the content is the result of the job + ( + examples_per_job[job_id], + bytes_per_job[job_id], + features_per_job[job_id], + shards_per_job[job_id], + shard_lengths_per_job[job_id], + original_shards_per_job[job_id], + original_shard_lengths_per_job[job_id], + ) = content + else: + # the content is the number of examples progress update + pbar.update(content) + + assert None not in examples_per_job, ( + f"Failed to retrieve results from prepare_split: result list {examples_per_job} still contains None - at least one worker failed to return its results" + ) + + total_shards = sum(shards_per_job) + total_original_shards = sum(original_shards_per_job) + total_num_examples = sum(examples_per_job) + total_num_bytes = sum(bytes_per_job) + features = features_per_job[0] + + split_generator.split_info.num_examples = total_num_examples + split_generator.split_info.num_bytes = total_num_bytes + + # should rename everything at the end + logger.debug(f"Renaming {total_shards} shards.") + if total_shards > 1: + # use the -SSSSS-of-NNNNN pattern + + def _rename_shard(shard_id_and_job: tuple[int]): + shard_id, job_id = shard_id_and_job + global_shard_id = sum(shards_per_job[:job_id]) + shard_id + self._rename( + fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + fpath.replace("JJJJJ-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"), + ) + + shard_ids_and_jobs = [ + (shard_id, job_id) + for job_id, num_shards in enumerate(shards_per_job) + for shard_id in range(num_shards) + ] + thread_map(_rename_shard, shard_ids_and_jobs, disable=True, max_workers=64) + + split_generator.split_info.shard_lengths = [ + shard_length for shard_lengths in shard_lengths_per_job for shard_length in shard_lengths + ] + else: + # don't use any pattern + shard_id, job_id = 0, 0 + self._rename( + fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + fpath.replace(SUFFIX, ""), + ) + + if total_original_shards > 1 and config.SAVE_ORIGINAL_SHARD_LENGTHS: + split_generator.split_info.original_shard_lengths = [ + original_shard_length + for original_shard_lengths in original_shard_lengths_per_job + for original_shard_length in original_shard_lengths + ] + + if self.info.features is None: + self.info.features = features + + def _prepare_split_single( + self, gen_kwargs: dict, fpath: str, file_format: str, max_shard_size: int, job_id: int + ) -> Iterator[tuple[int, bool, tuple[int, int, Features, int, int, int]]]: + gen_kwargs = {k: tracked_list(v) if isinstance(v, list) else v for k, v in gen_kwargs.items()} + generator = self._generate_tables(**gen_kwargs) + writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter + embed_local_files = file_format == "parquet" + shard_lengths = [] + original_shard_lengths = [] + total_num_examples, total_num_bytes = 0, 0 + + shard_id = 0 + original_shard_id = 0 + num_examples_progress_update = 0 + try: + writer = writer_class( + features=self.info.features, + path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + writer_batch_size=self._writer_batch_size, + storage_options=self._fs.storage_options, + embed_local_files=embed_local_files, + ) + try: + _time = time.time() + for key, table in generator: + if isinstance(key, Key): # old custom builders may not use Key + original_shard_id = key.original_shard_id + if max_shard_size is not None and writer._num_bytes > max_shard_size: + num_examples, num_bytes = writer.finalize() + writer.close() + shard_lengths.append(num_examples) + total_num_examples += num_examples + total_num_bytes += num_bytes + shard_id += 1 + writer = writer_class( + features=writer._features, + path=fpath.replace("SSSSS", f"{shard_id:05d}").replace("JJJJJ", f"{job_id:05d}"), + writer_batch_size=self._writer_batch_size, + storage_options=self._fs.storage_options, + embed_local_files=embed_local_files, + ) + try: + if len(table) == 1: + writer.write_row(table) + else: + writer.write_table(table) + except CastError as cast_error: + raise DatasetGenerationCastError.from_cast_error( + cast_error=cast_error, + builder_name=self.info.builder_name, + gen_kwargs=gen_kwargs, + token=self.token, + ) + if len(original_shard_lengths) == original_shard_id: + original_shard_lengths.append(len(table)) + else: + original_shard_lengths[original_shard_id] += len(table) + num_examples_progress_update += len(table) + if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL: + _time = time.time() + yield job_id, False, num_examples_progress_update + num_examples_progress_update = 0 + finally: + yield job_id, False, num_examples_progress_update + num_shards = shard_id + 1 + num_original_shards = original_shard_id + 1 + num_examples, num_bytes = writer.finalize() + writer.close() + shard_lengths.append(num_examples) + total_num_examples += num_examples + total_num_bytes += num_bytes + except Exception as e: + # Ignore the writer's error for no examples written to the file if this error was caused by the error in _generate_examples before the first example was yielded + if isinstance(e, SchemaInferenceError) and e.__context__ is not None: + e = e.__context__ + if isinstance(e, DatasetGenerationError): + raise + raise DatasetGenerationError("An error occurred while generating the dataset") from e + + yield ( + job_id, + True, + ( + total_num_examples, + total_num_bytes, + writer._features, + num_shards, + shard_lengths, + num_original_shards, + original_shard_lengths, + ), + ) + + def _get_examples_iterable_for_split(self, split_generator: SplitGenerator) -> ExamplesIterable: + return ArrowExamplesIterable( + self._generate_tables, + kwargs=split_generator.gen_kwargs, + generate_more_kwargs_fn=getattr(self, "_generate_more_gen_kwargs", None), + ) + + +class _CountableBuilderMixin(DatasetBuilder): + @abc.abstractmethod + def _generate_num_examples(self, **kwargs) -> Iterator[int]: + raise NotImplementedError() + + def count_examples(self, dl_manager: DownloadManager) -> dict[str, int]: + split_generators_kwargs = self._make_split_generators_kwargs({}) + split_generators: list[SplitGenerator] = self._split_generators(dl_manager, **split_generators_kwargs) + return {split_generator.name: self._count_examples(split_generator) for split_generator in split_generators} + + def _count_examples(self, split_generator: SplitGenerator) -> int: + max_workers = min(32, os.cpu_count() + 4) + return sum( + thread_map( + self._count_examples_single, + _split_gen_kwargs(split_generator.gen_kwargs, max_workers), + delay=5, + desc=f"Counting rows for split={split_generator.name}", + ) + ) + + def _count_examples_single(self, gen_kwargs: dict[str, Any]) -> int: + return sum(self._generate_num_examples(**gen_kwargs)) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/combine.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/combine.py new file mode 100644 index 0000000000000000000000000000000000000000..7b6c64b4cc16d83ac57bb9c4f6883e7ada7f19f7 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/combine.py @@ -0,0 +1,232 @@ +from typing import Optional, TypeVar + +from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets +from .dataset_dict import DatasetDict, IterableDatasetDict +from .info import DatasetInfo +from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets +from .splits import NamedSplit +from .utils import logging +from .utils.py_utils import Literal + + +logger = logging.get_logger(__name__) + + +DatasetType = TypeVar("DatasetType", Dataset, IterableDataset) + + +def interleave_datasets( + datasets: list[DatasetType], + probabilities: Optional[list[float]] = None, + seed: Optional[int] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + stopping_strategy: Literal[ + "first_exhausted", "all_exhausted", "all_exhausted_without_replacement" + ] = "first_exhausted", +) -> DatasetType: + """ + Interleave several datasets (sources) into a single dataset. + The new dataset is constructed by alternating between the sources to get the examples. + + You can use this function on a list of [`Dataset`] objects, or on a list of [`IterableDataset`] objects. + + - If `probabilities` is `None` (default) the new dataset is constructed by cycling between each source to get the examples. + - If `probabilities` is not `None`, the new dataset is constructed by getting examples from a random source at a time according to the provided probabilities. + + The resulting dataset ends when one of the source datasets runs out of examples except when `oversampling` is `True`, + in which case, the resulting dataset ends when all datasets have ran out of examples at least one time. + + Note for iterable datasets: + + * The resulting dataset's `num_shards` is the minimum of each dataset's `num_shards` to ensure good parallelism. + If some of your datasets have a very low number of shards, you may use [`IterableDataset.reshard`]. + * In a distributed setup or in PyTorch DataLoader workers, the stopping strategy is applied per process. + Therefore the "first_exhausted" strategy on an sharded iterable dataset can generate less samples in total (up to 1 missing sample per subdataset per worker). + + Args: + datasets (`List[Dataset]` or `List[IterableDataset]`): + List of datasets to interleave. + probabilities (`List[float]`, *optional*, defaults to `None`): + If specified, the new dataset is constructed by sampling + examples from one source at a time according to these probabilities. + seed (`int`, *optional*, defaults to `None`): + The random seed used to choose a source for each example. + info ([`DatasetInfo`], *optional*): + Dataset information, like description, citation, etc. + + split ([`NamedSplit`], *optional*): + Name of the dataset split. + + stopping_strategy (`str`, defaults to `first_exhausted`): + Three strategies are proposed right now, `first_exhausted`, `all_exhausted` and `all_exhausted_without_replacement`. + By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples. + If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once. + When strategy is `all_exhausted_without_replacement` we make sure that each sample in each dataset is sampled only once. + Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous: + - with no probabilities, the resulting dataset will have `max_length_datasets*nb_dataset` samples. + - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting. + Returns: + [`Dataset`] or [`IterableDataset`]: Return type depends on the input `datasets` + parameter. `Dataset` if the input is a list of `Dataset`, `IterableDataset` if the input is a list of + `IterableDataset`. + + Example: + + For regular datasets (map-style): + + ```python + >>> from datasets import Dataset, interleave_datasets + >>> d1 = Dataset.from_dict({"a": [0, 1, 2]}) + >>> d2 = Dataset.from_dict({"a": [10, 11, 12]}) + >>> d3 = Dataset.from_dict({"a": [20, 21, 22]}) + >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42, stopping_strategy="all_exhausted") + >>> dataset["a"] + [10, 0, 11, 1, 2, 20, 12, 10, 0, 1, 2, 21, 0, 11, 1, 2, 0, 1, 12, 2, 10, 0, 22] + >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42) + >>> dataset["a"] + [10, 0, 11, 1, 2] + >>> dataset = interleave_datasets([d1, d2, d3]) + >>> dataset["a"] + [0, 10, 20, 1, 11, 21, 2, 12, 22] + >>> dataset = interleave_datasets([d1, d2, d3], stopping_strategy="all_exhausted") + >>> dataset["a"] + [0, 10, 20, 1, 11, 21, 2, 12, 22] + >>> d1 = Dataset.from_dict({"a": [0, 1, 2]}) + >>> d2 = Dataset.from_dict({"a": [10, 11, 12, 13]}) + >>> d3 = Dataset.from_dict({"a": [20, 21, 22, 23, 24]}) + >>> dataset = interleave_datasets([d1, d2, d3]) + >>> dataset["a"] + [0, 10, 20, 1, 11, 21, 2, 12, 22] + >>> dataset = interleave_datasets([d1, d2, d3], stopping_strategy="all_exhausted") + >>> dataset["a"] + [0, 10, 20, 1, 11, 21, 2, 12, 22, 0, 13, 23, 1, 10, 24] + >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42) + >>> dataset["a"] + [10, 0, 11, 1, 2] + >>> dataset = interleave_datasets([d1, d2, d3], probabilities=[0.7, 0.2, 0.1], seed=42, stopping_strategy="all_exhausted") + >>> dataset["a"] + [10, 0, 11, 1, 2, 20, 12, 13, ..., 0, 1, 2, 0, 24] + For datasets in streaming mode (iterable): + + >>> from datasets import interleave_datasets + >>> d1 = load_dataset('allenai/c4', 'es', split='train', streaming=True) + >>> d2 = load_dataset('allenai/c4', 'fr', split='train', streaming=True) + >>> dataset = interleave_datasets([d1, d2]) + >>> iterator = iter(dataset) + >>> next(iterator) + {'text': 'Comprar Zapatillas para niña en chancla con goma por...'} + >>> next(iterator) + {'text': 'Le sacre de philippe ier, 23 mai 1059 - Compte Rendu...' + ``` + """ + from .arrow_dataset import Dataset + from .iterable_dataset import IterableDataset + + if not datasets: + raise ValueError("Unable to interleave an empty list of datasets.") + for i, dataset in enumerate(datasets): + if not isinstance(dataset, (Dataset, IterableDataset)): + if isinstance(dataset, (DatasetDict, IterableDatasetDict)): + if not dataset: + raise ValueError( + f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} " + "is an empty dataset dictionary." + ) + raise ValueError( + f"Dataset at position {i} has at least one split: {list(dataset)}\n" + f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(dataset))}']" + ) + raise ValueError( + f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(dataset).__name__}." + ) + if i == 0: + dataset_type, other_type = ( + (Dataset, IterableDataset) if isinstance(dataset, Dataset) else (IterableDataset, Dataset) + ) + elif not isinstance(dataset, dataset_type): + raise ValueError( + f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects." + ) + if stopping_strategy not in ["first_exhausted", "all_exhausted", "all_exhausted_without_replacement"]: + raise ValueError(f"{stopping_strategy} is not supported. Please enter a valid stopping_strategy.") + if dataset_type is Dataset: + return _interleave_map_style_datasets( + datasets, probabilities, seed, info=info, split=split, stopping_strategy=stopping_strategy + ) + else: + return _interleave_iterable_datasets( + datasets, + probabilities, + seed, + info=info, + split=split, + stopping_strategy=stopping_strategy, + ) + + +def concatenate_datasets( + dsets: list[DatasetType], + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + axis: int = 0, +) -> DatasetType: + """ + Concatenate several datasets (sources) into a single dataset. + + Use axis=0 to concatenate vertically (default), or axis=1 to concatenate horizontally. + + Note for iterable datasets: + + * if axis=0, the resulting dataset's `num_shards` is the sum of each dataset's `num_shards`. + * if axis=1, the resulting dataset has one (1) shard to not misalign data. + + Args: + dsets (`List[datasets.Dataset]` or `List[datasets.IterableDataset]`): + List of Datasets to concatenate. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + axis (`{0, 1}`, defaults to `0`): + Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns + (horizontally). + + + + Example: + + ```py + >>> ds3 = concatenate_datasets([ds1, ds2]) + ``` + """ + + if not dsets: + raise ValueError("Unable to concatenate an empty list of datasets.") + for i, dataset in enumerate(dsets): + if not isinstance(dataset, (Dataset, IterableDataset)): + if isinstance(dataset, (DatasetDict, IterableDatasetDict)): + if not dataset: + raise ValueError( + f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} " + "is an empty dataset dictionary." + ) + raise ValueError( + f"Dataset at position {i} has at least one split: {list(dataset)}\n" + f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(dataset))}']" + ) + raise ValueError( + f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(dataset).__name__}." + ) + if i == 0: + dataset_type, other_type = ( + (Dataset, IterableDataset) if isinstance(dataset, Dataset) else (IterableDataset, Dataset) + ) + elif not isinstance(dataset, dataset_type): + raise ValueError( + f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects." + ) + if dataset_type is Dataset: + return _concatenate_map_style_datasets(dsets, info=info, split=split, axis=axis) + else: + return _concatenate_iterable_datasets(dsets, info=info, split=split, axis=axis) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..905e753955a348a8e486302e1b6f5e8f53ec7bf4 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/__init__.py @@ -0,0 +1,13 @@ +from abc import ABC, abstractmethod +from argparse import ArgumentParser + + +class BaseDatasetsCLICommand(ABC): + @staticmethod + @abstractmethod + def register_subcommand(parser: ArgumentParser): + raise NotImplementedError() + + @abstractmethod + def run(self): + raise NotImplementedError() diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/datasets_cli.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/datasets_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..1d04d87ede1bfddc0886c0ae1686fa02ebd21768 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/datasets_cli.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +from argparse import ArgumentParser + +from datasets.commands.delete_from_hub import DeleteFromHubCommand +from datasets.commands.env import EnvironmentCommand +from datasets.commands.test import TestCommand +from datasets.utils.logging import set_verbosity_info + + +def parse_unknown_args(unknown_args): + return {key.lstrip("-"): value for key, value in zip(unknown_args[::2], unknown_args[1::2])} + + +def main(): + parser = ArgumentParser( + "HuggingFace Datasets CLI tool", usage="datasets-cli []", allow_abbrev=False + ) + commands_parser = parser.add_subparsers(help="datasets-cli command helpers") + set_verbosity_info() + + # Register commands + EnvironmentCommand.register_subcommand(commands_parser) + TestCommand.register_subcommand(commands_parser) + DeleteFromHubCommand.register_subcommand(commands_parser) + + # Parse args + args, unknown_args = parser.parse_known_args() + if not hasattr(args, "func"): + parser.print_help() + exit(1) + kwargs = parse_unknown_args(unknown_args) + + # Run + service = args.func(args, **kwargs) + service.run() + + +if __name__ == "__main__": + main() diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/delete_from_hub.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/delete_from_hub.py new file mode 100644 index 0000000000000000000000000000000000000000..126a029c246ab584df47c4ba1b908a798b9003b6 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/delete_from_hub.py @@ -0,0 +1,42 @@ +from argparse import ArgumentParser +from typing import Optional + +from datasets.commands import BaseDatasetsCLICommand +from datasets.hub import delete_from_hub + + +def _command_factory(args): + return DeleteFromHubCommand( + args.dataset_id, + args.config_name, + args.token, + args.revision, + ) + + +class DeleteFromHubCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser): + parser: ArgumentParser = parser.add_parser("delete_from_hub", help="Delete dataset config from the Hub") + parser.add_argument( + "dataset_id", help="source dataset ID, e.g. USERNAME/DATASET_NAME or ORGANIZATION/DATASET_NAME" + ) + parser.add_argument("config_name", help="config name to delete") + parser.add_argument("--token", help="access token to the Hugging Face Hub") + parser.add_argument("--revision", help="source revision") + parser.set_defaults(func=_command_factory) + + def __init__( + self, + dataset_id: str, + config_name: str, + token: Optional[str], + revision: Optional[str], + ): + self._dataset_id = dataset_id + self._config_name = config_name + self._token = token + self._revision = revision + + def run(self) -> None: + _ = delete_from_hub(self._dataset_id, self._config_name, revision=self._revision, token=self._token) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/env.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/env.py new file mode 100644 index 0000000000000000000000000000000000000000..40b2a3654c8f1c6c080222a80b12d108972a5dc9 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/env.py @@ -0,0 +1,41 @@ +import platform +from argparse import ArgumentParser + +import fsspec +import huggingface_hub +import pandas +import pyarrow + +from datasets import __version__ as version +from datasets.commands import BaseDatasetsCLICommand + + +def info_command_factory(_): + return EnvironmentCommand() + + +class EnvironmentCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser: ArgumentParser): + download_parser = parser.add_parser("env", help="Print relevant system environment info.") + download_parser.set_defaults(func=info_command_factory) + + def run(self): + info = { + "`datasets` version": version, + "Platform": platform.platform(), + "Python version": platform.python_version(), + "`huggingface_hub` version": huggingface_hub.__version__, + "PyArrow version": pyarrow.__version__, + "Pandas version": pandas.__version__, + "`fsspec` version": fsspec.__version__, + } + + print("\nCopy-and-paste the text below in your GitHub issue.\n") + print(self.format_dict(info)) + + return info + + @staticmethod + def format_dict(d): + return "\n".join([f"- {prop}: {val}" for prop, val in d.items()]) + "\n" diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/test.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/test.py new file mode 100644 index 0000000000000000000000000000000000000000..1aca3253df984fcbee2dec5e82f43e5bb71e6e56 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/commands/test.py @@ -0,0 +1,180 @@ +import logging +import os +from argparse import ArgumentParser +from collections.abc import Generator +from shutil import rmtree + +import datasets.config +from datasets.builder import DatasetBuilder +from datasets.commands import BaseDatasetsCLICommand +from datasets.download.download_manager import DownloadMode +from datasets.info import DatasetInfosDict +from datasets.load import dataset_module_factory, get_dataset_builder_class +from datasets.utils.info_utils import VerificationMode +from datasets.utils.logging import ERROR, get_logger + + +logger = get_logger(__name__) + + +def _test_command_factory(args): + return TestCommand( + args.dataset, + args.name, + args.cache_dir, + args.data_dir, + args.all_configs, + args.save_info or args.save_infos, + args.ignore_verifications, + args.force_redownload, + args.clear_cache, + args.num_proc, + ) + + +class TestCommand(BaseDatasetsCLICommand): + __test__ = False # to tell pytest it's not a test class + + @staticmethod + def register_subcommand(parser: ArgumentParser): + test_parser = parser.add_parser("test", help="Test dataset loading.") + test_parser.add_argument("--name", type=str, default=None, help="Dataset processing name") + test_parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="Cache directory where the datasets are stored.", + ) + test_parser.add_argument( + "--data_dir", + type=str, + default=None, + help="Can be used to specify a manual directory to get the files from.", + ) + test_parser.add_argument("--all_configs", action="store_true", help="Test all dataset configurations") + test_parser.add_argument( + "--save_info", action="store_true", help="Save the dataset infos in the dataset card (README.md)" + ) + test_parser.add_argument( + "--ignore_verifications", + action="store_true", + help="Run the test without checksums and splits checks.", + ) + test_parser.add_argument("--force_redownload", action="store_true", help="Force dataset redownload") + test_parser.add_argument( + "--clear_cache", + action="store_true", + help="Remove downloaded files and cached datasets after each config test", + ) + test_parser.add_argument("--num_proc", type=int, default=None, help="Number of processes") + # aliases + test_parser.add_argument("--save_infos", action="store_true", help="alias to save_info") + test_parser.add_argument("dataset", type=str, help="Name of the dataset to download") + test_parser.set_defaults(func=_test_command_factory) + + def __init__( + self, + dataset: str, + name: str, + cache_dir: str, + data_dir: str, + all_configs: bool, + save_infos: bool, + ignore_verifications: bool, + force_redownload: bool, + clear_cache: bool, + num_proc: int, + ): + self._dataset = dataset + self._name = name + self._cache_dir = cache_dir + self._data_dir = data_dir + self._all_configs = all_configs + self._save_infos = save_infos + self._ignore_verifications = ignore_verifications + self._force_redownload = force_redownload + self._clear_cache = clear_cache + self._num_proc = num_proc + if clear_cache and not cache_dir: + print( + "When --clear_cache is used, specifying a cache directory is mandatory.\n" + "The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.\n" + "Please provide a --cache_dir that will be used to test the dataset." + ) + exit(1) + if save_infos: + self._ignore_verifications = True + + def run(self): + logging.getLogger("filelock").setLevel(ERROR) + if self._name is not None and self._all_configs: + print("Both parameters `config` and `all_configs` can't be used at once.") + exit(1) + path, config_name = self._dataset, self._name + module = dataset_module_factory(path) + builder_cls = get_dataset_builder_class(module) + n_builders = len(builder_cls.BUILDER_CONFIGS) if self._all_configs and builder_cls.BUILDER_CONFIGS else 1 + + def get_builders() -> Generator[DatasetBuilder, None, None]: + if self._all_configs and builder_cls.BUILDER_CONFIGS: + for i, config in enumerate(builder_cls.BUILDER_CONFIGS): + if "config_name" in module.builder_kwargs: + yield builder_cls( + cache_dir=self._cache_dir, + data_dir=self._data_dir, + **module.builder_kwargs, + ) + else: + yield builder_cls( + config_name=config.name, + cache_dir=self._cache_dir, + data_dir=self._data_dir, + **module.builder_kwargs, + ) + else: + if "config_name" in module.builder_kwargs: + yield builder_cls(cache_dir=self._cache_dir, data_dir=self._data_dir, **module.builder_kwargs) + else: + yield builder_cls( + config_name=config_name, + cache_dir=self._cache_dir, + data_dir=self._data_dir, + **module.builder_kwargs, + ) + + for j, builder in enumerate(get_builders()): + print(f"Testing builder '{builder.config.name}' ({j + 1}/{n_builders})") + builder._record_checksums = os.path.exists( + os.path.join(builder.get_imported_module_dir(), datasets.config.DATASETDICT_INFOS_FILENAME) + ) # record checksums only if we need to update a (deprecated) dataset_infos.json + builder.download_and_prepare( + download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS + if not self._force_redownload + else DownloadMode.FORCE_REDOWNLOAD, + verification_mode=VerificationMode.NO_CHECKS + if self._ignore_verifications + else VerificationMode.ALL_CHECKS, + num_proc=self._num_proc, + ) + builder.as_dataset() + + # If save_infos=True, we create the dataset card (README.md) + # The dataset_infos are saved in the YAML part of the README.md + # This is to allow the user to upload them on HF afterwards. + if self._save_infos: + save_infos_dir = os.path.basename(path) if not os.path.isdir(path) else path + os.makedirs(save_infos_dir, exist_ok=True) + DatasetInfosDict(**{builder.config.name: builder.info}).write_to_directory(save_infos_dir) + print(f"Dataset card saved at {os.path.join(save_infos_dir, datasets.config.REPOCARD_FILENAME)}") + + # If clear_cache=True, the download folder and the dataset builder cache directory are deleted + if self._clear_cache: + if os.path.isdir(builder._cache_dir): + logger.warning(f"Clearing cache at {builder._cache_dir}") + rmtree(builder._cache_dir) + download_dir = os.path.join(self._cache_dir, datasets.config.DOWNLOADED_DATASETS_DIR) + if os.path.isdir(download_dir): + logger.warning(f"Clearing cache at {download_dir}") + rmtree(download_dir) + + print("Test successful.") diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/config.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/config.py new file mode 100644 index 0000000000000000000000000000000000000000..4764eda284fc3103dd55ac96cdb43ef6a0806f26 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/config.py @@ -0,0 +1,272 @@ +import importlib +import importlib.metadata +import logging +import os +import platform +from pathlib import Path +from typing import Optional + +from huggingface_hub import constants +from packaging import version + + +logger = logging.getLogger(__name__.split(".", 1)[0]) # to avoid circular import from .utils.logging + +# Datasets +S3_DATASETS_BUCKET_PREFIX = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets" +CLOUDFRONT_DATASETS_DISTRIB_PREFIX = "https://cdn-datasets.huggingface.co/datasets/datasets" +REPO_DATASETS_URL = "https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}" + +# Hub +HF_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co") +HUB_DATASETS_URL = HF_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}" +HUB_DATASETS_HFFS_URL = "hf://datasets/{repo_id}@{revision}/{path}" +HUB_DEFAULT_VERSION = "main" + +PY_VERSION = version.parse(platform.python_version()) + +# General environment variables accepted values for booleans +ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"} +ENV_VARS_FALSE_VALUES = {"0", "OFF", "NO", "FALSE"} +ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"}) +ENV_VARS_FALSE_AND_AUTO_VALUES = ENV_VARS_FALSE_VALUES.union({"AUTO"}) + + +# Imports +DILL_VERSION = version.parse(importlib.metadata.version("dill")) +FSSPEC_VERSION = version.parse(importlib.metadata.version("fsspec")) +PANDAS_VERSION = version.parse(importlib.metadata.version("pandas")) +PYARROW_VERSION = version.parse(importlib.metadata.version("pyarrow")) +HF_HUB_VERSION = version.parse(importlib.metadata.version("huggingface_hub")) + +USE_TF = os.environ.get("USE_TF", "AUTO").upper() +USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper() +USE_JAX = os.environ.get("USE_JAX", "AUTO").upper() + +TORCH_VERSION = "N/A" +TORCH_AVAILABLE = False + +if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES: + TORCH_AVAILABLE = importlib.util.find_spec("torch") is not None + if TORCH_AVAILABLE: + try: + TORCH_VERSION = version.parse(importlib.metadata.version("torch")) + logger.debug(f"PyTorch version {TORCH_VERSION} available.") + except importlib.metadata.PackageNotFoundError: + pass +else: + logger.info("Disabling PyTorch because USE_TF is set") + +POLARS_VERSION = "N/A" +POLARS_AVAILABLE = importlib.util.find_spec("polars") is not None + +if POLARS_AVAILABLE: + try: + POLARS_VERSION = version.parse(importlib.metadata.version("polars")) + logger.debug(f"Polars version {POLARS_VERSION} available.") + except importlib.metadata.PackageNotFoundError: + pass + + +DUCKDB_VERSION = "N/A" +DUCKDB_AVAILABLE = importlib.util.find_spec("duckdb") is not None + +if DUCKDB_AVAILABLE: + try: + DUCKDB_VERSION = version.parse(importlib.metadata.version("duckdb")) + logger.debug(f"Duckdb version {DUCKDB_VERSION} available.") + except importlib.metadata.PackageNotFoundError: + pass + +TF_VERSION = "N/A" +TF_AVAILABLE = False + +if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES: + TF_AVAILABLE = importlib.util.find_spec("tensorflow") is not None + if TF_AVAILABLE: + # For the metadata, we have to look for both tensorflow and tensorflow-cpu + for package in [ + "tensorflow", + "tensorflow-cpu", + "tensorflow-gpu", + "tf-nightly", + "tf-nightly-cpu", + "tf-nightly-gpu", + "intel-tensorflow", + "tensorflow-rocm", + "tensorflow-macos", + ]: + try: + TF_VERSION = version.parse(importlib.metadata.version(package)) + except importlib.metadata.PackageNotFoundError: + continue + else: + break + else: + TF_AVAILABLE = False + if TF_AVAILABLE: + if TF_VERSION.major < 2: + logger.info(f"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.") + TF_AVAILABLE = False + else: + logger.info(f"TensorFlow version {TF_VERSION} available.") +else: + logger.info("Disabling Tensorflow because USE_TORCH is set") + + +JAX_VERSION = "N/A" +JAX_AVAILABLE = False + +if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES: + JAX_AVAILABLE = importlib.util.find_spec("jax") is not None and importlib.util.find_spec("jaxlib") is not None + if JAX_AVAILABLE: + try: + JAX_VERSION = version.parse(importlib.metadata.version("jax")) + logger.info(f"JAX version {JAX_VERSION} available.") + except importlib.metadata.PackageNotFoundError: + pass +else: + logger.info("Disabling JAX because USE_JAX is set to False") + + +# Optional tools for data loading +SQLALCHEMY_AVAILABLE = importlib.util.find_spec("sqlalchemy") is not None + +# Optional tools for feature decoding +PIL_AVAILABLE = importlib.util.find_spec("PIL") is not None +IS_OPUS_SUPPORTED = True +IS_MP3_SUPPORTED = True +TORCHCODEC_AVAILABLE = importlib.util.find_spec("torchcodec") is not None +TORCHVISION_AVAILABLE = importlib.util.find_spec("torchvision") is not None +PDFPLUMBER_AVAILABLE = importlib.util.find_spec("pdfplumber") is not None +NIBABEL_AVAILABLE = importlib.util.find_spec("nibabel") is not None + +# Optional compression tools +RARFILE_AVAILABLE = importlib.util.find_spec("rarfile") is not None +ZSTANDARD_AVAILABLE = importlib.util.find_spec("zstandard") is not None +LZ4_AVAILABLE = importlib.util.find_spec("lz4") is not None +PY7ZR_AVAILABLE = importlib.util.find_spec("py7zr") is not None + +# Cache location +DEFAULT_XDG_CACHE_HOME = "~/.cache" +XDG_CACHE_HOME = os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME) +DEFAULT_HF_CACHE_HOME = os.path.join(XDG_CACHE_HOME, "huggingface") +HF_CACHE_HOME = os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME)) + +DEFAULT_HF_DATASETS_CACHE = os.path.join(HF_CACHE_HOME, "datasets") +HF_DATASETS_CACHE = Path(os.getenv("HF_DATASETS_CACHE", DEFAULT_HF_DATASETS_CACHE)) + +DEFAULT_HF_MODULES_CACHE = os.path.join(HF_CACHE_HOME, "modules") +HF_MODULES_CACHE = Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE)) + +DOWNLOADED_DATASETS_DIR = "downloads" +DEFAULT_DOWNLOADED_DATASETS_PATH = os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR) +DOWNLOADED_DATASETS_PATH = Path(os.getenv("HF_DATASETS_DOWNLOADED_DATASETS_PATH", DEFAULT_DOWNLOADED_DATASETS_PATH)) + +EXTRACTED_DATASETS_DIR = "extracted" +DEFAULT_EXTRACTED_DATASETS_PATH = os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR) +EXTRACTED_DATASETS_PATH = Path(os.getenv("HF_DATASETS_EXTRACTED_DATASETS_PATH", DEFAULT_EXTRACTED_DATASETS_PATH)) + +# Cached dataset info options +SAVE_ORIGINAL_SHARD_LENGTHS = False + +# Download count for the website +HF_UPDATE_DOWNLOAD_COUNTS = ( + os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES +) + +# For downloads and to check remote files metadata +HF_DATASETS_MULTITHREADING_MAX_WORKERS = 16 + +# Dataset viewer API +USE_PARQUET_EXPORT = True + +# Batch size constants. For more info, see: +# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations) +DEFAULT_MAX_BATCH_SIZE = 1000 + +DEFAULT_CDC_OPTIONS = {"min_chunk_size": 256 * 1024, "max_chunk_size": 1024 * 1024, "norm_level": 0} + +# Size of the preloaded record batch in `Dataset.__iter__` +ARROW_READER_BATCH_SIZE_IN_DATASET_ITER = 10 + +# Max uncompressed shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare) +MAX_SHARD_SIZE = "500MB" + +# Max uncompressed row group size in bytes (e.g. for parquet files in push_to_hub or download_and_prepare) +MAX_ROW_GROUP_SIZE = "100MB" + +# Parquet configuration +PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS = None +PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS = None +PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS = None +PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS = None + +# Arrow configuration +ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETS = 100 +ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETS = 100 +ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS = 100 +ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETS = 10 + +# Offline mode +_offline = os.environ.get("HF_DATASETS_OFFLINE") +HF_HUB_OFFLINE = constants.HF_HUB_OFFLINE if _offline is None else _offline.upper() in ENV_VARS_TRUE_VALUES +HF_DATASETS_OFFLINE = HF_HUB_OFFLINE # kept for backward-compatibility + +# Here, `True` will disable progress bars globally without possibility of enabling it +# programmatically. `False` will enable them without possibility of disabling them. +# If environment variable is not set (None), then the user is free to enable/disable +# them programmatically. +# TL;DR: env variable has priority over code +__HF_DATASETS_DISABLE_PROGRESS_BARS = os.environ.get("HF_DATASETS_DISABLE_PROGRESS_BARS") +HF_DATASETS_DISABLE_PROGRESS_BARS: Optional[bool] = ( + __HF_DATASETS_DISABLE_PROGRESS_BARS.upper() in ENV_VARS_TRUE_VALUES + if __HF_DATASETS_DISABLE_PROGRESS_BARS is not None + else None +) + +# In-memory +DEFAULT_IN_MEMORY_MAX_SIZE = 0 # Disabled +IN_MEMORY_MAX_SIZE = float(os.environ.get("HF_DATASETS_IN_MEMORY_MAX_SIZE", DEFAULT_IN_MEMORY_MAX_SIZE)) + +# File names +DATASET_ARROW_FILENAME = "dataset.arrow" +DATASET_INDICES_FILENAME = "indices.arrow" +DATASET_STATE_JSON_FILENAME = "state.json" +DATASET_INFO_FILENAME = "dataset_info.json" +DATASETDICT_INFOS_FILENAME = "dataset_infos.json" +LICENSE_FILENAME = "LICENSE" +DATASETDICT_JSON_FILENAME = "dataset_dict.json" +METADATA_CONFIGS_FIELD = "configs" +REPOCARD_FILENAME = "README.md" +REPOYAML_FILENAME = ".huggingface.yaml" + +MODULE_NAME_FOR_DYNAMIC_MODULES = "datasets_modules" + +MAX_DATASET_CONFIG_ID_READABLE_LENGTH = 255 + +# Temporary cache directory prefix +TEMP_CACHE_DIR_PREFIX = "hf_datasets-" + +# Streaming +STREAMING_READ_MAX_RETRIES = 20 +STREAMING_READ_RETRY_INTERVAL = 5 +STREAMING_READ_SERVER_UNAVAILABLE_RETRY_INTERVAL = 20 +STREAMING_READ_RATE_LIMIT_RETRY_INTERVAL = 60 +STREAMING_OPEN_MAX_RETRIES = 20 +STREAMING_OPEN_RETRY_INTERVAL = 5 + +# Datasets repositories exploration +ARCHIVES_MAX_NUMBER_FOR_MODULE_INFERENCE = 10 + +# Async map functions +MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL = 1000 + +# Progress bars +PBAR_REFRESH_TIME_INTERVAL = 0.05 # 20 progress updates per sec + +# Maximum number of uploaded files per commit +UPLOADS_MAX_NUMBER_PER_COMMIT = 50 + +# Backward compatibility +MAX_TABLE_NBYTES_FOR_PICKLING = 4 << 30 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/data_files.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/data_files.py new file mode 100644 index 0000000000000000000000000000000000000000..3e63c3ac65ffcaa6a6be5b9ef55c63a0c74ce4e6 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/data_files.py @@ -0,0 +1,829 @@ +import os +import re +from functools import partial +from glob import has_magic +from pathlib import Path, PurePath +from typing import Callable, Optional, Union + +import huggingface_hub +from fsspec.core import url_to_fs +from huggingface_hub import HfFileSystem +from packaging import version +from tqdm.contrib.concurrent import thread_map + +from . import config +from .download import DownloadConfig +from .naming import _split_re +from .splits import Split +from .utils import logging +from .utils import tqdm as hf_tqdm +from .utils.file_utils import _prepare_path_and_storage_options, is_local_path, is_relative_path, xbasename, xjoin +from .utils.py_utils import string_to_dict + + +SingleOriginMetadata = Union[tuple[str, str], tuple[str], tuple[()]] + + +SANITIZED_DEFAULT_SPLIT = str(Split.TRAIN) + + +logger = logging.get_logger(__name__) + + +class Url(str): + pass + + +class EmptyDatasetError(FileNotFoundError): + pass + + +SPLIT_PATTERN_SHARDED = "data/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*" + +SPLIT_KEYWORDS = { + Split.TRAIN: ["train", "training"], + Split.VALIDATION: ["validation", "valid", "dev", "val"], + Split.TEST: ["test", "testing", "eval", "evaluation"], +} +NON_WORDS_CHARS = "-._ 0-9" +if config.FSSPEC_VERSION < version.parse("2023.9.0"): + KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**[{sep}/]{keyword}[{sep}]*", "{keyword}[{sep}]*"] + KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [ + "{keyword}/**", + "{keyword}[{sep}]*/**", + "**[{sep}/]{keyword}/**", + "**[{sep}/]{keyword}[{sep}]*/**", + ] +elif config.FSSPEC_VERSION < version.parse("2023.12.0"): + KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**/*[{sep}/]{keyword}[{sep}]*", "{keyword}[{sep}]*"] + KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [ + "{keyword}/**/*", + "{keyword}[{sep}]*/**/*", + "**/*[{sep}/]{keyword}/**/*", + "**/*[{sep}/]{keyword}[{sep}]*/**/*", + ] +else: + KEYWORDS_IN_FILENAME_BASE_PATTERNS = ["**/{keyword}[{sep}]*", "**/*[{sep}]{keyword}[{sep}]*"] + KEYWORDS_IN_DIR_NAME_BASE_PATTERNS = [ + "**/{keyword}/**", + "**/{keyword}[{sep}]*/**", + "**/*[{sep}]{keyword}/**", + "**/*[{sep}]{keyword}[{sep}]*/**", + ] + +DEFAULT_SPLITS = [Split.TRAIN, Split.VALIDATION, Split.TEST] +DEFAULT_PATTERNS_SPLIT_IN_FILENAME = { + split: [ + pattern.format(keyword=keyword, sep=NON_WORDS_CHARS) + for keyword in SPLIT_KEYWORDS[split] + for pattern in KEYWORDS_IN_FILENAME_BASE_PATTERNS + ] + for split in DEFAULT_SPLITS +} +DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME = { + split: [ + pattern.format(keyword=keyword, sep=NON_WORDS_CHARS) + for keyword in SPLIT_KEYWORDS[split] + for pattern in KEYWORDS_IN_DIR_NAME_BASE_PATTERNS + ] + for split in DEFAULT_SPLITS +} + + +DEFAULT_PATTERNS_ALL = { + Split.TRAIN: ["**"], +} + +DEFAULT_PATTERNS_LOGS = {"logs": ["**/*.eval"]} + +ALL_SPLIT_PATTERNS = [SPLIT_PATTERN_SHARDED] +ALL_DEFAULT_PATTERNS = [ + DEFAULT_PATTERNS_LOGS, + DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME, + DEFAULT_PATTERNS_SPLIT_IN_FILENAME, + DEFAULT_PATTERNS_ALL, +] +WILDCARD_CHARACTERS = "*[]" +FILES_TO_IGNORE = [ + "README.md", + "config.json", + "dataset_info.json", + "dataset_infos.json", + "dummy_data.zip", + "dataset_dict.json", +] + + +def contains_wildcards(pattern: str) -> bool: + return any(wildcard_character in pattern for wildcard_character in WILDCARD_CHARACTERS) + + +def sanitize_patterns(patterns: Union[dict, list, str]) -> dict[str, Union[list[str], "DataFilesList"]]: + """ + Take the data_files patterns from the user, and format them into a dictionary. + Each key is the name of the split, and each value is a list of data files patterns (paths or urls). + The default split is "train". + + Returns: + patterns: dictionary of split_name -> list of patterns + """ + if isinstance(patterns, dict): + return {str(key): value if isinstance(value, list) else [value] for key, value in patterns.items()} + elif isinstance(patterns, str): + return {SANITIZED_DEFAULT_SPLIT: [patterns]} + elif isinstance(patterns, list): + if any(isinstance(pattern, dict) for pattern in patterns): + for pattern in patterns: + if not ( + isinstance(pattern, dict) + and len(pattern) == 2 + and "split" in pattern + and isinstance(pattern.get("path"), (str, list)) + ): + raise ValueError( + "Invalid format for data_files entry. " + "Each item must be a dictionary with the structure " + "{'split': , 'path': }.\n" + f"Received: {pattern}" + ) + splits = [pattern["split"] for pattern in patterns] + if len(set(splits)) != len(splits): + raise ValueError(f"Some splits are duplicated in data_files: {splits}") + return { + str(pattern["split"]): pattern["path"] if isinstance(pattern["path"], list) else [pattern["path"]] + for pattern in patterns + } + else: + return {SANITIZED_DEFAULT_SPLIT: patterns} + else: + return sanitize_patterns(list(patterns)) + + +def _is_inside_unrequested_special_dir(matched_rel_path: str, pattern: str) -> bool: + """ + When a path matches a pattern, we additionally check if it's inside a special directory + we ignore by default (if it starts with a double underscore). + + Users can still explicitly request a filepath inside such a directory if "__pycache__" is + mentioned explicitly in the requested pattern. + + Some examples: + + base directory: + + ./ + └── __pycache__ + └── b.txt + + >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "**") + True + >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "*/b.txt") + True + >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__pycache__/*") + False + >>> _is_inside_unrequested_special_dir("__pycache__/b.txt", "__*/*") + False + """ + # We just need to check if every special directories from the path is present explicitly in the pattern. + # Since we assume that the path matches the pattern, it's equivalent to counting that both + # the parent path and the parent pattern have the same number of special directories. + data_dirs_to_ignore_in_path = [part for part in PurePath(matched_rel_path).parent.parts if part.startswith("__")] + data_dirs_to_ignore_in_pattern = [part for part in PurePath(pattern).parent.parts if part.startswith("__")] + return len(data_dirs_to_ignore_in_path) != len(data_dirs_to_ignore_in_pattern) + + +def _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(matched_rel_path: str, pattern: str) -> bool: + """ + When a path matches a pattern, we additionally check if it's a hidden file or if it's inside + a hidden directory we ignore by default, i.e. if the file name or a parent directory name starts with a dot. + + Users can still explicitly request a filepath that is hidden or is inside a hidden directory + if the hidden part is mentioned explicitly in the requested pattern. + + Some examples: + + base directory: + + ./ + └── .hidden_file.txt + + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", "**") + True + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_file.txt", ".*") + False + + base directory: + + ./ + └── .hidden_dir + └── a.txt + + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", "**") + True + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".*/*") + False + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/a.txt", ".hidden_dir/*") + False + + base directory: + + ./ + └── .hidden_dir + └── .hidden_file.txt + + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", "**") + True + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/*") + True + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".*/.*") + False + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/*") + True + >>> _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(".hidden_dir/.hidden_file.txt", ".hidden_dir/.*") + False + """ + # We just need to check if every hidden part from the path is present explicitly in the pattern. + # Since we assume that the path matches the pattern, it's equivalent to counting that both + # the path and the pattern have the same number of hidden parts. + hidden_directories_in_path = [ + part for part in PurePath(matched_rel_path).parts if part.startswith(".") and not set(part) == {"."} + ] + hidden_directories_in_pattern = [ + part for part in PurePath(pattern).parts if part.startswith(".") and not set(part) == {"."} + ] + return len(hidden_directories_in_path) != len(hidden_directories_in_pattern) + + +def _get_data_files_patterns(pattern_resolver: Callable[[str], list[str]]) -> dict[str, list[str]]: + """ + Get the default pattern from a directory or repository by testing all the supported patterns. + The first patterns to return a non-empty list of data files is returned. + + In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS. + """ + # first check the split patterns like data/{split}-00000-of-00001.parquet + for split_pattern in ALL_SPLIT_PATTERNS: + pattern = split_pattern.replace("{split}", "*") + try: + data_files = pattern_resolver(pattern) + except FileNotFoundError: + continue + if len(data_files) > 0: + splits: set[str] = set() + for p in data_files: + p_parts = string_to_dict(xbasename(p), xbasename(split_pattern)) + assert p_parts is not None + splits.add(p_parts["split"]) + + if any(not re.match(_split_re, split) for split in splits): + raise ValueError(f"Split name should match '{_split_re}'' but got '{splits}'.") + sorted_splits = [str(split) for split in DEFAULT_SPLITS if split in splits] + sorted( + splits - {str(split) for split in DEFAULT_SPLITS} + ) + return {split: [split_pattern.format(split=split)] for split in sorted_splits} + # then check the default patterns based on train/valid/test splits + for patterns_dict in ALL_DEFAULT_PATTERNS: + non_empty_splits = [] + for split, patterns in patterns_dict.items(): + for pattern in patterns: + try: + data_files = pattern_resolver(pattern) + except FileNotFoundError: + continue + if len(data_files) > 0: + non_empty_splits.append(split) + break + if non_empty_splits: + return {split: patterns_dict[split] for split in non_empty_splits} + raise FileNotFoundError(f"Couldn't resolve pattern {pattern} with resolver {pattern_resolver}") + + +def resolve_pattern( + pattern: str, + base_path: str, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, +) -> list[str]: + """ + Resolve the paths and URLs of the data files from the pattern passed by the user. + + You can use patterns to resolve multiple local files. Here are a few examples: + - *.csv to match all the CSV files at the first level + - **.csv to match all the CSV files at any level + - data/* to match all the files inside "data" + - data/** to match all the files inside "data" and its subdirectories + + The patterns are resolved using the fsspec glob. In fsspec>=2023.12.0 this is equivalent to + Python's glob.glob, Path.glob, Path.match and fnmatch where ** is unsupported with a prefix/suffix + other than a forward slash /. + + More generally: + - '*' matches any character except a forward-slash (to match just the file or directory name) + - '**' matches any character including a forward-slash / + + Hidden files and directories (i.e. whose names start with a dot) are ignored, unless they are explicitly requested. + The same applies to special directories that start with a double underscore like "__pycache__". + You can still include one if the pattern explicitly mentions it: + - to include a hidden file: "*/.hidden.txt" or "*/.*" + - to include a hidden directory: ".hidden/*" or ".*/*" + - to include a special directory: "__special__/*" or "__*/*" + + Example:: + + >>> from datasets.data_files import resolve_pattern + >>> base_path = "." + >>> resolve_pattern("docs/**/*.py", base_path) + [/Users/mariosasko/Desktop/projects/datasets/docs/source/_config.py'] + + Args: + pattern (str): Unix pattern or paths or URLs of the data files to resolve. + The paths can be absolute or relative to base_path. + Remote filesystems using fsspec are supported, e.g. with the hf:// protocol. + base_path (str): Base path to use when resolving relative paths. + allowed_extensions (Optional[list], optional): White-list of file extensions to use. Defaults to None (all extensions). + For example: allowed_extensions=[".csv", ".json", ".txt", ".parquet"] + download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters. + Returns: + List[str]: List of paths or URLs to the local or remote files that match the patterns. + """ + if is_relative_path(pattern): + pattern = xjoin(base_path, pattern) + elif is_local_path(pattern): + base_path = os.path.splitdrive(pattern)[0] + os.sep + else: + base_path = "" + pattern, storage_options = _prepare_path_and_storage_options(pattern, download_config=download_config) + fs, fs_pattern = url_to_fs(pattern, **storage_options) + files_to_ignore = set(FILES_TO_IGNORE) - {xbasename(pattern)} + protocol = ( + pattern.split("://")[0] + if "://" in pattern + else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0]) + ) + protocol_prefix = protocol + "://" if protocol != "file" else "" + glob_kwargs = {} + if protocol == "hf": + # 10 times faster glob with detail=True (ignores costly info like lastCommit) + glob_kwargs["expand_info"] = False + + # if the pattern contains hops like "zip://csv/*.csv::data.zip", we need to keep them after globbing + _, *rest_hops = pattern.split("::") + matched_paths = [] + for filepath, info in fs.glob(fs_pattern, detail=True, **glob_kwargs).items(): + if not (info["type"] == "file" or (info.get("islink") and os.path.isfile(os.path.realpath(filepath)))) or ( + xbasename(filepath) in files_to_ignore + ): + continue + if _is_inside_unrequested_special_dir(filepath, fs_pattern): + continue + if _is_unrequested_hidden_file_or_is_inside_unrequested_hidden_dir(filepath, fs_pattern): + continue + filepath = filepath if "://" in filepath else protocol_prefix + filepath + if rest_hops: + filepath = "::".join([filepath] + rest_hops) + matched_paths.append(filepath) + # ignore .ipynb and __pycache__, but keep /../ + if allowed_extensions is not None: + out = [ + filepath + for filepath in matched_paths + if any("." + suffix in allowed_extensions for suffix in xbasename(filepath).split(".")[1:]) + ] + if len(out) < len(matched_paths): + invalid_matched_files = list(set(matched_paths) - set(out)) + logger.info( + f"Some files matched the pattern '{pattern}' but don't have valid data file extensions: {invalid_matched_files}" + ) + else: + out = matched_paths + if not out: + error_msg = f"Unable to find '{pattern}'" + if allowed_extensions is not None: + error_msg += f" with any supported extension {list(allowed_extensions)}" + raise FileNotFoundError(error_msg) + return out + + +def get_data_patterns(base_path: str, download_config: Optional[DownloadConfig] = None) -> dict[str, list[str]]: + """ + Get the default pattern from a directory testing all the supported patterns. + The first patterns to return a non-empty list of data files is returned. + + Some examples of supported patterns: + + Input: + + my_dataset_repository/ + ├── README.md + └── dataset.csv + + Output: + + {'train': ['**']} + + Input: + + my_dataset_repository/ + ├── README.md + ├── train.csv + └── test.csv + + my_dataset_repository/ + ├── README.md + └── data/ + ├── train.csv + └── test.csv + + my_dataset_repository/ + ├── README.md + ├── train_0.csv + ├── train_1.csv + ├── train_2.csv + ├── train_3.csv + ├── test_0.csv + └── test_1.csv + + Output: + + {'train': ['**/train[-._ 0-9]*', '**/*[-._ 0-9]train[-._ 0-9]*', '**/training[-._ 0-9]*', '**/*[-._ 0-9]training[-._ 0-9]*'], + 'test': ['**/test[-._ 0-9]*', '**/*[-._ 0-9]test[-._ 0-9]*', '**/testing[-._ 0-9]*', '**/*[-._ 0-9]testing[-._ 0-9]*', ...]} + + Input: + + my_dataset_repository/ + ├── README.md + └── data/ + ├── train/ + │ ├── shard_0.csv + │ ├── shard_1.csv + │ ├── shard_2.csv + │ └── shard_3.csv + └── test/ + ├── shard_0.csv + └── shard_1.csv + + Output: + + {'train': ['**/train/**', '**/train[-._ 0-9]*/**', '**/*[-._ 0-9]train/**', '**/*[-._ 0-9]train[-._ 0-9]*/**', ...], + 'test': ['**/test/**', '**/test[-._ 0-9]*/**', '**/*[-._ 0-9]test/**', '**/*[-._ 0-9]test[-._ 0-9]*/**', ...]} + + Input: + + my_dataset_repository/ + ├── README.md + └── data/ + ├── train-00000-of-00003.csv + ├── train-00001-of-00003.csv + ├── train-00002-of-00003.csv + ├── test-00000-of-00001.csv + ├── random-00000-of-00003.csv + ├── random-00001-of-00003.csv + └── random-00002-of-00003.csv + + Output: + + {'train': ['data/train-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'], + 'test': ['data/test-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*'], + 'random': ['data/random-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*']} + + In order, it first tests if SPLIT_PATTERN_SHARDED works, otherwise it tests the patterns in ALL_DEFAULT_PATTERNS. + """ + resolver = partial(resolve_pattern, base_path=base_path, download_config=download_config) + try: + return _get_data_files_patterns(resolver) + except FileNotFoundError: + raise EmptyDatasetError(f"The directory at {base_path} doesn't contain any data files") from None + + +def _get_single_origin_metadata( + data_file: str, + download_config: Optional[DownloadConfig] = None, +) -> SingleOriginMetadata: + if data_file.startswith(config.HF_ENDPOINT): + fs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=download_config.token) + data_file = "hf://" + data_file[len(config.HF_ENDPOINT) + 1 :] + data_file = data_file.replace("/resolve/", "/" if data_file.startswith("hf://buckets/") else "@", 1) + fs_path = data_file + else: + data_file, storage_options = _prepare_path_and_storage_options(data_file, download_config=download_config) + fs, fs_path = url_to_fs(data_file, **storage_options) + if isinstance(fs, HfFileSystem): + resolved_path = fs.resolve_path(fs_path) + if hasattr(resolved_path, "revision"): # no revision for buckets + return resolved_path.repo_id, resolved_path.revision + info = fs.info(fs_path) + # s3fs uses "ETag", gcsfs uses "etag", and for local we simply check mtime + for key in ["ETag", "etag", "mtime"]: + if key in info: + return (str(info[key]),) + return () + + +def _get_origin_metadata( + data_files: list[str], + download_config: Optional[DownloadConfig] = None, + max_workers: Optional[int] = None, +) -> list[SingleOriginMetadata]: + max_workers = max_workers if max_workers is not None else config.HF_DATASETS_MULTITHREADING_MAX_WORKERS + if all("hf://" in data_file for data_file in data_files): + # No need for multithreading here since the origin metadata of HF files + # is (repo_id, revision) and is cached after first .info() call. + return [ + _get_single_origin_metadata(data_file, download_config=download_config) + for data_file in hf_tqdm( + data_files, + desc="Resolving data files", + # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached + disable=len(data_files) <= 16 or None, + ) + ] + return thread_map( + partial(_get_single_origin_metadata, download_config=download_config), + data_files, + max_workers=max_workers, + tqdm_class=hf_tqdm, + desc="Resolving data files", + # set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached + disable=len(data_files) <= 16 or None, + ) + + +class DataFilesList(list[str]): + """ + List of data files (absolute local paths or URLs). + It has two construction methods given the user's data files patterns: + - ``from_hf_repo``: resolve patterns inside a dataset repository + - ``from_local_or_remote``: resolve patterns from a local path + + Moreover, DataFilesList has an additional attribute ``origin_metadata``. + It can store: + - the last modified time of local files + - ETag of remote files + - commit sha of a dataset repository + + Thanks to this additional attribute, it is possible to hash the list + and get a different hash if and only if at least one file changed. + This is useful for caching Dataset objects that are obtained from a list of data files. + """ + + def __init__(self, data_files: list[str], origin_metadata: list[SingleOriginMetadata]) -> None: + super().__init__(data_files) + self.origin_metadata = origin_metadata + + def __add__(self, other: "DataFilesList") -> "DataFilesList": + return DataFilesList([*self, *other], self.origin_metadata + other.origin_metadata) + + @classmethod + def from_hf_repo( + cls, + patterns: list[str], + dataset_info: huggingface_hub.hf_api.DatasetInfo, + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesList": + base_path = f"hf://datasets/{dataset_info.id}@{dataset_info.sha}/{base_path or ''}".rstrip("/") + return cls.from_patterns( + patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config + ) + + @classmethod + def from_local_or_remote( + cls, + patterns: list[str], + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesList": + base_path = base_path if base_path is not None else Path().resolve().as_posix() + return cls.from_patterns( + patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config + ) + + @classmethod + def from_patterns( + cls, + patterns: list[str], + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesList": + base_path = base_path if base_path is not None else Path().resolve().as_posix() + data_files = [] + for pattern in patterns: + try: + data_files.extend( + resolve_pattern( + pattern, + base_path=base_path, + allowed_extensions=allowed_extensions, + download_config=download_config, + ) + ) + except FileNotFoundError: + if not has_magic(pattern): + raise + origin_metadata = _get_origin_metadata(data_files, download_config=download_config) + return cls(data_files, origin_metadata) + + def filter( + self, *, extensions: Optional[list[str]] = None, file_names: Optional[list[str]] = None + ) -> "DataFilesList": + patterns = [] + if extensions: + ext_pattern = "|".join(re.escape(ext) for ext in extensions) + patterns.append(re.compile(f".*({ext_pattern})(\\..+)?$")) + if file_names: + fn_pattern = "|".join(re.escape(fn) for fn in file_names) + patterns.append(re.compile(rf".*[\/]?({fn_pattern})$")) + if patterns: + return DataFilesList( + [data_file for data_file in self if any(pattern.match(data_file) for pattern in patterns)], + origin_metadata=self.origin_metadata, + ) + else: + return DataFilesList(list(self), origin_metadata=self.origin_metadata) + + +class DataFilesDict(dict[str, DataFilesList]): + """ + Dict of split_name -> list of data files (absolute local paths or URLs). + It has two construction methods given the user's data files patterns : + - ``from_hf_repo``: resolve patterns inside a dataset repository + - ``from_local_or_remote``: resolve patterns from a local path + + Moreover, each list is a DataFilesList. It is possible to hash the dictionary + and get a different hash if and only if at least one file changed. + For more info, see [`DataFilesList`]. + + This is useful for caching Dataset objects that are obtained from a list of data files. + + Changing the order of the keys of this dictionary also doesn't change its hash. + """ + + @classmethod + def from_local_or_remote( + cls, + patterns: dict[str, Union[list[str], DataFilesList]], + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesDict": + out = cls() + for key, patterns_for_key in patterns.items(): + out[key] = ( + patterns_for_key + if isinstance(patterns_for_key, DataFilesList) + else DataFilesList.from_local_or_remote( + patterns_for_key, + base_path=base_path, + allowed_extensions=allowed_extensions, + download_config=download_config, + ) + ) + return out + + @classmethod + def from_hf_repo( + cls, + patterns: dict[str, Union[list[str], DataFilesList]], + dataset_info: huggingface_hub.hf_api.DatasetInfo, + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesDict": + out = cls() + for key, patterns_for_key in patterns.items(): + out[key] = ( + patterns_for_key + if isinstance(patterns_for_key, DataFilesList) + else DataFilesList.from_hf_repo( + patterns_for_key, + dataset_info=dataset_info, + base_path=base_path, + allowed_extensions=allowed_extensions, + download_config=download_config, + ) + ) + return out + + @classmethod + def from_patterns( + cls, + patterns: dict[str, Union[list[str], DataFilesList]], + base_path: Optional[str] = None, + allowed_extensions: Optional[list[str]] = None, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesDict": + out = cls() + for key, patterns_for_key in patterns.items(): + out[key] = ( + patterns_for_key + if isinstance(patterns_for_key, DataFilesList) + else DataFilesList.from_patterns( + patterns_for_key, + base_path=base_path, + allowed_extensions=allowed_extensions, + download_config=download_config, + ) + ) + return out + + def filter( + self, *, extensions: Optional[list[str]] = None, file_names: Optional[list[str]] = None + ) -> "DataFilesDict": + out = type(self)() + for key, data_files_list in self.items(): + out[key] = data_files_list.filter(extensions=extensions, file_names=file_names) + return out + + +class DataFilesPatternsList(list[str]): + """ + List of data files patterns (absolute local paths or URLs). + For each pattern there should also be a list of allowed extensions + to keep, or a None ot keep all the files for the pattern. + """ + + def __init__( + self, + patterns: list[str], + allowed_extensions: list[Optional[list[str]]], + ): + super().__init__(patterns) + self.allowed_extensions = allowed_extensions + + def __add__(self, other): + return DataFilesList([*self, *other], self.allowed_extensions + other.allowed_extensions) + + @classmethod + def from_patterns( + cls, patterns: list[str], allowed_extensions: Optional[list[str]] = None + ) -> "DataFilesPatternsList": + return cls(patterns, [allowed_extensions] * len(patterns)) + + def resolve( + self, + base_path: str, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesList": + base_path = base_path if base_path is not None else Path().resolve().as_posix() + data_files = [] + for pattern, allowed_extensions in zip(self, self.allowed_extensions): + try: + data_files.extend( + resolve_pattern( + pattern, + base_path=base_path, + allowed_extensions=allowed_extensions, + download_config=download_config, + ) + ) + except FileNotFoundError: + if not has_magic(pattern): + raise + origin_metadata = _get_origin_metadata(data_files, download_config=download_config) + return DataFilesList(data_files, origin_metadata) + + def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsList": + return DataFilesPatternsList( + self, [allowed_extensions + extensions for allowed_extensions in self.allowed_extensions] + ) + + +class DataFilesPatternsDict(dict[str, DataFilesPatternsList]): + """ + Dict of split_name -> list of data files patterns (absolute local paths or URLs). + """ + + @classmethod + def from_patterns( + cls, patterns: dict[str, list[str]], allowed_extensions: Optional[list[str]] = None + ) -> "DataFilesPatternsDict": + out = cls() + for key, patterns_for_key in patterns.items(): + out[key] = ( + patterns_for_key + if isinstance(patterns_for_key, DataFilesPatternsList) + else DataFilesPatternsList.from_patterns( + patterns_for_key, + allowed_extensions=allowed_extensions, + ) + ) + return out + + def resolve( + self, + base_path: str, + download_config: Optional[DownloadConfig] = None, + ) -> "DataFilesDict": + out = DataFilesDict() + for key, data_files_patterns_list in self.items(): + out[key] = data_files_patterns_list.resolve(base_path, download_config) + return out + + def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsDict": + out = type(self)() + for key, data_files_patterns_list in self.items(): + out[key] = data_files_patterns_list.filter_extensions(extensions) + return out diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/dataset_dict.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/dataset_dict.py new file mode 100644 index 0000000000000000000000000000000000000000..34f84f7fca8cdfa8c3eb508cfc345a830537ce62 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/dataset_dict.py @@ -0,0 +1,2774 @@ +import contextlib +import copy +import itertools +import json +import math +import posixpath +import random +import re +import time +from collections.abc import Sequence +from functools import partial +from typing import Callable, Literal, Optional, Union + +import fsspec +import numpy as np +from fsspec.core import url_to_fs +from fsspec.implementations.dirfs import DirFileSystem +from huggingface_hub import ( + CommitInfo, + CommitOperationAdd, + CommitOperationDelete, + HfApi, + HfFileSystem, + HfFileSystemResolvedPath, +) +from huggingface_hub.utils import EntryNotFoundError, HfHubHTTPError, RepositoryNotFoundError +from packaging import version + +from . import config +from .arrow_dataset import ( + Dataset, + _get_updated_dataset_card, +) +from .features import Features +from .features.features import FeatureType +from .iterable_dataset import IterableDataset +from .naming import _split_re +from .splits import NamedSplit, Split, SplitInfo +from .table import Table +from .utils import logging +from .utils.doc_utils import is_documented_by +from .utils.typing import PathLike + + +if config.HF_HUB_VERSION >= version.parse("1.6.0"): + from huggingface_hub.errors import BucketNotFoundError + from huggingface_hub.hf_file_system import HfFileSystemResolvedBucketPath, HfFileSystemResolvedRepositoryPath + +else: + BucketNotFoundError = None + HfFileSystemResolvedBucketPath = None + HfFileSystemResolvedRepositoryPath = HfFileSystemResolvedPath + + +logger = logging.get_logger(__name__) + + +class bind(partial): + def __call__(self, *fn_args, **fn_kwargs): + return self.func(*fn_args, *self.args, **fn_kwargs) + + +class DatasetDict(dict[Union[str, NamedSplit], "Dataset"]): + """A dictionary (dict of str: datasets.Dataset) with dataset transforms methods (map, filter, etc.)""" + + def _check_values_type(self): + for dataset in self.values(): + if not isinstance(dataset, Dataset): + raise TypeError(f"Values in `DatasetDict` should be of type `Dataset` but got type '{type(dataset)}'") + + def _check_values_features(self): + items = list(self.items()) + for item_a, item_b in zip(items[:-1], items[1:]): + if item_a[1].features != item_b[1].features: + raise ValueError( + f"All datasets in `DatasetDict` should have the same features but features for '{item_a[0]}' and '{item_b[0]}' don't match: {item_a[1].features} != {item_b[1].features}" + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Here `del` is used to del the pyarrow tables. This properly closes the files used for memory mapped tables + for dataset in self.values(): + if hasattr(dataset, "_data"): + del dataset._data + if hasattr(dataset, "_indices"): + del dataset._indices + + def __getitem__(self, k) -> Dataset: + if isinstance(k, (str, NamedSplit)) or len(self) == 0: + return super().__getitem__(k) + else: + available_suggested_splits = [ + split for split in (Split.TRAIN, Split.TEST, Split.VALIDATION) if split in self + ] + suggested_split = available_suggested_splits[0] if available_suggested_splits else list(self)[0] + raise KeyError( + f"Invalid key: {k}. Please first select a split. For example: " + f"`my_dataset_dictionary['{suggested_split}'][{k}]`. " + f"Available splits: {sorted(self)}" + ) + + @property + def data(self) -> dict[str, Table]: + """The Apache Arrow tables backing each split. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.data + ``` + """ + self._check_values_type() + return {k: dataset.data for k, dataset in self.items()} + + @property + def cache_files(self) -> dict[str, dict]: + """The cache files containing the Apache Arrow table backing each split. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.cache_files + {'test': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-test.arrow'}], + 'train': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-train.arrow'}], + 'validation': [{'filename': '/root/.cache/huggingface/datasets/rotten_tomatoes_movie_review/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/rotten_tomatoes_movie_review-validation.arrow'}]} + ``` + """ + self._check_values_type() + return {k: dataset.cache_files for k, dataset in self.items()} + + @property + def num_columns(self) -> dict[str, int]: + """Number of columns in each split of the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.num_columns + {'test': 2, 'train': 2, 'validation': 2} + ``` + """ + self._check_values_type() + return {k: dataset.num_columns for k, dataset in self.items()} + + @property + def num_rows(self) -> dict[str, int]: + """Number of rows in each split of the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.num_rows + {'test': 1066, 'train': 8530, 'validation': 1066} + ``` + """ + self._check_values_type() + return {k: dataset.num_rows for k, dataset in self.items()} + + @property + def column_names(self) -> dict[str, list[str]]: + """Names of the columns in each split of the dataset. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.column_names + {'test': ['text', 'label'], + 'train': ['text', 'label'], + 'validation': ['text', 'label']} + ``` + """ + self._check_values_type() + return {k: dataset.column_names for k, dataset in self.items()} + + @property + def shape(self) -> dict[str, tuple[int]]: + """Shape of each split of the dataset (number of rows, number of columns). + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.shape + {'test': (1066, 2), 'train': (8530, 2), 'validation': (1066, 2)} + ``` + """ + self._check_values_type() + return {k: dataset.shape for k, dataset in self.items()} + + def flatten(self, max_depth=16) -> "DatasetDict": + """Flatten the Apache Arrow Table of each split (nested features are flatten). + Each column with a struct type is flattened into one column per struct field. + Other columns are left unchanged. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("rajpurkar/squad") + >>> ds["train"].features + {'id': Value('string'), + 'title': Value('string'), + 'context': Value('string'), + 'question': Value('string'), + 'answers.text': List(Value('string')), + 'answers.answer_start': List(Value('int32'))} + >>> ds.flatten() + DatasetDict({ + train: Dataset({ + features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'], + num_rows: 87599 + }) + validation: Dataset({ + features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'], + num_rows: 10570 + }) + }) + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.flatten(max_depth=max_depth) for k, dataset in self.items()}) + + def unique(self, column: str) -> dict[str, list]: + """Return a list of the unique elements in a column for each split. + + This is implemented in the low-level backend and as such, very fast. + + Args: + column (`str`): + column name (list all the column names with [`~datasets.DatasetDict.column_names`]) + + Returns: + Dict[`str`, `list`]: Dictionary of unique elements in the given column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.unique("label") + {'test': [1, 0], 'train': [1, 0], 'validation': [1, 0]} + ``` + """ + self._check_values_type() + return {k: dataset.unique(column) for k, dataset in self.items()} + + def cleanup_cache_files(self) -> dict[str, int]: + """Clean up all cache files in the dataset cache directory, excepted the currently used cache file if there is one. + Be careful when running this command that no other process is currently using other cache files. + + Return: + `Dict` with the number of removed files for each split + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.cleanup_cache_files() + {'test': 0, 'train': 0, 'validation': 0} + ``` + """ + self._check_values_type() + return {k: dataset.cleanup_cache_files() for k, dataset in self.items()} + + def __repr__(self): + repr = "\n".join([f"{k}: {v}" for k, v in self.items()]) + repr = re.sub(r"^", " " * 4, repr, count=0, flags=re.M) + return f"DatasetDict({{\n{repr}\n}})" + + def cast(self, features: Features) -> "DatasetDict": + """ + Cast the dataset to a new set of features. + The transformation is applied to all the datasets of the dataset dictionary. + + Args: + features ([`Features`]): + New features to cast the dataset to. + The name and order of the fields in the features must match the current column names. + The type of the data must also be convertible from one type to the other. + For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`~DatasetDict.map`] to update the dataset. + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel, Value + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds["train"].features + {'label': ClassLabel(names=['neg', 'pos']), + 'text': Value('string')} + >>> new_features = ds["train"].features.copy() + >>> new_features['label'] = ClassLabel(names=['bad', 'good']) + >>> new_features['text'] = Value('large_string') + >>> ds = ds.cast(new_features) + >>> ds["train"].features + {'label': ClassLabel(names=['bad', 'good']), + 'text': Value('large_string')} + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.cast(features=features) for k, dataset in self.items()}) + + def cast_column(self, column: str, feature) -> "DatasetDict": + """Cast column to feature for decoding. + + Args: + column (`str`): + Column name. + feature ([`Feature`]): + Target feature. + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds["train"].features + {'label': ClassLabel(names=['neg', 'pos']), + 'text': Value('string')} + >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good'])) + >>> ds["train"].features + {'label': ClassLabel(names=['bad', 'good']), + 'text': Value('string')} + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.cast_column(column=column, feature=feature) for k, dataset in self.items()}) + + def remove_columns(self, column_names: Union[str, list[str]]) -> "DatasetDict": + """ + Remove one or several column(s) from each split in the dataset + and the features associated to the column(s). + + The transformation is applied to all the splits of the dataset dictionary. + + You can also remove a column using [`~DatasetDict.map`] with `remove_columns` but the present method + doesn't copy the data of the remaining columns and is thus faster. + + Args: + column_names (`Union[str, list[str]]`): + Name of the column(s) to remove. + + Returns: + [`DatasetDict`]: A copy of the dataset object without the columns to remove. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds = ds.remove_columns("label") + DatasetDict({ + train: Dataset({ + features: ['text'], + num_rows: 8530 + }) + validation: Dataset({ + features: ['text'], + num_rows: 1066 + }) + test: Dataset({ + features: ['text'], + num_rows: 1066 + }) + }) + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.remove_columns(column_names=column_names) for k, dataset in self.items()}) + + def rename_column(self, original_column_name: str, new_column_name: str) -> "DatasetDict": + """ + Rename a column in the dataset and move the features associated to the original column under the new column name. + The transformation is applied to all the datasets of the dataset dictionary. + + You can also rename a column using [`~DatasetDict.map`] with `remove_columns` but the present method: + - takes care of moving the original features under the new column name. + - doesn't copy the data to a new dataset and is thus much faster. + + Args: + original_column_name (`str`): + Name of the column to rename. + new_column_name (`str`): + New name for the column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds = ds.rename_column("label", "label_new") + DatasetDict({ + train: Dataset({ + features: ['text', 'label_new'], + num_rows: 8530 + }) + validation: Dataset({ + features: ['text', 'label_new'], + num_rows: 1066 + }) + test: Dataset({ + features: ['text', 'label_new'], + num_rows: 1066 + }) + }) + ``` + """ + self._check_values_type() + return DatasetDict( + { + k: dataset.rename_column( + original_column_name=original_column_name, + new_column_name=new_column_name, + ) + for k, dataset in self.items() + } + ) + + def rename_columns(self, column_mapping: dict[str, str]) -> "DatasetDict": + """ + Rename several columns in the dataset, and move the features associated to the original columns under + the new column names. + The transformation is applied to all the datasets of the dataset dictionary. + + Args: + column_mapping (`Dict[str, str]`): + A mapping of columns to rename to their new names. + + Returns: + [`DatasetDict`]: A copy of the dataset with renamed columns. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.rename_columns({'text': 'text_new', 'label': 'label_new'}) + DatasetDict({ + train: Dataset({ + features: ['text_new', 'label_new'], + num_rows: 8530 + }) + validation: Dataset({ + features: ['text_new', 'label_new'], + num_rows: 1066 + }) + test: Dataset({ + features: ['text_new', 'label_new'], + num_rows: 1066 + }) + }) + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.rename_columns(column_mapping=column_mapping) for k, dataset in self.items()}) + + def select_columns(self, column_names: Union[str, list[str]]) -> "DatasetDict": + """Select one or several column(s) from each split in the dataset and + the features associated to the column(s). + + The transformation is applied to all the splits of the dataset + dictionary. + + Args: + column_names (`Union[str, list[str]]`): + Name of the column(s) to keep. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.select_columns("text") + DatasetDict({ + train: Dataset({ + features: ['text'], + num_rows: 8530 + }) + validation: Dataset({ + features: ['text'], + num_rows: 1066 + }) + test: Dataset({ + features: ['text'], + num_rows: 1066 + }) + }) + ``` + """ + self._check_values_type() + return DatasetDict({k: dataset.select_columns(column_names=column_names) for k, dataset in self.items()}) + + def class_encode_column(self, column: str, include_nulls: bool = False) -> "DatasetDict": + """Casts the given column as [`~datasets.features.ClassLabel`] and updates the tables. + + Args: + column (`str`): + The name of the column to cast. + include_nulls (`bool`, defaults to `False`): + Whether to include null values in the class labels. If `True`, the null values will be encoded as the `"None"` class label. + + + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("google/boolq") + >>> ds["train"].features + {'answer': Value('bool'), + 'passage': Value('string'), + 'question': Value('string')} + >>> ds = ds.class_encode_column("answer") + >>> ds["train"].features + {'answer': ClassLabel(num_classes=2, names=['False', 'True']), + 'passage': Value('string'), + 'question': Value('string')} + ``` + """ + self._check_values_type() + return DatasetDict( + {k: dataset.class_encode_column(column=column, include_nulls=include_nulls) for k, dataset in self.items()} + ) + + @contextlib.contextmanager + def formatted_as( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ): + """To be used in a `with` statement. Set `__getitem__` return format (type and columns). + The transformation is applied to all the datasets of the dataset dictionary. + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__` returns python objects (default). + columns (`list[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to False): + Keep un-formatted columns as well in the output (as python objects). + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + """ + self._check_values_type() + old_format_type = {k: dataset._format_type for k, dataset in self.items()} + old_format_kwargs = {k: dataset._format_kwargs for k, dataset in self.items()} + old_format_columns = {k: dataset._format_columns for k, dataset in self.items()} + old_output_all_columns = {k: dataset._output_all_columns for k, dataset in self.items()} + try: + self.set_format(type, columns, output_all_columns, **format_kwargs) + yield + finally: + for k, dataset in self.items(): + dataset.set_format( + old_format_type[k], + old_format_columns[k], + old_output_all_columns[k], + **old_format_kwargs[k], + ) + + def set_format( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ): + """Set `__getitem__` return format (type and columns). + The format is set for every dataset in the dataset dictionary. + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__` returns python objects (default). + columns (`list[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to False): + Keep un-formatted columns as well in the output (as python objects), + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + + It is possible to call `map` after calling `set_format`. Since `map` may add new columns, then the list of formatted columns + gets updated. In this case, if you apply `map` on a dataset to add a new column, then this column will be formatted: + + `new formatted columns = (all columns - previously unformatted columns)` + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x["text"], truncation=True, padding=True), batched=True) + >>> ds.set_format(type="numpy", columns=['input_ids', 'token_type_ids', 'attention_mask', 'label']) + >>> ds["train"].format + {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': 'numpy'} + ``` + """ + self._check_values_type() + for dataset in self.values(): + dataset.set_format( + type=type, + columns=columns, + output_all_columns=output_all_columns, + **format_kwargs, + ) + + def reset_format(self): + """Reset `__getitem__` return format to python objects and all columns. + The transformation is applied to all the datasets of the dataset dictionary. + + Same as `self.set_format()` + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x["text"], truncation=True, padding=True), batched=True) + >>> ds.set_format(type="numpy", columns=['input_ids', 'token_type_ids', 'attention_mask', 'label']) + >>> ds["train"].format + {'columns': ['input_ids', 'token_type_ids', 'attention_mask', 'label'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': 'numpy'} + >>> ds.reset_format() + >>> ds["train"].format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': None} + ``` + """ + self._check_values_type() + for dataset in self.values(): + dataset.set_format() + + def set_transform( + self, + transform: Optional[Callable], + columns: Optional[list] = None, + output_all_columns: bool = False, + ): + """Set ``__getitem__`` return format using this transform. The transform is applied on-the-fly on batches when ``__getitem__`` is called. + The transform is set for every dataset in the dataset dictionary + As :func:`datasets.Dataset.set_format`, this can be reset using :func:`datasets.Dataset.reset_format` + + Args: + transform (`Callable`, optional): user-defined formatting transform, replaces the format defined by :func:`datasets.Dataset.set_format` + A formatting function is a callable that takes a batch (as a dict) as input and returns a batch. + This function is applied right before returning the objects in ``__getitem__``. + columns (`list[str]`, optional): columns to format in the output + If specified, then the input batch of the transform only contains those columns. + output_all_columns (`bool`, default to False): keep un-formatted columns as well in the output (as python objects) + If set to True, then the other un-formatted columns are kept with the output of the transform. + + """ + self._check_values_type() + for dataset in self.values(): + dataset.set_format( + "custom", + columns=columns, + output_all_columns=output_all_columns, + transform=transform, + ) + + def with_format( + self, + type: Optional[str] = None, + columns: Optional[list] = None, + output_all_columns: bool = False, + **format_kwargs, + ) -> "DatasetDict": + """Set `__getitem__` return format (type and columns). The data formatting is applied on-the-fly. + The format `type` (for example "numpy") is used to format batches when using `__getitem__`. + The format is set for every dataset in the dataset dictionary. + + It's also possible to use custom transforms for formatting using [`~datasets.Dataset.with_transform`]. + + Contrary to [`~datasets.DatasetDict.set_format`], `with_format` returns a new [`DatasetDict`] object with new [`Dataset`] objects. + + Args: + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means `__getitem__` returns python objects (default). + columns (`list[str]`, *optional*): + Columns to format in the output. + `None` means `__getitem__` returns all columns (default). + output_all_columns (`bool`, defaults to `False`): + Keep un-formatted columns as well in the output (as python objects). + **format_kwargs (additional keyword arguments): + Keywords arguments passed to the convert function like `np.array`, `torch.tensor` or `tensorflow.ragged.constant`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds["train"].format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': None} + >>> ds = ds.with_format("torch") + >>> ds["train"].format + {'columns': ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'], + 'format_kwargs': {}, + 'output_all_columns': False, + 'type': 'torch'} + >>> ds["train"][0] + {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .', + 'label': tensor(1), + 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617, + 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105, + 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])} + ``` + """ + dataset = copy.deepcopy(self) + dataset.set_format( + type=type, + columns=columns, + output_all_columns=output_all_columns, + **format_kwargs, + ) + return dataset + + def with_transform( + self, + transform: Optional[Callable], + columns: Optional[list] = None, + output_all_columns: bool = False, + ) -> "DatasetDict": + """Set `__getitem__` return format using this transform. The transform is applied on-the-fly on batches when `__getitem__` is called. + The transform is set for every dataset in the dataset dictionary + + As [`~datasets.Dataset.set_format`], this can be reset using [`~datasets.Dataset.reset_format`]. + + Contrary to [`~datasets.DatasetDict.set_transform`], `with_transform` returns a new [`DatasetDict`] object with new [`Dataset`] objects. + + Args: + transform (`Callable`, *optional*): + User-defined formatting transform, replaces the format defined by [`~datasets.Dataset.set_format`]. + A formatting function is a callable that takes a batch (as a dict) as input and returns a batch. + This function is applied right before returning the objects in `__getitem__`. + columns (`list[str]`, *optional*): + Columns to format in the output. + If specified, then the input batch of the transform only contains those columns. + output_all_columns (`bool`, defaults to False): + Keep un-formatted columns as well in the output (as python objects). + If set to `True`, then the other un-formatted columns are kept with the output of the transform. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> def encode(example): + ... return tokenizer(example['text'], truncation=True, padding=True, return_tensors="pt") + >>> ds = ds.with_transform(encode) + >>> ds["train"][0] + {'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1]), + 'input_ids': tensor([ 101, 1103, 2067, 1110, 17348, 1106, 1129, 1103, 6880, 1432, + 112, 188, 1207, 107, 14255, 1389, 107, 1105, 1115, 1119, + 112, 188, 1280, 1106, 1294, 170, 24194, 1256, 3407, 1190, + 170, 11791, 5253, 188, 1732, 7200, 10947, 12606, 2895, 117, + 179, 7766, 118, 172, 15554, 1181, 3498, 6961, 3263, 1137, + 188, 1566, 7912, 14516, 6997, 119, 102]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0])} + ``` + """ + dataset = copy.deepcopy(self) + dataset.set_transform(transform=transform, columns=columns, output_all_columns=output_all_columns) + return dataset + + def map( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + with_rank: bool = False, + with_split: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[Union[str, list[str]]] = None, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + cache_file_names: Optional[dict[str, Optional[str]]] = None, + writer_batch_size: Optional[int] = 1000, + features: Optional[Features] = None, + disable_nullable: bool = False, + fn_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + desc: Optional[str] = None, + try_original_type: Optional[bool] = True, + on_mixed_types: Optional[Literal["use_json"]] = None, + ) -> "DatasetDict": + """ + Apply a function to all the examples in the table (individually or in batches) and update the table. + If your function returns a column that already exists, then it overwrites it. + The transformation is applied to all the datasets of the dataset dictionary. + + You can specify whether the function should be batched or not with the `batched` parameter: + + - If batched is `False`, then the function takes 1 example in and should return 1 example. + An example is a dictionary, e.g. `{"text": "Hello there !"}`. + - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples. + A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`. + - If batched is `True` and `batch_size` is `n > 1`, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples. + Note that the last batch may have less than `n` examples. + A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`. + + If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simultaneous calls. + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`callable`): with one of the following signature: + - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` + - `function(example: Dict[str, Any], indices: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True` + - `function(batch: Dict[str, list]) -> Dict[str, list]` if `batched=True` and `with_indices=False` + - `function(batch: Dict[str, list], indices: list[int]) -> Dict[str, list]` if `batched=True` and `with_indices=True` + + For advanced usage, the function can also return a `pyarrow.Table`. + If the function is asynchronous, then `map` will run your function in parallel. + Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged. + If no function is provided, default to identity function: `lambda x: x`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`. + with_rank (`bool`, defaults to `False`): + Provide process rank to `function`. Note that in this case the + signature of `function` should be `def function(example[, idx], rank): ...`. + with_split (`bool`, defaults to `False`): + Provide process split to `function`. Note that in this case the + signature of `function` should be `def function(example[, idx], split): ...`. + input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`): + The columns to be passed into `function` as + positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True`, + `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`. + drop_last_batch (`bool`, defaults to `False`): + Whether a last batch smaller than the batch_size should be + dropped instead of being processed by the function. + remove_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`): + Remove a selection of columns while doing the mapping. + Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding + columns with names in `remove_columns`, these columns will be kept. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + You have to provide one `cache_file_name` per dataset in the dataset dictionary. + writer_batch_size (`int`, default `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + features (`[datasets.Features]`, *optional*, defaults to `None`): + Use a specific [`Features`] to store the cache file + instead of the automatically generated one. + disable_nullable (`bool`, defaults to `False`): + Disallow null values in the table. + fn_kwargs (`Dict`, *optional*, defaults to `None`): + Keyword arguments to be passed to `function` + num_proc (`int`, *optional*, defaults to `None`): + The number of processes to use for multiprocessing. + - If `None` or `0`, no multiprocessing is used and the operation runs in the main process. + - If greater than `1`, one or multiple worker processes are used to process data in parallel. + Note: The function passed to `map()` must be picklable for multiprocessing to work correctly + (i.e., prefer functions defined at the top level of a module, not inside another function or class). + desc (`str`, *optional*, defaults to `None`): + Meaningful description to be displayed alongside with the progress bar while mapping examples. + try_original_type (`Optional[bool]`, defaults to `True`): + Try to keep the types of the original columns (e.g. int32 -> int32). + Set to False if you want to always infer new types. + on_mixed_types (`Literal["use_json"]`, *optional*, defaults to `None`): + If "use_json", use the Json() type for mixed-types fields, + i.e. unstructured fields that contain data without a predefined schema. + In this case, a field with mixed type is set to Json(). + + This allow loading lists with a mix of strings/integers/floats + for example, or dictionaries with arbitrary value types. + + + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> def add_prefix(example): + ... example["text"] = "Review: " + example["text"] + ... return example + >>> ds = ds.map(add_prefix) + >>> ds["train"][0:3]["text"] + ['Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', + 'Review: the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .', + 'Review: effective but too-tepid biopic'] + + # process a batch of examples + >>> ds = ds.map(lambda example: tokenizer(example["text"]), batched=True) + # set number of processors + >>> ds = ds.map(add_prefix, num_proc=4) + ``` + """ + self._check_values_type() + if cache_file_names is None: + cache_file_names = dict.fromkeys(self) + + dataset_dict = {} + for split, dataset in self.items(): + if with_split: + function = bind(function, split) + + dataset_dict[split] = dataset.map( + function=function, + with_indices=with_indices, + with_rank=with_rank, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + drop_last_batch=drop_last_batch, + remove_columns=remove_columns, + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + cache_file_name=cache_file_names[split], + writer_batch_size=writer_batch_size, + features=features, + disable_nullable=disable_nullable, + fn_kwargs=fn_kwargs, + num_proc=num_proc, + desc=desc, + try_original_type=try_original_type, + on_mixed_types=on_mixed_types, + ) + + if with_split: + function = function.func + + return DatasetDict(dataset_dict) + + def filter( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + with_rank: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + cache_file_names: Optional[dict[str, Optional[str]]] = None, + writer_batch_size: Optional[int] = 1000, + fn_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + desc: Optional[str] = None, + ) -> "DatasetDict": + """Apply a filter function to all the elements in the table in batches + and update the table so that the dataset only includes examples according to the filter function. + The transformation is applied to all the datasets of the dataset dictionary. + + Args: + function (`Callable`): Callable with one of the following signatures: + + - `function(example: Dict[str, Any]) -> bool` if `batched=False` and `with_indices=False` and `with_rank=False` + - `function(example: Dict[str, Any], *extra_args) -> bool` if `batched=False` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + - `function(batch: Dict[str, list]) -> list[bool]` if `batched=True` and `with_indices=False` and `with_rank=False` + - `function(batch: Dict[str, list], *extra_args) -> list[bool]` if `batched=True` and `with_indices=True` and/or `with_rank=True` (one extra arg for each) + + If no function is provided, defaults to an always `True` function: `lambda x: True`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the + signature of `function` should be `def function(example, idx[, rank]): ...`. + with_rank (`bool`, defaults to `False`): + Provide process rank to `function`. Note that in this case the + signature of `function` should be `def function(example[, idx], rank): ...`. + input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`): + The columns to be passed into `function` as + positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True` + `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + You have to provide one `cache_file_name` per dataset in the dataset dictionary. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + fn_kwargs (`Dict`, *optional*, defaults to `None`): + Keyword arguments to be passed to `function` + num_proc (`int`, *optional*, defaults to `None`): + The number of processes to use for multiprocessing. + - If `None` or `0`, no multiprocessing is used and the operation runs in the main process. + - If greater than `1`, one or multiple worker processes are used to process data in parallel. + Note: The function passed to `map()` must be picklable for multiprocessing to work correctly + (i.e., prefer functions defined at the top level of a module, not inside another function or class). + desc (`str`, *optional*, defaults to `None`): + Meaningful description to be displayed alongside with the progress bar while filtering examples. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.filter(lambda x: x["label"] == 1) + DatasetDict({ + train: Dataset({ + features: ['text', 'label'], + num_rows: 4265 + }) + validation: Dataset({ + features: ['text', 'label'], + num_rows: 533 + }) + test: Dataset({ + features: ['text', 'label'], + num_rows: 533 + }) + }) + ``` + """ + self._check_values_type() + if cache_file_names is None: + cache_file_names = dict.fromkeys(self) + return DatasetDict( + { + k: dataset.filter( + function=function, + with_indices=with_indices, + with_rank=with_rank, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + cache_file_name=cache_file_names[k], + writer_batch_size=writer_batch_size, + fn_kwargs=fn_kwargs, + num_proc=num_proc, + desc=desc, + ) + for k, dataset in self.items() + } + ) + + def flatten_indices( + self, + keep_in_memory: bool = False, + cache_file_names: Optional[dict[str, Optional[str]]] = None, + writer_batch_size: Optional[int] = 1000, + features: Optional[Features] = None, + disable_nullable: bool = False, + num_proc: Optional[int] = None, + new_fingerprint: Optional[str] = None, + ) -> "DatasetDict": + """Create and cache a new Dataset by flattening the indices mapping. + + Args: + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + cache_file_names (`Dict[str, str]`, *optional*, default `None`): + Provide the name of a path for the cache file. It is used to store the + results of the computation instead of the automatically generated cache file name. + You have to provide one `cache_file_name` per dataset in the dataset dictionary. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + features (`Optional[datasets.Features]`, defaults to `None`): + Use a specific [`Features`] to store the cache file + instead of the automatically generated one. + disable_nullable (`bool`, defaults to `False`): + Allow null values in the table. + num_proc (`int`, optional, default `None`): + Max number of processes when generating cache. Already cached shards are loaded sequentially + new_fingerprint (`str`, *optional*, defaults to `None`): + The new fingerprint of the dataset after transform. + If `None`, the new fingerprint is computed using a hash of the previous fingerprint, and the transform arguments + """ + self._check_values_type() + if cache_file_names is None: + cache_file_names = dict.fromkeys(self) + return DatasetDict( + { + k: dataset.flatten_indices( + keep_in_memory=keep_in_memory, + cache_file_name=cache_file_names[k], + writer_batch_size=writer_batch_size, + features=features, + disable_nullable=disable_nullable, + num_proc=num_proc, + new_fingerprint=new_fingerprint, + ) + for k, dataset in self.items() + } + ) + + def sort( + self, + column_names: Union[str, Sequence[str]], + reverse: Union[bool, Sequence[bool]] = False, + null_placement: str = "at_end", + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + indices_cache_file_names: Optional[dict[str, Optional[str]]] = None, + writer_batch_size: Optional[int] = 1000, + ) -> "DatasetDict": + """Create a new dataset sorted according to a single or multiple columns. + + Args: + column_names (`Union[str, Sequence[str]]`): + Column name(s) to sort by. + reverse (`Union[bool, Sequence[bool]]`, defaults to `False`): + If `True`, sort by descending order rather than ascending. If a single bool is provided, + the value is applied to the sorting of all column names. Otherwise a list of bools with the + same length and order as column_names must be provided. + null_placement (`str`, defaults to `at_end`): + Put `None` values at the beginning if `at_start` or `first` or at the end if `at_end` or `last` + keep_in_memory (`bool`, defaults to `False`): + Keep the sorted indices in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the sorted indices + can be identified, use it instead of recomputing. + indices_cache_file_names (`[Dict[str, str]]`, *optional*, defaults to `None`): + Provide the name of a path for the cache file. It is used to store the + indices mapping instead of the automatically generated cache file name. + You have to provide one `cache_file_name` per dataset in the dataset dictionary. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + Higher value gives smaller cache files, lower value consume less temporary memory. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes') + >>> ds['train']['label'][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + >>> sorted_ds = ds.sort('label') + >>> sorted_ds['train']['label'][:10] + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + >>> another_sorted_ds = ds.sort(['label', 'text'], reverse=[True, False]) + >>> another_sorted_ds['train']['label'][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + ``` + """ + self._check_values_type() + if indices_cache_file_names is None: + indices_cache_file_names = dict.fromkeys(self) + return DatasetDict( + { + k: dataset.sort( + column_names=column_names, + reverse=reverse, + null_placement=null_placement, + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + indices_cache_file_name=indices_cache_file_names[k], + writer_batch_size=writer_batch_size, + ) + for k, dataset in self.items() + } + ) + + def shuffle( + self, + seeds: Optional[Union[int, dict[str, Optional[int]]]] = None, + seed: Optional[int] = None, + generators: Optional[dict[str, np.random.Generator]] = None, + keep_in_memory: bool = False, + load_from_cache_file: Optional[bool] = None, + indices_cache_file_names: Optional[dict[str, Optional[str]]] = None, + writer_batch_size: Optional[int] = 1000, + ) -> "DatasetDict": + """Create a new Dataset where the rows are shuffled. + + The transformation is applied to all the datasets of the dataset dictionary. + + Currently shuffling uses numpy random generators. + You can either supply a NumPy BitGenerator to use, or a seed to initiate NumPy's default random generator (PCG64). + + Args: + seeds (`Dict[str, int]` or `int`, *optional*): + A seed to initialize the default BitGenerator if `generator=None`. + If `None`, then fresh, unpredictable entropy will be pulled from the OS. + If an `int` or `array_like[ints]` is passed, then it will be passed to SeedSequence to derive the initial BitGenerator state. + You can provide one `seed` per dataset in the dataset dictionary. + seed (`int`, *optional*): + A seed to initialize the default BitGenerator if `generator=None`. Alias for seeds (a `ValueError` is raised if both are provided). + generators (`Dict[str, *optional*, np.random.Generator]`): + Numpy random Generator to use to compute the permutation of the dataset rows. + If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy). + You have to provide one `generator` per dataset in the dataset dictionary. + keep_in_memory (`bool`, defaults to `False`): + Keep the dataset in memory instead of writing it to a cache file. + load_from_cache_file (`Optional[bool]`, defaults to `True` if caching is enabled): + If a cache file storing the current computation from `function` + can be identified, use it instead of recomputing. + indices_cache_file_names (`Dict[str, str]`, *optional*): + Provide the name of a path for the cache file. It is used to store the + indices mappings instead of the automatically generated cache file name. + You have to provide one `cache_file_name` per dataset in the dataset dictionary. + writer_batch_size (`int`, defaults to `1000`): + Number of rows per write operation for the cache file writer. + This value is a good trade-off between memory usage during the processing, and processing speed. + Higher value makes the processing do fewer lookups, lower value consume less temporary memory while running `map`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds["train"]["label"][:10] + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + # set a seed + >>> shuffled_ds = ds.shuffle(seed=42) + >>> shuffled_ds["train"]["label"][:10] + [0, 1, 0, 1, 0, 0, 0, 0, 0, 0] + ``` + """ + self._check_values_type() + if seed is not None and seeds is not None: + raise ValueError("Please specify seed or seeds, but not both") + seeds = seed if seed is not None else seeds + if seeds is None: + seeds = dict.fromkeys(self) + elif not isinstance(seeds, dict): + seeds = dict.fromkeys(self, seeds) + if generators is None: + generators = dict.fromkeys(self) + if indices_cache_file_names is None: + indices_cache_file_names = dict.fromkeys(self) + return DatasetDict( + { + k: dataset.shuffle( + seed=seeds[k], + generator=generators[k], + keep_in_memory=keep_in_memory, + load_from_cache_file=load_from_cache_file, + indices_cache_file_name=indices_cache_file_names[k], + writer_batch_size=writer_batch_size, + ) + for k, dataset in self.items() + } + ) + + def save_to_disk( + self, + dataset_dict_path: PathLike, + max_shard_size: Optional[Union[str, int]] = None, + num_shards: Optional[dict[str, int]] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + ): + """ + Saves a dataset dict to a filesystem using `fsspec.spec.AbstractFileSystem`. + + For [`Image`], [`Audio`] and [`Video`] data: + + All the Image(), Audio() and Video() data are stored in the arrow files. + If you want to store paths or urls, please use the Value("string") type. + + Args: + dataset_dict_path (`path-like`): + Path (e.g. `dataset/train`) or remote URI (e.g. `s3://my-bucket/dataset/train`) + of the dataset dict directory where the dataset dict will be saved to. + max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`): + The maximum size of the dataset shards to be saved to the filesystem. If expressed as a string, needs to be digits followed by a unit + (like `"50MB"`). + num_shards (`Dict[str, int]`, *optional*): + Number of shards to write. By default the number of shards depends on `max_shard_size` and `num_proc`. + You need to provide the number of shards for each dataset in the dataset dictionary. + Use a dictionary to define a different num_shards for each split. + + + num_proc (`int`, *optional*, default `None`): + Number of processes when downloading and generating the dataset locally. + Multiprocessing is disabled by default. + + + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Example: + + ```python + >>> dataset_dict.save_to_disk("path/to/dataset/directory") + >>> dataset_dict.save_to_disk("path/to/dataset/directory", max_shard_size="1GB") + >>> dataset_dict.save_to_disk("path/to/dataset/directory", num_shards={"train": 1024, "test": 8}) + ``` + """ + fs: fsspec.AbstractFileSystem + fs, _ = url_to_fs(dataset_dict_path, **(storage_options or {})) + + if num_shards is None: + num_shards = dict.fromkeys(self) + elif not isinstance(num_shards, dict): + raise ValueError( + "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}" + ) + + fs.makedirs(dataset_dict_path, exist_ok=True) + + with fs.open( + posixpath.join(dataset_dict_path, config.DATASETDICT_JSON_FILENAME), + "w", + encoding="utf-8", + ) as f: + json.dump({"splits": list(self)}, f) + for k, dataset in self.items(): + dataset.save_to_disk( + posixpath.join(dataset_dict_path, k), + num_shards=num_shards.get(k), + max_shard_size=max_shard_size, + num_proc=num_proc, + storage_options=storage_options, + ) + + @staticmethod + def load_from_disk( + dataset_dict_path: PathLike, + keep_in_memory: Optional[bool] = None, + storage_options: Optional[dict] = None, + ) -> "DatasetDict": + """ + Load a dataset that was previously saved using [`save_to_disk`] from a filesystem using `fsspec.spec.AbstractFileSystem`. + + Args: + dataset_dict_path (`path-like`): + Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3//my-bucket/dataset/train"`) + of the dataset dict directory where the dataset dict will be loaded from. + keep_in_memory (`bool`, defaults to `None`): + Whether to copy the dataset in-memory. If `None`, the + dataset will not be copied in-memory unless explicitly enabled by setting + `datasets.config.IN_MEMORY_MAX_SIZE` to nonzero. See more details in the + [improve performance](../cache#improve-performance) section. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> ds = load_from_disk('path/to/dataset/directory') + ``` + """ + fs: fsspec.AbstractFileSystem + fs, dataset_dict_path = url_to_fs(dataset_dict_path, **(storage_options or {})) + + dataset_dict_json_path = posixpath.join(dataset_dict_path, config.DATASETDICT_JSON_FILENAME) + dataset_state_json_path = posixpath.join(dataset_dict_path, config.DATASET_STATE_JSON_FILENAME) + dataset_info_path = posixpath.join(dataset_dict_path, config.DATASET_INFO_FILENAME) + if not fs.isfile(dataset_dict_json_path): + if fs.isfile(dataset_info_path) and fs.isfile(dataset_state_json_path): + raise FileNotFoundError( + f"No such file: '{dataset_dict_json_path}'. Expected to load a `DatasetDict` object, but got a `Dataset`. Please use either `datasets.load_from_disk` or `Dataset.load_from_disk` instead." + ) + raise FileNotFoundError( + f"No such file: '{dataset_dict_json_path}'. Expected to load a `DatasetDict` object, but provided path is not a `DatasetDict`." + ) + + with fs.open(dataset_dict_json_path, "r", encoding="utf-8") as f: + splits = json.load(f)["splits"] + + dataset_dict = DatasetDict() + for k in splits: + dataset_dict_split_path = posixpath.join(fs.unstrip_protocol(dataset_dict_path), k) + dataset_dict[k] = Dataset.load_from_disk( + dataset_dict_split_path, + keep_in_memory=keep_in_memory, + storage_options=storage_options, + ) + return dataset_dict + + @staticmethod + def from_csv( + path_or_paths: dict[str, PathLike], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + **kwargs, + ) -> "DatasetDict": + """Create [`DatasetDict`] from CSV file(s). + + Args: + path_or_paths (`dict` of path-like): + Path(s) of the CSV file(s). + features ([`Features`], *optional*): + Dataset features. + cache_dir (str, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`pandas.read_csv`]. + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> from datasets import DatasetDict + >>> ds = DatasetDict.from_csv({'train': 'path/to/dataset.csv'}) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.csv import CsvDatasetReader + + return CsvDatasetReader( + path_or_paths, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + **kwargs, + ).read() + + @staticmethod + def from_json( + path_or_paths: dict[str, PathLike], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + **kwargs, + ) -> "DatasetDict": + """Create [`DatasetDict`] from JSON Lines file(s). + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the JSON Lines file(s). + features ([`Features`], *optional*): + Dataset features. + cache_dir (str, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`JsonConfig`]. + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> from datasets import DatasetDict + >>> ds = DatasetDict.from_json({'train': 'path/to/dataset.json'}) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.json import JsonDatasetReader + + return JsonDatasetReader( + path_or_paths, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + **kwargs, + ).read() + + @staticmethod + def from_parquet( + path_or_paths: dict[str, PathLike], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + columns: Optional[list[str]] = None, + **kwargs, + ) -> "DatasetDict": + """Create [`DatasetDict`] from Parquet file(s). + + Args: + path_or_paths (`dict` of path-like): + Path(s) of the CSV file(s). + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + columns (`list[str]`, *optional*): + If not `None`, only these columns will be read from the file. + A column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e'. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`ParquetConfig`]. + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> from datasets import DatasetDict + >>> ds = DatasetDict.from_parquet({'train': 'path/to/dataset/parquet'}) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.parquet import ParquetDatasetReader + + return ParquetDatasetReader( + path_or_paths, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + columns=columns, + **kwargs, + ).read() + + @staticmethod + def from_text( + path_or_paths: dict[str, PathLike], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + **kwargs, + ) -> "DatasetDict": + """Create [`DatasetDict`] from text file(s). + + Args: + path_or_paths (`dict` of path-like): + Path(s) of the text file(s). + features ([`Features`], *optional*): + Dataset features. + cache_dir (`str`, *optional*, defaults to `"~/.cache/huggingface/datasets"`): + Directory to cache data. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`TextConfig`]. + + Returns: + [`DatasetDict`] + + Example: + + ```py + >>> from datasets import DatasetDict + >>> ds = DatasetDict.from_text({'train': 'path/to/dataset.txt'}) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.text import TextDatasetReader + + return TextDatasetReader( + path_or_paths, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + **kwargs, + ).read() + + @is_documented_by(Dataset.align_labels_with_mapping) + def align_labels_with_mapping(self, label2id: dict, label_column: str) -> "DatasetDict": + self._check_values_type() + return DatasetDict( + { + k: dataset.align_labels_with_mapping(label2id=label2id, label_column=label_column) + for k, dataset in self.items() + } + ) + + def push_to_hub( + self, + repo_id: str, + config_name: str = "default", + set_default: Optional[bool] = None, + data_dir: Optional[str] = None, + commit_message: Optional[str] = None, + commit_description: Optional[str] = None, + private: Optional[bool] = None, + token: Optional[str] = None, + revision: Optional[str] = None, + create_pr: Optional[bool] = False, + max_shard_size: Optional[Union[int, str]] = None, + num_shards: Optional[dict[str, int]] = None, + embed_external_files: bool = True, + num_proc: Optional[int] = None, + ) -> Optional[CommitInfo]: + """Pushes the [`DatasetDict`] to the hub as a Parquet dataset. + The [`DatasetDict`] is pushed using HTTP requests and does not need to have neither git or git-lfs installed. + + Each dataset split will be pushed independently. The pushed dataset will keep the original split names. + + The resulting Parquet files are self-contained by default: if your dataset contains [`Image`] or [`Audio`] + data, the Parquet files will store the bytes of your images or audio files. + You can disable this by setting `embed_external_files` to False. + + Args: + repo_id (`str`): + The ID of the repository to push to in the following format: `/` or + `/`. Also accepts ``, which will default to the namespace + of the logged-in user. + + It could also be a location inside a bucket, e.g. `buckets///...` + config_name (`str`): + Configuration name of a dataset. Defaults to "default". + set_default (`bool`, *optional*): + Whether to set this configuration as the default one. Otherwise, the default configuration is the one + named "default". + data_dir (`str`, *optional*): + Directory name that will contain the uploaded data files. Defaults to the `config_name` if different + from "default", else "data". + + + commit_message (`str`, *optional*): + Message to commit while pushing. Will default to `"Upload dataset"`. + commit_description (`str`, *optional*): + Description of the commit that will be created. + Additionally, description of the PR if a PR is created (`create_pr` is True). + + + private (`bool`, *optional*): + Whether to make the repo private. If `None` (default), the repo will be public unless the + organization's default is private. This value is ignored if the repo already exists. + token (`str`, *optional*): + An optional authentication token for the Hugging Face Hub. If no token is passed, will default + to the token saved locally when logging in with `huggingface-cli login`. Will raise an error + if no token is passed and the user is not logged-in. + revision (`str`, *optional*): + Branch to push the uploaded files to. Defaults to the `"main"` branch. + + + create_pr (`bool`, *optional*, defaults to `False`): + Whether to create a PR with the uploaded files or directly commit. + + + max_shard_size (`int` or `str`, *optional*, defaults to `"500MB"`): + The maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed by a unit + (like `"500MB"` or `"1GB"`). + num_shards (`Dict[str, int]`, *optional*): + Number of shards to write. By default, the number of shards depends on `max_shard_size`. + Use a dictionary to define a different num_shards for each split. + + + embed_external_files (`bool`, defaults to `True`): + Whether to embed file bytes in the shards. + In particular, this will do the following before the push for the fields of type: + + - [`Audio`] and [`Image`] removes local path information and embed file content in the Parquet files. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when preparing and uploading the dataset. + This is helpful if the dataset is made of many samples or media files to embed. + I uses "spawn" context to work with hf_xet, the rust client for fast uploads to HF. + Multiprocessing is disabled by default. + + + + Return: + huggingface_hub.CommitInfo + + Example: + + ```python + >>> dataset_dict.push_to_hub("/") + >>> dataset_dict.push_to_hub("/", private=True) + >>> dataset_dict.push_to_hub("/", max_shard_size="1GB") + >>> dataset_dict.push_to_hub("/", num_shards={"train": 1024, "test": 8}) + ``` + + If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages): + + ```python + >>> english_dataset.push_to_hub("/", "en") + >>> french_dataset.push_to_hub("/", "fr") + >>> # later + >>> english_dataset = load_dataset("/", "en") + >>> french_dataset = load_dataset("/", "fr") + ``` + """ + if num_shards is None: + num_shards = dict.fromkeys(self) + elif not isinstance(num_shards, dict): + raise ValueError( + "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}" + ) + + self._check_values_type() + self._check_values_features() + if config_name == "data": + raise ValueError("`config_name` cannot be 'data'. Please, choose another name for configuration.") + + if max_shard_size is not None and num_shards is not None and any(x is not None for x in num_shards.values()): + raise ValueError( + "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both." + ) + + for split in self: + if not re.match(_split_re, split): + raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.") + + if not data_dir: + data_dir = config_name if config_name != "default" else "data" # for backward compatibility + + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + if repo_id.startswith("buckets/"): + if BucketNotFoundError is None: + raise ImportError("Pushing datasets to buckets requires huggingface_hub>=1.6.0") + _, _namespace, _bucket_name, *_path_segments = repo_id.split("/") + try: + bucket_id = api.bucket_info(_namespace + "/" + _bucket_name).id + except BucketNotFoundError: + bucket_url = api.create_bucket(_namespace + "/" + _bucket_name, private=private, exist_ok=True) + bucket_id = bucket_url.bucket_id + path = "/".join(s for s in _path_segments if s) + return _push_to_bucket( + self, + bucket_id=bucket_id, + path=path, + config_name=config_name, + set_default=set_default, + data_dir=data_dir, + token=token, + max_shard_size=max_shard_size, + num_shards=num_shards, + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + else: + try: + repo_id = api.repo_info(repo_id, repo_type="dataset").id + except RepositoryNotFoundError: + repo_url = api.create_repo( + repo_id, + repo_type="dataset", + private=private, + exist_ok=True, + ) + repo_id = repo_url.repo_id + + if revision is not None and not revision.startswith("refs/pr/"): + # We do not call create_branch for a PR reference: 400 Bad Request + api.create_branch(repo_id, branch=revision, repo_type="dataset", exist_ok=True) + return _push_to_repo( + self, + repo_id=repo_id, + config_name=config_name, + set_default=set_default, + data_dir=data_dir, + commit_message=commit_message, + commit_description=commit_description, + token=token, + revision=revision, + create_pr=create_pr, + max_shard_size=max_shard_size, + num_shards=num_shards, + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + + +class IterableDatasetDict(dict[Union[str, NamedSplit], IterableDataset]): + def _check_values_type(self): + for dataset in self.values(): + if not isinstance(dataset, IterableDataset): + raise TypeError(f"Values in `DatasetDict` should be of type `Dataset` but got type '{type(dataset)}'") + + def _check_values_features(self): + items = [(key, dataset._resolve_features()) for key, dataset in self.items()] + for item_a, item_b in zip(items[:-1], items[1:]): + if item_a[1].features != item_b[1].features: + raise ValueError( + f"All datasets in `DatasetDict` should have the same features but features for '{item_a[0]}' and '{item_b[0]}' don't match: {item_a[1].features} != {item_b[1].features}" + ) + + def __repr__(self): + repr = "\n".join([f"{k}: {v}" for k, v in self.items()]) + repr = re.sub(r"^", " " * 4, repr, count=0, flags=re.M) + return f"IterableDatasetDict({{\n{repr}\n}})" + + @property + def num_columns(self) -> dict[str, Optional[int]]: + """Number of columns in each split of the dataset. + This can contain None valies if some splits have unknown features (e.g. after a map() operation). + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.num_columns + {'test': 2, 'train': 2, 'validation': 2} + ``` + """ + self._check_values_type() + return {k: dataset.num_columns for k, dataset in self.items()} + + @property + def column_names(self) -> dict[str, Optional[list[str]]]: + """Names of the columns in each split of the dataset. + This can contain None valies if some splits have unknown features (e.g. after a map() operation). + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes") + >>> ds.column_names + {'test': ['text', 'label'], + 'train': ['text', 'label'], + 'validation': ['text', 'label']} + ``` + """ + self._check_values_type() + return {k: dataset.column_names for k, dataset in self.items()} + + def with_format( + self, + type: Optional[str] = None, + ) -> "IterableDatasetDict": + """ + Return a dataset with the specified format. + + Args: + + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means it returns python objects (default). + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True) + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds = ds.with_format("torch") + >>> next(iter(ds)) + {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .', + 'label': tensor(1), + 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617, + 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105, + 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])} + ``` + """ + return IterableDatasetDict({k: dataset.with_format(type=type) for k, dataset in self.items()}) + + def map( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + with_split: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: int = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[Union[str, list[str]]] = None, + fn_kwargs: Optional[dict] = None, + ) -> "IterableDatasetDict": + """ + Apply a function to all the examples in the iterable dataset (individually or in batches) and update them. + If your function returns a column that already exists, then it overwrites it. + The function is applied on-the-fly on the examples when iterating over the dataset. + The transformation is applied to all the datasets of the dataset dictionary. + + You can specify whether the function should be batched or not with the `batched` parameter: + + - If batched is `False`, then the function takes 1 example in and should return 1 example. + An example is a dictionary, e.g. `{"text": "Hello there !"}`. + - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples. + A batch is a dictionary, e.g. a batch of 1 example is `{"text": ["Hello there !"]}`. + - If batched is `True` and `batch_size` is `n` > 1, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples. + Note that the last batch may have less than `n` examples. + A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`. + + If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simultaneous calls. + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`Callable`, *optional*, defaults to `None`): + Function applied on-the-fly on the examples when you iterate on the dataset. + It must have one of the following signatures: + + - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` + - `function(example: Dict[str, Any], idx: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True` + - `function(batch: Dict[str, list]) -> Dict[str, list]` if `batched=True` and `with_indices=False` + - `function(batch: Dict[str, list], indices: list[int]) -> Dict[str, list]` if `batched=True` and `with_indices=True` + + For advanced usage, the function can also return a `pyarrow.Table`. + If the function is asynchronous, then `map` will run your function in parallel. + Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged. + If no function is provided, default to identity function: `lambda x: x`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`. + input_columns (`[Union[str, list[str]]]`, *optional*, defaults to `None`): + The columns to be passed into `function` + as positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True`. + drop_last_batch (`bool`, defaults to `False`): + Whether a last batch smaller than the `batch_size` should be + dropped instead of being processed by the function. + remove_columns (`[list[str]]`, *optional*, defaults to `None`): + Remove a selection of columns while doing the mapping. + Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding + columns with names in `remove_columns`, these columns will be kept. + fn_kwargs (`Dict`, *optional*, defaults to `None`): + Keyword arguments to be passed to `function` + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> def add_prefix(example): + ... example["text"] = "Review: " + example["text"] + ... return example + >>> ds = ds.map(add_prefix) + >>> next(iter(ds["train"])) + {'label': 1, + 'text': 'Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + + dataset_dict = {} + for split, dataset in self.items(): + if with_split: + function = bind(function, split) + + dataset_dict[split] = dataset.map( + function=function, + with_indices=with_indices, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + drop_last_batch=drop_last_batch, + remove_columns=remove_columns, + fn_kwargs=fn_kwargs, + ) + + if with_split: + function = function.func + + return IterableDatasetDict(dataset_dict) + + def filter( + self, + function: Optional[Callable] = None, + with_indices=False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + fn_kwargs: Optional[dict] = None, + ) -> "IterableDatasetDict": + """Apply a filter function to all the elements so that the dataset only includes examples according to the filter function. + The filtering is done on-the-fly when iterating over the dataset. + The filtering is applied to all the datasets of the dataset dictionary. + + Args: + function (`Callable`): + Callable with one of the following signatures: + + - `function(example: Dict[str, Any]) -> bool` if `with_indices=False, batched=False` + - `function(example: Dict[str, Any], indices: int) -> bool` if `with_indices=True, batched=False` + - `function(example: Dict[str, list]) -> list[bool]` if `with_indices=False, batched=True` + - `function(example: Dict[str, list], indices: list[int]) -> list[bool]` if `with_indices=True, batched=True` + + If no function is provided, defaults to an always True function: `lambda x: True`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`. + input_columns (`str` or `list[str]`, *optional*): + The columns to be passed into `function` as + positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function` + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True`. + fn_kwargs (`Dict`, *optional*, defaults to `None`): + Keyword arguments to be passed to `function` + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds = ds.filter(lambda x: x["label"] == 0) + >>> list(ds["train"].take(3)) + [{'label': 0, 'text': 'Review: simplistic , silly and tedious .'}, + {'label': 0, + 'text': "Review: it's so laddish and juvenile , only teenage boys could possibly find it funny ."}, + {'label': 0, + 'text': 'Review: exploitative and largely devoid of the depth or sophistication that would make watching such a graphic treatment of the crimes bearable .'}] + ``` + """ + return IterableDatasetDict( + { + k: dataset.filter( + function=function, + with_indices=with_indices, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + fn_kwargs=fn_kwargs, + ) + for k, dataset in self.items() + } + ) + + def shuffle( + self, + seed=None, + generator: Optional[np.random.Generator] = None, + buffer_size: int = 1000, + ) -> "IterableDatasetDict": + """ + Randomly shuffles the elements of this dataset. + The shuffling is applied to all the datasets of the dataset dictionary. + + This dataset fills a buffer with buffer_size elements, then randomly samples elements from this buffer, + replacing the selected elements with new elements. For perfect shuffling, a buffer size greater than or + equal to the full size of the dataset is required. + + For instance, if your dataset contains 10,000 elements but `buffer_size` is set to 1000, then `shuffle` will + initially select a random element from only the first 1000 elements in the buffer. Once an element is + selected, its space in the buffer is replaced by the next (i.e. 1,001-st) element, + maintaining the 1000 element buffer. + + If the dataset is made of several shards, it also does `shuffle` the order of the shards. + However if the order has been fixed by using [`~datasets.IterableDataset.skip`] or [`~datasets.IterableDataset.take`] + then the order of the shards is kept unchanged. + + Args: + seed (`int`, *optional*, defaults to `None`): + Random seed that will be used to shuffle the dataset. + It is used to sample from the shuffle buffer and also to shuffle the data shards. + generator (`numpy.random.Generator`, *optional*): + Numpy random Generator to use to compute the permutation of the dataset rows. + If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy). + buffer_size (`int`, defaults to `1000`): + Size of the buffer. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> list(ds["train"].take(3)) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + >>> ds = ds.shuffle(seed=42) + >>> list(ds["train"].take(3)) + [{'label': 1, + 'text': "a sports movie with action that's exciting on the field and a story you care about off it ."}, + {'label': 1, + 'text': 'at its best , the good girl is a refreshingly adult take on adultery . . .'}, + {'label': 1, + 'text': "sam jones became a very lucky filmmaker the day wilco got dropped from their record label , proving that one man's ruin may be another's fortune ."}] + ``` + """ + return IterableDatasetDict( + { + k: dataset.shuffle(seed=seed, generator=generator, buffer_size=buffer_size) + for k, dataset in self.items() + } + ) + + def rename_column(self, original_column_name: str, new_column_name: str) -> "IterableDatasetDict": + """ + Rename a column in the dataset, and move the features associated to the original column under the new column + name. + The renaming is applied to all the datasets of the dataset dictionary. + + Args: + original_column_name (`str`): + Name of the column to rename. + new_column_name (`str`): + New name for the column. + + Returns: + [`IterableDatasetDict`]: A copy of the dataset with a renamed column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds = ds.rename_column("text", "movie_review") + >>> next(iter(ds["train"])) + {'label': 1, + 'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + return IterableDatasetDict( + { + k: dataset.rename_column( + original_column_name=original_column_name, + new_column_name=new_column_name, + ) + for k, dataset in self.items() + } + ) + + def rename_columns(self, column_mapping: dict[str, str]) -> "IterableDatasetDict": + """ + Rename several columns in the dataset, and move the features associated to the original columns under + the new column names. + The renaming is applied to all the datasets of the dataset dictionary. + + Args: + column_mapping (`Dict[str, str]`): + A mapping of columns to rename to their new names. + + Returns: + [`IterableDatasetDict`]: A copy of the dataset with renamed columns + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds = ds.rename_columns({"text": "movie_review", "label": "rating"}) + >>> next(iter(ds["train"])) + {'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', + 'rating': 1} + ``` + """ + return IterableDatasetDict( + {k: dataset.rename_columns(column_mapping=column_mapping) for k, dataset in self.items()} + ) + + def remove_columns(self, column_names: Union[str, list[str]]) -> "IterableDatasetDict": + """ + Remove one or several column(s) in the dataset and the features associated to them. + The removal is done on-the-fly on the examples when iterating over the dataset. + The removal is applied to all the datasets of the dataset dictionary. + + + Args: + column_names (`Union[str, list[str]]`): + Name of the column(s) to remove. + + Returns: + [`IterableDatasetDict`]: A copy of the dataset object without the columns to remove. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds = ds.remove_columns("label") + >>> next(iter(ds["train"])) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + return IterableDatasetDict({k: dataset.remove_columns(column_names) for k, dataset in self.items()}) + + def select_columns(self, column_names: Union[str, list[str]]) -> "IterableDatasetDict": + """Select one or several column(s) in the dataset and the features + associated to them. The selection is done on-the-fly on the examples + when iterating over the dataset. The selection is applied to all the + datasets of the dataset dictionary. + + + Args: + column_names (`Union[str, list[str]]`): + Name of the column(s) to keep. + + Returns: + [`IterableDatasetDict`]: A copy of the dataset object with only selected columns. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds = ds.select("text") + >>> next(iter(ds["train"])) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + return IterableDatasetDict({k: dataset.select_columns(column_names) for k, dataset in self.items()}) + + def cast_column(self, column: str, feature: FeatureType) -> "IterableDatasetDict": + """Cast column to feature for decoding. + The type casting is applied to all the datasets of the dataset dictionary. + + Args: + column (`str`): + Column name. + feature ([`Feature`]): + Target feature. + + Returns: + [`IterableDatasetDict`] + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds["train"].features + {'label': ClassLabel(names=['neg', 'pos']), + 'text': Value('string')} + >>> ds = ds.cast_column('label', ClassLabel(names=['bad', 'good'])) + >>> ds["train"].features + {'label': ClassLabel(names=['bad', 'good']), + 'text': Value('string')} + ``` + """ + return IterableDatasetDict( + {k: dataset.cast_column(column=column, feature=feature) for k, dataset in self.items()} + ) + + def cast( + self, + features: Features, + ) -> "IterableDatasetDict": + """ + Cast the dataset to a new set of features. + The type casting is applied to all the datasets of the dataset dictionary. + + Args: + features (`Features`): + New features to cast the dataset to. + The name of the fields in the features must match the current column names. + The type of the data must also be convertible from one type to the other. + For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`map`] to update the Dataset. + + Returns: + [`IterableDatasetDict`]: A copy of the dataset with casted features. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", streaming=True) + >>> ds["train"].features + {'label': ClassLabel(names=['neg', 'pos']), + 'text': Value('string')} + >>> new_features = ds["train"].features.copy() + >>> new_features['label'] = ClassLabel(names=['bad', 'good']) + >>> new_features['text'] = Value('large_string') + >>> ds = ds.cast(new_features) + >>> ds["train"].features + {'label': ClassLabel(names=['bad', 'good']), + 'text': Value('large_string')} + ``` + """ + return IterableDatasetDict({k: dataset.cast(features=features) for k, dataset in self.items()}) + + def push_to_hub( + self, + repo_id: str, + config_name: str = "default", + set_default: Optional[bool] = None, + data_dir: Optional[str] = None, + commit_message: Optional[str] = None, + commit_description: Optional[str] = None, + private: Optional[bool] = None, + token: Optional[str] = None, + revision: Optional[str] = None, + create_pr: Optional[bool] = False, + max_shard_size: Optional[Union[int, str]] = None, + num_shards: Optional[dict[str, int]] = None, + embed_external_files: bool = True, + num_proc: Optional[int] = None, + ) -> CommitInfo: + """Pushes the [`IterableDatasetDict`] to the hub as a Parquet dataset. + The [`IterableDatasetDict`] is pushed using HTTP requests and does not need to have neither git or git-lfs installed. + + Each dataset split will be pushed independently. The pushed dataset will keep the original split names. + + The resulting Parquet files are self-contained by default: if your dataset contains [`Image`] or [`Audio`] + data, the Parquet files will store the bytes of your images or audio files. + You can disable this by setting `embed_external_files` to False. + + Args: + repo_id (`str`): + The ID of the repository to push to in the following format: `/` or + `/`. Also accepts ``, which will default to the namespace + of the logged-in user. + + It could also be a location inside a bucket, e.g. `buckets///...` + config_name (`str`): + Configuration name of a dataset. Defaults to "default". + set_default (`bool`, *optional*): + Whether to set this configuration as the default one. Otherwise, the default configuration is the one + named "default". + data_dir (`str`, *optional*): + Directory name that will contain the uploaded data files. Defaults to the `config_name` if different + from "default", else "data". + + + commit_message (`str`, *optional*): + Message to commit while pushing. Will default to `"Upload dataset"`. + commit_description (`str`, *optional*): + Description of the commit that will be created. + Additionally, description of the PR if a PR is created (`create_pr` is True). + + + private (`bool`, *optional*): + Whether to make the repo private. If `None` (default), the repo will be public unless the + organization's default is private. This value is ignored if the repo already exists. + token (`str`, *optional*): + An optional authentication token for the Hugging Face Hub. If no token is passed, will default + to the token saved locally when logging in with `huggingface-cli login`. Will raise an error + if no token is passed and the user is not logged-in. + revision (`str`, *optional*): + Branch to push the uploaded files to. Defaults to the `"main"` branch. + create_pr (`bool`, *optional*, defaults to `False`): + Whether to create a PR with the uploaded files or directly commit. + max_shard_size (`int` or `str`, *optional*): + Optional maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed + by a unit (like `"500MB"` or `"1GB"`). If not provided, each split keeps its dataset-native shard count. + num_shards (`Dict[str, int]`, *optional*): + Number of shards to write. Use a dictionary to define a different `num_shards` for each split. + If `max_shard_size` is provided and a split's `num_shards` is not, then the number of shards for that split is estimated + from `max_shard_size`. + embed_external_files (`bool`, defaults to `True`): + Whether to embed file bytes in the shards. + In particular, this will do the following before the push for the fields of type: + + - [`Audio`] and [`Image`] removes local path information and embed file content in the Parquet files. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when preparing and uploading the dataset. + This is helpful if the dataset is made of many samples or media files to embed. + I uses "spawn" context to work with hf_xet, the rust client for fast uploads to HF. + Multiprocessing is disabled by default. + + + + Return: + huggingface_hub.CommitInfo + + Example: + + ```python + >>> dataset_dict.push_to_hub("/") + >>> dataset_dict.push_to_hub("/", private=True) + >>> dataset_dict.push_to_hub("/", max_shard_size="1GB") + >>> dataset_dict.push_to_hub("/", num_shards={"train": 1024, "test": 8}) + ``` + + If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages): + + ```python + >>> english_dataset.push_to_hub("/", "en") + >>> french_dataset.push_to_hub("/", "fr") + >>> # later + >>> english_dataset = load_dataset("/", "en") + >>> french_dataset = load_dataset("/", "fr") + ``` + """ + # check to make sure that the user doesnt specify the # of shards and max shard sdize at same time, since these are 2 different ways to specify the same thing + if max_shard_size is not None and num_shards is not None: + raise ValueError( + "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both." + ) + + if num_shards is None: + num_shards = dict.fromkeys(self) + elif not isinstance(num_shards, dict): + raise ValueError( + "Please provide one `num_shards` per dataset in the dataset dictionary, e.g. {{'train': 128, 'test': 4}}" + ) + + self._check_values_type() + self._check_values_features() + if config_name == "data": + raise ValueError("`config_name` cannot be 'data'. Please, choose another name for configuration.") + + # if max_shard_size is not None and num_shards is not None and any(x is not None for x in num_shards.values()): + # raise ValueError( + # "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both." + # ) + + for split in self: + if not re.match(_split_re, split): + raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.") + + if not data_dir: + data_dir = config_name if config_name != "default" else "data" # for backward compatibility + + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + if repo_id.startswith("buckets/"): + if BucketNotFoundError is None: + raise ImportError("Pushing datasets to buckets requires huggingface_hub>=1.6.0") + _, _namespace, _bucket_name, *_path_segments = repo_id.split("/") + try: + bucket_id = api.bucket_info(_namespace + "/" + _bucket_name).id + except BucketNotFoundError: + bucket_url = api.create_bucket(_namespace + "/" + _bucket_name, private=private, exist_ok=True) + bucket_id = bucket_url.bucket_id + path = "/".join(s for s in _path_segments if s) + return _push_to_bucket( + self, + bucket_id=bucket_id, + path=path, + config_name=config_name, + set_default=set_default, + data_dir=data_dir, + token=token, + max_shard_size=max_shard_size, + num_shards=num_shards.get(split), + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + else: + try: + repo_id = api.repo_info(repo_id, repo_type="dataset").id + except RepositoryNotFoundError: + repo_url = api.create_repo( + repo_id, + repo_type="dataset", + private=private, + exist_ok=True, + ) + repo_id = repo_url.repo_id + + if revision is not None and not revision.startswith("refs/pr/"): + # We do not call create_branch for a PR reference: 400 Bad Request + api.create_branch(repo_id, branch=revision, repo_type="dataset", exist_ok=True) + return _push_to_repo( + self, + repo_id=repo_id, + config_name=config_name, + set_default=set_default, + data_dir=data_dir, + commit_message=commit_message, + commit_description=commit_description, + token=token, + create_pr=create_pr, + revision=revision, + max_shard_size=max_shard_size, + num_shards=num_shards, + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + + +def _push_to_repo( + dset_dict: Union["DatasetDict", "IterableDatasetDict"], + repo_id: str, + config_name: str = "default", + set_default: Optional[bool] = None, + data_dir: Optional[str] = None, + commit_message: Optional[str] = None, + commit_description: Optional[str] = None, + token: Optional[str] = None, + revision: Optional[str] = None, + create_pr: Optional[bool] = False, + max_shard_size: Optional[Union[int, str]] = None, + num_shards: Optional[dict[str, Optional[int]]] = None, + embed_external_files: bool = True, + num_proc: Optional[int] = None, +) -> CommitInfo: + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + resolved_output_path = HfFileSystemResolvedRepositoryPath( + repo_id=repo_id, repo_type="dataset", revision=revision or "main", path_in_repo="" + ) + + additions: list[CommitOperationAdd] = [] + new_parquet_paths: list[str] = [] + splits_info: list[SplitInfo] = [] + uploaded_sizes: list[int] = [] + + for split in dset_dict: + split_additions, split_new_parquet_paths, features, split_info, uploaded_size = dset_dict[ + split + ]._push_parquet_shards_to_hub( + resolved_output_path=resolved_output_path, + data_dir=data_dir, + split=split, + token=token, + max_shard_size=max_shard_size, + num_shards=(num_shards or {}).get(split), + create_pr=create_pr, + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + additions += split_additions + new_parquet_paths += split_new_parquet_paths + splits_info.append(split_info) + uploaded_sizes.append(uploaded_size) + + commit_message = commit_message if commit_message is not None else "Upload dataset" + if len(additions) > config.UPLOADS_MAX_NUMBER_PER_COMMIT: + logger.info( + f"Number of files to upload is larger than {config.UPLOADS_MAX_NUMBER_PER_COMMIT}. Splitting the push into multiple commits." + ) + num_commits = math.ceil(len(additions) / config.UPLOADS_MAX_NUMBER_PER_COMMIT) + for i in range(0, num_commits): + operations = additions[ + i * config.UPLOADS_MAX_NUMBER_PER_COMMIT : (i + 1) * config.UPLOADS_MAX_NUMBER_PER_COMMIT + ] + for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1): + # We need to retry if another commit happens at the same time + sleep_time *= 1 + random.random() + try: + commit_info = api.create_commit( + repo_id, + operations=operations, + commit_message=commit_message + f" (part {i:05d}-of-{num_commits:05d})", + commit_description=commit_description, + repo_type="dataset", + revision=revision, + create_pr=create_pr, + ) + except HfHubHTTPError as err: + if ( + err.__context__ + and isinstance(err.__context__, HfHubHTTPError) + and err.__context__.response.status_code == 409 + ): + # 409 is Conflict (another commit is in progress) + time.sleep(sleep_time) + logger.info( + f"Retrying intermediate commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})" + ) + continue + else: + raise + break + logger.info( + f"Commit #{i + 1} completed" + + (f" (still {num_commits - i - 1} to go)" if num_commits - i - 1 else "") + + "." + ) + last_commit_additions = [] + else: + last_commit_additions = additions + + for retry, sleep_time in enumerate(itertools.chain(range(10), itertools.repeat(30)), start=1): + # We need to retry if there was a commit in between in case it touched the dataset card data + sleep_time *= 1 + random.random() + + # We make sure to get info from this commit + parent_commit = api.repo_info(repo_id, repo_type="dataset", revision=revision).sha + hf_path = HfFileSystemResolvedRepositoryPath( + repo_type=resolved_output_path.repo_type, + repo_id=resolved_output_path.repo_id, + revision=parent_commit, + path_in_repo=resolved_output_path.path_in_repo, + ).unresolve() + hffs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token) + dirfs = DirFileSystem(fs=hffs, path=hf_path) + + # Check the files to delete + try: + files_to_delete = list(dirfs.glob(f"{data_dir}/*")) + except EntryNotFoundError: # needed for huggingface_hub<=1.7.1 + files_to_delete = [] + + # Don't delete the new files + deletions = [ + CommitOperationDelete(path_in_repo=file_to_delete) + for file_to_delete in files_to_delete + if file_to_delete not in new_parquet_paths + ] + + # Update the dataset card + new_dataset_card, new_legacy_dataset_infos = _get_updated_dataset_card( + fs=dirfs, + config_name=config_name, + splits_info=splits_info, + features=features, + data_dir=data_dir, + set_default=set_default, + uploaded_sizes=uploaded_sizes, + deleted_sizes=[], + remove_other_splits=True, + ) + dataset_card_additions = [ + CommitOperationAdd(path_in_repo=config.REPOCARD_FILENAME, path_or_fileobj=str(new_dataset_card).encode()) + ] + if new_legacy_dataset_infos: + dataset_card_additions.append( + CommitOperationAdd( + path_in_repo=config.DATASETDICT_INFOS_FILENAME, + path_or_fileobj=json.dumps(new_legacy_dataset_infos).encode("utf-8"), + ) + ) + operations = last_commit_additions + dataset_card_additions + deletions + + try: + commit_info = api.create_commit( + repo_id, + operations=operations, + commit_message=commit_message, + commit_description=commit_description, + repo_type="dataset", + revision=revision, + create_pr=create_pr, + parent_commit=parent_commit, + ) + except HfHubHTTPError as err: + if ( + err.__context__ + and isinstance(err.__context__, HfHubHTTPError) + and err.__context__.response.status_code in (412, 409) + ): + # 412 is Precondition failed (parent_commit isn't satisfied) + # 409 is Conflict (another commit is in progress) + time.sleep(sleep_time) + logger.info( + f"Retrying commit for {repo_id}, {config_name} ({retry}/n with status_code {err.__context__.response.status_code})" + ) + continue + else: + raise + break + + return commit_info + + +def _push_to_bucket( + dset_dict: Union["DatasetDict", "IterableDatasetDict"], + bucket_id: str, + path: str, + config_name: str = "default", + set_default: Optional[bool] = None, + data_dir: Optional[str] = None, + token: Optional[str] = None, + max_shard_size: Optional[Union[int, str]] = None, + num_shards: Optional[dict[str, Optional[int]]] = None, + embed_external_files: bool = True, + num_proc: Optional[int] = None, +) -> None: + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + resolved_output_path = HfFileSystemResolvedBucketPath(bucket_id=bucket_id, path=path) + hf_path = resolved_output_path.unresolve() + hffs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token) + dirfs = DirFileSystem(fs=hffs, path=hf_path) + + additions: list[CommitOperationAdd] = [] + new_parquet_paths: list[str] = [] + splits_info: list[SplitInfo] = [] + uploaded_sizes: list[int] = [] + + # Check the files to delete before uploading + try: + files_to_delete = list(dirfs.glob(f"{data_dir}/*")) + except EntryNotFoundError: # needed for huggingface_hub<=1.7.1 + files_to_delete = [] + + for split in dset_dict: + # Upload the Parquet files + split_additions, split_new_parquet_paths, features, split_info, uploaded_size = dset_dict[ + split + ]._push_parquet_shards_to_hub( + resolved_output_path=resolved_output_path, + data_dir=data_dir, + split=split, + token=token, + max_shard_size=max_shard_size, + num_shards=(num_shards or {}).get(split), + create_pr=False, + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + additions += split_additions + new_parquet_paths += split_new_parquet_paths + splits_info.append(split_info) + uploaded_sizes.append(uploaded_size) + + # Don't delete the new files + new_parquet_paths = set(new_parquet_paths) + delete = [file_to_delete for file_to_delete in files_to_delete if file_to_delete not in new_parquet_paths] + + # Update the dataset card + new_dataset_card, new_legacy_dataset_infos = _get_updated_dataset_card( + fs=dirfs, + config_name=config_name, + splits_info=splits_info, + features=features, + data_dir=data_dir, + set_default=set_default, + uploaded_sizes=uploaded_sizes, + deleted_sizes=[], + remove_other_splits=True, + ) + path_prefix = (path + "/") if path else "" + add = [(str(new_dataset_card).encode(), path_prefix + config.REPOCARD_FILENAME)] + if new_legacy_dataset_infos: + add.append( + (json.dumps(new_legacy_dataset_infos).encode("utf-8"), path_prefix + config.DATASETDICT_INFOS_FILENAME) + ) + + # Upload dataset card and delete old files + api.batch_bucket_files( + bucket_id=bucket_id, + add=add, + delete=delete, + ) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/distributed.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..a9ca59c72ff0987e3f06613a64283c79ce83e8db --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/distributed.py @@ -0,0 +1,43 @@ +from typing import TypeVar + +from .arrow_dataset import Dataset, _split_by_node_map_style_dataset +from .iterable_dataset import IterableDataset, _split_by_node_iterable_dataset + + +DatasetType = TypeVar("DatasetType", Dataset, IterableDataset) + + +def split_dataset_by_node(dataset: DatasetType, rank: int, world_size: int) -> DatasetType: + """ + Split a dataset for the node at rank `rank` in a pool of nodes of size `world_size`. + + For map-style datasets: + + Each node is assigned a chunk of data, e.g. rank 0 is given the first chunk of the dataset. + To maximize data loading throughput, chunks are made of contiguous data on disk if possible. + + For iterable datasets: + + If the dataset has a number of shards that is a factor of `world_size` (i.e. if `dataset.num_shards % world_size == 0`), + then the shards are evenly assigned across the nodes, which is the most optimized. + Otherwise, each node keeps 1 example out of `world_size`, skipping the other examples. + + > [!WARNING] + > If you shuffle your iterable dataset in a distributed setup, make sure to set a fixed `seed` in [`IterableDataset.shuffle`] + so the same shuffled list of shards is used on every node to know which shards the node should skip. + + Args: + dataset ([`Dataset`] or [`IterableDataset`]): + The dataset to split by node. + rank (`int`): + Rank of the current node. + world_size (`int`): + Total number of nodes. + + Returns: + [`Dataset`] or [`IterableDataset`]: The dataset to be used on the node at rank `rank`. + """ + if isinstance(dataset, Dataset): + return _split_by_node_map_style_dataset(dataset, rank=rank, world_size=world_size) + else: + return _split_by_node_iterable_dataset(dataset, rank=rank, world_size=world_size) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8ae0d436504dc2e609b0ca8851509c72a161dbde --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/__init__.py @@ -0,0 +1,10 @@ +__all__ = [ + "DownloadConfig", + "DownloadManager", + "DownloadMode", + "StreamingDownloadManager", +] + +from .download_config import DownloadConfig +from .download_manager import DownloadManager, DownloadMode +from .streaming_download_manager import StreamingDownloadManager diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/download_config.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/download_config.py new file mode 100644 index 0000000000000000000000000000000000000000..6efae72b671bd0977c52453582dc70561766334b --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/download_config.py @@ -0,0 +1,81 @@ +import copy +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional, Union + +from .. import config + + +@dataclass +class DownloadConfig: + """Configuration for our cached path manager. + + Attributes: + cache_dir (`str` or `Path`, *optional*): + Specify a cache directory to save the file to (overwrite the + default cache dir). + force_download (`bool`, defaults to `False`): + If `True`, re-download the file even if it's already cached in + the cache dir. + resume_download (`bool`, defaults to `False`): + If `True`, resume the download if an incompletely received file is + found. + proxies (`dict`, *optional*): + user_agent (`str`, *optional*): + Optional string or dict that will be appended to the user-agent on remote + requests. + extract_compressed_file (`bool`, defaults to `False`): + If `True` and the path point to a zip or tar file, + extract the compressed file in a folder along the archive. + force_extract (`bool`, defaults to `False`): + If `True` when `extract_compressed_file` is `True` and the archive + was already extracted, re-extract the archive and override the folder where it was extracted. + delete_extracted (`bool`, defaults to `False`): + Whether to delete (or keep) the extracted files. + extract_on_the_fly (`bool`, defaults to `False`): + If `True`, extract compressed files while they are being read. + use_etag (`bool`, defaults to `True`): + Whether to use the ETag HTTP response header to validate the cached files. + num_proc (`int`, *optional*): + The number of processes to launch to download the files in parallel. + max_retries (`int`, default to `1`): + The number of times to retry an HTTP request if it fails. + token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token + for remote files on the Datasets Hub. If `True`, or not specified, will get token from `~/.huggingface`. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the dataset file-system backend, if any. + download_desc (`str`, *optional*): + A description to be displayed alongside with the progress bar while downloading the files. + disable_tqdm (`bool`, defaults to `False`): + Whether to disable the individual files download progress bar + """ + + cache_dir: Optional[Union[str, Path]] = None + force_download: bool = False + resume_download: bool = False + local_files_only: bool = False + proxies: Optional[dict] = None + user_agent: Optional[str] = None + extract_compressed_file: bool = False + force_extract: bool = False + delete_extracted: bool = False + extract_on_the_fly: bool = False + use_etag: bool = True + num_proc: Optional[int] = None + max_retries: int = 1 + token: Optional[Union[str, bool]] = None + storage_options: dict[str, Any] = field(default_factory=dict) + download_desc: Optional[str] = None + disable_tqdm: bool = False + + def copy(self) -> "DownloadConfig": + return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()}) + + def __setattr__(self, name, value): + if name == "token" and getattr(self, "storage_options", None) is not None: + if "hf" not in self.storage_options: + self.storage_options["hf"] = {"endpoint": config.HF_ENDPOINT, "token": value} + elif getattr(self.storage_options["hf"], "token", None) is None: + self.storage_options["hf"]["token"] = value + super().__setattr__(name, value) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/download_manager.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/download_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..b3515bc0e08fd7f809dcbe7dc68dc825aad5f06d --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/download_manager.py @@ -0,0 +1,340 @@ +# Copyright 2020 The TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Download manager interface.""" + +import enum +import io +import multiprocessing +import os +from datetime import datetime +from functools import partial +from typing import Optional, Union + +import fsspec +from fsspec.core import url_to_fs +from tqdm.contrib.concurrent import thread_map + +from .. import config +from ..utils import tqdm as hf_tqdm +from ..utils.file_utils import ( + ArchiveIterable, + FilesIterable, + cached_path, + is_relative_path, + stack_multiprocessing_download_progress_bars, + url_or_path_join, +) +from ..utils.info_utils import get_size_checksum_dict +from ..utils.logging import get_logger, tqdm +from ..utils.py_utils import NestedDataStructure, map_nested +from ..utils.track import tracked_str +from .download_config import DownloadConfig + + +logger = get_logger(__name__) + + +class DownloadMode(enum.Enum): + """`Enum` for how to treat pre-existing downloads and data. + + The default mode is `REUSE_DATASET_IF_EXISTS`, which will reuse both + raw downloads and the prepared dataset if they exist. + + The generations modes: + + | | Downloads | Dataset | + |-------------------------------------|-----------|---------| + | `REUSE_DATASET_IF_EXISTS` (default) | Reuse | Reuse | + | `REUSE_CACHE_IF_EXISTS` | Reuse | Fresh | + | `FORCE_REDOWNLOAD` | Fresh | Fresh | + + """ + + REUSE_DATASET_IF_EXISTS = "reuse_dataset_if_exists" + REUSE_CACHE_IF_EXISTS = "reuse_cache_if_exists" + FORCE_REDOWNLOAD = "force_redownload" + + +class DownloadManager: + is_streaming = False + + def __init__( + self, + dataset_name: Optional[str] = None, + data_dir: Optional[str] = None, + download_config: Optional[DownloadConfig] = None, + base_path: Optional[str] = None, + record_checksums=False, + ): + """Download manager constructor. + + Args: + data_dir: + can be used to specify a manual directory to get the files from. + dataset_name (`str`): + name of dataset this instance will be used for. If + provided, downloads will contain which datasets they were used for. + download_config (`DownloadConfig`): + to specify the cache directory and other + download options + base_path (`str`): + base path that is used when relative paths are used to + download files. This can be a remote url. + record_checksums (`bool`, defaults to `False`): + Whether to record the checksums of the downloaded files. If None, the value is inferred from the builder. + """ + self._dataset_name = dataset_name + self._data_dir = data_dir + self._base_path = base_path or os.path.abspath(".") + # To record what is being used: {url: {num_bytes: int, checksum: str}} + self._recorded_sizes_checksums: dict[str, dict[str, Optional[Union[int, str]]]] = {} + self.record_checksums = record_checksums + self.download_config = download_config or DownloadConfig() + self.downloaded_paths = {} + self.extracted_paths = {} + + @property + def manual_dir(self): + return self._data_dir + + @property + def downloaded_size(self): + """Returns the total size of downloaded files.""" + return sum(checksums_dict["num_bytes"] for checksums_dict in self._recorded_sizes_checksums.values()) + + def _record_sizes_checksums(self, url_or_urls: NestedDataStructure, downloaded_path_or_paths: NestedDataStructure): + """Record size/checksum of downloaded files.""" + delay = 5 + for url, path in hf_tqdm( + list(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten())), + delay=delay, + desc="Computing checksums", + ): + # call str to support PathLike objects + self._recorded_sizes_checksums[str(url)] = get_size_checksum_dict( + path, record_checksum=self.record_checksums + ) + + def download(self, url_or_urls): + """Download given URL(s). + + By default, only one process is used for download. Pass customized `download_config.num_proc` to change this behavior. + + Args: + url_or_urls (`str` or `list` or `dict`): + URL or `list` or `dict` of URLs to download. Each URL is a `str`. + + Returns: + `str` or `list` or `dict`: + The downloaded paths matching the given input `url_or_urls`. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + ``` + """ + download_config = self.download_config.copy() + download_config.extract_compressed_file = False + if download_config.download_desc is None: + download_config.download_desc = "Downloading data" + + download_func = partial(self._download_batched, download_config=download_config) + + start_time = datetime.now() + with stack_multiprocessing_download_progress_bars(): + downloaded_path_or_paths = map_nested( + download_func, + url_or_urls, + map_tuple=True, + num_proc=download_config.num_proc, + desc="Downloading data files", + batched=True, + batch_size=-1, + ) + duration = datetime.now() - start_time + logger.info(f"Downloading took {duration.total_seconds() // 60} min") + url_or_urls = NestedDataStructure(url_or_urls) + downloaded_path_or_paths = NestedDataStructure(downloaded_path_or_paths) + self.downloaded_paths.update(dict(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten()))) + + start_time = datetime.now() + self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths) + duration = datetime.now() - start_time + logger.info(f"Checksum Computation took {duration.total_seconds() // 60} min") + + return downloaded_path_or_paths.data + + def _download_batched( + self, + url_or_filenames: list[str], + download_config: DownloadConfig, + ) -> list[str]: + if len(url_or_filenames) >= 16: + download_config = download_config.copy() + download_config.disable_tqdm = True + download_func = partial(self._download_single, download_config=download_config) + + fs: fsspec.AbstractFileSystem + path = str(url_or_filenames[0]) + if is_relative_path(path): + # append the relative path to the base_path + path = url_or_path_join(self._base_path, path) + fs, path = url_to_fs(path, **download_config.storage_options) + size = 0 + try: + size = fs.info(path).get("size", 0) + except Exception: + pass + max_workers = ( + config.HF_DATASETS_MULTITHREADING_MAX_WORKERS if size < (20 << 20) else 1 + ) # enable multithreading if files are small + + return thread_map( + download_func, + url_or_filenames, + desc=download_config.download_desc or "Downloading", + unit="files", + position=multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses + if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1" + and multiprocessing.current_process()._identity + else None, + max_workers=max_workers, + tqdm_class=tqdm, + ) + else: + return [ + self._download_single(url_or_filename, download_config=download_config) + for url_or_filename in url_or_filenames + ] + + def _download_single(self, url_or_filename: str, download_config: DownloadConfig) -> str: + url_or_filename = str(url_or_filename) + if is_relative_path(url_or_filename): + # append the relative path to the base_path + url_or_filename = url_or_path_join(self._base_path, url_or_filename) + out = cached_path(url_or_filename, download_config=download_config) + out = tracked_str(out) + out.set_origin(url_or_filename) + return out + + def iter_archive(self, path_or_buf: Union[str, io.BufferedReader]): + """Iterate over files within an archive. + + Args: + path_or_buf (`str` or `io.BufferedReader`): + Archive path or archive binary file object. + + Yields: + `tuple[str, io.BufferedReader]`: + 2-tuple (path_within_archive, file_object). + File object is opened in binary mode. + + Example: + + ```py + >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> files = dl_manager.iter_archive(archive) + ``` + """ + + if hasattr(path_or_buf, "read"): + return ArchiveIterable.from_buf(path_or_buf) + else: + return ArchiveIterable.from_urlpath(path_or_buf) + + def iter_files(self, paths: Union[str, list[str]]): + """Iterate over file paths. + + Args: + paths (`str` or `list` of `str`): + Root paths. + + Yields: + `str`: File path. + + Example: + + ```py + >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/AI-Lab-Makerere/beans/resolve/main/data/train.zip') + >>> files = dl_manager.iter_files(files) + ``` + """ + return FilesIterable.from_urlpaths(paths) + + def extract(self, path_or_paths): + """Extract given path(s). + + Args: + path_or_paths (path or `list` or `dict`): + Path of file to extract. Each path is a `str`. + + Returns: + extracted_path(s): `str`, The extracted paths matching the given input + path_or_paths. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> extracted_files = dl_manager.extract(downloaded_files) + ``` + """ + download_config = self.download_config.copy() + download_config.extract_compressed_file = True + extract_func = partial(self._download_single, download_config=download_config) + extracted_paths = map_nested( + extract_func, + path_or_paths, + num_proc=download_config.num_proc, + desc="Extracting data files", + ) + path_or_paths = NestedDataStructure(path_or_paths) + extracted_paths = NestedDataStructure(extracted_paths) + self.extracted_paths.update(dict(zip(path_or_paths.flatten(), extracted_paths.flatten()))) + return extracted_paths.data + + def download_and_extract(self, url_or_urls): + """Download and extract given `url_or_urls`. + + Is roughly equivalent to: + + ``` + extracted_paths = dl_manager.extract(dl_manager.download(url_or_urls)) + ``` + + Args: + url_or_urls (`str` or `list` or `dict`): + URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`. + + Returns: + extracted_path(s): `str`, extracted paths of given URL(s). + """ + return self.extract(self.download(url_or_urls)) + + def get_recorded_sizes_checksums(self): + return self._recorded_sizes_checksums.copy() + + def delete_extracted_files(self): + paths_to_delete = set(self.extracted_paths.values()) - set(self.downloaded_paths.values()) + for key, path in list(self.extracted_paths.items()): + if path in paths_to_delete and os.path.isfile(path): + os.remove(path) + del self.extracted_paths[key] + + def manage_extracted_files(self): + if self.download_config.delete_extracted: + self.delete_extracted_files() diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/streaming_download_manager.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/streaming_download_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..bef18148455d782b3dbd65c56d23209b00d369b0 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/download/streaming_download_manager.py @@ -0,0 +1,219 @@ +import io +import os +from collections.abc import Iterable +from typing import Optional, Union + +from ..utils.file_utils import ( # noqa: F401 # backward compatibility + SINGLE_FILE_COMPRESSION_PROTOCOLS, + ArchiveIterable, + FilesIterable, + _get_extraction_protocol, + _get_path_extension, + _prepare_path_and_storage_options, + is_relative_path, + url_or_path_join, + xbasename, + xdirname, + xet_parse, + xexists, + xgetsize, + xglob, + xgzip_open, + xisdir, + xisfile, + xjoin, + xlistdir, + xnumpy_load, + xopen, + xpandas_read_csv, + xpandas_read_excel, + xPath, + xpyarrow_parquet_read_table, + xrelpath, + xsio_loadmat, + xsplit, + xsplitext, + xwalk, + xxml_dom_minidom_parse, +) +from ..utils.logging import get_logger +from ..utils.py_utils import map_nested +from .download_config import DownloadConfig + + +logger = get_logger(__name__) + + +class StreamingDownloadManager: + """ + Download manager that uses the "::" separator to navigate through (possibly remote) compressed archives. + Contrary to the regular `DownloadManager`, the `download` and `extract` methods don't actually download nor extract + data, but they rather return the path or url that could be opened using the `xopen` function which extends the + built-in `open` function to stream data from remote files. + """ + + is_streaming = True + + def __init__( + self, + dataset_name: Optional[str] = None, + data_dir: Optional[str] = None, + download_config: Optional[DownloadConfig] = None, + base_path: Optional[str] = None, + ): + self._dataset_name = dataset_name + self._data_dir = data_dir + self._base_path = base_path or os.path.abspath(".") + self.download_config = download_config or DownloadConfig() + self.downloaded_size = None + self.record_checksums = False + + @property + def manual_dir(self): + return self._data_dir + + def download(self, url_or_urls): + """Normalize URL(s) of files to stream data from. + This is the lazy version of `DownloadManager.download` for streaming. + + Args: + url_or_urls (`str` or `list` or `dict`): + URL(s) of files to stream data from. Each url is a `str`. + + Returns: + url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input url_or_urls. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + ``` + """ + url_or_urls = map_nested(self._download_single, url_or_urls, map_tuple=True) + return url_or_urls + + def _download_single(self, urlpath: str) -> str: + urlpath = str(urlpath) + if is_relative_path(urlpath): + # append the relative path to the base_path + urlpath = url_or_path_join(self._base_path, urlpath) + return urlpath + + def extract(self, url_or_urls): + """Add extraction protocol for given url(s) for streaming. + + This is the lazy version of `DownloadManager.extract` for streaming. + + Args: + url_or_urls (`str` or `list` or `dict`): + URL(s) of files to stream data from. Each url is a `str`. + + Returns: + url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> extracted_files = dl_manager.extract(downloaded_files) + ``` + """ + urlpaths = map_nested(self._extract, url_or_urls, map_tuple=True) + return urlpaths + + def _extract(self, urlpath: str) -> str: + urlpath = str(urlpath) + # get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz + protocol = _get_extraction_protocol(urlpath, download_config=self.download_config) + path = urlpath.split("::")[0] + extension = _get_path_extension(path) + if extension in ["tgz", "tar"] or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz")): + raise NotImplementedError( + f"Extraction protocol for TAR archives like '{urlpath}' is not implemented in streaming mode. " + f"Please use `dl_manager.iter_archive` instead.\n\n" + f"Example usage:\n\n" + f"\turl = dl_manager.download(url)\n" + f"\ttar_archive_iterator = dl_manager.iter_archive(url)\n\n" + f"\tfor filename, file in tar_archive_iterator:\n" + f"\t\t..." + ) + if protocol is None: + # no extraction + return urlpath + elif protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS: + # there is one single file which is the uncompressed file + inner_file = os.path.basename(urlpath.split("::")[0]) + inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file + return f"{protocol}://{inner_file}::{urlpath}" + else: + return f"{protocol}://::{urlpath}" + + def download_and_extract(self, url_or_urls): + """Prepare given `url_or_urls` for streaming (add extraction protocol). + + This is the lazy version of `DownloadManager.download_and_extract` for streaming. + + Is equivalent to: + + ``` + urls = dl_manager.extract(dl_manager.download(url_or_urls)) + ``` + + Args: + url_or_urls (`str` or `list` or `dict`): + URL(s) to stream from data from. Each url is a `str`. + + Returns: + url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. + """ + return self.extract(self.download(url_or_urls)) + + def iter_archive(self, urlpath_or_buf: Union[str, io.BufferedReader]) -> Iterable[tuple]: + """Iterate over files within an archive. + + Args: + urlpath_or_buf (`str` or `io.BufferedReader`): + Archive path or archive binary file object. + + Yields: + `tuple[str, io.BufferedReader]`: + 2-tuple (path_within_archive, file_object). + File object is opened in binary mode. + + Example: + + ```py + >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> files = dl_manager.iter_archive(archive) + ``` + """ + + if hasattr(urlpath_or_buf, "read"): + return ArchiveIterable.from_buf(urlpath_or_buf) + else: + return ArchiveIterable.from_urlpath(urlpath_or_buf, download_config=self.download_config) + + def iter_files(self, urlpaths: Union[str, list[str]]) -> Iterable[str]: + """Iterate over files. + + Args: + urlpaths (`str` or `list` of `str`): + Root paths. + + Yields: + str: File URL path. + + Example: + + ```py + >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/AI-Lab-Makerere/beans/resolve/main/data/train.zip') + >>> files = dl_manager.iter_files(files) + ``` + """ + return FilesIterable.from_urlpaths(urlpaths, download_config=self.download_config) + + def manage_extracted_files(self): + pass + + def get_recorded_sizes_checksums(self): + pass diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/exceptions.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..7c1724fb572dc98a2b65ad07f0a1a08c5d2a32e5 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/exceptions.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2023 The HuggingFace Authors. +from typing import Any, Optional, Union + +from huggingface_hub import HfFileSystem + +from . import config +from .table import CastError +from .utils.track import TrackedIterableFromGenerator, tracked_list, tracked_str + + +class DatasetsError(Exception): + """Base class for exceptions in this library.""" + + +class DefunctDatasetError(DatasetsError): + """The dataset has been defunct.""" + + +class FileNotFoundDatasetsError(DatasetsError, FileNotFoundError): + """FileNotFoundError raised by this library.""" + + +class DataFilesNotFoundError(FileNotFoundDatasetsError): + """No (supported) data files found.""" + + +class DatasetNotFoundError(FileNotFoundDatasetsError): + """Dataset not found. + + Raised when trying to access: + - a missing dataset, or + - a private/gated dataset and the user is not authenticated. + """ + + +class DatasetBuildError(DatasetsError): + pass + + +class ManualDownloadError(DatasetBuildError): + pass + + +class FileFormatError(DatasetBuildError): + pass + + +class DatasetGenerationError(DatasetBuildError): + pass + + +class DatasetGenerationCastError(DatasetGenerationError): + @classmethod + def from_cast_error( + cls, + cast_error: CastError, + builder_name: str, + gen_kwargs: dict[str, Any], + token: Optional[Union[bool, str]], + ) -> "DatasetGenerationCastError": + explanation_message = ( + f"\n\nAll the data files must have the same columns, but at some point {cast_error.details()}" + ) + formatted_tracked_gen_kwargs: list[str] = [] + for gen_kwarg in gen_kwargs.values(): + if not isinstance(gen_kwarg, (tracked_str, tracked_list, TrackedIterableFromGenerator)): + continue + while ( + isinstance(gen_kwarg, (tracked_list, TrackedIterableFromGenerator)) and gen_kwarg.last_item is not None + ): + gen_kwarg = gen_kwarg.last_item + if isinstance(gen_kwarg, tracked_str): + gen_kwarg = gen_kwarg.get_origin() + if isinstance(gen_kwarg, str) and gen_kwarg.startswith("hf://"): + resolved_path = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token).resolve_path(gen_kwarg) + gen_kwarg = "hf://" + resolved_path.unresolve() + if "@" + resolved_path.revision in gen_kwarg: + gen_kwarg = ( + gen_kwarg.replace("@" + resolved_path.revision, "", 1) + + f" (at revision {resolved_path.revision})" + ) + formatted_tracked_gen_kwargs.append(str(gen_kwarg)) + if formatted_tracked_gen_kwargs: + explanation_message += f"\n\nThis happened while the {builder_name} dataset builder was generating data using\n\n{', '.join(formatted_tracked_gen_kwargs)}" + help_message = "\n\nPlease either edit the data files to have matching columns, or separate them into different configurations (see docs at https://hf.co/docs/hub/datasets-manual-configuration#multiple-configurations)" + return cls("An error occurred while generating the dataset" + explanation_message + help_message) + + +class ChecksumVerificationError(DatasetsError): + """Error raised during checksums verifications of downloaded files.""" + + +class UnexpectedDownloadedFileError(ChecksumVerificationError): + """Some downloaded files were not expected.""" + + +class ExpectedMoreDownloadedFilesError(ChecksumVerificationError): + """Some files were supposed to be downloaded but were not.""" + + +class NonMatchingChecksumError(ChecksumVerificationError): + """The downloaded file checksum don't match the expected checksum.""" + + +class SplitsVerificationError(DatasetsError): + """Error raised during splits verifications.""" + + +class UnexpectedSplitsError(SplitsVerificationError): + """The expected splits of the downloaded file is missing.""" + + +class ExpectedMoreSplitsError(SplitsVerificationError): + """Some recorded splits are missing.""" + + +class NonMatchingSplitsSizesError(SplitsVerificationError): + """The splits sizes don't match the expected splits sizes.""" diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d2c780f788ca55b4b1d1d22d5b9aa0ec582ea51c --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/__init__.py @@ -0,0 +1,27 @@ +__all__ = [ + "Audio", + "Array2D", + "Array3D", + "Array4D", + "Array5D", + "ClassLabel", + "Features", + "Json", + "LargeList", + "List", + "Sequence", + "Value", + "Image", + "Translation", + "TranslationVariableLanguages", + "Video", + "Pdf", + "Nifti", +] +from .audio import Audio +from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, Json, LargeList, List, Sequence, Value +from .image import Image +from .nifti import Nifti +from .pdf import Pdf +from .translation import Translation, TranslationVariableLanguages +from .video import Video diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/_torchcodec.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/_torchcodec.py new file mode 100644 index 0000000000000000000000000000000000000000..d71b4156a5124be4a6433ce76a3794d55393499c --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/_torchcodec.py @@ -0,0 +1,15 @@ +import numpy as np +from torchcodec.decoders import AudioDecoder as _AudioDecoder + + +class AudioDecoder(_AudioDecoder): + def __getitem__(self, key: str): + if key == "array": + y = self.get_all_samples().data.cpu().numpy() + return np.mean(y, axis=tuple(range(y.ndim - 1))) if y.ndim > 1 else y + elif key == "sampling_rate": + return self.get_samples_played_in_range(0, 0).sample_rate + elif hasattr(super(), "__getitem__"): + return super().__getitem__(key) + else: + raise TypeError("'torchcodec.decoders.AudioDecoder' object is not subscriptable") diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/audio.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/audio.py new file mode 100644 index 0000000000000000000000000000000000000000..ac2ddeef67cd1299f137c8f6d3c48d8c67229093 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/audio.py @@ -0,0 +1,365 @@ +import os +from dataclasses import dataclass, field +from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union + +import numpy as np +import pyarrow as pa + +from .. import config +from ..download.download_config import DownloadConfig +from ..table import array_cast +from ..utils.file_utils import is_local_path, is_remote_url, xopen +from ..utils.py_utils import no_op_if_value_is_null, string_to_dict + + +if TYPE_CHECKING: + from torchcodec.decoders import AudioDecoder + + from .features import FeatureType + + +@dataclass +class Audio: + """Audio [`Feature`] to extract audio data from an audio file. + + Input: The Audio feature accepts as input: + - A `str`: Absolute path to the audio file (i.e. random access is allowed). + - A `pathlib.Path`: path to the audio file (i.e. random access is allowed). + - A `dict` with the keys: + + - `path`: String with relative path of the audio file to the archive file. + - `bytes`: Bytes content of the audio file. + + This is useful for parquet or webdataset files which embed audio files. + + - A `dict` with the keys: + + - `array`: Array containing the audio sample + - `sampling_rate`: Integer corresponding to the sampling rate of the audio sample. + + - A `torchcodec.decoders.AudioDecoder`: torchcodec audio decoder object. + + Output: The Audio features output data as `torchcodec.decoders.AudioDecoder` objects, with additional keys: + + - `array`: Array containing the audio sample + - `sampling_rate`: Integer corresponding to the sampling rate of the audio sample. + + Args: + sampling_rate (`int`, *optional*): + Target sampling rate. If `None`, the native sampling rate is used. + num_channels (`int`, *optional*): + The desired number of channels of the samples. By default, the number of channels of the source is used. + Audio decoding will return samples with shape (num_channels, num_samples) + Currently `None` (number of channels of the source, default), `1` (mono) or `2` (stereo) channels are supported. + The `num_channels` argument is passed to `torchcodec.decoders.AudioDecoder`. + + + decode (`bool`, defaults to `True`): + Whether to decode the audio data. If `False`, + returns the underlying dictionary in the format `{"path": audio_path, "bytes": audio_bytes}`. + stream_index (`int`, *optional*): + The streaming index to use from the file. If `None` defaults to the "best" index. + + Example: + + ```py + >>> from datasets import load_dataset, Audio + >>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train") + >>> ds = ds.cast_column("audio", Audio(sampling_rate=44100, num_channels=2)) + >>> ds[0]["audio"] + + >>> audio = ds[0]["audio"] + >>> audio.get_samples_played_in_range(0, 10) + AudioSamples: + data (shape): torch.Size([2, 110592]) + pts_seconds: 0.0 + duration_seconds: 2.507755102040816 + sample_rate: 44100 + ``` + """ + + sampling_rate: Optional[int] = None + decode: bool = True + num_channels: Optional[int] = None + stream_index: Optional[int] = None + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + dtype: ClassVar[str] = "dict" + pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()}) + _type: str = field(default="Audio", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, value: Union[str, bytes, bytearray, dict, "AudioDecoder"]) -> dict: + """Encode example into a format for Arrow. + + Args: + value (`str`, `bytes`,`bytearray`,`dict`, `AudioDecoder`): + Data passed as input to Audio feature. + + Returns: + `dict` + """ + try: + import torch + from torchcodec.encoders import AudioEncoder # needed to write audio files + except ImportError as err: + raise ImportError("To support encoding audio data, please install 'torchcodec'.") from err + + if value is None: + raise ValueError("value must be provided") + + if config.TORCHCODEC_AVAILABLE: + from torchcodec.decoders import AudioDecoder + + else: + AudioDecoder = None + + if isinstance(value, str): + return {"bytes": None, "path": value} + elif isinstance(value, Path): + return {"bytes": None, "path": str(value.absolute())} + elif isinstance(value, (bytes, bytearray)): + return {"bytes": value, "path": None} + elif AudioDecoder is not None and isinstance(value, AudioDecoder): + return encode_torchcodec_audio(value) + elif "array" in value: + # convert the audio array to wav bytes + buffer = BytesIO() + AudioEncoder( + torch.from_numpy(value["array"].astype(np.float32)), sample_rate=value["sampling_rate"] + ).to_file_like(buffer, format="wav", num_channels=self.num_channels) + return {"bytes": buffer.getvalue(), "path": None} + elif value.get("path") is not None and os.path.isfile(value["path"]): + # we set "bytes": None to not duplicate the data if they're already available locally + if value["path"].endswith("pcm"): + # "PCM" only has raw audio bytes + if value.get("sampling_rate") is None: + # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate + raise KeyError("To use PCM files, please specify a 'sampling_rate' in Audio object") + if value.get("bytes"): + # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!) + bytes_value = np.frombuffer(value["bytes"], dtype=np.int16).astype(np.float32) / 32767 + else: + bytes_value = np.memmap(value["path"], dtype="h", mode="r").astype(np.float32) / 32767 + + buffer = BytesIO() + AudioEncoder(torch.from_numpy(bytes_value), sample_rate=value["sampling_rate"]).to_file_like( + buffer, format="wav", num_channels=self.num_channels + ) + return {"bytes": buffer.getvalue(), "path": None} + else: + return {"bytes": None, "path": value.get("path")} + elif value.get("bytes") is not None or value.get("path") is not None: + # store the audio bytes, and path is used to infer the audio format using the file extension + return {"bytes": value.get("bytes"), "path": value.get("path")} + else: + raise ValueError( + f"An audio sample should have one of 'path' or 'bytes' but they are missing or None in {value}." + ) + + def decode_example( + self, value: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None + ) -> "AudioDecoder": + """Decode example audio file into audio data. + + Args: + value (`dict`): + A dictionary with keys: + + - `path`: String with relative audio file path. + - `bytes`: Bytes of the audio file. + token_per_repo_id (`dict`, *optional*): + To access and decode + audio files from private repositories on the Hub, you can pass + a dictionary repo_id (`str`) -> token (`bool` or `str`) + + Returns: + `torchcodec.decoders.AudioDecoder` + """ + if config.TORCHCODEC_AVAILABLE: + from ._torchcodec import AudioDecoder + else: + raise ImportError("To support decoding audio data, please install 'torchcodec'.") + + if not self.decode: + raise RuntimeError("Decoding is disabled for this feature. Please use Audio(decode=True) instead.") + + path, bytes = (value["path"], value["bytes"]) if value["bytes"] is not None else (value["path"], None) + if path is None and bytes is None: + raise ValueError(f"An audio sample should have one of 'path' or 'bytes' but both are None in {value}.") + + if bytes is None and is_local_path(path): + audio = AudioDecoder( + path, stream_index=self.stream_index, sample_rate=self.sampling_rate, num_channels=self.num_channels + ) + + elif bytes is None: + token_per_repo_id = token_per_repo_id or {} + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL + ) + source_url_fields = string_to_dict(source_url, pattern) + token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + + download_config = DownloadConfig(token=token) + f = xopen(path, "rb", download_config=download_config) + audio = AudioDecoder( + f, stream_index=self.stream_index, sample_rate=self.sampling_rate, num_channels=self.num_channels + ) + + else: + audio = AudioDecoder( + bytes, stream_index=self.stream_index, sample_rate=self.sampling_rate, num_channels=self.num_channels + ) + audio._hf_encoded = {"path": path, "bytes": bytes} + audio.metadata.path = path + return audio + + def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]: + """If in the decodable state, raise an error, otherwise flatten the feature into a dictionary.""" + from .features import Value + + if self.decode: + raise ValueError("Cannot flatten a decoded Audio feature.") + return { + "bytes": Value("binary"), + "path": Value("string"), + } + + def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray]) -> pa.StructArray: + """Cast an Arrow array to the Audio arrow storage type. + The Arrow types that can be converted to the Audio pyarrow storage type are: + + - `pa.string()` - it must contain the "path" data + - `pa.binary()` - it must contain the audio bytes + - `pa.struct({"bytes": pa.binary()})` + - `pa.struct({"path": pa.string()})` + - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter + + Args: + storage (`Union[pa.StringArray, pa.StructArray]`): + PyArrow array to cast. + + Returns: + `pa.StructArray`: Array in the Audio arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})` + """ + if pa.types.is_string(storage.type): + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_large_binary(storage.type): + storage = array_cast( + storage, pa.binary() + ) # this can fail in case of big audios, paths should be used instead + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_binary(storage.type): + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_struct(storage.type) and storage.type.get_all_field_indices("array"): + storage = pa.array( + [Audio().encode_example(x) if x is not None else None for x in storage.to_numpy(zero_copy_only=False)] + ) + elif pa.types.is_struct(storage.type): + if storage.type.get_field_index("bytes") >= 0: + bytes_array = storage.field("bytes") + else: + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + if storage.type.get_field_index("path") >= 0: + path_array = storage.field("path") + else: + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + return array_cast(storage, self.pa_type) + + def embed_storage( + self, storage: pa.StructArray, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True + ) -> pa.StructArray: + """Embed audio files into the Arrow array. + + Args: + storage (`pa.StructArray`): + PyArrow array to embed. + token_per_repo_id (`dict`, optional): + Dictionary repo_id -> token to fetch the files bytes. + local_files (`bool`, defaults to `True`) + Whether to embed local files data in the array + + + remote_files (`bool`, defaults to `True`) + Whether to embed remote files data in the array. + E.g. files with paths that start with hf:// or https:// + + + + Returns: + `pa.StructArray`: Array in the Audio arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if token_per_repo_id is None: + token_per_repo_id = {} + + @no_op_if_value_is_null + def path_to_bytes(path): + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL + ) + source_url_fields = string_to_dict(source_url, pattern) + token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + download_config = DownloadConfig(token=token) + with xopen(path, "rb", download_config=download_config) as f: + return f.read() + + bytes_array = pa.array( + [ + ( + path_to_bytes(x["path"]) + if x["bytes"] is None + and ((local_files and is_local_path(x["path"])) or (remote_files and is_remote_url(x["path"]))) + else x["bytes"] + ) + if x is not None + else None + for x in storage.to_pylist() + ], + type=pa.binary(), + ) + path_array = pa.array( + [ + ( + os.path.basename(path) + if (local_files and is_local_path(path)) or (remote_files and is_remote_url(path)) + else path + ) + if path is not None + else None + for path in storage.field("path").to_pylist() + ], + type=pa.string(), + ) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + return array_cast(storage, self.pa_type) + + +def encode_torchcodec_audio(audio: "AudioDecoder") -> dict: + if hasattr(audio, "_hf_encoded"): + return audio._hf_encoded + else: + try: + from torchcodec.encoders import AudioEncoder # needed to write audio files + except ImportError as err: + raise ImportError("To support encoding audio data, please install 'torchcodec'.") from err + + samples = audio.get_all_samples() + buffer = BytesIO() + num_channels = samples.data.shape[0] + AudioEncoder(samples.data.cpu(), sample_rate=samples.sample_rate).to_file_like( + buffer, format="wav", num_channels=num_channels + ) + return {"bytes": buffer.getvalue(), "path": None} diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/features.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/features.py new file mode 100644 index 0000000000000000000000000000000000000000..36ed4b0b17c5d007e669a1aac2ebde9de2e63ea3 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/features.py @@ -0,0 +1,2439 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""This class handle features definition in datasets and some utilities to display table type.""" + +import copy +import json +import re +import sys +from collections.abc import Iterable, Mapping +from collections.abc import Sequence as SequenceABC +from collections.abc import Sequence as Sequence_ +from dataclasses import InitVar, dataclass, field, fields +from functools import reduce, wraps +from operator import mul +from typing import Any, Callable, ClassVar, Literal, Optional, Union + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.types +from pandas.api.extensions import ExtensionArray as PandasExtensionArray +from pandas.api.extensions import ExtensionDtype as PandasExtensionDtype + +from .. import config +from ..naming import camelcase_to_snakecase, snakecase_to_camelcase +from ..table import array_cast +from ..utils import experimental, logging +from ..utils.json import ujson_dumps, ujson_loads +from ..utils.py_utils import asdict, first_non_null_value, zip_dict +from .audio import Audio +from .image import Image, encode_pil_image +from .nifti import Nifti, encode_nibabel_image +from .pdf import Pdf, encode_pdfplumber_pdf +from .translation import Translation, TranslationVariableLanguages +from .video import Video + + +logger = logging.get_logger(__name__) + + +def _arrow_to_datasets_dtype(arrow_type: pa.DataType) -> str: + """ + _arrow_to_datasets_dtype takes a pyarrow.DataType and converts it to a datasets string dtype. + In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))` + """ + if pyarrow.types.is_null(arrow_type): + return "null" + elif pyarrow.types.is_boolean(arrow_type): + return "bool" + elif pyarrow.types.is_int8(arrow_type): + return "int8" + elif pyarrow.types.is_int16(arrow_type): + return "int16" + elif pyarrow.types.is_int32(arrow_type): + return "int32" + elif pyarrow.types.is_int64(arrow_type): + return "int64" + elif pyarrow.types.is_uint8(arrow_type): + return "uint8" + elif pyarrow.types.is_uint16(arrow_type): + return "uint16" + elif pyarrow.types.is_uint32(arrow_type): + return "uint32" + elif pyarrow.types.is_uint64(arrow_type): + return "uint64" + elif pyarrow.types.is_float16(arrow_type): + return "float16" # pyarrow dtype is "halffloat" + elif pyarrow.types.is_float32(arrow_type): + return "float32" # pyarrow dtype is "float" + elif pyarrow.types.is_float64(arrow_type): + return "float64" # pyarrow dtype is "double" + elif pyarrow.types.is_time32(arrow_type): + return f"time32[{pa.type_for_alias(str(arrow_type)).unit}]" + elif pyarrow.types.is_time64(arrow_type): + return f"time64[{pa.type_for_alias(str(arrow_type)).unit}]" + elif pyarrow.types.is_timestamp(arrow_type): + if arrow_type.tz is None: + return f"timestamp[{arrow_type.unit}]" + elif arrow_type.tz: + return f"timestamp[{arrow_type.unit}, tz={arrow_type.tz}]" + else: + raise ValueError(f"Unexpected timestamp object {arrow_type}.") + elif pyarrow.types.is_date32(arrow_type): + return "date32" # pyarrow dtype is "date32[day]" + elif pyarrow.types.is_date64(arrow_type): + return "date64" # pyarrow dtype is "date64[ms]" + elif pyarrow.types.is_duration(arrow_type): + return f"duration[{arrow_type.unit}]" + elif pyarrow.types.is_decimal128(arrow_type): + return f"decimal128({arrow_type.precision}, {arrow_type.scale})" + elif pyarrow.types.is_decimal256(arrow_type): + return f"decimal256({arrow_type.precision}, {arrow_type.scale})" + elif pyarrow.types.is_binary(arrow_type): + return "binary" + elif pyarrow.types.is_large_binary(arrow_type): + return "large_binary" + elif pyarrow.types.is_binary_view(arrow_type): + return "binary_view" + elif pyarrow.types.is_string(arrow_type): + return "string" + elif pyarrow.types.is_large_string(arrow_type): + return "large_string" + elif pyarrow.types.is_string_view(arrow_type): + return "string_view" + elif pyarrow.types.is_dictionary(arrow_type): + return _arrow_to_datasets_dtype(arrow_type.value_type) + else: + raise ValueError(f"Arrow type {arrow_type} does not have a datasets dtype equivalent.") + + +def string_to_arrow(datasets_dtype: str) -> pa.DataType: + """ + string_to_arrow takes a datasets string dtype and converts it to a pyarrow.DataType. + + In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))` + + This is necessary because the datasets.Value() primitive type is constructed using a string dtype + + Value(dtype=str) + + But Features.type (via `get_nested_type()` expects to resolve Features into a pyarrow Schema, + which means that each Value() must be able to resolve into a corresponding pyarrow.DataType, which is the + purpose of this function. + """ + if datasets_dtype == "json": + raise ValueError("'json' is not a valid dtype, use the Json() feature instead") + + def _dtype_error_msg(dtype, pa_dtype, examples=None, urls=None): + msg = f"{dtype} is not a validly formatted string representation of the pyarrow {pa_dtype} type." + if examples: + examples = ", ".join(examples[:-1]) + " or " + examples[-1] if len(examples) > 1 else examples[0] + msg += f"\nValid examples include: {examples}." + if urls: + urls = ", ".join(urls[:-1]) + " and " + urls[-1] if len(urls) > 1 else urls[0] + msg += f"\nFor more insformation, see: {urls}." + return msg + + if datasets_dtype in pa.__dict__: + return pa.__dict__[datasets_dtype]() + + if (datasets_dtype + "_") in pa.__dict__: + return pa.__dict__[datasets_dtype + "_"]() + + timestamp_matches = re.search(r"^timestamp\[(.*)\]$", datasets_dtype) + if timestamp_matches: + timestamp_internals = timestamp_matches.group(1) + internals_matches = re.search(r"^(s|ms|us|ns),\s*tz=([a-zA-Z0-9/_+\-:]*)$", timestamp_internals) + if timestamp_internals in ["s", "ms", "us", "ns"]: + return pa.timestamp(timestamp_internals) + elif internals_matches: + return pa.timestamp(internals_matches.group(1), internals_matches.group(2)) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "timestamp", + examples=["timestamp[us]", "timestamp[us, tz=America/New_York"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.timestamp.html"], + ) + ) + + duration_matches = re.search(r"^duration\[(.*)\]$", datasets_dtype) + if duration_matches: + duration_internals = duration_matches.group(1) + if duration_internals in ["s", "ms", "us", "ns"]: + return pa.duration(duration_internals) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "duration", + examples=["duration[s]", "duration[us]"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.duration.html"], + ) + ) + + time_matches = re.search(r"^time(.*)\[(.*)\]$", datasets_dtype) + if time_matches: + time_internals_bits = time_matches.group(1) + if time_internals_bits == "32": + time_internals_unit = time_matches.group(2) + if time_internals_unit in ["s", "ms"]: + return pa.time32(time_internals_unit) + else: + raise ValueError( + f"{time_internals_unit} is not a valid unit for the pyarrow time32 type. Supported units: s (second) and ms (millisecond)." + ) + elif time_internals_bits == "64": + time_internals_unit = time_matches.group(2) + if time_internals_unit in ["us", "ns"]: + return pa.time64(time_internals_unit) + else: + raise ValueError( + f"{time_internals_unit} is not a valid unit for the pyarrow time64 type. Supported units: us (microsecond) and ns (nanosecond)." + ) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "time", + examples=["time32[s]", "time64[us]"], + urls=[ + "https://arrow.apache.org/docs/python/generated/pyarrow.time32.html", + "https://arrow.apache.org/docs/python/generated/pyarrow.time64.html", + ], + ) + ) + + decimal_matches = re.search(r"^decimal(.*)\((.*)\)$", datasets_dtype) + if decimal_matches: + decimal_internals_bits = decimal_matches.group(1) + if decimal_internals_bits == "128": + decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2)) + if decimal_internals_precision_and_scale: + precision = decimal_internals_precision_and_scale.group(1) + scale = decimal_internals_precision_and_scale.group(2) + return pa.decimal128(int(precision), int(scale)) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "decimal128", + examples=["decimal128(10, 2)", "decimal128(4, -2)"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html"], + ) + ) + elif decimal_internals_bits == "256": + decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2)) + if decimal_internals_precision_and_scale: + precision = decimal_internals_precision_and_scale.group(1) + scale = decimal_internals_precision_and_scale.group(2) + return pa.decimal256(int(precision), int(scale)) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "decimal256", + examples=["decimal256(30, 2)", "decimal256(38, -4)"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html"], + ) + ) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "decimal", + examples=["decimal128(12, 3)", "decimal256(40, 6)"], + urls=[ + "https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html", + "https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html", + ], + ) + ) + + raise ValueError( + f"Neither {datasets_dtype} nor {datasets_dtype + '_'} seems to be a pyarrow data type. " + f"Please make sure to use a correct data type, see: " + f"https://arrow.apache.org/docs/python/api/datatypes.html#factory-functions" + ) + + +def _cast_to_python_objects(obj: Any, only_1d_for_numpy: bool, optimize_list_casting: bool) -> tuple[Any, bool]: + """ + Cast pytorch/tensorflow/pandas objects to python numpy array/lists. + It works recursively. + + If `optimize_list_casting` is True, to avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted. + If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same. + This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example. + + Args: + obj: the object (nested struct) to cast. + only_1d_for_numpy (bool): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to + nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays. + Indeed Arrow only support converting 1-dimensional array values. + optimize_list_casting (bool): whether to optimize list casting by checking the first non-null element to see if it needs to be casted + and if it doesn't, not checking the rest of the list elements. + + Returns: + casted_obj: the casted object + has_changed (bool): True if the object has been changed, False if it is identical + """ + + if config.TF_AVAILABLE and "tensorflow" in sys.modules: + import tensorflow as tf + + if config.TORCH_AVAILABLE and "torch" in sys.modules: + import torch + + if config.JAX_AVAILABLE and "jax" in sys.modules: + import jax.numpy as jnp + + if config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + if config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules: + import pdfplumber + + if config.NIBABEL_AVAILABLE and "nibabel" in sys.modules: + import nibabel as nib + + if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules: + from torchcodec.decoders import AudioDecoder, VideoDecoder + + if isinstance(obj, np.ndarray): + if obj.ndim == 0: + return obj[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return obj, False + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in obj + ], + True, + ) + elif config.TORCH_AVAILABLE and "torch" in sys.modules and isinstance(obj, torch.Tensor): + if obj.dtype == torch.bfloat16: + return _cast_to_python_objects( + obj.detach().to(torch.float).cpu().numpy(), + only_1d_for_numpy=only_1d_for_numpy, + optimize_list_casting=optimize_list_casting, + )[0], True + if obj.ndim == 0: + return obj.detach().cpu().numpy()[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return obj.detach().cpu().numpy(), True + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in obj.detach().cpu().numpy() + ], + True, + ) + elif config.TF_AVAILABLE and "tensorflow" in sys.modules and isinstance(obj, tf.Tensor): + if obj.ndim == 0: + return obj.numpy()[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return obj.numpy(), True + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in obj.numpy() + ], + True, + ) + elif config.JAX_AVAILABLE and "jax" in sys.modules and isinstance(obj, jnp.ndarray): + if obj.ndim == 0: + return np.asarray(obj)[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return np.asarray(obj), True + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in np.asarray(obj) + ], + True, + ) + elif config.PIL_AVAILABLE and "PIL" in sys.modules and isinstance(obj, PIL.Image.Image): + return encode_pil_image(obj), True + elif config.PDFPLUMBER_AVAILABLE and "pdfplumber" in sys.modules and isinstance(obj, pdfplumber.pdf.PDF): + return encode_pdfplumber_pdf(obj), True + elif config.NIBABEL_AVAILABLE and "nibabel" in sys.modules and isinstance(obj, nib.analyze.AnalyzeImage): + return encode_nibabel_image(obj, force_bytes=True), True + elif isinstance(obj, pd.Series): + return ( + _cast_to_python_objects( + obj.tolist(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0], + True, + ) + elif isinstance(obj, pd.DataFrame): + return ( + { + key: _cast_to_python_objects( + value, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for key, value in obj.to_dict("series").items() + }, + True, + ) + elif isinstance(obj, pd.Timestamp): + return obj.to_pydatetime(), True + elif isinstance(obj, pd.Timedelta): + return obj.to_pytimedelta(), True + elif isinstance(obj, Mapping): + has_changed = not isinstance(obj, dict) + output = {} + for k, v in obj.items(): + casted_v, has_changed_v = _cast_to_python_objects( + v, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + ) + has_changed |= has_changed_v + output[k] = casted_v + return output if has_changed else obj, has_changed + elif hasattr(obj, "__array__"): + if np.isscalar(obj): + return obj, False + else: + return ( + _cast_to_python_objects( + obj.__array__(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0], + True, + ) + elif isinstance(obj, (list, tuple)): + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt): + break + casted_first_elmt, has_changed_first_elmt = _cast_to_python_objects( + first_elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + ) + if has_changed_first_elmt or not optimize_list_casting: + return ( + [ + _cast_to_python_objects( + elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for elmt in obj + ], + True, + ) + else: + if isinstance(obj, (list, tuple)): + return obj, False + else: + return list(obj), True + else: + return obj, False + elif config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules and isinstance(obj, VideoDecoder): + v = Video() + return v.encode_example(obj), True + elif config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules and isinstance(obj, AudioDecoder): + a = Audio() + return a.encode_example(obj), True + else: + return obj, False + + +def cast_to_python_objects(obj: Any, only_1d_for_numpy=False, optimize_list_casting=True) -> Any: + """ + Cast numpy/pytorch/tensorflow/pandas objects to python lists. + It works recursively. + + If `optimize_list_casting` is True, To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted. + If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same. + This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example. + + Args: + obj: the object (nested struct) to cast + only_1d_for_numpy (bool, default ``False``): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to + nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays. + Indeed Arrow only support converting 1-dimensional array values. + optimize_list_casting (bool, default ``True``): whether to optimize list casting by checking the first non-null element to see if it needs to be casted + and if it doesn't, not checking the rest of the list elements. + + Returns: + casted_obj: the casted object + """ + return _cast_to_python_objects( + obj, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + + +@dataclass(repr=False) +class Value: + """ + Scalar feature value of a particular data type. + + The possible dtypes of `Value` are as follows: + - `null` + - `bool` + - `int8` + - `int16` + - `int32` + - `int64` + - `uint8` + - `uint16` + - `uint32` + - `uint64` + - `float16` + - `float32` (alias float) + - `float64` (alias double) + - `time32[(s|ms)]` + - `time64[(us|ns)]` + - `timestamp[(s|ms|us|ns)]` + - `timestamp[(s|ms|us|ns), tz=(tzstring)]` + - `date32` + - `date64` + - `duration[(s|ms|us|ns)]` + - `decimal128(precision, scale)` + - `decimal256(precision, scale)` + - `binary` + - `large_binary` + - `binary_view` + - `string` + - `large_string` + - `string_view` + + Args: + dtype (`str`): + Name of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'stars': Value('int32')}) + >>> features + {'stars': Value('int32')} + ``` + """ + + dtype: str + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + pa_type: ClassVar[Any] = None + _type: str = field(default="Value", init=False, repr=False) + + def __post_init__(self): + if self.dtype == "double": # fix inferred type + self.dtype = "float64" + if self.dtype == "float": # fix inferred type + self.dtype = "float32" + self.pa_type = string_to_arrow(self.dtype) + + def __call__(self): + return self.pa_type + + def encode_example(self, value): + if pa.types.is_boolean(self.pa_type): + return bool(value) + elif pa.types.is_integer(self.pa_type): + return int(value) + elif pa.types.is_floating(self.pa_type): + return float(value) + elif pa.types.is_string(self.pa_type): + return str(value) + elif pa.types.is_large_string(self.pa_type): + return str(value) + elif pa.types.is_string_view(self.pa_type): + return str(value) + else: + return value + + def __repr__(self): + return f"{type(self).__name__}('{self.dtype}')" + + +class _ArrayXD: + def __post_init__(self): + self.shape = tuple(self.shape) + + def __call__(self): + pa_type = globals()[self.__class__.__name__ + "ExtensionType"](self.shape, self.dtype) + return pa_type + + def encode_example(self, value): + return value + + +@dataclass +class Array2D(_ArrayXD): + """Create a two-dimensional array. + + Args: + shape (`tuple`): + Size of each dimension. + dtype (`str`): + Name of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array2D(shape=(1, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + _type: str = field(default="Array2D", init=False, repr=False) + + +@dataclass +class Array3D(_ArrayXD): + """Create a three-dimensional array. + + Args: + shape (`tuple`): + Size of each dimension. + dtype (`str`): + Name of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array3D(shape=(1, 2, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + _type: str = field(default="Array3D", init=False, repr=False) + + +@dataclass +class Array4D(_ArrayXD): + """Create a four-dimensional array. + + Args: + shape (`tuple`): + Size of each dimension. + dtype (`str`): + Name of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array4D(shape=(1, 2, 2, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + _type: str = field(default="Array4D", init=False, repr=False) + + +@dataclass +class Array5D(_ArrayXD): + """Create a five-dimensional array. + + Args: + shape (`tuple`): + Size of each dimension. + dtype (`str`): + Name of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array5D(shape=(1, 2, 2, 3, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + _type: str = field(default="Array5D", init=False, repr=False) + + +class _ArrayXDExtensionType(pa.ExtensionType): + ndims: Optional[int] = None + + def __init__(self, shape: tuple, dtype: str): + if self.ndims is None or self.ndims <= 1: + raise ValueError("You must instantiate an array type with a value for dim that is > 1") + if len(shape) != self.ndims: + raise ValueError(f"shape={shape} and ndims={self.ndims} don't match") + for dim in range(1, self.ndims): + if shape[dim] is None: + raise ValueError(f"Support only dynamic size on first dimension. Got: {shape}") + self.shape = tuple(shape) + self.value_type = dtype + self.storage_dtype = self._generate_dtype(self.value_type) + pa.ExtensionType.__init__(self, self.storage_dtype, f"{self.__class__.__module__}.{self.__class__.__name__}") + + def __arrow_ext_serialize__(self): + return json.dumps((self.shape, self.value_type)).encode() + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + args = json.loads(serialized) + return cls(*args) + + # This was added to pa.ExtensionType in pyarrow >= 13.0.0 + def __reduce__(self): + return self.__arrow_ext_deserialize__, (self.storage_type, self.__arrow_ext_serialize__()) + + def __hash__(self): + return hash((self.__class__, self.shape, self.value_type)) + + def __arrow_ext_class__(self): + return ArrayExtensionArray + + def _generate_dtype(self, dtype): + dtype = string_to_arrow(dtype) + for d in reversed(self.shape): + dtype = pa.list_(dtype) + # Don't specify the size of the list, since fixed length list arrays have issues + # being validated after slicing in pyarrow 0.17.1 + return dtype + + def to_pandas_dtype(self): + return PandasArrayExtensionDtype(self.value_type) + + +class Array2DExtensionType(_ArrayXDExtensionType): + ndims = 2 + + +class Array3DExtensionType(_ArrayXDExtensionType): + ndims = 3 + + +class Array4DExtensionType(_ArrayXDExtensionType): + ndims = 4 + + +class Array5DExtensionType(_ArrayXDExtensionType): + ndims = 5 + + +# Register the extension types for deserialization +pa.register_extension_type(Array2DExtensionType((1, 2), "int64")) +pa.register_extension_type(Array3DExtensionType((1, 2, 3), "int64")) +pa.register_extension_type(Array4DExtensionType((1, 2, 3, 4), "int64")) +pa.register_extension_type(Array5DExtensionType((1, 2, 3, 4, 5), "int64")) + + +def _is_zero_copy_only(pa_type: pa.DataType, unnest: bool = False) -> bool: + """ + When converting a pyarrow array to a numpy array, we must know whether this could be done in zero-copy or not. + This function returns the value of the ``zero_copy_only`` parameter to pass to ``.to_numpy()``, given the type of the pyarrow array. + + # zero copy is available for all primitive types except booleans and temporal types (date, time, timestamp or duration) + # primitive types are types for which the physical representation in arrow and in numpy + # https://github.com/wesm/arrow/blob/c07b9b48cf3e0bbbab493992a492ae47e5b04cad/python/pyarrow/types.pxi#L821 + # see https://arrow.apache.org/docs/python/generated/pyarrow.Array.html#pyarrow.Array.to_numpy + # and https://issues.apache.org/jira/browse/ARROW-2871?jql=text%20~%20%22boolean%20to_numpy%22 + """ + + def _unnest_pa_type(pa_type: pa.DataType) -> pa.DataType: + if pa.types.is_list(pa_type): + return _unnest_pa_type(pa_type.value_type) + return pa_type + + if unnest: + pa_type = _unnest_pa_type(pa_type) + return pa.types.is_primitive(pa_type) and not (pa.types.is_boolean(pa_type) or pa.types.is_temporal(pa_type)) + + +class ArrayExtensionArray(pa.ExtensionArray): + def __array__(self): + zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True) + return self.to_numpy(zero_copy_only=zero_copy_only) + + def __getitem__(self, i): + return self.storage[i] + + def to_numpy(self, zero_copy_only=True): + storage: pa.ListArray = self.storage + null_mask = storage.is_null().to_numpy(zero_copy_only=False) + + if self.type.shape[0] is not None: + size = 1 + null_indices = np.arange(len(storage))[null_mask] - np.arange(np.sum(null_mask)) + + for i in range(self.type.ndims): + size *= self.type.shape[i] + storage = storage.flatten() + numpy_arr = storage.to_numpy(zero_copy_only=zero_copy_only) + numpy_arr = numpy_arr.reshape(len(self) - len(null_indices), *self.type.shape) + + if len(null_indices): + numpy_arr = np.insert(numpy_arr.astype(np.float64), null_indices, np.nan, axis=0) + + else: + shape = self.type.shape + ndims = self.type.ndims + arrays = [] + first_dim_offsets = np.array([off.as_py() for off in storage.offsets]) + for i, is_null in enumerate(null_mask): + if is_null: + arrays.append(np.nan) + else: + storage_el = storage[i : i + 1] + first_dim = first_dim_offsets[i + 1] - first_dim_offsets[i] + # flatten storage + for _ in range(ndims): + storage_el = storage_el.flatten() + + numpy_arr = storage_el.to_numpy(zero_copy_only=zero_copy_only) + arrays.append(numpy_arr.reshape(first_dim, *shape[1:])) + + if len(np.unique(np.diff(first_dim_offsets))) > 1: + # ragged + numpy_arr = np.empty(len(arrays), dtype=object) + numpy_arr[:] = arrays + else: + numpy_arr = np.array(arrays) + + return numpy_arr + + def to_pylist(self, maps_as_pydicts: Optional[Literal["lossy", "strict"]] = None): + zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True) + numpy_arr = self.to_numpy(zero_copy_only=zero_copy_only) + if self.type.shape[0] is None and numpy_arr.dtype == object: + return [arr.tolist() for arr in numpy_arr.tolist()] + else: + return numpy_arr.tolist() + + +class PandasArrayExtensionDtype(PandasExtensionDtype): + _metadata = "value_type" + + def __init__(self, value_type: Union["PandasArrayExtensionDtype", np.dtype]): + self._value_type = value_type + + def __from_arrow__(self, array: Union[pa.Array, pa.ChunkedArray]): + if isinstance(array, pa.ChunkedArray): + array = array.type.wrap_array(pa.concat_arrays([chunk.storage for chunk in array.chunks])) + zero_copy_only = _is_zero_copy_only(array.storage.type, unnest=True) + numpy_arr = array.to_numpy(zero_copy_only=zero_copy_only) + return PandasArrayExtensionArray(numpy_arr) + + @classmethod + def construct_array_type(cls): + return PandasArrayExtensionArray + + @property + def type(self) -> type: + return np.ndarray + + @property + def kind(self) -> str: + return "O" + + @property + def name(self) -> str: + return f"array[{self.value_type}]" + + @property + def value_type(self) -> np.dtype: + return self._value_type + + +class PandasArrayExtensionArray(PandasExtensionArray): + def __init__(self, data: np.ndarray, copy: bool = False): + self._data = data if not copy else np.array(data) + self._dtype = PandasArrayExtensionDtype(data.dtype) + + def __array__(self, dtype=None): + """ + Convert to NumPy Array. + Note that Pandas expects a 1D array when dtype is set to object. + But for other dtypes, the returned shape is the same as the one of ``data``. + + More info about pandas 1D requirement for PandasExtensionArray here: + https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.api.extensions.ExtensionArray.html#pandas.api.extensions.ExtensionArray + + """ + if dtype == np.dtype(object): + out = np.empty(len(self._data), dtype=object) + for i in range(len(self._data)): + out[i] = self._data[i] + return out + if dtype is None: + return self._data + else: + return self._data.astype(dtype) + + def copy(self, deep: bool = False) -> "PandasArrayExtensionArray": + return PandasArrayExtensionArray(self._data, copy=True) + + @classmethod + def _from_sequence( + cls, scalars, dtype: Optional[PandasArrayExtensionDtype] = None, copy: bool = False + ) -> "PandasArrayExtensionArray": + if len(scalars) > 1 and all( + isinstance(x, np.ndarray) and x.shape == scalars[0].shape and x.dtype == scalars[0].dtype for x in scalars + ): + data = np.array(scalars, dtype=dtype if dtype is None else dtype.value_type, copy=copy) + else: + data = np.empty(len(scalars), dtype=object) + data[:] = scalars + return cls(data, copy=copy) + + @classmethod + def _concat_same_type(cls, to_concat: Sequence_["PandasArrayExtensionArray"]) -> "PandasArrayExtensionArray": + if len(to_concat) > 1 and all( + va._data.shape == to_concat[0]._data.shape and va._data.dtype == to_concat[0]._data.dtype + for va in to_concat + ): + data = np.vstack([va._data for va in to_concat]) + else: + data = np.empty(len(to_concat), dtype=object) + data[:] = [va._data for va in to_concat] + return cls(data, copy=False) + + @property + def dtype(self) -> PandasArrayExtensionDtype: + return self._dtype + + @property + def nbytes(self) -> int: + return self._data.nbytes + + def isna(self) -> np.ndarray: + return np.array([pd.isna(arr).any() for arr in self._data]) + + def __setitem__(self, key: Union[int, slice, np.ndarray], value: Any) -> None: + raise NotImplementedError() + + def __getitem__(self, item: Union[int, slice, np.ndarray]) -> Union[np.ndarray, "PandasArrayExtensionArray"]: + if isinstance(item, int): + return self._data[item] + return PandasArrayExtensionArray(self._data[item], copy=False) + + def take( + self, indices: Sequence_[int], allow_fill: bool = False, fill_value: bool = None + ) -> "PandasArrayExtensionArray": + indices: np.ndarray = np.asarray(indices, dtype=int) + if allow_fill: + fill_value = ( + self.dtype.na_value if fill_value is None else np.asarray(fill_value, dtype=self.dtype.value_type) + ) + mask = indices == -1 + if (indices < -1).any(): + raise ValueError("Invalid value in `indices`, must be all >= -1 for `allow_fill` is True") + elif len(self) > 0: + pass + elif not np.all(mask): + raise IndexError("Invalid take for empty PandasArrayExtensionArray, must be all -1.") + else: + data = np.array([fill_value] * len(indices), dtype=self.dtype.value_type) + return PandasArrayExtensionArray(data, copy=False) + took = self._data.take(indices, axis=0) + if allow_fill and mask.any(): + took[mask] = [fill_value] * np.sum(mask) + return PandasArrayExtensionArray(took, copy=False) + + def __len__(self) -> int: + return len(self._data) + + def __eq__(self, other) -> np.ndarray: + if not isinstance(other, PandasArrayExtensionArray): + raise NotImplementedError(f"Invalid type to compare to: {type(other)}") + return (self._data == other._data).all() + + +def pandas_types_mapper(dtype): + if isinstance(dtype, _ArrayXDExtensionType): + return PandasArrayExtensionDtype(dtype.value_type) + + +@dataclass +class ClassLabel: + """Feature type for integer class labels. + + There are 3 ways to define a `ClassLabel`, which correspond to the 3 arguments: + + * `num_classes`: Create 0 to (num_classes-1) labels. + * `names`: List of label strings. + * `names_file`: File containing the list of labels. + + Under the hood the labels are stored as integers. + You can use negative integers to represent unknown/missing labels. + + Args: + num_classes (`int`, *optional*): + Number of classes. All labels must be < `num_classes`. + names (`list` of `str`, *optional*): + String names for the integer classes. + The order in which the names are provided is kept. + names_file (`str`, *optional*): + Path to a file with names for the integer classes, one per line. + + Example: + + ```py + >>> from datasets import Features, ClassLabel + >>> features = Features({'label': ClassLabel(num_classes=3, names=['bad', 'ok', 'good'])}) + >>> features + {'label': ClassLabel(names=['bad', 'ok', 'good'])} + ``` + """ + + num_classes: InitVar[Optional[int]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict + names: list[str] = None + names_file: InitVar[Optional[str]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + dtype: ClassVar[str] = "int64" + pa_type: ClassVar[Any] = pa.int64() + _str2int: ClassVar[dict[str, int]] = None + _int2str: ClassVar[dict[int, int]] = None + _type: str = field(default="ClassLabel", init=False, repr=False) + + def __post_init__(self, num_classes, names_file): + self.num_classes = num_classes + self.names_file = names_file + if self.names_file is not None and self.names is not None: + raise ValueError("Please provide either names or names_file but not both.") + # Set self.names + if self.names is None: + if self.names_file is not None: + self.names = self._load_names_from_file(self.names_file) + elif self.num_classes is not None: + self.names = [str(i) for i in range(self.num_classes)] + else: + raise ValueError("Please provide either num_classes, names or names_file.") + elif not isinstance(self.names, SequenceABC): + raise TypeError(f"Please provide names as a list, is {type(self.names)}") + # Set self.num_classes + if self.num_classes is None: + self.num_classes = len(self.names) + elif self.num_classes != len(self.names): + raise ValueError( + "ClassLabel number of names do not match the defined num_classes. " + f"Got {len(self.names)} names VS {self.num_classes} num_classes" + ) + # Prepare mappings + self._int2str = [str(name) for name in self.names] + self._str2int = {name: i for i, name in enumerate(self._int2str)} + if len(self._int2str) != len(self._str2int): + raise ValueError("Some label names are duplicated. Each label name should be unique.") + + def __call__(self): + return self.pa_type + + def str2int(self, values: Union[str, Iterable]) -> Union[int, Iterable]: + """Conversion class name `string` => `integer`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> ds.features["label"].str2int('neg') + 0 + ``` + """ + if not isinstance(values, str) and not isinstance(values, Iterable): + raise ValueError( + f"Values {values} should be a string or an Iterable (list, numpy array, pytorch, tensorflow tensors)" + ) + return_list = True + if isinstance(values, str): + values = [values] + return_list = False + + output = [self._strval2int(value) for value in values] + return output if return_list else output[0] + + def _strval2int(self, value: str) -> int: + failed_parse = False + value = str(value) + # first attempt - raw string value + int_value = self._str2int.get(value) + if int_value is None: + # second attempt - strip whitespace + int_value = self._str2int.get(value.strip()) + if int_value is None: + # third attempt - convert str to int + try: + int_value = int(value) + except ValueError: + failed_parse = True + else: + if int_value < -1 or int_value >= self.num_classes: + failed_parse = True + if failed_parse: + raise ValueError(f"Invalid string class label {value}") + return int_value + + def int2str(self, values: Union[int, Iterable]) -> Union[str, Iterable]: + """Conversion `integer` => class name `string`. + + Regarding unknown/missing labels: passing negative integers raises `ValueError`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> ds.features["label"].int2str(0) + 'neg' + ``` + """ + if not isinstance(values, int) and not isinstance(values, Iterable): + raise ValueError( + f"Values {values} should be an integer or an Iterable (list, numpy array, pytorch, tensorflow tensors)" + ) + return_list = True + if isinstance(values, int): + values = [values] + return_list = False + + for v in values: + if not 0 <= v < self.num_classes: + raise ValueError(f"Invalid integer class label {v:d}") + + output = [self._int2str[int(v)] for v in values] + return output if return_list else output[0] + + def encode_example(self, example_data): + if self.num_classes is None: + raise ValueError( + "Trying to use ClassLabel feature with undefined number of class. " + "Please set ClassLabel.names or num_classes." + ) + + # If a string is given, convert to associated integer + if isinstance(example_data, str): + example_data = self.str2int(example_data) + + # Allowing -1 to mean no label. + if not -1 <= example_data < self.num_classes: + raise ValueError(f"Class label {example_data:d} greater than configured num_classes {self.num_classes}") + return example_data + + def cast_storage(self, storage: Union[pa.StringArray, pa.IntegerArray]) -> pa.Int64Array: + """Cast an Arrow array to the `ClassLabel` arrow storage type. + The Arrow types that can be converted to the `ClassLabel` pyarrow storage type are: + + - `pa.string()` + - `pa.int()` + + Args: + storage (`Union[pa.StringArray, pa.IntegerArray]`): + PyArrow array to cast. + + Returns: + `pa.Int64Array`: Array in the `ClassLabel` arrow storage type. + """ + if isinstance(storage, pa.IntegerArray) and len(storage) > 0: + min_max = pc.min_max(storage).as_py() + if min_max["max"] is not None and min_max["max"] >= self.num_classes: + raise ValueError( + f"Class label {min_max['max']} greater than configured num_classes {self.num_classes}" + ) + elif isinstance(storage, pa.StringArray): + storage = pa.array( + [self._strval2int(label) if label is not None else None for label in storage.to_pylist()] + ) + return array_cast(storage, self.pa_type) + + @staticmethod + def _load_names_from_file(names_filepath): + with open(names_filepath, encoding="utf-8") as f: + return [name.strip() for name in f.read().split("\n") if name.strip()] # Filter empty names + + +@dataclass +class Json: + """Feature type for JSON objects. + + Under the hood the objects are stored as JSON-encoded strings. + + Example: + + ```py + >>> from datasets import Features, Json + >>> features = Features({'json': Json()}) + >>> features + {'json': Json()} + ``` + + ```py + >>> from datasets import Dataset, Features, Json, List + >>> features = Features({"a": List(Json())}) + >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}, features=features) + >>> # OR + >>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}, on_mixed_types="use_json") + >>> ds.features + {'a': List(Json())} + >>> ds[0] + {'a': [{'b': 0}, {'c': 0}]} + >>> def f(x): + ... for y in x["a"]: + ... y["d"] = "foo" + ... return x + >>> ds = ds.map(f) + >>> ds.features + >>> ds[0] + {'a': [{'b': 0, 'd': 'foo'}, {'c': 0, 'd': 'foo'}]} + ``` + """ + + decode: bool = True + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + pa_type: ClassVar[Any] = pa.json_() + _type: str = field(default="Json", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, example_data): + if not isinstance(example_data, str): + example_data = ujson_dumps(example_data) + else: + try: + ujson_loads(example_data) + except Exception: + example_data = ujson_dumps(example_data) + return example_data + + def decode_example(self, example_data, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None): + if not self.decode: + raise RuntimeError("Decoding is disabled for this feature. Please use Json(decode=True) instead.") + return ujson_loads(example_data) + + def cast_storage(self, storage: Union[pa.Array]) -> pa.JsonArray: + """Cast an Arrow array to the `Json` arrow storage type. + + Args: + storage (`Union[pa.StringArray, pa.IntegerArray]`): + PyArrow array to cast. + + Returns: + `pa.JsonArray`: Array in the `Json` arrow storage type. + """ + if isinstance(storage, pa.JsonArray): + return storage + elif isinstance(storage, (pa.StringArray)): + items = storage[:5].to_pylist() + try: + for item in items: + ujson_loads(item) + except Exception: + storage = pa.array([ujson_dumps(x) for x in storage.to_pylist()], pa.json_()) + else: + storage = pa.array([ujson_dumps(x) for x in storage.to_pylist()], pa.json_()) + return array_cast(storage, self.pa_type) + + +class Sequence: + """ + A `Sequence` is a utility that automatically converts internal dictionary feature into a dictionary of + lists. This behavior is implemented to have a compatibility layer with the TensorFlow Datasets library but may be + un-wanted in some cases. If you don't want this behavior, you can use a [`List`] or a [`LargeList`] + instead of the [`Sequence`]. + + Args: + feature ([`FeatureType`]): + Child feature data type of each item within the large list. + length (optional `int`, default to -1): + Length of the list if it is fixed. + Defaults to -1 which means an arbitrary length. + + Returns: + [`List`] of the specified feature, except `dict` of sub-features + which are converted to `dict` of lists of sub-features for compatibility with TFDS. + + """ + + def __new__(cls, feature=None, length=-1, **kwargs): + # useful to still get isinstance(Sequence(Value("int64")), Sequence) + if ( + cls is Sequence + and isinstance(feature, dict) + and any(not isinstance(subfeature, List) for subfeature in feature.values()) + ): + out = {key: List(value, length=length, **kwargs) for key, value in feature.items()} + else: + out = super().__new__(List) + return out + + +@dataclass(repr=False) +class List(Sequence): + """Feature type for large list data composed of child feature data type. + + It is backed by `pyarrow.ListType`, which uses 32-bit offsets or a fixed length. + + Args: + feature ([`FeatureType`]): + Child feature data type of each item within the large list. + length (optional `int`, default to -1): + Length of the list if it is fixed. + Defaults to -1 which means an arbitrary length. + """ + + feature: Any + length: int = -1 + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + pa_type: ClassVar[Any] = None + _type: str = field(default="List", init=False, repr=False) + + def __repr__(self): + if self.length != -1: + return f"{type(self).__name__}({self.feature}, length={self.length})" + else: + return f"{type(self).__name__}({self.feature})" + + +@dataclass(repr=False) +class LargeList: + """Feature type for large list data composed of child feature data type. + + It is backed by `pyarrow.LargeListType`, which is like `pyarrow.ListType` but with 64-bit rather than 32-bit offsets. + + Args: + feature ([`FeatureType`]): + Child feature data type of each item within the large list. + """ + + feature: Any + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + pa_type: ClassVar[Any] = None + _type: str = field(default="LargeList", init=False, repr=False) + + def __repr__(self): + return f"{type(self).__name__}({self.feature})" + + +FeatureType = Union[ + dict, + list, + tuple, + Value, + ClassLabel, + Translation, + TranslationVariableLanguages, + LargeList, + List, + Array2D, + Array3D, + Array4D, + Array5D, + Audio, + Image, + Video, + Pdf, + Nifti, +] + + +def _check_non_null_non_empty_recursive(obj, schema: Optional[FeatureType] = None) -> bool: + """ + Check if the object is not None. + If the object is a list or a tuple, recursively check the first element of the sequence and stop if at any point the first element is not a sequence or is an empty sequence. + """ + if obj is None: + return False + elif isinstance(obj, (list, tuple)) and (schema is None or isinstance(schema, (list, tuple, LargeList, List))): + if len(obj) > 0: + if schema is None: + pass + elif isinstance(schema, (list, tuple)): + schema = schema[0] + else: + schema = schema.feature + return _check_non_null_non_empty_recursive(obj[0], schema) + else: + return False + else: + return True + + +def get_nested_type(schema: FeatureType) -> pa.DataType: + """ + get_nested_type() converts a datasets.FeatureType into a pyarrow.DataType, and acts as the inverse of + generate_from_arrow_type(). + + It performs double-duty as the implementation of Features.type and handles the conversion of + datasets.Feature->pa.struct + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(schema, Features): + return pa.struct( + {key: get_nested_type(schema[key]) for key in schema} + ) # Features is subclass of dict, and dict order is deterministic since Python 3.6 + elif isinstance(schema, dict): + return pa.struct( + {key: get_nested_type(schema[key]) for key in schema} + ) # however don't sort on struct types since the order matters + elif isinstance(schema, (list, tuple)): + if len(schema) != 1: + raise ValueError("When defining list feature, you should just provide one example of the inner type") + value_type = get_nested_type(schema[0]) + return pa.list_(value_type) + elif isinstance(schema, LargeList): + value_type = get_nested_type(schema.feature) + return pa.large_list(value_type) + elif isinstance(schema, List): + value_type = get_nested_type(schema.feature) + return pa.list_(value_type, schema.length) + + # Other objects are callable which returns their data type (ClassLabel, Array2D, Translation, Arrow datatype creation methods) + return schema() + + +def encode_nested_example(schema, obj, level=0): + """Encode a nested example. + This is used since some features (in particular ClassLabel) have some logic during encoding. + + To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be encoded. + If the first element needs to be encoded, then all the elements of the list will be encoded, otherwise they'll stay the same. + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(schema, dict): + if level == 0 and obj is None: + raise ValueError("Got None but expected a dictionary instead") + return ( + {k: encode_nested_example(schema[k], obj.get(k), level=level + 1) for k in schema} + if obj is not None + else None + ) + elif isinstance(schema, (LargeList, List)): + if obj is None: + return None + else: + if len(obj) > 0: + sub_schema = schema.feature + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, sub_schema): + break + try: + changed = bool(encode_nested_example(sub_schema, first_elmt, level=level + 1) != first_elmt) + except ValueError: # can happen when comparing arrays + changed = False + if changed: + return [encode_nested_example(sub_schema, o, level=level + 1) for o in obj] + return list(obj) + # Object with special encoding: + # ClassLabel will convert from string to int, TranslationVariableLanguages does some checks + elif hasattr(schema, "encode_example"): + return schema.encode_example(obj) if obj is not None else None + # Other object should be directly convertible to a native Arrow type (like Translation and Translation) + return obj + + +def decode_nested_example(schema, obj, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None): + """Decode a nested example. + This is used since some features (in particular Audio and Image) have some logic during decoding. + + To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be decoded. + If the first element needs to be decoded, then all the elements of the list will be decoded, otherwise they'll stay the same. + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(schema, dict): + return ( + {k: decode_nested_example(sub_schema, sub_obj) for k, (sub_schema, sub_obj) in zip_dict(schema, obj)} + if obj is not None + else None + ) + elif isinstance(schema, (list, tuple)): + sub_schema = schema[0] + if obj is None: + return None + else: + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, sub_schema): + break + if decode_nested_example(sub_schema, first_elmt) != first_elmt: + return [decode_nested_example(sub_schema, o) for o in obj] + return list(obj) + elif isinstance(schema, (LargeList, List)): + if obj is None: + return None + else: + sub_schema = schema.feature + if isinstance(sub_schema, dict): + return [decode_nested_example(sub_schema, o) for o in obj] + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, sub_schema): + break + if decode_nested_example(sub_schema, first_elmt) != first_elmt: + return [decode_nested_example(sub_schema, o) for o in obj] + return list(obj) + # Object with special decoding: + elif hasattr(schema, "decode_example") and getattr(schema, "decode", True): + # we pass the token to read and decode files from private repositories in streaming mode + return schema.decode_example(obj, token_per_repo_id=token_per_repo_id) if obj is not None else None + return obj + + +_FEATURE_TYPES: dict[str, FeatureType] = { + Value.__name__: Value, + ClassLabel.__name__: ClassLabel, + Translation.__name__: Translation, + TranslationVariableLanguages.__name__: TranslationVariableLanguages, + LargeList.__name__: LargeList, + List.__name__: List, + Array2D.__name__: Array2D, + Array3D.__name__: Array3D, + Array4D.__name__: Array4D, + Array5D.__name__: Array5D, + Audio.__name__: Audio, + Image.__name__: Image, + Video.__name__: Video, + Pdf.__name__: Pdf, + Nifti.__name__: Nifti, + Json.__name__: Json, +} + + +@experimental +def register_feature( + feature_cls: type, + feature_type: str, +): + """ + Register a Feature object using a name and class. + This function must be used on a Feature class. + """ + if feature_type in _FEATURE_TYPES: + logger.warning( + f"Overwriting feature type '{feature_type}' ({_FEATURE_TYPES[feature_type].__name__} -> {feature_cls.__name__})" + ) + _FEATURE_TYPES[feature_type] = feature_cls + + +def generate_from_dict(obj: Any): + """Regenerate the nested feature object from a deserialized dict. + We use the '_type' fields to get the dataclass name to load. + + generate_from_dict is the recursive helper for Features.from_dict, and allows for a convenient constructor syntax + to define features from deserialized JSON dictionaries. This function is used in particular when deserializing + a :class:`DatasetInfo` that was dumped to a JSON object. This acts as an analogue to + :meth:`Features.from_arrow_schema` and handles the recursive field-by-field instantiation, but doesn't require any + mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive dtypes + that :class:`Value` automatically performs. + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(obj, list): + return [generate_from_dict(value) for value in obj] + # Otherwise we have a dict or a dataclass + if "_type" not in obj or isinstance(obj["_type"], dict): + return {key: generate_from_dict(value) for key, value in obj.items()} + obj = dict(obj) + _type = obj.pop("_type") + class_type = _FEATURE_TYPES.get(_type, None) or globals().get(_type, None) + + if class_type is None: + raise ValueError(f"Feature type '{_type}' not found. Available feature types: {list(_FEATURE_TYPES.keys())}") + + if class_type == LargeList: + feature = obj.pop("feature") + return LargeList(generate_from_dict(feature), **obj) + if class_type == List: + feature = obj.pop("feature") + return List(generate_from_dict(feature), **obj) + if class_type == Sequence: # backward compatibility, this translates to a List or a dict + feature = obj.pop("feature") + return Sequence(feature=generate_from_dict(feature), **obj) + + field_names = {f.name for f in fields(class_type)} + return class_type(**{k: v for k, v in obj.items() if k in field_names}) + + +def generate_from_arrow_type(pa_type: pa.DataType) -> FeatureType: + """ + generate_from_arrow_type accepts an arrow DataType and returns a datasets FeatureType to be used as the type for + a single field. + + This is the high-level arrow->datasets type conversion and is inverted by get_nested_type(). + + This operates at the individual *field* level, whereas Features.from_arrow_schema() operates at the + full schema level and holds the methods that represent the bijection from Features<->pyarrow.Schema + """ + if isinstance(pa_type, pa.StructType): + return {field.name: generate_from_arrow_type(field.type) for field in pa_type} + elif isinstance(pa_type, pa.FixedSizeListType): + return List(generate_from_arrow_type(pa_type.value_type), length=pa_type.list_size) + elif isinstance(pa_type, pa.ListType): + return List(generate_from_arrow_type(pa_type.value_type)) + elif isinstance(pa_type, pa.LargeListType): + return LargeList(generate_from_arrow_type(pa_type.value_type)) + elif isinstance(pa_type, _ArrayXDExtensionType): + array_feature = [None, None, Array2D, Array3D, Array4D, Array5D][pa_type.ndims] + return array_feature(shape=pa_type.shape, dtype=pa_type.value_type) + elif isinstance(pa_type, pa.JsonType): + return Json() + elif isinstance(pa_type, pa.DataType): + return Value(dtype=_arrow_to_datasets_dtype(pa_type)) + else: + raise ValueError(f"Cannot convert {pa_type} to a Feature type.") + + +def numpy_to_pyarrow_listarray(arr: np.ndarray, type: pa.DataType = None) -> pa.ListArray: + """Build a PyArrow ListArray from a multidimensional NumPy array""" + arr = np.array(arr) + values = pa.array(arr.flatten(), type=type) + for i in range(arr.ndim - 1): + n_offsets = reduce(mul, arr.shape[: arr.ndim - i - 1], 1) + step_offsets = arr.shape[arr.ndim - i - 1] + offsets = pa.array(np.arange(n_offsets + 1) * step_offsets, type=pa.int32()) + values = pa.ListArray.from_arrays(offsets, values) + return values + + +def list_of_pa_arrays_to_pyarrow_listarray(l_arr: list[Optional[pa.Array]]) -> pa.ListArray: + null_mask = np.array([arr is None for arr in l_arr]) + null_indices = np.arange(len(null_mask))[null_mask] - np.arange(np.sum(null_mask)) + l_arr = [arr for arr in l_arr if arr is not None] + offsets = np.cumsum( + [0] + [len(arr) for arr in l_arr], dtype=object + ) # convert to dtype object to allow None insertion + offsets = np.insert(offsets, null_indices, None) + offsets = pa.array(offsets, type=pa.int32()) + values = pa.concat_arrays(l_arr) + return pa.ListArray.from_arrays(offsets, values) + + +def list_of_np_array_to_pyarrow_listarray(l_arr: list[np.ndarray], type: pa.DataType = None) -> pa.ListArray: + """Build a PyArrow ListArray from a possibly nested list of NumPy arrays""" + if len(l_arr) > 0: + return list_of_pa_arrays_to_pyarrow_listarray( + [numpy_to_pyarrow_listarray(arr, type=type) if arr is not None else None for arr in l_arr] + ) + else: + return pa.array([], type=type) + + +def contains_any_np_array(data: Any): + """Return `True` if data is a NumPy ndarray or (recursively) if first non-null value in list is a NumPy ndarray. + + Args: + data (Any): Data. + + Returns: + bool + """ + if isinstance(data, np.ndarray): + return True + elif isinstance(data, list): + return contains_any_np_array(first_non_null_value(data)[1]) + else: + return False + + +def any_np_array_to_pyarrow_listarray(data: Union[np.ndarray, list], type: pa.DataType = None) -> pa.ListArray: + """Convert to PyArrow ListArray either a NumPy ndarray or (recursively) a list that may contain any NumPy ndarray. + + Args: + data (Union[np.ndarray, List]): Data. + type (pa.DataType): Explicit PyArrow DataType passed to coerce the ListArray data type. + + Returns: + pa.ListArray + """ + if isinstance(data, np.ndarray): + return numpy_to_pyarrow_listarray(data, type=type) + elif isinstance(data, list): + return list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, type=type) for i in data]) + + +def to_pyarrow_listarray(data: Any, pa_type: _ArrayXDExtensionType) -> pa.Array: + """Convert to PyArrow ListArray. + + Args: + data (Any): List, iterable, np.ndarray or pd.Series. + pa_type (_ArrayXDExtensionType): Any of the ArrayNDExtensionType. + + Returns: + pyarrow.Array + """ + if contains_any_np_array(data): + return any_np_array_to_pyarrow_listarray(data, type=pa_type.value_type) + else: + return pa.array(data, pa_type.storage_dtype) + + +def _visit(feature: FeatureType, func: Callable[[FeatureType], Optional[FeatureType]]) -> FeatureType: + """Visit a (possibly nested) feature. + + Args: + feature (FeatureType): the feature type to be checked + Returns: + visited feature (FeatureType) + """ + if isinstance(feature, Features): + out = func(Features({k: _visit(f, func) for k, f in feature.items()})) + elif isinstance(feature, dict): + out = func({k: _visit(f, func) for k, f in feature.items()}) + elif isinstance(feature, LargeList): + out = func(LargeList(_visit(feature.feature, func))) + elif isinstance(feature, List): + out = func(List(_visit(feature.feature, func), length=feature.length)) + else: + out = func(feature) + return feature if out is None else out + + +_VisitPath = list[Union[str, Literal[0]]] + + +def _visit_with_path( + feature: FeatureType, func: Callable[[FeatureType, _VisitPath], Optional[FeatureType]], visit_path: _VisitPath = [] +) -> FeatureType: + """Visit a (possibly nested) feature with its path in the Feature object. + + A path in a nested feature object is the list of keys that need to be + sequentially accessed to get to the sub-feature. + + For example: + - ["foo"] corresponds to the column "foo" + - ["foo", 0] corresponds to the sub-feature of the lists in "foo" + - ["foo", "bar"] corresponds to the sub-feature of the dicts in "foo" with key "bar" + + Args: + feature (`FeatureType`): the feature type to be checked. + + Returns: + `FeatureType`: the visited feature. + """ + if isinstance(feature, Features): + out = func(Features({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}), visit_path) + elif isinstance(feature, dict): + out = func({k: _visit_with_path(f, func, visit_path + [k]) for k, f in feature.items()}, visit_path) + elif isinstance(feature, List): + out = func(List(_visit_with_path(feature.feature, func, visit_path + [0]), length=feature.length), visit_path) + elif isinstance(feature, LargeList): + out = func(LargeList(_visit_with_path(feature.feature, func, visit_path + [0])), visit_path) + else: + out = func(feature, visit_path) + return feature if out is None else out + + +def require_decoding(feature: FeatureType, ignore_decode_attribute: bool = False) -> bool: + """Check if a (possibly nested) feature requires decoding. + + Args: + feature (FeatureType): the feature type to be checked + ignore_decode_attribute (:obj:`bool`, default ``False``): Whether to ignore the current value + of the `decode` attribute of the decodable feature types. + Returns: + :obj:`bool` + """ + if isinstance(feature, dict): + return any(require_decoding(f) for f in feature.values()) + elif isinstance(feature, (list, tuple)): + return require_decoding(feature[0]) + elif isinstance(feature, LargeList): + return require_decoding(feature.feature) + elif isinstance(feature, List): + return require_decoding(feature.feature) + else: + return hasattr(feature, "decode_example") and ( + getattr(feature, "decode", True) if not ignore_decode_attribute else True + ) + + +def require_storage_cast(feature: FeatureType) -> bool: + """Check if a (possibly nested) feature requires storage casting. + + Args: + feature (FeatureType): the feature type to be checked + Returns: + :obj:`bool` + """ + if isinstance(feature, dict): + return any(require_storage_cast(f) for f in feature.values()) + elif isinstance(feature, LargeList): + return require_storage_cast(feature.feature) + elif isinstance(feature, List): + return require_storage_cast(feature.feature) + else: + return hasattr(feature, "cast_storage") + + +def require_storage_embed(feature: FeatureType) -> bool: + """Check if a (possibly nested) feature requires embedding data into storage. + + Args: + feature (FeatureType): the feature type to be checked + Returns: + :obj:`bool` + """ + if isinstance(feature, dict): + return any(require_storage_cast(f) for f in feature.values()) + elif isinstance(feature, LargeList): + return require_storage_cast(feature.feature) + elif isinstance(feature, List): + return require_storage_cast(feature.feature) + else: + return hasattr(feature, "embed_storage") + + +def keep_features_dicts_synced(func): + """ + Wrapper to keep the secondary dictionary, which tracks whether keys are decodable, of the :class:`datasets.Features` object + in sync with the main dictionary. + """ + + @wraps(func) + def wrapper(*args, **kwargs): + if args: + self: "Features" = args[0] + args = args[1:] + else: + self: "Features" = kwargs.pop("self") + out = func(self, *args, **kwargs) + assert hasattr(self, "_column_requires_decoding") + self._column_requires_decoding = {col: require_decoding(feature) for col, feature in self.items()} + return out + + wrapper._decorator_name_ = "_keep_dicts_synced" + return wrapper + + +class Features(dict): + """A special dictionary that defines the internal structure of a dataset. + + Instantiated with a dictionary of type `dict[str, FieldType]`, where keys are the desired column names, + and values are the type of that column. + + `FieldType` can be one of the following: + - [`Value`] feature specifies a single data type value, e.g. `int64` or `string`. + - [`ClassLabel`] feature specifies a predefined set of classes which can have labels associated to them and + will be stored as integers in the dataset. + - Python `dict` specifies a composite feature containing a mapping of sub-fields to sub-features. + It's possible to have nested fields of nested fields in an arbitrary manner. + - [`List`] or [`LargeList`] specifies a composite feature containing a sequence of + sub-features, all of the same feature type. + - [`Array2D`], [`Array3D`], [`Array4D`] or [`Array5D`] feature for multidimensional arrays. + - [`Audio`] feature to store the absolute path to an audio file or a dictionary with the relative path + to an audio file ("path" key) and its bytes content ("bytes" key). + This feature loads the audio lazily with a decoder. + - [`Image`] feature to store the absolute path to an image file, an `np.ndarray` object, a `PIL.Image.Image` object + or a dictionary with the relative path to an image file ("path" key) and its bytes content ("bytes" key). + This feature extracts the image data. + - [`Video`] feature to store the absolute path to a video file, a `torchcodec.decoders.VideoDecoder` object + or a dictionary with the relative path to a video file ("path" key) and its bytes content ("bytes" key). + This feature loads the video lazily with a decoder. + - [`Pdf`] feature to store the absolute path to a PDF file, a `pdfplumber.pdf.PDF` object + or a dictionary with the relative path to a PDF file ("path" key) and its bytes content ("bytes" key). + This feature loads the PDF lazily with a PDF reader. + - [`Nifti`] feature to store the absolute path to a NIfTI neuroimaging file, a `nibabel.Nifti1Image` object + or a dictionary with the relative path to a NIfTI file ("path" key) and its bytes content ("bytes" key). + This feature loads the NIfTI file lazily with nibabel. + - [`Translation`] or [`TranslationVariableLanguages`] feature specific to Machine Translation. + - [`Json`] feature to store unstructred data, e.g. containing mixed/abritrary types. Under the hood + """ + + def __init__(*args, **kwargs): + # self not in the signature to allow passing self as a kwarg + if not args: + raise TypeError("descriptor '__init__' of 'Features' object needs an argument") + self, *args = args + super(Features, self).__init__(*args, **kwargs) + # keep track of columns which require decoding + self._column_requires_decoding: dict[str, bool] = { + col: require_decoding(feature) for col, feature in self.items() + } + + # backward compatibility with datasets<4 : [feature] -> List(feature) + def _check_old_list(feature): + if isinstance(feature, list): + return List(_visit(feature[0], _check_old_list)) + return feature + + for column_name, feature in self.items(): + self[column_name] = _visit(feature, _check_old_list) + + __setitem__ = keep_features_dicts_synced(dict.__setitem__) + __delitem__ = keep_features_dicts_synced(dict.__delitem__) + update = keep_features_dicts_synced(dict.update) + setdefault = keep_features_dicts_synced(dict.setdefault) + pop = keep_features_dicts_synced(dict.pop) + popitem = keep_features_dicts_synced(dict.popitem) + clear = keep_features_dicts_synced(dict.clear) + + def __reduce__(self): + return Features, (dict(self),) + + @property + def type(self): + """ + Features field types. + + Returns: + :obj:`pyarrow.DataType` + """ + return get_nested_type(self) + + @property + def arrow_schema(self): + """ + Features schema. + + Returns: + :obj:`pyarrow.Schema` + """ + hf_metadata = {"info": {"features": self.to_dict()}} + return pa.schema(self.type).with_metadata({"huggingface": json.dumps(hf_metadata)}) + + @classmethod + def from_arrow_schema(cls, pa_schema: pa.Schema) -> "Features": + """ + Construct [`Features`] from Arrow Schema. + It also checks the schema metadata for Hugging Face Datasets features. + Non-nullable fields are not supported and set to nullable. + + Also, pa.dictionary is not supported and it uses its underlying type instead. + Therefore datasets convert DictionaryArray objects to their actual values. + + Args: + pa_schema (`pyarrow.Schema`): + Arrow Schema. + + Returns: + [`Features`] + """ + # try to load features from the arrow schema metadata + metadata_features = Features() + if pa_schema.metadata is not None and b"huggingface" in pa_schema.metadata: + metadata = json.loads(pa_schema.metadata[b"huggingface"].decode()) + if "info" in metadata and "features" in metadata["info"] and metadata["info"]["features"] is not None: + metadata_features = Features.from_dict(metadata["info"]["features"]) + metadata_features_schema = metadata_features.arrow_schema + obj = { + field.name: ( + metadata_features[field.name] + if field.name in metadata_features and metadata_features_schema.field(field.name) == field + else generate_from_arrow_type(field.type) + ) + for field in pa_schema + } + return cls(**obj) + + @classmethod + def from_dict(cls, dic) -> "Features": + """ + Construct [`Features`] from dict. + + Regenerate the nested feature object from a deserialized dict. + We use the `_type` key to infer the dataclass name of the feature `FieldType`. + + It allows for a convenient constructor syntax + to define features from deserialized JSON dictionaries. This function is used in particular when deserializing + a [`DatasetInfo`] that was dumped to a JSON object. This acts as an analogue to + [`Features.from_arrow_schema`] and handles the recursive field-by-field instantiation, but doesn't require + any mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive + dtypes that [`Value`] automatically performs. + + Args: + dic (`dict[str, Any]`): + Python dictionary. + + Returns: + `Features` + + Example:: + >>> Features.from_dict({'_type': {'dtype': 'string', 'id': None, '_type': 'Value'}}) + {'_type': Value('string')} + """ + obj = generate_from_dict(dic) + return cls(**obj) + + def to_dict(self): + return asdict(self) + + def _to_yaml_list(self) -> list: + # we compute the YAML list from the dict representation that is used for JSON dump + yaml_data = self.to_dict() + + def simplify(feature: dict) -> dict: + if not isinstance(feature, dict): + raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}") + + for list_type in ["large_list", "list", "sequence"]: + # + # list_type: -> list_type: int32 + # dtype: int32 -> + # + if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["dtype"]: + feature[list_type] = feature[list_type]["dtype"] + + # + # list_type: -> list_type: + # struct: -> - name: foo + # - name: foo -> dtype: int32 + # dtype: int32 -> + # + if isinstance(feature.get(list_type), dict) and list(feature[list_type]) == ["struct"]: + feature[list_type] = feature[list_type]["struct"] + + # + # class_label: -> class_label: + # names: -> names: + # - negative -> '0': negative + # - positive -> '1': positive + # + if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), list): + # server-side requirement: keys must be strings + feature["class_label"]["names"] = { + str(label_id): label_name for label_id, label_name in enumerate(feature["class_label"]["names"]) + } + return feature + + def to_yaml_inner(obj: Union[dict, list]) -> dict: + if isinstance(obj, dict): + _type = obj.pop("_type", None) + if _type == "LargeList": + _feature = obj.pop("feature") + return simplify({"large_list": to_yaml_inner(_feature), **obj}) + elif _type == "List": + _feature = obj.pop("feature") + return simplify({"list": to_yaml_inner(_feature), **obj}) + elif _type == "Value": + return obj + elif _type and not obj: + return {"dtype": camelcase_to_snakecase(_type)} + elif _type: + return {"dtype": simplify({camelcase_to_snakecase(_type): obj})} + else: + return {"struct": [{"name": name, **to_yaml_inner(_feature)} for name, _feature in obj.items()]} + elif isinstance(obj, list): + return simplify({"list": simplify(to_yaml_inner(obj[0]))}) + elif isinstance(obj, tuple): + return to_yaml_inner(list(obj)) + else: + raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}") + + def to_yaml_types(obj: dict) -> dict: + if isinstance(obj, dict): + return {k: to_yaml_types(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [to_yaml_types(v) for v in obj] + elif isinstance(obj, tuple): + return to_yaml_types(list(obj)) + else: + return obj + + return to_yaml_types(to_yaml_inner(yaml_data)["struct"]) + + @classmethod + def _from_yaml_list(cls, yaml_data: list) -> "Features": + yaml_data = copy.deepcopy(yaml_data) + + # we convert the list obtained from YAML data into the dict representation that is used for JSON dump + + def unsimplify(feature: dict) -> dict: + if not isinstance(feature, dict): + raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}") + + for list_type in ["large_list", "list", "sequence"]: + # + # list_type: int32 -> list_type: + # -> dtype: int32 + # + if isinstance(feature.get(list_type), str): + feature[list_type] = {"dtype": feature[list_type]} + + # + # class_label: -> class_label: + # names: -> names: + # '0': negative -> - negative + # '1': positive -> - positive + # + if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), dict): + label_ids = sorted(feature["class_label"]["names"], key=int) + if label_ids and [int(label_id) for label_id in label_ids] != list(range(int(label_ids[-1]) + 1)): + raise ValueError( + f"ClassLabel expected a value for all label ids [0:{int(label_ids[-1]) + 1}] but some ids are missing." + ) + feature["class_label"]["names"] = [feature["class_label"]["names"][label_id] for label_id in label_ids] + return feature + + def from_yaml_inner(obj: Union[dict, list]) -> Union[dict, list]: + if isinstance(obj, dict): + if not obj: + return {} + _type = next(iter(obj)) + if _type == "large_list": + _feature = from_yaml_inner(unsimplify(obj).pop(_type)) + return {"feature": _feature, **obj, "_type": "LargeList"} + if _type == "sequence": # backward compatibility + if isinstance(obj[_type], list): + _feature = from_yaml_inner(unsimplify(obj).pop(_type)) + return { + name: {"feature": _subfeature, **obj, "_type": "List"} + for name, _subfeature in _feature.items() + } + else: + _feature = from_yaml_inner(unsimplify(obj).pop(_type)) + return {"feature": _feature, **obj, "_type": "List"} + if _type == "list": + _feature = from_yaml_inner(unsimplify(obj).pop(_type)) + return {"feature": _feature, **obj, "_type": "List"} + if _type == "struct": + return from_yaml_inner(obj["struct"]) + elif _type == "dtype": + if isinstance(obj["dtype"], str): + # e.g. int32, float64, string, audio, image + try: + Value(obj["dtype"]) + return {**obj, "_type": "Value"} + except ValueError: + # e.g. Audio, Image, ArrayXD + return {"_type": snakecase_to_camelcase(obj["dtype"])} + else: + return from_yaml_inner(obj["dtype"]) + else: + return {"_type": snakecase_to_camelcase(_type), **unsimplify(obj)[_type]} + elif isinstance(obj, list): + names = [_feature.pop("name") for _feature in obj] + return {name: from_yaml_inner(_feature) for name, _feature in zip(names, obj)} + else: + raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}") + + return cls.from_dict(from_yaml_inner(yaml_data)) + + def encode_example(self, example): + """ + Encode example into a format for Arrow. + + Args: + example (`dict[str, Any]`): + Data in a Dataset row. + + Returns: + `dict[str, Any]` + """ + example = cast_to_python_objects(example) + return encode_nested_example(self, example) + + def encode_column(self, column, column_name: str): + """ + Encode column into a format for Arrow. + + Args: + column (`list[Any]`): + Data in a Dataset column. + column_name (`str`): + Dataset column name. + + Returns: + `list[Any]` + """ + column = cast_to_python_objects(column) + return [encode_nested_example(self[column_name], obj, level=1) for obj in column] + + def encode_batch(self, batch): + """ + Encode batch into a format for Arrow. + + Args: + batch (`dict[str, list[Any]]`): + Data in a Dataset batch. + + Returns: + `dict[str, list[Any]]` + """ + encoded_batch = {} + if set(batch) != set(self): + raise ValueError(f"Column mismatch between batch {set(batch)} and features {set(self)}") + for key, column in batch.items(): + column = cast_to_python_objects(column) + encoded_batch[key] = [encode_nested_example(self[key], obj, level=1) for obj in column] + return encoded_batch + + def decode_example(self, example: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None): + """Decode example with custom feature decoding. + + Args: + example (`dict[str, Any]`): + Dataset row data. + token_per_repo_id (`dict`, *optional*): + To access and decode audio or image files from private repositories on the Hub, you can pass + a dictionary `repo_id (str) -> token (bool or str)`. + + Returns: + `dict[str, Any]` + """ + + return { + column_name: decode_nested_example(feature, value, token_per_repo_id=token_per_repo_id) + if self._column_requires_decoding[column_name] + else value + for column_name, (feature, value) in zip_dict( + {key: value for key, value in self.items() if key in example}, example + ) + } + + def decode_column( + self, column: list, column_name: str, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None + ): + """Decode column with custom feature decoding. + + Args: + column (`list[Any]`): + Dataset column data. + column_name (`str`): + Dataset column name. + + Returns: + `list[Any]` + """ + return ( + [ + decode_nested_example(self[column_name], value, token_per_repo_id=token_per_repo_id) + if value is not None + else None + for value in column + ] + if self._column_requires_decoding[column_name] + else column + ) + + def decode_batch(self, batch: dict, token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None): + """Decode batch with custom feature decoding. + + Args: + batch (`dict[str, list[Any]]`): + Dataset batch data. + token_per_repo_id (`dict`, *optional*): + To access and decode audio or image files from private repositories on the Hub, you can pass + a dictionary repo_id (str) -> token (bool or str) + + Returns: + `dict[str, list[Any]]` + """ + decoded_batch = {} + for column_name, column in batch.items(): + decoded_batch[column_name] = ( + [ + decode_nested_example(self[column_name], value, token_per_repo_id=token_per_repo_id) + if value is not None + else None + for value in column + ] + if self._column_requires_decoding[column_name] + else column + ) + return decoded_batch + + def copy(self) -> "Features": + """ + Make a deep copy of [`Features`]. + + Returns: + [`Features`] + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> copy_of_features = ds.features.copy() + >>> copy_of_features + {'label': ClassLabel(names=['neg', 'pos']), + 'text': Value('string')} + ``` + """ + return copy.deepcopy(self) + + def reorder_fields_as(self, other: "Features") -> "Features": + """ + Reorder Features fields to match the field order of other [`Features`]. + + The order of the fields is important since it matters for the underlying arrow data. + Re-ordering the fields allows to make the underlying arrow data type match. + + Args: + other ([`Features`]): + The other [`Features`] to align with. + + Returns: + [`Features`] + + Example:: + + >>> from datasets import Features, List, Value + >>> # let's say we have two features with a different order of nested fields (for a and b for example) + >>> f1 = Features({"root": {"a": Value("string"), "b": Value("string")}}) + >>> f2 = Features({"root": {"b": Value("string"), "a": Value("string")}}) + >>> assert f1.type != f2.type + >>> # re-ordering keeps the base structure (here List is defined at the root level), but makes the fields order match + >>> f1.reorder_fields_as(f2) + {'root': List({'b': Value('string'), 'a': Value('string')})} + >>> assert f1.reorder_fields_as(f2).type == f2.type + """ + + def recursive_reorder(source, target, stack=""): + stack_position = " at " + stack[1:] if stack else "" + if isinstance(source, dict): + if not isinstance(target, dict): + raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position) + if sorted(source) != sorted(target): + message = ( + f"Keys mismatch: between {source} (source) and {target} (target).\n" + f"{source.keys() - target.keys()} are missing from target " + f"and {target.keys() - source.keys()} are missing from source" + stack_position + ) + raise ValueError(message) + return {key: recursive_reorder(source[key], target[key], stack + f".{key}") for key in target} + elif isinstance(source, List): + if not isinstance(target, List): + raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position) + return List(recursive_reorder(source.feature, target.feature, stack + "."), length=source.length) + elif isinstance(source, LargeList): + if not isinstance(target, LargeList): + raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position) + return LargeList(recursive_reorder(source.feature, target.feature, stack + ".")) + else: + return source + + return Features(recursive_reorder(self, other)) + + def flatten(self, max_depth=16) -> "Features": + """Flatten the features. Every dictionary column is removed and is replaced by + all the subfields it contains. The new fields are named by concatenating the + name of the original column and the subfield name like this: `.`. + + If a column contains nested dictionaries, then all the lower-level subfields names are + also concatenated to form new columns: `..`, etc. + + Returns: + [`Features`]: + The flattened features. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("rajpurkar/squad", split="train") + >>> ds.features.flatten() + {'answers.answer_start': List(Value('int32'), id=None), + 'answers.text': List(Value('string'), id=None), + 'context': Value('string'), + 'id': Value('string'), + 'question': Value('string'), + 'title': Value('string')} + ``` + """ + for depth in range(1, max_depth): + no_change = True + flattened = self.copy() + for column_name, subfeature in self.items(): + if isinstance(subfeature, dict): + no_change = False + flattened.update({f"{column_name}.{k}": v for k, v in subfeature.items()}) + del flattened[column_name] + elif hasattr(subfeature, "flatten") and subfeature.flatten() != subfeature: + no_change = False + flattened.update({f"{column_name}.{k}": v for k, v in subfeature.flatten().items()}) + del flattened[column_name] + self = flattened + if no_change: + break + return self + + +def _is_null_feature(feature) -> bool: + """Recursively check if a feature represents a null type. + + This handles not only top-level ``Value("null")`` but also nested null types + such as ``List(Value("null"))``, ``LargeList(Value("null"))``, and + ``Sequence(Value("null"))``, which can arise when a shard contains only + empty lists during multi-process ``Dataset.map()``. + """ + if isinstance(feature, Value) and feature.dtype == "null": + return True + if isinstance(feature, (Sequence, LargeList)) and hasattr(feature, "feature"): + return _is_null_feature(feature.feature) + return False + + +def _align_features(features_list: list[Features]) -> list[Features]: + """Align dictionaries of features so that the keys that are found in multiple dictionaries share the same feature.""" + name2feature = {} + for features in features_list: + for k, v in features.items(): + if k in name2feature and isinstance(v, dict): + # Recursively align features. + name2feature[k] = _align_features([name2feature[k], v])[0] + elif k not in name2feature or _is_null_feature(name2feature[k]): + name2feature[k] = v + + return [Features({k: name2feature[k] for k in features.keys()}) for features in features_list] + + +def _check_if_features_can_be_aligned(features_list: list[Features]): + """Check if the dictionaries of features can be aligned. + + Two dictionaries of features can be aligned if the keys they share have the same type or some of them is of + type ``Value("null")`` (or a container wrapping ``Value("null")``, such as ``List(Value("null"))``). + """ + name2feature = {} + for features in features_list: + for k, v in features.items(): + if k not in name2feature or _is_null_feature(name2feature[k]): + name2feature[k] = v + + for features in features_list: + for k, v in features.items(): + if isinstance(v, dict) and isinstance(name2feature[k], dict): + # Deep checks for structure. + _check_if_features_can_be_aligned([name2feature[k], v]) + elif not _is_null_feature(v) and name2feature[k] != v: + raise ValueError( + f'The features can\'t be aligned because the key {k} of features {features} has unexpected type - {v} (expected either {name2feature[k]} or Value("null").' + ) + + +def _fix_for_backward_compatible_features(feature: Any) -> FeatureType: + def _fix_old_list(feature): + if isinstance(feature, list): + return List(_fix_for_backward_compatible_features(feature[0])) + return feature + + return _visit(feature, _fix_old_list) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/image.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/image.py new file mode 100644 index 0000000000000000000000000000000000000000..9a7953dd6bc4b6638de7d53d23764a996964e75e --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/image.py @@ -0,0 +1,441 @@ +import os +import sys +import warnings +from dataclasses import dataclass, field +from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union + +import numpy as np +import pyarrow as pa + +from .. import config +from ..download.download_config import DownloadConfig +from ..table import array_cast +from ..utils.file_utils import is_local_path, is_remote_url, xopen +from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict + + +if TYPE_CHECKING: + import PIL.Image + + from .features import FeatureType + + +_IMAGE_COMPRESSION_FORMATS: Optional[list[str]] = None +_NATIVE_BYTEORDER = "<" if sys.byteorder == "little" else ">" +# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image +_VALID_IMAGE_ARRAY_DTPYES = [ + np.dtype("|b1"), + np.dtype("|u1"), + np.dtype("u2"), + np.dtype("i2"), + np.dtype("u4"), + np.dtype("i4"), + np.dtype("f4"), + np.dtype("f8"), +] + + +@dataclass +class Image: + """Image [`Feature`] to read image data from an image file. + + Input: The Image feature accepts as input: + - A `str`: Absolute path to the image file (i.e. random access is allowed). + - A `pathlib.Path`: path to the image file (i.e. random access is allowed). + - A `dict` with the keys: + + - `path`: String with relative path of the image file to the archive file. + - `bytes`: Bytes of the image file. + + This is useful for parquet or webdataset files which embed image files. + + - An `np.ndarray`: NumPy array representing an image. + - A `PIL.Image.Image`: PIL image object. + + Output: The Image features output data as `PIL.Image.Image` objects. + + Args: + mode (`str`, *optional*): + The mode to convert the image to. If `None`, the native mode of the image is used. + decode (`bool`, defaults to `True`): + Whether to decode the image data. If `False`, + returns the underlying dictionary in the format `{"path": image_path, "bytes": image_bytes}`. + + Examples: + + ```py + >>> from datasets import load_dataset, Image + >>> ds = load_dataset("AI-Lab-Makerere/beans", split="train") + >>> ds.features["image"] + Image(decode=True, id=None) + >>> ds[0]["image"] + + >>> ds = ds.cast_column('image', Image(decode=False)) + {'bytes': None, + 'path': '/root/.cache/huggingface/datasets/downloads/extracted/b0a21163f78769a2cf11f58dfc767fb458fc7cea5c05dccc0144a2c0f0bc1292/train/healthy/healthy_train.85.jpg'} + ``` + """ + + mode: Optional[str] = None + decode: bool = True + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + dtype: ClassVar[str] = "PIL.Image.Image" + pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()}) + _type: str = field(default="Image", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, value: Union[str, bytes, bytearray, dict, np.ndarray, "PIL.Image.Image"]) -> dict: + """Encode example into a format for Arrow. + + Args: + value (`str`, `np.ndarray`, `PIL.Image.Image` or `dict`): + Data passed as input to Image feature. + + Returns: + `dict` with "path" and "bytes" fields + """ + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + if isinstance(value, list): + value = np.array(value) + + if isinstance(value, str): + return {"path": value, "bytes": None} + elif isinstance(value, Path): + return {"path": str(value.absolute()), "bytes": None} + elif isinstance(value, (bytes, bytearray)): + return {"path": None, "bytes": value} + elif isinstance(value, np.ndarray): + # convert the image array to PNG/TIFF bytes + return encode_np_array(value) + elif isinstance(value, PIL.Image.Image): + # convert the PIL image to bytes (default format is PNG/TIFF) + return encode_pil_image(value) + elif value.get("path") is not None and os.path.isfile(value["path"]): + # we set "bytes": None to not duplicate the data if they're already available locally + return {"bytes": None, "path": value.get("path")} + elif value.get("bytes") is not None or value.get("path") is not None: + # store the image bytes, and path is used to infer the image format using the file extension + return {"bytes": value.get("bytes"), "path": value.get("path")} + else: + raise ValueError( + f"An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}." + ) + + def decode_example(self, value: dict, token_per_repo_id=None) -> "PIL.Image.Image": + """Decode example image file into image data. + + Args: + value (`str` or `dict`): + A string with the absolute image file path, a dictionary with + keys: + + - `path`: String with absolute or relative image file path. + - `bytes`: The bytes of the image file. + token_per_repo_id (`dict`, *optional*): + To access and decode + image files from private repositories on the Hub, you can pass + a dictionary repo_id (`str`) -> token (`bool` or `str`). + + Returns: + `PIL.Image.Image` + """ + if not self.decode: + raise RuntimeError("Decoding is disabled for this feature. Please use Image(decode=True) instead.") + + if config.PIL_AVAILABLE: + import PIL.Image + import PIL.ImageOps + else: + raise ImportError("To support decoding images, please install 'Pillow'.") + + if token_per_repo_id is None: + token_per_repo_id = {} + + path, bytes_ = value["path"], value["bytes"] + if bytes_ is None: + if path is None: + raise ValueError(f"An image should have one of 'path' or 'bytes' but both are None in {value}.") + else: + if is_local_path(path): + image = PIL.Image.open(path) + else: + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL + if source_url.startswith(config.HF_ENDPOINT) + else config.HUB_DATASETS_HFFS_URL + ) + source_url_fields = string_to_dict(source_url, pattern) + token = ( + token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + ) + download_config = DownloadConfig(token=token) + with xopen(path, "rb", download_config=download_config) as f: + bytes_ = BytesIO(f.read()) + image = PIL.Image.open(bytes_) + else: + image = PIL.Image.open(BytesIO(bytes_)) + image.load() # to avoid "Too many open files" errors + if image.getexif().get(PIL.Image.ExifTags.Base.Orientation) is not None: + image = PIL.ImageOps.exif_transpose(image) + if self.mode and self.mode != image.mode: + image = image.convert(self.mode) + return image + + def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]: + """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary.""" + from .features import Value + + return ( + self + if self.decode + else { + "bytes": Value("binary"), + "path": Value("string"), + } + ) + + def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray: + """Cast an Arrow array to the Image arrow storage type. + The Arrow types that can be converted to the Image pyarrow storage type are: + + - `pa.string()` - it must contain the "path" data + - `pa.large_string()` - it must contain the "path" data (will be cast to string if possible) + - `pa.binary()` - it must contain the image bytes + - `pa.struct({"bytes": pa.binary()})` + - `pa.struct({"path": pa.string()})` + - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter + - `pa.list(*)` - it must contain the image array data + + Args: + storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`): + PyArrow array to cast. + + Returns: + `pa.StructArray`: Array in the Image arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if pa.types.is_large_string(storage.type): + try: + storage = storage.cast(pa.string()) + except pa.ArrowInvalid as e: + raise ValueError( + f"Failed to cast large_string to string for Image feature. " + f"This can happen if string values exceed 2GB. " + f"Original error: {e}" + ) from e + if pa.types.is_string(storage.type): + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_large_binary(storage.type): + storage = array_cast( + storage, pa.binary() + ) # this can fail in case of big images, paths should be used instead + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_binary(storage.type): + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_struct(storage.type): + if storage.type.get_field_index("bytes") >= 0: + bytes_array = storage.field("bytes") + else: + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + if storage.type.get_field_index("path") >= 0: + path_array = storage.field("path") + else: + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_list(storage.type): + bytes_array = pa.array( + [encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()], + type=pa.binary(), + ) + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays( + [bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null() + ) + return array_cast(storage, self.pa_type) + + def embed_storage( + self, storage: pa.StructArray, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True + ) -> pa.StructArray: + """Embed image files into the Arrow array. + + Args: + storage (`pa.StructArray`): + PyArrow array to embed. + token_per_repo_id (`dict`, optional): + Dictionary repo_id -> token to fetch the files bytes. + local_files (`bool`, defaults to `True`) + Whether to embed local files data in the array + + + remote_files (`bool`, defaults to `True`) + Whether to embed remote files data in the array. + E.g. files with paths that start with hf:// or https:// + + + + Returns: + `pa.StructArray`: Array in the Image arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if token_per_repo_id is None: + token_per_repo_id = {} + + @no_op_if_value_is_null + def path_to_bytes(path): + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL + ) + source_url_fields = string_to_dict(source_url, pattern) + token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + download_config = DownloadConfig(token=token) + with xopen(path, "rb", download_config=download_config) as f: + return f.read() + + bytes_array = pa.array( + [ + ( + path_to_bytes(x["path"]) + if x["bytes"] is None + and ((local_files and is_local_path(x["path"])) or (remote_files and is_remote_url(x["path"]))) + else x["bytes"] + ) + if x is not None + else None + for x in storage.to_pylist() + ], + type=pa.binary(), + ) + path_array = pa.array( + [ + ( + os.path.basename(path) + if (local_files and is_local_path(path)) or (remote_files and is_remote_url(path)) + else path + ) + if path is not None + else None + for path in storage.field("path").to_pylist() + ], + type=pa.string(), + ) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + return array_cast(storage, self.pa_type) + + +def list_image_compression_formats() -> list[str]: + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + global _IMAGE_COMPRESSION_FORMATS + if _IMAGE_COMPRESSION_FORMATS is None: + PIL.Image.init() + _IMAGE_COMPRESSION_FORMATS = list(set(PIL.Image.OPEN.keys()) & set(PIL.Image.SAVE.keys())) + return _IMAGE_COMPRESSION_FORMATS + + +def image_to_bytes(image: "PIL.Image.Image") -> bytes: + """Convert a PIL Image object to bytes using native compression if possible, otherwise use PNG/TIFF compression.""" + buffer = BytesIO() + if image.format in list_image_compression_formats(): + format = image.format + else: + format = "PNG" if image.mode in ["1", "L", "LA", "RGB", "RGBA"] else "TIFF" + image.save(buffer, format=format) + return buffer.getvalue() + + +def encode_pil_image(image: "PIL.Image.Image") -> dict: + if hasattr(image, "filename") and image.filename != "": + return {"path": image.filename, "bytes": None} + else: + return {"path": None, "bytes": image_to_bytes(image)} + + +def encode_np_array(array: np.ndarray) -> dict: + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + dtype = array.dtype + dtype_byteorder = dtype.byteorder if dtype.byteorder != "=" else _NATIVE_BYTEORDER + dtype_kind = dtype.kind + dtype_itemsize = dtype.itemsize + + dest_dtype = None + + # Multi-channel array case (only np.dtype("|u1") is allowed) + if array.shape[2:]: + if dtype_kind not in ["u", "i"]: + raise TypeError( + f"Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays." + ) + dest_dtype = np.dtype("|u1") + if dtype != dest_dtype: + warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'") + # Exact match + elif dtype in _VALID_IMAGE_ARRAY_DTPYES: + dest_dtype = dtype + else: # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually) + while dtype_itemsize >= 1: + dtype_str = dtype_byteorder + dtype_kind + str(dtype_itemsize) + if np.dtype(dtype_str) in _VALID_IMAGE_ARRAY_DTPYES: + dest_dtype = np.dtype(dtype_str) + warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'") + break + else: + dtype_itemsize //= 2 + if dest_dtype is None: + raise TypeError( + f"Cannot downcast dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}" + ) + + image = PIL.Image.fromarray(array.astype(dest_dtype)) + return {"path": None, "bytes": image_to_bytes(image)} + + +def objects_to_list_of_image_dicts( + objs: Union[list[str], list[dict], list[np.ndarray], list["PIL.Image.Image"]], +) -> list[dict]: + """Encode a list of objects into a format suitable for creating an extension array of type `ImageExtensionType`.""" + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + if objs: + _, obj = first_non_null_value(objs) + if isinstance(obj, str): + return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs] + if isinstance(obj, np.ndarray): + obj_to_image_dict_func = no_op_if_value_is_null(encode_np_array) + return [obj_to_image_dict_func(obj) for obj in objs] + elif isinstance(obj, PIL.Image.Image): + obj_to_image_dict_func = no_op_if_value_is_null(encode_pil_image) + return [obj_to_image_dict_func(obj) for obj in objs] + else: + return objs + else: + return objs diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/nifti.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/nifti.py new file mode 100644 index 0000000000000000000000000000000000000000..1e78cad9e28a10aea73e4de15807da3d75672e6e --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/nifti.py @@ -0,0 +1,351 @@ +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union + +import pyarrow as pa + +from .. import config +from ..download.download_config import DownloadConfig +from ..table import array_cast +from ..utils.file_utils import is_local_path, is_remote_url, xopen +from ..utils.py_utils import no_op_if_value_is_null, string_to_dict + + +if TYPE_CHECKING: + import nibabel as nib + + from .features import FeatureType + +if config.NIBABEL_AVAILABLE: + import nibabel as nib + + class Nifti1ImageWrapper(nib.nifti1.Nifti1Image): + """ + A wrapper around nibabel's Nifti1Image to customize its representation. + """ + + def __init__(self, nifti_image: nib.nifti1.Nifti1Image): + super().__init__( + dataobj=nifti_image.dataobj, + affine=nifti_image.affine, + header=nifti_image.header, + extra=nifti_image.extra, + file_map=nifti_image.file_map, + dtype=nifti_image.get_data_dtype(), + ) + self.nifti_image = nifti_image + + def _repr_html_(self): + from ipyniivue import NiiVue, ShowRender, SliceType, Volume + from IPython.display import display + + bytes_ = self.nifti_image.to_bytes() + nv = NiiVue() + nv.set_slice_type(SliceType.MULTIPLANAR) + nv.opts.multiplanar_show_render = ShowRender.ALWAYS + nv.opts.show_3d_crosshair = True + nv.opts.multiplanar_force_render = True + name = None + if hasattr(self.nifti_image, "file_map"): + if ( + "image" in self.nifti_image.file_map + and getattr(self.nifti_image.file_map["image"], "filename", None) is not None + ): + name = self.nifti_image.file_map["image"].filename + if name is None: + name = "volume.nii.gz" + volume = Volume(name=name, data=bytes_) + nv.load_volumes([volume]) + display(nv) + + +@dataclass +class Nifti: + """ + **Experimental.** + Nifti [`Feature`] to read NIfTI neuroimaging files. + + Input: The Nifti feature accepts as input: + - A `str`: Absolute path to the NIfTI file (i.e. random access is allowed). + - A `pathlib.Path`: path to the NIfTI file (i.e. random access is allowed). + - A `dict` with the keys: + - `path`: String with relative path of the NIfTI file in a dataset repository. + - `bytes`: Bytes of the NIfTI file. + This is useful for archived files with sequential access. + + - A `nibabel` image object (e.g., `nibabel.nifti1.Nifti1Image`). + + Args: + decode (`bool`, defaults to `True`): + Whether to decode the NIfTI data. If `False` a string with the bytes is returned. `decode=False` is not supported when decoding examples. + + Examples: + + ```py + >>> from datasets import Dataset, Nifti + >>> ds = Dataset.from_dict({"nifti": ["path/to/file.nii.gz"]}).cast_column("nifti", Nifti()) + >>> ds.features["nifti"] + Nifti(decode=True, id=None) + >>> ds[0]["nifti"] + + >>> ds = ds.cast_column("nifti", Nifti(decode=False)) + >>> ds[0]["nifti"] + {'bytes': None, + 'path': 'path/to/file.nii.gz'} + ``` + """ + + decode: bool = True + id: Optional[str] = field(default=None, repr=False) + + # Automatically constructed + dtype: ClassVar[str] = "nibabel.nifti1.Nifti1Image" + pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()}) + _type: str = field(default="Nifti", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, value: Union[str, bytes, bytearray, dict, "nib.Nifti1Image"]) -> dict: + """Encode example into a format for Arrow. + + Args: + value (`str`, `bytes`, `nibabel.Nifti1Image` or `dict`): + Data passed as input to Nifti feature. + + Returns: + `dict` with "path" and "bytes" fields + """ + if config.NIBABEL_AVAILABLE: + import nibabel as nib + else: + nib = None + + if isinstance(value, str): + return {"path": value, "bytes": None} + elif isinstance(value, Path): + return {"path": str(value.absolute()), "bytes": None} + elif isinstance(value, (bytes, bytearray)): + return {"path": None, "bytes": value} + elif nib is not None and isinstance(value, nib.spatialimages.SpatialImage): + # nibabel image object - try to get path or convert to bytes + return encode_nibabel_image(value) + elif isinstance(value, dict): + if value.get("path") is not None and os.path.isfile(value["path"]): + # we set "bytes": None to not duplicate the data if they're already available locally + return {"bytes": None, "path": value.get("path")} + elif value.get("bytes") is not None or value.get("path") is not None: + # store the nifti bytes, and path is used to infer the format using the file extension + return {"bytes": value.get("bytes"), "path": value.get("path")} + else: + raise ValueError( + f"A nifti sample should have one of 'path' or 'bytes' but they are missing or None in {value}." + ) + else: + raise ValueError( + f"A nifti sample should be a string, bytes, Path, nibabel image, or dict, but got {type(value)}." + ) + + def decode_example(self, value: dict, token_per_repo_id=None) -> "Nifti1ImageWrapper": + """Decode example NIfTI file into nibabel image object. + + Args: + value (`str` or `dict`): + A string with the absolute NIfTI file path, a dictionary with + keys: + + - `path`: String with absolute or relative NIfTI file path. + - `bytes`: The bytes of the NIfTI file. + + token_per_repo_id (`dict`, *optional*): + To access and decode NIfTI files from private repositories on + the Hub, you can pass a dictionary + repo_id (`str`) -> token (`bool` or `str`). + + Returns: + `nibabel.Nifti1Image` objects + """ + if config.NIBABEL_AVAILABLE: + import nibabel as nib + else: + raise ImportError("To support decoding NIfTI files, please install 'nibabel'.") + + if token_per_repo_id is None: + token_per_repo_id = {} + + path, bytes_ = value["path"], value["bytes"] + if bytes_ is None: + if path is None: + raise ValueError(f"A nifti should have one of 'path' or 'bytes' but both are None in {value}.") + else: + # gzipped files have the structure: 'gzip://T1.nii::' + if path.startswith("gzip://") and is_local_path(path.split("::")[-1]): + path = path.split("::")[-1] + if is_local_path(path): + nifti = nib.load(path) + else: + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL + if source_url.startswith(config.HF_ENDPOINT) + else config.HUB_DATASETS_HFFS_URL + ) + source_url_fields = string_to_dict(source_url, pattern) + token = ( + token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + ) + download_config = DownloadConfig(token=token) + with xopen(path, "rb", download_config=download_config) as f: + nifti = nib.load(f) + else: + import gzip + + if ( + bytes_[:2] == b"\x1f\x8b" + ): # gzip magic number, see https://stackoverflow.com/a/76055284/9534390 or "Magic number" on https://en.wikipedia.org/wiki/Gzip + bytes_ = gzip.decompress(bytes_) + + nifti = nib.Nifti1Image.from_bytes(bytes_) + + return Nifti1ImageWrapper(nifti) + + def embed_storage( + self, storage: pa.StructArray, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True + ) -> pa.StructArray: + """Embed NifTI files into the Arrow array. + + Args: + storage (`pa.StructArray`): + PyArrow array to embed. + token_per_repo_id (`dict`, optional): + Dictionary repo_id -> token to fetch the files bytes. + local_files (`bool`, defaults to `True`) + Whether to embed local files data in the array + + + remote_files (`bool`, defaults to `True`) + Whether to embed remote files data in the array. + E.g. files with paths that start with hf:// or https:// + + + + Returns: + `pa.StructArray`: Array in the NifTI arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if token_per_repo_id is None: + token_per_repo_id = {} + + @no_op_if_value_is_null + def path_to_bytes(path): + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL + ) + source_url_fields = string_to_dict(source_url, pattern) + token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + download_config = DownloadConfig(token=token) + with xopen(path, "rb", download_config=download_config) as f: + return f.read() + + bytes_array = pa.array( + [ + ( + path_to_bytes(x["path"]) + if x["bytes"] is None + and ((local_files and is_local_path(x["path"])) or (remote_files and is_remote_url(x["path"]))) + else x["bytes"] + ) + if x is not None + else None + for x in storage.to_pylist() + ], + type=pa.binary(), + ) + path_array = pa.array( + [ + ( + os.path.basename(path) + if (local_files and is_local_path(path)) or (remote_files and is_remote_url(path)) + else path + ) + if path is not None + else None + for path in storage.field("path").to_pylist() + ], + type=pa.string(), + ) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + return array_cast(storage, self.pa_type) + + def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]: + """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary.""" + from .features import Value + + return ( + self + if self.decode + else { + "bytes": Value("binary"), + "path": Value("string"), + } + ) + + def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.BinaryArray]) -> pa.StructArray: + """Cast an Arrow array to the Nifti arrow storage type. + The Arrow types that can be converted to the Nifti pyarrow storage type are: + + - `pa.string()` - it must contain the "path" data + - `pa.binary()` - it must contain the NIfTI bytes + - `pa.struct({"bytes": pa.binary()})` + - `pa.struct({"path": pa.string()})` + - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter + + Args: + storage (`Union[pa.StringArray, pa.StructArray, pa.BinaryArray]`): + PyArrow array to cast. + + Returns: + `pa.StructArray`: Array in the Nifti arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if pa.types.is_string(storage.type): + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_binary(storage.type): + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_struct(storage.type): + if storage.type.get_field_index("bytes") >= 0: + bytes_array = storage.field("bytes") + else: + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + if storage.type.get_field_index("path") >= 0: + path_array = storage.field("path") + else: + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + return array_cast(storage, self.pa_type) + + +def encode_nibabel_image(img: "nib.Nifti1Image", force_bytes: bool = False) -> dict[str, Optional[Union[str, bytes]]]: + """ + Encode a nibabel image object into a dictionary. + + If the image has an associated file path, returns the path. Otherwise, serializes + the image content into bytes. + + Args: + img: A nibabel image object (e.g., Nifti1Image). + force_bytes: If `True`, always serialize to bytes even if a file path exists. Needed to upload bytes properly. + + Returns: + dict: A dictionary with "path" or "bytes" field. + """ + if hasattr(img, "file_map") and img.file_map is not None and not force_bytes: + filename = img.file_map["image"].filename + return {"path": filename, "bytes": None} + + bytes_data = img.to_bytes() + return {"path": None, "bytes": bytes_data} diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/pdf.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/pdf.py new file mode 100644 index 0000000000000000000000000000000000000000..74c7560d5ffdd4dda9adf7e4f4808b1a70842894 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/pdf.py @@ -0,0 +1,309 @@ +import os +from dataclasses import dataclass, field +from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union + +import pyarrow as pa + +from .. import config +from ..download.download_config import DownloadConfig +from ..table import array_cast +from ..utils.file_utils import is_local_path, is_remote_url, xopen +from ..utils.py_utils import no_op_if_value_is_null, string_to_dict + + +if TYPE_CHECKING: + import pdfplumber + + from .features import FeatureType + + +def pdf_to_bytes(pdf: "pdfplumber.pdf.PDF") -> bytes: + """Convert a pdfplumber.pdf.PDF object to bytes.""" + with BytesIO() as buffer: + for page in pdf.pages: + buffer.write(page.pdf.stream) + return buffer.getvalue() + + +@dataclass +class Pdf: + """ + **Experimental.** + Pdf [`Feature`] to read pdf documents from a pdf file. + + Input: The Pdf feature accepts as input: + - A `str`: Absolute path to the pdf file (i.e. random access is allowed). + - A `pathlib.Path`: path to the pdf file (i.e. random access is allowed). + - A `dict` with the keys: + - `path`: String with relative path of the pdf file in a dataset repository. + - `bytes`: Bytes of the pdf file. + This is useful for archived files with sequential access. + + - A `pdfplumber.pdf.PDF`: pdfplumber pdf object. + + Args: + decode (`bool`, defaults to `True`): + Whether to decode the pdf data. If `False`, + returns the underlying dictionary in the format `{"path": pdf_path, "bytes": pdf_bytes}`. + + Examples: + + ```py + >>> from datasets import Dataset, Pdf + >>> ds = Dataset.from_dict({"pdf": ["path/to/pdf/file.pdf"]}).cast_column("pdf", Pdf()) + >>> ds.features["pdf"] + Pdf(decode=True, id=None) + >>> ds[0]["pdf"] + + >>> ds = ds.cast_column("pdf", Pdf(decode=False)) + >>> ds[0]["pdf"] + {'bytes': None, + 'path': 'path/to/pdf/file.pdf'} + ``` + """ + + decode: bool = True + id: Optional[str] = field(default=None, repr=False) + + # Automatically constructed + dtype: ClassVar[str] = "pdfplumber.pdf.PDF" + pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()}) + _type: str = field(default="Pdf", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, value: Union[str, bytes, bytearray, dict, "pdfplumber.pdf.PDF"]) -> dict: + """Encode example into a format for Arrow. + + Args: + value (`str`, `bytes`, `pdfplumber.pdf.PDF` or `dict`): + Data passed as input to Pdf feature. + + Returns: + `dict` with "path" and "bytes" fields + """ + if config.PDFPLUMBER_AVAILABLE: + import pdfplumber + else: + pdfplumber = None + + if isinstance(value, str): + return {"path": value, "bytes": None} + elif isinstance(value, Path): + return {"path": str(value.absolute()), "bytes": None} + elif isinstance(value, (bytes, bytearray)): + return {"path": None, "bytes": value} + elif pdfplumber is not None and isinstance(value, pdfplumber.pdf.PDF): + # convert the pdfplumber.pdf.PDF to bytes + return encode_pdfplumber_pdf(value) + elif value.get("path") is not None and os.path.isfile(value["path"]): + # we set "bytes": None to not duplicate the data if they're already available locally + return {"bytes": None, "path": value.get("path")} + elif value.get("bytes") is not None or value.get("path") is not None: + # store the pdf bytes, and path is used to infer the pdf format using the file extension + return {"bytes": value.get("bytes"), "path": value.get("path")} + else: + raise ValueError( + f"A pdf sample should have one of 'path' or 'bytes' but they are missing or None in {value}." + ) + + def decode_example(self, value: dict, token_per_repo_id=None) -> "pdfplumber.pdf.PDF": + """Decode example pdf file into pdf data. + + Args: + value (`str` or `dict`): + A string with the absolute pdf file path, a dictionary with + keys: + + - `path`: String with absolute or relative pdf file path. + - `bytes`: The bytes of the pdf file. + + token_per_repo_id (`dict`, *optional*): + To access and decode pdf files from private repositories on + the Hub, you can pass a dictionary + repo_id (`str`) -> token (`bool` or `str`). + + Returns: + `pdfplumber.pdf.PDF` + """ + if not self.decode: + raise RuntimeError("Decoding is disabled for this feature. Please use Pdf(decode=True) instead.") + + if config.PDFPLUMBER_AVAILABLE: + import pdfplumber + else: + raise ImportError("To support decoding pdfs, please install 'pdfplumber'.") + + if token_per_repo_id is None: + token_per_repo_id = {} + + path, bytes_ = value["path"], value["bytes"] + if bytes_ is None: + if path is None: + raise ValueError(f"A pdf should have one of 'path' or 'bytes' but both are None in {value}.") + else: + if is_local_path(path): + pdf = pdfplumber.open(path) + else: + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL + if source_url.startswith(config.HF_ENDPOINT) + else config.HUB_DATASETS_HFFS_URL + ) + try: + repo_id = string_to_dict(source_url, pattern)["repo_id"] + token = token_per_repo_id.get(repo_id) + except ValueError: + token = None + download_config = DownloadConfig(token=token) + f = xopen(path, "rb", download_config=download_config) + return pdfplumber.open(f) + else: + with pdfplumber.open(BytesIO(bytes_)) as p: + pdf = p + + return pdf + + def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]: + """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary.""" + from .features import Value + + return ( + self + if self.decode + else { + "bytes": Value("binary"), + "path": Value("string"), + } + ) + + def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray: + """Cast an Arrow array to the Pdf arrow storage type. + The Arrow types that can be converted to the Pdf pyarrow storage type are: + + - `pa.string()` - it must contain the "path" data + - `pa.binary()` - it must contain the image bytes + - `pa.struct({"bytes": pa.binary()})` + - `pa.struct({"path": pa.string()})` + - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter + - `pa.list(*)` - it must contain the pdf array data + + Args: + storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`): + PyArrow array to cast. + + Returns: + `pa.StructArray`: Array in the Pdf arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if pa.types.is_string(storage.type): + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_binary(storage.type): + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_struct(storage.type): + if storage.type.get_field_index("bytes") >= 0: + bytes_array = storage.field("bytes") + else: + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + if storage.type.get_field_index("path") >= 0: + path_array = storage.field("path") + else: + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + return array_cast(storage, self.pa_type) + + def embed_storage( + self, storage: pa.StructArray, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True + ) -> pa.StructArray: + """Embed PDF files into the Arrow array. + + Args: + storage (`pa.StructArray`): + PyArrow array to embed. + token_per_repo_id (`dict`, optional): + Dictionary repo_id -> token to fetch the files bytes. + local_files (`bool`, defaults to `True`) + Whether to embed local files data in the array + + + remote_files (`bool`, defaults to `True`) + Whether to embed remote files data in the array. + E.g. files with paths that start with hf:// or https:// + + + + Returns: + `pa.StructArray`: Array in the PDF arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if token_per_repo_id is None: + token_per_repo_id = {} + + @no_op_if_value_is_null + def path_to_bytes(path): + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL + ) + source_url_fields = string_to_dict(source_url, pattern) + token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + download_config = DownloadConfig(token=token) + with xopen(path, "rb", download_config=download_config) as f: + return f.read() + + bytes_array = pa.array( + [ + ( + path_to_bytes(x["path"]) + if x["bytes"] is None + and ((local_files and is_local_path(x["path"])) or (remote_files and is_remote_url(x["path"]))) + else x["bytes"] + ) + if x is not None + else None + for x in storage.to_pylist() + ], + type=pa.binary(), + ) + path_array = pa.array( + [ + ( + os.path.basename(path) + if (local_files and is_local_path(path)) or (remote_files and is_remote_url(path)) + else path + ) + if path is not None + else None + for path in storage.field("path").to_pylist() + ], + type=pa.string(), + ) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + return array_cast(storage, self.pa_type) + + +def encode_pdfplumber_pdf(pdf: "pdfplumber.pdf.PDF") -> dict: + """ + Encode a pdfplumber.pdf.PDF object into a dictionary. + + If the PDF has an associated file path, returns the path. Otherwise, serializes + the PDF content into bytes. + + Args: + pdf (pdfplumber.pdf.PDF): A pdfplumber PDF object. + + Returns: + dict: A dictionary with "path" or "bytes" field. + """ + if hasattr(pdf, "stream") and hasattr(pdf.stream, "name") and pdf.stream.name: + # Return the path if the PDF has an associated file path + return {"path": pdf.stream.name, "bytes": None} + else: + # Convert the PDF to bytes if no path is available + return {"path": None, "bytes": pdf_to_bytes(pdf)} diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/translation.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/translation.py new file mode 100644 index 0000000000000000000000000000000000000000..9bee3aa2c11f26edbe0d17138a71fde96e3d3541 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/translation.py @@ -0,0 +1,129 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union + +import pyarrow as pa + + +if TYPE_CHECKING: + from .features import FeatureType + + +@dataclass +class Translation: + """`Feature` for translations with fixed languages per example. + Here for compatibility with tfds. + + Args: + languages (`dict`): + A dictionary for each example mapping string language codes to string translations. + + Example: + + ```python + >>> # At construction time: + >>> datasets.features.Translation(languages=['en', 'fr', 'de']) + >>> # During data generation: + >>> yield { + ... 'en': 'the cat', + ... 'fr': 'le chat', + ... 'de': 'die katze' + ... } + ``` + """ + + languages: list[str] + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + dtype: ClassVar[str] = "dict" + pa_type: ClassVar[Any] = None + _type: str = field(default="Translation", init=False, repr=False) + + def __call__(self): + return pa.struct({lang: pa.string() for lang in sorted(self.languages)}) + + def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]: + """Flatten the Translation feature into a dictionary.""" + from .features import Value + + return {k: Value("string") for k in sorted(self.languages)} + + +@dataclass +class TranslationVariableLanguages: + """`Feature` for translations with variable languages per example. + Here for compatibility with tfds. + + Args: + languages (`dict`): + A dictionary for each example mapping string language codes to one or more string translations. + The languages present may vary from example to example. + + Returns: + - `language` or `translation` (variable-length 1D `tf.Tensor` of `tf.string`): + Language codes sorted in ascending order or plain text translations, sorted to align with language codes. + + Example: + + ```python + >>> # At construction time: + >>> datasets.features.TranslationVariableLanguages(languages=['en', 'fr', 'de']) + >>> # During data generation: + >>> yield { + ... 'en': 'the cat', + ... 'fr': ['le chat', 'la chatte,'] + ... 'de': 'die katze' + ... } + >>> # Tensor returned : + >>> { + ... 'language': ['en', 'de', 'fr', 'fr'], + ... 'translation': ['the cat', 'die katze', 'la chatte', 'le chat'], + ... } + ``` + """ + + languages: Optional[list] = None + num_languages: Optional[int] = None + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + dtype: ClassVar[str] = "dict" + pa_type: ClassVar[Any] = None + _type: str = field(default="TranslationVariableLanguages", init=False, repr=False) + + def __post_init__(self): + self.languages = sorted(set(self.languages)) if self.languages else None + self.num_languages = len(self.languages) if self.languages else None + + def __call__(self): + return pa.struct({"language": pa.list_(pa.string()), "translation": pa.list_(pa.string())}) + + def encode_example(self, translation_dict): + lang_set = set(self.languages) + if set(translation_dict) == {"language", "translation"}: + return translation_dict + elif self.languages and set(translation_dict) - lang_set: + raise ValueError( + f"Some languages in example ({', '.join(sorted(set(translation_dict) - lang_set))}) are not in valid set ({', '.join(lang_set)})." + ) + + # Convert dictionary into tuples, splitting out cases where there are + # multiple translations for a single language. + translation_tuples = [] + for lang, text in translation_dict.items(): + if isinstance(text, str): + translation_tuples.append((lang, text)) + else: + translation_tuples.extend([(lang, el) for el in text]) + + # Ensure translations are in ascending order by language code. + languages, translations = zip(*sorted(translation_tuples)) + + return {"language": languages, "translation": translations} + + def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]: + """Flatten the TranslationVariableLanguages feature into a dictionary.""" + from .features import List, Value + + return { + "language": List(Value("string")), + "translation": List(Value("string")), + } diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/video.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/video.py new file mode 100644 index 0000000000000000000000000000000000000000..8522c2cb1225aff2f8bb21a28a8a22570c8650bd --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/features/video.py @@ -0,0 +1,415 @@ +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, TypedDict, Union + +import numpy as np +import pyarrow as pa + +from .. import config +from ..download.download_config import DownloadConfig +from ..table import array_cast +from ..utils.file_utils import is_local_path, is_remote_url, xopen +from ..utils.py_utils import no_op_if_value_is_null, string_to_dict + + +if TYPE_CHECKING: + import torch + from torchcodec.decoders import VideoDecoder + + from .features import FeatureType + + +class Example(TypedDict): + path: Optional[str] + bytes: Optional[bytes] + + +@dataclass +class Video: + """ + Video [`Feature`] to read video data from a video file. + + Input: The Video feature accepts as input: + - A `str`: Absolute path to the video file (i.e. random access is allowed). + - A `pathlib.Path`: path to the video file (i.e. random access is allowed). + - A `dict` with the keys: + + - `path`: String with relative path of the video file in a dataset repository. + - `bytes`: Bytes of the video file. + + This is useful for parquet or webdataset files which embed video files. + + - A `torchcodec.decoders.VideoDecoder`: torchcodec video decoder object. + + Output: The Video features output data as `torchcodec.decoders.VideoDecoder` objects. + + Args: + decode (`bool`, defaults to `True`): + Whether to decode the video data. If `False`, + returns the underlying dictionary in the format `{"path": video_path, "bytes": video_bytes}`. + stream_index (`int`, *optional*): + The streaming index to use from the file. If `None` defaults to the "best" index. + dimension_order (`str`, defaults to `NCHW`): + The dimension order of the decoded frames. + where N is the batch size, C is the number of channels, + H is the height, and W is the width of the frames. + num_ffmpeg_threads (`int`, defaults to `1`): + The number of threads to use for decoding the video. (Recommended to keep this at 1) + device (`str` or `torch.device`, defaults to `cpu`): + The device to use for decoding the video. + seek_mode (`str`, defaults to `exact`): + Determines if frame access will be “exact” or “approximate”. + Exact guarantees that requesting frame i will always return frame i, but doing so requires an initial scan of the file. + Approximate is faster as it avoids scanning the file, but less accurate as it uses the file's metadata to calculate where i probably is. + read more [here](https://docs.pytorch.org/torchcodec/stable/generated_examples/approximate_mode.html#sphx-glr-generated-examples-approximate-mode-py) + + Examples: + + ```py + >>> from datasets import Dataset, Video + >>> ds = Dataset.from_dict({"video":["path/to/Screen Recording.mov"]}).cast_column("video", Video()) + >>> ds.features["video"] + Video(decode=True, id=None) + >>> ds[0]["video"] + + >>> video = ds[0]["video"] + >>> video.get_frames_in_range(0, 10) + FrameBatch: + data (shape): torch.Size([10, 3, 50, 66]) + pts_seconds: tensor([0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333, 0.4333, + 0.4333], dtype=torch.float64) + duration_seconds: tensor([0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, 0.0167, + 0.0167], dtype=torch.float64) + >>> ds.cast_column('video', Video(decode=False))[0]["video] + {'bytes': None, + 'path': 'path/to/Screen Recording.mov'} + ``` + """ + + decode: bool = True + stream_index: Optional[int] = None + dimension_order: Literal["NCHW", "NHWC"] = "NCHW" + num_ffmpeg_threads: int = 1 + device: Optional[Union[str, "torch.device"]] = "cpu" + seek_mode: Literal["exact", "approximate"] = "exact" + id: Optional[str] = field(default=None, repr=False) + # Automatically constructed + dtype: ClassVar[str] = "torchcodec.decoders.VideoDecoder" + pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()}) + _type: str = field(default="Video", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, value: Union[str, bytes, bytearray, Example, np.ndarray, "VideoDecoder"]) -> Example: + """Encode example into a format for Arrow. + + Args: + value (`str`, `np.ndarray`, `bytes`, `bytearray`, `VideoDecoder` or `dict`): + Data passed as input to Video feature. + + Returns: + `dict` with "path" and "bytes" fields + """ + if value is None: + raise ValueError("value must be provided") + + if config.TORCHCODEC_AVAILABLE: + from torchcodec.decoders import VideoDecoder + else: + VideoDecoder = None + + if isinstance(value, list): + value = np.array(value) + + if isinstance(value, str): + return {"path": value, "bytes": None} + elif isinstance(value, Path): + return {"path": str(value.absolute()), "bytes": None} + elif isinstance(value, (bytes, bytearray)): + return {"path": None, "bytes": value} + elif isinstance(value, np.ndarray): + # convert the video array to bytes + return encode_np_array(value) + elif VideoDecoder is not None and isinstance(value, VideoDecoder): + # convert the torchcodec video decoder to bytes + return encode_torchcodec_video(value) + elif isinstance(value, dict): + path, bytes_ = value.get("path"), value.get("bytes") + if path is not None and os.path.isfile(path): + # we set "bytes": None to not duplicate the data if they're already available locally + return {"bytes": None, "path": path} + elif bytes_ is not None or path is not None: + # store the video bytes, and path is used to infer the video format using the file extension + return {"bytes": bytes_, "path": path} + else: + raise ValueError( + f"A video sample should have one of 'path' or 'bytes' but they are missing or None in {value}." + ) + else: + raise TypeError(f"Unsupported encode_example type: {type(value)}") + + def decode_example( + self, + value: Union[str, Example], + token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None, + ) -> "VideoDecoder": + """Decode example video file into video data. + + Args: + value (`str` or `dict`): + A string with the absolute video file path, a dictionary with + keys: + + - `path`: String with absolute or relative video file path. + - `bytes`: The bytes of the video file. + token_per_repo_id (`dict`, *optional*): + To access and decode + video files from private repositories on the Hub, you can pass + a dictionary repo_id (`str`) -> token (`bool` or `str`). + + Returns: + `torchcodec.decoders.VideoDecoder` + """ + if not self.decode: + raise RuntimeError("Decoding is disabled for this feature. Please use Video(decode=True) instead.") + + if config.TORCHCODEC_AVAILABLE: + from torchcodec.decoders import VideoDecoder + + else: + raise ImportError("To support decoding videos, please install 'torchcodec'.") + + if token_per_repo_id is None: + token_per_repo_id = {} + + if isinstance(value, str): + path, bytes_ = value, None + else: + path, bytes_ = value["path"], value["bytes"] + + if bytes_ is None: + if path is None: + raise ValueError(f"A video should have one of 'path' or 'bytes' but both are None in {value}.") + elif is_local_path(path): + video = VideoDecoder( + path, + stream_index=self.stream_index, + dimension_order=self.dimension_order, + num_ffmpeg_threads=self.num_ffmpeg_threads, + device=self.device, + seek_mode=self.seek_mode, + ) + else: + video = hf_video_reader( + path, + token_per_repo_id=token_per_repo_id, + dimension_order=self.dimension_order, + num_ffmpeg_threads=self.num_ffmpeg_threads, + device=self.device, + seek_mode=self.seek_mode, + ) + else: + video = VideoDecoder( + bytes_, + stream_index=self.stream_index, + dimension_order=self.dimension_order, + num_ffmpeg_threads=self.num_ffmpeg_threads, + device=self.device, + seek_mode=self.seek_mode, + ) + video._hf_encoded = {"path": path, "bytes": bytes_} + video.metadata.path = path + return video + + def flatten(self) -> Union["FeatureType", dict[str, "FeatureType"]]: + """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary.""" + from .features import Value + + return ( + self + if self.decode + else { + "bytes": Value("binary"), + "path": Value("string"), + } + ) + + def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray: + """Cast an Arrow array to the Video arrow storage type. + The Arrow types that can be converted to the Video pyarrow storage type are: + + - `pa.string()` - it must contain the "path" data + - `pa.binary()` - it must contain the video bytes + - `pa.struct({"bytes": pa.binary()})` + - `pa.struct({"path": pa.string()})` + - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter + - `pa.list(*)` - it must contain the video array data + + Args: + storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`): + PyArrow array to cast. + + Returns: + `pa.StructArray`: Array in the Video arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if pa.types.is_string(storage.type): + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_large_binary(storage.type): + storage = array_cast( + storage, pa.binary() + ) # this can fail in case of big videos, paths should be used instead + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_binary(storage.type): + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_struct(storage.type): + if storage.type.get_field_index("bytes") >= 0: + bytes_array = storage.field("bytes") + else: + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + if storage.type.get_field_index("path") >= 0: + path_array = storage.field("path") + else: + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_list(storage.type): + bytes_array = pa.array( + [encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()], + type=pa.binary(), + ) + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays( + [bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null() + ) + return array_cast(storage, self.pa_type) + + def embed_storage( + self, storage: pa.StructArray, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True + ) -> pa.StructArray: + """Embed image files into the Arrow array. + + Args: + storage (`pa.StructArray`): + PyArrow array to embed. + token_per_repo_id (`dict`, optional): + Dictionary repo_id -> token to fetch the files bytes. + local_files (`bool`, defaults to `True`) + Whether to embed local files data in the array + + + remote_files (`bool`, defaults to `True`) + Whether to embed remote files data in the array. + E.g. files with paths that start with hf:// or https:// + + + + Returns: + `pa.StructArray`: Array in the Video arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if token_per_repo_id is None: + token_per_repo_id = {} + + @no_op_if_value_is_null + def path_to_bytes(path): + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL + ) + source_url_fields = string_to_dict(source_url, pattern) + token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + download_config = DownloadConfig(token=token) + with xopen(path, "rb", download_config=download_config) as f: + return f.read() + + bytes_array = pa.array( + [ + ( + path_to_bytes(x["path"]) + if x["bytes"] is None + and ((local_files and is_local_path(x["path"])) or (remote_files and is_remote_url(x["path"]))) + else x["bytes"] + ) + if x is not None + else None + for x in storage.to_pylist() + ], + type=pa.binary(), + ) + path_array = pa.array( + [ + ( + os.path.basename(path) + if (local_files and is_local_path(path)) or (remote_files and is_remote_url(path)) + else path + ) + if path is not None + else None + for path in storage.field("path").to_pylist() + ], + type=pa.string(), + ) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + return array_cast(storage, self.pa_type) + + +def video_to_bytes(video: "VideoDecoder") -> bytes: + """Convert a torchcodec Video object to bytes using native compression if possible""" + raise NotImplementedError() + + +def encode_torchcodec_video(video: "VideoDecoder") -> Example: + if hasattr(video, "_hf_encoded"): + return video._hf_encoded + else: + raise NotImplementedError( + "Encoding a VideoDecoder that doesn't come from datasets.Video.decode() is not implemented" + ) + + +def encode_np_array(array: np.ndarray) -> Example: + raise NotImplementedError() + + +# No monkey patch needed! +# 1. store the encoded video data {"path": ..., "bytes": ...} in `video._hf_encoded`` +# 2. add support for hf:// files + + +def hf_video_reader( + path: str, + token_per_repo_id: Optional[dict[str, Union[bool, str]]] = None, + stream: str = "video", + dimension_order: Literal["NCHW", "NHWC"] = "NCHW", + num_ffmpeg_threads: int = 1, + device: Optional[Union[str, "torch.device"]] = "cpu", + seek_mode: Literal["exact", "approximate"] = "exact", +) -> "VideoDecoder": + from torchcodec.decoders import VideoDecoder + + # Load the file from HF + if token_per_repo_id is None: + token_per_repo_id = {} + source_url = path.split("::")[-1] + pattern = config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL + source_url_fields = string_to_dict(source_url, pattern) + token = token_per_repo_id.get(source_url_fields["repo_id"]) if source_url_fields is not None else None + download_config = DownloadConfig(token=token) + f = xopen(path, "rb", download_config=download_config) + + # Instantiate the VideoDecoder + stream_id = 0 if len(stream.split(":")) == 1 else int(stream.split(":")[1]) + vd = VideoDecoder( + f, + stream_index=stream_id, + dimension_order=dimension_order, + num_ffmpeg_threads=num_ffmpeg_threads, + device=device, + seek_mode=seek_mode, + ) + return vd diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/filesystems/compression.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/filesystems/compression.py new file mode 100644 index 0000000000000000000000000000000000000000..237b06d75008f8354a4b43d19f6a023e02991393 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/filesystems/compression.py @@ -0,0 +1,127 @@ +import os +from functools import partial +from typing import Optional + +import fsspec +from fsspec.archive import AbstractArchiveFileSystem + + +class BaseCompressedFileFileSystem(AbstractArchiveFileSystem): + """Read contents of compressed file as a filesystem with one file inside.""" + + root_marker = "" + protocol: str = ( + None # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz + ) + compression: str = None # compression type in fsspec. ex: "gzip" + extensions: list[str] = None # extensions of the filename to strip. ex: ".gz" to get file.txt from file.txt.gz + + def __init__( + self, fo: str = "", target_protocol: Optional[str] = None, target_options: Optional[dict] = None, **kwargs + ): + """ + The compressed file system can be instantiated from any compressed file. + It reads the contents of compressed file as a filesystem with one file inside, as if it was an archive. + + The single file inside the filesystem is named after the compresssed file, + without the compression extension at the end of the filename. + + Args: + fo (:obj:``str``): Path to compressed file. Will fetch file using ``fsspec.open()`` + mode (:obj:``str``): Currently, only 'rb' accepted + target_protocol(:obj:``str``, optional): To override the FS protocol inferred from a URL. + target_options (:obj:``dict``, optional): Kwargs passed when instantiating the target FS. + """ + super().__init__(self, **kwargs) + self.fo = fo.__fspath__() if hasattr(fo, "__fspath__") else fo + # always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode + self._open_with_fsspec = partial( + fsspec.open, + self.fo, + mode="rb", + protocol=target_protocol, + compression=self.compression, + client_kwargs={ + "requote_redirect_url": False, # see https://github.com/huggingface/datasets/pull/5459 + "trust_env": True, # Enable reading proxy env variables. + **(target_options or {}).pop("client_kwargs", {}), # To avoid issues if it was already passed. + }, + **(target_options or {}), + ) + self.compressed_name = os.path.basename(self.fo.split("::")[0]) + self.uncompressed_name = ( + self.compressed_name[: self.compressed_name.rindex(".")] + if "." in self.compressed_name + else self.compressed_name + ) + self.dir_cache = None + + @classmethod + def _strip_protocol(cls, path): + # compressed file paths are always relative to the archive root + return super()._strip_protocol(path).lstrip("/") + + def _get_dirs(self): + if self.dir_cache is None: + f = {**self._open_with_fsspec().fs.info(self.fo), "name": self.uncompressed_name} + self.dir_cache = {f["name"]: f} + + def cat(self, path: str): + with self._open_with_fsspec().open() as f: + return f.read() + + def _open( + self, + path: str, + mode: str = "rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + path = self._strip_protocol(path) + if mode != "rb": + raise ValueError(f"Tried to read with mode {mode} on file {self.fo} opened with mode 'rb'") + return self._open_with_fsspec().open() + + +class Bz2FileSystem(BaseCompressedFileFileSystem): + """Read contents of BZ2 file as a filesystem with one file inside.""" + + protocol = "bz2" + compression = "bz2" + extensions = [".bz2"] + + +class GzipFileSystem(BaseCompressedFileFileSystem): + """Read contents of GZIP file as a filesystem with one file inside.""" + + protocol = "gzip" + compression = "gzip" + extensions = [".gz", ".gzip"] + + +class Lz4FileSystem(BaseCompressedFileFileSystem): + """Read contents of LZ4 file as a filesystem with one file inside.""" + + protocol = "lz4" + compression = "lz4" + extensions = [".lz4"] + + +class XzFileSystem(BaseCompressedFileFileSystem): + """Read contents of .xz (LZMA) file as a filesystem with one file inside.""" + + protocol = "xz" + compression = "xz" + extensions = [".xz"] + + +class ZstdFileSystem(BaseCompressedFileFileSystem): + """ + Read contents of .zstd file as a filesystem with one file inside. + """ + + protocol = "zstd" + compression = "zstd" + extensions = [".zst", ".zstd"] diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/fingerprint.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/fingerprint.py new file mode 100644 index 0000000000000000000000000000000000000000..13b801621bb5945c61c34f615e9dfa5ab70234ae --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/fingerprint.py @@ -0,0 +1,480 @@ +import inspect +import os +import random +import shutil +import tempfile +import weakref +from functools import wraps +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Optional, Union + +import numpy as np +import xxhash + +from . import config +from .naming import INVALID_WINDOWS_CHARACTERS_IN_PATH +from .utils._dill import dumps +from .utils.logging import get_logger + + +if TYPE_CHECKING: + from .arrow_dataset import Dataset + + +logger = get_logger(__name__) + + +# Fingerprinting allows to have one deterministic fingerprint per dataset state. +# A dataset fingerprint is updated after each transform. +# Re-running the same transforms on a dataset in a different session results in the same fingerprint. +# This is possible thanks to a custom hashing function that works with most python objects. + +# Fingerprinting is the main mechanism that enables caching. +# The caching mechanism allows to reload an existing cache file if it's already been computed. + + +################# +# Caching +################# + +_CACHING_ENABLED = True +_TEMP_DIR_FOR_TEMP_CACHE_FILES: Optional["_TempCacheDir"] = None +_DATASETS_WITH_TABLE_IN_TEMP_DIR: Optional[weakref.WeakSet] = None + + +class _TempCacheDir: + """ + A temporary directory for storing cached Arrow files with a cleanup that frees references to the Arrow files + before deleting the directory itself to avoid permission errors on Windows. + """ + + def __init__(self): + # Check if TMPDIR is set and handle the case where it doesn't exist + tmpdir = os.environ.get("TMPDIR") or os.environ.get("TEMP") or os.environ.get("TMP") + # Normalize the path to handle any path resolution issues + if tmpdir: + tmpdir = os.path.normpath(tmpdir) + if not os.path.exists(tmpdir): + # Auto-create the directory if it doesn't exist + # This prevents tempfile from silently falling back to /tmp + try: + os.makedirs(tmpdir, exist_ok=True) + logger.info(f"Created TMPDIR directory: {tmpdir}") + except OSError as e: + raise OSError( + f"TMPDIR is set to '{tmpdir}' but the directory does not exist and could not be created: {e}. " + "Please create it manually or unset TMPDIR to fall back to the default temporary directory." + ) from e + # If tmpdir exists, verify it's actually a directory and writable + elif not os.path.isdir(tmpdir): + raise OSError( + f"TMPDIR is set to '{tmpdir}' but it is not a directory. " + "Please point TMPDIR to a writable directory or unset it to fall back to the default temporary directory." + ) + + # Explicitly pass the directory to mkdtemp to ensure TMPDIR is respected + # This works even if tempfile.gettempdir() was already called and cached + # Pass dir=None if tmpdir is None to use default temp directory + self.name = tempfile.mkdtemp(prefix=config.TEMP_CACHE_DIR_PREFIX, dir=tmpdir) + self._finalizer = weakref.finalize(self, self._cleanup) + + def _cleanup(self): + for dset in get_datasets_with_cache_file_in_temp_dir(): + dset.__del__() + if os.path.exists(self.name): + try: + shutil.rmtree(self.name) + except Exception as e: + raise OSError( + f"An error occurred while trying to delete temporary cache directory {self.name}. Please delete it manually." + ) from e + + def cleanup(self): + if self._finalizer.detach(): + self._cleanup() + + +def maybe_register_dataset_for_temp_dir_deletion(dataset): + """ + This function registers the datasets that have cache files in _TEMP_DIR_FOR_TEMP_CACHE_FILES in order + to properly delete them before deleting the temporary directory. + The temporary directory _TEMP_DIR_FOR_TEMP_CACHE_FILES is used when caching is disabled. + """ + if _TEMP_DIR_FOR_TEMP_CACHE_FILES is None: + return + + global _DATASETS_WITH_TABLE_IN_TEMP_DIR + if _DATASETS_WITH_TABLE_IN_TEMP_DIR is None: + _DATASETS_WITH_TABLE_IN_TEMP_DIR = weakref.WeakSet() + if any( + Path(_TEMP_DIR_FOR_TEMP_CACHE_FILES.name) in Path(cache_file["filename"]).parents + for cache_file in dataset.cache_files + ): + _DATASETS_WITH_TABLE_IN_TEMP_DIR.add(dataset) + + +def get_datasets_with_cache_file_in_temp_dir(): + return list(_DATASETS_WITH_TABLE_IN_TEMP_DIR) if _DATASETS_WITH_TABLE_IN_TEMP_DIR is not None else [] + + +def enable_caching(): + """ + When applying transforms on a dataset, the data are stored in cache files. + The caching mechanism allows to reload an existing cache file if it's already been computed. + + Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated + after each transform. + + If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets. + More precisely, if the caching is disabled: + - cache files are always recreated + - cache files are written to a temporary directory that is deleted when session closes + - cache files are named using a random hash instead of the dataset fingerprint + - use [`~datasets.Dataset.save_to_disk`] to save a transformed dataset or it will be deleted when session closes + - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use + the `download_mode` parameter in [`~datasets.load_dataset`]. + """ + global _CACHING_ENABLED + _CACHING_ENABLED = True + + +def disable_caching(): + """ + When applying transforms on a dataset, the data are stored in cache files. + The caching mechanism allows to reload an existing cache file if it's already been computed. + + Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated + after each transform. + + If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets. + More precisely, if the caching is disabled: + - cache files are always recreated + - cache files are written to a temporary directory that is deleted when session closes + - cache files are named using a random hash instead of the dataset fingerprint + - use [`~datasets.Dataset.save_to_disk`] to save a transformed dataset or it will be deleted when session closes + - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use + the `download_mode` parameter in [`~datasets.load_dataset`]. + """ + global _CACHING_ENABLED + _CACHING_ENABLED = False + + +def is_caching_enabled() -> bool: + """ + When applying transforms on a dataset, the data are stored in cache files. + The caching mechanism allows to reload an existing cache file if it's already been computed. + + Reloading a dataset is possible since the cache files are named using the dataset fingerprint, which is updated + after each transform. + + If disabled, the library will no longer reload cached datasets files when applying transforms to the datasets. + More precisely, if the caching is disabled: + - cache files are always recreated + - cache files are written to a temporary directory that is deleted when session closes + - cache files are named using a random hash instead of the dataset fingerprint + - use [`~datasets.Dataset.save_to_disk`]] to save a transformed dataset or it will be deleted when session closes + - caching doesn't affect [`~datasets.load_dataset`]. If you want to regenerate a dataset from scratch you should use + the `download_mode` parameter in [`~datasets.load_dataset`]. + """ + global _CACHING_ENABLED + return bool(_CACHING_ENABLED) + + +def get_temporary_cache_files_directory() -> str: + """Return a directory that is deleted when session closes.""" + global _TEMP_DIR_FOR_TEMP_CACHE_FILES + if _TEMP_DIR_FOR_TEMP_CACHE_FILES is None: + _TEMP_DIR_FOR_TEMP_CACHE_FILES = _TempCacheDir() + return _TEMP_DIR_FOR_TEMP_CACHE_FILES.name + + +################# +# Hashing +################# + + +class Hasher: + """Hasher that accepts python objects as inputs.""" + + dispatch: dict = {} + + def __init__(self): + self.m = xxhash.xxh64() + + @classmethod + def hash_bytes(cls, value: Union[bytes, list[bytes]]) -> str: + value = [value] if isinstance(value, bytes) else value + m = xxhash.xxh64() + for x in value: + m.update(x) + return m.hexdigest() + + @classmethod + def hash(cls, value: Any) -> str: + return cls.hash_bytes(dumps(value)) + + def update(self, value: Any) -> None: + header_for_update = f"=={type(value)}==" + value_for_update = self.hash(value) + self.m.update(header_for_update.encode("utf8")) + self.m.update(value_for_update.encode("utf-8")) + + def hexdigest(self) -> str: + return self.m.hexdigest() + + +################# +# Fingerprinting +################# + +fingerprint_rng = random.Random() +# we show a warning only once when fingerprinting fails to avoid spam +fingerprint_warnings: dict[str, bool] = {} + + +def generate_fingerprint(dataset: "Dataset") -> str: + state = dataset.__dict__ + hasher = Hasher() + for key in sorted(state): + if key == "_fingerprint": + continue + hasher.update(key) + hasher.update(state[key]) + # hash data files last modification timestamps as well + for cache_file in dataset.cache_files: + hasher.update(os.path.getmtime(cache_file["filename"])) + return hasher.hexdigest() + + +def generate_random_fingerprint(nbits: int = 64) -> str: + return f"{fingerprint_rng.getrandbits(nbits):0{nbits // 4}x}" + + +def update_fingerprint(fingerprint, transform, transform_args): + global fingerprint_warnings + hasher = Hasher() + hasher.update(fingerprint) + try: + hasher.update(transform) + except: # noqa various errors might raise here from pickle or dill + if _CACHING_ENABLED: + if not fingerprint_warnings.get("update_fingerprint_transform_hash_failed", False): + logger.warning( + f"Transform {transform} couldn't be hashed properly, a random hash was used instead. " + "Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. " + "If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. " + "This warning is only shown once. Subsequent hashing failures won't be shown." + ) + fingerprint_warnings["update_fingerprint_transform_hash_failed"] = True + else: + logger.info(f"Transform {transform} couldn't be hashed properly, a random hash was used instead.") + else: + logger.info( + f"Transform {transform} couldn't be hashed properly, a random hash was used instead. This doesn't affect caching since it's disabled." + ) + + return generate_random_fingerprint() + for key in sorted(transform_args): + hasher.update(key) + try: + hasher.update(transform_args[key]) + except: # noqa various errors might raise here from pickle or dill + if _CACHING_ENABLED: + if not fingerprint_warnings.get("update_fingerprint_transform_hash_failed", False): + logger.warning( + f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead. " + "Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. " + "If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. " + "This warning is only shown once. Subsequent hashing failures won't be shown." + ) + fingerprint_warnings["update_fingerprint_transform_hash_failed"] = True + else: + logger.info( + f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead." + ) + else: + logger.info( + f"Parameter '{key}'={transform_args[key]} of the transform {transform} couldn't be hashed properly, a random hash was used instead. This doesn't affect caching since it's disabled." + ) + return generate_random_fingerprint() + return hasher.hexdigest() + + +def validate_fingerprint(fingerprint: str, max_length=64): + """ + Make sure the fingerprint is a non-empty string that is not longer that max_length=64 by default, + so that the fingerprint can be used to name cache files without issues. + """ + if not isinstance(fingerprint, str) or not fingerprint: + raise ValueError(f"Invalid fingerprint '{fingerprint}': it should be a non-empty string.") + for invalid_char in INVALID_WINDOWS_CHARACTERS_IN_PATH: + if invalid_char in fingerprint: + raise ValueError( + f"Invalid fingerprint. Bad characters from black list '{INVALID_WINDOWS_CHARACTERS_IN_PATH}' found in '{fingerprint}'. " + f"They could create issues when creating cache files." + ) + if len(fingerprint) > max_length: + raise ValueError( + f"Invalid fingerprint. Maximum lenth is {max_length} but '{fingerprint}' has length {len(fingerprint)}." + "It could create issues when creating cache files." + ) + + +def format_transform_for_fingerprint(func: Callable, version: Optional[str] = None) -> str: + """ + Format a transform to the format that will be used to update the fingerprint. + """ + transform = f"{func.__module__}.{func.__qualname__}" + if version is not None: + transform += f"@{version}" + return transform + + +def format_kwargs_for_fingerprint( + func: Callable, + args: tuple, + kwargs: dict[str, Any], + use_kwargs: Optional[list[str]] = None, + ignore_kwargs: Optional[list[str]] = None, + randomized_function: bool = False, +) -> dict[str, Any]: + """ + Format the kwargs of a transform to the format that will be used to update the fingerprint. + """ + kwargs_for_fingerprint = kwargs.copy() + if args: + params = [p.name for p in inspect.signature(func).parameters.values() if p != p.VAR_KEYWORD] + args = args[1:] # assume the first argument is the dataset + params = params[1:] + kwargs_for_fingerprint.update(zip(params, args)) + else: + del kwargs_for_fingerprint[ + next(iter(inspect.signature(func).parameters)) + ] # assume the first key is the dataset + + # keep the right kwargs to be hashed to generate the fingerprint + + if use_kwargs: + kwargs_for_fingerprint = {k: v for k, v in kwargs_for_fingerprint.items() if k in use_kwargs} + if ignore_kwargs: + kwargs_for_fingerprint = {k: v for k, v in kwargs_for_fingerprint.items() if k not in ignore_kwargs} + if randomized_function: # randomized functions have `seed` and `generator` parameters + if kwargs_for_fingerprint.get("seed") is None and kwargs_for_fingerprint.get("generator") is None: + _, seed, pos, *_ = np.random.get_state() + seed = seed[pos] if pos < 624 else seed[0] + kwargs_for_fingerprint["generator"] = np.random.default_rng(seed) + + # remove kwargs that are the default values + + default_values = { + p.name: p.default for p in inspect.signature(func).parameters.values() if p.default != inspect._empty + } + for default_varname, default_value in default_values.items(): + if default_varname in kwargs_for_fingerprint and kwargs_for_fingerprint[default_varname] == default_value: + kwargs_for_fingerprint.pop(default_varname) + return kwargs_for_fingerprint + + +def fingerprint_transform( + inplace: bool, + use_kwargs: Optional[list[str]] = None, + ignore_kwargs: Optional[list[str]] = None, + fingerprint_names: Optional[list[str]] = None, + randomized_function: bool = False, + version: Optional[str] = None, +): + """ + Wrapper for dataset transforms to update the dataset fingerprint using ``update_fingerprint`` + Args: + inplace (:obj:`bool`): If inplace is True, the fingerprint of the dataset is updated inplace. + Otherwise, a parameter "new_fingerprint" is passed to the wrapped method that should take care of + setting the fingerprint of the returned Dataset. + use_kwargs (:obj:`List[str]`, optional): optional white list of argument names to take into account + to update the fingerprint to the wrapped method that should take care of + setting the fingerprint of the returned Dataset. By default all the arguments are used. + ignore_kwargs (:obj:`List[str]`, optional): optional black list of argument names to take into account + to update the fingerprint. Note that ignore_kwargs prevails on use_kwargs. + fingerprint_names (:obj:`List[str]`, optional, defaults to ["new_fingerprint"]): + If the dataset transforms is not inplace and returns a DatasetDict, then it can require + several fingerprints (one per dataset in the DatasetDict). By specifying fingerprint_names, + one fingerprint named after each element of fingerprint_names is going to be passed. + randomized_function (:obj:`bool`, defaults to False): If the dataset transform is random and has + optional parameters "seed" and "generator", then you can set randomized_function to True. + This way, even if users set "seed" and "generator" to None, then the fingerprint is + going to be randomly generated depending on numpy's current state. In this case, the + generator is set to np.random.default_rng(np.random.get_state()[1][0]). + version (:obj:`str`, optional): version of the transform. The version is taken into account when + computing the fingerprint. If a datase transform changes (or at least if the output data + that are cached changes), then one should increase the version. If the version stays the + same, then old cached data could be reused that are not compatible with the new transform. + It should be in the format "MAJOR.MINOR.PATCH". + """ + + if use_kwargs is not None and not isinstance(use_kwargs, list): + raise ValueError(f"use_kwargs is supposed to be a list, not {type(use_kwargs)}") + + if ignore_kwargs is not None and not isinstance(ignore_kwargs, list): + raise ValueError(f"ignore_kwargs is supposed to be a list, not {type(use_kwargs)}") + + if inplace and fingerprint_names: + raise ValueError("fingerprint_names are only used when inplace is False") + + fingerprint_names = fingerprint_names if fingerprint_names is not None else ["new_fingerprint"] + + def _fingerprint(func): + if not inplace and not all(name in func.__code__.co_varnames for name in fingerprint_names): + raise ValueError(f"function {func} is missing parameters {fingerprint_names} in signature") + + if randomized_function: # randomized function have seed and generator parameters + if "seed" not in func.__code__.co_varnames: + raise ValueError(f"'seed' must be in {func}'s signature") + if "generator" not in func.__code__.co_varnames: + raise ValueError(f"'generator' must be in {func}'s signature") + # this call has to be outside the wrapper or since __qualname__ changes in multiprocessing + transform = format_transform_for_fingerprint(func, version=version) + + @wraps(func) + def wrapper(*args, **kwargs): + kwargs_for_fingerprint = format_kwargs_for_fingerprint( + func, + args, + kwargs, + use_kwargs=use_kwargs, + ignore_kwargs=ignore_kwargs, + randomized_function=randomized_function, + ) + + if args: + dataset: Dataset = args[0] + args = args[1:] + else: + dataset: Dataset = kwargs.pop(next(iter(inspect.signature(func).parameters))) + + # compute new_fingerprint and add it to the args of not in-place transforms + if inplace: + new_fingerprint = update_fingerprint(dataset._fingerprint, transform, kwargs_for_fingerprint) + else: + for fingerprint_name in fingerprint_names: # transforms like `train_test_split` have several hashes + if kwargs.get(fingerprint_name) is None: + kwargs_for_fingerprint["fingerprint_name"] = fingerprint_name + kwargs[fingerprint_name] = update_fingerprint( + dataset._fingerprint, transform, kwargs_for_fingerprint + ) + else: + validate_fingerprint(kwargs[fingerprint_name]) + + # Call actual function + + out = func(dataset, *args, **kwargs) + + # Update fingerprint of in-place transforms + update in-place history of transforms + + if inplace: # update after calling func so that the fingerprint doesn't change if the function fails + dataset._fingerprint = new_fingerprint + + return out + + wrapper._decorator_name_ = "fingerprint" + return wrapper + + return _fingerprint diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/hub.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/hub.py new file mode 100644 index 0000000000000000000000000000000000000000..c6594760de4bb1da4d2726791f4e9726ffaaa611 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/hub.py @@ -0,0 +1,124 @@ +from itertools import chain +from typing import Optional, Union + +from huggingface_hub import ( + CommitInfo, + CommitOperationAdd, + CommitOperationDelete, + DatasetCard, + DatasetCardData, + HfApi, + HfFileSystem, +) + +import datasets.config +from datasets.info import DatasetInfosDict +from datasets.load import load_dataset_builder +from datasets.utils.metadata import MetadataConfigs + + +def delete_from_hub( + repo_id: str, + config_name: str, + revision: Optional[str] = None, + token: Optional[Union[bool, str]] = None, +) -> CommitInfo: + """Delete a dataset configuration from a [data-only dataset](repository_structure) on the Hub. + + Args: + repo_id (`str`): ID of the Hub dataset repository, in the following format: `/` or + `/`. + config_name (`str`): Name of the dataset configuration. + revision (`str`, *optional*): Branch to delete the configuration from. Defaults to the `"main"` branch. + token (`bool` or `str`, *optional*): Authentication token for the Hugging Face Hub. + + Returns: + `huggingface_hub.CommitInfo` + """ + operations = [] + # data_files + fs = HfFileSystem(endpoint=datasets.config.HF_ENDPOINT, token=token) + builder = load_dataset_builder(repo_id, config_name, revision=revision, token=token) + for data_file in chain(*builder.config.data_files.values()): + data_file_resolved_path = fs.resolve_path(data_file) + if data_file_resolved_path.repo_id == repo_id: + operations.append(CommitOperationDelete(path_in_repo=data_file_resolved_path.path_in_repo)) + # README.md + dataset_card = DatasetCard.load(repo_id) + # config_names + if dataset_card.data.get("config_names", None) and config_name in dataset_card.data["config_names"]: + dataset_card.data["config_names"].remove(config_name) + # metadata_configs + metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card.data) + if metadata_configs: + _ = metadata_configs.pop(config_name, None) + dataset_card_data = DatasetCardData() + metadata_configs.to_dataset_card_data(dataset_card_data) + if datasets.config.METADATA_CONFIGS_FIELD in dataset_card_data: + dataset_card.data[datasets.config.METADATA_CONFIGS_FIELD] = dataset_card_data[ + datasets.config.METADATA_CONFIGS_FIELD + ] + else: + _ = dataset_card.data.pop(datasets.config.METADATA_CONFIGS_FIELD, None) + # dataset_info + dataset_infos: DatasetInfosDict = DatasetInfosDict.from_dataset_card_data(dataset_card.data) + if dataset_infos: + _ = dataset_infos.pop(config_name, None) + dataset_card_data = DatasetCardData() + dataset_infos.to_dataset_card_data(dataset_card_data) + if "dataset_info" in dataset_card_data: + dataset_card.data["dataset_info"] = dataset_card_data["dataset_info"] + else: + _ = dataset_card.data.pop("dataset_info", None) + # Commit + operations.append( + CommitOperationAdd(path_in_repo=datasets.config.REPOCARD_FILENAME, path_or_fileobj=str(dataset_card).encode()) + ) + api = HfApi(endpoint=datasets.config.HF_ENDPOINT, token=token) + commit_info = api.create_commit( + repo_id, + operations=operations, + commit_message=f"Delete '{config_name}' config", + commit_description=f"Delete '{config_name}' config.", + token=token, + repo_type="dataset", + revision=revision, + create_pr=True, + ) + print(f"You can find your PR to delete the dataset config at: {commit_info.pr_url}") + return commit_info + + +def _delete_files(dataset_id, revision=None, token=None): + hf_api = HfApi(endpoint=datasets.config.HF_ENDPOINT, token=token) + repo_files = hf_api.list_repo_files( + dataset_id, + repo_type="dataset", + ) + if repo_files: + legacy_json_file = [] + data_files = [] + for filename in repo_files: + if filename in {".gitattributes", "README.md"}: + continue + elif filename == "dataset_infos.json": + legacy_json_file.append(filename) + else: + data_files.append(filename) + if legacy_json_file: + hf_api.delete_file( + "dataset_infos.json", + dataset_id, + repo_type="dataset", + revision=revision, + commit_message="Delete legacy dataset_infos.json", + ) + if data_files: + for filename in data_files: + hf_api.delete_file( + filename, + dataset_id, + repo_type="dataset", + revision=revision, + commit_message="Delete data file", + ) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/info.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/info.py new file mode 100644 index 0000000000000000000000000000000000000000..13fd972cb1a57f47a8ff581d7937f7a3411698a7 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/info.py @@ -0,0 +1,440 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""DatasetInfo record information we know about a dataset. + +This includes things that we know about the dataset statically, i.e.: + - description + - canonical location + - does it have validation and tests splits + - size + - etc. + +This also includes the things that can and should be computed once we've +processed the dataset as well: + - number of examples (in each split) + - etc. +""" + +import copy +import dataclasses +import json +import os +import posixpath +from dataclasses import dataclass +from pathlib import Path +from typing import ClassVar, Optional, Union + +import fsspec +from fsspec.core import url_to_fs +from huggingface_hub import DatasetCard, DatasetCardData + +from . import config +from .features import Features +from .splits import SplitDict +from .utils import Version +from .utils.logging import get_logger +from .utils.py_utils import asdict, unique_values + + +logger = get_logger(__name__) + + +@dataclass +class SupervisedKeysData: + input: str = "" + output: str = "" + + +@dataclass +class DownloadChecksumsEntryData: + key: str = "" + value: str = "" + + +class MissingCachedSizesConfigError(Exception): + """The expected cached sizes of the download file are missing.""" + + +class NonMatchingCachedSizesError(Exception): + """The prepared split doesn't have expected sizes.""" + + +@dataclass +class PostProcessedInfo: + features: Optional[Features] = None + resources_checksums: Optional[dict] = None + + def __post_init__(self): + # Convert back to the correct classes when we reload from dict + if self.features is not None and not isinstance(self.features, Features): + self.features = Features.from_dict(self.features) + + @classmethod + def from_dict(cls, post_processed_info_dict: dict) -> "PostProcessedInfo": + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in post_processed_info_dict.items() if k in field_names}) + + +@dataclass +class DatasetInfo: + """Information about a dataset. + + `DatasetInfo` documents datasets, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Not all fields are known on construction and may be updated later. + + Attributes: + description (`str`): + A description of the dataset. + citation (`str`): + A BibTeX citation of the dataset. + homepage (`str`): + A URL to the official homepage for the dataset. + license (`str`): + The dataset's license. It can be the name of the license or a paragraph containing the terms of the license. + features ([`Features`], *optional*): + The features used to specify the dataset's column types. + post_processed (`PostProcessedInfo`, *optional*): + Deprecated. Information regarding the resources of a possible post-processing of a dataset. For example, it can contain the information of an index. + supervised_keys (`SupervisedKeysData`, *optional*): + Specifies the input feature and the label for supervised learning if applicable for the dataset (legacy from TFDS). + builder_name (`str`, *optional*): + The name of the `GeneratorBasedBuilder` subclass used to create the dataset. It is also the snake_case version of the dataset builder class name. + config_name (`str`, *optional*): + The name of the configuration derived from [`BuilderConfig`]. + version (`str` or [`Version`], *optional*): + The version of the dataset. + splits (`dict`, *optional*): + The mapping between split name and metadata. + download_checksums (`dict`, *optional*): + The mapping between the URL to download the dataset's checksums and corresponding metadata. + download_size (`int`, *optional*): + The size of the files to download to generate the dataset, in bytes. + post_processing_size (`int`, *optional*): + Deprecated. Size of the dataset in bytes after post-processing, if any. + dataset_size (`int`, *optional*): + The combined size in bytes of the Arrow tables for all splits. + size_in_bytes (`int`, *optional*): + The combined size in bytes of all files associated with the dataset (downloaded files + Arrow files). + **config_kwargs (additional keyword arguments): + Keyword arguments to be passed to the [`BuilderConfig`] and used in the [`DatasetBuilder`]. + """ + + # Set in the dataset builders + description: str = dataclasses.field(default_factory=str) + citation: str = dataclasses.field(default_factory=str) + homepage: str = dataclasses.field(default_factory=str) + license: str = dataclasses.field(default_factory=str) + features: Optional[Features] = None + post_processed: Optional[PostProcessedInfo] = None # kept for bawkard compat + supervised_keys: Optional[SupervisedKeysData] = None + + # Set later by the builder + builder_name: Optional[str] = None + dataset_name: Optional[str] = None # for packaged builders, to be different from builder_name + config_name: Optional[str] = None + version: Optional[Union[str, Version]] = None + # Set later by `download_and_prepare` + splits: Optional[SplitDict] = None + download_checksums: Optional[dict] = None + download_size: Optional[int] = None + post_processing_size: Optional[int] = None + dataset_size: Optional[int] = None + size_in_bytes: Optional[int] = None + + _INCLUDED_INFO_IN_YAML: ClassVar[list[str]] = [ + "config_name", + "download_size", + "dataset_size", + "features", + "splits", + ] + + def __post_init__(self): + # Convert back to the correct classes when we reload from dict + if self.features is not None and not isinstance(self.features, Features): + self.features = Features.from_dict(self.features) + if self.post_processed is not None and not isinstance(self.post_processed, PostProcessedInfo): + self.post_processed = PostProcessedInfo.from_dict(self.post_processed) + if self.version is not None and not isinstance(self.version, Version): + if isinstance(self.version, str): + self.version = Version(self.version) + else: + self.version = Version.from_dict(self.version) + if self.splits is not None and not isinstance(self.splits, SplitDict): + self.splits = SplitDict.from_split_dict(self.splits) + if self.supervised_keys is not None and not isinstance(self.supervised_keys, SupervisedKeysData): + if isinstance(self.supervised_keys, (tuple, list)): + self.supervised_keys = SupervisedKeysData(*self.supervised_keys) + else: + self.supervised_keys = SupervisedKeysData(**self.supervised_keys) + + def write_to_directory(self, dataset_info_dir, pretty_print=False, storage_options: Optional[dict] = None): + """Write `DatasetInfo` and license (if present) as JSON files to `dataset_info_dir`. + + Args: + dataset_info_dir (`str`): + Destination directory. + pretty_print (`bool`, defaults to `False`): + If `True`, the JSON will be pretty-printed with the indent level of 4. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.info.write_to_directory("/path/to/directory/") + ``` + """ + fs: fsspec.AbstractFileSystem + fs, *_ = url_to_fs(dataset_info_dir, **(storage_options or {})) + with fs.open(posixpath.join(dataset_info_dir, config.DATASET_INFO_FILENAME), "wb") as f: + self._dump_info(f, pretty_print=pretty_print) + if self.license: + with fs.open(posixpath.join(dataset_info_dir, config.LICENSE_FILENAME), "wb") as f: + self._dump_license(f) + + def _dump_info(self, file, pretty_print=False): + """Dump info in `file` file-like object open in bytes mode (to support remote files)""" + file.write(json.dumps(asdict(self), indent=4 if pretty_print else None).encode("utf-8")) + + def _dump_license(self, file): + """Dump license in `file` file-like object open in bytes mode (to support remote files)""" + file.write(self.license.encode("utf-8")) + + @classmethod + def from_merge(cls, dataset_infos: list["DatasetInfo"]): + dataset_infos = [dset_info.copy() for dset_info in dataset_infos if dset_info is not None] + + if len(dataset_infos) > 0 and all(dataset_infos[0] == dset_info for dset_info in dataset_infos): + # if all dataset_infos are equal we don't need to merge. Just return the first. + return dataset_infos[0] + + description = "\n\n".join(unique_values(info.description for info in dataset_infos)).strip() + citation = "\n\n".join(unique_values(info.citation for info in dataset_infos)).strip() + homepage = "\n\n".join(unique_values(info.homepage for info in dataset_infos)).strip() + license = "\n\n".join(unique_values(info.license for info in dataset_infos)).strip() + features = None + supervised_keys = None + + return cls( + description=description, + citation=citation, + homepage=homepage, + license=license, + features=features, + supervised_keys=supervised_keys, + ) + + @classmethod + def from_directory(cls, dataset_info_dir: str, storage_options: Optional[dict] = None) -> "DatasetInfo": + """Create [`DatasetInfo`] from the JSON file in `dataset_info_dir`. + + This function updates all the dynamically generated fields (num_examples, + hash, time of creation,...) of the [`DatasetInfo`]. + + This will overwrite all previous metadata. + + Args: + dataset_info_dir (`str`): + The directory containing the metadata file. This + should be the root directory of a specific dataset version. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Example: + + ```py + >>> from datasets import DatasetInfo + >>> ds_info = DatasetInfo.from_directory("/path/to/directory/") + ``` + """ + fs: fsspec.AbstractFileSystem + fs, *_ = url_to_fs(dataset_info_dir, **(storage_options or {})) + logger.debug(f"Loading Dataset info from {dataset_info_dir}") + if not dataset_info_dir: + raise ValueError("Calling DatasetInfo.from_directory() with undefined dataset_info_dir.") + with fs.open(posixpath.join(dataset_info_dir, config.DATASET_INFO_FILENAME), "r", encoding="utf-8") as f: + dataset_info_dict = json.load(f) + return cls.from_dict(dataset_info_dict) + + @classmethod + def from_dict(cls, dataset_info_dict: dict) -> "DatasetInfo": + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in dataset_info_dict.items() if k in field_names}) + + def update(self, other_dataset_info: "DatasetInfo", ignore_none=True): + self_dict = self.__dict__ + self_dict.update( + **{ + k: copy.deepcopy(v) + for k, v in other_dataset_info.__dict__.items() + if (v is not None or not ignore_none) + } + ) + + def copy(self) -> "DatasetInfo": + return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()}) + + def _to_yaml_dict(self) -> dict: + yaml_dict = {} + dataset_info_dict = asdict(self) + for key in dataset_info_dict: + if key in self._INCLUDED_INFO_IN_YAML: + value = getattr(self, key) + if hasattr(value, "_to_yaml_list"): # Features, SplitDict + yaml_dict[key] = value._to_yaml_list() + elif hasattr(value, "_to_yaml_string"): # Version + yaml_dict[key] = value._to_yaml_string() + else: + yaml_dict[key] = value + return yaml_dict + + @classmethod + def _from_yaml_dict(cls, yaml_data: dict) -> "DatasetInfo": + yaml_data = copy.deepcopy(yaml_data) + if yaml_data.get("features") is not None: + yaml_data["features"] = Features._from_yaml_list(yaml_data["features"]) + if yaml_data.get("splits") is not None: + yaml_data["splits"] = SplitDict._from_yaml_list(yaml_data["splits"]) + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in yaml_data.items() if k in field_names}) + + def __repr__(self): + return ( + self.__class__.__qualname__ + + "(" + + ", ".join( + [f"{f.name}={repr(getattr(self, f.name))}" for f in dataclasses.fields(self) if getattr(self, f.name)] + ) + + ")" + ) + + +class DatasetInfosDict(dict[str, DatasetInfo]): + def write_to_directory(self, dataset_infos_dir, overwrite=False, pretty_print=False) -> None: + total_dataset_infos = {} + dataset_infos_path = os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME) + dataset_readme_path = os.path.join(dataset_infos_dir, config.REPOCARD_FILENAME) + if not overwrite: + total_dataset_infos = self.from_directory(dataset_infos_dir) + total_dataset_infos.update(self) + if os.path.exists(dataset_infos_path): + # for backward compatibility, let's update the JSON file if it exists + with open(dataset_infos_path, "w", encoding="utf-8") as f: + dataset_infos_dict = { + config_name: asdict(dset_info) for config_name, dset_info in total_dataset_infos.items() + } + json.dump(dataset_infos_dict, f, indent=4 if pretty_print else None) + # Dump the infos in the YAML part of the README.md file + if os.path.exists(dataset_readme_path): + dataset_card = DatasetCard.load(dataset_readme_path) + dataset_card_data = dataset_card.data + else: + dataset_card = None + dataset_card_data = DatasetCardData() + if total_dataset_infos: + total_dataset_infos.to_dataset_card_data(dataset_card_data) + dataset_card = ( + DatasetCard("---\n" + str(dataset_card_data) + "\n---\n") if dataset_card is None else dataset_card + ) + dataset_card.save(Path(dataset_readme_path)) + + @classmethod + def from_directory(cls, dataset_infos_dir) -> "DatasetInfosDict": + logger.debug(f"Loading Dataset Infos from {dataset_infos_dir}") + # Load the info from the YAML part of README.md + if os.path.exists(os.path.join(dataset_infos_dir, config.REPOCARD_FILENAME)): + dataset_card_data = DatasetCard.load(Path(dataset_infos_dir) / config.REPOCARD_FILENAME).data + if "dataset_info" in dataset_card_data: + return cls.from_dataset_card_data(dataset_card_data) + if os.path.exists(os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME)): + # this is just to have backward compatibility with dataset_infos.json files + with open(os.path.join(dataset_infos_dir, config.DATASETDICT_INFOS_FILENAME), encoding="utf-8") as f: + return cls( + { + config_name: DatasetInfo.from_dict(dataset_info_dict) + for config_name, dataset_info_dict in json.load(f).items() + } + ) + else: + return cls() + + @classmethod + def from_dataset_card_data(cls, dataset_card_data: DatasetCardData) -> "DatasetInfosDict": + if isinstance(dataset_card_data.get("dataset_info"), (list, dict)): + if isinstance(dataset_card_data["dataset_info"], list): + return cls( + { + dataset_info_yaml_dict.get("config_name", "default"): DatasetInfo._from_yaml_dict( + dataset_info_yaml_dict + ) + for dataset_info_yaml_dict in dataset_card_data["dataset_info"] + } + ) + else: + dataset_info = DatasetInfo._from_yaml_dict(dataset_card_data["dataset_info"]) + dataset_info.config_name = dataset_card_data["dataset_info"].get("config_name", "default") + return cls({dataset_info.config_name: dataset_info}) + else: + return cls() + + def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None: + if self: + # first get existing metadata info + if "dataset_info" in dataset_card_data and isinstance(dataset_card_data["dataset_info"], dict): + dataset_metadata_infos = { + dataset_card_data["dataset_info"].get("config_name", "default"): dataset_card_data["dataset_info"] + } + elif "dataset_info" in dataset_card_data and isinstance(dataset_card_data["dataset_info"], list): + dataset_metadata_infos = { + config_metadata["config_name"]: config_metadata + for config_metadata in dataset_card_data["dataset_info"] + } + else: + dataset_metadata_infos = {} + # update/rewrite existing metadata info with the one to dump + total_dataset_infos = { + **dataset_metadata_infos, + **{config_name: dset_info._to_yaml_dict() for config_name, dset_info in self.items()}, + } + # the config_name from the dataset_infos_dict takes over the config_name of the DatasetInfo + for config_name, dset_info_yaml_dict in total_dataset_infos.items(): + dset_info_yaml_dict["config_name"] = config_name + if len(total_dataset_infos) == 1: + # use a struct instead of a list of configurations, since there's only one + dataset_card_data["dataset_info"] = next(iter(total_dataset_infos.values())) + config_name = dataset_card_data["dataset_info"].pop("config_name", None) + if config_name != "default": + # if config_name is not "default" preserve it and put at the first position + dataset_card_data["dataset_info"] = { + "config_name": config_name, + **dataset_card_data["dataset_info"], + } + else: + dataset_card_data["dataset_info"] = [] + for config_name, dataset_info_yaml_dict in sorted(total_dataset_infos.items()): + # add the config_name field in first position + dataset_info_yaml_dict.pop("config_name", None) + dataset_info_yaml_dict = {"config_name": config_name, **dataset_info_yaml_dict} + dataset_card_data["dataset_info"].append(dataset_info_yaml_dict) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/inspect.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/inspect.py new file mode 100644 index 0000000000000000000000000000000000000000..f07c825d02c2c04112579631db84d93ef2de5c28 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/inspect.py @@ -0,0 +1,350 @@ +# Copyright 2020 The HuggingFace Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""List and inspect datasets.""" + +import os +from collections.abc import Mapping, Sequence +from typing import Optional, Union + +from .download.download_config import DownloadConfig +from .download.download_manager import DownloadMode +from .download.streaming_download_manager import StreamingDownloadManager +from .info import DatasetInfo +from .load import ( + dataset_module_factory, + get_dataset_builder_class, + load_dataset_builder, +) +from .utils.logging import get_logger +from .utils.version import Version + + +logger = get_logger(__name__) + + +class SplitsNotFoundError(ValueError): + pass + + +def get_dataset_infos( + path: str, + data_files: Optional[Union[dict, list, str]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + revision: Optional[Union[str, Version]] = None, + token: Optional[Union[bool, str]] = None, + **config_kwargs, +): + """Get the meta information about a dataset, returned as a dict mapping config name to DatasetInfoDict. + + Args: + path (`str`): path to the dataset repository. Can be either: + + - a local path to the dataset directory containing the data files, + e.g. `'./dataset/squad'` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]), + e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'` + revision (`Union[str, datasets.Version]`, *optional*): + If specified, the dataset module will be loaded from the datasets repository at this version. + By default: + - it is set to the local version of the lib. + - it will also try to load it from the main branch if it's not available at the local version of the lib. + Specifying a version that is different from your local version of the lib might cause compatibility issues. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + data_files (`Union[Dict, List, str]`, *optional*): + Defining the data_files of the dataset configuration. + token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. + If `True`, or not specified, will get token from `"~/.huggingface"`. + **config_kwargs (additional keyword arguments): + Optional attributes for builder class which will override the attributes if supplied. + + Example: + + ```py + >>> from datasets import get_dataset_infos + >>> get_dataset_infos('cornell-movie-review-data/rotten_tomatoes') + {'default': DatasetInfo(description="Movie Review Dataset.\nThis is a dataset of containing 5,331 positive and 5,331 negative processed\nsentences from Rotten Tomatoes movie reviews...), ...} + ``` + """ + config_names = get_dataset_config_names( + path=path, + revision=revision, + download_config=download_config, + download_mode=download_mode, + data_files=data_files, + token=token, + ) + return { + config_name: get_dataset_config_info( + path=path, + config_name=config_name, + data_files=data_files, + download_config=download_config, + download_mode=download_mode, + revision=revision, + token=token, + **config_kwargs, + ) + for config_name in config_names + } + + +def get_dataset_config_names( + path: str, + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + data_files: Optional[Union[dict, list, str]] = None, + **download_kwargs, +): + """Get the list of available config names for a particular dataset. + + Args: + path (`str`): path to the dataset repository. Can be either: + + - a local path to the dataset directory containing the data files, + e.g. `'./dataset/squad'` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]), + e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'` + revision (`Union[str, datasets.Version]`, *optional*): + If specified, the dataset module will be loaded from the datasets repository at this version. + By default: + - it is set to the local version of the lib. + - it will also try to load it from the main branch if it's not available at the local version of the lib. + Specifying a version that is different from your local version of the lib might cause compatibility issues. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + data_files (`Union[Dict, List, str]`, *optional*): + Defining the data_files of the dataset configuration. + **download_kwargs (additional keyword arguments): + Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied, + for example `token`. + + Example: + + ```py + >>> from datasets import get_dataset_config_names + >>> get_dataset_config_names("nyu-mll/glue") + ['cola', + 'sst2', + 'mrpc', + 'qqp', + 'stsb', + 'mnli', + 'mnli_mismatched', + 'mnli_matched', + 'qnli', + 'rte', + 'wnli', + 'ax'] + ``` + """ + dataset_module = dataset_module_factory( + path, + revision=revision, + download_config=download_config, + download_mode=download_mode, + data_files=data_files, + **download_kwargs, + ) + builder_cls = get_dataset_builder_class(dataset_module, dataset_name=os.path.basename(path)) + return list(builder_cls.builder_configs.keys()) or [ + dataset_module.builder_kwargs.get("config_name", builder_cls.DEFAULT_CONFIG_NAME or "default") + ] + + +def get_dataset_default_config_name( + path: str, + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + data_files: Optional[Union[dict, list, str]] = None, + **download_kwargs, +) -> Optional[str]: + """Get the default config name for a particular dataset. + Can return None only if the dataset has multiple configurations and no default configuration. + + Args: + path (`str`): path to the dataset repository. Can be either: + + - a local path to the dataset directory containing the data files, + e.g. `'./dataset/squad'` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]), + e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'` + revision (`Union[str, datasets.Version]`, *optional*): + If specified, the dataset module will be loaded from the datasets repository at this version. + By default: + - it is set to the local version of the lib. + - it will also try to load it from the main branch if it's not available at the local version of the lib. + Specifying a version that is different from your local version of the lib might cause compatibility issues. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + data_files (`Union[Dict, List, str]`, *optional*): + Defining the data_files of the dataset configuration. + **download_kwargs (additional keyword arguments): + Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied, + for example `token`. + + Returns: + Optional[str]: the default config name if there is one + + Example: + + ```py + >>> from datasets import get_dataset_default_config_name + >>> get_dataset_default_config_name("openbookqa") + 'main' + ``` + """ + dataset_module = dataset_module_factory( + path, + revision=revision, + download_config=download_config, + download_mode=download_mode, + data_files=data_files, + **download_kwargs, + ) + builder_cls = get_dataset_builder_class(dataset_module, dataset_name=os.path.basename(path)) + builder_configs = list(builder_cls.builder_configs.keys()) + if builder_configs: + default_config_name = builder_configs[0] if len(builder_configs) == 1 else None + else: + default_config_name = "default" + return builder_cls.DEFAULT_CONFIG_NAME or default_config_name + + +def get_dataset_config_info( + path: str, + config_name: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + revision: Optional[Union[str, Version]] = None, + token: Optional[Union[bool, str]] = None, + **config_kwargs, +) -> DatasetInfo: + """Get the meta information (DatasetInfo) about a dataset for a particular config + + Args: + path (`str`): path to the dataset repository. Can be either: + + - a local path to the dataset directory containing the data files, + e.g. `'./dataset/squad'` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]), + e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'` + config_name (:obj:`str`, optional): Defining the name of the dataset configuration. + data_files (:obj:`str` or :obj:`Sequence` or :obj:`Mapping`, optional): Path(s) to source data file(s). + download_config (:class:`~download.DownloadConfig`, optional): Specific download configuration parameters. + download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. + revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset to load. + As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. + You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. + token (``str`` or :obj:`bool`, optional): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. + If True, or not specified, will get token from `"~/.huggingface"`. + **config_kwargs (additional keyword arguments): optional attributes for builder class which will override the attributes if supplied. + + """ + builder = load_dataset_builder( + path, + name=config_name, + data_files=data_files, + download_config=download_config, + download_mode=download_mode, + revision=revision, + token=token, + **config_kwargs, + ) + info = builder.info + if info.splits is None: + download_config = download_config.copy() if download_config else DownloadConfig() + if token is not None: + download_config.token = token + try: + info.splits = { + split_generator.name: {"name": split_generator.name, "dataset_name": path} + for split_generator in builder._split_generators( + StreamingDownloadManager(base_path=builder.base_path, download_config=download_config) + ) + } + except Exception as err: + raise SplitsNotFoundError("The split names could not be parsed from the dataset config.") from err + return info + + +def get_dataset_split_names( + path: str, + config_name: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + revision: Optional[Union[str, Version]] = None, + token: Optional[Union[bool, str]] = None, + **config_kwargs, +): + """Get the list of available splits for a particular config and dataset. + + Args: + path (`str`): path to the dataset repository. Can be either: + + - a local path to the dataset directory containing the data files, + e.g. `'./dataset/squad'` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`huggingface_hub.list_datasets`]), + e.g. `'rajpurkar/squad'`, `'nyu-mll/glue'` or``'openai/webtext'` + config_name (`str`, *optional*): + Defining the name of the dataset configuration. + data_files (`str` or `Sequence` or `Mapping`, *optional*): + Path(s) to source data file(s). + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + revision ([`Version`] or `str`, *optional*): + Version of the dataset to load. + As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. + You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. + token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. + If `True`, or not specified, will get token from `"~/.huggingface"`. + **config_kwargs (additional keyword arguments): + Optional attributes for builder class which will override the attributes if supplied. + + Example: + + ```py + >>> from datasets import get_dataset_split_names + >>> get_dataset_split_names('cornell-movie-review-data/rotten_tomatoes') + ['train', 'validation', 'test'] + ``` + """ + info = get_dataset_config_info( + path, + config_name=config_name, + data_files=data_files, + download_config=download_config, + download_mode=download_mode, + revision=revision, + token=token, + **config_kwargs, + ) + return list(info.splits.keys()) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/csv.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/csv.py new file mode 100644 index 0000000000000000000000000000000000000000..13d87d0401fafaa20c32908063e41bba8e349c24 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/csv.py @@ -0,0 +1,143 @@ +import multiprocessing +import os +from typing import BinaryIO, Optional, Union + +import fsspec + +from .. import Dataset, Features, NamedSplit, config +from ..formatting import query_table +from ..packaged_modules.csv.csv import Csv +from ..utils import tqdm as hf_tqdm +from ..utils.typing import NestedDataStructureLike, PathLike +from .abc import AbstractDatasetReader + + +class CsvDatasetReader(AbstractDatasetReader): + def __init__( + self, + path_or_paths: NestedDataStructureLike[PathLike], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + super().__init__( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths} + self.builder = Csv( + cache_dir=cache_dir, + data_files=path_or_paths, + features=features, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split=self.split) + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory) + return dataset + + +class CsvDatasetWriter: + def __init__( + self, + dataset: Dataset, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_csv_kwargs, + ): + if num_proc is not None and num_proc <= 0: + raise ValueError(f"num_proc {num_proc} must be an integer > 0.") + + self.dataset = dataset + self.path_or_buf = path_or_buf + self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + self.num_proc = num_proc + self.encoding = "utf-8" + self.storage_options = storage_options or {} + self.to_csv_kwargs = to_csv_kwargs + + def write(self) -> int: + _ = self.to_csv_kwargs.pop("path_or_buf", None) + header = self.to_csv_kwargs.pop("header", True) + index = self.to_csv_kwargs.pop("index", False) + + if isinstance(self.path_or_buf, (str, bytes, os.PathLike)): + with fsspec.open(self.path_or_buf, "wb", **(self.storage_options or {})) as buffer: + written = self._write(file_obj=buffer, header=header, index=index, **self.to_csv_kwargs) + else: + written = self._write(file_obj=self.path_or_buf, header=header, index=index, **self.to_csv_kwargs) + return written + + def _batch_csv(self, args): + offset, header, index, to_csv_kwargs = args + + batch = query_table( + table=self.dataset.data, + key=slice(offset, offset + self.batch_size), + indices=self.dataset._indices, + ) + csv_str = batch.to_pandas().to_csv( + path_or_buf=None, header=header if (offset == 0) else False, index=index, **to_csv_kwargs + ) + return csv_str.encode(self.encoding) + + def _write(self, file_obj: BinaryIO, header, index, **to_csv_kwargs) -> int: + """Writes the pyarrow table as CSV to a binary file handle. + + Caller is responsible for opening and closing the handle. + """ + written = 0 + + if self.num_proc is None or self.num_proc == 1: + for offset in hf_tqdm( + range(0, len(self.dataset), self.batch_size), + unit="ba", + desc="Creating CSV from Arrow format", + ): + csv_str = self._batch_csv((offset, header, index, to_csv_kwargs)) + written += file_obj.write(csv_str) + + else: + num_rows, batch_size = len(self.dataset), self.batch_size + with multiprocessing.Pool(self.num_proc) as pool: + for csv_str in hf_tqdm( + pool.imap( + self._batch_csv, + [(offset, header, index, to_csv_kwargs) for offset in range(0, num_rows, batch_size)], + ), + total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size, + unit="ba", + desc="Creating CSV from Arrow format", + ): + written += file_obj.write(csv_str) + + return written diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/json.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/json.py new file mode 100644 index 0000000000000000000000000000000000000000..eb347aea7aa54fee182496a35d7fdf0fef06c787 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/json.py @@ -0,0 +1,178 @@ +import multiprocessing +import os +from functools import partial +from typing import BinaryIO, Optional, Union + +import fsspec + +from .. import Dataset, Features, NamedSplit, config +from ..formatting import query_table +from ..packaged_modules.json.json import Json +from ..utils import tqdm as hf_tqdm +from ..utils.json import get_json_field_paths_from_feature, json_decode_field +from ..utils.typing import NestedDataStructureLike, PathLike +from .abc import AbstractDatasetReader + + +class JsonDatasetReader(AbstractDatasetReader): + def __init__( + self, + path_or_paths: NestedDataStructureLike[PathLike], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + field: Optional[str] = None, + num_proc: Optional[int] = None, + **kwargs, + ): + super().__init__( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + self.field = field + path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths} + self.builder = Json( + cache_dir=cache_dir, + data_files=path_or_paths, + features=features, + field=field, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split=self.split) + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory) + return dataset + + +class JsonDatasetWriter: + def __init__( + self, + dataset: Dataset, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_json_kwargs, + ): + if num_proc is not None and num_proc <= 0: + raise ValueError(f"num_proc {num_proc} must be an integer > 0.") + + self.dataset = dataset + self.path_or_buf = path_or_buf + self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + self.num_proc = num_proc + self.encoding = "utf-8" + self.storage_options = storage_options or {} + self.to_json_kwargs = to_json_kwargs + + def write(self) -> int: + _ = self.to_json_kwargs.pop("path_or_buf", None) + orient = self.to_json_kwargs.pop("orient", "records") + lines = self.to_json_kwargs.pop("lines", True if orient == "records" else False) + if "index" not in self.to_json_kwargs and orient in ["split", "table"]: + self.to_json_kwargs["index"] = False + + # Determine the default compression value based on self.path_or_buf type + default_compression = "infer" if isinstance(self.path_or_buf, (str, bytes, os.PathLike)) else None + compression = self.to_json_kwargs.pop("compression", default_compression) + + if compression not in [None, "infer", "gzip", "bz2", "xz"]: + raise NotImplementedError(f"`datasets` currently does not support {compression} compression") + + if not lines and self.batch_size < self.dataset.num_rows: + raise NotImplementedError( + "Output JSON will not be formatted correctly when lines = False and batch_size < number of rows in the dataset. Use pandas.DataFrame.to_json() instead." + ) + + if isinstance(self.path_or_buf, (str, bytes, os.PathLike)): + with fsspec.open( + self.path_or_buf, "wb", compression=compression, **(self.storage_options or {}) + ) as buffer: + written = self._write(file_obj=buffer, orient=orient, lines=lines, **self.to_json_kwargs) + else: + if compression: + raise NotImplementedError( + f"The compression parameter is not supported when writing to a buffer, but compression={compression}" + " was passed. Please provide a local path instead." + ) + written = self._write(file_obj=self.path_or_buf, orient=orient, lines=lines, **self.to_json_kwargs) + return written + + def _batch_json(self, args): + offset, orient, lines, to_json_kwargs = args + + batch = query_table( + table=self.dataset.data, + key=slice(offset, offset + self.batch_size), + indices=self.dataset._indices, + ) + batch = batch.to_pandas() + for json_field_path in get_json_field_paths_from_feature(self.dataset.features): + col, *json_field_subpath = json_field_path + batch[col] = batch[col].apply(partial(json_decode_field, json_field_path=json_field_subpath)) + json_str = batch.to_json(path_or_buf=None, orient=orient, lines=lines, **to_json_kwargs) + if not json_str.endswith("\n"): + json_str += "\n" + return json_str.encode(self.encoding) + + def _write( + self, + file_obj: BinaryIO, + orient, + lines, + **to_json_kwargs, + ) -> int: + """Writes the pyarrow table as JSON lines to a binary file handle. + + Caller is responsible for opening and closing the handle. + """ + written = 0 + + if self.num_proc is None or self.num_proc == 1: + for offset in hf_tqdm( + range(0, len(self.dataset), self.batch_size), + unit="ba", + desc="Creating json from Arrow format", + ): + json_str = self._batch_json((offset, orient, lines, to_json_kwargs)) + written += file_obj.write(json_str) + else: + num_rows, batch_size = len(self.dataset), self.batch_size + with multiprocessing.Pool(self.num_proc) as pool: + for json_str in hf_tqdm( + pool.imap( + self._batch_json, + [(offset, orient, lines, to_json_kwargs) for offset in range(0, num_rows, batch_size)], + ), + total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size, + unit="ba", + desc="Creating json from Arrow format", + ): + written += file_obj.write(json_str) + + return written diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/sql.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/sql.py new file mode 100644 index 0000000000000000000000000000000000000000..f74374a5ad07f9a9767976752fd34fdc8ad71540 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/sql.py @@ -0,0 +1,122 @@ +import multiprocessing +from typing import TYPE_CHECKING, Optional, Union + +from .. import Dataset, Features, config +from ..formatting import query_table +from ..packaged_modules.sql.sql import Sql +from ..utils import tqdm as hf_tqdm +from .abc import AbstractDatasetInputStream + + +if TYPE_CHECKING: + import sqlite3 + + import sqlalchemy + + +class SqlDatasetReader(AbstractDatasetInputStream): + def __init__( + self, + sql: Union[str, "sqlalchemy.sql.Selectable"], + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + **kwargs, + ): + super().__init__(features=features, cache_dir=cache_dir, keep_in_memory=keep_in_memory, **kwargs) + self.builder = Sql( + cache_dir=cache_dir, + features=features, + sql=sql, + con=con, + **kwargs, + ) + + def read(self): + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + ) + + # Build dataset for splits + dataset = self.builder.as_dataset(split="train", in_memory=self.keep_in_memory) + return dataset + + +class SqlDatasetWriter: + def __init__( + self, + dataset: Dataset, + name: str, + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + **to_sql_kwargs, + ): + if num_proc is not None and num_proc <= 0: + raise ValueError(f"num_proc {num_proc} must be an integer > 0.") + + self.dataset = dataset + self.name = name + self.con = con + self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + self.num_proc = num_proc + self.to_sql_kwargs = to_sql_kwargs + + def write(self) -> int: + _ = self.to_sql_kwargs.pop("sql", None) + _ = self.to_sql_kwargs.pop("con", None) + index = self.to_sql_kwargs.pop("index", False) + + written = self._write(index=index, **self.to_sql_kwargs) + return written + + def _batch_sql(self, args): + offset, index, to_sql_kwargs = args + to_sql_kwargs = {**to_sql_kwargs, "if_exists": "append"} if offset > 0 else to_sql_kwargs + batch = query_table( + table=self.dataset.data, + key=slice(offset, offset + self.batch_size), + indices=self.dataset._indices, + ) + df = batch.to_pandas() + num_rows = df.to_sql(self.name, self.con, index=index, **to_sql_kwargs) + return num_rows or len(df) + + def _write(self, index, **to_sql_kwargs) -> int: + """Writes the pyarrow table as SQL to a database. + + Caller is responsible for opening and closing the SQL connection. + """ + written = 0 + + if self.num_proc is None or self.num_proc == 1: + for offset in hf_tqdm( + range(0, len(self.dataset), self.batch_size), + unit="ba", + desc="Creating SQL from Arrow format", + ): + written += self._batch_sql((offset, index, to_sql_kwargs)) + else: + num_rows, batch_size = len(self.dataset), self.batch_size + with multiprocessing.Pool(self.num_proc) as pool: + for num_rows in hf_tqdm( + pool.imap( + self._batch_sql, + [(offset, index, to_sql_kwargs) for offset in range(0, num_rows, batch_size)], + ), + total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size, + unit="ba", + desc="Creating SQL from Arrow format", + ): + written += num_rows + + return written diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/iterable_dataset.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/iterable_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..840662b7793289033e900a28d388240789d080bd --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/iterable_dataset.py @@ -0,0 +1,5181 @@ +import asyncio +import contextlib +import copy +import inspect +import itertools +import multiprocessing.pool +import re +import sys +import tempfile +from collections import Counter +from collections.abc import Iterable, Iterator +from copy import deepcopy +from dataclasses import dataclass +from functools import partial +from itertools import cycle, islice +from pathlib import Path +from typing import TYPE_CHECKING, Any, BinaryIO, Callable, Optional, Union + +import fsspec.asyn +import multiprocess as mp +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.dataset as pds +import pyarrow.parquet as pq +from huggingface_hub import ( + CommitInfo, + CommitOperationAdd, + HfApi, + HfFileSystem, + HfFileSystemResolvedPath, +) +from huggingface_hub.utils import RepositoryNotFoundError +from packaging import version + +from . import config +from .arrow_dataset import Dataset, DatasetInfoMixin, _push_to_bucket, _push_to_repo +from .features import Features +from .features.features import ( + FeatureType, + List, + Value, + _align_features, + _check_if_features_can_be_aligned, + _fix_for_backward_compatible_features, + _visit, + cast_to_python_objects, + require_decoding, +) +from .formatting import ( + ArrowFormatter, + PythonFormatter, + TableFormatter, + TensorFormatter, + get_format_type_from_alias, + get_formatter, +) +from .info import DatasetInfo +from .naming import _split_re +from .splits import NamedSplit, Split, SplitInfo +from .table import _batch_arrow_table, cast_table_to_features, embed_table_storage, read_schema_from_file, table_cast +from .utils import tqdm as hf_tqdm +from .utils.logging import get_logger +from .utils.py_utils import ( + Literal, + convert_file_size_to_int, + iflatmap_unordered, +) +from .utils.sharding import _merge_gen_kwargs, _number_of_shards_in_gen_kwargs, _shuffle_gen_kwargs, _split_gen_kwargs +from .utils.typing import PathLike + + +if config.HF_HUB_VERSION >= version.parse("1.6.0"): + from huggingface_hub.errors import BucketNotFoundError + from huggingface_hub.hf_file_system import HfFileSystemResolvedBucketPath, HfFileSystemResolvedRepositoryPath + +else: + BucketNotFoundError = None + HfFileSystemResolvedBucketPath = None + HfFileSystemResolvedRepositoryPath = HfFileSystemResolvedPath + +if TYPE_CHECKING: + import sqlite3 + + import polars as pl + import sqlalchemy + import torch + + from .builder import Key as BuilderKey + +logger = get_logger(__name__) + +Key = Union[int, str, tuple[int, int], "BuilderKey"] + + +def identity_func(x): + return x + + +def _rename_columns_fn(example: dict, column_mapping: dict[str, str]): + if any(col not in example for col in column_mapping): + raise ValueError( + f"Error when renaming {list(column_mapping)} to {list(column_mapping.values())}: columns {set(column_mapping) - set(example)} are not in the dataset." + ) + if any(col in example for col in column_mapping.values()): + raise ValueError( + f"Error when renaming {list(column_mapping)} to {list(column_mapping.values())}: columns {set(example) - set(column_mapping.values())} are already in the dataset." + ) + return { + new_column_name: example[original_column_name] + for original_column_name, new_column_name in column_mapping.items() + } + + +def add_column_fn(example: dict, idx: int, name: str, column: list[dict]): + if name in example: + raise ValueError(f"Error when adding {name}: column {name} is already in the dataset.") + return {name: column[idx]} + + +def _infer_features_from_batch(batch: dict[str, list], try_features: Optional[Features] = None) -> Features: + pa_table = pa.Table.from_pydict(batch) + if try_features is not None: + try: + pa_table = table_cast(pa_table, pa.schema(try_features.type)) + except (TypeError, pa.ArrowInvalid, pa.ArrowNotImplementedError): + pass + return Features.from_arrow_schema(pa_table.schema) + + +def _examples_to_batch(examples: list[dict[str, Any]]) -> dict[str, list]: + # we order the columns by order of appearance + # to do so, we use a dict as an ordered set + cols = {col: None for example in examples for col in example} + # when an example is missing a column, we set the value to None with .get() + arrays = [[example.get(col) for example in examples] for col in cols] + return dict(zip(cols, arrays)) + + +def _batch_to_examples(batch: dict[str, list]) -> Iterator[dict[str, Any]]: + """Convert a batch (dict of examples) to examples list""" + n_examples = 0 if len(batch) == 0 else len(batch[next(iter(batch))]) + for i in range(n_examples): + yield {col: array[i] for col, array in batch.items()} + + +def _convert_to_arrow( + iterable: Iterable[tuple[Key, dict]], + batch_size: int, + drop_last_batch: bool = False, +) -> Iterator[tuple[Key, pa.Table]]: + """Convert and group examples in Arrow tables of size `batch_size`. + + Args: + iterable (`Iterable[Tuple[Key, dict]]`): + An examples iterable containing tuples (example_key, example) of type (int/str, dict) + batch_size (`Optional[int]`): + Size of each sub-table to yield. If None or <= 0, yields the full table. + drop_last_batch (`bool`, defaults to `False`): + Drop the last batch if it is smaller than `batch_size`. + """ + if batch_size is None or batch_size <= 0: + yield ( + "all", + pa.Table.from_pylist(cast_to_python_objects([example for _, example in iterable], only_1d_for_numpy=True)), + ) + return + iterator = iter(iterable) + for key, example in iterator: + iterator_batch = islice(iterator, batch_size - 1) + key_examples_list = [(key, example)] + list(iterator_batch) + if len(key_examples_list) < batch_size and drop_last_batch: + return + keys, examples = zip(*key_examples_list) + new_key = "_".join(str(key) for key in keys) + yield new_key, pa.Table.from_pylist(cast_to_python_objects(examples, only_1d_for_numpy=True)) + + +def shift_ex_examples_rngs(ex_iterable: "_BaseExamplesIterable", value: int) -> "_BaseExamplesIterable": + """We need to go through the ex_iterables recursively, create a new seed and return a new iterable, then set it to the containing ex_iterable.""" + + def set_seed_recursively(ex_iterable): + if hasattr(ex_iterable, "shift_rngs"): + ex_iterable = ex_iterable.shift_rngs(value) + if hasattr(ex_iterable, "ex_iterable"): + ex_iterable.ex_iterable = set_seed_recursively(ex_iterable.ex_iterable) + if hasattr(ex_iterable, "ex_iterables"): + ex_iterable.ex_iterables = [set_seed_recursively(ei) for ei in ex_iterable.ex_iterables] + return ex_iterable + + return set_seed_recursively(ex_iterable) + + +class _BaseExamplesIterable: + """Base class for the examples iterable used by an IterableDataset""" + + def __init__(self) -> None: + self._state_dict: Optional[Union[list, dict]] = None + + def __iter__(self) -> Iterator[tuple[Key, dict]]: + """An examples iterable should yield tuples (example_key, example) of type (int/str, dict)""" + raise NotImplementedError(f"{type(self)} doesn't implement __iter__ yet") + + @property + def iter_arrow(self) -> Optional[Callable[[], Iterator[tuple[Key, pa.Table]]]]: + return None + + @property + def is_typed(self) -> bool: + return False + + @property + def features(self) -> Optional[Features]: + return None + + def shuffle_data_sources(self, generator: np.random.Generator) -> "_BaseExamplesIterable": + """ + Either shuffle the shards/sources of the dataset, or propagate the shuffling to the underlying iterable. + If the order of the shards must stay fixed (when using .skip or .take for example), then this method returns self. + """ + raise NotImplementedError(f"{type(self)} doesn't implement shuffle_data_sources yet") + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "_BaseExamplesIterable": + """Either keep only the requested shard, or propagate the request to the underlying iterable.""" + raise NotImplementedError(f"{type(self)} doesn't implement shard_data_sources yet") + + def reshard_data_sources(self) -> "_BaseExamplesIterable": + """ + Either reshard the shards/sources of the dataset, i.e. further split the current shards into more shards, + or propagate the resharding to the underlying iterable. + If the examples iterable can't be further resharded, then this method returns self. + """ + raise NotImplementedError(f"{type(self)} doesn't implement reshard_data_sources yet") + + def split_shard_indices_by_worker(self, num_shards: int, index: int, contiguous=True) -> list[int]: + if contiguous: + div = self.num_shards // num_shards + mod = self.num_shards % num_shards + start = div * index + min(index, mod) + end = start + div + (1 if index < mod else 0) + return list(range(start, end)) + else: + return list(range(index, self.num_shards, num_shards)) + + @property + def num_shards(self) -> int: + raise NotImplementedError(f"{type(self)} doesn't implement num_shards yet") + + def _init_state_dict(self) -> dict: + raise NotImplementedError(f"{type(self)} doesn't implement _init_state_dict yet") + + def load_state_dict(self, state_dict: dict) -> dict: + def _inner_load_state_dict(state, new_state): + if new_state is not None and isinstance(state, dict): + for key in new_state: + state[key] = _inner_load_state_dict(state[key], new_state[key]) + return state + elif new_state is not None and isinstance(state, list): + for i in range(len(state)): + state[i] = _inner_load_state_dict(state[i], new_state[i]) + return state + return new_state + + return _inner_load_state_dict(self._state_dict, state_dict) + + def state_dict(self) -> dict: + if self._state_dict: + return copy.deepcopy(self._state_dict) + raise RuntimeError("State dict is not initialized, please call ex_iterable._init_state_dict() first.") + + +class ExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + generate_examples_fn: Callable[..., Iterator[tuple[Key, dict]]], + kwargs: dict, + generate_more_kwargs_fn: Optional[Callable[..., Iterator[dict]]] = None, + ): + super().__init__() + self.generate_examples_fn = generate_examples_fn + self.kwargs = kwargs + + # for resharding + self.generate_more_kwargs_fn = generate_more_kwargs_fn + + def _init_state_dict(self) -> dict: + self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__} + return self._state_dict + + def __iter__(self): + shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0 + for gen_kwargs in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None): + shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0 + for key_example in islice(self.generate_examples_fn(**gen_kwargs), shard_example_idx_start, None): + if self._state_dict: + self._state_dict["shard_example_idx"] += 1 + yield key_example + if self._state_dict: + self._state_dict["shard_idx"] += 1 + self._state_dict["shard_example_idx"] = 0 + + def shuffle_data_sources(self, generator: np.random.Generator) -> "ExamplesIterable": + return ExamplesIterable( + self.generate_examples_fn, + _shuffle_gen_kwargs(copy.deepcopy(generator), self.kwargs), + self.generate_more_kwargs_fn, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ExamplesIterable": + """Keep only the requested shard.""" + gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards) + shard_indices = self.split_shard_indices_by_worker(num_shards, index, contiguous=contiguous) + requested_gen_kwargs = _merge_gen_kwargs([gen_kwargs_list[i] for i in shard_indices]) + return ExamplesIterable(self.generate_examples_fn, requested_gen_kwargs, self.generate_more_kwargs_fn) + + def reshard_data_sources(self) -> "ExamplesIterable": + """Split shars into more shards if possible.""" + if not self.generate_more_kwargs_fn: + return ExamplesIterable(self.generate_examples_fn, self.kwargs, self.generate_more_kwargs_fn) + gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards) + new_gen_kwargs = _merge_gen_kwargs( + [ + new_gen_kwargs + for gen_kwargs in gen_kwargs_list + for new_gen_kwargs in self.generate_more_kwargs_fn(**gen_kwargs) + ] + ) + return ExamplesIterable(self.generate_examples_fn, new_gen_kwargs, self.generate_more_kwargs_fn) + + @property + def num_shards(self) -> int: + return _number_of_shards_in_gen_kwargs(self.kwargs) + + +class ArrowExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + generate_tables_fn: Callable[..., Iterator[tuple[Key, pa.Table]]], + kwargs: dict, + generate_more_kwargs_fn: Optional[Callable[..., Iterator[dict]]] = None, + ): + super().__init__() + self.generate_tables_fn = generate_tables_fn + self.kwargs = kwargs + + # for resharding + self.generate_more_kwargs_fn = generate_more_kwargs_fn + + @property + def iter_arrow(self): + return self._iter_arrow + + def _init_state_dict(self) -> dict: + self._state_dict = {"shard_idx": 0, "shard_example_idx": 0, "type": self.__class__.__name__} + return self._state_dict + + def __iter__(self): + formatter = PythonFormatter() + shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0 + for gen_kwags in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None): + shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0 + shard_example_idx = 0 + for key, pa_table in self.generate_tables_fn(**gen_kwags): + if shard_example_idx + len(pa_table) <= shard_example_idx_start: + shard_example_idx += len(pa_table) + continue + for pa_subtable in pa_table.to_reader(max_chunksize=config.ARROW_READER_BATCH_SIZE_IN_DATASET_ITER): + formatted_batch = formatter.format_batch(pa_subtable) + for example in _batch_to_examples(formatted_batch): + if shard_example_idx >= shard_example_idx_start: + if self._state_dict: + self._state_dict["shard_example_idx"] += 1 + yield key, example + shard_example_idx += 1 + if self._state_dict: + self._state_dict["shard_idx"] += 1 + self._state_dict["shard_example_idx"] = 0 + + def _iter_arrow(self): + shard_idx_start = self._state_dict["shard_idx"] if self._state_dict else 0 + for gen_kwags in islice(_split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards), shard_idx_start, None): + shard_example_idx_start = self._state_dict["shard_example_idx"] if self._state_dict else 0 + shard_example_idx = 0 + for key, pa_table in self.generate_tables_fn(**gen_kwags): + shard_example_idx += len(pa_table) + if shard_example_idx <= shard_example_idx_start: + continue + if self._state_dict: + self._state_dict["shard_example_idx"] += len(pa_table) + yield key, pa_table + if self._state_dict: + self._state_dict["shard_idx"] += 1 + self._state_dict["shard_example_idx"] = 0 + + def shuffle_data_sources(self, generator: np.random.Generator) -> "ArrowExamplesIterable": + return ArrowExamplesIterable( + self.generate_tables_fn, _shuffle_gen_kwargs(copy.deepcopy(generator), self.kwargs), generator + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "ArrowExamplesIterable": + """Keep only the requested shard.""" + gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards) + shard_indices = self.split_shard_indices_by_worker(num_shards, index, contiguous=contiguous) + requested_gen_kwargs = _merge_gen_kwargs([gen_kwargs_list[i] for i in shard_indices]) + return ArrowExamplesIterable(self.generate_tables_fn, requested_gen_kwargs) + + def reshard_data_sources(self) -> "ArrowExamplesIterable": + """Split shars into more shards if possible.""" + if not self.generate_more_kwargs_fn: + return ArrowExamplesIterable(self.generate_tables_fn, self.kwargs, self.generate_more_kwargs_fn) + gen_kwargs_list = _split_gen_kwargs(self.kwargs, max_num_jobs=self.num_shards) + new_gen_kwargs = _merge_gen_kwargs( + [ + new_gen_kwargs + for gen_kwargs in gen_kwargs_list + for new_gen_kwargs in self.generate_more_kwargs_fn(**gen_kwargs) + ] + ) + return ArrowExamplesIterable(self.generate_tables_fn, new_gen_kwargs, self.generate_more_kwargs_fn) + + @property + def num_shards(self) -> int: + return _number_of_shards_in_gen_kwargs(self.kwargs) + + +class RebatchedArrowExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + batch_size: Optional[int], + drop_last_batch: bool = False, + force_convert_to_arrow: bool = False, + ): + super().__init__() + self.ex_iterable = ex_iterable + self.batch_size = batch_size + self.drop_last_batch = drop_last_batch + self.force_convert_to_arrow = force_convert_to_arrow + + @property + def iter_arrow(self): + return self._iter_arrow if self.ex_iterable.iter_arrow or self.force_convert_to_arrow else None + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + def _init_state_dict(self) -> dict: + self._state_dict = { + "examples_iterable": self.ex_iterable._init_state_dict(), + "previous_state": None, + "batch_idx": 0, + "num_chunks_since_previous_state": 0, + "cropped_chunk_length": 0, + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + yield from self.ex_iterable + + def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]: + """Iterate over sub-tables of size `batch_size`.""" + if self._state_dict and self._state_dict["previous_state"]: + self.ex_iterable.load_state_dict(self._state_dict["previous_state"]) + if self.ex_iterable.iter_arrow: + iterator = self.ex_iterable.iter_arrow() + elif self.force_convert_to_arrow: + iterator = _convert_to_arrow(self.ex_iterable, batch_size=1) + else: + raise RuntimeError( + "_iter_arrow is not available in RebatchedArrowExamplesIterable, use an examples iterable that implements _iter_arrow() or pass force_convert_to_arrow=True" + ) + if self.batch_size is None or self.batch_size <= 0: + if self._state_dict and self._state_dict["batch_idx"] > 0: + return + all_pa_table = pa.concat_tables([pa_table for _, pa_table in iterator]) + if self._state_dict: + self._state_dict["batch_idx"] = 1 + yield "all", all_pa_table + return + keys_buffer = [] + chunks_buffer = [] + chunks_buffer_size = 0 + num_chunks_to_skip = self._state_dict["num_chunks_since_previous_state"] if self._state_dict else 0 + chunk_length_to_crop = self._state_dict["cropped_chunk_length"] if self._state_dict else 0 + if self._state_dict: + previous_state = self.ex_iterable.state_dict() + self._state_dict["previous_state"] = previous_state + for key, pa_table in iterator: + for num_chunks_since_previous_state, chunk in enumerate(pa_table.to_reader(max_chunksize=self.batch_size)): + if num_chunks_to_skip > 1: + num_chunks_to_skip -= 1 + continue + elif num_chunks_to_skip == 1 and chunk_length_to_crop == 0: + num_chunks_to_skip -= 1 + continue + elif num_chunks_to_skip == 1 and chunk_length_to_crop > 0: + chunk = chunk.slice(chunk_length_to_crop, len(chunk) - chunk_length_to_crop) + num_chunks_to_skip = 0 + chunk_length_to_crop = 0 + if len(chunk) == 0: + continue + + if chunks_buffer_size + len(chunk) < self.batch_size: + keys_buffer.append(key) + chunks_buffer.append(chunk) + chunks_buffer_size += len(chunk) + continue + elif chunks_buffer_size + len(chunk) == self.batch_size: + keys_buffer.append(key) + chunks_buffer.append(chunk) + new_key = "_".join(str(_key) for _key in keys_buffer) + if self._state_dict: + self._state_dict["batch_idx"] += 1 + self._state_dict["num_chunks_since_previous_state"] += len(chunks_buffer) + self._state_dict["cropped_chunk_length"] = 0 + yield new_key, pa.Table.from_batches(chunks_buffer) + keys_buffer = [] + chunks_buffer = [] + chunks_buffer_size = 0 + if self._state_dict: + self._state_dict["previous_state"] = previous_state + self._state_dict["num_chunks_since_previous_state"] = num_chunks_since_previous_state + 1 + else: + cropped_chunk_length = self.batch_size - chunks_buffer_size + keys_buffer.append(f"{key}[:{cropped_chunk_length}]") + chunks_buffer.append(chunk.slice(0, cropped_chunk_length)) + new_key = "_".join(str(_key) for _key in keys_buffer) + if self._state_dict: + self._state_dict["batch_idx"] += 1 + self._state_dict["num_chunks_since_previous_state"] += len(chunks_buffer) + self._state_dict["cropped_chunk_length"] = cropped_chunk_length + yield new_key, pa.Table.from_batches(chunks_buffer) + keys_buffer = [f"{key}[{cropped_chunk_length}:]"] + chunks_buffer = [chunk.slice(cropped_chunk_length, len(chunk) - cropped_chunk_length)] + chunks_buffer_size = len(chunk) - cropped_chunk_length + if self._state_dict: + self._state_dict["previous_state"] = previous_state + self._state_dict["num_chunks_since_previous_state"] = num_chunks_since_previous_state + if self._state_dict: + previous_state = self.ex_iterable.state_dict() + if not self.drop_last_batch and chunks_buffer: + new_key = "_".join(str(_key) for _key in keys_buffer) + if self._state_dict: + self._state_dict["previous_state"] = previous_state + self._state_dict["batch_idx"] += 1 + self._state_dict["num_chunks_since_previous_state"] = 0 + self._state_dict["cropped_chunk_length"] = 0 + yield new_key, pa.Table.from_batches(chunks_buffer) + + def shuffle_data_sources(self, generator: np.random.Generator) -> "RebatchedArrowExamplesIterable": + return RebatchedArrowExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), + self.batch_size, + self.drop_last_batch, + self.force_convert_to_arrow, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "RebatchedArrowExamplesIterable": + return RebatchedArrowExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + self.batch_size, + self.drop_last_batch, + self.force_convert_to_arrow, + ) + + def reshard_data_sources(self) -> "RebatchedArrowExamplesIterable": + return RebatchedArrowExamplesIterable( + self.ex_iterable.reshard_data_sources(), self.batch_size, self.drop_last_batch, self.force_convert_to_arrow + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class SelectColumnsIterable(_BaseExamplesIterable): + def __init__(self, ex_iterable: _BaseExamplesIterable, column_names: list[str]): + super().__init__() + self.ex_iterable = ex_iterable + self.column_names = column_names + + @property + def iter_arrow(self): + if self.ex_iterable.iter_arrow: + return self._iter_arrow + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + def _init_state_dict(self) -> dict: + self._state_dict = self.ex_iterable._init_state_dict() + return self._state_dict + + def __iter__(self): + for idx, row in self.ex_iterable: + yield idx, {c: row[c] for c in self.column_names} + + def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]: + for idx, pa_table in self.ex_iterable.iter_arrow(): + if len(pa_table) > 0: # empty tables have no schema + yield idx, pa_table.select(self.column_names) + + def shuffle_data_sources(self, generator: np.random.Generator) -> "SelectColumnsIterable": + return SelectColumnsIterable(self.ex_iterable.shuffle_data_sources(generator), self.column_names) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SelectColumnsIterable": + return SelectColumnsIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), self.column_names + ) + + def reshard_data_sources(self) -> "SelectColumnsIterable": + return SelectColumnsIterable(self.ex_iterable.reshard_data_sources(), self.column_names) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class StepExamplesIterable(_BaseExamplesIterable): + def __init__(self, ex_iterable: _BaseExamplesIterable, step: int, offset: int): + super().__init__() + self.ex_iterable = ex_iterable + self.step = step + self.offset = offset + + @property + def iter_arrow(self): + return self._iter_arrow if self.ex_iterable.iter_arrow else None + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + def _init_state_dict(self) -> dict: + self._state_dict = { + "examples_iterable": self.ex_iterable._init_state_dict(), + "stepped": 0, + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + ex_iterator = iter(self.ex_iterable) + while True: + batch = list(islice(ex_iterator, self.step)) + if len(batch) > self.offset: + yield batch[self.offset] + else: + break + + def _iter_arrow(self): + stepped = self._state_dict["stepped"] if self._state_dict else 0 + for key, pa_table in self.ex_iterable.iter_arrow(): + stepped_pa_table = pa_table.take( + pa.array(range((self.offset - stepped) % self.step, len(pa_table), self.step), type=pa.int64()) + ) + stepped = (stepped + len(pa_table)) % self.step + if self._state_dict: + self._state_dict["stepped"] = stepped + yield key, stepped_pa_table + + def shuffle_data_sources(self, generator: np.random.Generator) -> "StepExamplesIterable": + return StepExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), step=self.step, offset=self.offset + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "StepExamplesIterable": + return StepExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + step=self.step, + offset=self.offset, + ) + + def reshard_data_sources(self) -> "StepExamplesIterable": + return StepExamplesIterable( + self.ex_iterable.reshard_data_sources(), + step=self.step, + offset=self.offset, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class CyclingMultiSourcesExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + ex_iterables: list[_BaseExamplesIterable], + stopping_strategy: Literal[ + "first_exhausted", "all_exhausted", "all_exhausted_without_replacement" + ] = "first_exhausted", + ): + super().__init__() + self.ex_iterables = ex_iterables + self.stopping_strategy = stopping_strategy + + # if undersampling ("first_exhausted"), we stop as soon as one dataset is exhausted + # if oversampling ("all_exhausted"), we stop as soons as every dataset is exhausted, i.e as soon as every samples of every dataset has been visited at least once + # if sampling without replacement ("all_exhausted_without_replacement"), we stop once all samples of every dataset has been visited exactly once. + self.bool_strategy_func = ( + np.all if (stopping_strategy in ("all_exhausted", "all_exhausted_without_replacement")) else np.any + ) + + @property + def is_typed(self): + return self.ex_iterables[0].is_typed + + @property + def features(self): + return self.ex_iterables[0].features + + @property + def iter_arrow(self): + # iterate on arrow tables if all ex_iterables can iterate + return self._iter_arrow if all(ex_iterable.iter_arrow for ex_iterable in self.ex_iterables) else None + + def _get_indices_iterator(self): + # this is an infinite iterator to keep track of which iterator we want to pick examples from + ex_iterable_idx = self._state_dict["ex_iterable_idx"] if self._state_dict else 0 + for next_ex_iterable_idx in islice(cycle(range(len(self.ex_iterables))), ex_iterable_idx + 1, None): + if self._state_dict: + self._state_dict["ex_iterable_idx"] = next_ex_iterable_idx + yield ex_iterable_idx + ex_iterable_idx = next_ex_iterable_idx + + def _init_state_dict(self) -> dict: + self._state_dict = { + "ex_iterable_idx": 0, + "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables], + "previous_states": [None] * len(self.ex_iterables), + "is_exhausted": [False] * len(self.ex_iterables), + "type": self.__class__.__name__, + } + return self._state_dict + + def _iter_arrow(self): + # we use this to buffer one example of each iterator to know if an iterator is exhausted + nexts = [None] * len(self.ex_iterables) + # because of that, we need to rewind 1 example when reloading the state dict + if self._state_dict: + for i in range(len(self.ex_iterables)): + if self._state_dict["previous_states"][i] is not None: + self.ex_iterables[i].load_state_dict(self._state_dict["previous_states"][i]) + iterators = [ex_iterable.iter_arrow() for ex_iterable in self.ex_iterables] + + indices_iterator = self._get_indices_iterator() + + is_exhausted = ( + np.array(self._state_dict["is_exhausted"]) if self._state_dict else np.full(len(self.ex_iterables), False) + ) + for i in indices_iterator: + # if the stopping criteria is met, break the main for loop + if self.bool_strategy_func(is_exhausted): + break + # Skip exhausted iterators if we sample without replacement + if is_exhausted[i] and self.stopping_strategy in ["all_exhausted_without_replacement"]: + continue + # let's pick one example from the iterator at index i + if nexts[i] is None: + nexts[i] = next(iterators[i], False) + result = nexts[i] + if self._state_dict: + self._state_dict["previous_states"][i] = deepcopy(self._state_dict["ex_iterables"][i]) + nexts[i] = next(iterators[i], False) + + # the iterator is exhausted + if nexts[i] is False: + is_exhausted[i] = True + if self._state_dict: + self._state_dict["is_exhausted"][i] = True + # we reset it in case the stopping crtieria isn't met yet and we sample with replacement + if self.stopping_strategy not in ["all_exhausted_without_replacement"]: + nexts[i] = None + if self._state_dict: + self._state_dict["ex_iterables"][i] = self.ex_iterables[i]._init_state_dict() + self._state_dict["previous_states"][i] = None + iterators[i] = self.ex_iterables[i]._iter_arrow() + + if result is not False: + yield result + + def __iter__(self): + # we use this to buffer one example of each iterator to know if an iterator is exhausted + nexts = [None] * len(self.ex_iterables) + # because of that, we need to rewind 1 example when reloading the state dict + if self._state_dict: + for i in range(len(self.ex_iterables)): + if self._state_dict["previous_states"][i] is not None: + self.ex_iterables[i].load_state_dict(self._state_dict["previous_states"][i]) + iterators = [iter(ex_iterable) for ex_iterable in self.ex_iterables] + + indices_iterator = self._get_indices_iterator() + + is_exhausted = ( + np.array(self._state_dict["is_exhausted"]) if self._state_dict else np.full(len(self.ex_iterables), False) + ) + for i in indices_iterator: + # if the stopping criteria is met, break the main for loop + if self.bool_strategy_func(is_exhausted): + break + # let's pick one example from the iterator at index i + if is_exhausted[i] and self.stopping_strategy in ["all_exhausted_without_replacement"]: + continue + if nexts[i] is None: + nexts[i] = next(iterators[i], False) + result = nexts[i] + if self._state_dict: + self._state_dict["previous_states"][i] = deepcopy(self._state_dict["ex_iterables"][i]) + nexts[i] = next(iterators[i], False) + + # the iterator is exhausted + if nexts[i] is False: + is_exhausted[i] = True + if self._state_dict: + self._state_dict["is_exhausted"][i] = True + # we reset it in case the stopping crtieria isn't met yet + if self.stopping_strategy not in ["all_exhausted_without_replacement"]: + nexts[i] = None + if self._state_dict: + self._state_dict["ex_iterables"][i] = self.ex_iterables[i]._init_state_dict() + self._state_dict["previous_states"][i] = None + iterators[i] = iter(self.ex_iterables[i]) + if result is not False: + yield result + + def shuffle_data_sources(self, generator: np.random.Generator) -> "CyclingMultiSourcesExamplesIterable": + """Shuffle each underlying examples iterable.""" + ex_iterables = [ex_iterable.shuffle_data_sources(generator) for ex_iterable in self.ex_iterables] + return CyclingMultiSourcesExamplesIterable(ex_iterables, self.stopping_strategy) + + @property + def num_shards(self) -> int: + return min(ex_iterable.num_shards for ex_iterable in self.ex_iterables) if self.ex_iterables else 0 + + def shard_data_sources( + self, num_shards: int, index: int, contiguous=True + ) -> "CyclingMultiSourcesExamplesIterable": + """Either keep only the requested shard, or propagate the request to the underlying iterable.""" + if num_shards < self.num_shards: + return CyclingMultiSourcesExamplesIterable( + [ + iterable.shard_data_sources(num_shards, index, contiguous=contiguous) + for iterable in self.ex_iterables + ], + stopping_strategy=self.stopping_strategy, + ) + elif index < self.num_shards: + return CyclingMultiSourcesExamplesIterable( + [ + iterable.shard_data_sources(self.num_shards, index, contiguous=contiguous) + for iterable in self.ex_iterables + ], + stopping_strategy=self.stopping_strategy, + ) + else: + return CyclingMultiSourcesExamplesIterable( + [], + stopping_strategy=self.stopping_strategy, + ) + + def reshard_data_sources(self) -> "CyclingMultiSourcesExamplesIterable": + return CyclingMultiSourcesExamplesIterable( + [iterable.reshard_data_sources() for iterable in self.ex_iterables], + stopping_strategy=self.stopping_strategy, + ) + + +class VerticallyConcatenatedMultiSourcesExamplesIterable(_BaseExamplesIterable): + """ + VerticallyConcatenatedMultiSourcesExamplesIterable simply chains the input iterables. + It doesn't require the examples iterables to always yield the same columns. + Instead, this is handled by the `IterableDataset` class or `FormattedExamplesIterable`. + + For information, `IterableDataset` merges the features of all the datasets to concatenate into one. + We use `IterableDataset._resolve_features` to obtain the features of all the datasets to concatenate. + + Then for each example, `IterableDataset` and `FormattedExamplesIterable` automatically fill missing columns with None. + This is done with `_apply_feature_types_on_example`. + """ + + def __init__(self, ex_iterables: list[_BaseExamplesIterable]): + super().__init__() + self.ex_iterables = ex_iterables + + @property + def is_typed(self): + return self.ex_iterables[0].is_typed + + @property + def features(self): + return self.ex_iterables[0].features + + @property + def iter_arrow(self): + if all(ex_iterable.iter_arrow is not None for ex_iterable in self.ex_iterables): + return self._iter_arrow + + def _init_state_dict(self) -> dict: + self._state_dict = { + "ex_iterable_idx": 0, + "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables], + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + ex_iterable_idx_start = self._state_dict["ex_iterable_idx"] if self._state_dict else 0 + for ex_iterable in islice(self.ex_iterables, ex_iterable_idx_start, None): + yield from ex_iterable + if self._state_dict: + self._state_dict["ex_iterable_idx"] += 1 + + def _iter_arrow(self): + ex_iterable_idx_start = self._state_dict["ex_iterable_idx"] if self._state_dict else 0 + for ex_iterable in islice(self.ex_iterables, ex_iterable_idx_start, None): + yield from ex_iterable.iter_arrow() + if self._state_dict: + self._state_dict["ex_iterable_idx"] += 1 + + def shuffle_data_sources( + self, generator: np.random.Generator + ) -> "VerticallyConcatenatedMultiSourcesExamplesIterable": + """Shuffle all shards.""" + rng = deepcopy(generator) + single_shard_ex_iterables = [ + ex_iterable.shard_data_sources(num_shards=ex_iterable.num_shards, index=index) + for ex_iterable in self.ex_iterables + for index in range(ex_iterable.num_shards) + ] + rng.shuffle(single_shard_ex_iterables) + return VerticallyConcatenatedMultiSourcesExamplesIterable(single_shard_ex_iterables) + + @property + def num_shards(self) -> int: + return sum(ex_iterable.num_shards for ex_iterable in self.ex_iterables) + + def shard_data_sources( + self, num_shards: int, index: int, contiguous=True + ) -> "VerticallyConcatenatedMultiSourcesExamplesIterable": + """Keep only the requested shard""" + single_shard_ex_iterables = [ + ex_iterable.shard_data_sources(num_shards=ex_iterable.num_shards, index=index) + for ex_iterable in self.ex_iterables + for index in range(ex_iterable.num_shards) + ] + shard_indices = self.split_shard_indices_by_worker(num_shards, index, contiguous=contiguous) + return VerticallyConcatenatedMultiSourcesExamplesIterable( + [single_shard_ex_iterables[i] for i in shard_indices] + ) + + def reshard_data_sources(self) -> "VerticallyConcatenatedMultiSourcesExamplesIterable": + return VerticallyConcatenatedMultiSourcesExamplesIterable( + [iterable.reshard_data_sources() for iterable in self.ex_iterables] + ) + + +def _check_column_names(column_names: list[str]): + """Check the column names to make sure they don't contain duplicates.""" + counter = Counter(column_names) + if not all(count == 1 for count in counter.values()): + duplicated_columns = [col for col in counter if counter[col] > 1] + raise ValueError( + f"The examples iterables can't have duplicated columns but columns {duplicated_columns} are duplicated." + ) + + +class HorizontallyConcatenatedMultiSourcesExamplesIterable(_BaseExamplesIterable): + """ + HorizontallyConcatenatedMultiSourcesExamplesIterable merges examples together for the input list of iterables. + It also checks that there are no duplicate columns (otherwise we don't know which one to keep). + This check is done once when yielding the first example. + + However it doesn't fill missing columns with None. + Instead, this is handled by the `IterableDataset` class or `FormattedExamplesIterable`. + + For information, `IterableDataset` merges the features of all the datasets to concatenate into one. + We use `IterableDataset._resolve_features` to obtain the features of all the datasets to concatenate. + + Then for each example, `IterableDataset` and `FormattedExamplesIterable` automatically fill missing columns with None. + This is done with `_apply_feature_types_on_example`. + """ + + def __init__(self, ex_iterables: list[_BaseExamplesIterable]): + super().__init__() + self.ex_iterables = ex_iterables + + @property + def iter_arrow(self): + return ( + self._iter_arrow + if all( + isinstance(ex_iterable, RebatchedArrowExamplesIterable) and ex_iterable.ex_iterable.iter_arrow + for ex_iterable in self.ex_iterables + ) + or (len(self.ex_iterables) < 2 and all(ex_iterable.iter_arrow for ex_iterable in self.ex_iterables)) + else None + ) + + @property + def is_typed(self): + return self.ex_iterables[0].is_typed + + @property + def features(self): + return self.ex_iterables[0].features + + def _init_state_dict(self) -> dict: + self._state_dict = { + "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables], + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + ex_iterators = [iter(ex_iterable) for ex_iterable in self.ex_iterables] + for i in itertools.count(): + keys = [] + examples = [] + for ex_iterator in list(ex_iterators): + try: + key, example = next(ex_iterator) + keys.append(key) + examples.append(example) + except StopIteration: + ex_iterators.remove(ex_iterator) + if ex_iterators: + if i == 0: + _check_column_names([column_name for example in examples for column_name in example]) + new_example = {} + for example in examples: + new_example.update(example) + new_key = "_".join(str(key) for key in keys) + yield new_key, new_example + else: + break + + def _iter_arrow(self): + pa_table_iterators = [iter(ex_iterable.iter_arrow()) for ex_iterable in self.ex_iterables] + for i in itertools.count(): + keys = [] + pa_tables = [] + for pa_table_iterator in list(pa_table_iterators): + try: + key, pa_table = next(pa_table_iterator) + keys.append(key) + pa_tables.append(pa_table) + except StopIteration: + pa_table_iterators.remove(pa_table_iterator) + if pa_table_iterators: + if i == 0: + _check_column_names( + [column_name for pa_table in pa_tables for column_name in pa_table.column_names] + ) + for j, table in enumerate(pa_tables): + if j == 0: + new_pa_table = table + else: + for name, col in zip(table.column_names, table.columns): + new_pa_table = pa_table.append_column(name, col) + new_key = "_".join(str(key) for key in keys) + yield new_key, new_pa_table + else: + break + + def shuffle_data_sources( + self, generator: np.random.Generator + ) -> "HorizontallyConcatenatedMultiSourcesExamplesIterable": + """Doesn't shuffle the wrapped examples iterable since it would break the alignment between them.""" + return self + + @property + def num_shards(self) -> int: + return 1 + + def shard_data_sources( + self, num_shards: int, index: int, contiguous=True + ) -> "HorizontallyConcatenatedMultiSourcesExamplesIterable": + """Doesn't shard the wrapped examples iterable since it would break the alignment between them.""" + return self + + def reshard_data_sources(self) -> "HorizontallyConcatenatedMultiSourcesExamplesIterable": + """Doesn't reshard the wrapped examples iterable since it would break the alignment between them.""" + return self + + +class RandomlyCyclingMultiSourcesExamplesIterable(CyclingMultiSourcesExamplesIterable): + def __init__( + self, + ex_iterables: list[_BaseExamplesIterable], + generator: np.random.Generator, + probabilities: Optional[list[float]] = None, + stopping_strategy: Literal[ + "first_exhausted", "all_exhausted", "all_exhausted_without_replacement" + ] = "first_exhausted", + ): + super().__init__(ex_iterables, stopping_strategy) + self.generator = deepcopy(generator) + self.probabilities = probabilities + + def shift_rngs(self, value: int) -> "_BaseExamplesIterable": + rng = deepcopy(self.generator) + new_seed = rng.integers(0, 1 << 63) - value + return RandomlyCyclingMultiSourcesExamplesIterable( + ex_iterables=self.ex_iterables, + generator=np.random.default_rng(seed=new_seed), + probabilities=self.probabilities, + stopping_strategy=self.stopping_strategy, + ) + + @property + def is_typed(self): + return self.ex_iterables[0].is_typed + + @property + def features(self): + return self.ex_iterables[0].features + + def _get_indices_iterator(self): + rng = deepcopy(self.generator) + num_sources = len(self.ex_iterables) + random_batch_size = 1000 + # this is an infinite iterator that randomly samples the index of the source to pick examples from + index_offset = self._state_dict["bit_generator_index_offset"] if self._state_dict else 0 + if self._state_dict: + rng.bit_generator.state = self._state_dict["bit_generator_state"] + if self.probabilities is None: + while True: + for i in islice(rng.integers(0, num_sources, size=random_batch_size), index_offset, None): + index_offset = (index_offset + 1) % random_batch_size + if self._state_dict: + self._state_dict["bit_generator_index_offset"] = index_offset + if index_offset == 0: + self._state_dict["bit_generator_state"] = rng.bit_generator.state + yield int(i) + else: + while True: + for i in islice( + rng.choice(num_sources, size=random_batch_size, p=self.probabilities), index_offset, None + ): + index_offset = (index_offset + 1) % random_batch_size + if self._state_dict: + self._state_dict["bit_generator_index_offset"] = index_offset + if index_offset == 0: + self._state_dict["bit_generator_state"] = rng.bit_generator.state + yield int(i) + + def _init_state_dict(self) -> dict: + self._state_dict = { + "bit_generator_state": self.generator.bit_generator.state, + "bit_generator_index_offset": 0, + "ex_iterables": [ex_iterable._init_state_dict() for ex_iterable in self.ex_iterables], + "previous_states": [None] * len(self.ex_iterables), + "is_exhausted": [False] * len(self.ex_iterables), + "type": self.__class__.__name__, + } + return self._state_dict + + def shuffle_data_sources(self, generator: np.random.Generator) -> "RandomlyCyclingMultiSourcesExamplesIterable": + """Shuffle the data sources of each wrapped examples iterable.""" + ex_iterables = [ex_iterable.shuffle_data_sources(generator) for ex_iterable in self.ex_iterables] + return RandomlyCyclingMultiSourcesExamplesIterable( + ex_iterables, + generator=generator, + probabilities=self.probabilities, + stopping_strategy=self.stopping_strategy, + ) + + def shard_data_sources( + self, num_shards: int, index: int, contiguous=True + ) -> "RandomlyCyclingMultiSourcesExamplesIterable": + """Either keep only the requested shard, or propagate the request to the underlying iterable.""" + if num_shards < self.num_shards: + return RandomlyCyclingMultiSourcesExamplesIterable( + [ + iterable.shard_data_sources(num_shards, index, contiguous=contiguous) + for iterable in self.ex_iterables + ], + self.generator, + self.probabilities, + self.stopping_strategy, + ) + elif index < self.num_shards: + return RandomlyCyclingMultiSourcesExamplesIterable( + [ + iterable.shard_data_sources(self.num_shards, index, contiguous=contiguous) + for iterable in self.ex_iterables + ], + self.generator, + self.probabilities, + self.stopping_strategy, + ) + else: + return RandomlyCyclingMultiSourcesExamplesIterable( + [], + self.generator, + self.probabilities, + self.stopping_strategy, + ) + + def reshard_data_sources(self) -> "RandomlyCyclingMultiSourcesExamplesIterable": + """Either keep only the requested shard, or propagate the request to the underlying iterable.""" + return RandomlyCyclingMultiSourcesExamplesIterable( + [iterable.reshard_data_sources() for iterable in self.ex_iterables], + self.generator, + self.probabilities, + self.stopping_strategy, + ) + + +def _table_output_to_arrow(output) -> pa.Table: + if isinstance(output, pa.Table): + return output + if isinstance(output, (pd.DataFrame, pd.Series)): + return pa.Table.from_pandas(output) + if config.POLARS_AVAILABLE and "polars" in sys.modules: + import polars as pl + + if isinstance(output, (pl.DataFrame, pl.Series)): + return output.to_arrow() + return output + + +class MappedExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + function: Callable, + with_indices: bool = False, + input_columns: Optional[list[str]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[list[str]] = None, + fn_kwargs: Optional[dict] = None, + formatting: Optional["FormattingConfig"] = None, + features: Optional[Features] = None, + max_num_running_async_map_functions_in_parallel: Optional[int] = None, + ): + super().__init__() + self.ex_iterable = ex_iterable + self.function = function + self.batched = batched + self.batch_size = batch_size + self.drop_last_batch = drop_last_batch + self.remove_columns = remove_columns + self.with_indices = with_indices + self.input_columns = input_columns + self.fn_kwargs = fn_kwargs or {} + self.formatting = formatting # required for iter_arrow + self._features = features + self.max_num_running_async_map_functions_in_parallel = ( + max_num_running_async_map_functions_in_parallel or config.MAX_NUM_RUNNING_ASYNC_MAP_FUNCTIONS_IN_PARALLEL + ) + # sanity checks + if formatting and formatting.is_table: + # batch_size should match for iter_arrow + if not isinstance(ex_iterable, RebatchedArrowExamplesIterable): + raise ValueError( + f"The {formatting.format_type.capitalize()}-formatted {type(self).__name__} has underlying iterable " + f"that is a {type(ex_iterable).__name__} instead of a RebatchedArrowExamplesIterable." + ) + elif not ex_iterable.iter_arrow: + raise ValueError( + f"The {formatting.format_type.capitalize()}-formatted {type(self).__name__} has underlying iterable " + f"that is a {type(ex_iterable).__name__} but doesnt' implement iter_arrow(), a possible fix could be " + "to use RebatchedArrowExamplesIterable(..., force_convert_to_arrow=True)." + ) + elif ex_iterable.batch_size != (batch_size if batched else 1): + raise ValueError( + f"The {formatting.format_type.capitalize()}-formatted {type(self).__name__} has batch_size={batch_size if batched else 1} which is " + f"different from {ex_iterable.batch_size=} from its underlying iterable." + ) + # to enable graceful ends + self._owned_loops_and_tasks: list[tuple[asyncio.AbstractEventLoop, list[asyncio.Task]]] = [] + + @property + def iter_arrow(self): + if self.formatting and self.formatting.is_table: + return self._iter_arrow + + @property + def is_typed(self): + return self.features is not None # user has extracted features + + @property + def features(self): + return self._features + + def _init_state_dict(self) -> dict: + self._state_dict = { + "examples_iterable": self.ex_iterable._init_state_dict(), + "previous_state": None, + "num_examples_since_previous_state": 0, + "previous_state_example_idx": 0, + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + if self.formatting and self.formatting.is_table: + formatter = PythonFormatter() + for key, pa_table in self._iter_arrow(max_chunksize=1): + yield key, formatter.format_row(pa_table) + else: + yield from self._iter() + + def _iter(self): + current_idx = self._state_dict["previous_state_example_idx"] if self._state_dict else 0 + if self._state_dict and self._state_dict["previous_state"]: + self.ex_iterable.load_state_dict(self._state_dict["previous_state"]) + num_examples_to_skip = self._state_dict["num_examples_since_previous_state"] + else: + num_examples_to_skip = 0 + iterator = iter(self.ex_iterable) + + # We use the same logic as in Dataset.map, but with less features/formatting + # since they're handled by FormattedExamplesIterable + + if self.formatting: + formatter = get_formatter(self.formatting.format_type) + format_dict = formatter.recursive_tensorize if isinstance(formatter, TensorFormatter) else None + else: + format_dict = None + + def iter_batched_inputs(): + nonlocal current_idx + for key, example in iterator: + # If `batched`, first build the batch, if `batch_size` is None or <=0, then the batch is the whole dataset + iterator_batch = ( + iterator + if self.batch_size is None or self.batch_size <= 0 + else islice(iterator, self.batch_size - 1) + ) + key_examples_list = [(key, example)] + list(iterator_batch) + keys, examples = zip(*key_examples_list) + # the new key is the concatenation of the examples keys from the batch + key = "_".join(str(key) for key in keys) + if ( + self.drop_last_batch + and self.batch_size is not None + and self.batch_size > 0 + and len(examples) < self.batch_size + ): # ignore last batch + return + batch = _examples_to_batch(examples) + # we need to format here in case we need to stack tensors together + batch = format_dict(batch) if format_dict else batch + indices = [current_idx + i for i in range(len(key_examples_list))] + current_idx += len(indices) + yield indices, (key, batch) + + def iter_inputs(): + nonlocal current_idx + for key, example in iterator: + # If not batched, we can apply the transform and yield the example directly + # first copy the example, since we might drop some keys + example = dict(example) + # no need to do formatting here + current_idx += 1 + yield current_idx - 1, (key, example) + + def validate_function_output(processed_inputs): + if self.batched and processed_inputs: + first_col = next(iter(processed_inputs)) + bad_cols = [ + col for col in processed_inputs if len(processed_inputs[col]) != len(processed_inputs[first_col]) + ] + if bad_cols: + raise ValueError( + f"Column lengths mismatch: columns {bad_cols} have length {[len(processed_inputs[col]) for col in bad_cols]} " + f"while {first_col} has length {len(processed_inputs[first_col])}." + ) + + def prepare_inputs(key_example, indices): + key, example = key_example + fn_args = [example] if self.input_columns is None else [example[col] for col in self.input_columns] + additional_args = () + if self.with_indices: + fn_args += (indices,) + inputs = dict(example) + return inputs, fn_args, additional_args, self.fn_kwargs + + def prepare_outputs(key_example, inputs, processed_inputs): + validate_function_output(processed_inputs) + # this logic mimics the one in Dataset.map + if self.remove_columns: + for c in self.remove_columns: + if c in inputs: + del inputs[c] + if processed_inputs is key_example[1] and c in processed_inputs: + del processed_inputs[c] + transformed_inputs = {**inputs, **processed_inputs} + # no need to do features decoding here + return transformed_inputs + + def apply_function(key_example, indices): + """Utility to apply the function on a selection of columns.""" + inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(key_example, indices) + processed_inputs = self.function(*fn_args, *additional_args, **fn_kwargs) + return prepare_outputs(key_example, inputs, processed_inputs) + + async def async_apply_function(key_example, indices): + """Utility to apply the function on a selection of columns. Same code but async""" + inputs, fn_args, additional_args, fn_kwargs = prepare_inputs(key_example, indices) + processed_inputs = await self.function(*fn_args, *additional_args, **fn_kwargs) + return prepare_outputs(key_example, inputs, processed_inputs) + + tasks: list[asyncio.Task] = [] + if inspect.iscoroutinefunction(self.function): + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + self._owned_loops_and_tasks.append((loop, tasks)) + else: + loop = None + + def iter_outputs(): + nonlocal tasks, loop + inputs_iterator = iter_batched_inputs() if self.batched else iter_inputs() + if inspect.iscoroutinefunction(self.function): + if self._state_dict: + previous_state = self.ex_iterable.state_dict() + self._state_dict["previous_state"] = previous_state + previous_state_task = None + previous_state_example_idx = self._state_dict["previous_state_example_idx"] + indices: Union[list[int], list[list[int]]] = [] + for i, key_example in inputs_iterator: + indices.append(i) + tasks.append(loop.create_task(async_apply_function(key_example, i))) + # keep the total active tasks under a certain number + if len(tasks) >= self.max_num_running_async_map_functions_in_parallel: + done, pending = loop.run_until_complete( + asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + ) + while tasks and len(pending) >= self.max_num_running_async_map_functions_in_parallel: + done, pending = loop.run_until_complete( + asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + ) + if len(tasks) >= 10 * self.max_num_running_async_map_functions_in_parallel: + loop.run_until_complete(tasks[0]) + # yield finished tasks + while tasks and tasks[0].done(): + i, task = indices.pop(0), tasks.pop(0) + yield i, task.result() + if self._state_dict and task is previous_state_task: + self._state_dict["previous_state"] = previous_state + self._state_dict["num_examples_since_previous_state"] = 0 + self._state_dict["previous_state_example_idx"] = previous_state_example_idx + previous_state, previous_state_task = None, None + # checkpoint + if self._state_dict and previous_state_task is None and tasks: + previous_state = self.ex_iterable.state_dict() + previous_state_task = tasks[-1] + previous_state_example_idx = current_idx + while tasks: + yield indices[0], loop.run_until_complete(tasks[0]) + indices.pop(0), tasks.pop(0) + else: + if self._state_dict: + if self.batched: + self._state_dict["previous_state"] = self.ex_iterable.state_dict() + self._state_dict["num_examples_since_previous_state"] = 0 + self._state_dict["previous_state_example_idx"] = current_idx + for i, key_example in inputs_iterator: + if self._state_dict: + if not self.batched: + self._state_dict["previous_state_example_idx"] = current_idx + yield i, apply_function(key_example, i) + if self._state_dict: + if self.batched: + self._state_dict["previous_state"] = self.ex_iterable.state_dict() + self._state_dict["num_examples_since_previous_state"] = 0 + self._state_dict["previous_state_example_idx"] = current_idx + + try: + outputs = iter_outputs() + if self.batched: + outputs = ( + (key, transformed_example) + for key, transformed_batch in outputs + for transformed_example in _batch_to_examples(transformed_batch) + ) + for key, transformed_example in outputs: + if self._state_dict and self._state_dict["previous_state"] is not None: + self._state_dict["num_examples_since_previous_state"] += 1 + if num_examples_to_skip > 0: + num_examples_to_skip -= 1 + continue + yield key, transformed_example + except (Exception, KeyboardInterrupt): + if loop: + logger.debug(f"Canceling {len(tasks)} async tasks.") + for task in tasks: + task.cancel(msg="KeyboardInterrupt") + try: + loop.run_until_complete(asyncio.gather(*tasks)) + except (asyncio.CancelledError, ValueError): + logger.debug("Tasks canceled.") + raise + + def _iter_arrow(self, max_chunksize: Optional[int] = None) -> Iterator[tuple[Key, pa.Table]]: + formatter: TableFormatter = get_formatter(self.formatting.format_type) if self.formatting else ArrowFormatter() + if self.ex_iterable.iter_arrow: + iterator = self.ex_iterable.iter_arrow() + else: + iterator = _convert_to_arrow( + self.ex_iterable, + batch_size=self.batch_size if self.batched else 1, + drop_last_batch=self.drop_last_batch, + ) + if self._state_dict and self._state_dict["previous_state"]: + self.ex_iterable.load_state_dict(self._state_dict["previous_state"]) + num_examples_to_skip = self._state_dict["num_examples_since_previous_state"] + else: + num_examples_to_skip = 0 + if self._state_dict and max_chunksize is not None: + self._state_dict["previous_state"] = self.ex_iterable.state_dict() + self._state_dict["num_examples_since_previous_state"] = 0 + current_idx = self._state_dict["previous_state_example_idx"] if self._state_dict else 0 + for key, pa_table in iterator: + if ( + self.batched + and self.batch_size is not None + and len(pa_table) < self.batch_size + and self.drop_last_batch + ): + return + # first build the batch + function_args = ( + [formatter.format_batch(pa_table)] + if self.input_columns is None + else [pa_table[col] for col in self.input_columns] + ) + if self.with_indices: + if self.batched: + function_args.append([current_idx + i for i in range(len(pa_table))]) + else: + function_args.append(current_idx) + # then apply the transform + output = self.function(*function_args, **self.fn_kwargs) + output_table = _table_output_to_arrow(output) + if not isinstance(output_table, pa.Table): + raise TypeError( + f"Provided `function` which is applied to {formatter.table_type} returns a variable of type " + f"{type(output)}. Make sure provided `function` returns a {formatter.table_type} to update the dataset." + ) + # we don't need to merge results for consistency with Dataset.map which merges iif both input and output are dicts + # then remove the unwanted columns + if self.remove_columns: + for column in self.remove_columns: + if column in output_table.column_names: + output_table = output_table.remove_column(output_table.column_names.index(column)) + # return output + if max_chunksize is None: + current_idx += len(pa_table) + if self._state_dict: + self._state_dict["previous_state_example_idx"] += len(pa_table) + yield key, output_table + else: + for i, pa_subtable in enumerate(output_table.to_reader(max_chunksize=max_chunksize)): + current_idx += 1 + if self._state_dict: + self._state_dict["num_examples_since_previous_state"] += 1 + if num_examples_to_skip > 0: + num_examples_to_skip -= 1 + continue + yield f"{key}_{i}", pa_subtable + if self._state_dict: + self._state_dict["previous_state"] = self.ex_iterable.state_dict() + self._state_dict["num_examples_since_previous_state"] = 0 + self._state_dict["previous_state_example_idx"] += len(pa_table) + + def shuffle_data_sources(self, generator: np.random.Generator) -> "MappedExamplesIterable": + """Shuffle the wrapped examples iterable.""" + return MappedExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), + function=self.function, + with_indices=self.with_indices, + input_columns=self.input_columns, + batched=self.batched, + batch_size=self.batch_size, + drop_last_batch=self.drop_last_batch, + remove_columns=self.remove_columns, + fn_kwargs=self.fn_kwargs, + formatting=self.formatting, + features=self.features, + max_num_running_async_map_functions_in_parallel=self.max_num_running_async_map_functions_in_parallel, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "MappedExamplesIterable": + """Keep only the requested shard.""" + return MappedExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + function=self.function, + with_indices=self.with_indices, + input_columns=self.input_columns, + batched=self.batched, + batch_size=self.batch_size, + drop_last_batch=self.drop_last_batch, + remove_columns=self.remove_columns, + fn_kwargs=self.fn_kwargs, + formatting=self.formatting, + features=self.features, + max_num_running_async_map_functions_in_parallel=self.max_num_running_async_map_functions_in_parallel, + ) + + def reshard_data_sources(self) -> "MappedExamplesIterable": + return MappedExamplesIterable( + self.ex_iterable.reshard_data_sources(), + function=self.function, + with_indices=self.with_indices, + input_columns=self.input_columns, + batched=self.batched, + batch_size=self.batch_size, + drop_last_batch=self.drop_last_batch, + remove_columns=self.remove_columns, + fn_kwargs=self.fn_kwargs, + formatting=self.formatting, + features=self.features, + max_num_running_async_map_functions_in_parallel=self.max_num_running_async_map_functions_in_parallel, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +def _add_mask( + input: Union[dict, pa.Table], + mask: Union[bool, list, pa.Array, pa.ChunkedArray, pa.BooleanScalar], + mask_column_name: str, +): + if isinstance(input, pa.Table): + if not isinstance(mask, (list, pa.Array, pa.ChunkedArray)): + mask = pa.array([mask], type=pa.bool_()) + return input.append_column(mask_column_name, mask) + else: + return {mask_column_name: mask} + + +def add_mask(mask_function: Callable, input: Union[dict, pa.Table], *args, mask_column_name: str, **kwargs): + mask = mask_function(input, *args, **kwargs) + return _add_mask(input, mask, mask_column_name) + + +async def async_add_mask( + mask_function: Callable, input: Union[dict, pa.Table], *args, mask_column_name: str, **kwargs +): + mask = await mask_function(input, *args, **kwargs) + return _add_mask(input, mask, mask_column_name) + + +class FilteredExamplesIterable(MappedExamplesIterable): + mask_column_name = "===MASK===" + + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + function: Callable, + with_indices: bool = False, + input_columns: Optional[list[str]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + fn_kwargs: Optional[dict] = None, + formatting: Optional["FormattingConfig"] = None, + ): + self.mask_function = function + if ex_iterable.is_typed: + features = Features({**ex_iterable.features, self.mask_column_name: Value("bool")}) + else: + features = None + super().__init__( + ex_iterable=ex_iterable, + function=partial( + async_add_mask if inspect.iscoroutinefunction(function) else add_mask, + function, + mask_column_name=self.mask_column_name, + ), + with_indices=with_indices, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + fn_kwargs=fn_kwargs, + formatting=formatting, + features=features, + ) + + def _iter(self): + for key, example in super()._iter(): + example = dict(example) + if example.pop(self.mask_column_name): + yield key, example + + def _iter_arrow(self, max_chunksize: Optional[int] = None): + for key, pa_table in super()._iter_arrow(max_chunksize=max_chunksize): + mask = pa_table[self.mask_column_name] + yield key, pa_table.drop(self.mask_column_name).filter(mask) + + def shuffle_data_sources(self, seed: Optional[int]) -> "FilteredExamplesIterable": + """Shuffle the wrapped examples iterable.""" + return FilteredExamplesIterable( + self.ex_iterable.shuffle_data_sources(seed), + function=self.mask_function, + with_indices=self.with_indices, + input_columns=self.input_columns, + batched=self.batched, + batch_size=self.batch_size, + fn_kwargs=self.fn_kwargs, + formatting=self.formatting, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "FilteredExamplesIterable": + """Keep only the requested shard.""" + return FilteredExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + function=self.mask_function, + with_indices=self.with_indices, + input_columns=self.input_columns, + batched=self.batched, + batch_size=self.batch_size, + fn_kwargs=self.fn_kwargs, + formatting=self.formatting, + ) + + def reshard_data_sources(self) -> "FilteredExamplesIterable": + return FilteredExamplesIterable( + self.ex_iterable.reshard_data_sources(), + function=self.mask_function, + with_indices=self.with_indices, + input_columns=self.input_columns, + batched=self.batched, + batch_size=self.batch_size, + fn_kwargs=self.fn_kwargs, + formatting=self.formatting, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class BufferShuffledExamplesIterable(_BaseExamplesIterable): + def __init__(self, ex_iterable: _BaseExamplesIterable, buffer_size: int, generator: np.random.Generator): + super().__init__() + self.ex_iterable = ex_iterable + self.buffer_size = buffer_size + self.generator = generator + + def shift_rngs(self, value: int) -> "_BaseExamplesIterable": + rng = deepcopy(self.generator) + new_seed = rng.integers(0, 1 << 63) - value + return BufferShuffledExamplesIterable( + ex_iterable=self.ex_iterable, + buffer_size=self.buffer_size, + generator=np.random.default_rng(seed=new_seed), + ) + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + @property + def iter_arrow(self): + return self._iter_arrow if self.ex_iterable.iter_arrow else None + + def _init_state_dict(self) -> dict: + self._state_dict = self.ex_iterable._init_state_dict() + self._original_state_dict = self.state_dict() + return self._state_dict + + def load_state_dict(self, state_dict: dict) -> dict: + if self._state_dict: + if state_dict != self._original_state_dict: + logger.warning( + "Loading a state dict of a shuffle buffer of a dataset without the buffer content." + "The shuffle buffer will be refilled before starting to yield new examples." + ) + return super().load_state_dict(state_dict) + + @staticmethod + def _iter_random_indices(rng: np.random.Generator, buffer_size: int, random_batch_size=1000) -> Iterator[int]: + while True: + yield from (int(i) for i in rng.integers(0, buffer_size, size=random_batch_size)) + + def __iter__(self): + buffer_size = self.buffer_size + rng = deepcopy(self.generator) + indices_iterator = self._iter_random_indices(rng, buffer_size) + # this is the shuffle buffer that we keep in memory + mem_buffer = [] + for x in self.ex_iterable: + if len(mem_buffer) == buffer_size: # if the buffer is full, pick and example from it + i = next(indices_iterator) + yield mem_buffer[i] + mem_buffer[i] = x # replace the picked example by a new one + else: # otherwise, keep filling the buffer + mem_buffer.append(x) + # when we run out of examples, we shuffle the remaining examples in the buffer and yield them + rng.shuffle(mem_buffer) + yield from mem_buffer + + def _iter_arrow(self): + buffer_size = self.buffer_size + rng = deepcopy(self.generator) + indices_iterator = self._iter_random_indices(rng, buffer_size) + # this is the shuffle buffer that we keep in memory + mem_buffer = [] + for key, pa_table in self.ex_iterable.iter_arrow(): + if len(mem_buffer) == buffer_size: # if the buffer is full, pick and example from it + i = next(indices_iterator) + yield mem_buffer[i] + mem_buffer[i] = (key, pa_table) # replace the picked example by a new one + else: # otherwise, keep filling the buffer + mem_buffer.append((key, pa_table)) + # when we run out of examples, we shuffle the remaining examples in the buffer and yield them + rng.shuffle(mem_buffer) + yield from mem_buffer + + def shuffle_data_sources(self, generator: np.random.Generator) -> "BufferShuffledExamplesIterable": + """Shuffle the wrapped examples iterable as well as the shuffling buffer.""" + return BufferShuffledExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), buffer_size=self.buffer_size, generator=self.generator + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "BufferShuffledExamplesIterable": + """Keep only the requested shard.""" + return BufferShuffledExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + buffer_size=self.buffer_size, + generator=self.generator, + ) + + def reshard_data_sources(self) -> "BufferShuffledExamplesIterable": + return BufferShuffledExamplesIterable( + self.ex_iterable.reshard_data_sources(), + buffer_size=self.buffer_size, + generator=self.generator, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class SkipExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + n: int, + block_sources_order_when_shuffling: bool = True, + split_when_sharding: bool = True, + ): + super().__init__() + self.ex_iterable = ex_iterable + self.n = n + self.block_sources_order_when_shuffling = block_sources_order_when_shuffling + self.split_when_sharding = split_when_sharding + + @property + def iter_arrow(self): + return self._iter_arrow if self.ex_iterable.iter_arrow else None + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + def _init_state_dict(self) -> dict: + self._state_dict = { + "skipped": 0, + "examples_iterable": self.ex_iterable._init_state_dict(), + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + skipped = self._state_dict["skipped"] if self._state_dict else 0 + for key_example in self.ex_iterable: + if skipped + 1 <= self.n: + skipped += 1 + if self._state_dict: + self._state_dict["skipped"] = skipped + else: + yield key_example + + def _iter_arrow(self): + skipped = self._state_dict["skipped"] if self._state_dict else 0 + for key, pa_table in self.ex_iterable.iter_arrow(): + if len(pa_table) == 0: + continue + elif skipped + len(pa_table) <= self.n: + skipped += len(pa_table) + if self._state_dict: + self._state_dict["skipped"] = skipped + if skipped + 1 <= self.n: + offset = self.n - skipped + skipped = self.n + if self._state_dict: + self._state_dict["skipped"] = skipped + yield key, pa_table.slice(offset, len(pa_table) - offset) + else: + yield key, pa_table + + @staticmethod + def split_number(num, n): + quotient = num // n + remainder = num % n + result = [quotient] * n + for i in range(remainder): + result[i] += 1 + return result + + def shuffle_data_sources(self, generator: np.random.Generator) -> "SkipExamplesIterable": + """May not shuffle the wrapped examples iterable since it would skip examples from other shards instead.""" + if self.block_sources_order_when_shuffling: + return self + else: + return SkipExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), + n=self.n, + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SkipExamplesIterable": + """Keep only the requested shard.""" + if self.split_when_sharding: + return SkipExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + n=self.split_number(self.n, num_shards)[index], + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + else: + return self + + def reshard_data_sources(self) -> "SkipExamplesIterable": + return SkipExamplesIterable( + self.ex_iterable.reshard_data_sources(), + n=self.n, + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class RepeatExamplesIterable(_BaseExamplesIterable): + """ + Iterable that repeats the underlying iterable a given number of times. + """ + + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + num_times: Optional[int], + ): + super().__init__() + self.ex_iterable = ex_iterable + self.num_times = num_times + + def _init_state_dict(self) -> dict: + self._state_dict = { + "repeat_index": 0, + "examples_iterable": self.ex_iterable._init_state_dict(), + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + repeat_index = self._state_dict["repeat_index"] if self._state_dict else 0 + while True: + if self.num_times is not None and repeat_index >= max(self.num_times, 0): + break + yield from self.ex_iterable + repeat_index += 1 + if self._state_dict: + self._state_dict["repeat_index"] = repeat_index + self._state_dict["examples_iterable"] = self.ex_iterable._init_state_dict() + + def shuffle_data_sources(self, generator: np.random.Generator) -> "RepeatExamplesIterable": + """Shuffle the underlying iterable, then repeat.""" + return RepeatExamplesIterable(self.ex_iterable.shuffle_data_sources(generator), num_times=self.num_times) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "RepeatExamplesIterable": + """Shard, then repeat shards.""" + return RepeatExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + num_times=self.num_times, + ) + + def reshard_data_sources(self) -> "RepeatExamplesIterable": + return RepeatExamplesIterable( + self.ex_iterable.reshard_data_sources(), + num_times=self.num_times, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +class TakeExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + n: int, + block_sources_order_when_shuffling: bool = True, + split_when_sharding: bool = True, + ): + super().__init__() + self.ex_iterable = ex_iterable + self.n = n + self.block_sources_order_when_shuffling = block_sources_order_when_shuffling + self.split_when_sharding = split_when_sharding + + @property + def iter_arrow(self): + return self._iter_arrow if self.ex_iterable.iter_arrow else None + + @property + def is_typed(self): + return self.ex_iterable.is_typed + + @property + def features(self): + return self.ex_iterable.features + + def _init_state_dict(self) -> dict: + self._state_dict = { + "taken": 0, + "examples_iterable": self.ex_iterable._init_state_dict(), + "type": self.__class__.__name__, + } + return self._state_dict + + def __iter__(self): + taken = self._state_dict["taken"] if self._state_dict else 0 + if taken >= self.n: + return + for key_example in self.ex_iterable: + if taken + 1 <= self.n: + taken += 1 + if self._state_dict: + self._state_dict["taken"] = taken + yield key_example + else: + break + + def _iter_arrow(self): + taken = self._state_dict["taken"] if self._state_dict else 0 + if taken >= self.n: + return + for key, pa_table in self.ex_iterable.iter_arrow(): + if len(pa_table) == 0: + continue + elif taken + len(pa_table) <= self.n: + taken += len(pa_table) + if self._state_dict: + self._state_dict["taken"] = taken + yield key, pa_table + elif taken + 1 <= self.n: + length = self.n - taken + taken = self.n + if self._state_dict: + self._state_dict["taken"] = taken + yield key, pa_table.slice(0, length) + else: + break + + @staticmethod + def split_number(num, n): + quotient = num // n + remainder = num % n + result = [quotient] * n + for i in range(remainder): + result[i] += 1 + return result + + def shuffle_data_sources(self, generator: np.random.Generator) -> "TakeExamplesIterable": + """May not shuffle the wrapped examples iterable since it would take examples from other shards instead.""" + if self.block_sources_order_when_shuffling: + return self + else: + return TakeExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), + n=self.n, + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "TakeExamplesIterable": + """Keep only the requested shard.""" + if self.split_when_sharding: + return TakeExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + n=self.split_number(self.n, num_shards)[index], + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + else: + return TakeExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + n=self.n, + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + + def reshard_data_sources(self) -> "TakeExamplesIterable": + return TakeExamplesIterable( + self.ex_iterable.reshard_data_sources(), + n=self.n, + block_sources_order_when_shuffling=self.block_sources_order_when_shuffling, + split_when_sharding=self.split_when_sharding, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +def _apply_feature_types_on_example( + example: dict, features: Features, token_per_repo_id: dict[str, Union[str, bool, None]] +) -> dict: + example = dict(example) + # add missing columns + for column_name in features: + if column_name not in example: + example[column_name] = None + # we encode the example for ClassLabel feature types for example + encoded_example = features.encode_example(example) + # Decode example for Audio feature, e.g. + decoded_example = features.decode_example(encoded_example, token_per_repo_id=token_per_repo_id) + return decoded_example + + +@dataclass +class FormattingConfig: + format_type: Optional[str] + + @property + def is_table(self) -> bool: + return isinstance(get_formatter(self.format_type), TableFormatter) + + @property + def is_tensor(self) -> bool: + return isinstance(get_formatter(self.format_type), TensorFormatter) + + +class FormattedExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + formatting: Optional[FormattingConfig], + features: Optional[Features], + token_per_repo_id: dict[str, Union[str, bool, None]], + force_convert_to_python: bool = False, + ): + super().__init__() + self.ex_iterable = ex_iterable + self._features = features + self.formatting = formatting + self.token_per_repo_id = token_per_repo_id + self.force_convert_to_python = force_convert_to_python + + @property + def iter_arrow(self): + if self.ex_iterable.iter_arrow and not self.force_convert_to_python: + return self._iter_arrow + + @property + def is_typed(self): + return self.ex_iterable.is_typed or self._features is not None + + @property + def features(self): + return self._features + + def _init_state_dict(self) -> dict: + self._state_dict = self.ex_iterable._init_state_dict() + return self._state_dict + + def __iter__(self): + if not self.formatting or self.formatting.is_table: + formatter = PythonFormatter( + features=self._features if not self.ex_iterable.is_typed else None, + token_per_repo_id=self.token_per_repo_id, + ) + else: + formatter = get_formatter( + self.formatting.format_type, + features=self._features if not self.ex_iterable.is_typed else None, + token_per_repo_id=self.token_per_repo_id, + ) + + # It's ok to use _iter_arrow here without fancy state_dict logic since it's + # used with RebatchedArrowExamplesIterable with the right batch_size to + # never lose examples + if self.ex_iterable.iter_arrow: + # feature casting (inc column addition) handled within self._iter_arrow() + for key, pa_table in self._iter_arrow(): + batch = formatter.format_batch(pa_table) + for example in _batch_to_examples(batch): + yield key, example + else: + format_dict = ( + formatter.recursive_tensorize + if isinstance(formatter, TensorFormatter) + else None # cast in case features is None + ) + for key, example in self.ex_iterable: + # don't apply feature types if already applied by ex_iterable (e.g. in case of chained with_format) + if self.features and not self.ex_iterable.is_typed: + example = _apply_feature_types_on_example( + example, self.features, token_per_repo_id=self.token_per_repo_id + ) + if format_dict: + example = format_dict(example) + yield key, example + + def _iter_arrow(self) -> Iterator[tuple[Key, pa.Table]]: + if not self.features: + yield from self.ex_iterable._iter_arrow() + return + for key, pa_table in self.ex_iterable._iter_arrow(): + columns = set(pa_table.column_names) + schema = self.features.arrow_schema + # add missing columns + for column_name in self.features: + if column_name not in columns: + col = pa.NullArray.from_buffers(pa.null(), len(pa_table), [None]) + pa_table = pa_table.append_column(column_name, col) + if pa_table.schema != schema: + pa_table = cast_table_to_features(pa_table, self.features) + yield key, pa_table + + def shuffle_data_sources(self, generator: np.random.Generator) -> "FormattedExamplesIterable": + """Shuffle the wrapped examples iterable.""" + return FormattedExamplesIterable( + self.ex_iterable.shuffle_data_sources(generator), + features=self.features, + token_per_repo_id=self.token_per_repo_id, + formatting=self.formatting, + force_convert_to_python=self.force_convert_to_python, + ) + + def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "FormattedExamplesIterable": + """Keep only the requested shard.""" + return FormattedExamplesIterable( + self.ex_iterable.shard_data_sources(num_shards, index, contiguous=contiguous), + features=self.features, + token_per_repo_id=self.token_per_repo_id, + formatting=self.formatting, + force_convert_to_python=self.force_convert_to_python, + ) + + def reshard_data_sources(self) -> "FormattedExamplesIterable": + return FormattedExamplesIterable( + self.ex_iterable.reshard_data_sources(), + features=self.features, + token_per_repo_id=self.token_per_repo_id, + formatting=self.formatting, + force_convert_to_python=self.force_convert_to_python, + ) + + @property + def num_shards(self) -> int: + return self.ex_iterable.num_shards + + +@dataclass +class DistributedConfig: + rank: int + world_size: int + + +def _maybe_add_torch_iterable_dataset_parent_class(cls): + """Add torch.utils.data.IterableDataset as a parent class if 'torch' is available""" + if config.TORCH_AVAILABLE: + import torch.utils.data + + if torch.utils.data.IterableDataset not in cls.__bases__: + cls.__bases__ += (torch.utils.data.IterableDataset,) + + +def _maybe_share_with_torch_persistent_workers(value: Union[int, "torch.Tensor"]) -> Union[int, "torch.Tensor"]: + if config.TORCH_AVAILABLE: + import torch + + if isinstance(value, torch.Tensor): + return value.share_memory_() + else: + return torch.tensor(value).share_memory_() + else: + return value + + +class IterableColumn: + """ + An iterable for a specific column of an [`IterableDataset`]. + + Example: + + Iterate on the texts of the "text" column of a dataset: + + ```python + for text in dataset["text"]: + ... + ``` + + It also works with nested columns: + + ```python + for source in dataset["metadata"]["source"]: + ... + ``` + """ + + def __init__(self, source: Union["IterableDataset", "IterableColumn"], column_name: str): + self.source = source + self.column_name = column_name + + def __iter__(self) -> Iterator[Any]: + for example in self.source: + yield example[self.column_name] + + def __getitem__(self, column_name: str) -> "IterableColumn": + return IterableColumn(self, column_name) + + +class IterableDataset(DatasetInfoMixin): + """A Dataset backed by an iterable.""" + + def __init__( + self, + ex_iterable: _BaseExamplesIterable, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + formatting: Optional[FormattingConfig] = None, + distributed: Optional[DistributedConfig] = None, + token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None, + ): + info = info.copy() if info is not None else DatasetInfo() + DatasetInfoMixin.__init__(self, info=info, split=split) + + self._ex_iterable = copy.copy(ex_iterable) + self._formatting = formatting + self._distributed = distributed + self._token_per_repo_id: dict[str, Union[str, bool, None]] = token_per_repo_id or {} + self._epoch: Union[int, "torch.Tensor"] = _maybe_share_with_torch_persistent_workers(0) + self._starting_state_dict: Optional[dict] = None + self.__hffs_cache = HfFileSystem._cache # keep the cache on pickling (e.g. for dataloader workers) + self._prepare_ex_iterable_for_iteration() # set state_dict + _maybe_add_torch_iterable_dataset_parent_class(self.__class__) # subclass of torch IterableDataset + + @property + def num_columns(self) -> Optional[int]: + """Number of columns in the dataset. + This can be None if the dataset has unknown features (e.g. after a map() operation). + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation") + >>> ds.num_columns + 2 + ``` + """ + return None if self.features is None else len(self.features) + + @property + def column_names(self) -> Optional[list[str]]: + """Names of the columns in the dataset. + This can be None if the dataset has unknown features (e.g. after a map() operation). + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True) + >>> ds.column_names + ['text', 'label'] + ``` + """ + return None if self.features is None else list(self.features) + + def state_dict(self) -> dict: + """Get the current state_dict of the dataset. + It corresponds to the state at the latest example it yielded. + + Resuming returns exactly where the checkpoint was saved except in two cases: + + 1. examples from shuffle buffers are lost when resuming and the buffers are refilled with new data + 2. combinations of `.with_format(arrow)` and batched `.map()` may skip one batch. + + Returns: + `dict` + + Example: + + ```py + >>> from datasets import Dataset, concatenate_datasets + >>> ds = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3) + >>> for idx, example in enumerate(ds): + ... print(example) + ... if idx == 2: + ... state_dict = ds.state_dict() + ... print("checkpoint") + ... break + >>> ds.load_state_dict(state_dict) + >>> print(f"restart from checkpoint") + >>> for example in ds: + ... print(example) + ``` + + which returns: + ``` + {'a': 0} + {'a': 1} + {'a': 2} + checkpoint + restart from checkpoint + {'a': 3} + {'a': 4} + {'a': 5} + ``` + + ```py + >>> from torchdata.stateful_dataloader import StatefulDataLoader + >>> ds = load_dataset("deepmind/code_contests", streaming=True, split="train") + >>> dataloader = StatefulDataLoader(ds, batch_size=32, num_workers=4) + >>> # checkpoint + >>> state_dict = dataloader.state_dict() # uses ds.state_dict() under the hood + >>> # resume from checkpoint + >>> dataloader.load_state_dict(state_dict) # uses ds.load_state_dict() under the hood + ``` + """ + return copy.deepcopy(self._state_dict) + + def load_state_dict(self, state_dict: dict) -> None: + """Load the state_dict of the dataset. + The iteration will restart at the next example from when the state was saved. + + Resuming returns exactly where the checkpoint was saved except in two cases: + + 1. examples from shuffle buffers are lost when resuming and the buffers are refilled with new data + 2. combinations of `.with_format(arrow)` and batched `.map()` may skip one batch. + + Example: + + ```py + >>> from datasets import Dataset, concatenate_datasets + >>> ds = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3) + >>> for idx, example in enumerate(ds): + ... print(example) + ... if idx == 2: + ... state_dict = ds.state_dict() + ... print("checkpoint") + ... break + >>> ds.load_state_dict(state_dict) + >>> print(f"restart from checkpoint") + >>> for example in ds: + ... print(example) + ``` + + which returns: + ``` + {'a': 0} + {'a': 1} + {'a': 2} + checkpoint + restart from checkpoint + {'a': 3} + {'a': 4} + {'a': 5} + ``` + + ```py + >>> from torchdata.stateful_dataloader import StatefulDataLoader + >>> ds = load_dataset("deepmind/code_contests", streaming=True, split="train") + >>> dataloader = StatefulDataLoader(ds, batch_size=32, num_workers=4) + >>> # checkpoint + >>> state_dict = dataloader.state_dict() # uses ds.state_dict() under the hood + >>> # resume from checkpoint + >>> dataloader.load_state_dict(state_dict) # uses ds.load_state_dict() under the hood + ``` + """ + self._starting_state_dict = state_dict + + def __repr__(self): + return f"IterableDataset({{\n features: {list(self._info.features.keys()) if self._info.features is not None else 'Unknown'},\n num_shards: {self.num_shards}\n}})" + + def __getstate__(self): + return self.__dict__ + + def __setstate__(self, d): + self.__dict__ = d + # Re-add torch shared memory, since shared memory is not always kept when pickling + self._epoch = _maybe_share_with_torch_persistent_workers(self._epoch) + # Re-add the cache to keep on pickling (e.g. for dataloader workers) + self.__hffs_cache = HfFileSystem._cache + # Re-add torch iterable dataset as a parent class, since dynamically added parent classes are not kept when pickling + _maybe_add_torch_iterable_dataset_parent_class(self.__class__) + + def _head(self, n=5): + return next(iter(self.iter(batch_size=n))) + + @property + def epoch(self) -> int: + return int(self._epoch) + + @property + def num_shards(self) -> int: + if self._distributed and self._ex_iterable.num_shards % self._distributed.world_size == 0: + return self._ex_iterable.num_shards // self._distributed.world_size + return self._ex_iterable.num_shards + + @property + def n_shards(self) -> int: # backward compatibility + return self.num_shards + + def _iter_pytorch(self): + ex_iterable = self._prepare_ex_iterable_for_iteration() + # Fix for fsspec when using multiprocess to avoid hanging in the ML training loop. (only required for fsspec >= 0.9.0) + # See https://github.com/fsspec/gcsfs/issues/379 + fsspec.asyn.reset_lock() + # check if there aren't too many workers + import torch.utils.data + + worker_info = torch.utils.data.get_worker_info() + if self._is_main_process() and ex_iterable.num_shards < worker_info.num_workers: + logger.warning( + f"Too many dataloader workers: {worker_info.num_workers} (max is dataset.num_shards={ex_iterable.num_shards}). " + f"Stopping {worker_info.num_workers - ex_iterable.num_shards} dataloader workers." + ) + logger.info( + f"To parallelize data loading, we give each process some shards (or data sources) to process. " + f"Therefore it's unnecessary to have a number of workers greater than dataset.num_shards={ex_iterable.num_shards}. " + f"To enable more parallelism, please split the dataset in more files than {ex_iterable.num_shards} or try `dataset = dataset.reshard()` which may increase `num_shards` depending on the dataset file format." + ) + # split workload + _log_prefix = f"node#{self._distributed.rank} " if self._distributed else "" + shards_indices = ex_iterable.split_shard_indices_by_worker( + num_shards=worker_info.num_workers, index=worker_info.id, contiguous=False + ) + if shards_indices: + logger.debug( + f"{_log_prefix}dataloader worker#{worker_info.id}, ': Starting to iterate over {len(shards_indices)}/{ex_iterable.num_shards} shards." + ) + ex_iterable = ex_iterable.shard_data_sources( + num_shards=worker_info.num_workers, index=worker_info.id, contiguous=False + ) + ex_iterable = shift_ex_examples_rngs(ex_iterable=ex_iterable, value=worker_info.id) + self._state_dict = { + "examples_iterable": ex_iterable._init_state_dict(), + "epoch": self.epoch, + } + if self._starting_state_dict and self.epoch == self._starting_state_dict["epoch"]: + ex_iterable.load_state_dict(self._starting_state_dict["examples_iterable"]) + + if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table): + formatter = get_formatter(self._formatting.format_type, features=self.features) + for key, pa_table in ex_iterable.iter_arrow(): + yield formatter.format_row(pa_table) + return + else: + for key, example in ex_iterable: + # no need to format thanks to FormattedExamplesIterable + yield example + logger.debug( + f"{_log_prefix}dataloader worker#{worker_info.id}, ': Finished iterating over {len(shards_indices)}/{ex_iterable.num_shards} shards." + ) + else: + logger.debug( + f"{_log_prefix}dataloader worker#{worker_info.id}, ': Stopping... Number of dataset shards < num_workers ({ex_iterable.num_shards}<{worker_info.num_workers})." + ) + + def _is_main_process(self): + if self._distributed and self._distributed.rank > 0: + return False + if "torch" in sys.modules: + import torch.utils.data + + worker_info = torch.utils.data.get_worker_info() + if worker_info is not None and worker_info.id > 0: + return False + return True + + def _prepare_ex_iterable_for_iteration( + self, batch_size: int = 1, drop_last_batch: bool = False + ) -> _BaseExamplesIterable: + ex_iterable = self._ex_iterable + + if self.epoch: + ex_iterable = ex_iterable.shuffle_data_sources(np.random.default_rng(self.epoch)) + ex_iterable = shift_ex_examples_rngs(ex_iterable, self.epoch) + + if self._distributed: + rank = self._distributed.rank + world_size = self._distributed.world_size + if ex_iterable.num_shards % world_size == 0: + if self._is_main_process(): + num_shards_per_node = ex_iterable.num_shards // world_size + plural = "s" if num_shards_per_node > 1 else "" + logger.info( + f"Assigning {num_shards_per_node} shard{plural} (or data source{plural}) of the dataset to each node." + ) + ex_iterable = ex_iterable.shard_data_sources(num_shards=world_size, index=rank, contiguous=False) + else: + if self._is_main_process(): + logger.info( + f"Assigning 1 out of {world_size} examples of the dataset to each node. The others are skipped during the iteration." + ) + logger.info( + f"It is more optimized to distribute the dataset shards (or data sources) across nodes. " + f"You can do that by using a dataset with number of shards that is a factor of world_size={world_size}. " + f"The current dataset has {ex_iterable.num_shards} which is not a factor of {world_size}" + ) + ex_iterable = StepExamplesIterable(ex_iterable, step=world_size, offset=rank) + + if ex_iterable.iter_arrow: + ex_iterable = RebatchedArrowExamplesIterable( + ex_iterable, batch_size=batch_size, drop_last_batch=drop_last_batch + ) + elif self._formatting and self._formatting.is_table: + ex_iterable = RebatchedArrowExamplesIterable( + ex_iterable, batch_size=batch_size, drop_last_batch=drop_last_batch, force_convert_to_arrow=True + ) + + if self._formatting or (self.features and ex_iterable.features != self.features): + ex_iterable = FormattedExamplesIterable( + ex_iterable, + formatting=self._formatting, + features=self.features, + token_per_repo_id=self._token_per_repo_id, + ) + + self._state_dict = { + "examples_iterable": ex_iterable._init_state_dict(), + "epoch": self.epoch, + } + if self._starting_state_dict and self.epoch == self._starting_state_dict["epoch"]: + ex_iterable.load_state_dict(self._starting_state_dict["examples_iterable"]) + return ex_iterable + + def __iter__(self): + if "torch" in sys.modules: + import torch.utils.data + + worker_info = torch.utils.data.get_worker_info() + if isinstance(self, torch.utils.data.IterableDataset) and worker_info is not None: + # We're a torch.utils.data.IterableDataset in a PyTorch worker process + yield from self._iter_pytorch() + return + + ex_iterable = self._prepare_ex_iterable_for_iteration() + if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table): + formatter = get_formatter(self._formatting.format_type, features=self.features) + for key, pa_table in ex_iterable.iter_arrow(): + yield formatter.format_row(pa_table) + return + + for key, example in ex_iterable: + # no need to format thanks to FormattedExamplesIterable + yield example + + def iter(self, batch_size: int, drop_last_batch: bool = False): + """Iterate through the batches of size `batch_size`. + + Args: + batch_size (:obj:`int`): size of each batch to yield. + drop_last_batch (:obj:`bool`, default `False`): Whether a last batch smaller than the batch_size should be + dropped + """ + + if self._formatting: + formatter = get_formatter(self._formatting.format_type, features=self.features) + format_dict = formatter.recursive_tensorize if isinstance(formatter, TensorFormatter) else None + else: + format_dict = None + + ex_iterable = self._prepare_ex_iterable_for_iteration(batch_size=batch_size, drop_last_batch=drop_last_batch) + if self._formatting and (ex_iterable.iter_arrow or self._formatting.is_table): + for key, pa_table in ex_iterable.iter_arrow(): + yield formatter.format_batch(pa_table) + return + + iterator = iter(ex_iterable) + for key, example in iterator: + # If batched, first build the batch + examples = [example] + [example for key, example in islice(iterator, batch_size - 1)] + if drop_last_batch and len(examples) < batch_size: # ignore last batch + return + batch = _examples_to_batch(examples) + # we need to format here in case we need to stack tensors together + yield format_dict(batch) if format_dict else batch + + def __getitem__(self, column_name: str) -> IterableColumn: + return IterableColumn(self, column_name) + + @staticmethod + def from_generator( + generator: Callable, + features: Optional[Features] = None, + gen_kwargs: Optional[dict] = None, + split: NamedSplit = Split.TRAIN, + ) -> "IterableDataset": + """Create an Iterable Dataset from a generator. + + Args: + generator (`Callable`): + A generator function that `yields` examples. + features (`Features`, *optional*): + Dataset features. + gen_kwargs(`dict`, *optional*): + Keyword arguments to be passed to the `generator` callable. + You can define a sharded iterable dataset by passing the list of shards in `gen_kwargs`. + This can be used to improve shuffling and when iterating over the dataset with multiple workers. + split ([`NamedSplit`], defaults to `Split.TRAIN`): + Split name to be assigned to the dataset. + + + Returns: + [`IterableDataset`] + + Example: + + ```py + >>> def gen(): + ... yield {"text": "Good", "label": 0} + ... yield {"text": "Bad", "label": 1} + ... + >>> ds = IterableDataset.from_generator(gen) + ``` + + ```py + >>> def gen(shards): + ... for shard in shards: + ... with open(shard) as f: + ... for line in f: + ... yield {"line": line} + ... + >>> shards = [f"data{i}.txt" for i in range(32)] + >>> ds = IterableDataset.from_generator(gen, gen_kwargs={"shards": shards}) + >>> ds = ds.shuffle(seed=42, buffer_size=10_000) # shuffles the shards order + uses a shuffle buffer + >>> from torch.utils.data import DataLoader + >>> dataloader = DataLoader(ds.with_format("torch"), num_workers=4) # give each worker a subset of 32/4=8 shards + ``` + """ + from .io.generator import GeneratorDatasetInputStream + + return GeneratorDatasetInputStream( + generator=generator, features=features, gen_kwargs=gen_kwargs, streaming=True, split=split + ).read() + + @staticmethod + def from_spark( + df: "pyspark.sql.DataFrame", + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + **kwargs, + ) -> "IterableDataset": + """Create an IterableDataset from Spark DataFrame. The dataset is streamed to the driver in batches. + + Args: + df (`pyspark.sql.DataFrame`): + The DataFrame containing the desired data. + split (`NamedSplit`, *optional*): + Split name to be assigned to the dataset. + features (`Features`, *optional*): + Dataset features. + + Returns: + [`IterableDataset`] + + Example: + + ```py + >>> df = spark.createDataFrame( + >>> data=[[1, "Elia"], [2, "Teo"], [3, "Fang"]], + >>> columns=["id", "name"], + >>> ) + >>> ds = IterableDataset.from_spark(df) + ``` + """ + from .io.spark import SparkDatasetReader + + if sys.platform == "win32": + raise OSError("IterableDataset.from_spark is not currently supported on Windows") + + return SparkDatasetReader( + df, + split=split, + features=features, + streaming=True, + **kwargs, + ).read() + + @staticmethod + def from_file(filename: str) -> "IterableDataset": + """Instantiate a IterableDataset from Arrow table at filename. + + Args: + filename (`str`): + File name of the dataset. + + Returns: + [`IterableDataset`] + """ + pa_table_schema = read_schema_from_file(filename) + inferred_features = Features.from_arrow_schema(pa_table_schema) + ex_iterable = ArrowExamplesIterable(Dataset._generate_tables_from_cache_file, kwargs={"filename": filename}) + return IterableDataset(ex_iterable=ex_iterable, info=DatasetInfo(features=inferred_features)) + + @classmethod + def from_pandas( + cls, + df: pd.DataFrame, + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + preserve_index: Optional[bool] = None, + num_shards: Optional[int] = 1, + ) -> "IterableDataset": + """ + Convert `pandas.DataFrame` to a `pyarrow.Table` to create an [`IterableDataset`]. + + The column types in the resulting Arrow Table are inferred from the dtypes of the `pandas.Series` in the + DataFrame. In the case of non-object Series, the NumPy dtype is translated to its Arrow equivalent. In the + case of `object`, we need to guess the datatype by looking at the Python objects in this Series. + + Be aware that Series of the `object` dtype don't carry enough information to always lead to a meaningful Arrow + type. In the case that we cannot infer a type, e.g. because the DataFrame is of length 0 or the Series only + contains `None/nan` objects, the type is set to `null`. This behavior can be avoided by constructing explicit + features and passing it to this function. + + Important: a dataset created with from_pandas() lives in memory. + This may change in the future, but in the meantime if you + want to reduce memory usage you should write it on disk + and reload using e.g. to_parquet / from_parquet. + + Args: + df (`pandas.DataFrame`): + Dataframe that contains the dataset. + features ([`Features`], *optional*): + Dataset features. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + preserve_index (`bool`, *optional*): + Whether to store the index as an additional column in the resulting Dataset. + The default of `None` will store the index as a column, except for `RangeIndex` which is stored as metadata only. + Use `preserve_index=True` to force it to be stored as a column. + num_shards (`int`, default to `1`): + Number of shards to define when instantiating the iterable dataset. This is especially useful for big datasets to be able to shuffle properly, + and also to enable fast parallel loading using a PyTorch DataLoader or in distributed setups for example. + + Returns: + [`IterableDataset`] + + Example: + + ```py + >>> ds = IterableDataset.from_pandas(df) + ``` + """ + return Dataset.from_pandas( + df, + features=features, + info=info, + split=split, + preserve_index=preserve_index, + ).to_iterable_dataset(num_shards=num_shards) + + @classmethod + def from_polars( + cls, + df: Union["pl.DataFrame", "pl.LazyFrame"], + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + ) -> "IterableDataset": + """ + Create an IterableDataset from a polars DataFrame or LazyFrame. + + Iterating over the dataset is mostly zero copy. + Under the hood, the dataset iterates over the polars DataFrame batches/slices. + + Data types that do copy: + * CategoricalType + + Args: + df (`polars.DataFrame`): DataFrame to convert to Arrow Table + features (`Features`, optional): Dataset features. + info (`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (`NamedSplit`, optional): Name of the dataset split. + + Returns: + [`IterableDataset`] + + Examples: + ```py + >>> ds = IterableDataset.from_polars(df) + ``` + """ + import polars as pl + + if info is not None and features is not None and info.features != features: + raise ValueError( + f"Features specified in `features` and `info.features` can't be different:\n{features}\n{info.features}" + ) + features = features if features is not None else info.features if info is not None else None + if features is not None: + features = _fix_for_backward_compatible_features(features) + if info is None: + info = DatasetInfo() + info.features = features or Features.from_arrow_schema( + (df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema).to_arrow() + ) + return IterableDataset( + ArrowExamplesIterable(_generate_tables_from_polars, kwargs={"df": df}), + info=info, + split=split, + ) + + @classmethod + def from_dict( + cls, + mapping: dict, + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + num_shards: Optional[int] = 1, + ) -> "IterableDataset": + """ + Convert `dict` to a `pyarrow.Table` to create an [`IterableDataset`]. + + Important: a dataset created with from_dict() lives in memory. + This may change in the future, but in the meantime if you + want to reduce memory usage you should write it back on disk + and reload using e.g. to_parquet / from_parquet. + + Args: + mapping (`Mapping`): + Mapping of strings to Arrays or Python lists. + features ([`Features`], *optional*): + Dataset features. + info (`DatasetInfo`, *optional*): + Dataset information, like description, citation, etc. + split (`NamedSplit`, *optional*): + Name of the dataset split. + num_shards (`int`, default to `1`): + Number of shards to define when instantiating the iterable dataset. This is especially useful for big datasets to be able to shuffle properly, + and also to enable fast parallel loading using a PyTorch DataLoader or in distributed setups for example. + + Returns: + [`IterableDataset`] + """ + return Dataset.from_dict(mapping, features=features, info=info, split=split).to_iterable_dataset( + num_shards=num_shards + ) + + @classmethod + def from_list( + cls, + mapping: list[dict], + features: Optional[Features] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + num_shards: Optional[int] = 1, + ) -> "IterableDataset": + """ + Convert a list of dicts to a `pyarrow.Table` to create an [`IterableDataset`]`. + + Note that the keys of the first entry will be used to determine the dataset columns, + regardless of what is passed to features. + + Important: a dataset created with from_list() lives in memory. + This may change in the future, but in the meantime if you + want to reduce memory usage you should write it back on disk + and reload using e.g. from_parquet / to_parquet. + + Args: + mapping (`List[dict]`): A list of mappings of strings to row values. + features (`Features`, optional): Dataset features. + info (`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (`NamedSplit`, optional): Name of the dataset split. + num_shards (`int`, default to `1`): + Number of shards to define when instantiating the iterable dataset. This is especially useful for big datasets to be able to shuffle properly, + and also to enable fast parallel loading using a PyTorch DataLoader or in distributed setups for example. + + Returns: + [`IterableDataset`] + """ + return Dataset.from_list( + mapping, + features=features, + info=info, + split=split, + ).to_iterable_dataset(num_shards=num_shards) + + @staticmethod + def from_csv( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + keep_in_memory: bool = False, + **kwargs, + ) -> "IterableDataset": + """Create an IterableDataset from CSV file(s). + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the CSV file(s). + split ([`NamedSplit`], *optional*): + Split name to be assigned to the dataset. + features ([`Features`], *optional*): + Dataset features. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`pandas.read_csv`]. + + Returns: + [`IterableDataset`] + + Example: + + ```py + >>> ds = IterableDataset.from_csv('path/to/dataset.csv') + ``` + """ + # Dynamic import to avoid circular dependency + from .io.csv import CsvDatasetReader + + return CsvDatasetReader( + path_or_paths, + split=split, + features=features, + keep_in_memory=keep_in_memory, + streaming=True, + **kwargs, + ).read() + + @staticmethod + def from_json( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + keep_in_memory: bool = False, + field: Optional[str] = None, + **kwargs, + ) -> "IterableDataset": + """Create an IterableDataset from JSON or JSON Lines file(s). + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the JSON or JSON Lines file(s). + split ([`NamedSplit`], *optional*): + Split name to be assigned to the dataset. + features ([`Features`], *optional*): + Dataset features. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + field (`str`, *optional*): + Field name of the JSON file where the dataset is contained in. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`JsonConfig`]. + + Returns: + [`IterableDataset`] + + Example: + + ```py + >>> ds = IterableDataset.from_json('path/to/dataset.json') + ``` + """ + # Dynamic import to avoid circular dependency + from .io.json import JsonDatasetReader + + return JsonDatasetReader( + path_or_paths, + split=split, + features=features, + keep_in_memory=keep_in_memory, + field=field, + streaming=True, + **kwargs, + ).read() + + @staticmethod + def from_parquet( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + keep_in_memory: bool = False, + columns: Optional[list[str]] = None, + filters: Optional[Union[pds.Expression, list[tuple], list[list[tuple]]]] = None, + fragment_scan_options: Optional[pds.ParquetFragmentScanOptions] = None, + on_bad_files: Literal["error", "warn", "skip"] = "error", + **kwargs, + ) -> "IterableDataset": + """Create an IterableDataset from Parquet file(s). + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the Parquet file(s). + split (`NamedSplit`, *optional*): + Split name to be assigned to the dataset. + features (`Features`, *optional*): + Dataset features. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + columns (`List[str]`, *optional*): + If not `None`, only these columns will be read from the file. + A column name may be a prefix of a nested field, e.g. 'a' will select + 'a.b', 'a.c', and 'a.d.e'. + filters (`Union[pyarrow.dataset.Expression, list[tuple], list[list[tuple]]]`, *optional*): + Return only the rows matching the filter. + If possible the predicate will be pushed down to exploit the partition information + or internal metadata found in the data source, e.g. Parquet statistics. + Otherwise filters the loaded RecordBatches before yielding them. + fragment_scan_options (`pyarrow.dataset.ParquetFragmentScanOptions`, *optional*) + Scan-specific options for Parquet fragments. + This is especially useful to configure buffering and caching. + + + on_bad_files (`Literal["error", "warn", "skip"]`, *optional*, defaults to "error") + Specify what to do upon encountering a bad file (a file that can't be read). Allowed values are : + * 'error', raise an Exception when a bad file is encountered. + * 'warn', raise a warning when a bad file is encountered and skip that file. + * 'skip', skip bad files without raising or warning when they are encountered. + + + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`ParquetConfig`]. + + Returns: + [`IterableDataset`] + + Example: + + ```py + >>> ds = IterableDataset.from_parquet('path/to/dataset.parquet') + ``` + + Load a subset of columns: + + ```python + >>> ds = IterableDataset.from_parquet('path/to/dataset.parquet', columns=["col_0", "col_1"]) + ``` + + Efficiently filter data, possibly skipping entire files or row groups: + + ```python + >>> filters = [("col_0", "==", 0)] + >>> ds = IterableDataset.from_parquet(parquet_files_list, filters=filters) + ``` + """ + # Dynamic import to avoid circular dependency + from .io.parquet import ParquetDatasetReader + + return ParquetDatasetReader( + path_or_paths, + split=split, + features=features, + keep_in_memory=keep_in_memory, + columns=columns, + streaming=True, + filters=filters, + fragment_scan_options=fragment_scan_options, + on_bad_files=on_bad_files, + **kwargs, + ).read() + + @staticmethod + def from_text( + path_or_paths: Union[PathLike, list[PathLike]], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + keep_in_memory: bool = False, + keep_linebreaks: bool = False, + sample_by: Literal["line", "paragraph", "document"] = "line", + **kwargs, + ) -> "IterableDataset": + """Create an IterableDataset from text file(s). + + Args: + path_or_paths (`path-like` or list of `path-like`): + Path(s) of the text file(s). + split (`NamedSplit`, *optional*): + Split name to be assigned to the dataset. + features (`Features`, *optional*): + Dataset features. + keep_in_memory (`bool`, defaults to `False`): + Whether to copy the data in-memory. + keep_linebreaks: (`bool`, defaults to False): + Whether to keep line breaks. + sample_by (`Literal["line", "paragraph", "document"]`, defaults to "line"): + Whether to load data per line, praragraph or document. + By default one row in the dataset = one line. + **kwargs (additional keyword arguments): + Keyword arguments to be passed to [`TextConfig`]. + + Returns: + [`IterableDataset`] + + Example: + + ```py + >>> ds = IterableDataset.from_text('path/to/dataset.txt') + ``` + """ + # Dynamic import to avoid circular dependency + from .io.text import TextDatasetReader + + return TextDatasetReader( + path_or_paths, + split=split, + features=features, + keep_in_memory=keep_in_memory, + streaming=True, + keep_linebreaks=keep_linebreaks, + sample_by=sample_by, + **kwargs, + ).read() + + def with_format( + self, + type: Optional[str] = None, + ) -> "IterableDataset": + """ + Return a dataset with the specified format. + + Args: + + type (`str`, *optional*): + Either output type selected in `[None, 'numpy', 'torch', 'tensorflow', 'jax', 'arrow', 'pandas', 'polars']`. + `None` means it returns python objects (default). + + Example: + + ```py + >>> from datasets import load_dataset + >>> from transformers import AutoTokenizer + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation", streaming=True) + >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + >>> ds = ds.map(lambda x: tokenizer(x['text'], truncation=True, padding=True), batched=True) + >>> ds = ds.with_format("torch") + >>> next(iter(ds)) + {'text': 'compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .', + 'label': tensor(1), + 'input_ids': tensor([ 101, 18027, 16310, 16001, 1103, 9321, 178, 11604, 7235, 6617, + 1742, 2165, 2820, 1206, 6588, 22572, 12937, 1811, 2153, 1105, + 1147, 12890, 19587, 6463, 1105, 15026, 1482, 119, 102, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0]), + 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])} + ``` + """ + type = get_format_type_from_alias(type) + # TODO(QL): add format_kwargs + # TODO(QL): add format_columns and return_all_columns + # TODO(QL): add pandas format + return IterableDataset( + ex_iterable=self._ex_iterable, + info=self._info.copy(), + split=self._split, + formatting=FormattingConfig(format_type=type), + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def map( + self, + function: Optional[Callable] = None, + with_indices: bool = False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + drop_last_batch: bool = False, + remove_columns: Optional[Union[str, list[str]]] = None, + features: Optional[Features] = None, + fn_kwargs: Optional[dict] = None, + ) -> "IterableDataset": + """ + Apply a function to all the examples in the iterable dataset (individually or in batches) and update them. + If your function returns a column that already exists, then it overwrites it. + The function is applied on-the-fly on the examples when iterating over the dataset. + + You can specify whether the function should be batched or not with the `batched` parameter: + + - If batched is `False`, then the function takes 1 example in and should return 1 example. + An example is a dictionary, e.g. `{"text": "Hello there !"}`. + - If batched is `True` and `batch_size` is 1, then the function takes a batch of 1 example as input and can return a batch with 1 or more examples. + A batch is a dictionary, e.g. a batch of 1 example is {"text": ["Hello there !"]}. + - If batched is `True` and `batch_size` is `n` > 1, then the function takes a batch of `n` examples as input and can return a batch with `n` examples, or with an arbitrary number of examples. + Note that the last batch may have less than `n` examples. + A batch is a dictionary, e.g. a batch of `n` examples is `{"text": ["Hello there !"] * n}`. + + If the function is asynchronous, then `map` will run your function in parallel, with up to one thousand simulatenous calls. + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`Callable`, *optional*, defaults to `None`): + Function applied on-the-fly on the examples when you iterate on the dataset. + It must have one of the following signatures: + + - `function(example: Dict[str, Any]) -> Dict[str, Any]` if `batched=False` and `with_indices=False` + - `function(example: Dict[str, Any], idx: int) -> Dict[str, Any]` if `batched=False` and `with_indices=True` + - `function(batch: Dict[str, List]) -> Dict[str, List]` if `batched=True` and `with_indices=False` + - `function(batch: Dict[str, List], indices: List[int]) -> Dict[str, List]` if `batched=True` and `with_indices=True` + + For advanced usage, the function can also return a `pyarrow.Table`. + If the function is asynchronous, then `map` will run your function in parallel. + Moreover if your function returns nothing (`None`), then `map` will run your function and return the dataset unchanged. + If no function is provided, default to identity function: `lambda x: x`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx[, rank]): ...`. + input_columns (`Optional[Union[str, List[str]]]`, defaults to `None`): + The columns to be passed into `function` + as positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, defaults to `1000`): + Number of examples per batch provided to `function` if `batched=True`. + `batch_size <= 0` or `batch_size == None` then provide the full dataset as a single batch to `function`. + drop_last_batch (`bool`, defaults to `False`): + Whether a last batch smaller than the batch_size should be + dropped instead of being processed by the function. + remove_columns (`[List[str]]`, *optional*, defaults to `None`): + Remove a selection of columns while doing the mapping. + Columns will be removed before updating the examples with the output of `function`, i.e. if `function` is adding + columns with names in `remove_columns`, these columns will be kept. + features (`[Features]`, *optional*, defaults to `None`): + Feature types of the resulting dataset. + fn_kwargs (`Dict`, *optional*, default `None`): + Keyword arguments to be passed to `function`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> def add_prefix(example): + ... example["text"] = "Review: " + example["text"] + ... return example + >>> ds = ds.map(add_prefix) + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'Review: the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'Review: the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'Review: effective but too-tepid biopic'}] + ``` + """ + if isinstance(input_columns, str): + input_columns = [input_columns] + if isinstance(remove_columns, str): + remove_columns = [remove_columns] + if function is None: + function = identity_func + if fn_kwargs is None: + fn_kwargs = {} + if features is not None: + features = _fix_for_backward_compatible_features(features) + + ex_iterable = self._ex_iterable + # no need to apply features if ex_iterable is typed and if there was no cast_column() + input_features = ( + None + if (ex_iterable.is_typed and (self._info.features is None or self._info.features == ex_iterable.features)) + else self._info.features + ) + + if self._formatting and self._formatting.is_table: + # apply formatting before iter_arrow to keep map examples iterable happy + ex_iterable = FormattedExamplesIterable( + ex_iterable, + formatting=copy.deepcopy(self._formatting), + features=input_features, + token_per_repo_id=self._token_per_repo_id, + ) + ex_iterable = RebatchedArrowExamplesIterable( + ex_iterable, + batch_size=batch_size if batched else 1, + drop_last_batch=drop_last_batch, + force_convert_to_arrow=True, + ) + else: + if self._ex_iterable.iter_arrow: + if self._formatting or input_features: + ex_iterable = RebatchedArrowExamplesIterable( + self._ex_iterable, batch_size=batch_size if batched else 1, drop_last_batch=drop_last_batch + ) + if self._formatting or input_features: + # apply formatting after iter_arrow to avoid re-encoding the examples + ex_iterable = FormattedExamplesIterable( + ex_iterable, + formatting=copy.deepcopy(self._formatting), + features=input_features, + token_per_repo_id=self._token_per_repo_id, + force_convert_to_python=True, + ) + + ex_iterable = MappedExamplesIterable( + ex_iterable, + function=function, + with_indices=with_indices, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + drop_last_batch=drop_last_batch, + remove_columns=remove_columns, + fn_kwargs=fn_kwargs, + formatting=self._formatting, + features=features, + ) + info = self.info.copy() + info.features = features + return IterableDataset( + ex_iterable=ex_iterable, + info=info, + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def filter( + self, + function: Optional[Callable] = None, + with_indices=False, + input_columns: Optional[Union[str, list[str]]] = None, + batched: bool = False, + batch_size: Optional[int] = 1000, + fn_kwargs: Optional[dict] = None, + ) -> "IterableDataset": + """Apply a filter function to all the elements so that the dataset only includes examples according to the filter function. + The filtering is done on-the-fly when iterating over the dataset. + + If the function is asynchronous, then `filter` will run your function in parallel, with up to one thousand simulatenous calls (configurable). + It is recommended to use a `asyncio.Semaphore` in your function if you want to set a maximum number of operations that can run at the same time. + + Args: + function (`Callable`): + Callable with one of the following signatures: + + - `function(example: Dict[str, Any]) -> bool` if `with_indices=False, batched=False` + - `function(example: Dict[str, Any], indices: int) -> bool` if `with_indices=True, batched=False` + - `function(example: Dict[str, List]) -> List[bool]` if `with_indices=False, batched=True` + - `function(example: Dict[str, List], indices: List[int]) -> List[bool]` if `with_indices=True, batched=True` + + If the function is asynchronous, then `filter` will run your function in parallel. + If no function is provided, defaults to an always True function: `lambda x: True`. + with_indices (`bool`, defaults to `False`): + Provide example indices to `function`. Note that in this case the signature of `function` should be `def function(example, idx): ...`. + input_columns (`str` or `List[str]`, *optional*): + The columns to be passed into `function` as + positional arguments. If `None`, a dict mapping to all formatted columns is passed as one argument. + batched (`bool`, defaults to `False`): + Provide batch of examples to `function`. + batch_size (`int`, *optional*, default `1000`): + Number of examples per batch provided to `function` if `batched=True`. + fn_kwargs (`Dict`, *optional*, default `None`): + Keyword arguments to be passed to `function`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> ds = ds.filter(lambda x: x["label"] == 0) + >>> list(ds.take(3)) + [{'label': 0, 'movie_review': 'simplistic , silly and tedious .'}, + {'label': 0, + 'movie_review': "it's so laddish and juvenile , only teenage boys could possibly find it funny ."}, + {'label': 0, + 'movie_review': 'exploitative and largely devoid of the depth or sophistication that would make watching such a graphic treatment of the crimes bearable .'}] + ``` + """ + if isinstance(input_columns, str): + input_columns = [input_columns] + + # We need the examples to be decoded for certain feature types like Image or Audio, + # format and type before filtering + ex_iterable = self._ex_iterable + if self._info.features or self._formatting: + ex_iterable = FormattedExamplesIterable( + ex_iterable, + formatting=self._formatting, + features=ex_iterable.features if ex_iterable.is_typed else self._info.features, + token_per_repo_id=self._token_per_repo_id, + ) + + ex_iterable = FilteredExamplesIterable( + ex_iterable, + function=function, + with_indices=with_indices, + input_columns=input_columns, + batched=batched, + batch_size=batch_size, + fn_kwargs=fn_kwargs, + formatting=self._formatting, + ) + return IterableDataset( + ex_iterable=ex_iterable, + info=self._info, + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def shuffle( + self, seed=None, generator: Optional[np.random.Generator] = None, buffer_size: int = 1000 + ) -> "IterableDataset": + """ + Randomly shuffles the elements of this dataset. + + This dataset fills a buffer with `buffer_size` elements, then randomly samples elements from this buffer, + replacing the selected elements with new elements. For perfect shuffling, a buffer size greater than or + equal to the full size of the dataset is required. + + For instance, if your dataset contains 10,000 elements but `buffer_size` is set to 1000, then `shuffle` will + initially select a random element from only the first 1000 elements in the buffer. Once an element is + selected, its space in the buffer is replaced by the next (i.e. 1,001-st) element, + maintaining the 1000 element buffer. + + If the dataset is made of several shards, it also does shuffle the order of the shards. + However if the order has been fixed by using [`~datasets.IterableDataset.skip`] or [`~datasets.IterableDataset.take`] + then the order of the shards is kept unchanged. + + Args: + seed (`int`, *optional*, defaults to `None`): + Random seed that will be used to shuffle the dataset. + It is used to sample from the shuffle buffer and also to shuffle the data shards. + generator (`numpy.random.Generator`, *optional*): + Numpy random Generator to use to compute the permutation of the dataset rows. + If `generator=None` (default), uses `np.random.default_rng` (the default BitGenerator (PCG64) of NumPy). + buffer_size (`int`, defaults to `1000`): + Size of the buffer. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + >>> shuffled_ds = ds.shuffle(seed=42) + >>> list(shuffled_ds.take(3)) + [{'label': 1, + 'text': "a sports movie with action that's exciting on the field and a story you care about off it ."}, + {'label': 1, + 'text': 'at its best , the good girl is a refreshingly adult take on adultery . . .'}, + {'label': 1, + 'text': "sam jones became a very lucky filmmaker the day wilco got dropped from their record label , proving that one man's ruin may be another's fortune ."}] + ``` + """ + if generator is None: + generator = np.random.default_rng(seed) + else: + generator = deepcopy(generator) + return IterableDataset( + BufferShuffledExamplesIterable( + RebatchedArrowExamplesIterable(self._ex_iterable.shuffle_data_sources(generator), batch_size=1) + if self._ex_iterable.iter_arrow + else self._ex_iterable.shuffle_data_sources(generator), + buffer_size=buffer_size, + generator=generator, + ), + info=self._info.copy(), + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def set_epoch(self, epoch: int): + self._epoch += epoch - self._epoch # update torch value in shared memory in-place + + def skip(self, n: int) -> "IterableDataset": + """ + Create a new [`IterableDataset`] that skips the first `n` elements. + + Args: + n (`int`): + Number of elements to skip. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + >>> ds = ds.skip(1) + >>> list(ds.take(3)) + [{'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}, + {'label': 1, + 'text': 'if you sometimes like to go to the movies to have fun , wasabi is a good place to start .'}] + ``` + """ + ex_iterable = SkipExamplesIterable( + self._ex_iterable, + n, + split_when_sharding=self._distributed is None, + ) + return IterableDataset( + ex_iterable=ex_iterable, + info=self._info.copy(), + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def repeat(self, num_times: Optional[int]) -> "IterableDataset": + """ + Create a new [`IterableDataset`] that repeats the underlying dataset `num_times` times. + + N.B. The effect of calling shuffle after repeat depends significantly on buffer size. + With buffer_size 1, duplicate data is never seen in the same iteration, even after shuffling: + ds.repeat(n).shuffle(seed=42, buffer_size=1) is equivalent to ds.shuffle(seed=42, buffer_size=1).repeat(n), + and only shuffles shard orders within each iteration. + With buffer size >= (num samples in the dataset * num_times), we get full shuffling of the repeated data, i.e. we can observe duplicates in + the same iteration. + + Args: + num_times (`int`) or (`None`): + Number of times to repeat the dataset. If `None`, the dataset will be repeated indefinitely. + + Example: + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train") + >>> ds = ds.take(2).repeat(2) + >>> list(ds) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}, + {'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}, + {'label': 1, 'text': 'effective but too-tepid biopic'}] + ``` + """ + return IterableDataset( + ex_iterable=RepeatExamplesIterable(self._ex_iterable, num_times=num_times), + info=self._info, + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def take(self, n: int) -> "IterableDataset": + """ + Create a new [`IterableDataset`] with only the first `n` elements. + + Args: + n (`int`): + Number of elements to take. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> small_ds = ds.take(2) + >>> list(small_ds) + [{'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}, + {'label': 1, + 'text': 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .'}] + ``` + """ + ex_iterable = TakeExamplesIterable( + self._ex_iterable, + n, + split_when_sharding=self._distributed is None, + ) + return IterableDataset( + ex_iterable=ex_iterable, + info=self._info.copy(), + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def shard( + self, + num_shards: int, + index: int, + contiguous: bool = True, + ) -> "IterableDataset": + """Return the `index`-nth shard from dataset split into `num_shards` pieces. + + This shards deterministically. `dataset.shard(n, i)` splits the dataset into contiguous chunks, + so it can be easily concatenated back together after processing. If `dataset.num_shards % n == l`, then the + first `l` datasets each have `(dataset.num_shards // n) + 1` shards, and the remaining datasets have `(dataset.num_shards // n)` shards. + `datasets.concatenate_datasets([dset.shard(n, i) for i in range(n)])` returns a dataset with the same order as the original. + In particular, `dataset.shard(dataset.num_shards, i)` returns a dataset with 1 shard. + + Note: n should be less or equal to the number of shards in the dataset `dataset.num_shards`. + + On the other hand, `dataset.shard(n, i, contiguous=False)` contains all the shards of the dataset whose index mod `n = i`. + + Be sure to shard before using any randomizing operator (such as `shuffle`). + It is best if the shard operator is used early in the dataset pipeline. + + Args: + num_shards (`int`): + How many shards to split the dataset into. + index (`int`): + Which shard to select and return. + contiguous: (`bool`, defaults to `True`): + Whether to select contiguous blocks of indices for shards. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("fancyzhx/amazon_polarity", split="train", streaming=True) + >>> ds + IterableDataset({ + features: ['label', 'title', 'content'], + num_shards: 4 + }) + >>> ds.shard(num_shards=2, index=0) + IterableDataset({ + features: ['label', 'title', 'content'], + num_shards: 2 + }) + ``` + """ + ex_iterable = self._ex_iterable.shard_data_sources(num_shards=num_shards, index=index, contiguous=contiguous) + return IterableDataset( + ex_iterable=ex_iterable, + info=self._info.copy(), + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def reshard(self) -> "IterableDataset": + """Reshard the dataset if possible, i.e. split the current shards further into more shards. + This increases the number of shards and the resulting dataset has num_shards >= previous_num_shards. + Equality may happen if no shard can be split further. + + The resharding mechanism depends on the dataset file format: + + * Parquet: shard per row group instead of per file + * Other: not implemented yet (contributions are welcome !) + + Be sure to reshard/shard before using any randomizing operator (such as `shuffle`). + It is best if the shard operator is used early in the dataset pipeline. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("fancyzhx/amazon_polarity", split="train", streaming=True) + >>> ds + IterableDataset({ + features: ['label', 'title', 'content'], + num_shards: 4 + }) + >>> ds.reshard() + IterableDataset({ + features: ['label', 'title', 'content'], + num_shards: 3600 + }) + ``` + """ + ex_iterable = self._ex_iterable.reshard_data_sources() + return IterableDataset( + ex_iterable=ex_iterable, + info=self._info.copy(), + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def add_column(self, name: str, column: Union[list, np.array]) -> "IterableDataset": + """Add column to Dataset. + + Args: + name (str): Column name. + column (list or np.array): Column data to be added. + + Returns: + `IterableDataset` + """ + return self.map(partial(add_column_fn, name=name, column=column), with_indices=True) + + def rename_column(self, original_column_name: str, new_column_name: str) -> "IterableDataset": + """ + Rename a column in the dataset, and move the features associated to the original column under the new column + name. + + Args: + original_column_name (`str`): + Name of the column to rename. + new_column_name (`str`): + New name for the column. + + Returns: + `IterableDataset`: A copy of the dataset with a renamed column. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> next(iter(ds)) + {'label': 1, + 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + >>> ds = ds.rename_column("text", "movie_review") + >>> next(iter(ds)) + {'label': 1, + 'movie_review': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + return self.rename_columns({original_column_name: new_column_name}) + + def rename_columns(self, column_mapping: dict[str, str]) -> "IterableDataset": + """ + Rename several columns in the dataset, and move the features associated to the original columns under + the new column names. + + Args: + column_mapping (`Dict[str, str]`): A mapping of columns to rename to their new names + + Returns: + `IterableDataset`: A copy of the dataset with renamed columns + """ + + original_features = self._info.features.copy() if self._info.features else None + ds_iterable = self.map( + partial(_rename_columns_fn, column_mapping=column_mapping), remove_columns=list(column_mapping) + ) + if original_features is not None: + ds_iterable._info.features = Features( + { + column_mapping[col] if col in column_mapping.keys() else col: feature + for col, feature in original_features.items() + } + ) + return ds_iterable + + def remove_columns(self, column_names: Union[str, list[str]]) -> "IterableDataset": + """ + Remove one or several column(s) in the dataset and the features associated to them. + The removal is done on-the-fly on the examples when iterating over the dataset. + + + Args: + column_names (`Union[str, List[str]]`): + Name of the column(s) to remove. + + Returns: + `IterableDataset`: A copy of the dataset object without the columns to remove. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> next(iter(ds)) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', 'label': 1} + >>> ds = ds.remove_columns("label") + >>> next(iter(ds)) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + original_features = self._info.features.copy() if self._info.features else None + ds_iterable = self.map(remove_columns=column_names) + if original_features is not None: + ds_iterable._info.features = original_features.copy() + for col, _ in original_features.items(): + if col in column_names: + del ds_iterable._info.features[col] + + return ds_iterable + + def select_columns(self, column_names: Union[str, list[str]]) -> "IterableDataset": + """Select one or several column(s) in the dataset and the features + associated to them. The selection is done on-the-fly on the examples + when iterating over the dataset. + + + Args: + column_names (`Union[str, List[str]]`): + Name of the column(s) to select. + + Returns: + `IterableDataset`: A copy of the dataset object with selected columns. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> next(iter(ds)) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', 'label': 1} + >>> ds = ds.select_columns("text") + >>> next(iter(ds)) + {'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'} + ``` + """ + if isinstance(column_names, str): + column_names = [column_names] + + if self._info: + info = copy.deepcopy(self._info) + if self._info.features is not None: + missing_columns = set(column_names) - set(self._info.features.keys()) + if missing_columns: + raise ValueError( + f"Column name {list(missing_columns)} not in the " + "dataset. Columns in the dataset: " + f"{list(self._info.features.keys())}." + ) + info.features = Features({c: info.features[c] for c in column_names}) + + ex_iterable = SelectColumnsIterable(self._ex_iterable, column_names) + return IterableDataset( + ex_iterable=ex_iterable, + info=info, + split=self._split, + formatting=self._formatting, + distributed=self._distributed, + token_per_repo_id=self._token_per_repo_id, + ) + + def cast_column(self, column: str, feature: FeatureType) -> "IterableDataset": + """Cast column to feature for decoding. + + Args: + column (`str`): + Column name. + feature (`Feature`): + Target feature. + + Returns: + `IterableDataset` + + Example: + + ```py + >>> from datasets import load_dataset, Audio + >>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train", streaming=True) + >>> ds.features + {'audio': Audio(sampling_rate=8000, mono=True, decode=True, id=None), + 'english_transcription': Value('string'), + 'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill']), + 'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN']), + 'path': Value('string'), + 'transcription': Value('string')} + >>> ds = ds.cast_column("audio", Audio(sampling_rate=16000)) + >>> ds.features + {'audio': Audio(sampling_rate=16000, mono=True, decode=True, id=None), + 'english_transcription': Value('string'), + 'intent_class': ClassLabel(num_classes=14, names=['abroad', 'address', 'app_error', 'atm_limit', 'balance', 'business_loan', 'card_issues', 'cash_deposit', 'direct_debit', 'freeze', 'high_value_payment', 'joint_account', 'latest_transactions', 'pay_bill']), + 'lang_id': ClassLabel(num_classes=14, names=['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN']), + 'path': Value('string'), + 'transcription': Value('string')} + ``` + """ + feature = _fix_for_backward_compatible_features(feature) + info = self._info.copy() + info.features[column] = feature + return IterableDataset( + ex_iterable=self._ex_iterable, + info=info, + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def cast( + self, + features: Features, + ) -> "IterableDataset": + """ + Cast the dataset to a new set of features. + + Args: + features ([`Features`]): + New features to cast the dataset to. + The name of the fields in the features must match the current column names. + The type of the data must also be convertible from one type to the other. + For non-trivial conversion, e.g. `string` <-> `ClassLabel` you should use [`~Dataset.map`] to update the Dataset. + + Returns: + `IterableDataset`: A copy of the dataset with casted features. + + Example: + + ```py + >>> from datasets import load_dataset, ClassLabel, Value + >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train", streaming=True) + >>> ds.features + {'label': ClassLabel(names=['neg', 'pos']), + 'text': Value('string')} + >>> new_features = ds.features.copy() + >>> new_features["label"] = ClassLabel(names=["bad", "good"]) + >>> new_features["text"] = Value("large_string") + >>> ds = ds.cast(new_features) + >>> ds.features + {'label': ClassLabel(names=['bad', 'good']), + 'text': Value('large_string')} + ``` + """ + features = _fix_for_backward_compatible_features(features) + info = self._info.copy() + info.features = features + return IterableDataset( + ex_iterable=self._ex_iterable, + info=info, + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def decode(self, enable: bool = True, num_threads: int = 0) -> "IterableDataset": + """ + Enable or disable the dataset features decoding for audio, image, video. + + When enabled (default), media types are decoded: + + * audio -> dict of "array" and "sampling_rate" and "path" + * image -> PIL.Image + * video -> torchcodec.decoders.VideoDecoder + + You can enable multithreading using `num_threads`. This is especially useful to speed up remote + data streaming. However it can be slower than `num_threads=0` for local data on fast disks. + + Disabling decoding is useful if you want to iterate on the paths or bytes of the media files + without actually decoding their content. To disable decoding you can use `.decode(False)`, which + is equivalent to calling `.cast()` or `.cast_column()` with all the Audio, Image and Video types + set to `decode=False`. + + Args: + enable (`bool`, defaults to `True`): + Enable or disable features decoding. + num_threads (`int`, defaults to `0`): + Enable multithreading for features decoding. + + Returns: + `IterableDataset`: A copy of the dataset with casted features. + + Examples: + + Disable decoding: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("sshh12/planet-textures", split="train", streaming=True) + >>> next(iter(ds)) + {'image': , + 'text': 'A distant celestial object with an icy crust, displaying a light blue shade, covered with round pits and rugged terrains.'} + >>> ds = ds.decode(False) + >>> ds.features + {'image': Image(mode=None, decode=False, id=None), + 'text': Value('string')} + >>> next(iter(ds)) + { + 'image': { + 'path': 'hf://datasets/sshh12/planet-textures@69dc4cef7a5c4b2cfe387727ec8ea73d4bff7302/train/textures/0000.png', + 'bytes': None + }, + 'text': 'A distant celestial object with an icy crust, displaying a light blue shade, covered with round pits and rugged terrains.' + } + ``` + + Speed up streaming with multithreading: + + ```py + >>> import os + >>> from datasets import load_dataset + >>> from tqdm import tqdm + >>> ds = load_dataset("sshh12/planet-textures", split="train", streaming=True) + >>> num_threads = min(32, (os.cpu_count() or 1) + 4) + >>> ds = ds.decode(num_threads=num_threads) + >>> for _ in tqdm(ds): # 20 times faster ! + ... ... + ``` + """ + if not self.features: + raise ValueError( + "Features decoding is only available for datasets with known features, but features are Unknown. " + "Please set the datasets features with `ds = ds.cast(features)`." + ) + ds = self + + def set_decoding(decode: bool, feature): + if hasattr(feature, "decode"): + feature.decode = decode + + if enable and num_threads > 0: + disabled_decoding_features = self.features.copy() + enabled_decoding_features = self.features.copy() + + _visit(disabled_decoding_features, partial(set_decoding, False)) + _visit(enabled_decoding_features, partial(set_decoding, True)) + ds = ds.cast(disabled_decoding_features) + pool = multiprocessing.pool.ThreadPool(num_threads) + func = partial(_apply_async, pool, enabled_decoding_features.decode_example) + ds = ds.map(func, features=enabled_decoding_features) + assert isinstance(ds._ex_iterable, MappedExamplesIterable) + ds._ex_iterable.max_num_running_async_map_functions_in_parallel = 2 * num_threads + else: + features = ds.features.copy() + _visit(features, partial(set_decoding, enable)) + ds = ds.cast(features) + return ds + + def _step(self, step: int, offset: int) -> "IterableDataset": + ex_iterable = StepExamplesIterable(self._ex_iterable, step=step, offset=offset) + return IterableDataset( + ex_iterable=ex_iterable, + info=self._info.copy(), + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def _resolve_features(self): + if self.features is not None: + return self + elif self._ex_iterable.is_typed: + features = self._ex_iterable.features + else: + features = _infer_features_from_batch(self.with_format(None)._head()) + info = self.info.copy() + info.features = features + return IterableDataset( + ex_iterable=self._ex_iterable, + info=info, + split=self._split, + formatting=self._formatting, + distributed=copy.deepcopy(self._distributed), + token_per_repo_id=self._token_per_repo_id, + ) + + def batch(self, batch_size: int, drop_last_batch: bool = False) -> "IterableDataset": + """ + Group samples from the dataset into batches. + + Args: + batch_size (`int`): The number of samples in each batch. + drop_last_batch (`bool`, defaults to `False`): Whether to drop the last incomplete batch. + + Example: + ```py + >>> ds = load_dataset("some_dataset", streaming=True) + >>> batched_ds = ds.batch(batch_size=32) + ``` + """ + + if self.features: + features = Features({col: List(feature) for col, feature in self.features.items()}) + else: + features = None + if self._formatting and self._formatting.is_table: + return ( + self.with_format("arrow") + .map( + _batch_arrow_table, + batched=True, + batch_size=batch_size, + drop_last_batch=drop_last_batch, + features=features, + ) + .with_format(self._formatting.format_type) + ) + return self.map( + _batch_fn, batched=True, batch_size=batch_size, drop_last_batch=drop_last_batch, features=features + ) + + def to_dict(self, batch_size: Optional[int] = None, batched: bool = False) -> Union[dict, Iterator[dict]]: + """Returns the dataset as a Python dict. Can also return a generator for large datasets. + + Args: + batch_size (`int`, *optional*): The size (number of rows) of the batches if `batched` is `True`. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + + Returns: + `dict` or `Iterator[dict]` + + Example: + + ```py + >>> ds.to_dict() + ``` + """ + if batched: + for table in self.with_format("arrow").iter(batch_size=batch_size): + yield Dataset(table, fingerprint="unset").to_dict() + else: + table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000))) + return Dataset(table, fingerprint="unset").to_dict() + + def to_list(self) -> list: + """Returns the dataset as a Python list. + + Returns: + `list` + + Example: + + ```py + >>> ds.to_list() + ``` + """ + table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000))) + return Dataset(table, fingerprint="unset").to_list() + + def to_pandas( + self, batch_size: Optional[int] = None, batched: bool = False + ) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]: + """Returns the dataset as a `pandas.DataFrame`. Can also return a generator for large datasets. + + Args: + batch_size (`int`, *optional*): + The size (number of rows) of the batches if `batched` is `True`. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + batched (`bool`): + Set to `True` to return a generator that yields the dataset as batches + of `batch_size` rows. Defaults to `False` (returns the whole datasets once). + + Returns: + `pandas.DataFrame` or `Iterator[pandas.DataFrame]` + + Example: + + ```py + >>> ds.to_pandas() + ``` + """ + info = DatasetInfo(features=self.features.copy()) if self.features is not None else None + + def maybe_cast_to_declared_features(table: pa.Table): + if self.features is not None and table.schema != self.features.arrow_schema: + return cast_table_to_features(table, self.features) + return table + + if batched: + return ( + Dataset(maybe_cast_to_declared_features(table), info=info, fingerprint="unset").to_pandas() + for table in self.with_format("arrow").iter(batch_size=batch_size) + ) + else: + table = pa.concat_tables( + [maybe_cast_to_declared_features(table) for table in self.with_format("arrow").iter(batch_size=1000)] + ) + return Dataset(table, info=info, fingerprint="unset").to_pandas() + + def to_polars( + self, + batch_size: Optional[int] = None, + batched: bool = False, + schema_overrides: Optional[dict] = None, + rechunk: bool = True, + ) -> Union["pl.DataFrame", Iterator["pl.DataFrame"]]: + """Returns the dataset as a `polars.DataFrame`. Can also return a generator for large datasets. + + Args: + batch_size (`int`, *optional*): + The size (number of rows) of the batches if `batched` is `True`. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + batched (`bool`): + Set to `True` to return a generator that yields the dataset as batches + of `batch_size` rows. Defaults to `False` (returns the whole datasets once). + schema_overrides (`dict`, *optional*): + Support type specification or override of one or more columns; note that + any dtypes inferred from the schema param will be overridden. + rechunk (`bool`): + Make sure that all data is in contiguous memory. Defaults to `True`. + Returns: + `polars.DataFrame` or `Iterator[polars.DataFrame]` + + Example: + + ```py + >>> ds.to_polars() + ``` + """ + if batched: + for table in self.with_format("arrow").iter(batch_size=batch_size): + yield Dataset(table, fingerprint="unset").to_polars(schema_overrides=schema_overrides, rechunk=rechunk) + else: + table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000))) + return Dataset(table, fingerprint="unset").to_polars(schema_overrides=schema_overrides, rechunk=rechunk) + + def to_csv( + self, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_csv_kwargs, + ) -> int: + """Exports the dataset to csv. + + This iterates on the dataset and loads it completely in memory before writing it. + + Args: + path_or_buf (`PathLike` or `FileOrBuffer`): + Either a path to a file (e.g. `file.csv`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.csv`), + or a BinaryIO, where the dataset will be saved to in the specified format. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + **to_csv_kwargs (additional keyword arguments): + Parameters to pass to pandas's [`pandas.DataFrame.to_csv`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_csv.html). + The parameter `index` defaults to `False` if not specified. + If you would like to write the index, pass `index=True` and also set a name for the index column by + passing `index_label`. + + Returns: + `int`: The number of characters or bytes written. + + Example: + + ```py + >>> ds.to_csv("path/to/dataset/directory") + ``` + """ + table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000))) + return Dataset(table, fingerprint="unset").to_csv( + path_or_buf, + batch_size=batch_size, + storage_options=storage_options, + **to_csv_kwargs, + ) + + def to_json( + self, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_json_kwargs, + ) -> int: + """Export the dataset to JSON Lines or JSON. + + This iterates on the dataset and loads it completely in memory before writing it. + + The default output format is [JSON Lines](https://jsonlines.org/). + To export to [JSON](https://www.json.org), pass `lines=False` argument and the desired `orient`. + + Args: + path_or_buf (`PathLike` or `FileOrBuffer`): + Either a path to a file (e.g. `file.json`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.json`), + or a BinaryIO, where the dataset will be saved to in the specified format. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + **to_json_kwargs (additional keyword arguments): + Parameters to pass to pandas's [`pandas.DataFrame.to_json`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_json.html). + Default arguments are `lines=True` and `orient="records". + The parameter `index` defaults to `False` if `orient` is `"split"` or `"table"`. + If you would like to write the index, pass `index=True`. + + Returns: + `int`: The number of characters or bytes written. + + Example: + + ```py + >>> ds.to_json("path/to/dataset/directory/filename.jsonl") + ``` + + ```py + >>> num_shards = dataset.num_shards + >>> for index in range(num_shards): + ... shard = dataset.shard(index, num_shards) + ... shard.to_json(f"path/of/my/dataset/data-{index:05d}.jsonl") + ``` + + """ + table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000))) + return Dataset(table, fingerprint="unset").to_json( + path_or_buf, + batch_size=batch_size, + storage_options=storage_options, + **to_json_kwargs, + ) + + def to_sql( + self, + name: str, + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"], + batch_size: Optional[int] = None, + **sql_writer_kwargs, + ) -> int: + """Exports the dataset to a SQL database. + + Args: + name (`str`): + Name of SQL table. + con (`str` or `sqlite3.Connection` or `sqlalchemy.engine.Connection` or `sqlalchemy.engine.Connection`): + A [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) or a SQLite3/SQLAlchemy connection object used to write to a database. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + **sql_writer_kwargs (additional keyword arguments): + Parameters to pass to pandas's [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_sql.html). + The parameter `index` defaults to `False` if not specified. + If you would like to write the index, pass `index=True` and also set a name for the index column by + passing `index_label`. + + + Returns: + `int`: The number of records written. + + Example: + + ```py + >>> # con provided as a connection URI string + >>> ds.to_sql("data", "sqlite:///my_own_db.sql") + >>> # con provided as a sqlite3 connection object + >>> import sqlite3 + >>> con = sqlite3.connect("my_own_db.sql") + >>> with con: + ... ds.to_sql("data", con) + ``` + """ + table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=1000))) + return Dataset(table, fingerprint="unset").to_sql(name, con, batch_size=batch_size, **sql_writer_kwargs) + + def to_parquet( + self, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + storage_options: Optional[dict] = None, + **parquet_writer_kwargs, + ) -> int: + """Exports the dataset to parquet + + Args: + path_or_buf (`PathLike` or `FileOrBuffer`): + Either a path to a file (e.g. `file.parquet`), a remote URI (e.g. `hf://datasets/username/my_dataset_name/data.parquet`), + or a BinaryIO, where the dataset will be saved to in the specified format. + batch_size (`int`, *optional*): + Size of the batch to load in memory and write at once. + Defaults to `datasets.config.DEFAULT_MAX_BATCH_SIZE`. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + **parquet_writer_kwargs (additional keyword arguments): + Parameters to pass to PyArrow's `pyarrow.parquet.ParquetWriter`. + + Returns: + `int`: The number of characters or bytes written. + + Example: + + ```py + >>> ds.to_parquet("path/to/dataset/directory") + ``` + + ```py + >>> num_shards = dataset.num_shards + >>> for index in range(num_shards): + ... shard = dataset.shard(index, num_shards) + ... shard.to_parquet(f"path/of/my/dataset/data-{index:05d}.parquet") + ``` + + """ + from .arrow_writer import get_arrow_writer_batch_size_from_features + + batch_size = get_arrow_writer_batch_size_from_features(self.features) or config.DEFAULT_MAX_BATCH_SIZE + table = pa.concat_tables(list(self.with_format("arrow").iter(batch_size=batch_size))) + return Dataset(table, fingerprint="unset").to_parquet( + path_or_buf, storage_options=storage_options, **parquet_writer_kwargs + ) + + def _push_parquet_shards_to_hub_single( + self, + job_id: int, + num_jobs: int, + resolved_output_path: HfFileSystemResolvedPath, + data_dir: str, + split: str, + token: Optional[str], + create_pr: Optional[bool], + # max_shard_size: Optional[Union[int, str]] = None, # TODO(QL): add arg + num_shards: int, + embed_external_files: bool, + ) -> Iterable[tuple[list[CommitOperationAdd], list[str], int, int]]: + """Pushes the dataset shards as Parquet files to the hub. + + Returns: + additions (`List[CommitOperation]`): list of the `CommitOperationAdd` of the uploaded shards + new_parquet_paths (`List[str]`): list of the paths of the uploaded parquet files + features (`Features`): features of the uploaded dataset + dataset_nbytes (`int`): approximate size in bytes of the uploaded dataset after uncompression + num_examples (`int`): number of examples of th euploaded shards + """ + + div = num_shards // num_jobs + mod = num_shards % num_jobs + start = div * job_id + min(job_id, mod) + end = start + div + (1 if job_id < mod else 0) + + index_shards = ( + (start + i, self.shard(num_shards=end - start, index=i, contiguous=True)) for i in range(end - start) + ) + + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + + dataset_nbytes = 0 + num_examples = 0 + additions: list[CommitOperationAdd] = [] + new_parquet_paths: list[str] = [] + features = self.features + for index, shard in index_shards: + if embed_external_files: + from .arrow_writer import get_arrow_writer_batch_size_from_features + + shard = shard.with_format("arrow") + shard = shard.map( + partial(embed_table_storage, token_per_repo_id=self._token_per_repo_id), + batched=True, + batch_size=get_arrow_writer_batch_size_from_features(shard.features), + ) + shard_path_in_repo = f"{data_dir}/{split}-{index:05d}-of-{num_shards:05d}.parquet" + tmp_file = tempfile.NamedTemporaryFile(suffix=".parquet", delete=False) + try: + shard.to_parquet(tmp_file) + tmp_file.close() + parquet_metadata = pq.read_metadata(tmp_file.name) + if features is None: + features = Features.from_arrow_schema(parquet_metadata.schema.to_arrow_schema()) + num_examples += parquet_metadata.num_rows + dataset_nbytes += sum( + parquet_metadata.row_group(i).total_byte_size for i in range(parquet_metadata.num_row_groups) + ) + new_parquet_paths.append(shard_path_in_repo) + if ( + isinstance(resolved_output_path, HfFileSystemResolvedRepositoryPath) + and not resolved_output_path.path_in_repo + ): + shard_addition = CommitOperationAdd(path_in_repo=shard_path_in_repo, path_or_fileobj=tmp_file.name) + api.preupload_lfs_files( + repo_id=resolved_output_path.repo_id, + additions=[shard_addition], + repo_type=resolved_output_path.repo_type, + revision=resolved_output_path.revision, + create_pr=create_pr, + ) + additions.append(shard_addition) + elif isinstance(resolved_output_path, HfFileSystemResolvedBucketPath): + if resolved_output_path.path: + shard_path_in_repo = resolved_output_path.path + "/" + shard_path_in_repo + api.batch_bucket_files( + bucket_id=resolved_output_path.bucket_id, add=[(tmp_file.name, shard_path_in_repo)] + ) + else: + raise NotImplementedError(f"Bad HF path: {resolved_output_path}") + except (Exception, KeyboardInterrupt): + tmp_file.close() + Path(tmp_file.name).unlink() + raise + tmp_file.close() + Path(tmp_file.name).unlink() + yield job_id, False, 1 + + yield job_id, True, (additions, new_parquet_paths, features, dataset_nbytes, num_examples) + + def _push_parquet_shards_to_hub( + self, + resolved_output_path: HfFileSystemResolvedPath, + data_dir: str, + split: str, + token: Optional[str], + create_pr: Optional[bool], + max_shard_size: Optional[Union[int, str]], + num_shards: Optional[int], + embed_external_files: bool, + num_proc: Optional[int], + ) -> tuple[list[CommitOperationAdd], list[str], Features, SplitInfo, int]: + """Pushes the dataset shards as Parquet files to the hub. + + Returns: + additions (`List[CommitOperation]`): list of the `CommitOperationAdd` of the uploaded shards + new_parquet_paths (`List[str]`): list of paths of the new files uploaded to the output path, + relative to output path + features (`features`): features of the uploaded dataset + split_info (`int`): info of the uploaded split, including the approximate size in bytes of + the uploaded dataset after uncompression + uploaded_size (`int`): number of uploaded bytes to the repository or bucket + """ + + # Find decodable columns, because if there are any, we need to: + # embed the bytes from the files in the shards + decodable_columns = ( + [k for k, v in self._info.features.items() if require_decoding(v, ignore_decode_attribute=True)] + if embed_external_files + else [] + ) + embed_external_files = embed_external_files and bool(decodable_columns) + + if num_shards is None: + if max_shard_size is None: + num_shards = self.num_shards + else: + max_shard_size = convert_file_size_to_int(max_shard_size or config.MAX_SHARD_SIZE) + estimated_nbytes = 0 + for pa_table in self.with_format("arrow").iter(batch_size=config.DEFAULT_MAX_BATCH_SIZE): + estimated_nbytes += pa_table.nbytes + num_shards = int(estimated_nbytes / max_shard_size) + 1 + num_shards = max(num_shards, num_proc or 1) + + additions: list[CommitOperationAdd] = [] + new_parquet_paths: list[str] = [] + uploaded_size = 0 + dataset_nbytes = 0 + num_examples = 0 + features = self.features + + num_jobs = num_proc or 1 + if num_shards <= 1: + logger.warning( + f"Setting num_proc from {num_jobs} back to 1 for the {split} split to disable multiprocessing as it only contains one shard." + ) + num_proc = None + num_jobs = 1 + elif num_shards < num_jobs: + logger.warning( + f"Setting num_proc from {num_jobs} to {num_shards} for the {split} split as it only contains {num_shards} shards." + ) + num_proc = num_shards + num_jobs = num_shards + kwargs_iterable = [ + { + "self": self.shard(num_shards=num_jobs, index=job_id, contiguous=True), + "job_id": job_id, + "num_jobs": num_jobs, + "resolved_output_path": resolved_output_path, + "data_dir": data_dir, + "split": split, + "token": token, + "create_pr": create_pr, + "num_shards": num_shards, + "embed_external_files": embed_external_files, + } + for job_id in range(num_jobs) + ] + desc = "Uploading the dataset shards" + desc += f" (num_proc={num_proc})" if num_proc is not None and num_proc >= 1 else "" + pbar = hf_tqdm( + unit=" shards", + total=num_shards, + desc=desc, + ) + with ( + contextlib.nullcontext() + if num_proc is None or num_proc < 1 + else mp.get_context("spawn").Pool(num_proc) as pool + ): + update_stream = ( + IterableDataset._push_parquet_shards_to_hub_single(**kwargs_iterable[0]) + if pool is None + else iflatmap_unordered( + pool, + IterableDataset._push_parquet_shards_to_hub_single, + kwargs_iterable=kwargs_iterable, + ) + ) + for job_id, done, content in update_stream: + if not done: + pbar.update(content) + else: + job_additions, job_new_parquet_paths, job_features, job_uploaded_size, job_num_examples = content + additions += job_additions + new_parquet_paths += job_new_parquet_paths + uploaded_size += job_uploaded_size + num_examples += job_num_examples + features = job_features + if pool is not None: + pool.close() + pool.join() + + uploaded_size = sum(addition.upload_info.size for addition in additions) + split_info = SplitInfo(split, num_bytes=dataset_nbytes, num_examples=num_examples) + return additions, new_parquet_paths, features, split_info, uploaded_size + + def push_to_hub( + self, + repo_id: str, + config_name: str = "default", + set_default: Optional[bool] = None, + split: Optional[str] = None, + data_dir: Optional[str] = None, + commit_message: Optional[str] = None, + commit_description: Optional[str] = None, + private: Optional[bool] = None, + token: Optional[str] = None, + revision: Optional[str] = None, + create_pr: Optional[bool] = False, + max_shard_size: Optional[Union[int, str]] = None, + num_shards: Optional[int] = None, + embed_external_files: bool = True, + num_proc: Optional[int] = None, + ) -> CommitInfo: + """Pushes the dataset to the hub as a Parquet dataset. + The dataset is pushed using HTTP requests and does not need to have neither git or git-lfs installed. + + The resulting Parquet files are self-contained by default. If your dataset contains [`Image`], [`Audio`] or [`Video`] + data, the Parquet files will store the bytes of your images or audio files. + You can disable this by setting `embed_external_files` to `False`. + + Args: + repo_id (`str`): + The ID of the repository to push to in the following format: `/` or + `/`. Also accepts ``, which will default to the namespace + of the logged-in user. + + It could also be a location inside a bucket, e.g. `buckets///...` + config_name (`str`, defaults to "default"): + The configuration name (or subset) of a dataset. Defaults to "default". + set_default (`bool`, *optional*): + Whether to set this configuration as the default one. Otherwise, the default configuration is the one + named "default". + split (`str`, *optional*): + The name of the split that will be given to that dataset. Defaults to `self.split`. + data_dir (`str`, *optional*): + Directory name that will contain the uploaded data files. Defaults to the `config_name` if different + from "default", else "data". + commit_message (`str`, *optional*): + Message to commit while pushing. Will default to `"Upload dataset"`. + commit_description (`str`, *optional*): + Description of the commit that will be created. + Additionally, description of the PR if a PR is created (`create_pr` is True). + private (`bool`, *optional*): + Whether to make the repo private. If `None` (default), the repo will be public unless the + organization's default is private. This value is ignored if the repo already exists. + token (`str`, *optional*): + An optional authentication token for the Hugging Face Hub. If no token is passed, will default + to the token saved locally when logging in with `huggingface-cli login`. Will raise an error + if no token is passed and the user is not logged-in. + revision (`str`, *optional*): + Branch to push the uploaded files to. Defaults to the `"main"` branch. + create_pr (`bool`, *optional*, defaults to `False`): + Whether to create a PR with the uploaded files or directly commit. + max_shard_size (`int` or `str`, *optional*): + Optional maximum size of the dataset shards to be uploaded to the hub. If expressed as a string, needs to be digits followed + by a unit (like `"5MB"`). If not provided, shard count defaults to this dataset's `.num_shards`. + num_shards (`int`, *optional*): + Number of shards to write. If `max_shard_size` is provided and `num_shards` is not, then the number of shards is estimated + from `max_shard_size`. + embed_external_files (`bool`, defaults to `True`): + Whether to embed file bytes in the shards. + In particular, this will do the following before the push for the fields of type: + + - [`Audio`] and [`Image`]: remove local path information and embed file content in the Parquet files. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when preparing and uploading the dataset. + This is helpful if the dataset is made of many samples and transformations. + I uses "spawn" context to work with hf_xet, the rust client for fast uploads to HF. + Multiprocessing is disabled by default. + + Return: + huggingface_hub.CommitInfo + + Example: + + ```python + >>> dataset.push_to_hub("/") + >>> dataset_dict.push_to_hub("/", private=True) + >>> dataset.push_to_hub("/", max_shard_size="1GB") + >>> dataset.push_to_hub("/", num_shards=1024) + ``` + + If your dataset has multiple splits (e.g. train/validation/test): + + ```python + >>> train_dataset.push_to_hub("/", split="train") + >>> val_dataset.push_to_hub("/", split="validation") + >>> # later + >>> dataset = load_dataset("/") + >>> train_dataset = dataset["train"] + >>> val_dataset = dataset["validation"] + ``` + + If you want to add a new configuration (or subset) to a dataset (e.g. if the dataset has multiple tasks/versions/languages): + + ```python + >>> english_dataset.push_to_hub("/", "en") + >>> french_dataset.push_to_hub("/", "fr") + >>> # later + >>> english_dataset = load_dataset("/", "en") + >>> french_dataset = load_dataset("/", "fr") + ``` + """ + if num_proc is not None and num_proc > self.num_shards: + logger.warning( + f"Too many num_proc: {num_proc} (max is dataset.num_shards={self.num_shards}). " + f"Stopping {num_proc - self.num_shards} processes." + ) + logger.info( + f"To parallelize data loading, we give each process some shards (or data sources) to process. " + f"Therefore it's unnecessary to have a number of processes greater than dataset.num_shards={self.num_shards}. " + f"To enable more parallelism, please split the dataset in more files than {self.num_shards}." + ) + num_proc = self.num_shards + + if config_name == "data": + raise ValueError("`config_name` cannot be 'data'. Please, choose another name for configuration.") + + if max_shard_size is not None and num_shards is not None: + raise ValueError( + "Failed to push_to_hub: please specify either max_shard_size or num_shards, but not both." + ) + + if split is None: + split = str(self.split) if self.split is not None else "train" + + if not re.match(_split_re, split): + raise ValueError(f"Split name should match '{_split_re}' but got '{split}'.") + + if not data_dir: + data_dir = config_name if config_name != "default" else "data" # for backward compatibility + + api = HfApi(endpoint=config.HF_ENDPOINT, token=token) + if repo_id.startswith("buckets/"): + if BucketNotFoundError is None: + raise ImportError("Pushing datasets to buckets requires huggingface_hub>=1.6.0") + _, _namespace, _bucket_name, *_path_segments = repo_id.split("/") + try: + bucket_id = api.bucket_info(_namespace + "/" + _bucket_name).id + except BucketNotFoundError: + bucket_url = api.create_bucket(_namespace + "/" + _bucket_name, private=private, exist_ok=True) + bucket_id = bucket_url.bucket_id + path = "/".join(s for s in _path_segments if s) + return _push_to_bucket( + self, + bucket_id=bucket_id, + path=path, + config_name=config_name, + set_default=set_default, + split=split, + data_dir=data_dir, + token=token, + max_shard_size=max_shard_size, + num_shards=num_shards, + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + else: + try: + repo_id = api.repo_info(repo_id, repo_type="dataset").id + except RepositoryNotFoundError: + repo_url = api.create_repo( + repo_id, + repo_type="dataset", + private=private, + exist_ok=True, + ) + repo_id = repo_url.repo_id + + if revision is not None and not revision.startswith("refs/pr/"): + # We do not call create_branch for a PR reference: 400 Bad Request + api.create_branch(repo_id, branch=revision, repo_type="dataset", exist_ok=True) + return _push_to_repo( + self, + repo_id=repo_id, + config_name=config_name, + set_default=set_default, + split=split, + data_dir=data_dir, + commit_message=commit_message, + commit_description=commit_description, + token=token, + revision=revision, + create_pr=create_pr, + max_shard_size=max_shard_size, + num_shards=num_shards, + embed_external_files=embed_external_files, + num_proc=num_proc, + ) + + +def _concatenate_iterable_datasets( + dsets: list[IterableDataset], + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + axis: int = 0, +) -> IterableDataset: + """ + Converts a list of `IterableDataset` with the same schema into a single `IterableDataset`. + Missing data are filled with None values. + + + + Args: + dsets (`List[datasets.IterableDataset]`): List of Datasets to concatenate. + info (`DatasetInfo`, optional): Dataset information, like description, citation, etc. + split (`NamedSplit`, optional): Name of the dataset split. + axis (``{0, 1}``, default ``0``, meaning over rows): + Axis to concatenate over, where ``0`` means over rows (vertically) and ``1`` means over columns + (horizontally). + + *New in version 1.6.0* + + Example: + + ```py + >>> ds3 = _concatenate_iterable_datasets([ds1, ds2]) + ``` + """ + dsets = [d._resolve_features() for d in dsets] + + # Perform checks (and a potentional cast if axis=0) + if axis == 0: + _check_if_features_can_be_aligned([dset.features for dset in dsets]) + else: + _check_column_names([col_name for dset in dsets for col_name in dset.features]) + + # Check format is consistent; if so, will set format for concatenated dataset + if all(dset._formatting is None for dset in dsets): + formatting = None + elif any(dset._formatting is None for dset in dsets): + formatting = None + logger.info( + "Some of the datasets have disparate format or format not set. Resetting the format of the concatenated dataset." + ) + else: + format_type_set = {dset._formatting.format_type for dset in dsets} + if len(format_type_set) == 1: + format_type = format_type_set.pop() + formatting = FormattingConfig(format_type=format_type) + else: + formatting = None + logger.info( + "Some of the datasets have disparate format or format not set. Resetting the format of the concatenated dataset." + ) + + # TODO: improve this to account for a mix of ClassLabel and Value for example + # right now it would keep the type of the first dataset in the list + features = Features( + {k: v for features in _align_features([dset.features for dset in dsets]) for k, v in features.items()} + ) + + ex_iterables = [copy.deepcopy(d._ex_iterable) for d in dsets] + if axis == 0: + ex_iterable = VerticallyConcatenatedMultiSourcesExamplesIterable(ex_iterables) + else: + if all(ex_iterable.iter_arrow for ex_iterable in ex_iterables): + from .arrow_writer import get_arrow_writer_batch_size_from_features + + batch_size = get_arrow_writer_batch_size_from_features(features) or config.DEFAULT_MAX_BATCH_SIZE + ex_iterables = [ + RebatchedArrowExamplesIterable(ex_iterable, batch_size=batch_size) for ex_iterable in ex_iterables + ] + ex_iterable = HorizontallyConcatenatedMultiSourcesExamplesIterable(ex_iterables) + # Set new info - we update the features + # setting the features also ensures to fill missing columns with None + if info is None: + info = DatasetInfo.from_merge([d.info for d in dsets]) + else: + info = info.copy() + info.features = features + # Get all the auth tokens per repository - in case the datasets come from different private repositories + token_per_repo_id = {repo_id: token for dataset in dsets for repo_id, token in dataset._token_per_repo_id.items()} + # Return new daset + return IterableDataset( + ex_iterable=ex_iterable, + info=info, + split=split, + token_per_repo_id=token_per_repo_id, + formatting=formatting, + ) + + +def _interleave_iterable_datasets( + datasets: list[IterableDataset], + probabilities: Optional[list[float]] = None, + seed: Optional[int] = None, + info: Optional[DatasetInfo] = None, + split: Optional[NamedSplit] = None, + stopping_strategy: Literal[ + "first_exhausted", "all_exhausted", "all_exhausted_without_replacement" + ] = "first_exhausted", +) -> IterableDataset: + """ + Interleave several iterable datasets (sources) into a single iterable dataset. + The new iterable dataset alternates between the sources to yield examples. + If `probabilities = None` (default) the iterable dataset will cycles through the sources in order for each next example in the iteration. + If `probabilities` is not `None, the iterable dataset will sample a random source according to the provided probabilities for each next examples in the iteration. + + + + Args: + datasets (`List[IterableDataset]`): list of datasets to interleave + probabilities (`List[float]`, optional, default None): If specified, the new iterable dataset samples + examples from one source at a time according to these probabilities. + seed (`int`, optional, default None): The random seed used to choose a source for each example. + stopping_strategy (`str`, defaults to `first_exhausted`): + Two strategies are proposed right now. + By default, `first_exhausted` is an undersampling strategy, i.e the dataset construction is stopped as soon as one dataset has ran out of samples. + If the strategy is `all_exhausted`, we use an oversampling strategy, i.e the dataset construction is stopped as soon as every samples of every dataset has been added at least once. + Note that if the strategy is `all_exhausted`, the interleaved dataset size can get enormous: + - with no probabilities, the resulting dataset will have max_length_datasets*nb_dataset samples. + - with given probabilities, the resulting dataset will have more samples if some datasets have really low probability of visiting. + + Output: + `datasets.IterableDataset` + """ + datasets = [d._resolve_features() for d in datasets] + + # Perform checks + _check_if_features_can_be_aligned([dset.features for dset in datasets]) + for i, dset in enumerate(datasets): + if datasets[0]._distributed != dset._distributed: + raise ValueError( + f"Datasets should be identically split_by_node before interleaving, but got {datasets[0]._distributed}!={dset._distributed} at index 0 and {i}" + ) + + # TODO: improve this to account for a mix of ClassLabel and Value for example + # right now it would keep the type of the first dataset in the list + features = Features( + {k: v for features in _align_features([dset.features for dset in datasets]) for k, v in features.items()} + ) + + ex_iterables = [copy.deepcopy(d._ex_iterable) for d in datasets] + if all(ex_iterable.iter_arrow for ex_iterable in ex_iterables): + ex_iterables = [RebatchedArrowExamplesIterable(ex_iterable, batch_size=1) for ex_iterable in ex_iterables] + # Use cycling or random cycling of sources + if probabilities is None: + ex_iterable = CyclingMultiSourcesExamplesIterable(ex_iterables, stopping_strategy=stopping_strategy) + else: + generator = np.random.default_rng(seed) + ex_iterable = RandomlyCyclingMultiSourcesExamplesIterable( + ex_iterables, + generator=generator, + probabilities=probabilities, + stopping_strategy=stopping_strategy, + ) + # Set new info - we update the features + # setting the features also ensures to fill missing columns with None + if info is None: + info = DatasetInfo.from_merge([d.info for d in datasets]) + else: + info = info.copy() + info.features = features + # Get all the auth tokens per repository - in case the datasets come from different private repositories + token_per_repo_id = { + repo_id: token for dataset in datasets for repo_id, token in dataset._token_per_repo_id.items() + } + # Return new daset + return IterableDataset( + ex_iterable=ex_iterable, + info=info, + split=split, + token_per_repo_id=token_per_repo_id, + distributed=datasets[0]._distributed, + ) + + +def _split_by_node_iterable_dataset(dataset: IterableDataset, rank: int, world_size: int) -> IterableDataset: + """ + Split an iterable dataset for the node at rank `rank` in a pool of nodes of size `world_size`. + + If the dataset has a number of shards that is a factor of `world_size` (i.e. if `dataset.num_shards % world_size == 0`), + then the shards are evenly assigned across the nodes, which is the most optimized. + Otherwise, each node keeps 1 example out of `world_size`, skipping the other examples. + + Args: + dataset ([`IterableDataset`]): + The iterable dataset to split by node. + rank (`int`): + Rank of the current node. + world_size (`int`): + Total number of nodes. + + Returns: + [`IterableDataset`]: The iterable dataset to be used on the node at rank `rank`. + """ + if dataset._distributed: + rank = world_size * dataset._distributed.rank + rank + world_size = world_size * dataset._distributed.world_size + distributed = DistributedConfig(rank=rank, world_size=world_size) + return IterableDataset( + ex_iterable=dataset._ex_iterable, + info=dataset._info.copy(), + split=dataset._split, + formatting=dataset._formatting, + distributed=distributed, + token_per_repo_id=dataset._token_per_repo_id, + ) + + +async def _apply_async(pool, func, x): + future = pool.apply_async(func, (x,)) + while True: + if future.ready(): + return future.get() + else: + await asyncio.sleep(0) + + +def _batch_fn(unbatched): + return {k: [v] for k, v in unbatched.items()} + + +def _generate_tables_from_polars(df: Union["pl.DataFrame", "pl.LazyFrame"]) -> Iterator[tuple["BuilderKey", pa.Table]]: + import polars as pl + + from .builder import Key as BuilderKey + + for slice_idx, df_slice in enumerate(df.collect_batches() if isinstance(df, pl.LazyFrame) else df.iter_slices()): + yield BuilderKey(0, slice_idx), df_slice.to_arrow() diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/load.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/load.py new file mode 100644 index 0000000000000000000000000000000000000000..560bcad3a44d4fe225d962e2b2d4e0ae88da586f --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/load.py @@ -0,0 +1,1772 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Access datasets.""" + +import glob +import importlib +import inspect +import json +import os +import posixpath +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal, Optional, Union, overload + +import fsspec +import httpx +import requests +import yaml +from fsspec.core import url_to_fs +from huggingface_hub import DatasetCard, DatasetCardData, HfApi, HfFileSystem +from huggingface_hub.utils import ( + EntryNotFoundError, + GatedRepoError, + LocalEntryNotFoundError, + OfflineModeIsEnabled, + RepositoryNotFoundError, + RevisionNotFoundError, + get_session, +) +from packaging import version + +from . import __version__, config +from .arrow_dataset import Dataset +from .builder import BuilderConfig, DatasetBuilder +from .data_files import ( + DataFilesDict, + DataFilesList, + DataFilesPatternsDict, + EmptyDatasetError, + get_data_patterns, + sanitize_patterns, +) +from .dataset_dict import DatasetDict, IterableDatasetDict +from .download.download_config import DownloadConfig +from .download.download_manager import DownloadMode +from .download.streaming_download_manager import StreamingDownloadManager, xbasename, xglob, xjoin +from .exceptions import DataFilesNotFoundError, DatasetNotFoundError +from .features import Features +from .features.features import _fix_for_backward_compatible_features +from .fingerprint import Hasher +from .info import DatasetInfo, DatasetInfosDict +from .iterable_dataset import IterableDataset +from .naming import camelcase_to_snakecase, snakecase_to_camelcase +from .packaged_modules import ( + _ALL_ALLOWED_EXTENSIONS, + _EXTENSION_TO_MODULE, + _MODULE_TO_EXTENSIONS, + _MODULE_TO_METADATA_EXTENSIONS, + _MODULE_TO_METADATA_FILE_NAMES, + _PACKAGED_DATASETS_MODULES, +) +from .packaged_modules.folder_based_builder.folder_based_builder import FolderBasedBuilder +from .splits import Split +from .utils import _dataset_viewer +from .utils.file_utils import ( + _raise_if_offline_mode_is_enabled, + cached_path, + get_datasets_user_agent, + is_relative_path, + relative_to_absolute_path, +) +from .utils.hub import hf_dataset_url +from .utils.info_utils import VerificationMode, is_small_dataset +from .utils.logging import get_logger +from .utils.metadata import MetadataConfigs +from .utils.typing import PathLike +from .utils.version import Version + + +if config.HF_HUB_VERSION >= version.parse("1.6.0"): + from huggingface_hub.errors import BucketNotFoundError + +else: + BucketNotFoundError = None + + +logger = get_logger(__name__) + + +class _InitializeConfiguredDatasetBuilder: + """ + From https://stackoverflow.com/questions/4647566/pickle-a-dynamically-parameterized-sub-class + See also ConfiguredDatasetBuilder.__reduce__ + When called with the param value as the only argument, returns an + un-initialized instance of the parameterized class. Subsequent __setstate__ + will be called by pickle. + """ + + def __call__(self, builder_cls, metadata_configs, default_config_name, name): + # make a simple object which has no complex __init__ (this one will do) + obj = _InitializeConfiguredDatasetBuilder() + obj.__class__ = configure_builder_class( + builder_cls, metadata_configs, default_config_name=default_config_name, dataset_name=name + ) + return obj + + +def configure_builder_class( + builder_cls: type[DatasetBuilder], + builder_configs: list[BuilderConfig], + default_config_name: Optional[str], + dataset_name: str, +) -> type[DatasetBuilder]: + """ + Dynamically create a builder class with custom builder configs parsed from README.md file, + i.e. set BUILDER_CONFIGS class variable of a builder class to custom configs list. + """ + + class ConfiguredDatasetBuilder(builder_cls): + BUILDER_CONFIGS = builder_configs + DEFAULT_CONFIG_NAME = default_config_name + + __module__ = builder_cls.__module__ # so that the actual packaged builder can be imported + + def __reduce__(self): # to make dynamically created class pickable, see _InitializeParameterizedDatasetBuilder + parent_builder_cls = self.__class__.__mro__[1] + return ( + _InitializeConfiguredDatasetBuilder(), + ( + parent_builder_cls, + self.BUILDER_CONFIGS, + self.DEFAULT_CONFIG_NAME, + self.dataset_name, + ), + self.__dict__.copy(), + ) + + ConfiguredDatasetBuilder.__name__ = ( + f"{builder_cls.__name__.lower().capitalize()}{snakecase_to_camelcase(dataset_name)}" + ) + ConfiguredDatasetBuilder.__qualname__ = ( + f"{builder_cls.__name__.lower().capitalize()}{snakecase_to_camelcase(dataset_name)}" + ) + + return ConfiguredDatasetBuilder + + +def import_main_class(module_path) -> Optional[type[DatasetBuilder]]: + """Import a module at module_path and return its main class: a DatasetBuilder""" + module = importlib.import_module(module_path) + # Find the main class in our imported module + module_main_cls = None + for name, obj in module.__dict__.items(): + if inspect.isclass(obj) and issubclass(obj, DatasetBuilder): + if inspect.isabstract(obj): + continue + module_main_cls = obj + obj_module = inspect.getmodule(obj) + if obj_module is not None and module == obj_module: + break + + return module_main_cls + + +def get_dataset_builder_class( + dataset_module: "DatasetModule", dataset_name: Optional[str] = None +) -> type[DatasetBuilder]: + builder_cls = import_main_class(dataset_module.module_path) + if dataset_module.builder_configs_parameters.builder_configs: + dataset_name = dataset_name or dataset_module.builder_kwargs.get("dataset_name") + if dataset_name is None: + raise ValueError("dataset_name should be specified but got None") + builder_cls = configure_builder_class( + builder_cls, + builder_configs=dataset_module.builder_configs_parameters.builder_configs, + default_config_name=dataset_module.builder_configs_parameters.default_config_name, + dataset_name=dataset_name, + ) + return builder_cls + + +def increase_load_count(name: str): + """Update the download count of a dataset.""" + if not config.HF_HUB_OFFLINE and config.HF_UPDATE_DOWNLOAD_COUNTS: + try: + get_session().head( + "/".join((config.S3_DATASETS_BUCKET_PREFIX, name, name + ".py")), + headers={"User-Agent": get_datasets_user_agent()}, + timeout=3, + ) + except Exception: + pass + + +def infer_module_for_data_files_list( + data_files_list: DataFilesList, download_config: Optional[DownloadConfig] = None +) -> tuple[Optional[str], dict]: + """Infer module (and builder kwargs) from list of data files. + + It picks the module based on the most common file extension. + In case of a draw ".parquet" is the favorite, and then alphabetical order. + + Args: + data_files_list (DataFilesList): List of data files. + download_config (bool or str, optional): Mainly use `token` or `storage_options` to support different platforms and auth types. + + Returns: + tuple[str, dict[str, Any]]: Tuple with + - inferred module name + - dict of builder kwargs + """ + extensions_counter = Counter( + ("." + suffix.lower(), xbasename(filepath) in FolderBasedBuilder.METADATA_FILENAMES) + for filepath in data_files_list + for suffix in xbasename(filepath).split(".")[1:] + ) + if extensions_counter: + + def sort_key(ext_count: tuple[tuple[str, bool], int]) -> tuple[int, bool]: + """Sort by count and set ".parquet" as the favorite in case of a draw, and ignore metadata files""" + (ext, is_metadata), count = ext_count + return (not is_metadata, count, ext == ".parquet", ext == ".jsonl", ext == ".json", ext == ".csv", ext) + + for (ext, _), _ in sorted(extensions_counter.items(), key=sort_key, reverse=True): + if ext in _EXTENSION_TO_MODULE: + return _EXTENSION_TO_MODULE[ext] + elif ext == ".zip": + return infer_module_for_data_files_list_in_archives(data_files_list, download_config=download_config) + return None, {} + + +def infer_module_for_data_files_list_in_archives( + data_files_list: DataFilesList, download_config: Optional[DownloadConfig] = None +) -> tuple[Optional[str], dict]: + """Infer module (and builder kwargs) from list of archive data files. + + Args: + data_files_list (DataFilesList): List of data files. + download_config (bool or str, optional): Mainly use `token` or `storage_options` to support different platforms and auth types. + + Returns: + tuple[str, dict[str, Any]]: Tuple with + - inferred module name + - dict of builder kwargs + """ + archived_files = [] + archive_files_counter = 0 + for filepath in data_files_list: + if str(filepath).endswith(".zip"): + archive_files_counter += 1 + if archive_files_counter > config.ARCHIVES_MAX_NUMBER_FOR_MODULE_INFERENCE: + break + extracted = xjoin(StreamingDownloadManager().extract(filepath), "**") + archived_files += [ + f.split("::")[0] for f in xglob(extracted, recursive=True, download_config=download_config) + ] + extensions_counter = Counter( + "." + suffix.lower() for filepath in archived_files for suffix in xbasename(filepath).split(".")[1:] + ) + if extensions_counter: + most_common = extensions_counter.most_common(1)[0][0] + if most_common in _EXTENSION_TO_MODULE: + return _EXTENSION_TO_MODULE[most_common] + return None, {} + + +def infer_module_for_data_files( + data_files: DataFilesDict, path: Optional[str] = None, download_config: Optional[DownloadConfig] = None +) -> tuple[Optional[str], dict[str, Any]]: + """Infer module (and builder kwargs) from data files. Raise if module names for different splits don't match. + + Args: + data_files ([`DataFilesDict`]): Dict of list of data files. + path (str, *optional*): Dataset name or path. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters to authenticate on the Hugging Face Hub for private remote files. + + Returns: + tuple[str, dict[str, Any]]: Tuple with + - inferred module name + - builder kwargs + """ + split_modules = { + split: infer_module_for_data_files_list(data_files_list, download_config=download_config) + for split, data_files_list in data_files.items() + } + module_name, default_builder_kwargs = next(iter(split_modules.values())) + if any((module_name, default_builder_kwargs) != split_module for split_module in split_modules.values()): + raise ValueError(f"Couldn't infer the same data file format for all splits. Got {split_modules}") + if not module_name: + raise DataFilesNotFoundError("No (supported) data files found" + (f" in {path}" if path else "")) + return module_name, default_builder_kwargs + + +def create_builder_configs_from_metadata_configs( + module_path: str, + metadata_configs: MetadataConfigs, + base_path: Optional[str] = None, + default_builder_kwargs: dict[str, Any] = None, + download_config: Optional[DownloadConfig] = None, +) -> tuple[list[BuilderConfig], str]: + builder_cls = import_main_class(module_path) + builder_config_cls = builder_cls.BUILDER_CONFIG_CLASS + default_config_name = metadata_configs.get_default_config_name() + builder_configs = [] + default_builder_kwargs = {} if default_builder_kwargs is None else default_builder_kwargs + + base_path = base_path if base_path is not None else "" + for config_name, config_params in metadata_configs.items(): + config_data_files = config_params.get("data_files") + config_data_dir = config_params.get("data_dir") + config_base_path = xjoin(base_path, config_data_dir) if config_data_dir else base_path + try: + config_patterns = ( + sanitize_patterns(config_data_files) + if config_data_files is not None + else get_data_patterns(config_base_path, download_config=download_config) + ) + config_data_files_dict = DataFilesPatternsDict.from_patterns( + config_patterns, + allowed_extensions=_ALL_ALLOWED_EXTENSIONS, + ) + except EmptyDatasetError as e: + raise EmptyDatasetError( + f"Dataset at '{base_path}' doesn't contain data files matching the patterns for config '{config_name}'," + f" check `data_files` and `data_fir` parameters in the `configs` YAML field in README.md. " + ) from e + ignored_params = [ + param for param in config_params if not hasattr(builder_config_cls, param) and param != "default" + ] + if ignored_params: + logger.warning( + f"Some datasets params were ignored: {ignored_params}. " + "Make sure to use only valid params for the dataset builder and to have " + "a up-to-date version of the `datasets` library." + ) + builder_configs.append( + builder_config_cls( + name=config_name, + data_files=config_data_files_dict, + data_dir=config_data_dir, + **{ + param: value + for param, value in {**default_builder_kwargs, **config_params}.items() + if hasattr(builder_config_cls, param) and param not in ("default", "data_files", "data_dir") + }, + ) + ) + return builder_configs, default_config_name + + +@dataclass +class BuilderConfigsParameters: + """Dataclass containing objects related to creation of builder configurations from yaml's metadata content. + + Attributes: + metadata_configs (`MetadataConfigs`, *optional*): + Configs parsed from yaml's metadata. + builder_configs (`list[BuilderConfig]`, *optional*): + List of BuilderConfig objects created from metadata_configs above. + default_config_name (`str`): + Name of default config taken from yaml's metadata. + """ + + metadata_configs: Optional[MetadataConfigs] = None + builder_configs: Optional[list[BuilderConfig]] = None + default_config_name: Optional[str] = None + + +@dataclass +class DatasetModule: + module_path: str + hash: str + builder_kwargs: dict + builder_configs_parameters: BuilderConfigsParameters = field(default_factory=BuilderConfigsParameters) + dataset_infos: Optional[DatasetInfosDict] = None + + +class _DatasetModuleFactory: + def get_module(self) -> DatasetModule: + raise NotImplementedError + + +class LocalDatasetModuleFactory(_DatasetModuleFactory): + """Get the module of a dataset loaded from the user's data files. The dataset builder module to use is inferred + from the data files extensions.""" + + def __init__( + self, + path: str, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, list, dict]] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + ): + if data_dir and os.path.isabs(data_dir): + raise ValueError(f"`data_dir` must be relative to a dataset directory's root: {path}") + + self.path = Path(path).as_posix() + self.name = Path(path).stem + self.data_files = data_files + self.data_dir = data_dir + self.download_mode = download_mode + + def get_module(self) -> DatasetModule: + readme_path = os.path.join(self.path, config.REPOCARD_FILENAME) + standalone_yaml_path = os.path.join(self.path, config.REPOYAML_FILENAME) + dataset_card_data = DatasetCard.load(readme_path).data if os.path.isfile(readme_path) else DatasetCardData() + if os.path.exists(standalone_yaml_path): + with open(standalone_yaml_path, encoding="utf-8") as f: + standalone_yaml_data = yaml.safe_load(f.read()) + if standalone_yaml_data: + _dataset_card_data_dict = dataset_card_data.to_dict() + _dataset_card_data_dict.update(standalone_yaml_data) + dataset_card_data = DatasetCardData(**_dataset_card_data_dict) + metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data) + dataset_infos = DatasetInfosDict.from_dataset_card_data(dataset_card_data) + # we need a set of data files to find which dataset builder to use + # because we need to infer module name by files extensions + base_path = Path(self.path, self.data_dir or "").expanduser().resolve().as_posix() + if self.data_files is not None: + patterns = sanitize_patterns(self.data_files) + elif metadata_configs and not self.data_dir and "data_files" in next(iter(metadata_configs.values())): + patterns = sanitize_patterns(next(iter(metadata_configs.values()))["data_files"]) + else: + patterns = get_data_patterns(base_path) + data_files = DataFilesDict.from_patterns( + patterns, + base_path=base_path, + allowed_extensions=_ALL_ALLOWED_EXTENSIONS, + ) + module_name, default_builder_kwargs = infer_module_for_data_files( + data_files=data_files, + path=self.path, + ) + data_files = data_files.filter( + extensions=_MODULE_TO_EXTENSIONS[module_name] + _MODULE_TO_METADATA_EXTENSIONS[module_name], + file_names=_MODULE_TO_METADATA_FILE_NAMES[module_name], + ) + module_path, _ = _PACKAGED_DATASETS_MODULES[module_name] + if metadata_configs: + builder_configs, default_config_name = create_builder_configs_from_metadata_configs( + module_path, + metadata_configs, + base_path=base_path, + default_builder_kwargs=default_builder_kwargs, + ) + else: + builder_configs: list[BuilderConfig] = [ + import_main_class(module_path).BUILDER_CONFIG_CLASS( + data_files=data_files, + **default_builder_kwargs, + ) + ] + default_config_name = None + builder_kwargs = { + "base_path": self.path, + "dataset_name": camelcase_to_snakecase(Path(self.path).name), + } + if self.data_dir: + builder_kwargs["data_files"] = data_files + # this file is deprecated and was created automatically in old versions of push_to_hub + if os.path.isfile(os.path.join(self.path, config.DATASETDICT_INFOS_FILENAME)): + with open(os.path.join(self.path, config.DATASETDICT_INFOS_FILENAME), encoding="utf-8") as f: + legacy_dataset_infos = DatasetInfosDict( + { + config_name: DatasetInfo.from_dict(dataset_info_dict) + for config_name, dataset_info_dict in json.load(f).items() + } + ) + if len(legacy_dataset_infos) == 1: + # old config e.g. named "username--dataset_name" + legacy_config_name = next(iter(legacy_dataset_infos)) + legacy_dataset_infos["default"] = legacy_dataset_infos.pop(legacy_config_name) + legacy_dataset_infos.update(dataset_infos) + dataset_infos = legacy_dataset_infos + if default_config_name is None and len(dataset_infos) == 1: + default_config_name = next(iter(dataset_infos)) + + hash = Hasher.hash({"dataset_infos": dataset_infos, "builder_configs": builder_configs}) + return DatasetModule( + module_path, + hash, + builder_kwargs, + dataset_infos=dataset_infos, + builder_configs_parameters=BuilderConfigsParameters( + metadata_configs=metadata_configs, + builder_configs=builder_configs, + default_config_name=default_config_name, + ), + ) + + +class PackagedDatasetModuleFactory(_DatasetModuleFactory): + """Get the dataset builder module from the ones that are packaged with the library: csv, json, etc.""" + + def __init__( + self, + name: str, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, list, dict]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + ): + self.name = name + self.data_files = data_files + self.data_dir = data_dir + self.download_config = download_config + self.download_mode = download_mode + increase_load_count(name) + + def get_module(self) -> DatasetModule: + base_path = Path(self.data_dir or "").expanduser().resolve().as_posix() + patterns = ( + sanitize_patterns(self.data_files) + if self.data_files is not None + else get_data_patterns(base_path, download_config=self.download_config) + ) + data_files = DataFilesDict.from_patterns( + patterns, + download_config=self.download_config, + base_path=base_path, + ) + + module_path, hash = _PACKAGED_DATASETS_MODULES[self.name] + + builder_kwargs = { + "data_files": data_files, + "dataset_name": self.name, + } + + return DatasetModule(module_path, hash, builder_kwargs) + + +class HubDatasetModuleFactory(_DatasetModuleFactory): + """ + Get the module of a dataset loaded from data files of a dataset repository. + The dataset builder module to use is inferred from the data files extensions. + """ + + def __init__( + self, + name: str, + commit_hash: str, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, list, dict]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + use_exported_dataset_infos: bool = False, + ): + self.name = name + self.commit_hash = commit_hash + self.data_files = data_files + self.data_dir = data_dir + self.download_config = download_config or DownloadConfig() + self.download_mode = download_mode + self.use_exported_dataset_infos = use_exported_dataset_infos + increase_load_count(name) + + def get_module(self) -> DatasetModule: + # Get the Dataset Card and fix the revision in case there are new commits in the meantime + api = HfApi( + endpoint=config.HF_ENDPOINT, + token=self.download_config.token, + library_name="datasets", + library_version=__version__, + user_agent=get_datasets_user_agent(self.download_config.user_agent), + ) + try: + dataset_readme_path = api.hf_hub_download( + repo_id=self.name, + filename=config.REPOCARD_FILENAME, + repo_type="dataset", + revision=self.commit_hash, + proxies=self.download_config.proxies, + ) + dataset_card_data = DatasetCard.load(dataset_readme_path).data + except EntryNotFoundError: + dataset_card_data = DatasetCardData() + download_config = self.download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading standalone yaml" + try: + standalone_yaml_path = cached_path( + hf_dataset_url(self.name, config.REPOYAML_FILENAME, revision=self.commit_hash), + download_config=download_config, + ) + with open(standalone_yaml_path, encoding="utf-8") as f: + standalone_yaml_data = yaml.safe_load(f.read()) + if standalone_yaml_data: + _dataset_card_data_dict = dataset_card_data.to_dict() + _dataset_card_data_dict.update(standalone_yaml_data) + dataset_card_data = DatasetCardData(**_dataset_card_data_dict) + except FileNotFoundError: + pass + base_path = f"hf://datasets/{self.name}@{self.commit_hash}/{self.data_dir or ''}".rstrip("/") + metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data) + dataset_infos = DatasetInfosDict.from_dataset_card_data(dataset_card_data) + if config.USE_PARQUET_EXPORT and self.use_exported_dataset_infos: + try: + exported_dataset_infos = _dataset_viewer.get_exported_dataset_infos( + dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token + ) + exported_dataset_infos = DatasetInfosDict( + { + config_name: DatasetInfo.from_dict(exported_dataset_infos[config_name]) + for config_name in exported_dataset_infos + } + ) + except _dataset_viewer.DatasetViewerError: + exported_dataset_infos = None + else: + exported_dataset_infos = None + if exported_dataset_infos: + exported_dataset_infos.update(dataset_infos) + dataset_infos = exported_dataset_infos + # we need a set of data files to find which dataset builder to use + # because we need to infer module name by files extensions + if self.data_files is not None: + patterns = sanitize_patterns(self.data_files) + elif metadata_configs and not self.data_dir and "data_files" in next(iter(metadata_configs.values())): + patterns = sanitize_patterns(next(iter(metadata_configs.values()))["data_files"]) + else: + patterns = get_data_patterns(base_path, download_config=self.download_config) + data_files = DataFilesDict.from_patterns( + patterns, + base_path=base_path, + allowed_extensions=_ALL_ALLOWED_EXTENSIONS, + download_config=self.download_config, + ) + module_name, default_builder_kwargs = infer_module_for_data_files( + data_files=data_files, + path=self.name, + download_config=self.download_config, + ) + data_files = data_files.filter( + extensions=_MODULE_TO_EXTENSIONS[module_name] + _MODULE_TO_METADATA_EXTENSIONS[module_name], + file_names=_MODULE_TO_METADATA_FILE_NAMES[module_name], + ) + module_path, _ = _PACKAGED_DATASETS_MODULES[module_name] + if metadata_configs: + builder_configs, default_config_name = create_builder_configs_from_metadata_configs( + module_path, + metadata_configs, + base_path=base_path, + default_builder_kwargs=default_builder_kwargs, + download_config=self.download_config, + ) + else: + builder_configs: list[BuilderConfig] = [ + import_main_class(module_path).BUILDER_CONFIG_CLASS( + data_files=data_files, + **default_builder_kwargs, + ) + ] + default_config_name = None + builder_kwargs = { + "base_path": base_path, + "repo_id": self.name, + "dataset_name": camelcase_to_snakecase(Path(self.name).name), + } + if self.data_dir: + builder_kwargs["data_files"] = data_files + download_config = self.download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading metadata" + try: + # this file is deprecated and was created automatically in old versions of push_to_hub + dataset_infos_path = cached_path( + hf_dataset_url(self.name, config.DATASETDICT_INFOS_FILENAME, revision=self.commit_hash), + download_config=download_config, + ) + with open(dataset_infos_path, encoding="utf-8") as f: + legacy_dataset_infos = DatasetInfosDict( + { + config_name: DatasetInfo.from_dict(dataset_info_dict) + for config_name, dataset_info_dict in json.load(f).items() + } + ) + if len(legacy_dataset_infos) == 1: + # old config e.g. named "username--dataset_name" + legacy_config_name = next(iter(legacy_dataset_infos)) + legacy_dataset_infos["default"] = legacy_dataset_infos.pop(legacy_config_name) + legacy_dataset_infos.update(dataset_infos) + dataset_infos = legacy_dataset_infos + except FileNotFoundError: + pass + if default_config_name is None and len(dataset_infos) == 1: + default_config_name = next(iter(dataset_infos)) + + return DatasetModule( + module_path, + self.commit_hash, + builder_kwargs, + dataset_infos=dataset_infos, + builder_configs_parameters=BuilderConfigsParameters( + metadata_configs=metadata_configs, + builder_configs=builder_configs, + default_config_name=default_config_name, + ), + ) + + +class HubDatasetModuleFactoryWithParquetExport(_DatasetModuleFactory): + """ + Get the module of a dataset loaded from parquet files of a dataset repository parquet export. + """ + + def __init__( + self, + name: str, + commit_hash: str, + download_config: Optional[DownloadConfig] = None, + ): + self.name = name + self.commit_hash = commit_hash + self.download_config = download_config or DownloadConfig() + increase_load_count(name) + + def get_module(self) -> DatasetModule: + exported_parquet_files = _dataset_viewer.get_exported_parquet_files( + dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token + ) + exported_dataset_infos = _dataset_viewer.get_exported_dataset_infos( + dataset=self.name, commit_hash=self.commit_hash, token=self.download_config.token + ) + dataset_infos = DatasetInfosDict( + { + config_name: DatasetInfo.from_dict(exported_dataset_infos[config_name]) + for config_name in exported_dataset_infos + } + ) + parquet_commit_hash = ( + HfApi( + endpoint=config.HF_ENDPOINT, + token=self.download_config.token, + library_name="datasets", + library_version=__version__, + user_agent=get_datasets_user_agent(self.download_config.user_agent), + ) + .dataset_info( + self.name, + revision="refs/convert/parquet", + token=self.download_config.token, + timeout=100.0, + ) + .sha + ) # fix the revision in case there are new commits in the meantime + metadata_configs = MetadataConfigs._from_exported_parquet_files_and_dataset_infos( + parquet_commit_hash=parquet_commit_hash, + exported_parquet_files=exported_parquet_files, + dataset_infos=dataset_infos, + ) + module_path, _ = _PACKAGED_DATASETS_MODULES["parquet"] + builder_configs, default_config_name = create_builder_configs_from_metadata_configs( + module_path, + metadata_configs, + download_config=self.download_config, + ) + builder_kwargs = { + "repo_id": self.name, + "dataset_name": camelcase_to_snakecase(Path(self.name).name), + } + + return DatasetModule( + module_path, + self.commit_hash, + builder_kwargs, + dataset_infos=dataset_infos, + builder_configs_parameters=BuilderConfigsParameters( + metadata_configs=metadata_configs, + builder_configs=builder_configs, + default_config_name=default_config_name, + ), + ) + + +class CachedDatasetModuleFactory(_DatasetModuleFactory): + """ + Get the module of a dataset that has been loaded once already and cached. + """ + + def __init__( + self, + name: str, + cache_dir: Optional[str] = None, + ): + self.name = name + self.cache_dir = cache_dir + assert self.name.count("/") <= 1 + + def get_module(self) -> DatasetModule: + cache_dir = os.path.expanduser(str(self.cache_dir or config.HF_DATASETS_CACHE)) + namespace_and_dataset_name = self.name.split("/") + namespace_and_dataset_name[-1] = camelcase_to_snakecase(namespace_and_dataset_name[-1]) + cached_relative_path = "___".join(namespace_and_dataset_name) + cached_datasets_directory_path_root = os.path.join(cache_dir, cached_relative_path) + cached_directory_paths = [ + cached_directory_path + for cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", "*", "*")) + if os.path.isdir(cached_directory_path) + ] + if cached_directory_paths: + builder_kwargs = { + "repo_id": self.name, + "dataset_name": self.name.split("/")[-1], + } + warning_msg = f"Using the latest cached version of the dataset since {self.name} couldn't be found on the Hugging Face Hub" + if config.HF_HUB_OFFLINE: + warning_msg += " (offline mode is enabled)." + logger.warning(warning_msg) + return DatasetModule( + "datasets.packaged_modules.cache.cache", + "auto", + {**builder_kwargs, "version": "auto"}, + ) + raise FileNotFoundError(f"Dataset {self.name} is not cached in {self.cache_dir}") + + +class HubBucketDatasetModuleFactory(_DatasetModuleFactory): + """ + Get the module of a dataset loaded from data files of a a Storage Bucket. + The dataset builder module to use is inferred from the data files extensions. + """ + + def __init__( + self, + path: str, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, list, dict]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + ): + self.path = Path(path).as_posix() + self.name = Path(path).stem + self.data_files = data_files + self.data_dir = data_dir + self.download_config = download_config + self.download_mode = download_mode + + def get_module(self) -> DatasetModule: + hffs = HfFileSystem( + endpoint=config.HF_ENDPOINT, + token=self.download_config.token, + ) + readme_path = xjoin(self.path, config.REPOCARD_FILENAME) + standalone_yaml_path = xjoin(self.path, config.REPOYAML_FILENAME) + try: + dataset_card_data = DatasetCard(hffs.read_text(readme_path, newline="", encoding="utf-8")) + except FileNotFoundError: + dataset_card_data = DatasetCardData() + try: + standalone_yaml_data = yaml.safe_load(hffs.read_text(standalone_yaml_path, newline="", encoding="utf-8")) + except FileNotFoundError: + dataset_card_data = DatasetCardData() + if hffs.exists(standalone_yaml_path): + with hffs.open(standalone_yaml_path, "r", encoding="utf-8") as f: + standalone_yaml_data = yaml.safe_load(f.read()) + if standalone_yaml_data: + _dataset_card_data_dict = dataset_card_data.to_dict() + _dataset_card_data_dict.update(standalone_yaml_data) + dataset_card_data = DatasetCardData(**_dataset_card_data_dict) + metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data) + dataset_infos = DatasetInfosDict.from_dataset_card_data(dataset_card_data) + # we need a set of data files to find which dataset builder to use + # because we need to infer module name by files extensions + base_path = "hf://" + Path(self.path, self.data_dir or "").as_posix() + if self.data_files is not None: + patterns = sanitize_patterns(self.data_files) + elif metadata_configs and not self.data_dir and "data_files" in next(iter(metadata_configs.values())): + patterns = sanitize_patterns(next(iter(metadata_configs.values()))["data_files"]) + else: + patterns = get_data_patterns(base_path, download_config=self.download_config) + data_files = DataFilesDict.from_patterns( + patterns, + base_path=base_path, + allowed_extensions=_ALL_ALLOWED_EXTENSIONS, + ) + module_name, default_builder_kwargs = infer_module_for_data_files( + data_files=data_files, + path=self.path, + ) + data_files = data_files.filter( + extensions=_MODULE_TO_EXTENSIONS[module_name] + _MODULE_TO_METADATA_EXTENSIONS[module_name], + file_names=_MODULE_TO_METADATA_FILE_NAMES[module_name], + ) + module_path, _ = _PACKAGED_DATASETS_MODULES[module_name] + if metadata_configs: + builder_configs, default_config_name = create_builder_configs_from_metadata_configs( + module_path, + metadata_configs, + base_path=base_path, + default_builder_kwargs=default_builder_kwargs, + ) + else: + builder_configs: list[BuilderConfig] = [ + import_main_class(module_path).BUILDER_CONFIG_CLASS( + data_files=data_files, + **default_builder_kwargs, + ) + ] + default_config_name = None + builder_kwargs = { + "base_path": base_path, + "dataset_name": camelcase_to_snakecase(Path(self.path).name), + } + if self.data_dir: + builder_kwargs["data_files"] = data_files + # this file is deprecated and was created automatically in old versions of push_to_hub + if hffs.isfile(xjoin(self.path, config.DATASETDICT_INFOS_FILENAME)): + with hffs.open(xjoin(self.path, config.DATASETDICT_INFOS_FILENAME), "r", encoding="utf-8") as f: + legacy_dataset_infos = DatasetInfosDict( + { + config_name: DatasetInfo.from_dict(dataset_info_dict) + for config_name, dataset_info_dict in json.load(f).items() + } + ) + if len(legacy_dataset_infos) == 1: + # old config e.g. named "username--dataset_name" + legacy_config_name = next(iter(legacy_dataset_infos)) + legacy_dataset_infos["default"] = legacy_dataset_infos.pop(legacy_config_name) + legacy_dataset_infos.update(dataset_infos) + dataset_infos = legacy_dataset_infos + if default_config_name is None and len(dataset_infos) == 1: + default_config_name = next(iter(dataset_infos)) + + hash = Hasher.hash({"dataset_infos": dataset_infos, "builder_configs": builder_configs}) + return DatasetModule( + module_path, + hash, + builder_kwargs, + dataset_infos=dataset_infos, + builder_configs_parameters=BuilderConfigsParameters( + metadata_configs=metadata_configs, + builder_configs=builder_configs, + default_config_name=default_config_name, + ), + ) + + +def dataset_module_factory( + path: str, + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + data_dir: Optional[str] = None, + data_files: Optional[Union[dict, list, str, DataFilesDict]] = None, + cache_dir: Optional[str] = None, + **download_kwargs, +) -> DatasetModule: + """ + Download/extract/cache a dataset module. + + Dataset codes are cached inside the dynamic modules cache to allow easy import (avoid ugly sys.path tweaks). + + Args: + + path (str): Path or name of the dataset. + Depending on ``path``, the dataset builder that is used comes from one of the generic dataset builders (JSON, CSV, Parquet, text etc.). + + For local datasets: + + - if ``path`` is a local directory (containing data files only) + -> load a generic dataset builder (csv, json, text etc.) based on the content of the directory + e.g. ``'./path/to/directory/with/my/csv/data'``. + + For datasets on the Hugging Face Hub (list all available datasets with ``huggingface_hub.list_datasets()``) + + - if ``path`` is a dataset repository on the HF hub (containing data files only) + -> load a generic dataset builder (csv, text etc.) based on the content of the repository + e.g. ``'username/dataset_name'``, a dataset repository on the HF hub containing your data files. + + For datasets in Storage Buckets on the Hugging Face Hub (list all available buckets with ``huggingface_hub.list_buckets()``) + + - if `path` is a directory within a Storage Bucket on the HF Hub (containing data files only) + -> load the dataset from supported files in the directory (csv, json, parquet, etc.) + e.g. `'buckets/username/bucket_name/my_dataset'`. + + revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset to load. + As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. + You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. + download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters. + download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. + data_dir (:obj:`str`, optional): Directory with the data files. Used only if `data_files` is not specified, + in which case it's equal to pass `os.path.join(data_dir, "**")` as `data_files`. + data_files (:obj:`Union[Dict, List, str]`, optional): Defining the data_files of the dataset configuration. + cache_dir (`str`, *optional*): + Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`. + + + + **download_kwargs (additional keyword arguments): optional attributes for DownloadConfig() which will override + the attributes in download_config if supplied. + + Returns: + DatasetModule + """ + if download_config is None: + download_config = DownloadConfig(**download_kwargs) + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + download_config.extract_compressed_file = True + download_config.force_extract = True + download_config.force_download = download_mode == DownloadMode.FORCE_REDOWNLOAD + + filename = list(filter(lambda x: x, path.replace(os.sep, "/").split("/")))[-1] + if not filename.endswith(".py"): + filename = filename + ".py" + combined_path = os.path.join(path, filename) + + # We have several ways to get a dataset builder: + # + # - if path is the name of a packaged dataset module + # -> use the packaged module (json, csv, etc.) + # + # - if path is a local directory (but no python file) + # -> use a packaged module (csv, text etc.) based on content of the directory + # + # - if path has one "/" and is dataset repository on the HF hub + # -> use a packaged module (csv, text etc.) based on content of the repository + # + # - if path starts with "buckets/" and points to a Storage Bucket on the HF hub + # -> use a packaged module (csv, text etc.) based on content of the directory in the bucket + + if path.startswith("hf://datasets/"): + path = path[len("hf://datasets/") :] + remote_only = True + elif path.startswith("hf://buckets/"): + path = path[len("hf://") :] + remote_only = True + else: + remote_only = False + + # Try packaged + if path in _PACKAGED_DATASETS_MODULES: + return PackagedDatasetModuleFactory( + path, + data_dir=data_dir, + data_files=data_files, + download_config=download_config, + download_mode=download_mode, + ).get_module() + # Try locally + elif path.endswith(filename): + raise RuntimeError(f"Dataset scripts are no longer supported, but found {filename}") + elif os.path.isfile(combined_path): + raise RuntimeError(f"Dataset scripts are no longer supported, but found {filename}") + elif os.path.isdir(path) and not remote_only: + return LocalDatasetModuleFactory( + path, data_dir=data_dir, data_files=data_files, download_mode=download_mode + ).get_module() + # Try remotely + elif path.startswith("buckets/"): + if BucketNotFoundError is None: + raise ImportError("Loading datasets from buckets requires huggingface_hub>=1.6.0") + # We check that the bucket exists, and the directory exists, and authentication in one call + api = HfApi( + endpoint=config.HF_ENDPOINT, + token=download_config.token, + library_name="datasets", + library_version=__version__, + user_agent=get_datasets_user_agent(download_config.user_agent), + ) + _, _namespace, _bucket_name, *_path_segments = path.split("/") + bucket_id = _namespace + "/" + _bucket_name + prefix = "/".join(s for s in _path_segments if s) + try: + next(iter(api.list_bucket_tree(bucket_id, prefix))) + except ( + OfflineModeIsEnabled, + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + httpx.ConnectError, + httpx.TimeoutException, + ) as e: + raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e + except StopIteration as e: + raise DatasetNotFoundError(f"Bucket directory at {path} doesn't exist") from e + except BucketNotFoundError as e: + raise DatasetNotFoundError(f"Bucket '{bucket_id}' doesn't exist on the Hub or cannot be accessed.") from e + return HubBucketDatasetModuleFactory( + path, + data_dir=data_dir, + data_files=data_files, + download_config=download_config, + download_mode=download_mode, + ).get_module() + elif is_relative_path(path) and path.count("/") <= 1: + try: + # Get the Dataset Card + get the revision + check authentication all at in one call + # We fix the commit_hash in case there are new commits in the meantime + api = HfApi( + endpoint=config.HF_ENDPOINT, + token=download_config.token, + library_name="datasets", + library_version=__version__, + user_agent=get_datasets_user_agent(download_config.user_agent), + ) + try: + _raise_if_offline_mode_is_enabled() + dataset_readme_path = api.hf_hub_download( + repo_id=path, + filename=config.REPOCARD_FILENAME, + repo_type="dataset", + revision=revision, + proxies=download_config.proxies, + ) + commit_hash = os.path.basename(os.path.dirname(dataset_readme_path)) + except LocalEntryNotFoundError as e: + if isinstance( + e.__cause__, + ( + OfflineModeIsEnabled, + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + httpx.ConnectError, + httpx.TimeoutException, + ), + ): + raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e + else: + raise + except EntryNotFoundError: + commit_hash = api.dataset_info( + path, + revision=revision, + timeout=100.0, + ).sha + except ( + OfflineModeIsEnabled, + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + httpx.ConnectError, + httpx.TimeoutException, + ) as e: + raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e + except GatedRepoError as e: + message = f"Dataset '{path}' is a gated dataset on the Hub." + if e.response.status_code == 401: + message += " You must be authenticated to access it." + elif e.response.status_code == 403: + message += f" Visit the dataset page at https://huggingface.co/datasets/{path} to ask for access." + raise DatasetNotFoundError(message) from e + except RepositoryNotFoundError as e: + raise DatasetNotFoundError(f"Dataset '{path}' doesn't exist on the Hub or cannot be accessed.") from e + try: + api.hf_hub_download( + repo_id=path, + filename=filename, + repo_type="dataset", + revision=commit_hash, + proxies=download_config.proxies, + ) + raise RuntimeError(f"Dataset scripts are no longer supported, but found {filename}") + except EntryNotFoundError: + # Use the infos from the parquet export except in some cases: + if data_dir or data_files or (revision and revision != "main"): + use_exported_dataset_infos = False + else: + use_exported_dataset_infos = True + return HubDatasetModuleFactory( + path, + commit_hash=commit_hash, + data_dir=data_dir, + data_files=data_files, + download_config=download_config, + download_mode=download_mode, + use_exported_dataset_infos=use_exported_dataset_infos, + ).get_module() + except GatedRepoError as e: + message = f"Dataset '{path}' is a gated dataset on the Hub." + if e.response.status_code == 401: + message += " You must be authenticated to access it." + elif e.response.status_code == 403: + message += f" Visit the dataset page at https://huggingface.co/datasets/{path} to ask for access." + raise DatasetNotFoundError(message) from e + except RevisionNotFoundError as e: + raise DatasetNotFoundError(f"Revision '{revision}' doesn't exist for dataset '{path}' on the Hub.") from e + except Exception as e1: + # All the attempts failed, before raising the error we should check if the module is already cached + try: + return CachedDatasetModuleFactory(path, cache_dir=cache_dir).get_module() + except Exception: + # If it's not in the cache, then it doesn't exist. + if isinstance(e1, OfflineModeIsEnabled): + raise ConnectionError(f"Couldn't reach the Hugging Face Hub for dataset '{path}': {e1}") from None + if isinstance(e1, (DataFilesNotFoundError, DatasetNotFoundError, EmptyDatasetError)): + raise e1 from None + if isinstance(e1, FileNotFoundError): + raise FileNotFoundError( + f"Couldn't find any data file at {relative_to_absolute_path(path)}. " + f"Couldn't find '{path}' on the Hugging Face Hub either: {type(e1).__name__}: {e1}" + ) from None + raise e1 from None + else: + raise FileNotFoundError(f"Couldn't find any data file at {relative_to_absolute_path(path)}.") + + +def load_dataset_builder( + path: str, + name: Optional[str] = None, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + cache_dir: Optional[str] = None, + features: Optional[Features] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + revision: Optional[Union[str, Version]] = None, + token: Optional[Union[bool, str]] = None, + storage_options: Optional[dict] = None, + **config_kwargs, +) -> DatasetBuilder: + """Load a dataset builder which can be used to: + + - Inspect general information that is required to build a dataset (cache directory, config, dataset info, features, data files, etc.) + - Download and prepare the dataset as Arrow files in the cache + - Get a streaming dataset without downloading or caching anything + + You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`huggingface_hub.list_datasets`]. + + A dataset is a directory that contains some data files in generic formats (JSON, CSV, Parquet, etc.) and possibly + in a generic structure (Webdataset, ImageFolder, AudioFolder, VideoFolder, etc.) + + Args: + + path (`str`): + Path or name of the dataset. + + - if `path` is a dataset repository on the HF hub (list all available datasets with [`huggingface_hub.list_datasets`]) + -> load the dataset builder from supported files in the repository (csv, json, parquet, etc.) + e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing the data files. + + - if `path` is a directory within a Storage Bucket on the HF Hub (list your buckets with [`huggingface_hub.list_buckets`]) + -> load the dataset from supported files in the directory (csv, json, parquet, etc.) + e.g. `'buckets/username/bucket_name/my_dataset'`. + + - if `path` is a local directory + -> load the dataset builder from supported files in the directory (csv, json, parquet, etc.) + e.g. `'./path/to/directory/with/my/csv/data'`. + + - if `path` is the name of a dataset builder and `data_files` or `data_dir` is specified + (available builders are "json", "csv", "parquet", "arrow", "text", "xml", "webdataset", "imagefolder", "audiofolder", "videofolder") + -> load the dataset builder from the files in `data_files` or `data_dir` + e.g. `'parquet'`. + + Use a `hf://` path like `'hf://datasets/username/dataset_name'` to allow remote only. + Use an absolute path to allow local only. + + name (`str`, *optional*): + Defining the name of the dataset configuration. + data_dir (`str`, *optional*): + Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`, + the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory. + data_files (`str` or `Sequence` or `Mapping`, *optional*): + Path(s) to source data file(s). + cache_dir (`str`, *optional*): + Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`. + features ([`Features`], *optional*): + Set the features type to use for this dataset. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + revision ([`Version`] or `str`, *optional*): + Version of the dataset to load. + As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. + You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. + token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. + If `True`, or not specified, will get token from `"~/.huggingface"`. + storage_options (`dict`, *optional*, defaults to `None`): + **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any. + + + + **config_kwargs (additional keyword arguments): + Keyword arguments to be passed to the [`BuilderConfig`] + and used in the [`DatasetBuilder`]. + + Returns: + [`DatasetBuilder`] + + Example: + + ```py + >>> from datasets import load_dataset_builder + >>> ds_builder = load_dataset_builder('cornell-movie-review-data/rotten_tomatoes') + >>> ds_builder.info.features + {'label': ClassLabel(names=['neg', 'pos']), + 'text': Value('string')} + ``` + """ + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + if token is not None: + download_config = download_config.copy() if download_config else DownloadConfig() + download_config.token = token + if storage_options is not None: + download_config = download_config.copy() if download_config else DownloadConfig() + download_config.storage_options.update(storage_options) + if features is not None: + features = _fix_for_backward_compatible_features(features) + dataset_module = dataset_module_factory( + path, + revision=revision, + download_config=download_config, + download_mode=download_mode, + data_dir=data_dir, + data_files=data_files, + cache_dir=cache_dir, + ) + # Get dataset builder class + builder_kwargs = dataset_module.builder_kwargs + data_dir = builder_kwargs.pop("data_dir", data_dir) + data_files = builder_kwargs.pop("data_files", data_files) + config_name = builder_kwargs.pop( + "config_name", name or dataset_module.builder_configs_parameters.default_config_name + ) + dataset_name = builder_kwargs.pop("dataset_name", None) + info = dataset_module.dataset_infos.get(config_name) if dataset_module.dataset_infos else None + + if ( + path in _PACKAGED_DATASETS_MODULES + and data_files is None + and dataset_module.builder_configs_parameters.builder_configs[0].data_files is None + ): + error_msg = f"Please specify the data files or data directory to load for the {path} dataset builder." + example_extensions = [ + extension for extension in _EXTENSION_TO_MODULE if _EXTENSION_TO_MODULE[extension] == path + ] + if example_extensions: + error_msg += f'\nFor example `data_files={{"train": "path/to/data/train/*.{example_extensions[0]}"}}`' + raise ValueError(error_msg) + + # When users pass config kwargs, they should override module-provided defaults + # instead of colliding at constructor call time. + builder_kwargs = {key: value for key, value in builder_kwargs.items() if key not in config_kwargs} + + builder_cls = get_dataset_builder_class(dataset_module, dataset_name=dataset_name) + # Instantiate the dataset builder + builder_instance: DatasetBuilder = builder_cls( + cache_dir=cache_dir, + dataset_name=dataset_name, + config_name=config_name, + data_dir=data_dir, + data_files=data_files, + hash=dataset_module.hash, + info=info, + features=features, + token=token, + storage_options=storage_options, + **builder_kwargs, + **config_kwargs, + ) + builder_instance._use_legacy_cache_dir_if_possible(dataset_module) + + return builder_instance + + +@overload +def load_dataset( + path: str, + name: Optional[str] = None, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + split: None = None, + cache_dir: Optional[str] = None, + features: Optional[Features] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + verification_mode: Optional[Union[VerificationMode, str]] = None, + keep_in_memory: Optional[bool] = None, + save_infos: bool = False, + revision: Optional[Union[str, Version]] = None, + token: Optional[Union[bool, str]] = None, + streaming: Literal[False] = False, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **config_kwargs: Any, +) -> DatasetDict: ... + + +@overload +def load_dataset( + path: str, + name: Optional[str] = None, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + *, + split: Union[str, Split, list[str], list[Split]], + cache_dir: Optional[str] = None, + features: Optional[Features] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + verification_mode: Optional[Union[VerificationMode, str]] = None, + keep_in_memory: Optional[bool] = None, + save_infos: bool = False, + revision: Optional[Union[Version, str]] = None, + token: Optional[Union[bool, str]] = None, + streaming: Literal[False] = False, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **config_kwargs: Any, +) -> Dataset: ... + + +@overload +def load_dataset( + path: str, + name: Optional[str] = None, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + split: None = None, + cache_dir: Optional[str] = None, + features: Optional[Features] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + verification_mode: Optional[Union[VerificationMode, str]] = None, + keep_in_memory: Optional[bool] = None, + save_infos: bool = False, + revision: Optional[Union[Version, str]] = None, + token: Optional[Union[bool, str]] = None, + *, + streaming: Literal[True], + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **config_kwargs: Any, +) -> IterableDatasetDict: ... + + +@overload +def load_dataset( + path: str, + name: Optional[str] = None, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + *, + split: Union[str, Split, list[str], list[Split]], + cache_dir: Optional[str] = None, + features: Optional[Features] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + verification_mode: Optional[Union[VerificationMode, str]] = None, + keep_in_memory: Optional[bool] = None, + save_infos: bool = False, + revision: Optional[Union[Version, str]] = None, + token: Optional[Union[bool, str]] = None, + streaming: Literal[True], + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **config_kwargs: Any, +) -> IterableDataset: ... + + +def load_dataset( + path: str, + name: Optional[str] = None, + data_dir: Optional[str] = None, + data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, + split: Optional[Union[str, Split, list[str], list[Split]]] = None, + cache_dir: Optional[str] = None, + features: Optional[Features] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[Union[DownloadMode, str]] = None, + verification_mode: Optional[Union[VerificationMode, str]] = None, + keep_in_memory: Optional[bool] = None, + save_infos: bool = False, + revision: Optional[Union[str, Version]] = None, + token: Optional[Union[bool, str]] = None, + streaming: bool = False, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **config_kwargs, +) -> Union[DatasetDict, Dataset, IterableDatasetDict, IterableDataset]: + """Load a dataset from the Hugging Face Hub, or a local dataset. + + You can find the list of datasets on the [Hub](https://huggingface.co/datasets) or with [`huggingface_hub.list_datasets`]. + + A dataset is a directory that contains some data files in generic formats (JSON, CSV, Parquet, etc.) and possibly + in a generic structure (Webdataset, ImageFolder, AudioFolder, VideoFolder, etc.) + + This function does the following under the hood: + + 1. Load a dataset builder: + + * Find the most common data format in the dataset and pick its associated builder (JSON, CSV, Parquet, Webdataset, ImageFolder, AudioFolder, etc.) + * Find which file goes into which split (e.g. train/test) based on file and directory names or on the YAML configuration + * It is also possible to specify `data_files` manually, and which dataset builder to use (e.g. "parquet"). + + 2. Run the dataset builder: + + In the general case: + + * Download the data files from the dataset if they are not already available locally or cached. + * Process and cache the dataset in typed Arrow tables for caching. + + Arrow table are arbitrarily long, typed tables which can store nested objects and be mapped to numpy/pandas/python generic types. + They can be directly accessed from disk, loaded in RAM or even streamed over the web. + + In the streaming case: + + * Don't download or cache anything. Instead, the dataset is lazily loaded and will be streamed on-the-fly when iterating on it. + + 3. Return a dataset built from the requested splits in `split` (default: all). + + Args: + + path (`str`): + Path or name of the dataset. + + - if `path` is a dataset repository on the HF hub (list all available datasets with [`huggingface_hub.list_datasets`]) + -> load the dataset from supported files in the repository (csv, json, parquet, etc.) + e.g. `'username/dataset_name'`, a dataset repository on the HF hub containing the data files. + + - if `path` is a directory within a Storage Bucket on the HF Hub (list your buckets with [`huggingface_hub.list_buckets`]) + -> load the dataset from supported files in the directory (csv, json, parquet, etc.) + e.g. `'buckets/username/bucket_name/my_dataset'`. + + - if `path` is a local directory + -> load the dataset from supported files in the directory (csv, json, parquet, etc.) + e.g. `'./path/to/directory/with/my/csv/data'`. + + - if `path` is the name of a dataset builder and `data_files` or `data_dir` is specified + (available builders are "json", "csv", "parquet", "arrow", "text", "xml", "webdataset", "imagefolder", "audiofolder", "videofolder") + -> load the dataset from the files in `data_files` or `data_dir` + e.g. `'parquet'`. + + Use a `hf://` path like `'hf://datasets/username/dataset_name'` to allow remote only. + Use an absolute path to allow local only. + + name (`str`, *optional*): + Defining the name of the dataset configuration. + data_dir (`str`, *optional*): + Defining the `data_dir` of the dataset configuration. If specified for the generic builders (csv, text etc.) or the Hub datasets and `data_files` is `None`, + the behavior is equal to passing `os.path.join(data_dir, **)` as `data_files` to reference all the files in a directory. + data_files (`str` or `Sequence` or `Mapping`, *optional*): + Path(s) to source data file(s). + split (`Split` or `str`): + Which split of the data to load. + If `None`, will return a `dict` with all splits (typically `datasets.Split.TRAIN` and `datasets.Split.TEST`). + If given, will return a single Dataset. + Splits can be combined and specified like in tensorflow-datasets. + cache_dir (`str`, *optional*): + Directory to read/write data. Defaults to `"~/.cache/huggingface/datasets"`. + features (`Features`, *optional*): + Set the features type to use for this dataset. + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + verification_mode ([`VerificationMode`] or `str`, defaults to `BASIC_CHECKS`): + Verification mode determining the checks to run on the downloaded/processed dataset information (checksums/size/splits/...). + + + keep_in_memory (`bool`, defaults to `None`): + Whether to copy the dataset in-memory. If `None`, the dataset + will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to + nonzero. See more details in the [improve performance](../cache#improve-performance) section. + revision ([`Version`] or `str`, *optional*): + Version of the dataset to load. + As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. + You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. + token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. + If `True`, or not specified, will get token from `"~/.huggingface"`. + streaming (`bool`, defaults to `False`): + If set to `True`, don't download the data files. Instead, it streams the data progressively while + iterating on the dataset. An [`IterableDataset`] or [`IterableDatasetDict`] is returned instead in this case. + + Note that streaming works for datasets that use data formats that support being iterated over like txt, csv, jsonl for example. + Json files may be downloaded completely. Also streaming from remote zip or gzip files is supported but other compressed formats + like rar and xz are not yet supported. The tgz format doesn't allow streaming. + num_proc (`int`, *optional*, defaults to `None`): + Number of processes when downloading and generating the dataset locally. + Multiprocessing is disabled by default. + + + storage_options (`dict`, *optional*, defaults to `None`): + **Experimental**. Key/value pairs to be passed on to the dataset file-system backend, if any. + + + **config_kwargs (additional keyword arguments): + Keyword arguments to be passed to the `BuilderConfig` + and used in the [`DatasetBuilder`]. + + Returns: + [`Dataset`] or [`DatasetDict`]: + - if `split` is not `None`: the dataset requested, + - if `split` is `None`, a [`~datasets.DatasetDict`] with each split. + + or [`IterableDataset`] or [`IterableDatasetDict`]: if `streaming=True` + + - if `split` is not `None`, the dataset is requested + - if `split` is `None`, a [`~datasets.streaming.IterableDatasetDict`] with each split. + + Example: + + Load a dataset from the Hugging Face Hub: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train') + + # Load a subset or dataset configuration (here 'sst2') + >>> from datasets import load_dataset + >>> ds = load_dataset('nyu-mll/glue', 'sst2', split='train') + + # Manual mapping of data files to splits + >>> data_files = {'train': 'train.csv', 'test': 'test.csv'} + >>> ds = load_dataset('namespace/your_dataset_name', data_files=data_files) + + # Manual selection of a directory to load + >>> ds = load_dataset('namespace/your_dataset_name', data_dir='folder_name') + ``` + + Load a dataset from a Storage Bucket on the Hugging Face Hub: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset('buckets/username/bucket_name/rotten_tomatoes', split='train') + ``` + + Load a local dataset: + + ```py + # Load a CSV file + >>> from datasets import load_dataset + >>> ds = load_dataset('csv', data_files='path/to/local/my_dataset.csv') + + # Load a JSON file + >>> from datasets import load_dataset + >>> ds = load_dataset('json', data_files='path/to/local/my_dataset.json') + ``` + + Load an [`~datasets.IterableDataset`]: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset('cornell-movie-review-data/rotten_tomatoes', split='train', streaming=True) + ``` + + Load an image dataset with the `ImageFolder` dataset builder: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset('imagefolder', data_dir='/path/to/images', split='train') + ``` + """ + if "trust_remote_code" in config_kwargs: + if config_kwargs.pop("trust_remote_code"): + logger.error( + "`trust_remote_code` is not supported anymore.\n" + f"Please check that the Hugging Face dataset '{path}' isn't based on a loading script and remove `trust_remote_code`.\n" + "If the dataset is based on a loading script, please ask the dataset author to remove it and convert it to a standard format like Parquet." + ) + if data_files is not None and not data_files: + raise ValueError(f"Empty 'data_files': '{data_files}'. It should be either non-empty or None (default).") + if Path(path, config.DATASET_STATE_JSON_FILENAME).exists(): + raise ValueError( + "You are trying to load a dataset that was saved using `save_to_disk`. " + "Please use `load_from_disk` instead." + ) + + if streaming and num_proc is not None: + raise NotImplementedError( + "Loading a streaming dataset in parallel with `num_proc` is not implemented. " + "To parallelize streaming, you can wrap the dataset with a PyTorch DataLoader using `num_workers` > 1 instead." + ) + + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + verification_mode = VerificationMode( + (verification_mode or VerificationMode.BASIC_CHECKS) if not save_infos else VerificationMode.ALL_CHECKS + ) + + # Create a dataset builder + builder_instance = load_dataset_builder( + path=path, + name=name, + data_dir=data_dir, + data_files=data_files, + cache_dir=cache_dir, + features=features, + download_config=download_config, + download_mode=download_mode, + revision=revision, + token=token, + storage_options=storage_options, + **config_kwargs, + ) + + # Return iterable dataset in case of streaming + if streaming: + return builder_instance.as_streaming_dataset(split=split) + + # Download and prepare data + builder_instance.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + num_proc=num_proc, + storage_options=storage_options, + ) + + # Build dataset for splits + keep_in_memory = ( + keep_in_memory if keep_in_memory is not None else is_small_dataset(builder_instance.info.dataset_size) + ) + ds = builder_instance.as_dataset(split=split, in_memory=keep_in_memory) + + return ds + + +def load_from_disk( + dataset_path: PathLike, keep_in_memory: Optional[bool] = None, storage_options: Optional[dict] = None +) -> Union[Dataset, DatasetDict]: + """ + Loads a dataset that was previously saved using [`~Dataset.save_to_disk`] from a dataset directory, or + from a filesystem using any implementation of `fsspec.spec.AbstractFileSystem`. + + Args: + dataset_path (`path-like`): + Path (e.g. `"dataset/train"`) or remote URI (e.g. `"s3://my-bucket/dataset/train"`) + of the [`Dataset`] or [`DatasetDict`] directory where the dataset/dataset-dict will be + loaded from. + keep_in_memory (`bool`, defaults to `None`): + Whether to copy the dataset in-memory. If `None`, the dataset + will not be copied in-memory unless explicitly enabled by setting `datasets.config.IN_MEMORY_MAX_SIZE` to + nonzero. See more details in the [improve performance](../cache#improve-performance) section. + + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + Returns: + [`Dataset`] or [`DatasetDict`]: + - If `dataset_path` is a path of a dataset directory: the dataset requested. + - If `dataset_path` is a path of a dataset dict directory, a [`DatasetDict`] with each split. + + Example: + + ```py + >>> from datasets import load_from_disk + >>> ds = load_from_disk('path/to/dataset/directory') + ``` + """ + fs: fsspec.AbstractFileSystem + fs, *_ = url_to_fs(dataset_path, **(storage_options or {})) + if not fs.exists(dataset_path): + raise FileNotFoundError(f"Directory {dataset_path} not found") + if fs.isfile(posixpath.join(dataset_path, config.DATASET_INFO_FILENAME)) and fs.isfile( + posixpath.join(dataset_path, config.DATASET_STATE_JSON_FILENAME) + ): + return Dataset.load_from_disk(dataset_path, keep_in_memory=keep_in_memory, storage_options=storage_options) + elif fs.isfile(posixpath.join(dataset_path, config.DATASETDICT_JSON_FILENAME)): + return DatasetDict.load_from_disk(dataset_path, keep_in_memory=keep_in_memory, storage_options=storage_options) + else: + raise FileNotFoundError( + f"Directory {dataset_path} is neither a `Dataset` directory nor a `DatasetDict` directory." + ) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/naming.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/naming.py new file mode 100644 index 0000000000000000000000000000000000000000..29216bbd7de3b27993f821a1d26604ab40337f3c --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/naming.py @@ -0,0 +1,84 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Utilities for file names.""" + +import itertools +import os +import re + + +_uppercase_uppercase_re = re.compile(r"([A-Z]+)([A-Z][a-z])") +_lowercase_uppercase_re = re.compile(r"([a-z\d])([A-Z])") + +_single_underscore_re = re.compile(r"(?:/\|?*" + + +def camelcase_to_snakecase(name): + """Convert camel-case string to snake-case.""" + name = _uppercase_uppercase_re.sub(r"\1_\2", name) + name = _lowercase_uppercase_re.sub(r"\1_\2", name) + return name.lower() + + +def snakecase_to_camelcase(name): + """Convert snake-case string to camel-case string.""" + name = _single_underscore_re.split(name) + name = [_multiple_underscores_re.split(n) for n in name] + return "".join(n.capitalize() for n in itertools.chain.from_iterable(name) if n != "") + + +def filename_prefix_for_name(name): + if os.path.basename(name) != name: + raise ValueError(f"Should be a dataset name, not a path: {name}") + return camelcase_to_snakecase(name) + + +def filename_prefix_for_split(name, split): + if os.path.basename(name) != name: + raise ValueError(f"Should be a dataset name, not a path: {name}") + if not re.match(_split_re, split): + raise ValueError(f"Split name should match '{_split_re}'' but got '{split}'.") + return f"{filename_prefix_for_name(name)}-{split}" + + +def filepattern_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=None): + prefix = filename_prefix_for_split(dataset_name, split) + if filetype_suffix: + prefix += f".{filetype_suffix}" + filepath = os.path.join(data_dir, prefix) + return f"{filepath}*" + + +def filenames_for_dataset_split(path, dataset_name, split, filetype_suffix=None, shard_lengths=None): + prefix = filename_prefix_for_split(dataset_name, split) + prefix = os.path.join(path, prefix) + + if shard_lengths and len(shard_lengths) > 1: + num_shards = len(shard_lengths) + filenames = [f"{prefix}-{shard_id:05d}-of-{num_shards:05d}" for shard_id in range(num_shards)] + if filetype_suffix: + filenames = [filename + f".{filetype_suffix}" for filename in filenames] + return filenames + else: + filename = prefix + if filetype_suffix: + filename += f".{filetype_suffix}" + return [filename] diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f0ebcb79693a01b2a076e3bebd2be2c466958c1a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/__init__.py @@ -0,0 +1,130 @@ +import inspect +import re +from typing import Dict, List, Tuple + +from huggingface_hub.utils import insecure_hashlib + +from .arrow import arrow +from .audiofolder import audiofolder +from .cache import cache +from .csv import csv +from .eval import eval +from .hdf5 import hdf5 +from .imagefolder import imagefolder +from .json import json +from .lance import lance +from .niftifolder import niftifolder +from .pandas import pandas +from .parquet import parquet +from .pdffolder import pdffolder +from .sql import sql +from .text import text +from .videofolder import videofolder +from .webdataset import webdataset +from .xml import xml + + +def _hash_python_lines(lines: list[str]) -> str: + filtered_lines = [] + for line in lines: + line = re.sub(r"#.*", "", line) # remove comments + if line: + filtered_lines.append(line) + full_str = "\n".join(filtered_lines) + + # Make a hash from all this code + full_bytes = full_str.encode("utf-8") + return insecure_hashlib.sha256(full_bytes).hexdigest() + + +# get importable module names and hash for caching +_PACKAGED_DATASETS_MODULES = { + "csv": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())), + "json": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())), + "pandas": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())), + "parquet": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())), + "arrow": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())), + "text": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())), + "imagefolder": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())), + "audiofolder": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())), + "videofolder": (videofolder.__name__, _hash_python_lines(inspect.getsource(videofolder).splitlines())), + "pdffolder": (pdffolder.__name__, _hash_python_lines(inspect.getsource(pdffolder).splitlines())), + "niftifolder": (niftifolder.__name__, _hash_python_lines(inspect.getsource(niftifolder).splitlines())), + "webdataset": (webdataset.__name__, _hash_python_lines(inspect.getsource(webdataset).splitlines())), + "xml": (xml.__name__, _hash_python_lines(inspect.getsource(xml).splitlines())), + "hdf5": (hdf5.__name__, _hash_python_lines(inspect.getsource(hdf5).splitlines())), + "eval": (eval.__name__, _hash_python_lines(inspect.getsource(eval).splitlines())), + "lance": (lance.__name__, _hash_python_lines(inspect.getsource(lance).splitlines())), +} + +# get importable module names and hash for caching +_PACKAGED_DATASETS_MODULES_2_15_HASHES = { + "csv": "eea64c71ca8b46dd3f537ed218fc9bf495d5707789152eb2764f5c78fa66d59d", + "json": "8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96", + "pandas": "3ac4ffc4563c796122ef66899b9485a3f1a977553e2d2a8a318c72b8cc6f2202", + "parquet": "ca31c69184d9832faed373922c2acccec0b13a0bb5bbbe19371385c3ff26f1d1", + "arrow": "74f69db2c14c2860059d39860b1f400a03d11bf7fb5a8258ca38c501c878c137", + "text": "c4a140d10f020282918b5dd1b8a49f0104729c6177f60a6b49ec2a365ec69f34", + "imagefolder": "7b7ce5247a942be131d49ad4f3de5866083399a0f250901bd8dc202f8c5f7ce5", + "audiofolder": "d3c1655c66c8f72e4efb5c79e952975fa6e2ce538473a6890241ddbddee9071c", +} + +# Used to infer the module to use based on the data files extensions +_EXTENSION_TO_MODULE: dict[str, tuple[str, dict]] = { + ".csv": ("csv", {}), + ".tsv": ("csv", {"sep": "\t"}), + ".json": ("json", {}), + ".jsonl": ("json", {}), + # ndjson is no longer maintained (see: https://github.com/ndjson/ndjson-spec/issues/35#issuecomment-1285673417) + ".ndjson": ("json", {}), + ".parquet": ("parquet", {}), + ".geoparquet": ("parquet", {}), + ".gpq": ("parquet", {}), + ".arrow": ("arrow", {}), + ".txt": ("text", {}), + ".tar": ("webdataset", {}), + ".xml": ("xml", {}), + ".hdf5": ("hdf5", {}), + ".h5": ("hdf5", {}), + ".eval": ("eval", {}), + ".lance": ("lance", {}), +} +_EXTENSION_TO_MODULE.update({ext: ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext.upper(): ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext: ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext.upper(): ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext: ("videofolder", {}) for ext in videofolder.VideoFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext.upper(): ("videofolder", {}) for ext in videofolder.VideoFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext: ("pdffolder", {}) for ext in pdffolder.PdfFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext.upper(): ("pdffolder", {}) for ext in pdffolder.PdfFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext: ("niftifolder", {}) for ext in niftifolder.NiftiFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext.upper(): ("niftifolder", {}) for ext in niftifolder.NiftiFolder.EXTENSIONS}) + +# Used to filter data files based on extensions given a module name +_MODULE_TO_EXTENSIONS: dict[str, list[str]] = {} +for _ext, (_module, _) in _EXTENSION_TO_MODULE.items(): + _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext) + +for _module in _MODULE_TO_EXTENSIONS: + _MODULE_TO_EXTENSIONS[_module].append(".zip") + +# Used to filter data files based on file names +_MODULE_TO_METADATA_FILE_NAMES: Dict[str, List[str]] = {} +for _module in _MODULE_TO_EXTENSIONS: + _MODULE_TO_METADATA_FILE_NAMES[_module] = [] +_MODULE_TO_METADATA_FILE_NAMES["imagefolder"] = imagefolder.ImageFolder.METADATA_FILENAMES +_MODULE_TO_METADATA_FILE_NAMES["audiofolder"] = imagefolder.ImageFolder.METADATA_FILENAMES +_MODULE_TO_METADATA_FILE_NAMES["videofolder"] = imagefolder.ImageFolder.METADATA_FILENAMES +_MODULE_TO_METADATA_FILE_NAMES["pdffolder"] = imagefolder.ImageFolder.METADATA_FILENAMES +_MODULE_TO_METADATA_FILE_NAMES["niftifolder"] = imagefolder.ImageFolder.METADATA_FILENAMES + +_MODULE_TO_METADATA_EXTENSIONS: Dict[str, List[str]] = {} +for _module in _MODULE_TO_EXTENSIONS: + _MODULE_TO_METADATA_EXTENSIONS[_module] = [] +_MODULE_TO_METADATA_EXTENSIONS["lance"] = lance.Lance.METADATA_EXTENSIONS + +# Total + +_ALL_EXTENSIONS = list(_EXTENSION_TO_MODULE.keys()) + [".zip"] +_ALL_METADATA_EXTENSIONS = sorted({_ext for _exts in _MODULE_TO_METADATA_EXTENSIONS.values() for _ext in _exts}) +_ALL_ALLOWED_EXTENSIONS = _ALL_EXTENSIONS + _ALL_METADATA_EXTENSIONS diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/csv/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/csv/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/csv/csv.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/csv/csv.py new file mode 100644 index 0000000000000000000000000000000000000000..249df36e2950025c0a05c298752de1c1dd5df7b1 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/csv/csv.py @@ -0,0 +1,206 @@ +from dataclasses import dataclass +from typing import Any, Callable, Optional, Union + +import pandas as pd +import pyarrow as pa + +import datasets +import datasets.config +from datasets.builder import Key +from datasets.features.features import require_storage_cast +from datasets.table import table_cast +from datasets.utils.py_utils import Literal + + +logger = datasets.utils.logging.get_logger(__name__) + +_PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS = ["names", "prefix"] +_PANDAS_READ_CSV_DEPRECATED_PARAMETERS = ["warn_bad_lines", "error_bad_lines", "mangle_dupe_cols"] +_PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS = ["encoding_errors", "on_bad_lines"] +_PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS = ["date_format"] +_PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS = ["verbose"] + + +@dataclass +class CsvConfig(datasets.BuilderConfig): + """BuilderConfig for CSV.""" + + sep: str = "," + delimiter: Optional[str] = None + header: Optional[Union[int, list[int], str]] = "infer" + names: Optional[list[str]] = None + column_names: Optional[list[str]] = None + index_col: Optional[Union[int, str, list[int], list[str]]] = None + usecols: Optional[Union[list[int], list[str]]] = None + prefix: Optional[str] = None + mangle_dupe_cols: bool = True + engine: Optional[Literal["c", "python", "pyarrow"]] = None + converters: dict[Union[int, str], Callable[[Any], Any]] = None + true_values: Optional[list] = None + false_values: Optional[list] = None + skipinitialspace: bool = False + skiprows: Optional[Union[int, list[int]]] = None + nrows: Optional[int] = None + na_values: Optional[Union[str, list[str]]] = None + keep_default_na: bool = True + na_filter: bool = True + verbose: bool = False + skip_blank_lines: bool = True + thousands: Optional[str] = None + decimal: str = "." + lineterminator: Optional[str] = None + quotechar: str = '"' + quoting: int = 0 + escapechar: Optional[str] = None + comment: Optional[str] = None + encoding: Optional[str] = None + dialect: Optional[str] = None + error_bad_lines: bool = True + warn_bad_lines: bool = True + skipfooter: int = 0 + doublequote: bool = True + memory_map: bool = False + float_precision: Optional[str] = None + chunksize: int = 10_000 + features: Optional[datasets.Features] = None + encoding_errors: Optional[str] = "strict" + on_bad_lines: Literal["error", "warn", "skip"] = "error" + date_format: Optional[str] = None + + def __post_init__(self): + super().__post_init__() + if self.delimiter is not None: + self.sep = self.delimiter + if self.column_names is not None: + self.names = self.column_names + + @property + def pd_read_csv_kwargs(self): + pd_read_csv_kwargs = { + "sep": self.sep, + "header": self.header, + "names": self.names, + "index_col": self.index_col, + "usecols": self.usecols, + "prefix": self.prefix, + "mangle_dupe_cols": self.mangle_dupe_cols, + "engine": self.engine, + "converters": self.converters, + "true_values": self.true_values, + "false_values": self.false_values, + "skipinitialspace": self.skipinitialspace, + "skiprows": self.skiprows, + "nrows": self.nrows, + "na_values": self.na_values, + "keep_default_na": self.keep_default_na, + "na_filter": self.na_filter, + "verbose": self.verbose, + "skip_blank_lines": self.skip_blank_lines, + "thousands": self.thousands, + "decimal": self.decimal, + "lineterminator": self.lineterminator, + "quotechar": self.quotechar, + "quoting": self.quoting, + "escapechar": self.escapechar, + "comment": self.comment, + "encoding": self.encoding, + "dialect": self.dialect, + "error_bad_lines": self.error_bad_lines, + "warn_bad_lines": self.warn_bad_lines, + "skipfooter": self.skipfooter, + "doublequote": self.doublequote, + "memory_map": self.memory_map, + "float_precision": self.float_precision, + "chunksize": self.chunksize, + "encoding_errors": self.encoding_errors, + "on_bad_lines": self.on_bad_lines, + "date_format": self.date_format, + } + + # some kwargs must not be passed if they don't have a default value + # some others are deprecated and we can also not pass them if they are the default value + for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS: + if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter): + del pd_read_csv_kwargs[pd_read_csv_parameter] + + # Remove 1.3 new arguments + if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3): + for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS: + del pd_read_csv_kwargs[pd_read_csv_parameter] + + # Remove 2.0 new arguments + if not (datasets.config.PANDAS_VERSION.major >= 2): + for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS: + del pd_read_csv_kwargs[pd_read_csv_parameter] + + # Remove 2.2 deprecated arguments + if datasets.config.PANDAS_VERSION.release >= (2, 2): + for pd_read_csv_parameter in _PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS: + if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter): + del pd_read_csv_kwargs[pd_read_csv_parameter] + + return pd_read_csv_kwargs + + +class Csv(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = CsvConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + base_data_files = dl_manager.download(self.config.data_files) + extracted_data_files = dl_manager.extract(base_data_files) + splits = [] + for split_name, extracted_files in extracted_data_files.items(): + files_iterables = [dl_manager.iter_files(extracted_file) for extracted_file in extracted_files] + splits.append( + datasets.SplitGenerator( + name=split_name, + gen_kwargs={"files_iterables": files_iterables, "base_files": base_data_files[split_name]}, + ) + ) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + schema = self.config.features.arrow_schema + if all(not require_storage_cast(feature) for feature in self.config.features.values()): + # cheaper cast + pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema) + else: + # more expensive cast; allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, schema) + return pa_table + + def _generate_shards(self, base_files, files_iterables): + yield from base_files + + def _generate_tables(self, base_files, files_iterables): + schema = self.config.features.arrow_schema if self.config.features else None + # dtype allows reading an int column as str + dtype = ( + { + name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object + for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values()) + } + if schema is not None + else None + ) + for shard_idx, files_iterable in enumerate(files_iterables): + for file in files_iterable: + csv_file_reader = pd.read_csv(file, iterator=True, dtype=dtype, **self.config.pd_read_csv_kwargs) + try: + for batch_idx, df in enumerate(csv_file_reader): + pa_table = pa.Table.from_pandas(df) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield Key(shard_idx, batch_idx), self._cast_table(pa_table) + except ValueError as e: + logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}") + raise diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..82586c648cbf6039625ac1de7dfc50b2106ca7a7 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py @@ -0,0 +1,444 @@ +import collections +import io +import itertools +import os +from dataclasses import dataclass +from typing import Any, Callable, Iterator, Optional, Union + +import pandas as pd +import pyarrow as pa +import pyarrow.dataset as ds +import pyarrow.json as paj +import pyarrow.parquet as pq + +import datasets +from datasets import config +from datasets.builder import Key +from datasets.features.features import FeatureType, _visit, _visit_with_path, _VisitPath, require_storage_cast +from datasets.utils.file_utils import readline + + +logger = datasets.utils.logging.get_logger(__name__) + + +def count_path_segments(path): + return path.replace("\\", "/").count("/") + + +@dataclass +class FolderBasedBuilderConfig(datasets.BuilderConfig): + """BuilderConfig for AutoFolder.""" + + features: Optional[datasets.Features] = None + drop_labels: bool = None + drop_metadata: bool = None + metadata_filenames: list[str] = None + filters: Optional[Union[ds.Expression, list[tuple], list[list[tuple]]]] = None + + def __post_init__(self): + super().__post_init__() + + +class FolderBasedBuilder(datasets.GeneratorBasedBuilder): + """ + Base class for generic data loaders for vision and image data. + + + Abstract class attributes to be overridden by a child class: + BASE_FEATURE: feature object to decode data (i.e. datasets.Image, datasets.Audio, ...) + BASE_COLUMN_NAME: string key name of a base feature (i.e. "image", "audio", ...) + BUILDER_CONFIG_CLASS: builder config inherited from `folder_based_builder.FolderBasedBuilderConfig` + EXTENSIONS: list of allowed extensions (only files with these extensions and METADATA_FILENAME files + will be included in a dataset) + """ + + BASE_FEATURE: type[FeatureType] + BASE_COLUMN_NAME: str + BUILDER_CONFIG_CLASS: FolderBasedBuilderConfig + EXTENSIONS: list[str] + + METADATA_FILENAMES: list[str] = ["metadata.csv", "metadata.jsonl", "metadata.parquet"] + + def _info(self): + if not self.config.data_dir and not self.config.data_files: + raise ValueError( + "Folder-based datasets require either `data_dir` or `data_files` to be specified. " + "Neither was provided." + ) + + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + # Do an early pass if: + # * `drop_labels` is None (default) or False, to infer the class labels + # * `drop_metadata` is None (default) or False, to find the metadata files + do_analyze = not self.config.drop_labels or not self.config.drop_metadata + labels, path_depths = set(), set() + all_metadata_files = collections.defaultdict(set) + metadata_filenames = self.config.metadata_filenames or self.METADATA_FILENAMES + + def analyze(files_or_archives, downloaded_files_or_dirs, split): + if len(downloaded_files_or_dirs) == 0: + return + # The files are separated from the archives at this point, so check the first sample + # to see if it's a file or a directory and iterate accordingly + if os.path.isfile(downloaded_files_or_dirs[0]): + original_files, downloaded_files = files_or_archives, downloaded_files_or_dirs + for original_file, downloaded_file in zip(original_files, downloaded_files): + original_file, downloaded_file = str(original_file), str(downloaded_file) + _, original_file_ext = os.path.splitext(original_file) + if original_file_ext.lower() in self.EXTENSIONS: + if not self.config.drop_labels: + labels.add(os.path.basename(os.path.dirname(original_file))) + path_depths.add(count_path_segments(original_file)) + elif os.path.basename(original_file) in metadata_filenames: + all_metadata_files[split].add((original_file, None, downloaded_file)) + else: + original_file_name = os.path.basename(original_file) + logger.debug( + f"The file '{original_file_name}' was ignored: it is not a {self.BASE_COLUMN_NAME}, and is not {metadata_filenames} either." + ) + else: + archives, downloaded_dirs = files_or_archives, downloaded_files_or_dirs + for archive, downloaded_dir in zip(archives, downloaded_dirs): + archive, downloaded_dir = str(archive), str(downloaded_dir) + for downloaded_dir_file in dl_manager.iter_files(downloaded_dir): + _, downloaded_dir_file_ext = os.path.splitext(downloaded_dir_file) + if downloaded_dir_file_ext in self.EXTENSIONS: + if not self.config.drop_labels: + labels.add(os.path.basename(os.path.dirname(downloaded_dir_file))) + path_depths.add(count_path_segments(downloaded_dir_file)) + elif os.path.basename(downloaded_dir_file) in metadata_filenames: + all_metadata_files[split].add((None, downloaded_dir, downloaded_dir_file)) + else: + archive_file_name = os.path.basename(archive) + original_file_name = os.path.basename(downloaded_dir_file) + logger.debug( + f"The file '{original_file_name}' from the archive '{archive_file_name}' was ignored: it is not a {self.BASE_COLUMN_NAME}, and is not {metadata_filenames} either." + ) + + data_files = self.config.data_files + splits = [] + for split_name, files in data_files.items(): + files, metadata_files, archives = self._split_files_and_metadata_and_archives(files) + downloaded_files = dl_manager.download(files) + downloaded_metadata_files = dl_manager.download(metadata_files) + downloaded_dirs = dl_manager.download_and_extract(archives) + if do_analyze: # drop_metadata is None or False, drop_labels is None or False + logger.info(f"Searching for labels and/or metadata files in {split_name} data files...") + analyze(files, downloaded_files, split_name) + analyze(metadata_files, downloaded_metadata_files, split_name) + analyze(archives, downloaded_dirs, split_name) + + if all_metadata_files: + # add metadata if `all_metadata_files` are found and `drop_metadata` is None (default) or False + add_metadata = not self.config.drop_metadata + # if `all_metadata_files` are found, don't add labels + add_labels = False + else: + # if `all_metadata_files` are not found, don't add metadata + add_metadata = False + # if `all_metadata_files` are not found and `drop_labels` is None (default) - + # add labels if files are on the same level in directory hierarchy and there is more than one label + add_labels = ( + (len(labels) > 1 and len(path_depths) == 1) + if self.config.drop_labels is None + else not self.config.drop_labels + ) + + if add_labels: + logger.info("Adding the labels inferred from data directories to the dataset's features...") + if add_metadata: + logger.info("Adding metadata to the dataset...") + else: + add_labels, add_metadata, all_metadata_files = False, False, {} + + # files info (original_file, None, downloaded_file) + files = tuple(zip(files, [None] * len(files), downloaded_files)) + # archives info (original_archive_file, downloaded_dir, downloaded_files) + files += tuple( + (archive, downloaded_dir, dl_manager.iter_files(downloaded_dir)) + for archive, downloaded_dir in zip(archives, downloaded_dirs) + ) + splits.append( + datasets.SplitGenerator( + name=split_name, + gen_kwargs={ + "files": files, + "metadata_files": all_metadata_files.get(split_name, []), + "add_labels": add_labels, + "add_metadata": add_metadata, + }, + ) + ) + + if add_metadata: + # Verify that: + # * all metadata files have the same set of features in each split + # * the `file_name` key is one of the metadata keys and is of type string + features_per_metadata_file: list[tuple[str, datasets.Features]] = [] + + # Check that all metadata files share the same format + metadata_ext = { + os.path.splitext(original_metadata_file or downloaded_metadata_file)[-1] + for original_metadata_file, _, downloaded_metadata_file in itertools.chain.from_iterable( + all_metadata_files.values() + ) + } + if len(metadata_ext) > 1: + raise ValueError(f"Found metadata files with different extensions: {list(metadata_ext)}") + metadata_ext = metadata_ext.pop() + + for split_metadata_files in all_metadata_files.values(): + pa_metadata_table = None + for _, _, downloaded_metadata_file in split_metadata_files: + for pa_metadata_table in self._read_metadata(downloaded_metadata_file, metadata_ext=metadata_ext): + break # just fetch the first rows + if pa_metadata_table is not None: + features_per_metadata_file.append( + (downloaded_metadata_file, datasets.Features.from_arrow_schema(pa_metadata_table.schema)) + ) + break # no need to fetch all the files + for downloaded_metadata_file, metadata_features in features_per_metadata_file: + if metadata_features != features_per_metadata_file[0][1]: + raise ValueError( + f"Metadata files {downloaded_metadata_file} and {features_per_metadata_file[0][0]} have different features: {features_per_metadata_file[0]} != {metadata_features}" + ) + metadata_features = features_per_metadata_file[0][1] + feature_not_found = True + + def _set_feature(feature): + nonlocal feature_not_found + if isinstance(feature, dict): + out = type(feature)() + for key in feature: + if (key == "file_name" or key.endswith("_file_name")) and ( + feature[key] == datasets.Value("string") or feature[key] == datasets.Value("large_string") + ): + key = key[: -len("_file_name")] or self.BASE_COLUMN_NAME + out[key] = self.BASE_FEATURE() + feature_not_found = False + elif (key == "file_names" or key.endswith("_file_names")) and ( + feature[key] + in [datasets.List(datasets.Value("string")), datasets.List(datasets.Value("large_string"))] + ): + key = key[: -len("_file_names")] or (self.BASE_COLUMN_NAME + "s") + out[key] = datasets.List(self.BASE_FEATURE()) + feature_not_found = False + elif (key == "file_names" or key.endswith("_file_names")) and ( + feature[key] == [datasets.Value("string")] + or feature[key] == [datasets.Value("large_string")] + ): + key = key[: -len("_file_names")] or (self.BASE_COLUMN_NAME + "s") + out[key] = [self.BASE_FEATURE()] + feature_not_found = False + else: + out[key] = feature[key] + return out + return feature + + metadata_features = _visit(metadata_features, _set_feature) + + if feature_not_found: + raise ValueError( + "`file_name`, `*_file_name`, `file_names` or `*_file_names` must be present as dictionary key in metadata files" + ) + else: + metadata_features = None + + # Normally, we would do this in _info, but we need to know the labels and/or metadata + # before building the features + if self.config.features is None: + if add_metadata: + self.info.features = metadata_features + elif add_labels: + self.info.features = datasets.Features( + { + self.BASE_COLUMN_NAME: self.BASE_FEATURE(), + "label": datasets.ClassLabel(names=sorted(labels)), + } + ) + else: + self.info.features = datasets.Features({self.BASE_COLUMN_NAME: self.BASE_FEATURE()}) + + return splits + + def _split_files_and_metadata_and_archives(self, data_files): + files, metadata_files, archives = [], [], [] + metadata_filenames = self.config.metadata_filenames or self.METADATA_FILENAMES + for data_file in data_files: + data_file_root, data_file_ext = os.path.splitext(data_file) + _, second_data_file_ext = os.path.splitext(data_file_root) + if data_file_ext.lower() in self.EXTENSIONS or second_data_file_ext.lower() in self.EXTENSIONS: + files.append(data_file) + elif os.path.basename(data_file) in metadata_filenames: + metadata_files.append(data_file) + elif data_file_ext.lower() == ".zip": + archives.append(data_file) + return files, metadata_files, archives + + def _read_metadata(self, metadata_file: str, metadata_ext: str = "") -> Iterator[pa.Table]: + """using the same logic as the Csv, Json and Parquet dataset builders to stream the data""" + if self.config.filters is not None: + filter_expr = ( + pq.filters_to_expression(self.config.filters) + if isinstance(self.config.filters, list) + else self.config.filters + ) + else: + filter_expr = None + if metadata_ext == ".csv": + chunksize = 10_000 # 10k lines + schema = self.config.features.arrow_schema if self.config.features else None + # dtype allows reading an int column as str + dtype = ( + { + name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object + for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values()) + } + if schema is not None + else None + ) + csv_file_reader = pd.read_csv(metadata_file, iterator=True, dtype=dtype, chunksize=chunksize) + for df in csv_file_reader: + pa_table = pa.Table.from_pandas(df) + if self.config.filters is not None: + pa_table = pa_table.filter(filter_expr) + if len(pa_table) > 0: + yield pa_table + elif metadata_ext == ".jsonl": + with open(metadata_file, "rb") as f: + chunksize: int = 10 << 20 # 10MB + # Use block_size equal to the chunk size divided by 32 to leverage multithreading + # Set a default minimum value of 16kB if the chunk size is really small + block_size = max(chunksize // 32, 16 << 10) + while True: + batch = f.read(chunksize) + if not batch: + break + # Finish current line + try: + batch += f.readline() + except (AttributeError, io.UnsupportedOperation): + batch += readline(f) + while True: + try: + pa_table = paj.read_json( + io.BytesIO(batch), read_options=paj.ReadOptions(block_size=block_size) + ) + break + except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e: + if ( + isinstance(e, pa.ArrowInvalid) + and "straddling" not in str(e) + or block_size > len(batch) + ): + raise + else: + # Increase the block size in case it was too small. + # The block size will be reset for the next file. + logger.debug( + f"Batch of {len(batch)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}." + ) + block_size *= 2 + if self.config.filters is not None: + pa_table = pa_table.filter(filter_expr) + if len(pa_table) > 0: + yield pa_table + else: + with open(metadata_file, "rb") as f: + parquet_fragment = ds.ParquetFileFormat().make_fragment(f) + if parquet_fragment.row_groups: + batch_size = parquet_fragment.row_groups[0].num_rows + else: + batch_size = config.DEFAULT_MAX_BATCH_SIZE + for record_batch in parquet_fragment.to_batches( + batch_size=batch_size, + filter=filter_expr, + batch_readahead=0, + fragment_readahead=0, + ): + yield pa.Table.from_batches([record_batch]) + + def _generate_shards(self, files, metadata_files, add_metadata, add_labels): + if add_metadata: + for _, _, downloaded_metadata_file in metadata_files: + yield downloaded_metadata_file + else: + for _, downloaded_dir, downloaded_file in files: + yield downloaded_dir or downloaded_file + + def _generate_examples(self, files, metadata_files, add_metadata, add_labels): + if add_metadata: + feature_paths = [] + + def find_feature_path(feature, feature_path): + nonlocal feature_paths + if feature_path and isinstance(feature, self.BASE_FEATURE): + feature_paths.append(feature_path) + + _visit_with_path(self.info.features, find_feature_path) + + for shard_idx, metadata_file_info in enumerate(metadata_files): + if len(metadata_file_info) == 2: + original_metadata_file, downloaded_metadata_file = metadata_file_info + else: + original_metadata_file, downloaded_metadata_dir, downloaded_metadata_file = metadata_file_info + metadata_ext = os.path.splitext(original_metadata_file or downloaded_metadata_file)[-1] + downloaded_metadata_dir = os.path.dirname(downloaded_metadata_file) + + def set_feature(item, feature_path: _VisitPath): + if len(feature_path) == 2 and isinstance(feature_path[0], str) and feature_path[1] == 0: + item[feature_path[0]] = item.pop("file_names", None) or item.pop( + feature_path[0] + "_file_names", None + ) + elif len(feature_path) == 1 and isinstance(feature_path[0], str): + item[feature_path[0]] = item.pop("file_name", None) or item.pop( + feature_path[0] + "_file_name", None + ) + elif len(feature_path) == 0: + if item is not None: + file_relpath = os.path.normpath(item).replace("\\", "/") + item = os.path.join(downloaded_metadata_dir, file_relpath) + return item + + for pa_metadata_table in self._read_metadata(downloaded_metadata_file, metadata_ext=metadata_ext): + for sample_idx, sample in enumerate(pa_metadata_table.to_pylist()): + for feature_path in feature_paths: + _nested_apply(sample, feature_path, set_feature) + yield Key(shard_idx, sample_idx), sample + else: + if self.config.filters is not None: + filter_expr = ( + pq.filters_to_expression(self.config.filters) + if isinstance(self.config.filters, list) + else self.config.filters + ) + for shard_idx, (original_file, _, downloaded_files) in enumerate(files): + if isinstance(downloaded_files, str): + downloaded_files = [downloaded_files] + for sample_idx, downloaded_file in enumerate(downloaded_files): + sample = {self.BASE_COLUMN_NAME: downloaded_file} + if add_labels: + sample["label"] = os.path.basename(os.path.dirname(original_file or downloaded_file)) + if self.config.filters is not None: + pa_table = pa.Table.from_pylist([sample]).filter(filter_expr) + if len(pa_table) == 0: + continue + yield Key(shard_idx, sample_idx), sample + + +def _nested_apply(item: Any, feature_path: _VisitPath, func: Callable[[Any, _VisitPath], Any]): + # see _visit_with_path() to see how feature paths are constructed + item = func(item, feature_path) + if feature_path: + key = feature_path[0] + if key == 0: + for i in range(len(item)): + item[i] = _nested_apply(item[i], feature_path[1:], func) + else: + item[key] = _nested_apply(item[key], feature_path[1:], func) + return item diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/generator/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/generator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/generator/generator.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/generator/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..e20cada94efd6cf33bbe02b05438eced4e967f54 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/generator/generator.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass +from typing import Callable, Optional + +import datasets +from datasets.builder import Key +from datasets.utils.sharding import _number_of_shards_in_gen_kwargs, _split_gen_kwargs + + +@dataclass +class GeneratorConfig(datasets.BuilderConfig): + generator: Optional[Callable] = None + gen_kwargs: Optional[dict] = None + features: Optional[datasets.Features] = None + split: datasets.NamedSplit = datasets.Split.TRAIN + + def __post_init__(self): + super().__post_init__() + if self.generator is None: + raise ValueError("generator must be specified") + + if self.gen_kwargs is None: + self.gen_kwargs = {} + + +class Generator(datasets.GeneratorBasedBuilder): + BUILDER_CONFIG_CLASS = GeneratorConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + return [datasets.SplitGenerator(name=self.config.split, gen_kwargs=self.config.gen_kwargs)] + + def _generate_examples(self, **gen_kwargs): + num_shards = _number_of_shards_in_gen_kwargs(gen_kwargs) + for shard_idx, shard_gen_kwargs in enumerate(_split_gen_kwargs(gen_kwargs, max_num_jobs=num_shards)): + for sample_idx, sample in enumerate(self.config.generator(**shard_gen_kwargs)): + yield Key(shard_idx, sample_idx), sample diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/hdf5/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/hdf5/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/imagefolder/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/imagefolder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py new file mode 100644 index 0000000000000000000000000000000000000000..aba9452ad4e12833c60ef78df8b3a0cae519f4a4 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py @@ -0,0 +1,103 @@ +import datasets + +from ..folder_based_builder import folder_based_builder + + +logger = datasets.utils.logging.get_logger(__name__) + + +class ImageFolderConfig(folder_based_builder.FolderBasedBuilderConfig): + """BuilderConfig for ImageFolder.""" + + drop_labels: bool = None + drop_metadata: bool = None + + def __post_init__(self): + super().__post_init__() + + +class ImageFolder(folder_based_builder.FolderBasedBuilder): + BASE_FEATURE = datasets.Image + BASE_COLUMN_NAME = "image" + BUILDER_CONFIG_CLASS = ImageFolderConfig + EXTENSIONS: list[str] # definition at the bottom of the script + + +# Obtained with: +# ``` +# import PIL.Image +# IMAGE_EXTENSIONS = [] +# PIL.Image.init() +# for ext, format in PIL.Image.EXTENSION.items(): +# if format in PIL.Image.OPEN: +# IMAGE_EXTENSIONS.append(ext[1:]) +# ``` +# We intentionally do not run this code on launch because: +# (1) Pillow is an optional dependency, so importing Pillow in global namespace is not allowed +# (2) To ensure the list of supported extensions is deterministic +IMAGE_EXTENSIONS = [ + ".blp", + ".bmp", + ".dib", + ".bufr", + ".cur", + ".pcx", + ".dcx", + ".dds", + ".ps", + ".eps", + ".fit", + ".fits", + ".fli", + ".flc", + ".ftc", + ".ftu", + ".gbr", + ".gif", + ".grib", + # ".h5", # may contain zero or several images + # ".hdf", # may contain zero or several images + ".png", + ".apng", + ".jp2", + ".j2k", + ".jpc", + ".jpf", + ".jpx", + ".j2c", + ".icns", + ".ico", + ".im", + ".iim", + ".tif", + ".tiff", + ".jfif", + ".jpe", + ".jpg", + ".jpeg", + ".mpg", + ".mpeg", + ".msp", + ".pcd", + ".pxr", + ".pbm", + ".pgm", + ".ppm", + ".pnm", + ".psd", + ".bw", + ".rgb", + ".rgba", + ".sgi", + ".ras", + ".tga", + ".icb", + ".vda", + ".vst", + ".webp", + ".wmf", + ".emf", + ".xbm", + ".xpm", +] +ImageFolder.EXTENSIONS = IMAGE_EXTENSIONS diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/pandas/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/pandas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/pandas/pandas.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/pandas/pandas.py new file mode 100644 index 0000000000000000000000000000000000000000..13577949f0621781600f8eb88b67b6cab1527eb3 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/pandas/pandas.py @@ -0,0 +1,57 @@ +import warnings +from dataclasses import dataclass +from typing import Optional + +import pandas as pd +import pyarrow as pa + +import datasets +from datasets.builder import Key +from datasets.table import table_cast + + +@dataclass +class PandasConfig(datasets.BuilderConfig): + """BuilderConfig for Pandas.""" + + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + + +class Pandas(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = PandasConfig + + def _info(self): + warnings.warn( + "The Pandas builder is deprecated and will be removed in the next major version of datasets.", + FutureWarning, + ) + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + data_files = dl_manager.download(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + # more expensive cast to support nested features with keys in a different order + # allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, self.config.features.arrow_schema) + return pa_table + + def _generate_shards(self, files): + yield from files + + def _generate_tables(self, files): + for i, file in enumerate(files): + with open(file, "rb") as f: + pa_table = pa.Table.from_pandas(pd.read_pickle(f)) + yield Key(i, 0), self._cast_table(pa_table) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/pdffolder/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/pdffolder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py new file mode 100644 index 0000000000000000000000000000000000000000..5aaf9c71a4df6551a53a2625b332bc51a2f02837 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/pdffolder/pdffolder.py @@ -0,0 +1,23 @@ +import datasets + +from ..folder_based_builder import folder_based_builder + + +logger = datasets.utils.logging.get_logger(__name__) + + +class PdfFolderConfig(folder_based_builder.FolderBasedBuilderConfig): + """BuilderConfig for ImageFolder.""" + + drop_labels: bool = None + drop_metadata: bool = None + + def __post_init__(self): + super().__post_init__() + + +class PdfFolder(folder_based_builder.FolderBasedBuilder): + BASE_FEATURE = datasets.Pdf + BASE_COLUMN_NAME = "pdf" + BUILDER_CONFIG_CLASS = PdfFolderConfig + EXTENSIONS: list[str] = [".pdf"] diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/spark/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/spark/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/sql/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/sql/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/sql/sql.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/sql/sql.py new file mode 100644 index 0000000000000000000000000000000000000000..deebd41428d8f89d74a75e67729be1dc1ed1a468 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/sql/sql.py @@ -0,0 +1,120 @@ +import sys +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Union + +import pandas as pd +import pyarrow as pa + +import datasets +import datasets.config +from datasets.builder import Key +from datasets.features.features import require_storage_cast +from datasets.table import table_cast + + +if TYPE_CHECKING: + import sqlite3 + + import sqlalchemy + + +logger = datasets.utils.logging.get_logger(__name__) + + +@dataclass +class SqlConfig(datasets.BuilderConfig): + """BuilderConfig for SQL.""" + + sql: Union[str, "sqlalchemy.sql.Selectable"] = None + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"] = None + index_col: Optional[Union[str, list[str]]] = None + coerce_float: bool = True + params: Optional[Union[list, tuple, dict]] = None + parse_dates: Optional[Union[list, dict]] = None + columns: Optional[list[str]] = None + chunksize: Optional[int] = 10_000 + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + if self.sql is None: + raise ValueError("sql must be specified") + if self.con is None: + raise ValueError("con must be specified") + + def create_config_id( + self, + config_kwargs: dict, + custom_features: Optional[datasets.Features] = None, + ) -> str: + config_kwargs = config_kwargs.copy() + # We need to stringify the Selectable object to make its hash deterministic + + # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html + sql = config_kwargs["sql"] + if not isinstance(sql, str): + if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules: + import sqlalchemy + + if isinstance(sql, sqlalchemy.sql.Selectable): + engine = sqlalchemy.create_engine(config_kwargs["con"].split("://")[0] + "://") + sql_str = str(sql.compile(dialect=engine.dialect)) + config_kwargs["sql"] = sql_str + else: + raise TypeError( + f"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(sql)}: {sql}" + ) + else: + raise TypeError( + f"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(sql)}: {sql}" + ) + con = config_kwargs["con"] + if not isinstance(con, str): + config_kwargs["con"] = id(con) + logger.info( + f"SQL connection 'con' of type {type(con)} couldn't be hashed properly. To enable hashing, specify 'con' as URI string instead." + ) + + return super().create_config_id(config_kwargs, custom_features=custom_features) + + @property + def pd_read_sql_kwargs(self): + pd_read_sql_kwargs = { + "index_col": self.index_col, + "columns": self.columns, + "params": self.params, + "coerce_float": self.coerce_float, + "parse_dates": self.parse_dates, + } + return pd_read_sql_kwargs + + +class Sql(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = SqlConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={})] + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + schema = self.config.features.arrow_schema + if all(not require_storage_cast(feature) for feature in self.config.features.values()): + # cheaper cast + pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema) + else: + # more expensive cast; allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, schema) + return pa_table + + def _generate_tables(self): + chunksize = self.config.chunksize + sql_reader = pd.read_sql( + self.config.sql, self.config.con, chunksize=chunksize, **self.config.pd_read_sql_kwargs + ) + sql_reader = [sql_reader] if chunksize is None else sql_reader + for chunk_idx, df in enumerate(sql_reader): + pa_table = pa.Table.from_pandas(df) + yield Key(0, chunk_idx), self._cast_table(pa_table) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/videofolder/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/videofolder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/videofolder/videofolder.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/videofolder/videofolder.py new file mode 100644 index 0000000000000000000000000000000000000000..099a17a8d09e981b34c4a40a3f5e8c948d0efa5d --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/videofolder/videofolder.py @@ -0,0 +1,34 @@ +import datasets + +from ..folder_based_builder import folder_based_builder + + +logger = datasets.utils.logging.get_logger(__name__) + + +class VideoFolderConfig(folder_based_builder.FolderBasedBuilderConfig): + """BuilderConfig for ImageFolder.""" + + drop_labels: bool = None + drop_metadata: bool = None + + def __post_init__(self): + super().__post_init__() + + +class VideoFolder(folder_based_builder.FolderBasedBuilder): + BASE_FEATURE = datasets.Video + BASE_COLUMN_NAME = "video" + BUILDER_CONFIG_CLASS = VideoFolderConfig + EXTENSIONS: list[str] # definition at the bottom of the script + + +# TODO: initial list, we should check the compatibility of other formats +VIDEO_EXTENSIONS = [ + ".mkv", + ".mp4", + ".avi", + ".mpeg", + ".mov", +] +VideoFolder.EXTENSIONS = VIDEO_EXTENSIONS diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/xml/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/xml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/xml/xml.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/xml/xml.py new file mode 100644 index 0000000000000000000000000000000000000000..769ca7c6e0094a16d297b1c0976facedf4a35801 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/xml/xml.py @@ -0,0 +1,70 @@ +from dataclasses import dataclass +from typing import Optional + +import pyarrow as pa + +import datasets +from datasets.features.features import require_storage_cast +from datasets.table import table_cast + + +logger = datasets.utils.logging.get_logger(__name__) + + +@dataclass +class XmlConfig(datasets.BuilderConfig): + """BuilderConfig for xml files.""" + + features: Optional[datasets.Features] = None + encoding: str = "utf-8" + encoding_errors: Optional[str] = None + + +class Xml(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = XmlConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """The `data_files` kwarg in load_dataset() can be a str, List[str], Dict[str,str], or Dict[str,List[str]]. + + If str or List[str], then the dataset returns only the 'train' split. + If dict, then keys should be from the `datasets.Split` enum. + """ + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + files = [dl_manager.iter_files(file) for file in files] + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + schema = self.config.features.arrow_schema + if all(not require_storage_cast(feature) for feature in self.config.features.values()): + # cheaper cast + pa_table = pa_table.cast(schema) + else: + # more expensive cast; allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, schema) + return pa_table + else: + return pa_table.cast(pa.schema({"xml": pa.string()})) + + def _generate_shards(self, files): + yield from files + + def _generate_tables(self, files): + pa_table_names = list(self.config.features) if self.config.features is not None else ["xml"] + for file_idx, file in enumerate(files): + # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n" + with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f: + xml = f.read() + pa_table = pa.Table.from_arrays([pa.array([xml])], names=pa_table_names) + yield (file_idx, 0), self._cast_table(pa_table) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/search.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/search.py new file mode 100644 index 0000000000000000000000000000000000000000..cb994d24e41e96174386de79057441e4436cc19a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/search.py @@ -0,0 +1,785 @@ +import importlib.util +import os +import tempfile +from pathlib import PurePath +from typing import TYPE_CHECKING, NamedTuple, Optional, Union + +import fsspec +import numpy as np + +from .features import List +from .utils import logging +from .utils import tqdm as hf_tqdm + + +if TYPE_CHECKING: + from .arrow_dataset import Dataset # noqa: F401 + + try: + from elasticsearch import Elasticsearch # noqa: F401 + + except ImportError: + pass + try: + import faiss # noqa: F401 + + except ImportError: + pass + +_has_elasticsearch = importlib.util.find_spec("elasticsearch") is not None +_has_faiss = importlib.util.find_spec("faiss") is not None + + +logger = logging.get_logger(__name__) + + +class MissingIndex(Exception): + pass + + +class SearchResults(NamedTuple): + scores: list[float] + indices: list[int] + + +class BatchedSearchResults(NamedTuple): + total_scores: list[list[float]] + total_indices: list[list[int]] + + +class NearestExamplesResults(NamedTuple): + scores: list[float] + examples: dict + + +class BatchedNearestExamplesResults(NamedTuple): + total_scores: list[list[float]] + total_examples: list[dict] + + +class BaseIndex: + """Base class for indexing""" + + def search(self, query, k: int = 10, **kwargs) -> SearchResults: + """ + To implement. + This method has to return the scores and the indices of the retrieved examples given a certain query. + """ + raise NotImplementedError + + def search_batch(self, queries, k: int = 10, **kwargs) -> BatchedSearchResults: + """Find the nearest examples indices to the query. + + Args: + queries (`Union[List[str], np.ndarray]`): The queries as a list of strings if `column` is a text index or as a numpy array if `column` is a vector index. + k (`int`): The number of examples to retrieve per query. + + Output: + total_scores (`List[List[float]`): The retrieval scores of the retrieved examples per query. + total_indices (`List[List[int]]`): The indices of the retrieved examples per query. + """ + total_scores, total_indices = [], [] + for query in queries: + scores, indices = self.search(query, k) + total_scores.append(scores) + total_indices.append(indices) + return BatchedSearchResults(total_scores, total_indices) + + def save(self, file: Union[str, PurePath]): + """Serialize the index on disk""" + raise NotImplementedError + + @classmethod + def load(cls, file: Union[str, PurePath]) -> "BaseIndex": + """Deserialize the index from disk""" + raise NotImplementedError + + +class ElasticSearchIndex(BaseIndex): + """ + Sparse index using Elasticsearch. It is used to index text and run queries based on BM25 similarity. + An Elasticsearch server needs to be accessible, and a python client is declared with + ``` + es_client = Elasticsearch([{'host': 'localhost', 'port': '9200'}]) + ``` + for example. + """ + + def __init__( + self, + host: Optional[str] = None, + port: Optional[int] = None, + es_client: Optional["Elasticsearch"] = None, + es_index_name: Optional[str] = None, + es_index_config: Optional[dict] = None, + ): + if not _has_elasticsearch: + raise ImportError( + "You must install ElasticSearch to use ElasticSearchIndex. To do so you can run `pip install elasticsearch==7.7.1 for example`" + ) + if es_client is not None and (host is not None or port is not None): + raise ValueError("Please specify either `es_client` or `(host, port)`, but not both.") + host = host or "localhost" + port = port or 9200 + + import elasticsearch.helpers # noqa: F401 - need this to properly load all the es features + from elasticsearch import Elasticsearch # noqa: F811 + + self.es_client = es_client if es_client is not None else Elasticsearch([{"host": host, "port": str(port)}]) + self.es_index_name = ( + es_index_name + if es_index_name is not None + else "huggingface_datasets_" + os.path.basename(tempfile.NamedTemporaryFile().name) + ) + self.es_index_config = ( + es_index_config + if es_index_config is not None + else { + "settings": { + "number_of_shards": 1, + "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}}, + }, + "mappings": {"properties": {"text": {"type": "text", "analyzer": "standard", "similarity": "BM25"}}}, + } + ) + + def add_documents(self, documents: Union[list[str], "Dataset"], column: Optional[str] = None): + """ + Add documents to the index. + If the documents are inside a certain column, you can specify it using the `column` argument. + """ + index_name = self.es_index_name + index_config = self.es_index_config + self.es_client.indices.create(index=index_name, body=index_config) + number_of_docs = len(documents) + progress = hf_tqdm(unit="docs", total=number_of_docs) + successes = 0 + + def passage_generator(): + if column is not None: + for i, example in enumerate(documents): + yield {"text": example[column], "_id": i} + else: + for i, example in enumerate(documents): + yield {"text": example, "_id": i} + + # create the ES index + import elasticsearch as es + + for ok, action in es.helpers.streaming_bulk( + client=self.es_client, + index=index_name, + actions=passage_generator(), + ): + progress.update(1) + successes += ok + if successes != len(documents): + logger.warning( + f"Some documents failed to be added to ElasticSearch. Failures: {len(documents) - successes}/{len(documents)}" + ) + logger.info(f"Indexed {successes:d} documents") + + def search(self, query: str, k=10, **kwargs) -> SearchResults: + """Find the nearest examples indices to the query. + + Args: + query (`str`): The query as a string. + k (`int`): The number of examples to retrieve. + + Output: + scores (`List[List[float]`): The retrieval scores of the retrieved examples. + indices (`List[List[int]]`): The indices of the retrieved examples. + """ + response = self.es_client.search( + index=self.es_index_name, + body={"query": {"multi_match": {"query": query, "fields": ["text"], "type": "cross_fields"}}, "size": k}, + **kwargs, + ) + hits = response["hits"]["hits"] + return SearchResults([hit["_score"] for hit in hits], [int(hit["_id"]) for hit in hits]) + + def search_batch(self, queries, k: int = 10, max_workers=10, **kwargs) -> BatchedSearchResults: + import concurrent.futures + + total_scores, total_indices = [None] * len(queries), [None] * len(queries) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_index = {executor.submit(self.search, query, k, **kwargs): i for i, query in enumerate(queries)} + for future in concurrent.futures.as_completed(future_to_index): + index = future_to_index[future] + results: SearchResults = future.result() + total_scores[index] = results.scores + total_indices[index] = results.indices + return BatchedSearchResults(total_indices=total_indices, total_scores=total_scores) + + +class FaissIndex(BaseIndex): + """ + Dense index using Faiss. It is used to index vectors. + Faiss is a library for efficient similarity search and clustering of dense vectors. + It contains algorithms that search in sets of vectors of any size, up to ones that possibly do not fit in RAM. + You can find more information about Faiss here: + - For index types and the string factory: https://github.com/facebookresearch/faiss/wiki/The-index-factory + - For GPU settings: https://github.com/facebookresearch/faiss/wiki/Faiss-on-the-GPU + """ + + def __init__( + self, + device: Optional[Union[int, list[int]]] = None, + string_factory: Optional[str] = None, + metric_type: Optional[int] = None, + custom_index: Optional["faiss.Index"] = None, + ): + """ + Create a Dense index using Faiss. You can specify `device` if you want to run it on GPU (`device` must be the GPU index). + You can find more information about Faiss here: + - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory + """ + if string_factory is not None and custom_index is not None: + raise ValueError("Please specify either `string_factory` or `custom_index` but not both.") + if device is not None and custom_index is not None: + raise ValueError( + "Cannot pass both 'custom_index' and 'device'. " + "Pass 'custom_index' already transferred to the target device instead." + ) + self.device = device + self.string_factory = string_factory + self.metric_type = metric_type + self.faiss_index = custom_index + if not _has_faiss: + raise ImportError( + "You must install Faiss to use FaissIndex. To do so you can run `conda install -c pytorch faiss-cpu` or `conda install -c pytorch faiss-gpu`. " + "A community supported package is also available on pypi: `pip install faiss-cpu` or `pip install faiss-gpu`. " + "Note that pip may not have the latest version of FAISS, and thus, some of the latest features and bug fixes may not be available." + ) + + def add_vectors( + self, + vectors: Union[np.array, "Dataset"], + column: Optional[str] = None, + batch_size: int = 1000, + train_size: Optional[int] = None, + faiss_verbose: Optional[bool] = None, + ): + """ + Add vectors to the index. + If the arrays are inside a certain column, you can specify it using the `column` argument. + """ + import faiss # noqa: F811 + + if column and not isinstance(vectors.features[column], List): + raise ValueError( + f"Wrong feature type for column '{column}'. Expected 1d array, got {vectors.features[column]}" + ) + + # Create index + if self.faiss_index is None: + size = len(vectors[0]) if column is None else len(vectors[0][column]) + if self.string_factory is not None: + if self.metric_type is None: + index = faiss.index_factory(size, self.string_factory) + else: + index = faiss.index_factory(size, self.string_factory, self.metric_type) + else: + if self.metric_type is None: + index = faiss.IndexFlat(size) + else: + index = faiss.IndexFlat(size, self.metric_type) + + self.faiss_index = self._faiss_index_to_device(index, self.device) + logger.info(f"Created faiss index of type {type(self.faiss_index)}") + + # Set verbosity level + if faiss_verbose is not None: + self.faiss_index.verbose = faiss_verbose + if hasattr(self.faiss_index, "index") and self.faiss_index.index is not None: + self.faiss_index.index.verbose = faiss_verbose + if hasattr(self.faiss_index, "quantizer") and self.faiss_index.quantizer is not None: + self.faiss_index.quantizer.verbose = faiss_verbose + if hasattr(self.faiss_index, "clustering_index") and self.faiss_index.clustering_index is not None: + self.faiss_index.clustering_index.verbose = faiss_verbose + + # Train + if train_size is not None: + train_vecs = vectors[:train_size] if column is None else vectors[:train_size][column] + logger.info(f"Training the index with the first {len(train_vecs)} vectors") + self.faiss_index.train(train_vecs) + else: + logger.info("Ignored the training step of the faiss index as `train_size` is None.") + + # Add vectors + logger.info(f"Adding {len(vectors)} vectors to the faiss index") + for i in hf_tqdm(range(0, len(vectors), batch_size)): + vecs = vectors[i : i + batch_size] if column is None else vectors[i : i + batch_size][column] + self.faiss_index.add(vecs) + + @staticmethod + def _faiss_index_to_device(index: "faiss.Index", device: Optional[Union[int, list[int]]] = None) -> "faiss.Index": + """ + Sends a faiss index to a device. + A device can either be a positive integer (GPU id), a negative integer (all GPUs), + or a list of positive integers (select GPUs to use), or `None` for CPU. + """ + + # If device is not specified, then it runs on CPU. + if device is None: + return index + + import faiss # noqa: F811 + + # If the device id is given as an integer + if isinstance(device, int): + # Positive integers are directly mapped to GPU ids + if device > -1: + faiss_res = faiss.StandardGpuResources() + index = faiss.index_cpu_to_gpu(faiss_res, device, index) + # And negative integers mean using all GPUs + else: + index = faiss.index_cpu_to_all_gpus(index) + # Device ids given as a list mean mapping to those devices specified. + elif isinstance(device, (list, tuple)): + index = faiss.index_cpu_to_gpus_list(index, gpus=list(device)) + else: + raise TypeError( + f"The argument type: {type(device)} is not expected. " + + "Please pass in either nothing, a positive int, a negative int, or a list of positive ints." + ) + + return index + + def search(self, query: np.array, k=10, **kwargs) -> SearchResults: + """Find the nearest examples indices to the query. + + Args: + query (`np.array`): The query as a numpy array. + k (`int`): The number of examples to retrieve. + + Output: + scores (`List[List[float]`): The retrieval scores of the retrieved examples. + indices (`List[List[int]]`): The indices of the retrieved examples. + """ + if len(query.shape) != 1 and (len(query.shape) != 2 or query.shape[0] != 1): + raise ValueError("Shape of query is incorrect, it has to be either a 1D array or 2D (1, N)") + + queries = query.reshape(1, -1) + if not queries.flags.c_contiguous: + queries = np.asarray(queries, order="C") + scores, indices = self.faiss_index.search(queries, k, **kwargs) + return SearchResults(scores[0], indices[0].astype(int)) + + def search_batch(self, queries: np.array, k=10, **kwargs) -> BatchedSearchResults: + """Find the nearest examples indices to the queries. + + Args: + queries (`np.array`): The queries as a numpy array. + k (`int`): The number of examples to retrieve. + + Output: + total_scores (`List[List[float]`): The retrieval scores of the retrieved examples per query. + total_indices (`List[List[int]]`): The indices of the retrieved examples per query. + """ + if len(queries.shape) != 2: + raise ValueError("Shape of query must be 2D") + if not queries.flags.c_contiguous: + queries = np.asarray(queries, order="C") + scores, indices = self.faiss_index.search(queries, k, **kwargs) + return BatchedSearchResults(scores, indices.astype(int)) + + def save(self, file: Union[str, PurePath], storage_options: Optional[dict] = None): + """Serialize the FaissIndex on disk""" + import faiss # noqa: F811 + + if self.device is not None and isinstance(self.device, (int, list, tuple)): + index = faiss.index_gpu_to_cpu(self.faiss_index) + else: + index = self.faiss_index + + with fsspec.open(str(file), "wb", **(storage_options or {})) as f: + faiss.write_index(index, faiss.BufferedIOWriter(faiss.PyCallbackIOWriter(f.write))) + + @classmethod + def load( + cls, + file: Union[str, PurePath], + device: Optional[Union[int, list[int]]] = None, + storage_options: Optional[dict] = None, + ) -> "FaissIndex": + """Deserialize the FaissIndex from disk""" + import faiss # noqa: F811 + + # Instances of FaissIndex is essentially just a wrapper for faiss indices. + faiss_index = cls(device=device) + with fsspec.open(str(file), "rb", **(storage_options or {})) as f: + index = faiss.read_index(faiss.BufferedIOReader(faiss.PyCallbackIOReader(f.read))) + faiss_index.faiss_index = faiss_index._faiss_index_to_device(index, faiss_index.device) + return faiss_index + + +class IndexableMixin: + """Add indexing features to `datasets.Dataset`""" + + def __init__(self): + self._indexes: dict[str, BaseIndex] = {} + + def __len__(self): + raise NotImplementedError + + def __getitem__(self, key): + raise NotImplementedError + + def is_index_initialized(self, index_name: str) -> bool: + return index_name in self._indexes + + def _check_index_is_initialized(self, index_name: str): + if not self.is_index_initialized(index_name): + raise MissingIndex( + f"Index with index_name '{index_name}' not initialized yet. Please make sure that you call `add_faiss_index` or `add_elasticsearch_index` first." + ) + + def list_indexes(self) -> list[str]: + """List the `colindex_nameumns`/identifiers of all the attached indexes.""" + return list(self._indexes) + + def get_index(self, index_name: str) -> BaseIndex: + """List the `index_name`/identifiers of all the attached indexes. + + Args: + index_name (`str`): Index name. + + Returns: + [`BaseIndex`] + """ + self._check_index_is_initialized(index_name) + return self._indexes[index_name] + + def add_faiss_index( + self, + column: str, + index_name: Optional[str] = None, + device: Optional[Union[int, list[int]]] = None, + string_factory: Optional[str] = None, + metric_type: Optional[int] = None, + custom_index: Optional["faiss.Index"] = None, + batch_size: int = 1000, + train_size: Optional[int] = None, + faiss_verbose: bool = False, + ): + """Add a dense index using Faiss for fast retrieval. + The index is created using the vectors of the specified column. + You can specify `device` if you want to run it on GPU (`device` must be the GPU index, see more below). + You can find more information about Faiss here: + - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory + + Args: + column (`str`): The column of the vectors to add to the index. + index_name (Optional `str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`. + By default it corresponds to `column`. + device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs. + If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU. + string_factory (Optional `str`): This is passed to the index factory of Faiss to create the index. Default index class is IndexFlatIP. + metric_type (Optional `int`): Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`. + custom_index (Optional `faiss.Index`): Custom Faiss index that you already have instantiated and configured for your needs. + batch_size (Optional `int`): Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000. + + train_size (Optional `int`): If the index needs a training step, specifies how many vectors will be used to train the index. + faiss_verbose (`bool`, defaults to False): Enable the verbosity of the Faiss index. + """ + index_name = index_name if index_name is not None else column + faiss_index = FaissIndex( + device=device, string_factory=string_factory, metric_type=metric_type, custom_index=custom_index + ) + faiss_index.add_vectors( + self, column=column, batch_size=batch_size, train_size=train_size, faiss_verbose=faiss_verbose + ) + self._indexes[index_name] = faiss_index + + def add_faiss_index_from_external_arrays( + self, + external_arrays: np.array, + index_name: str, + device: Optional[Union[int, list[int]]] = None, + string_factory: Optional[str] = None, + metric_type: Optional[int] = None, + custom_index: Optional["faiss.Index"] = None, + batch_size: int = 1000, + train_size: Optional[int] = None, + faiss_verbose: bool = False, + ): + """Add a dense index using Faiss for fast retrieval. + The index is created using the vectors of `external_arrays`. + You can specify `device` if you want to run it on GPU (`device` must be the GPU index). + You can find more information about Faiss here: + - For `string factory`: https://github.com/facebookresearch/faiss/wiki/The-index-factory + + Args: + external_arrays (`np.array`): If you want to use arrays from outside the lib for the index, you can set `external_arrays`. + It will use `external_arrays` to create the Faiss index instead of the arrays in the given `column`. + index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`. + device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs. + If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU. + string_factory (Optional `str`): This is passed to the index factory of Faiss to create the index. Default index class is IndexFlatIP. + metric_type (Optional `int`): Type of metric. Ex: `faiss.METRIC_INNER_PRODUCT` or `faiss.METRIC_L2`. + custom_index (Optional `faiss.Index`): Custom Faiss index that you already have instantiated and configured for your needs. + batch_size (Optional `int`): Size of the batch to use while adding vectors to the FaissIndex. Default value is 1000. + + train_size (Optional `int`): If the index needs a training step, specifies how many vectors will be used to train the index. + faiss_verbose (`bool`, defaults to False): Enable the verbosity of the Faiss index. + """ + faiss_index = FaissIndex( + device=device, string_factory=string_factory, metric_type=metric_type, custom_index=custom_index + ) + faiss_index.add_vectors( + external_arrays, column=None, batch_size=batch_size, train_size=train_size, faiss_verbose=faiss_verbose + ) + self._indexes[index_name] = faiss_index + + def save_faiss_index(self, index_name: str, file: Union[str, PurePath], storage_options: Optional[dict] = None): + """Save a FaissIndex on disk. + + Args: + index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to call `.get_nearest` or `.search`. + file (`str`): The path to the serialized faiss index on disk or remote URI (e.g. `"s3://my-bucket/index.faiss"`). + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + """ + index = self.get_index(index_name) + if not isinstance(index, FaissIndex): + raise ValueError(f"Index '{index_name}' is not a FaissIndex but a '{type(index)}'") + index.save(file, storage_options=storage_options) + logger.info(f"Saved FaissIndex {index_name} at {file}") + + def load_faiss_index( + self, + index_name: str, + file: Union[str, PurePath], + device: Optional[Union[int, list[int]]] = None, + storage_options: Optional[dict] = None, + ): + """Load a FaissIndex from disk. + + If you want to do additional configurations, you can have access to the faiss index object by doing + `.get_index(index_name).faiss_index` to make it fit your needs. + + Args: + index_name (`str`): The index_name/identifier of the index. This is the index_name that is used to + call `.get_nearest` or `.search`. + file (`str`): The path to the serialized faiss index on disk or remote URI (e.g. `"s3://my-bucket/index.faiss"`). + device (Optional `Union[int, List[int]]`): If positive integer, this is the index of the GPU to use. If negative integer, use all GPUs. + If a list of positive integers is passed in, run only on those GPUs. By default it uses the CPU. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the file-system backend, if any. + + + + """ + index = FaissIndex.load(file, device=device, storage_options=storage_options) + if index.faiss_index.ntotal != len(self): + raise ValueError( + f"Index size should match Dataset size, but Index '{index_name}' at {file} has {index.faiss_index.ntotal} elements while the dataset has {len(self)} examples." + ) + self._indexes[index_name] = index + logger.info(f"Loaded FaissIndex {index_name} from {file}") + + def add_elasticsearch_index( + self, + column: str, + index_name: Optional[str] = None, + host: Optional[str] = None, + port: Optional[int] = None, + es_client: Optional["Elasticsearch"] = None, + es_index_name: Optional[str] = None, + es_index_config: Optional[dict] = None, + ): + """Add a text index using ElasticSearch for fast retrieval. + + Args: + column (`str`): The column of the documents to add to the index. + index_name (Optional `str`): The index_name/identifier of the index. This is the index name that is used to call `.get_nearest` or `.search`. + By default it corresponds to `column`. + host (Optional `str`, defaults to localhost): + host of where ElasticSearch is running + port (Optional `str`, defaults to 9200): + port of where ElasticSearch is running + es_client (Optional `elasticsearch.Elasticsearch`): + The elasticsearch client used to create the index if host and port are None. + es_index_name (Optional `str`): The elasticsearch index name used to create the index. + es_index_config (Optional `dict`): + The configuration of the elasticsearch index. + Default config is: + + Config:: + + { + "settings": { + "number_of_shards": 1, + "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}}, + }, + "mappings": { + "properties": { + "text": { + "type": "text", + "analyzer": "standard", + "similarity": "BM25" + }, + } + }, + } + """ + index_name = index_name if index_name is not None else column + es_index = ElasticSearchIndex( + host=host, port=port, es_client=es_client, es_index_name=es_index_name, es_index_config=es_index_config + ) + es_index.add_documents(self, column=column) + self._indexes[index_name] = es_index + + def load_elasticsearch_index( + self, + index_name: str, + es_index_name: str, + host: Optional[str] = None, + port: Optional[int] = None, + es_client: Optional["Elasticsearch"] = None, + es_index_config: Optional[dict] = None, + ): + """Load an existing text index using ElasticSearch for fast retrieval. + + Args: + index_name (`str`): + The `index_name`/identifier of the index. This is the index name that is used to call `get_nearest` or `search`. + es_index_name (`str`): + The name of elasticsearch index to load. + host (`str`, *optional*, defaults to `localhost`): + Host of where ElasticSearch is running. + port (`str`, *optional*, defaults to `9200`): + Port of where ElasticSearch is running. + es_client (`elasticsearch.Elasticsearch`, *optional*): + The elasticsearch client used to create the index if host and port are `None`. + es_index_config (`dict`, *optional*): + The configuration of the elasticsearch index. + Default config is: + ``` + { + "settings": { + "number_of_shards": 1, + "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}}, + }, + "mappings": { + "properties": { + "text": { + "type": "text", + "analyzer": "standard", + "similarity": "BM25" + }, + } + }, + } + ``` + """ + self._indexes[index_name] = ElasticSearchIndex( + host=host, port=port, es_client=es_client, es_index_name=es_index_name, es_index_config=es_index_config + ) + + def drop_index(self, index_name: str): + """Drop the index with the specified column. + + Args: + index_name (`str`): + The `index_name`/identifier of the index. + """ + del self._indexes[index_name] + + def search(self, index_name: str, query: Union[str, np.array], k: int = 10, **kwargs) -> SearchResults: + """Find the nearest examples indices in the dataset to the query. + + Args: + index_name (`str`): + The name/identifier of the index. + query (`Union[str, np.ndarray]`): + The query as a string if `index_name` is a text index or as a numpy array if `index_name` is a vector index. + k (`int`): + The number of examples to retrieve. + + Returns: + `(scores, indices)`: + A tuple of `(scores, indices)` where: + - **scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples + - **indices** (`List[List[int]]`): the indices of the retrieved examples + """ + self._check_index_is_initialized(index_name) + return self._indexes[index_name].search(query, k, **kwargs) + + def search_batch( + self, index_name: str, queries: Union[list[str], np.array], k: int = 10, **kwargs + ) -> BatchedSearchResults: + """Find the nearest examples indices in the dataset to the query. + + Args: + index_name (`str`): + The `index_name`/identifier of the index. + queries (`Union[List[str], np.ndarray]`): + The queries as a list of strings if `index_name` is a text index or as a numpy array if `index_name` is a vector index. + k (`int`): + The number of examples to retrieve per query. + + Returns: + `(total_scores, total_indices)`: + A tuple of `(total_scores, total_indices)` where: + - **total_scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples per query + - **total_indices** (`List[List[int]]`): the indices of the retrieved examples per query + """ + self._check_index_is_initialized(index_name) + return self._indexes[index_name].search_batch(queries, k, **kwargs) + + def get_nearest_examples( + self, index_name: str, query: Union[str, np.array], k: int = 10, **kwargs + ) -> NearestExamplesResults: + """Find the nearest examples in the dataset to the query. + + Args: + index_name (`str`): + The index_name/identifier of the index. + query (`Union[str, np.ndarray]`): + The query as a string if `index_name` is a text index or as a numpy array if `index_name` is a vector index. + k (`int`): + The number of examples to retrieve. + + Returns: + `(scores, examples)`: + A tuple of `(scores, examples)` where: + - **scores** (`List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples + - **examples** (`dict`): the retrieved examples + """ + self._check_index_is_initialized(index_name) + scores, indices = self.search(index_name, query, k, **kwargs) + top_indices = [i for i in indices if i >= 0] + return NearestExamplesResults(scores[: len(top_indices)], self[top_indices]) + + def get_nearest_examples_batch( + self, index_name: str, queries: Union[list[str], np.array], k: int = 10, **kwargs + ) -> BatchedNearestExamplesResults: + """Find the nearest examples in the dataset to the query. + + Args: + index_name (`str`): + The `index_name`/identifier of the index. + queries (`Union[List[str], np.ndarray]`): + The queries as a list of strings if `index_name` is a text index or as a numpy array if `index_name` is a vector index. + k (`int`): + The number of examples to retrieve per query. + + Returns: + `(total_scores, total_examples)`: + A tuple of `(total_scores, total_examples)` where: + - **total_scores** (`List[List[float]`): the retrieval scores from either FAISS (`IndexFlatL2` by default) or ElasticSearch of the retrieved examples per query + - **total_examples** (`List[dict]`): the retrieved examples per query + """ + self._check_index_is_initialized(index_name) + total_scores, total_indices = self.search_batch(index_name, queries, k, **kwargs) + total_scores = [ + scores_i[: len([i for i in indices_i if i >= 0])] + for scores_i, indices_i in zip(total_scores, total_indices) + ] + total_samples = [self[[i for i in indices if i >= 0]] for indices in total_indices] + return BatchedNearestExamplesResults(total_scores, total_samples) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/splits.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/splits.py new file mode 100644 index 0000000000000000000000000000000000000000..7e8ea953afdbb76240bafa272a33b27ef237359d --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/splits.py @@ -0,0 +1,647 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Splits related API.""" + +import abc +import collections +import copy +import dataclasses +import re +from dataclasses import dataclass +from typing import Optional, Union + +from .arrow_reader import FileInstructions, make_file_instructions +from .naming import _split_re +from .utils.py_utils import NonMutableDict, asdict + + +@dataclass +class SplitInfo: + name: str = dataclasses.field(default="", metadata={"include_in_asdict_even_if_is_default": True}) + num_bytes: int = dataclasses.field(default=0, metadata={"include_in_asdict_even_if_is_default": True}) + num_examples: int = dataclasses.field(default=0, metadata={"include_in_asdict_even_if_is_default": True}) + shard_lengths: Optional[list[int]] = None + original_shard_lengths: Optional[list[int]] = None + + # Deprecated + # For backward compatibility, this field needs to always be included in files like + # dataset_infos.json and dataset_info.json files + # To do so, we always include it in the output of datasets.utils.py_utils.asdict(split_info) + dataset_name: Optional[str] = dataclasses.field( + default=None, metadata={"include_in_asdict_even_if_is_default": True} + ) + + @property + def file_instructions(self): + """Returns the list of dict(filename, take, skip).""" + # `self.dataset_name` is assigned in `SplitDict.add()`. + instructions = make_file_instructions( + name=self.dataset_name, + split_infos=[self], + instruction=str(self.name), + ) + return instructions.file_instructions + + def __repr__(self): + return ( + self.__class__.__qualname__ + + "(" + + ", ".join( + [f"{f.name}={repr(getattr(self, f.name))}" for f in dataclasses.fields(self) if getattr(self, f.name)] + ) + + ")" + ) + + +@dataclass +class SubSplitInfo: + """Wrapper around a sub split info. + This class exposes info on the subsplit: + ``` + ds, info = datasets.load_dataset(..., split='train[75%:]', with_info=True) + info.splits['train[75%:]'].num_examples + ``` + """ + + instructions: FileInstructions + + @property + def num_examples(self): + """Returns the number of examples in the subsplit.""" + return self.instructions.num_examples + + @property + def file_instructions(self): + """Returns the list of dict(filename, take, skip).""" + return self.instructions.file_instructions + + +class SplitBase(metaclass=abc.ABCMeta): + # pylint: disable=line-too-long + """Abstract base class for Split compositionality. + + See the + [guide on splits](../loading#slice-splits) + for more information. + + There are three parts to the composition: + 1) The splits are composed (defined, merged, split,...) together before + calling the `.as_dataset()` function. This is done with the `__add__`, + `__getitem__`, which return a tree of `SplitBase` (whose leaf + are the `NamedSplit` objects) + + ``` + split = datasets.Split.TRAIN + datasets.Split.TEST.subsplit(datasets.percent[:50]) + ``` + + 2) The `SplitBase` is forwarded to the `.as_dataset()` function + to be resolved into actual read instruction. This is done by the + `.get_read_instruction()` method which takes the real dataset splits + (name, number of shards,...) and parse the tree to return a + `SplitReadInstruction()` object + + ``` + read_instruction = split.get_read_instruction(self.info.splits) + ``` + + 3) The `SplitReadInstruction` is then used in the `tf.data.Dataset` pipeline + to define which files to read and how to skip examples within file. + + """ + + # pylint: enable=line-too-long + + @abc.abstractmethod + def get_read_instruction(self, split_dict): + """Parse the descriptor tree and compile all read instructions together. + + Args: + split_dict: `dict`, The `dict[split_name, SplitInfo]` of the dataset + + Returns: + split_read_instruction: `SplitReadInstruction` + """ + raise NotImplementedError("Abstract method") + + def __eq__(self, other): + """Equality: datasets.Split.TRAIN == 'train'.""" + if isinstance(other, (NamedSplit, str)): + return False + raise NotImplementedError("Equality is not implemented between merged/sub splits.") + + def __ne__(self, other): + """InEquality: datasets.Split.TRAIN != 'test'.""" + return not self.__eq__(other) + + def __add__(self, other): + """Merging: datasets.Split.TRAIN + datasets.Split.TEST.""" + return _SplitMerged(self, other) + + def subsplit(self, arg=None, k=None, percent=None, weighted=None): # pylint: disable=redefined-outer-name + """Divides this split into subsplits. + + There are 3 ways to define subsplits, which correspond to the 3 + arguments `k` (get `k` even subsplits), `percent` (get a slice of the + dataset with `datasets.percent`), and `weighted` (get subsplits with proportions + specified by `weighted`). + + Example:: + + ``` + # 50% train, 50% test + train, test = split.subsplit(k=2) + # 50% train, 25% test, 25% validation + train, test, validation = split.subsplit(weighted=[2, 1, 1]) + # Extract last 20% + subsplit = split.subsplit(datasets.percent[-20:]) + ``` + + Warning: k and weighted will be converted into percent which mean that + values below the percent will be rounded up or down. The final split may be + bigger to deal with remainders. For instance: + + ``` + train, test, valid = split.subsplit(k=3) # 33%, 33%, 34% + s1, s2, s3, s4 = split.subsplit(weighted=[2, 2, 1, 1]) # 33%, 33%, 16%, 18% + ``` + + Args: + arg: If no kwargs are given, `arg` will be interpreted as one of + `k`, `percent`, or `weighted` depending on the type. + For example: + ``` + split.subsplit(10) # Equivalent to split.subsplit(k=10) + split.subsplit(datasets.percent[:-20]) # percent=datasets.percent[:-20] + split.subsplit([1, 1, 2]) # weighted=[1, 1, 2] + ``` + k: `int` If set, subdivide the split into `k` equal parts. + percent: `datasets.percent slice`, return a single subsplit corresponding to + a slice of the original split. For example: + `split.subsplit(datasets.percent[-20:]) # Last 20% of the dataset`. + weighted: `list[int]`, return a list of subsplits whose proportions match + the normalized sum of the list. For example: + `split.subsplit(weighted=[1, 1, 2]) # 25%, 25%, 50%`. + + Returns: + A subsplit or list of subsplits extracted from this split object. + """ + # Note that the percent kwargs redefine the outer name datasets.percent. This + # is done for consistency (.subsplit(percent=datasets.percent[:40])) + if sum(bool(x) for x in (arg, k, percent, weighted)) != 1: + raise ValueError("Only one argument of subsplit should be set.") + + # Auto deduce k + if isinstance(arg, int): + k = arg + elif isinstance(arg, slice): + percent = arg + elif isinstance(arg, list): + weighted = arg + + if not (k or percent or weighted): + raise ValueError( + f"Invalid split argument {arg}. Only list, slice and int supported. " + "One of k, weighted or percent should be set to a non empty value." + ) + + def assert_slices_coverage(slices): + # Ensure that the expended slices cover all percents. + assert sum((list(range(*s.indices(100))) for s in slices), []) == list(range(100)) + + if k: + if not 0 < k <= 100: + raise ValueError(f"Subsplit k should be between 0 and 100, got {k}") + shift = 100 // k + slices = [slice(i * shift, (i + 1) * shift) for i in range(k)] + # Round up last element to ensure all elements are taken + slices[-1] = slice(slices[-1].start, 100) + # Internal check to ensure full coverage + assert_slices_coverage(slices) + return tuple(_SubSplit(self, s) for s in slices) + elif percent: + return _SubSplit(self, percent) + elif weighted: + # Normalize the weighted sum + total = sum(weighted) + weighted = [100 * x // total for x in weighted] + # Create the slice for each of the elements + start = 0 + stop = 0 + slices = [] + for v in weighted: + stop += v + slices.append(slice(start, stop)) + start = stop + # Round up last element to ensure all elements are taken + slices[-1] = slice(slices[-1].start, 100) + # Internal check to ensure full coverage + assert_slices_coverage(slices) + return tuple(_SubSplit(self, s) for s in slices) + else: + # Should not be possible + raise ValueError("Could not determine the split") + + +# 2 requirements: +# 1. datasets.percent be sliceable +# 2. datasets.percent be documented +# +# Instances are not documented, so we want datasets.percent to be a class, but to +# have it be sliceable, we need this metaclass. +class PercentSliceMeta(type): + def __getitem__(cls, slice_value): + if not isinstance(slice_value, slice): + raise ValueError(f"datasets.percent should only be called with slice, not {slice_value}") + return slice_value + + +class PercentSlice(metaclass=PercentSliceMeta): + # pylint: disable=line-too-long + """Syntactic sugar for defining slice subsplits: `datasets.percent[75:-5]`. + + See the + [guide on splits](../loading#slice-splits) + for more information. + """ + + # pylint: enable=line-too-long + pass + + +percent = PercentSlice # pylint: disable=invalid-name + + +class _SplitMerged(SplitBase): + """Represent two split descriptors merged together.""" + + def __init__(self, split1, split2): + self._split1 = split1 + self._split2 = split2 + + def get_read_instruction(self, split_dict): + read_instruction1 = self._split1.get_read_instruction(split_dict) + read_instruction2 = self._split2.get_read_instruction(split_dict) + return read_instruction1 + read_instruction2 + + def __repr__(self): + return f"({repr(self._split1)} + {repr(self._split2)})" + + +class _SubSplit(SplitBase): + """Represent a sub split of a split descriptor.""" + + def __init__(self, split, slice_value): + self._split = split + self._slice_value = slice_value + + def get_read_instruction(self, split_dict): + return self._split.get_read_instruction(split_dict)[self._slice_value] + + def __repr__(self): + slice_str = "{start}:{stop}" + if self._slice_value.step is not None: + slice_str += ":{step}" + slice_str = slice_str.format( + start="" if self._slice_value.start is None else self._slice_value.start, + stop="" if self._slice_value.stop is None else self._slice_value.stop, + step=self._slice_value.step, + ) + return f"{repr(self._split)}(datasets.percent[{slice_str}])" + + +class NamedSplit(SplitBase): + """Descriptor corresponding to a named split (train, test, ...). + + Example: + Each descriptor can be composed with other using addition or slice: + + ```py + split = datasets.Split.TRAIN.subsplit(datasets.percent[0:25]) + datasets.Split.TEST + ``` + + The resulting split will correspond to 25% of the train split merged with + 100% of the test split. + + A split cannot be added twice, so the following will fail: + + ```py + split = ( + datasets.Split.TRAIN.subsplit(datasets.percent[:25]) + + datasets.Split.TRAIN.subsplit(datasets.percent[75:]) + ) # Error + split = datasets.Split.TEST + datasets.Split.ALL # Error + ``` + + The slices can be applied only one time. So the following are valid: + + ```py + split = ( + datasets.Split.TRAIN.subsplit(datasets.percent[:25]) + + datasets.Split.TEST.subsplit(datasets.percent[:50]) + ) + split = (datasets.Split.TRAIN + datasets.Split.TEST).subsplit(datasets.percent[:50]) + ``` + + But this is not valid: + + ```py + train = datasets.Split.TRAIN + test = datasets.Split.TEST + split = train.subsplit(datasets.percent[:25]).subsplit(datasets.percent[:25]) + split = (train.subsplit(datasets.percent[:25]) + test).subsplit(datasets.percent[:50]) + ``` + """ + + def __init__(self, name: str): + self._name = name + split_names_from_instruction = [split_instruction.split("[")[0] for split_instruction in name.split("+")] + for split_name in split_names_from_instruction: + if not re.match(_split_re, split_name): + raise ValueError(f"Split name should match '{_split_re}' but got '{split_name}'.") + + def __str__(self): + return self._name + + def __repr__(self): + return f"NamedSplit({self._name!r})" + + def __eq__(self, other): + """Equality: datasets.Split.TRAIN == 'train'.""" + if isinstance(other, NamedSplit): + return self._name == other._name # pylint: disable=protected-access + elif isinstance(other, SplitBase): + return False + elif isinstance(other, str): # Other should be string + return self._name == other + else: + return False + + def __lt__(self, other): + return self._name < other._name # pylint: disable=protected-access + + def __hash__(self): + return hash(self._name) + + def get_read_instruction(self, split_dict): + return SplitReadInstruction(split_dict[self._name]) + + +class NamedSplitAll(NamedSplit): + """Split corresponding to the union of all defined dataset splits.""" + + def __init__(self): + super().__init__("all") + + def __repr__(self): + return "NamedSplitAll()" + + def get_read_instruction(self, split_dict): + # Merge all dataset splits together + read_instructions = [SplitReadInstruction(s) for s in split_dict.values()] + return sum(read_instructions, SplitReadInstruction()) + + +class Split: + # pylint: disable=line-too-long + """`Enum` for dataset splits. + + Datasets are typically split into different subsets to be used at various + stages of training and evaluation. + + - `TRAIN`: the training data. + - `VALIDATION`: the validation data. If present, this is typically used as + evaluation data while iterating on a model (e.g. changing hyperparameters, + model architecture, etc.). + - `TEST`: the testing data. This is the data to report metrics on. Typically + you do not want to use this during model iteration as you may overfit to it. + - `ALL`: the union of all defined dataset splits. + + All splits, including compositions inherit from `datasets.SplitBase`. + + See the [guide](../load_hub#splits) on splits for more information. + + Example: + + ```py + >>> datasets.SplitGenerator( + ... name=datasets.Split.TRAIN, + ... gen_kwargs={"split_key": "train", "files": dl_manager.download_and extract(url)}, + ... ), + ... datasets.SplitGenerator( + ... name=datasets.Split.VALIDATION, + ... gen_kwargs={"split_key": "validation", "files": dl_manager.download_and extract(url)}, + ... ), + ... datasets.SplitGenerator( + ... name=datasets.Split.TEST, + ... gen_kwargs={"split_key": "test", "files": dl_manager.download_and extract(url)}, + ... ) + ``` + """ + + # pylint: enable=line-too-long + TRAIN = NamedSplit("train") + TEST = NamedSplit("test") + VALIDATION = NamedSplit("validation") + ALL = NamedSplitAll() + + def __new__(cls, name): + """Create a custom split with datasets.Split('custom_name').""" + return NamedSplitAll() if name == "all" else NamedSplit(name) + + +# Similar to SplitInfo, but contain an additional slice info +SlicedSplitInfo = collections.namedtuple( + "SlicedSplitInfo", + [ + "split_info", + "slice_value", + ], +) # noqa: E231 + + +class SplitReadInstruction: + """Object containing the reading instruction for the dataset. + + Similarly to `SplitDescriptor` nodes, this object can be composed with itself, + but the resolution happens instantaneously, instead of keeping track of the + tree, such as all instructions are compiled and flattened in a single + SplitReadInstruction object containing the list of files and slice to use. + + Once resolved, the instructions can be accessed with: + + ``` + read_instructions.get_list_sliced_split_info() # List of splits to use + ``` + + """ + + def __init__(self, split_info=None): + self._splits = NonMutableDict(error_msg="Overlap between splits. Split {key} has been added with itself.") + + if split_info: + self.add(SlicedSplitInfo(split_info=split_info, slice_value=None)) + + def add(self, sliced_split): + """Add a SlicedSplitInfo the read instructions.""" + # TODO(epot): Check that the number of examples per shard % 100 == 0 + # Otherwise the slices value may be unbalanced and not exactly reflect the + # requested slice. + self._splits[sliced_split.split_info.name] = sliced_split + + def __add__(self, other): + """Merging split together.""" + # Will raise error if a split has already be added (NonMutableDict) + # TODO(epot): If a split is already added but there is no overlap between + # the slices, should merge the slices (ex: [:10] + [80:]) + split_instruction = SplitReadInstruction() + split_instruction._splits.update(self._splits) # pylint: disable=protected-access + split_instruction._splits.update(other._splits) # pylint: disable=protected-access + return split_instruction + + def __getitem__(self, slice_value): + """Sub-splits.""" + # Will raise an error if a split has already been sliced + split_instruction = SplitReadInstruction() + for v in self._splits.values(): + if v.slice_value is not None: + raise ValueError(f"Trying to slice Split {v.split_info.name} which has already been sliced") + v = v._asdict() + v["slice_value"] = slice_value + split_instruction.add(SlicedSplitInfo(**v)) + return split_instruction + + def get_list_sliced_split_info(self): + return list(self._splits.values()) + + +class SplitDict(dict[str, SplitInfo]): + """Split info object.""" + + def __init__(self, *args, dataset_name=None, **kwargs): + super().__init__(*args, **kwargs) + self.dataset_name = dataset_name + + def __getitem__(self, key: Union[SplitBase, str]): + # 1st case: The key exists: `info.splits['train']` + if str(key) in self: + return super().__getitem__(str(key)) + # 2nd case: Uses instructions: `info.splits['train[50%]']` + else: + instructions = make_file_instructions( + name=self.dataset_name, + split_infos=self.values(), + instruction=key, + ) + return SubSplitInfo(instructions) + + def __setitem__(self, key: Union[SplitBase, str], value: SplitInfo): + if key != value.name: + raise ValueError(f"Cannot add elem. (key mismatch: '{key}' != '{value.name}')") + super().__setitem__(key, value) + + def add(self, split_info: SplitInfo): + """Add the split info.""" + if split_info.name in self: + raise ValueError(f"Split {split_info.name} already present") + split_info.dataset_name = self.dataset_name + super().__setitem__(split_info.name, split_info) + + @property + def total_num_examples(self): + """Return the total number of examples.""" + return sum(s.num_examples for s in self.values()) + + @classmethod + def from_split_dict(cls, split_infos: Union[list, dict], dataset_name: Optional[str] = None): + """Returns a new SplitDict initialized from a Dict or List of `split_infos`.""" + if isinstance(split_infos, dict): + split_infos = list(split_infos.values()) + + if dataset_name is None: + dataset_name = split_infos[0].get("dataset_name") if split_infos else None + + split_dict = cls(dataset_name=dataset_name) + + for split_info in split_infos: + if isinstance(split_info, dict): + split_info = SplitInfo(**split_info) + split_dict.add(split_info) + + return split_dict + + def to_split_dict(self): + """Returns a list of SplitInfo protos that we have.""" + out = [] + for split_name, split_info in self.items(): + split_info = copy.deepcopy(split_info) + split_info.name = split_name + out.append(split_info) + return out + + def copy(self): + return SplitDict.from_split_dict(self.to_split_dict(), self.dataset_name) + + def _to_yaml_list(self) -> list: + out = [asdict(s) for s in self.to_split_dict()] + # we don't need the shard lengths in YAML + for split_info_dict in out: + split_info_dict.pop("shard_lengths", None) + split_info_dict.pop("original_shard_lengths", None) + # we don't need the dataset_name attribute that is deprecated + for split_info_dict in out: + split_info_dict.pop("dataset_name", None) + return out + + @classmethod + def _from_yaml_list(cls, yaml_data: list) -> "SplitDict": + return cls.from_split_dict(yaml_data) + + +@dataclass +class SplitGenerator: + """Defines the split information for the generator. + + This should be used as returned value of + `GeneratorBasedBuilder._split_generators`. + See `GeneratorBasedBuilder._split_generators` for more info and example + of usage. + + Args: + name (`str`): + Name of the `Split` for which the generator will + create the examples. + **gen_kwargs (additional keyword arguments): + Keyword arguments to forward to the `DatasetBuilder._generate_examples` method + of the builder. + + Example: + + ```py + >>> datasets.SplitGenerator( + ... name=datasets.Split.TRAIN, + ... gen_kwargs={"split_key": "train", "files": dl_manager.download_and_extract(url)}, + ... ) + ``` + """ + + name: str + gen_kwargs: dict = dataclasses.field(default_factory=dict) + split_info: SplitInfo = dataclasses.field(init=False) + + def __post_init__(self): + self.name = str(self.name) # Make sure we convert NamedSplits in strings + NamedSplit(self.name) # check that it's a valid split name + self.split_info = SplitInfo(name=self.name) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/streaming.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/streaming.py new file mode 100644 index 0000000000000000000000000000000000000000..6e5e8ee15f9f529d93d43dc003ccdef3436fd725 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/streaming.py @@ -0,0 +1,131 @@ +import importlib +from functools import wraps +from typing import TYPE_CHECKING, Optional + +from .download.download_config import DownloadConfig +from .utils.file_utils import ( + xbasename, + xdirname, + xet_parse, + xexists, + xgetsize, + xglob, + xgzip_open, + xisdir, + xisfile, + xjoin, + xlistdir, + xnumpy_load, + xopen, + xpandas_read_csv, + xpandas_read_excel, + xPath, + xpyarrow_parquet_read_table, + xrelpath, + xsio_loadmat, + xsplit, + xsplitext, + xwalk, + xxml_dom_minidom_parse, +) +from .utils.logging import get_logger +from .utils.patching import patch_submodule + + +logger = get_logger(__name__) + + +if TYPE_CHECKING: + from .builder import DatasetBuilder + + +def extend_module_for_streaming(module_path, download_config: Optional[DownloadConfig] = None): + """Extend the module to support streaming. + + We patch some functions in the module to use `fsspec` to support data streaming: + - We use `fsspec.open` to open and read remote files. We patch the module function: + - `open` + - We use the "::" hop separator to join paths and navigate remote compressed/archive files. We patch the module + functions: + - `os.path.join` + - `pathlib.Path.joinpath` and `pathlib.Path.__truediv__` (called when using the "/" operator) + + The patched functions are replaced with custom functions defined to work with the + :class:`~download.streaming_download_manager.StreamingDownloadManager`. + + Args: + module_path: Path to the module to be extended. + download_config: Mainly use `token` or `storage_options` to support different platforms and auth types. + """ + + module = importlib.import_module(module_path) + + # TODO(QL): always update the module to add subsequent new authentication without removing old ones + if hasattr(module, "_patched_for_streaming") and module._patched_for_streaming: + if isinstance(module._patched_for_streaming, DownloadConfig): + module._patched_for_streaming.token = download_config.token + module._patched_for_streaming.storage_options = download_config.storage_options + return + + def wrap_auth(function): + @wraps(function) + def wrapper(*args, **kwargs): + return function(*args, download_config=download_config, **kwargs) + + wrapper._decorator_name_ = "wrap_auth" + return wrapper + + # open files in a streaming fashion + patch_submodule(module, "open", wrap_auth(xopen)).start() + patch_submodule(module, "os.listdir", wrap_auth(xlistdir)).start() + patch_submodule(module, "os.walk", wrap_auth(xwalk)).start() + patch_submodule(module, "glob.glob", wrap_auth(xglob)).start() + # allow to navigate in remote zip files + patch_submodule(module, "os.path.join", xjoin).start() + patch_submodule(module, "os.path.dirname", xdirname).start() + patch_submodule(module, "os.path.basename", xbasename).start() + patch_submodule(module, "os.path.relpath", xrelpath).start() + patch_submodule(module, "os.path.split", xsplit).start() + patch_submodule(module, "os.path.splitext", xsplitext).start() + # allow checks on paths + patch_submodule(module, "os.path.exists", wrap_auth(xexists)).start() + patch_submodule(module, "os.path.isdir", wrap_auth(xisdir)).start() + patch_submodule(module, "os.path.isfile", wrap_auth(xisfile)).start() + patch_submodule(module, "os.path.getsize", wrap_auth(xgetsize)).start() + patch_submodule(module, "pathlib.Path", xPath).start() + # file readers + patch_submodule(module, "gzip.open", wrap_auth(xgzip_open)).start() + patch_submodule(module, "numpy.load", wrap_auth(xnumpy_load)).start() + patch_submodule(module, "pandas.read_csv", wrap_auth(xpandas_read_csv), attrs=["__version__"]).start() + patch_submodule(module, "pandas.read_excel", wrap_auth(xpandas_read_excel), attrs=["__version__"]).start() + patch_submodule(module, "scipy.io.loadmat", wrap_auth(xsio_loadmat), attrs=["__version__"]).start() + patch_submodule(module, "xml.etree.ElementTree.parse", wrap_auth(xet_parse)).start() + patch_submodule(module, "xml.dom.minidom.parse", wrap_auth(xxml_dom_minidom_parse)).start() + # pyarrow: do not patch pyarrow attribute in packaged modules + if not module.__name__.startswith("datasets.packaged_modules."): + patch_submodule(module, "pyarrow.parquet.read_table", wrap_auth(xpyarrow_parquet_read_table)).start() + module._patched_for_streaming = download_config + + +def extend_dataset_builder_for_streaming(builder: "DatasetBuilder"): + """Extend the dataset builder module and the modules imported by it to support streaming. + + Args: + builder (:class:`DatasetBuilder`): Dataset builder instance. + """ + # this extends the open and os.path.join functions for data streaming + download_config = DownloadConfig(storage_options=builder.storage_options, token=builder.token) + extend_module_for_streaming(builder.__module__, download_config=download_config) + + # builders can inherit from other builders that might use streaming functionality + # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation) + # but these parents builders are not patched automatically as they are not instantiated, so we patch them here + from .builder import DatasetBuilder + + parent_builder_modules = [ + cls.__module__ + for cls in type(builder).__mro__[1:] # make sure it's not the same module we've already patched + if issubclass(cls, DatasetBuilder) and cls.__module__ != DatasetBuilder.__module__ + ] # check it's not a standard builder from datasets.builder + for module in parent_builder_modules: + extend_module_for_streaming(module, download_config=download_config) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/table.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/table.py new file mode 100644 index 0000000000000000000000000000000000000000..ca8949d391f4d845b516de5334379b48746ce370 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/table.py @@ -0,0 +1,2434 @@ +import copy +import os +from collections.abc import Iterator +from functools import partial +from itertools import groupby +from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, Union + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc + +from .utils.logging import get_logger + + +if TYPE_CHECKING: + from .features.features import Features, FeatureType + + +logger = get_logger(__name__) + + +def inject_arrow_table_documentation(arrow_table_method): + def wrapper(fn): + fn.__doc__ = arrow_table_method.__doc__ + (fn.__doc__ if fn.__doc__ is not None else "") + fn.__doc__ = fn.__doc__.replace("pyarrow.Table", "Table") + if hasattr(arrow_table_method, "__annotations__"): + fn.__annotations__ = arrow_table_method.__annotations__ + return fn + + return wrapper + + +def _in_memory_arrow_table_from_file(filename: str) -> pa.Table: + in_memory_stream = pa.input_stream(filename) + opened_stream = pa.ipc.open_stream(in_memory_stream) + pa_table = opened_stream.read_all() + return pa_table + + +def _in_memory_arrow_table_from_buffer(buffer: pa.Buffer) -> pa.Table: + stream = pa.BufferReader(buffer) + opened_stream = pa.ipc.open_stream(stream) + table = opened_stream.read_all() + return table + + +def _memory_mapped_record_batch_reader_from_file(filename: str) -> pa.RecordBatchStreamReader: + memory_mapped_stream = pa.memory_map(filename) + return pa.ipc.open_stream(memory_mapped_stream) + + +def read_schema_from_file(filename: str) -> pa.Schema: + """ + Infer arrow table schema from file without loading whole file into memory. + Useful especially while having very big files. + """ + with pa.memory_map(filename) as memory_mapped_stream: + schema = pa.ipc.open_stream(memory_mapped_stream).schema + return schema + + +def _batch_arrow_table(table: pa.Table) -> pa.Table: + batched_columns = [] + for column_name in table.column_names: + column = table[column_name].combine_chunks() + offsets = pa.array([0, len(column)], type=pa.int32()) + batched_columns.append(pa.ListArray.from_arrays(offsets, column)) + return pa.Table.from_arrays(batched_columns, names=table.column_names) + + +def _memory_mapped_arrow_table_from_file(filename: str) -> pa.Table: + opened_stream = _memory_mapped_record_batch_reader_from_file(filename) + pa_table = opened_stream.read_all() + return pa_table + + +def _deepcopy(x, memo: dict): + """deepcopy a regular class instance""" + cls = x.__class__ + result = cls.__new__(cls) + memo[id(x)] = result + for k, v in x.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + return result + + +def _interpolation_search(arr: list[int], x: int) -> int: + """ + Return the position i of a sorted array so that arr[i] <= x < arr[i+1] + + Args: + arr (`List[int]`): non-empty sorted list of integers + x (`int`): query + + Returns: + `int`: the position i so that arr[i] <= x < arr[i+1] + + Raises: + `IndexError`: if the array is empty or if the query is outside the array values + """ + i, j = 0, len(arr) - 1 + while i < j and arr[i] <= x < arr[j]: + k = i + ((j - i) * (x - arr[i]) // (arr[j] - arr[i])) + if arr[k] <= x < arr[k + 1]: + return k + elif arr[k] < x: + i, j = k + 1, j + else: + i, j = i, k + raise IndexError(f"Invalid query '{x}' for size {arr[-1] if len(arr) else 'none'}.") + + +class IndexedTableMixin: + def __init__(self, table: pa.Table): + self._schema: pa.Schema = table.schema + self._batches: list[pa.RecordBatch] = [ + recordbatch for recordbatch in table.to_batches() if len(recordbatch) > 0 + ] + self._offsets: np.ndarray = np.cumsum([0] + [len(b) for b in self._batches], dtype=np.int64) + + def fast_gather(self, indices: Union[list[int], np.ndarray]) -> pa.Table: + """ + Create a pa.Table by gathering the records at the records at the specified indices. Should be faster + than pa.concat_tables(table.fast_slice(int(i) % table.num_rows, 1) for i in indices) since NumPy can compute + the binary searches in parallel, highly optimized C + """ + if not len(indices): + raise ValueError("Indices must be non-empty") + batch_indices = np.searchsorted(self._offsets, indices, side="right") - 1 + return pa.Table.from_batches( + [ + self._batches[batch_idx].slice(i - self._offsets[batch_idx], 1) + for batch_idx, i in zip(batch_indices, indices) + ], + schema=self._schema, + ) + + def fast_slice(self, offset=0, length=None) -> pa.Table: + """ + Slice the Table using interpolation search. + The behavior is the same as `pyarrow.Table.slice` but it's significantly faster. + + Interpolation search is used to find the start and end indexes of the batches we want to keep. + The batches to keep are then concatenated to form the sliced Table. + """ + if offset < 0: + raise IndexError("Offset must be non-negative") + elif offset >= self._offsets[-1] or (length is not None and length <= 0): + return pa.Table.from_batches([], schema=self._schema) + i = _interpolation_search(self._offsets, offset) + if length is None or length + offset >= self._offsets[-1]: + batches = self._batches[i:] + batches[0] = batches[0].slice(offset - self._offsets[i]) + else: + j = _interpolation_search(self._offsets, offset + length - 1) + batches = self._batches[i : j + 1] + batches[-1] = batches[-1].slice(0, offset + length - self._offsets[j]) + batches[0] = batches[0].slice(offset - self._offsets[i]) + return pa.Table.from_batches(batches, schema=self._schema) + + +class Table(IndexedTableMixin): + """ + Wraps a pyarrow Table by using composition. + This is the base class for `InMemoryTable`, `MemoryMappedTable` and `ConcatenationTable`. + + It implements all the basic attributes/methods of the pyarrow Table class except + the Table transforms: `slice, filter, flatten, combine_chunks, cast, add_column, + append_column, remove_column, set_column, rename_columns` and `drop`. + + The implementation of these methods differs for the subclasses. + """ + + def __init__(self, table: pa.Table): + super().__init__(table) + self.table = table + + def __deepcopy__(self, memo: dict): + # arrow tables are immutable, so there's no need to copy self.table + # moreover calling deepcopy on a pyarrow table seems to make pa.total_allocated_bytes() decrease for some reason + # by adding it to the memo, self.table won't be copied + memo[id(self.table)] = self.table + # same for the recordbatches used by the index + memo[id(self._batches)] = list(self._batches) + return _deepcopy(self, memo) + + def validate(self, *args, **kwargs): + """ + Perform validation checks. An exception is raised if validation fails. + + By default only cheap validation checks are run. Pass `full=True` + for thorough validation checks (potentially `O(n)`). + + Args: + full (`bool`, defaults to `False`): + If `True`, run expensive checks, otherwise cheap checks only. + + Raises: + `pa.lib.ArrowInvalid`: if validation fails + """ + return self.table.validate(*args, **kwargs) + + def equals(self, *args, **kwargs): + """ + Check if contents of two tables are equal. + + Args: + other ([`~datasets.table.Table`]): + Table to compare against. + check_metadata `bool`, defaults to `False`): + Whether schema metadata equality should be checked as well. + + Returns: + `bool` + """ + args = tuple(arg.table if isinstance(arg, Table) else arg for arg in args) + kwargs = {k: v.table if isinstance(v, Table) else v for k, v in kwargs} + return self.table.equals(*args, **kwargs) + + def to_batches(self, *args, **kwargs): + """ + Convert Table to list of (contiguous) `RecordBatch` objects. + + Args: + max_chunksize (`int`, defaults to `None`): + Maximum size for `RecordBatch` chunks. Individual chunks may be + smaller depending on the chunk layout of individual columns. + + Returns: + `List[pyarrow.RecordBatch]` + """ + return self.table.to_batches(*args, **kwargs) + + def to_pydict(self, *args, **kwargs): + """ + Convert the Table to a `dict` or `OrderedDict`. + + Returns: + `dict` + """ + return self.table.to_pydict(*args, **kwargs) + + def to_pylist(self, *args, **kwargs): + """ + Convert the Table to a list + + Returns: + `list` + """ + return self.table.to_pylist(*args, **kwargs) + + def to_pandas(self, *args, **kwargs): + """ + Convert to a pandas-compatible NumPy array or DataFrame, as appropriate. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + Arrow MemoryPool to use for allocations. Uses the default memory + pool is not passed. + strings_to_categorical (`bool`, defaults to `False`): + Encode string (UTF8) and binary types to `pandas.Categorical`. + categories (`list`, defaults to `empty`): + List of fields that should be returned as `pandas.Categorical`. Only + applies to table-like data structures. + zero_copy_only (`bool`, defaults to `False`): + Raise an `ArrowException` if this function call would require copying + the underlying data. + integer_object_nulls (`bool`, defaults to `False`): + Cast integers with nulls to objects. + date_as_object (`bool`, defaults to `True`): + Cast dates to objects. If `False`, convert to `datetime64[ns]` dtype. + timestamp_as_object (`bool`, defaults to `False`): + Cast non-nanosecond timestamps (`np.datetime64`) to objects. This is + useful if you have timestamps that don't fit in the normal date + range of nanosecond timestamps (1678 CE-2262 CE). + If `False`, all timestamps are converted to `datetime64[ns]` dtype. + use_threads (`bool`, defaults to `True`): + Whether to parallelize the conversion using multiple threads. + deduplicate_objects (`bool`, defaults to `False`): + Do not create multiple copies Python objects when created, to save + on memory use. Conversion will be slower. + ignore_metadata (`bool`, defaults to `False`): + If `True`, do not use the 'pandas' metadata to reconstruct the + DataFrame index, if present. + safe (`bool`, defaults to `True`): + For certain data types, a cast is needed in order to store the + data in a pandas DataFrame or Series (e.g. timestamps are always + stored as nanoseconds in pandas). This option controls whether it + is a safe cast or not. + split_blocks (`bool`, defaults to `False`): + If `True`, generate one internal "block" for each column when + creating a pandas.DataFrame from a `RecordBatch` or `Table`. While this + can temporarily reduce memory note that various pandas operations + can trigger "consolidation" which may balloon memory use. + self_destruct (`bool`, defaults to `False`): + EXPERIMENTAL: If `True`, attempt to deallocate the originating Arrow + memory while converting the Arrow object to pandas. If you use the + object after calling `to_pandas` with this option it will crash your + program. + types_mapper (`function`, defaults to `None`): + A function mapping a pyarrow DataType to a pandas `ExtensionDtype`. + This can be used to override the default pandas type for conversion + of built-in pyarrow types or in absence of `pandas_metadata` in the + Table schema. The function receives a pyarrow DataType and is + expected to return a pandas `ExtensionDtype` or `None` if the + default conversion should be used for that type. If you have + a dictionary mapping, you can pass `dict.get` as function. + + Returns: + `pandas.Series` or `pandas.DataFrame`: `pandas.Series` or `pandas.DataFrame` depending on type of object + """ + return self.table.to_pandas(*args, **kwargs) + + def to_string(self, *args, **kwargs): + return self.table.to_string(*args, **kwargs) + + def to_reader(self, max_chunksize: Optional[int] = None): + """ + Convert the Table to a RecordBatchReader. + + Note that this method is zero-copy, it merely exposes the same data under a different API. + + Args: + max_chunksize (`int`, defaults to `None`) + Maximum size for RecordBatch chunks. Individual chunks may be smaller depending + on the chunk layout of individual columns. + + Returns: + `pyarrow.RecordBatchReader` + """ + return self.table.to_reader(max_chunksize=max_chunksize) + + def field(self, *args, **kwargs): + """ + Select a schema field by its column name or numeric index. + + Args: + i (`Union[int, str]`): + The index or name of the field to retrieve. + + Returns: + `pyarrow.Field` + """ + return self.table.field(*args, **kwargs) + + def column(self, *args, **kwargs): + """ + Select a column by its column name, or numeric index. + + Args: + i (`Union[int, str]`): + The index or name of the column to retrieve. + + Returns: + `pyarrow.ChunkedArray` + """ + return self.table.column(*args, **kwargs) + + def itercolumns(self, *args, **kwargs): + """ + Iterator over all columns in their numerical order. + + Yields: + `pyarrow.ChunkedArray` + """ + return self.table.itercolumns(*args, **kwargs) + + @property + def schema(self): + """ + Schema of the table and its columns. + + Returns: + `pyarrow.Schema` + """ + return self.table.schema + + @property + def columns(self): + """ + List of all columns in numerical order. + + Returns: + `List[pa.ChunkedArray]` + """ + return self.table.columns + + @property + def num_columns(self): + """ + Number of columns in this table. + + Returns: + int + """ + return self.table.num_columns + + @property + def num_rows(self): + """ + Number of rows in this table. + + Due to the definition of a table, all columns have the same number of + rows. + + Returns: + int + """ + return self.table.num_rows + + @property + def shape(self): + """ + Dimensions of the table: (#rows, #columns). + + Returns: + `(int, int)`: Number of rows and number of columns. + """ + return self.table.shape + + @property + def nbytes(self): + """ + Total number of bytes consumed by the elements of the table. + """ + return self.table.nbytes + + @property + def column_names(self): + """ + Names of the table's columns. + """ + return self.table.column_names + + def __eq__(self, other): + return self.equals(other) + + def __getitem__(self, i): + return self.table[i] + + def __len__(self): + return len(self.table) + + def __repr__(self): + return self.table.__repr__().replace("pyarrow.Table", self.__class__.__name__) + + def __str__(self): + return self.table.__str__().replace("pyarrow.Table", self.__class__.__name__) + + def slice(self, *args, **kwargs): + """ + Compute zero-copy slice of this Table. + + Args: + offset (`int`, defaults to `0`): + Offset from start of table to slice. + length (`int`, defaults to `None`): + Length of slice (default is until end of table starting from + offset). + + Returns: + `datasets.table.Table` + """ + raise NotImplementedError() + + def filter(self, *args, **kwargs): + """ + Select records from a Table. See `pyarrow.compute.filter` for full usage. + """ + raise NotImplementedError() + + def flatten(self, *args, **kwargs): + """ + Flatten this Table. Each column with a struct type is flattened + into one column per struct field. Other columns are left unchanged. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + raise NotImplementedError() + + def combine_chunks(self, *args, **kwargs): + """ + Make a new table by combining the chunks this table has. + + All the underlying chunks in the `ChunkedArray` of each column are + concatenated into zero or one chunk. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + raise NotImplementedError() + + def cast(self, *args, **kwargs): + """ + Cast table values to another schema. + + Args: + target_schema (`Schema`): + Schema to cast to, the names and order of fields must match. + safe (`bool`, defaults to `True`): + Check for overflows or other unsafe conversions. + + Returns: + `datasets.table.Table` + """ + raise NotImplementedError() + + def replace_schema_metadata(self, *args, **kwargs): + """ + EXPERIMENTAL: Create shallow copy of table by replacing schema + key-value metadata with the indicated new metadata (which may be None, + which deletes any existing metadata + + Args: + metadata (`dict`, defaults to `None`): + + Returns: + `datasets.table.Table`: shallow_copy + """ + raise NotImplementedError() + + def add_column(self, *args, **kwargs): + """ + Add column to Table at position. + + A new table is returned with the column added, the original table + object is left unchanged. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column added. + """ + raise NotImplementedError() + + def append_column(self, *args, **kwargs): + """ + Append column at end of columns. + + Args: + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column added. + """ + raise NotImplementedError() + + def remove_column(self, *args, **kwargs): + """ + Create new Table with the indicated column removed. + + Args: + i (`int`): + Index of column to remove. + + Returns: + `datasets.table.Table`: New table without the column. + """ + raise NotImplementedError() + + def set_column(self, *args, **kwargs): + """ + Replace column in Table at position. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column set. + """ + raise NotImplementedError() + + def rename_columns(self, *args, **kwargs): + """ + Create new table with columns renamed to provided names. + """ + raise NotImplementedError() + + def drop(self, *args, **kwargs): + """ + Drop one or more columns and return a new table. + + Args: + columns (`List[str]`): + List of field names referencing existing columns. + + Raises: + `KeyError` : if any of the passed columns name are not existing. + + Returns: + `datasets.table.Table`: New table without the columns. + """ + raise NotImplementedError() + + def select(self, *args, **kwargs): + """ + Select columns of the table. + + Returns a new table with the specified columns, and metadata preserved. + + Args: + columns (:obj:`Union[List[str], List[int]]`): + The column names or integer indices to select. + + Returns: + `datasets.table.Table`: table with only a subset of the columns + """ + raise NotImplementedError() + + +class TableBlock(Table): + """ + `TableBlock` is the allowed class inside a `ConcanetationTable`. + Only `MemoryMappedTable` and `InMemoryTable` are `TableBlock`. + This is because we don't want a `ConcanetationTable` made out of other `ConcanetationTables`. + """ + + pass + + +class InMemoryTable(TableBlock): + """ + The table is said in-memory when it is loaded into the user's RAM. + + Pickling it does copy all the data using memory. + Its implementation is simple and uses the underlying pyarrow Table methods directly. + + This is different from the `MemoryMapped` table, for which pickling doesn't copy all the + data in memory. For a `MemoryMapped`, unpickling instead reloads the table from the disk. + + `InMemoryTable` must be used when data fit in memory, while `MemoryMapped` are reserved for + data bigger than memory or when you want the memory footprint of your application to + stay low. + """ + + @classmethod + def from_file(cls, filename: str): + table = _in_memory_arrow_table_from_file(filename) + return cls(table) + + @classmethod + def from_buffer(cls, buffer: pa.Buffer): + table = _in_memory_arrow_table_from_buffer(buffer) + return cls(table) + + @classmethod + def from_pandas(cls, *args, **kwargs): + """ + Convert pandas.DataFrame to an Arrow Table. + + The column types in the resulting Arrow Table are inferred from the + dtypes of the pandas.Series in the DataFrame. In the case of non-object + Series, the NumPy dtype is translated to its Arrow equivalent. In the + case of `object`, we need to guess the datatype by looking at the + Python objects in this Series. + + Be aware that Series of the `object` dtype don't carry enough + information to always lead to a meaningful Arrow type. In the case that + we cannot infer a type, e.g. because the DataFrame is of length 0 or + the Series only contains `None/nan` objects, the type is set to + null. This behavior can be avoided by constructing an explicit schema + and passing it to this function. + + Args: + df (`pandas.DataFrame`): + schema (`pyarrow.Schema`, *optional*): + The expected schema of the Arrow Table. This can be used to + indicate the type of columns if we cannot infer it automatically. + If passed, the output will have exactly this schema. Columns + specified in the schema that are not found in the DataFrame columns + or its index will raise an error. Additional columns or index + levels in the DataFrame which are not specified in the schema will + be ignored. + preserve_index (`bool`, *optional*): + Whether to store the index as an additional column in the resulting + `Table`. The default of None will store the index as a column, + except for RangeIndex which is stored as metadata only. Use + `preserve_index=True` to force it to be stored as a column. + nthreads (`int`, defaults to `None` (may use up to system CPU count threads)) + If greater than 1, convert columns to Arrow in parallel using + indicated number of threads. + columns (`List[str]`, *optional*): + List of column to be converted. If `None`, use all columns. + safe (`bool`, defaults to `True`): + Check for overflows or other unsafe conversions, + + Returns: + `datasets.table.Table`: + + Examples: + ```python + >>> import pandas as pd + >>> import pyarrow as pa + >>> df = pd.DataFrame({ + ... 'int': [1, 2], + ... 'str': ['a', 'b'] + ... }) + >>> pa.Table.from_pandas(df) + + ``` + """ + return cls(pa.Table.from_pandas(*args, **kwargs)) + + @classmethod + def from_arrays(cls, *args, **kwargs): + """ + Construct a Table from Arrow arrays. + + Args: + arrays (`List[Union[pyarrow.Array, pyarrow.ChunkedArray]]`): + Equal-length arrays that should form the table. + names (`List[str]`, *optional*): + Names for the table columns. If not passed, schema must be passed. + schema (`Schema`, defaults to `None`): + Schema for the created table. If not passed, names must be passed. + metadata (`Union[dict, Mapping]`, defaults to `None`): + Optional metadata for the schema (if inferred). + + Returns: + `datasets.table.Table` + """ + return cls(pa.Table.from_arrays(*args, **kwargs)) + + @classmethod + def from_pydict(cls, *args, **kwargs): + """ + Construct a Table from Arrow arrays or columns. + + Args: + mapping (`Union[dict, Mapping]`): + A mapping of strings to Arrays or Python lists. + schema (`Schema`, defaults to `None`): + If not passed, will be inferred from the Mapping values + metadata (`Union[dict, Mapping]`, defaults to `None`): + Optional metadata for the schema (if inferred). + + Returns: + `datasets.table.Table` + """ + return cls(pa.Table.from_pydict(*args, **kwargs)) + + @classmethod + def from_pylist(cls, mapping, *args, **kwargs): + """ + Construct a Table from list of rows / dictionaries. + + Args: + mapping (`List[dict]`): + A mapping of strings to row values. + schema (`Schema`, defaults to `None`): + If not passed, will be inferred from the Mapping values + metadata (`Union[dict, Mapping]`, defaults to `None`): + Optional metadata for the schema (if inferred). + + Returns: + `datasets.table.Table` + """ + return cls(pa.Table.from_pylist(mapping, *args, **kwargs)) + + @classmethod + def from_batches(cls, *args, **kwargs): + """ + Construct a Table from a sequence or iterator of Arrow `RecordBatches`. + + Args: + batches (`Union[Sequence[pyarrow.RecordBatch], Iterator[pyarrow.RecordBatch]]`): + Sequence of `RecordBatch` to be converted, all schemas must be equal. + schema (`Schema`, defaults to `None`): + If not passed, will be inferred from the first `RecordBatch`. + + Returns: + `datasets.table.Table`: + """ + return cls(pa.Table.from_batches(*args, **kwargs)) + + def slice(self, offset=0, length=None): + """ + Compute zero-copy slice of this Table. + + Args: + offset (`int`, defaults to `0`): + Offset from start of table to slice. + length (`int`, defaults to `None`): + Length of slice (default is until end of table starting from + offset). + + Returns: + `datasets.table.Table` + """ + # Use fast slicing here + return InMemoryTable(self.fast_slice(offset=offset, length=length)) + + def filter(self, *args, **kwargs): + """ + Select records from a Table. See `pyarrow.compute.filter` for full usage. + """ + return InMemoryTable(self.table.filter(*args, **kwargs)) + + def flatten(self, *args, **kwargs): + """ + Flatten this Table. Each column with a struct type is flattened + into one column per struct field. Other columns are left unchanged. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + return InMemoryTable(table_flatten(self.table, *args, **kwargs)) + + def combine_chunks(self, *args, **kwargs): + """ + Make a new table by combining the chunks this table has. + + All the underlying chunks in the `ChunkedArray` of each column are + concatenated into zero or one chunk. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + return InMemoryTable(self.table.combine_chunks(*args, **kwargs)) + + def cast(self, *args, **kwargs): + """ + Cast table values to another schema. + + Args: + target_schema (`Schema`): + Schema to cast to, the names and order of fields must match. + safe (`bool`, defaults to `True`): + Check for overflows or other unsafe conversions. + + Returns: + `datasets.table.Table` + """ + return InMemoryTable(table_cast(self.table, *args, **kwargs)) + + def replace_schema_metadata(self, *args, **kwargs): + """ + EXPERIMENTAL: Create shallow copy of table by replacing schema + key-value metadata with the indicated new metadata (which may be `None`, + which deletes any existing metadata). + + Args: + metadata (`dict`, defaults to `None`): + + Returns: + `datasets.table.Table`: shallow_copy + """ + return InMemoryTable(self.table.replace_schema_metadata(*args, **kwargs)) + + def add_column(self, *args, **kwargs): + """ + Add column to Table at position. + + A new table is returned with the column added, the original table + object is left unchanged. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column added. + """ + return InMemoryTable(self.table.add_column(*args, **kwargs)) + + def append_column(self, *args, **kwargs): + """ + Append column at end of columns. + + Args: + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column added. + """ + return InMemoryTable(self.table.append_column(*args, **kwargs)) + + def remove_column(self, *args, **kwargs): + """ + Create new Table with the indicated column removed. + + Args: + i (`int`): + Index of column to remove. + + Returns: + `datasets.table.Table`: + New table without the column. + """ + return InMemoryTable(self.table.remove_column(*args, **kwargs)) + + def set_column(self, *args, **kwargs): + """ + Replace column in Table at position. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column set. + """ + return InMemoryTable(self.table.set_column(*args, **kwargs)) + + def rename_columns(self, *args, **kwargs): + """ + Create new table with columns renamed to provided names. + """ + return InMemoryTable(self.table.rename_columns(*args, **kwargs)) + + def drop(self, *args, **kwargs): + """ + Drop one or more columns and return a new table. + + Args: + columns (`List[str]`): + List of field names referencing existing columns. + + Raises: + `KeyError` : if any of the passed columns name are not existing. + + Returns: + `datasets.table.Table`: + New table without the columns. + """ + return InMemoryTable(self.table.drop(*args, **kwargs)) + + def select(self, *args, **kwargs): + """ + Select columns of the table. + + Returns a new table with the specified columns, and metadata preserved. + + Args: + columns (:obj:`Union[List[str], List[int]]`): + The column names or integer indices to select. + + Returns: + :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved. + """ + return InMemoryTable(self.table.select(*args, **kwargs)) + + +# The MemoryMappedTable needs replays to properly reload tables from the disk +Replay = tuple[str, tuple, dict] + + +class MemoryMappedTable(TableBlock): + """ + The table is said memory mapped when it doesn't use the user's RAM but loads the data + from the disk instead. + + Pickling it doesn't copy the data into memory. + Instead, only the path to the memory mapped arrow file is pickled, as well as the list + of transforms to "replay" when reloading the table from the disk. + + Its implementation requires to store an history of all the transforms that were applied + to the underlying pyarrow Table, so that they can be "replayed" when reloading the Table + from the disk. + + This is different from the `InMemoryTable` table, for which pickling does copy all the + data in memory. + + `InMemoryTable` must be used when data fit in memory, while `MemoryMapped` are reserved for + data bigger than memory or when you want the memory footprint of your application to + stay low. + """ + + def __init__(self, table: pa.Table, path: str, replays: Optional[list[Replay]] = None): + super().__init__(table) + self.path = os.path.abspath(path) + self.replays: list[Replay] = replays if replays is not None else [] + + @classmethod + def from_file(cls, filename: str, replays=None): + table = _memory_mapped_arrow_table_from_file(filename) + table = cls._apply_replays(table, replays) + return cls(table, filename, replays) + + def __getstate__(self): + return {"path": self.path, "replays": self.replays} + + def __setstate__(self, state): + path = state["path"] + replays = state["replays"] + table = _memory_mapped_arrow_table_from_file(path) + table = self._apply_replays(table, replays) + MemoryMappedTable.__init__(self, table, path=path, replays=replays) + + @staticmethod + def _apply_replays(table: pa.Table, replays: Optional[list[Replay]] = None) -> pa.Table: + if replays is not None: + for name, args, kwargs in replays: + if name == "cast": + table = table_cast(table, *args, **kwargs) + elif name == "flatten": + table = table_flatten(table, *args, **kwargs) + else: + table = getattr(table, name)(*args, **kwargs) + return table + + def _append_replay(self, replay: Replay) -> list[Replay]: + replays = copy.deepcopy(self.replays) + replays.append(replay) + return replays + + def slice(self, offset=0, length=None): + """ + Compute zero-copy slice of this Table. + + Args: + offset (`int`, defaults to `0`): + Offset from start of table to slice. + length (`int`, defaults to `None`): + Length of slice (default is until end of table starting from + offset). + + Returns: + `datasets.table.Table` + """ + replay = ("slice", (offset, length), {}) + replays = self._append_replay(replay) + # Use fast slicing here + return MemoryMappedTable(self.fast_slice(offset=offset, length=length), self.path, replays) + + def filter(self, *args, **kwargs): + """ + Select records from a Table. See `pyarrow.compute.filter` for full usage. + """ + replay = ("filter", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.filter(*args, **kwargs), self.path, replays) + + def flatten(self, *args, **kwargs): + """ + Flatten this Table. Each column with a struct type is flattened + into one column per struct field. Other columns are left unchanged. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + replay = ("flatten", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(table_flatten(self.table, *args, **kwargs), self.path, replays) + + def combine_chunks(self, *args, **kwargs): + """ + Make a new table by combining the chunks this table has. + + All the underlying chunks in the ChunkedArray of each column are + concatenated into zero or one chunk. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + replay = ("combine_chunks", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.combine_chunks(*args, **kwargs), self.path, replays) + + def cast(self, *args, **kwargs): + """ + Cast table values to another schema + + Args: + target_schema (`Schema`): + Schema to cast to, the names and order of fields must match. + safe (`bool`, defaults to `True`): + Check for overflows or other unsafe conversions. + + Returns: + `datasets.table.Table` + """ + replay = ("cast", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(table_cast(self.table, *args, **kwargs), self.path, replays) + + def replace_schema_metadata(self, *args, **kwargs): + """ + EXPERIMENTAL: Create shallow copy of table by replacing schema + key-value metadata with the indicated new metadata (which may be None, + which deletes any existing metadata. + + Args: + metadata (`dict`, defaults to `None`): + + Returns: + `datasets.table.Table`: shallow_copy + """ + replay = ("replace_schema_metadata", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.replace_schema_metadata(*args, **kwargs), self.path, replays) + + def add_column(self, *args, **kwargs): + """ + Add column to Table at position. + + A new table is returned with the column added, the original table + object is left unchanged. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column added. + """ + replay = ("add_column", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.add_column(*args, **kwargs), self.path, replays) + + def append_column(self, *args, **kwargs): + """ + Append column at end of columns. + + Args: + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column added. + """ + replay = ("append_column", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.append_column(*args, **kwargs), self.path, replays) + + def remove_column(self, *args, **kwargs): + """ + Create new Table with the indicated column removed. + + Args: + i (`int`): + Index of column to remove. + + Returns: + `datasets.table.Table`: + New table without the column. + """ + replay = ("remove_column", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.remove_column(*args, **kwargs), self.path, replays) + + def set_column(self, *args, **kwargs): + """ + Replace column in Table at position. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column set. + """ + replay = ("set_column", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.set_column(*args, **kwargs), self.path, replays) + + def rename_columns(self, *args, **kwargs): + """ + Create new table with columns renamed to provided names. + """ + replay = ("rename_columns", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.rename_columns(*args, **kwargs), self.path, replays) + + def drop(self, *args, **kwargs): + """ + Drop one or more columns and return a new table. + + Args: + columns (`List[str]`): + List of field names referencing existing columns. + + Raises: + `KeyError` : if any of the passed columns name are not existing. + + Returns: + `datasets.table.Table`: + New table without the columns. + """ + replay = ("drop", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.drop(*args, **kwargs), self.path, replays) + + def select(self, *args, **kwargs): + """ + Select columns of the table. + + Returns a new table with the specified columns, and metadata preserved. + + Args: + columns (:obj:`Union[List[str], List[int]]`): + The column names or integer indices to select. + + Returns: + :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved. + """ + replay = ("select", copy.deepcopy(args), copy.deepcopy(kwargs)) + replays = self._append_replay(replay) + return MemoryMappedTable(self.table.select(*args, **kwargs), self.path, replays) + + +# A ConcatenationTable is the concatenation of several tables. +# The ``blocks`` attributes stores a list of list of blocks. +# The first axis concatenates the tables along the axis 0 (it appends rows), +# while the second axis concatenates tables along the axis 1 (it appends columns). +TableBlockContainer = TypeVar("TableBlockContainer", TableBlock, list[TableBlock], list[list[TableBlock]]) + + +class ConcatenationTable(Table): + """ + The table comes from the concatenation of several tables called blocks. + It enables concatenation on both axis 0 (append rows) and axis 1 (append columns). + + The underlying tables are called "blocks" and can be either `InMemoryTable` + or `MemoryMappedTable` objects. + This allows to combine tables that come from memory or that are memory mapped. + When a `ConcatenationTable` is pickled, then each block is pickled: + - the `InMemoryTable` objects are pickled by copying all the data in memory. + - the MemoryMappedTable objects are pickled without copying the data into memory. + Instead, only the path to the memory mapped arrow file is pickled, as well as the list + of transforms to "replays" when reloading the table from the disk. + + Its implementation requires to store each block separately. + The `blocks` attributes stores a list of list of blocks. + The first axis concatenates the tables along the axis 0 (it appends rows), + while the second axis concatenates tables along the axis 1 (it appends columns). + + If some columns are missing when concatenating on axis 0, they are filled with null values. + This is done using `pyarrow.concat_tables(tables, promote=True)`. + + You can access the fully combined table by accessing the `ConcatenationTable.table` attribute, + and the blocks by accessing the `ConcatenationTable.blocks` attribute. + """ + + def __init__(self, table: pa.Table, blocks: list[list[TableBlock]]): + super().__init__(table) + self.blocks = blocks + # Check that all the blocks have the right type. + # Only InMemoryTable and MemoryMappedTable are allowed. + for subtables in blocks: + for subtable in subtables: + if not isinstance(subtable, TableBlock): + raise TypeError( + "The blocks of a ConcatenationTable must be InMemoryTable or MemoryMappedTable objects" + f", but got {_short_str(subtable)}." + ) + + def __getstate__(self): + return {"blocks": self.blocks, "schema": self.table.schema} + + def __setstate__(self, state): + blocks = state["blocks"] + schema = state["schema"] + table = self._concat_blocks_horizontally_and_vertically(blocks) + if schema is not None and table.schema != schema: + # We fix the columns by concatenating with an empty table with the right columns + empty_table = pa.Table.from_batches([], schema=schema) + # We set promote_options="default" to fill missing columns with null values + table = pa.concat_tables([table, empty_table], promote_options="default") + ConcatenationTable.__init__(self, table, blocks=blocks) + + @staticmethod + def _concat_blocks(blocks: list[Union[TableBlock, pa.Table]], axis: int = 0) -> pa.Table: + pa_tables = [table.table if hasattr(table, "table") else table for table in blocks] + if axis == 0: + # We set promote_options="default" to fill missing columns with null values + return pa.concat_tables(pa_tables, promote_options="default") + elif axis == 1: + for i, table in enumerate(pa_tables): + if i == 0: + pa_table = table + else: + for name, col in zip(table.column_names, table.columns): + pa_table = pa_table.append_column(name, col) + return pa_table + else: + raise ValueError("'axis' must be either 0 or 1") + + @classmethod + def _concat_blocks_horizontally_and_vertically(cls, blocks: list[list[TableBlock]]) -> pa.Table: + pa_tables_to_concat_vertically = [] + for i, tables in enumerate(blocks): + if not tables: + continue + pa_table_horizontally_concatenated = cls._concat_blocks(tables, axis=1) + pa_tables_to_concat_vertically.append(pa_table_horizontally_concatenated) + return cls._concat_blocks(pa_tables_to_concat_vertically, axis=0) + + @classmethod + def _merge_blocks(cls, blocks: TableBlockContainer, axis: Optional[int] = None) -> TableBlockContainer: + if axis is not None: + merged_blocks = [] + for is_in_memory, block_group in groupby(blocks, key=lambda x: isinstance(x, InMemoryTable)): + if is_in_memory: + block_group = [InMemoryTable(cls._concat_blocks(list(block_group), axis=axis))] + merged_blocks += list(block_group) + else: # both + merged_blocks = [cls._merge_blocks(row_block, axis=1) for row_block in blocks] + if all(len(row_block) == 1 for row_block in merged_blocks): + merged_blocks = cls._merge_blocks( + [block for row_block in merged_blocks for block in row_block], axis=0 + ) + return merged_blocks + + @classmethod + def _consolidate_blocks(cls, blocks: TableBlockContainer) -> TableBlockContainer: + if isinstance(blocks, TableBlock): + return blocks + elif isinstance(blocks[0], TableBlock): + return cls._merge_blocks(blocks, axis=0) + else: + return cls._merge_blocks(blocks) + + @classmethod + def from_blocks(cls, blocks: TableBlockContainer) -> "ConcatenationTable": + blocks = cls._consolidate_blocks(blocks) + if isinstance(blocks, TableBlock): + table = blocks + return cls(table.table, [[table]]) + elif isinstance(blocks[0], TableBlock): + table = cls._concat_blocks(blocks, axis=0) + blocks = [[t] for t in blocks] + return cls(table, blocks) + else: + table = cls._concat_blocks_horizontally_and_vertically(blocks) + return cls(table, blocks) + + @classmethod + def from_tables(cls, tables: list[Union[pa.Table, Table]], axis: int = 0) -> "ConcatenationTable": + """Create `ConcatenationTable` from list of tables. + + Args: + tables (list of `Table` or list of `pyarrow.Table`): + List of tables. + axis (`{0, 1}`, defaults to `0`, meaning over rows): + Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns + (horizontally). + + + """ + + def to_blocks(table: Union[pa.Table, Table]) -> list[list[TableBlock]]: + if isinstance(table, pa.Table): + return [[InMemoryTable(table)]] + elif isinstance(table, ConcatenationTable): + return copy.deepcopy(table.blocks) + else: + return [[table]] + + def _slice_row_block(row_block: list[TableBlock], length: int) -> tuple[list[TableBlock], list[TableBlock]]: + sliced = [table.slice(0, length) for table in row_block] + remainder = [table.slice(length, len(row_block[0]) - length) for table in row_block] + return sliced, remainder + + def _split_both_like( + result: list[list[TableBlock]], blocks: list[list[TableBlock]] + ) -> tuple[list[list[TableBlock]], list[list[TableBlock]]]: + """ + Make sure each row_block contain the same num_rows to be able to concatenate them on axis=1. + + To do so, we modify both blocks sets to have the same row_blocks boundaries. + For example, if `result` has 2 row_blocks of 3 rows and `blocks` has 3 row_blocks of 2 rows, + we modify both to have 4 row_blocks of size 2, 1, 1 and 2: + + [ x x x | x x x ] + + [ y y | y y | y y ] + ----------------------------- + = [ x x | x | x | x x ] + [ y y | y | y | y y ] + + """ + result, blocks = list(result), list(blocks) + new_result, new_blocks = [], [] + while result and blocks: + # we slice the longest row block to save two row blocks of same length + # and we replace the long row block by its remainder if necessary + if len(result[0][0]) > len(blocks[0][0]): + new_blocks.append(blocks[0]) + sliced, result[0] = _slice_row_block(result[0], len(blocks.pop(0)[0])) + new_result.append(sliced) + elif len(result[0][0]) < len(blocks[0][0]): + new_result.append(result[0]) + sliced, blocks[0] = _slice_row_block(blocks[0], len(result.pop(0)[0])) + new_blocks.append(sliced) + else: + new_result.append(result.pop(0)) + new_blocks.append(blocks.pop(0)) + if result or blocks: + raise ValueError("Failed to concatenate on axis=1 because tables don't have the same number of rows") + return new_result, new_blocks + + def _extend_blocks( + result: list[list[TableBlock]], blocks: list[list[TableBlock]], axis: int = 0 + ) -> list[list[TableBlock]]: + if axis == 0: + result.extend(blocks) + elif axis == 1: + # We make sure each row_block have the same num_rows + result, blocks = _split_both_like(result, blocks) + for i, row_block in enumerate(blocks): + result[i].extend(row_block) + return result + + blocks = to_blocks(tables[0]) + for table in tables[1:]: + table_blocks = to_blocks(table) + blocks = _extend_blocks(blocks, table_blocks, axis=axis) + return cls.from_blocks(blocks) + + @property + def _slices(self): + offset = 0 + for tables in self.blocks: + length = len(tables[0]) + yield (offset, length) + offset += length + + def slice(self, offset=0, length=None): + """ + Compute zero-copy slice of this Table. + + Args: + offset (`int`, defaults to `0`): + Offset from start of table to slice. + length (`int`, defaults to `None`): + Length of slice (default is until end of table starting from + offset). + + Returns: + `datasets.table.Table` + """ + table = self.table.slice(offset, length=length) + length = length if length is not None else self.num_rows - offset + blocks = [] + for tables in self.blocks: + n_rows = len(tables[0]) + if length == 0: + break + elif n_rows <= offset: + offset = offset - n_rows + elif n_rows <= offset + length: + blocks.append([t.slice(offset) for t in tables]) + length, offset = length + offset - n_rows, 0 + else: + blocks.append([t.slice(offset, length) for t in tables]) + length, offset = 0, 0 + return ConcatenationTable(table, blocks) + + def filter(self, mask, *args, **kwargs): + """ + Select records from a Table. See `pyarrow.compute.filter` for full usage. + """ + table = self.table.filter(mask, *args, **kwargs) + blocks = [] + for (offset, length), tables in zip(self._slices, self.blocks): + submask = mask.slice(offset, length) + blocks.append([t.filter(submask, *args, **kwargs) for t in tables]) + return ConcatenationTable(table, blocks) + + def flatten(self, *args, **kwargs): + """ + Flatten this Table. Each column with a struct type is flattened + into one column per struct field. Other columns are left unchanged. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + table = table_flatten(self.table, *args, **kwargs) + blocks = [] + for tables in self.blocks: + blocks.append([t.flatten(*args, **kwargs) for t in tables]) + return ConcatenationTable(table, blocks) + + def combine_chunks(self, *args, **kwargs): + """ + Make a new table by combining the chunks this table has. + + All the underlying chunks in the `ChunkedArray` of each column are + concatenated into zero or one chunk. + + Args: + memory_pool (`MemoryPool`, defaults to `None`): + For memory allocations, if required, otherwise use default pool. + + Returns: + `datasets.table.Table` + """ + table = self.table.combine_chunks(*args, **kwargs) + blocks = [] + for tables in self.blocks: + blocks.append([t.combine_chunks(*args, **kwargs) for t in tables]) + return ConcatenationTable(table, blocks) + + def cast(self, target_schema, *args, **kwargs): + """ + Cast table values to another schema. + + Args: + target_schema (`Schema`): + Schema to cast to, the names and order of fields must match. + safe (`bool`, defaults to `True`): + Check for overflows or other unsafe conversions. + + Returns: + `datasets.table.Table` + """ + from .features import Features + + table = table_cast(self.table, target_schema, *args, **kwargs) + target_features = Features.from_arrow_schema(target_schema) + blocks = [] + for subtables in self.blocks: + new_tables = [] + fields = list(target_schema) + for subtable in subtables: + subfields = [] + for name in subtable.column_names: + subfields.append(fields.pop(next(i for i, field in enumerate(fields) if field.name == name))) + subfeatures = Features({subfield.name: target_features[subfield.name] for subfield in subfields}) + subschema = subfeatures.arrow_schema + new_tables.append(subtable.cast(subschema, *args, **kwargs)) + blocks.append(new_tables) + return ConcatenationTable(table, blocks) + + def replace_schema_metadata(self, *args, **kwargs): + """ + EXPERIMENTAL: Create shallow copy of table by replacing schema + key-value metadata with the indicated new metadata (which may be `None`, + which deletes any existing metadata). + + Args: + metadata (`dict`, defaults to `None`): + + Returns: + `datasets.table.Table`: shallow_copy + """ + table = self.table.replace_schema_metadata(*args, **kwargs) + blocks = [] + for tables in self.blocks: + blocks.append([t.replace_schema_metadata(*args, **kwargs) for t in tables]) + return ConcatenationTable(table, self.blocks) + + def add_column(self, *args, **kwargs): + """ + Add column to Table at position. + + A new table is returned with the column added, the original table + object is left unchanged. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: New table with the passed column added. + """ + raise NotImplementedError() + + def append_column(self, *args, **kwargs): + """ + Append column at end of columns. + + Args: + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column added. + """ + raise NotImplementedError() + + def remove_column(self, i, *args, **kwargs): + """ + Create new Table with the indicated column removed. + + Args: + i (`int`): + Index of column to remove. + + Returns: + `datasets.table.Table`: + New table without the column. + """ + table = self.table.remove_column(i, *args, **kwargs) + name = self.table.column_names[i] + blocks = [] + for tables in self.blocks: + blocks.append( + [ + t.remove_column(t.column_names.index(name), *args, **kwargs) if name in t.column_names else t + for t in tables + ] + ) + return ConcatenationTable(table, blocks) + + def set_column(self, *args, **kwargs): + """ + Replace column in Table at position. + + Args: + i (`int`): + Index to place the column at. + field_ (`Union[str, pyarrow.Field]`): + If a string is passed then the type is deduced from the column + data. + column (`Union[pyarrow.Array, List[pyarrow.Array]]`): + Column data. + + Returns: + `datasets.table.Table`: + New table with the passed column set. + """ + raise NotImplementedError() + + def rename_columns(self, names, *args, **kwargs): + """ + Create new table with columns renamed to provided names. + """ + table = self.table.rename_columns(names, *args, **kwargs) + names = dict(zip(self.table.column_names, names)) + blocks = [] + for tables in self.blocks: + blocks.append( + [t.rename_columns([names[name] for name in t.column_names], *args, **kwargs) for t in tables] + ) + return ConcatenationTable(table, blocks) + + def drop(self, columns, *args, **kwargs): + """ + Drop one or more columns and return a new table. + + Args: + columns (`List[str]`): + List of field names referencing existing columns. + + Raises: + `KeyError` : if any of the passed columns name are not existing. + + Returns: + `datasets.table.Table`: + New table without the columns. + """ + table = self.table.drop(columns, *args, **kwargs) + blocks = [] + for tables in self.blocks: + blocks.append([t.drop([c for c in columns if c in t.column_names], *args, **kwargs) for t in tables]) + return ConcatenationTable(table, blocks) + + def select(self, columns, *args, **kwargs): + """ + Select columns of the table. + + Returns a new table with the specified columns, and metadata preserved. + + Args: + columns (:obj:`Union[List[str], List[int]]`): + The column names or integer indices to select. + + Returns: + :class:`datasets.table.Table`: New table with the specified columns, and metadata preserved. + """ + table = self.table.select(columns, *args, **kwargs) + blocks = [] + for tables in self.blocks: + blocks.append([t.select([c for c in columns if c in t.column_names], *args, **kwargs) for t in tables]) + return ConcatenationTable(table, blocks) + + +def concat_tables(tables: list[Table], axis: int = 0) -> Table: + """ + Concatenate tables. + + Args: + tables (list of `Table`): + List of tables to be concatenated. + axis (`{0, 1}`, defaults to `0`, meaning over rows): + Axis to concatenate over, where `0` means over rows (vertically) and `1` means over columns + (horizontally). + + + Returns: + `datasets.table.Table`: + If the number of input tables is > 1, then the returned table is a `datasets.table.ConcatenationTable`. + Otherwise if there's only one table, it is returned as is. + """ + tables = list(tables) + if len(tables) == 1: + return tables[0] + return ConcatenationTable.from_tables(tables, axis=axis) + + +def list_table_cache_files(table: Table) -> list[str]: + """ + Get the cache files that are loaded by the table. + Cache file are used when parts of the table come from the disk via memory mapping. + + Returns: + `List[str]`: + A list of paths to the cache files loaded by the table. + """ + if isinstance(table, ConcatenationTable): + cache_files = [] + for subtables in table.blocks: + for subtable in subtables: + cache_files += list_table_cache_files(subtable) + return cache_files + elif isinstance(table, MemoryMappedTable): + return [table.path] + else: + return [] + + +def _wrap_for_chunked_arrays(func): + """Apply the function on each chunk of a `pyarrow.ChunkedArray`, or on the array directly""" + + def wrapper(array, *args, **kwargs): + if isinstance(array, pa.ChunkedArray): + return pa.chunked_array([func(chunk, *args, **kwargs) for chunk in array.chunks]) + else: + return func(array, *args, **kwargs) + + return wrapper + + +def _are_list_values_of_length(array: pa.ListArray, length: int) -> bool: + """Check if all the sub-lists of a `pa.ListArray` have the specified length.""" + return pc.all(pc.equal(array.value_lengths(), length)).as_py() or array.null_count == len(array) + + +def _combine_list_array_offsets_with_mask(array: pa.ListArray) -> pa.Array: + """Add the null bitmap to the offsets of a `pa.ListArray`.""" + offsets = array.offsets + if array.null_count > 0: + offsets = pa.concat_arrays( + [ + pc.replace_with_mask(offsets[:-1], array.is_null(), pa.nulls(len(array), pa.int32())), + offsets[-1:], + ] + ) + return offsets + + +def _storage_type(type: pa.DataType) -> pa.DataType: + """Convert a (possibly nested) `pa.ExtensionType` to its storage type.""" + if isinstance(type, pa.ExtensionType): + return _storage_type(type.storage_type) + elif isinstance(type, pa.StructType): + return pa.struct([pa.field(field.name, _storage_type(field.type)) for field in type]) + elif isinstance(type, pa.ListType): + return pa.list_(_storage_type(type.value_type)) + elif isinstance(type, pa.FixedSizeListType): + return pa.list_(_storage_type(type.value_type), type.list_size) + return type + + +def _short_str(value: Any) -> str: + out = str(value) + if len(out) > 3000: + out = out[:1500] + "\n...\n" + out[-1500:] + return out + + +@_wrap_for_chunked_arrays +def array_cast( + array: pa.Array, pa_type: pa.DataType, allow_primitive_to_str: bool = True, allow_decimal_to_str: bool = True +) -> Union[pa.Array, pa.FixedSizeListArray, pa.ListArray, pa.StructArray, pa.ExtensionArray]: + """Improved version of `pa.Array.cast` + + It supports casting `pa.StructArray` objects to re-order the fields. + It also let you control certain aspects of the casting, e.g. whether + to disable casting primitives (`booleans`, `floats` or `ints`) or + disable casting decimals to strings. + + Args: + array (`pa.Array`): + PyArrow array to cast + pa_type (`pa.DataType`): + Target PyArrow type + allow_primitive_to_str (`bool`, defaults to `True`): + Whether to allow casting primitives to strings. + Defaults to `True`. + allow_decimal_to_str (`bool`, defaults to `True`): + Whether to allow casting decimals to strings. + Defaults to `True`. + + Raises: + `pa.ArrowInvalidError`: if the arrow data casting fails + `TypeError`: if the target type is not supported according, e.g. + + - if a field is missing + - if casting from primitives to strings and `allow_primitive_to_str` is `False` + - if casting from decimals to strings and `allow_decimal_to_str` is `False` + + Returns: + `List[pyarrow.Array]`: the casted array + """ + _c = partial(array_cast, allow_primitive_to_str=allow_primitive_to_str, allow_decimal_to_str=allow_decimal_to_str) + if isinstance(array, pa.ExtensionArray): + array = array.storage + if isinstance(pa_type, pa.ExtensionType): + return pa_type.wrap_array(_c(array, pa_type.storage_type)) + elif array.type == pa_type: + return array + elif pa.types.is_struct(array.type): + if pa.types.is_struct(pa_type) and ({field.name for field in pa_type} == {field.name for field in array.type}): + if array.type.num_fields == 0: + return array + arrays = [_c(array.field(field.name), field.type) for field in pa_type] + return pa.StructArray.from_arrays(arrays, fields=list(pa_type), mask=array.is_null()) + elif pa.types.is_list(array.type) or pa.types.is_large_list(array.type): + if pa.types.is_fixed_size_list(pa_type): + if _are_list_values_of_length(array, pa_type.list_size): + if array.null_count > 0: + # Ensure each null value in the array translates to [null] * pa_type.list_size in the array's values array + array_type = array.type + storage_type = _storage_type(array_type) + if array_type != storage_type: + # Temporarily convert to the storage type to support extension types in the slice operation + array = _c(array, storage_type) + array = pc.list_slice(array, 0, pa_type.list_size, return_fixed_size_list=True) + array = _c(array, array_type) + else: + array = pc.list_slice(array, 0, pa_type.list_size, return_fixed_size_list=True) + array_values = array.values + return pa.FixedSizeListArray.from_arrays( + _c(array_values, pa_type.value_type), pa_type.list_size, mask=array.is_null() + ) + else: + array_values = array.values[ + array.offset * pa_type.list_size : (array.offset + len(array)) * pa_type.list_size + ] + return pa.FixedSizeListArray.from_arrays(_c(array_values, pa_type.value_type), pa_type.list_size) + elif pa.types.is_list(pa_type): + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + return pa.ListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type)) + elif pa.types.is_large_list(pa_type): + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + return pa.LargeListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type)) + elif pa.types.is_fixed_size_list(array.type): + if pa.types.is_fixed_size_list(pa_type): + if pa_type.list_size == array.type.list_size: + array_values = array.values[ + array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size + ] + return pa.FixedSizeListArray.from_arrays( + _c(array_values, pa_type.value_type), pa_type.list_size, mask=array.is_null() + ) + elif pa.types.is_list(pa_type): + array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size + return pa.ListArray.from_arrays(array_offsets, _c(array.values, pa_type.value_type), mask=array.is_null()) + elif pa.types.is_large_list(pa_type): + array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size + return pa.LargeListArray.from_arrays( + array_offsets, _c(array.values, pa_type.value_type), mask=array.is_null() + ) + else: + if pa.types.is_string(pa_type): + if not allow_primitive_to_str and pa.types.is_primitive(array.type): + raise TypeError( + f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)} " + f"since allow_primitive_to_str is set to {allow_primitive_to_str} " + ) + if not allow_decimal_to_str and pa.types.is_decimal(array.type): + raise TypeError( + f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)} " + f"and allow_decimal_to_str is set to {allow_decimal_to_str}" + ) + if pa.types.is_null(pa_type) and not pa.types.is_null(array.type): + raise TypeError(f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)}") + return array.cast(pa_type) + raise TypeError(f"Couldn't cast array of type {_short_str(array.type)} to {_short_str(pa_type)}") + + +@_wrap_for_chunked_arrays +def cast_array_to_feature( + array: pa.Array, feature: "FeatureType", allow_primitive_to_str: bool = True, allow_decimal_to_str: bool = True +) -> pa.Array: + """Cast an array to the arrow type that corresponds to the requested feature type. + For custom features like [`Audio`] or [`Image`], it takes into account the "cast_storage" methods + they defined to enable casting from other arrow types. + + Args: + array (`pa.Array`): + The PyArrow array to cast. + feature (`datasets.features.FeatureType`): + The target feature type. + allow_primitive_to_str (`bool`, defaults to `True`): + Whether to allow casting primitives to strings. + Defaults to `True`. + allow_decimal_to_str (`bool`, defaults to `True`): + Whether to allow casting decimals to strings. + Defaults to `True`. + + Raises: + `pa.ArrowInvalidError`: if the arrow data casting fails + `TypeError`: if the target type is not supported according, e.g. + + - if a field is missing + - if casting from primitives and `allow_primitive_to_str` is `False` + - if casting from decimals and `allow_decimal_to_str` is `False` + + Returns: + array (`pyarrow.Array`): the casted array + """ + from .features.features import LargeList, List, get_nested_type + + _c = partial( + cast_array_to_feature, + allow_primitive_to_str=allow_primitive_to_str, + allow_decimal_to_str=allow_decimal_to_str, + ) + + if isinstance(array, pa.ExtensionArray): + array = array.storage + if hasattr(feature, "cast_storage"): + return feature.cast_storage(array) + + if pa.types.is_struct(array.type): + # feature must be a dict + if isinstance(feature, dict) and (array_fields := {field.name for field in array.type}) <= set(feature): + null_array = pa.array([None] * len(array)) + arrays = [ + _c(array.field(name) if name in array_fields else null_array, subfeature) + for name, subfeature in feature.items() + ] + return pa.StructArray.from_arrays(arrays, names=list(feature), mask=array.is_null()) + elif pa.types.is_list(array.type) or pa.types.is_large_list(array.type): + # feature must be either List(subfeature) or LargeList(subfeature) + if isinstance(feature, LargeList): + casted_array_values = _c(array.values, feature.feature) + if pa.types.is_large_list(array.type) and casted_array_values.type == array.values.type: + # Both array and feature have equal large_list type and values (within the list) type + return array + else: + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + return pa.LargeListArray.from_arrays(array_offsets, casted_array_values) + elif isinstance(feature, List): + if feature.length > -1: + if _are_list_values_of_length(array, feature.length): + if array.null_count > 0: + # Ensure each null value in the array translates to [null] * pa_type.list_size in the array's values array + array_type = array.type + storage_type = _storage_type(array_type) + if array_type != storage_type: + # Temporarily convert to the storage type to support extension types in the slice operation + array = array_cast( + array, + storage_type, + allow_primitive_to_str=allow_primitive_to_str, + allow_decimal_to_str=allow_decimal_to_str, + ) + array = pc.list_slice(array, 0, feature.length, return_fixed_size_list=True) + array = array_cast( + array, + array_type, + allow_primitive_to_str=allow_primitive_to_str, + allow_decimal_to_str=allow_decimal_to_str, + ) + else: + array = pc.list_slice(array, 0, feature.length, return_fixed_size_list=True) + array_values = array.values + casted_array_values = _c(array_values, feature.feature) + return pa.FixedSizeListArray.from_arrays( + casted_array_values, feature.length, mask=array.is_null() + ) + else: + array_values = array.values[ + array.offset * feature.length : (array.offset + len(array)) * feature.length + ] + return pa.FixedSizeListArray.from_arrays(_c(array_values, feature.feature), feature.length) + else: + casted_array_values = _c(array.values, feature.feature) + if pa.types.is_list(array.type) and casted_array_values.type == array.values.type: + # Both array and feature have equal list type and values (within the list) type + return array + else: + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + return pa.ListArray.from_arrays(array_offsets, casted_array_values) + elif pa.types.is_fixed_size_list(array.type): + # feature must be List(subfeature) + if isinstance(feature, LargeList): + array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size + return pa.LargeListArray.from_arrays( + array_offsets, _c(array.values, feature.feature), mask=array.is_null() + ) + elif isinstance(feature, List): + if feature.length > -1: + if feature.length == array.type.list_size: + array_values = array.values[ + array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size + ] + casted_array_values = _c(array_values, feature.feature) + return pa.FixedSizeListArray.from_arrays(casted_array_values, feature.length, mask=array.is_null()) + else: + array_offsets = (np.arange(len(array) + 1) + array.offset) * array.type.list_size + return pa.ListArray.from_arrays(array_offsets, _c(array.values, feature.feature), mask=array.is_null()) + if pa.types.is_null(array.type): + return array_cast( + array, + get_nested_type(feature), + allow_primitive_to_str=allow_primitive_to_str, + allow_decimal_to_str=allow_decimal_to_str, + ) + elif not isinstance(feature, (List, LargeList, dict)): + return array_cast( + array, + feature(), + allow_primitive_to_str=allow_primitive_to_str, + allow_decimal_to_str=allow_decimal_to_str, + ) + raise TypeError(f"Couldn't cast array of type\n{_short_str(array.type)}\nto\n{_short_str(feature)}") + + +@_wrap_for_chunked_arrays +def embed_array_storage( + array: pa.Array, + feature: "FeatureType", + token_per_repo_id=None, + local_files: bool = True, + remote_files: bool = True, +): + """Embed data into an arrays's storage. + For custom features like Audio or Image, it takes into account the "embed_storage" methods + they define to embed external data (e.g. an image file) into an array. + + + + Args: + array (`pa.Array`): + The PyArrow array in which to embed data. + feature (`datasets.features.FeatureType`): + Array features. + local_files (`bool`, defaults to `True`) + Whether to embed local files data in the array + + + remote_files (`bool`, defaults to `True`) + Whether to embed remote files data in the array. + E.g. files with paths that start with hf:// or https:// + + + + Raises: + `TypeError`: if the target type is not supported according, e.g. + + - if a field is missing + + Returns: + array (`pyarrow.Array`): the casted array + """ + if not local_files and not remote_files: + return array + + from .features import LargeList, List + + _e = partial( + embed_array_storage, token_per_repo_id=token_per_repo_id, local_files=local_files, remote_files=remote_files + ) + + if isinstance(array, pa.ExtensionArray): + array = array.storage + if hasattr(feature, "embed_storage"): + return feature.embed_storage( + array, token_per_repo_id=token_per_repo_id, local_files=local_files, remote_files=remote_files + ) + elif pa.types.is_struct(array.type): + # feature must be a dict + if isinstance(feature, dict): + arrays = [_e(array.field(name), subfeature) for name, subfeature in feature.items()] + return pa.StructArray.from_arrays(arrays, names=list(feature), mask=array.is_null()) + elif pa.types.is_list(array.type): + # feature must be either List(subfeature) + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + if isinstance(feature, List) and feature.length == -1: + return pa.ListArray.from_arrays(array_offsets, _e(array.values, feature.feature)) + elif pa.types.is_large_list(array.type): + # feature must be LargeList(subfeature) + # Merge offsets with the null bitmap to avoid the "Null bitmap with offsets slice not supported" ArrowNotImplementedError + array_offsets = _combine_list_array_offsets_with_mask(array) + return pa.LargeListArray.from_arrays(array_offsets, _e(array.values, feature.feature)) + elif pa.types.is_fixed_size_list(array.type): + # feature must be List(subfeature) + if isinstance(feature, List) and feature.length > -1: + array_values = array.values[ + array.offset * array.type.list_size : (array.offset + len(array)) * array.type.list_size + ] + embedded_array_values = _e(array_values, feature.feature) + return pa.FixedSizeListArray.from_arrays(embedded_array_values, feature.length, mask=array.is_null()) + if not isinstance(feature, (List, LargeList, dict)): + return array + raise TypeError(f"Couldn't embed array of type\n{_short_str(array.type)}\nwith\n{_short_str(feature)}") + + +class CastError(ValueError): + """When it's not possible to cast an Arrow table to a specific schema or set of features""" + + def __init__(self, *args, table_column_names: list[str], requested_column_names: list[str]) -> None: + super().__init__(*args) + self.table_column_names = table_column_names + self.requested_column_names = requested_column_names + + def __reduce__(self): + # Fix unpickling: TypeError: __init__() missing 2 required keyword-only arguments: 'table_column_names' and 'requested_column_names' + return partial( + CastError, table_column_names=self.table_column_names, requested_column_names=self.requested_column_names + ), () + + def details(self): + new_columns = set(self.table_column_names) - set(self.requested_column_names) + missing_columns = set(self.requested_column_names) - set(self.table_column_names) + if new_columns and missing_columns: + return f"there are {len(new_columns)} new columns ({_short_str(new_columns)}) and {len(missing_columns)} missing columns ({_short_str(missing_columns)})." + elif new_columns: + return f"there are {len(new_columns)} new columns ({_short_str(new_columns)})" + else: + return f"there are {len(missing_columns)} missing columns ({_short_str(missing_columns)})" + + +def cast_table_to_features(table: pa.Table, features: "Features"): + """Cast a table to the arrow schema that corresponds to the requested features. + + Args: + table (`pyarrow.Table`): + PyArrow table to cast. + features ([`Features`]): + Target features. + + Returns: + table (`pyarrow.Table`): the casted table + """ + if sorted(table.column_names) != sorted(features): + raise CastError( + f"Couldn't cast\n{_short_str(table.schema)}\nto\n{_short_str(features)}\nbecause column names don't match", + table_column_names=table.column_names, + requested_column_names=list(features), + ) + arrays = [cast_array_to_feature(table[name], feature) for name, feature in features.items()] + return pa.Table.from_arrays(arrays, schema=features.arrow_schema) + + +def cast_table_to_schema(table: pa.Table, schema: pa.Schema): + """Cast a table to the arrow schema. Different from `cast_table_to_features`, this method can preserve nullability. + + Args: + table (`pa.Table`): + PyArrow table to cast. + features ([`Features`]): + Target features. + + Returns: + `pa.Table`: the casted table + """ + from .features import Features + + features = Features.from_arrow_schema(schema) + table_column_names = set(table.column_names) + if not table_column_names <= set(schema.names): + raise CastError( + f"Couldn't cast\n{_short_str(table.schema)}\nto\n{_short_str(features)}\nbecause column names don't match", + table_column_names=table.column_names, + requested_column_names=list(features), + ) + arrays = [ + cast_array_to_feature( + table[name] if name in table_column_names else pa.array([None] * len(table), type=schema.field(name).type), + feature, + ) + for name, feature in features.items() + ] + return pa.Table.from_arrays(arrays, schema=schema) + + +def embed_table_storage(table: pa.Table, token_per_repo_id=None, local_files: bool = True, remote_files: bool = True): + """Embed external data into a table's storage. + + + + Args: + table (`pyarrow.Table`): + PyArrow table in which to embed data. + local_files (`bool`, defaults to `True`) + Whether to embed local files data in the table + + + remote_files (`bool`, defaults to `True`) + Whether to embed remote files data in the table. + E.g. files with paths that start with hf:// or https:// + + + + Returns: + table (`pyarrow.Table`): the table with embedded data + """ + if not local_files and not remote_files: + return table + + from .features.features import Features, require_storage_embed + + features = Features.from_arrow_schema(table.schema) + arrays = [ + embed_array_storage( + table[name], + feature, + token_per_repo_id=token_per_repo_id, + local_files=local_files, + remote_files=remote_files, + ) + if require_storage_embed(feature) + else table[name] + for name, feature in features.items() + ] + return pa.Table.from_arrays(arrays, schema=features.arrow_schema) + + +def table_cast(table: pa.Table, schema: pa.Schema): + """Improved version of `pa.Table.cast`. + + It supports casting to feature types stored in the schema metadata. + + Args: + table (`pyarrow.Table`): + PyArrow table to cast. + schema (`pyarrow.Schema`): + Target PyArrow schema. + + Returns: + table (`pyarrow.Table`): the casted table + """ + if table.schema != schema: + return cast_table_to_schema(table, schema) + elif table.schema.metadata != schema.metadata: + return table.replace_schema_metadata(schema.metadata) + else: + return table + + +def table_flatten(table: pa.Table): + """Improved version of `pa.Table.flatten`. + + It behaves as `pa.Table.flatten` in a sense it does 1-step flatten of the columns with a struct type into one column per struct field, + but updates the metadata and skips decodable features unless the `decode` attribute of these features is set to False. + + Args: + table (`pa.Table`): + PyArrow table to flatten. + + Returns: + `Table`: the flattened table + """ + from .features import Features + + features = Features.from_arrow_schema(table.schema) + if any(hasattr(subfeature, "flatten") and subfeature.flatten() == subfeature for subfeature in features.values()): + flat_arrays = [] + flat_column_names = [] + for field in table.schema: + array = table.column(field.name) + subfeature = features[field.name] + if pa.types.is_struct(field.type) and ( + not hasattr(subfeature, "flatten") or subfeature.flatten() != subfeature + ): + flat_arrays.extend(array.flatten()) + flat_column_names.extend([f"{field.name}.{subfield.name}" for subfield in field.type]) + else: + flat_arrays.append(array) + flat_column_names.append(field.name) + flat_table = pa.Table.from_arrays( + flat_arrays, + names=flat_column_names, + ) + else: + flat_table = table.flatten() + # Preserve complex types in the metadata + flat_features = features.flatten(max_depth=2) + flat_features = Features({column_name: flat_features[column_name] for column_name in flat_table.column_names}) + return flat_table.replace_schema_metadata(flat_features.arrow_schema.metadata) + + +def table_visitor(table: pa.Table, function: Callable[[pa.Array], None]): + """Visit all arrays in a table and apply a function to them. + + Args: + table (`pyarrow.Table`): + PyArrow table to visit. + function (`Callable[[pa.Array], None]`): + Function to apply to each array. + """ + from .features import Features, LargeList, List + + features = Features.from_arrow_schema(table.schema) + + def _visit(array, feature): + if isinstance(array, pa.ChunkedArray): + for chunk in array.chunks: + _visit(chunk, feature) + else: + if isinstance(array, pa.ExtensionArray): + array = array.storage + function(array, feature) + if pa.types.is_struct(array.type) and not hasattr(feature, "cast_storage"): + for name, subfeature in feature.items(): + _visit(array.field(name), subfeature) + elif pa.types.is_list(array.type): + if isinstance(feature, (LargeList, List)): + _visit(array.values, feature.feature) + + for name, feature in features.items(): + _visit(table[name], feature) + + +def table_iter(table: Table, batch_size: int, drop_last_batch=False) -> Iterator[pa.Table]: + """Iterate over sub-tables of size `batch_size`. + + Args: + table (`pyarrow.Table`): + PyArrow table to iterate over. + batch_size (`int`): + Size of each sub-table to yield. + drop_last_batch (`bool`, defaults to `False`): + Drop the last batch if it is smaller than `batch_size`. + """ + chunks_buffer = [] + chunks_buffer_size = 0 + for chunk in table.to_reader(max_chunksize=batch_size): + if len(chunk) == 0: + continue + elif chunks_buffer_size + len(chunk) < batch_size: + chunks_buffer.append(chunk) + chunks_buffer_size += len(chunk) + continue + elif chunks_buffer_size + len(chunk) == batch_size: + chunks_buffer.append(chunk) + yield pa.Table.from_batches(chunks_buffer) + chunks_buffer = [] + chunks_buffer_size = 0 + else: + cropped_chunk_length = batch_size - chunks_buffer_size + chunks_buffer.append(chunk.slice(0, cropped_chunk_length)) + yield pa.Table.from_batches(chunks_buffer) + chunks_buffer = [chunk.slice(cropped_chunk_length, len(chunk) - cropped_chunk_length)] + chunks_buffer_size = len(chunk) - cropped_chunk_length + if not drop_last_batch and chunks_buffer: + yield pa.Table.from_batches(chunks_buffer) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1c3d2935994956e230a441f3f2d65695ebaaf2d --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import tqdm as _tqdm # _tqdm is the module +from .experimental import experimental +from .info_utils import VerificationMode +from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled +from .tqdm import ( + are_progress_bars_disabled, + disable_progress_bars, + enable_progress_bars, + tqdm, +) +from .version import Version diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/_dataset_viewer.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/_dataset_viewer.py new file mode 100644 index 0000000000000000000000000000000000000000..0f4c298646c5534a93ecfd217b84987b4786cec6 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/_dataset_viewer.py @@ -0,0 +1,95 @@ +from typing import Any, Optional, Union + +from huggingface_hub.utils import get_session + +from .. import config +from ..exceptions import DatasetsError +from .file_utils import ( + get_authentication_headers_for_url, +) +from .logging import get_logger + + +logger = get_logger(__name__) + + +class DatasetViewerError(DatasetsError): + """Dataset viewer error. + + Raised when trying to use the dataset viewer HTTP API and when trying to access: + - a missing dataset, or + - a private/gated dataset and the user is not authenticated. + - unavailable /parquet or /info responses + """ + + +def get_exported_parquet_files( + dataset: str, commit_hash: str, token: Optional[Union[str, bool]] +) -> list[dict[str, Any]]: + """ + Get the dataset exported parquet files + Docs: https://huggingface.co/docs/datasets-server/parquet + """ + dataset_viewer_parquet_url = config.HF_ENDPOINT.replace("://", "://datasets-server.") + "/parquet?dataset=" + try: + parquet_data_files_response = get_session().get( + url=dataset_viewer_parquet_url + dataset, + headers=get_authentication_headers_for_url(config.HF_ENDPOINT + f"datasets/{dataset}", token=token), + timeout=100.0, + ) + parquet_data_files_response.raise_for_status() + if "X-Revision" in parquet_data_files_response.headers: + if parquet_data_files_response.headers["X-Revision"] == commit_hash or commit_hash is None: + parquet_data_files_response_json = parquet_data_files_response.json() + if ( + parquet_data_files_response_json.get("partial") is False + and not parquet_data_files_response_json.get("pending", True) + and not parquet_data_files_response_json.get("failed", True) + and "parquet_files" in parquet_data_files_response_json + ): + return parquet_data_files_response_json["parquet_files"] + else: + logger.debug(f"Parquet export for {dataset} is not completely ready yet.") + else: + logger.debug( + f"Parquet export for {dataset} is available but outdated (commit_hash='{parquet_data_files_response.headers['X-Revision']}')" + ) + except Exception as e: # noqa catch any exception of the dataset viewer API and consider the parquet export doesn't exist + logger.debug(f"No parquet export for {dataset} available ({type(e).__name__}: {e})") + raise DatasetViewerError("No exported Parquet files available.") + + +def get_exported_dataset_infos( + dataset: str, commit_hash: str, token: Optional[Union[str, bool]] +) -> dict[str, dict[str, Any]]: + """ + Get the dataset information, can be useful to get e.g. the dataset features. + Docs: https://huggingface.co/docs/datasets-server/info + """ + dataset_viewer_info_url = config.HF_ENDPOINT.replace("://", "://datasets-server.") + "/info?dataset=" + try: + info_response = get_session().get( + url=dataset_viewer_info_url + dataset, + headers=get_authentication_headers_for_url(config.HF_ENDPOINT + f"datasets/{dataset}", token=token), + timeout=100.0, + ) + info_response.raise_for_status() + if "X-Revision" in info_response.headers: + if info_response.headers["X-Revision"] == commit_hash or commit_hash is None: + info_response = info_response.json() + if ( + info_response.get("partial") is False + and not info_response.get("pending", True) + and not info_response.get("failed", True) + and "dataset_info" in info_response + ): + return info_response["dataset_info"] + else: + logger.debug(f"Dataset info for {dataset} is not completely ready yet.") + else: + logger.debug( + f"Dataset info for {dataset} is available but outdated (commit_hash='{info_response.headers['X-Revision']}')" + ) + except Exception as e: # noqa catch any exception of the dataset viewer API and consider the dataset info doesn't exist + logger.debug(f"No dataset info for {dataset} available ({type(e).__name__}: {e})") + raise DatasetViewerError("No exported dataset infos available.") diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/_dill.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/_dill.py new file mode 100644 index 0000000000000000000000000000000000000000..18300966b089483c692e84f9aa854b34d6a6958a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/_dill.py @@ -0,0 +1,505 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Extends `dill` to support pickling more types and produce more consistent dumps.""" + +import os +import sys +from io import BytesIO +from types import CodeType, FunctionType + +import dill +from packaging import version + +from .. import config + + +class Pickler(dill.Pickler): + dispatch = dill._dill.MetaCatchingDict(dill.Pickler.dispatch.copy()) + _legacy_no_dict_keys_sorting = False + + def save(self, obj, save_persistent_id=True): + obj_type = type(obj) + if obj_type not in self.dispatch: + if "regex" in sys.modules: + import regex # type: ignore + + if obj_type is regex.Pattern: + pklregister(obj_type)(_save_regexPattern) + if "spacy" in sys.modules: + import spacy # type: ignore + + if issubclass(obj_type, spacy.Language): + pklregister(obj_type)(_save_spacyLanguage) + if "tiktoken" in sys.modules: + import tiktoken # type: ignore + + if obj_type is tiktoken.Encoding: + pklregister(obj_type)(_save_tiktokenEncoding) + if "torch" in sys.modules: + import torch # type: ignore + + if issubclass(obj_type, torch.Tensor): + pklregister(obj_type)(_save_torchTensor) + + if obj_type is torch.Generator: + pklregister(obj_type)(_save_torchGenerator) + + # Unwrap `torch.compile`-ed modules + if issubclass(obj_type, torch.nn.Module): + obj = getattr(obj, "_orig_mod", obj) + if "transformers" in sys.modules: + import transformers # type: ignore + + if issubclass(obj_type, transformers.PreTrainedTokenizerBase): + pklregister(obj_type)(_save_transformersPreTrainedTokenizerBase) + + # Unwrap `torch.compile`-ed functions + if obj_type is FunctionType: + obj = getattr(obj, "_torchdynamo_orig_callable", obj) + dill.Pickler.save(self, obj, save_persistent_id=save_persistent_id) + + def _batch_setitems(self, items, *args, **kwargs): + # Ignore the order of keys in a dict + try: + # Faster, but fails for unorderable elements + items = sorted(items) + except Exception: # TypeError, decimal.InvalidOperation, etc. + from datasets.fingerprint import Hasher + + items = sorted(items, key=lambda x: Hasher.hash(x[0])) + return super()._batch_setitems(items, *args, **kwargs) + + def memoize(self, obj): + # Don't memoize strings since two identical strings can have different Python ids + if type(obj) is not str: # noqa: E721 + dill.Pickler.memoize(self, obj) + + +def pklregister(t): + """Register a custom reducer for the type.""" + + def proxy(func): + Pickler.dispatch[t] = func + return func + + return proxy + + +def _is_supported_dill_version(): + """Check if the current dill version is in the supported range.""" + return config.DILL_VERSION.release[:3] in [ + version.parse("0.3.6").release, + version.parse("0.3.7").release, + version.parse("0.3.8").release, + version.parse("0.3.9").release, + version.parse("0.4.0").release, + version.parse("0.4.1").release, + ] + + +def dump(obj, file): + """Pickle an object to a file.""" + Pickler(file, recurse=True).dump(obj) + + +def dumps(obj): + """Pickle an object to a string.""" + file = BytesIO() + dump(obj, file) + return file.getvalue() + + +if config.DILL_VERSION < version.parse("0.3.6"): + + def log(pickler, msg): + dill._dill.log.info(msg) + +elif _is_supported_dill_version(): + + def log(pickler, msg): + dill._dill.logger.trace(pickler, msg) + + +@pklregister(set) +def _save_set(pickler, obj): + log(pickler, f"Se: {obj}") + try: + # Faster, but fails for unorderable elements + args = (sorted(obj),) + except Exception: # TypeError, decimal.InvalidOperation, etc. + from datasets.fingerprint import Hasher + + args = (sorted(obj, key=Hasher.hash),) + + pickler.save_reduce(set, args, obj=obj) + log(pickler, "# Se") + + +def _save_regexPattern(pickler, obj): + import regex # type: ignore + + log(pickler, f"Re: {obj}") + args = (obj.pattern, obj.flags) + pickler.save_reduce(regex.compile, args, obj=obj) + log(pickler, "# Re") + + +def _save_tiktokenEncoding(pickler, obj): + import tiktoken # type: ignore + + log(pickler, f"Enc: {obj}") + args = (obj.name, obj._pat_str, obj._mergeable_ranks, obj._special_tokens) + pickler.save_reduce(tiktoken.Encoding, args, obj=obj) + log(pickler, "# Enc") + + +def _save_torchTensor(pickler, obj): + import torch # type: ignore + + # `torch.from_numpy` is not picklable in `torch>=1.11.0` + def create_torchTensor(np_array, dtype=None): + tensor = torch.from_numpy(np_array) + if dtype: + tensor = tensor.type(dtype) + return tensor + + log(pickler, f"To: {obj}") + if obj.dtype == torch.bfloat16: + args = (obj.detach().to(torch.float).cpu().numpy(), torch.bfloat16) + else: + args = (obj.detach().cpu().numpy(),) + pickler.save_reduce(create_torchTensor, args, obj=obj) + log(pickler, "# To") + + +def _save_torchGenerator(pickler, obj): + import torch # type: ignore + + def create_torchGenerator(state): + generator = torch.Generator() + generator.set_state(state) + return generator + + log(pickler, f"Ge: {obj}") + args = (obj.get_state(),) + pickler.save_reduce(create_torchGenerator, args, obj=obj) + log(pickler, "# Ge") + + +def _save_spacyLanguage(pickler, obj): + import spacy # type: ignore + + def create_spacyLanguage(config, bytes): + lang_cls = spacy.util.get_lang_class(config["nlp"]["lang"]) + lang_inst = lang_cls.from_config(config) + return lang_inst.from_bytes(bytes) + + log(pickler, f"Sp: {obj}") + args = (obj.config, obj.to_bytes()) + pickler.save_reduce(create_spacyLanguage, args, obj=obj) + log(pickler, "# Sp") + + +def _save_transformersPreTrainedTokenizerBase(pickler, obj): + log(pickler, f"Tok: {obj}") + # Ignore the `cache` attribute and make hashing stable. + # + # Some tokenizers backed by the `tokenizers` library mutate their internal `_tokenizer` state when called + # (e.g. by enabling truncation/padding). This can change the serialized bytes across runs and make dataset + # fingerprints unstable, which prevents `.map(load_from_cache_file=True)` from reusing cache files. + # + # For hashing/fingerprinting, we temporarily disable backend truncation/padding to avoid these runtime settings + # affecting the fingerprint, then restore the original settings. + state = obj.__dict__.copy() + if "cache" in state and isinstance(state["cache"], dict): + state["cache"] = {} + if "deprecation_warnings" in state and isinstance(state["deprecation_warnings"], dict): + state["deprecation_warnings"] = {} + + backend_tokenizer = obj.__dict__.get("_tokenizer") + truncation = padding = None + if ( + backend_tokenizer is not None + and hasattr(backend_tokenizer, "truncation") + and hasattr(backend_tokenizer, "padding") + ): + truncation = backend_tokenizer.truncation + padding = backend_tokenizer.padding + try: + if truncation is not None and hasattr(backend_tokenizer, "no_truncation"): + backend_tokenizer.no_truncation() + if padding is not None and hasattr(backend_tokenizer, "no_padding"): + backend_tokenizer.no_padding() + except Exception: + truncation = padding = None + + try: + pickler.save_reduce(type(obj), (), state=state, obj=obj) + finally: + try: + if backend_tokenizer is not None: + if truncation is not None and hasattr(backend_tokenizer, "enable_truncation"): + backend_tokenizer.enable_truncation(**truncation) + if padding is not None and hasattr(backend_tokenizer, "enable_padding"): + backend_tokenizer.enable_padding(**padding) + except Exception: + pass + log(pickler, "# Tok") + + +if config.DILL_VERSION < version.parse("0.3.6"): + + @pklregister(CodeType) + def _save_code(pickler, obj): + """ + From dill._dill.save_code + This is a modified version that removes the origin (filename + line no.) + of functions created in notebooks or shells for example. + """ + dill._dill.log.info(f"Co: {obj}") + # The filename of a function is the .py file where it is defined. + # Filenames of functions created in notebooks or shells start with '<' + # ex: for ipython, and for shell + # Filenames of functions created in ipykernel the filename + # look like f"{tempdir}/ipykernel_{id1}/{id2}.py" + # Moreover lambda functions have a special name: '' + # ex: (lambda x: x).__code__.co_name == "" # True + # + # For the hashing mechanism we ignore where the function has been defined + # More specifically: + # - we ignore the filename of special functions (filename starts with '<') + # - we always ignore the line number + # - we only use the base name of the file instead of the whole path, + # to be robust in case a script is moved for example. + # + # Only those two lines are different from the original implementation: + co_filename = ( + "" + if obj.co_filename.startswith("<") + or ( + len(obj.co_filename.split(os.path.sep)) > 1 + and obj.co_filename.split(os.path.sep)[-2].startswith("ipykernel_") + ) + or obj.co_name == "" + else os.path.basename(obj.co_filename) + ) + co_firstlineno = 1 + # The rest is the same as in the original dill implementation (with also a version check for 3.10) + if dill._dill.PY3: + if hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) + args = ( + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, + obj.co_name, + co_firstlineno, + obj.co_linetable if sys.version_info >= (3, 10) else obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + else: # python 3.7 (15 args) + args = ( + obj.co_argcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, + obj.co_name, + co_firstlineno, + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + else: + args = ( + obj.co_argcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, + obj.co_name, + co_firstlineno, + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + pickler.save_reduce(CodeType, args, obj=obj) + dill._dill.log.info("# Co") + return + +elif _is_supported_dill_version(): + # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1104 + @pklregister(CodeType) + def save_code(pickler, obj): + dill._dill.logger.trace(pickler, "Co: %s", obj) + + ############################################################################################################ + # Modification here for huggingface/datasets + # The filename of a function is the .py file where it is defined. + # Filenames of functions created in notebooks or shells start with '<' + # ex: for ipython, and for shell + # Filenames of functions created in ipykernel the filename + # look like f"{tempdir}/ipykernel_{id1}/{id2}.py" + # Moreover lambda functions have a special name: '' + # ex: (lambda x: x).__code__.co_name == "" # True + # + # For the hashing mechanism we ignore where the function has been defined + # More specifically: + # - we ignore the filename of special functions (filename starts with '<') + # - we always ignore the line number + # - we only use the base name of the file instead of the whole path, + # to be robust in case a script is moved for example. + # + # Only those two lines are different from the original implementation: + co_filename = ( + "" + if obj.co_filename.startswith("<") + or ( + len(obj.co_filename.split(os.path.sep)) > 1 + and obj.co_filename.split(os.path.sep)[-2].startswith("ipykernel_") + ) + or obj.co_name == "" + else os.path.basename(obj.co_filename) + ) + co_firstlineno = 1 + # The rest is the same as in the original dill implementation, except for the replacements: + # - obj.co_filename => co_filename + # - obj.co_firstlineno => co_firstlineno + # - obj.co_lnotab => obj.co_linetable for >= 3.10 since co_lnotab was deprecated + ############################################################################################################ + + if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) + args = ( + obj.co_linetable, # Modification for huggingface/datasets ############################################ + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + obj.co_qualname, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_linetable, + obj.co_endlinetable, + obj.co_columntable, + obj.co_exceptiontable, + obj.co_freevars, + obj.co_cellvars, + ) + elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) + args = ( + obj.co_linetable, # Modification for huggingface/datasets ####################################### + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + obj.co_qualname, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_linetable, + obj.co_exceptiontable, + obj.co_freevars, + obj.co_cellvars, + ) + elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) + args = ( + obj.co_linetable, # Modification for huggingface/datasets ####################################### + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_linetable, + obj.co_freevars, + obj.co_cellvars, + ) + elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) + args = ( + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + else: # python 3.7 (15 args) + args = ( + obj.co_argcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + + pickler.save_reduce(dill._dill._create_code, args, obj=obj) + dill._dill.logger.trace(pickler, "# Co") + return diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/_filelock.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/_filelock.py new file mode 100644 index 0000000000000000000000000000000000000000..803ce77f5a4168ac9023e1d4a4548db2bd154126 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/_filelock.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +"""Utilities to handle file locking in `datasets`.""" + +import os + +from filelock import FileLock as FileLock_ +from filelock import UnixFileLock +from filelock import __version__ as _filelock_version +from packaging import version + + +class FileLock(FileLock_): + """ + A `filelock.FileLock` initializer that handles long paths. + It also uses the current umask for lock files. + """ + + MAX_FILENAME_LENGTH = 255 + + def __init__(self, lock_file, *args, **kwargs): + # The "mode" argument is required if we want to use the current umask in filelock >= 3.10 + # In previous previous it was already using the current umask. + if "mode" not in kwargs and version.parse(_filelock_version) >= version.parse("3.10.0"): + umask = os.umask(0o666) + os.umask(umask) + kwargs["mode"] = 0o666 & ~umask + lock_file = self.hash_filename_if_too_long(lock_file) + super().__init__(lock_file, *args, **kwargs) + + @classmethod + def hash_filename_if_too_long(cls, path: str) -> str: + path = os.path.abspath(os.path.expanduser(path)) + filename = os.path.basename(path) + max_filename_length = cls.MAX_FILENAME_LENGTH + if issubclass(cls, UnixFileLock): + max_filename_length = min(max_filename_length, os.statvfs(os.path.dirname(path)).f_namemax) + if len(filename) > max_filename_length: + dirname = os.path.dirname(path) + hashed_filename = str(hash(filename)) + new_filename = ( + filename[: max_filename_length - len(hashed_filename) - 8] + "..." + hashed_filename + ".lock" + ) + return os.path.join(dirname, new_filename) + else: + return path diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/deprecation_utils.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/deprecation_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f05ecbeaa3eae5476e99c461dbede9ebfa111eb0 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/deprecation_utils.py @@ -0,0 +1,105 @@ +import enum +import inspect +import warnings +from functools import wraps +from typing import Callable, Optional + +from .logging import get_logger + + +_emitted_deprecation_warnings = set() +logger = get_logger(__name__) + + +def deprecated(help_message: Optional[str] = None): + """Decorator to mark a class or a function as deprecated. + + Args: + help_message (:obj:`str`, optional): An optional message to guide the user on how to + switch to non-deprecated usage of the library. + """ + + def decorator(deprecated_class_or_function: Callable): + global _emitted_deprecation_warnings + + if inspect.isclass(deprecated_class_or_function): + deprecated_function = deprecated_class_or_function.__init__ + name = deprecated_class_or_function.__name__ + else: + deprecated_function = deprecated_class_or_function + name = deprecated_function.__name__ + # Support deprecating __init__ class method: class name instead + name = name if name != "__init__" else deprecated_function.__qualname__.split(".")[-2] + + warning_msg = ( + f"{name} is deprecated and will be removed in the next major version of datasets." + f" {help_message}" + if help_message + else "" + ) + + @wraps(deprecated_function) + def wrapper(*args, **kwargs): + func_hash = hash(deprecated_function) + if func_hash not in _emitted_deprecation_warnings: + warnings.warn(warning_msg, category=FutureWarning, stacklevel=2) + _emitted_deprecation_warnings.add(func_hash) + return deprecated_function(*args, **kwargs) + + wrapper._decorator_name_ = "deprecated" + + if inspect.isclass(deprecated_class_or_function): + deprecated_class_or_function.__init__ = wrapper + return deprecated_class_or_function + else: + return wrapper + + return decorator + + +class OnAccess(enum.EnumMeta): + """ + Enum metaclass that calls a user-specified function whenever a member is accessed. + """ + + def __getattribute__(cls, name): + obj = super().__getattribute__(name) + if isinstance(obj, enum.Enum) and obj._on_access: + obj._on_access() + return obj + + def __getitem__(cls, name): + member = super().__getitem__(name) + if member._on_access: + member._on_access() + return member + + def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1): + obj = super().__call__(value, names, module=module, qualname=qualname, type=type, start=start) + if isinstance(obj, enum.Enum) and obj._on_access: + obj._on_access() + return obj + + +class DeprecatedEnum(enum.Enum, metaclass=OnAccess): + """ + Enum class that calls `deprecate` method whenever a member is accessed. + """ + + def __new__(cls, value): + member = object.__new__(cls) + member._value_ = value + member._on_access = member.deprecate + return member + + @property + def help_message(self): + return "" + + def deprecate(self): + help_message = f" {self.help_message}" if self.help_message else "" + warnings.warn( + f"'{self.__objclass__.__name__}' is deprecated and will be removed in the next major version of datasets." + + help_message, + FutureWarning, + stacklevel=3, + ) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/doc_utils.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/doc_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6ef8bcb4e70725ad086cb817e0ec4551d1c0966e --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/doc_utils.py @@ -0,0 +1,15 @@ +from typing import Callable + + +def is_documented_by(function_with_docstring: Callable): + """Decorator to share docstrings across common functions. + + Args: + function_with_docstring (`Callable`): Name of the function with the docstring. + """ + + def wrapper(target_function): + target_function.__doc__ = function_with_docstring.__doc__ + return target_function + + return wrapper diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/experimental.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/experimental.py new file mode 100644 index 0000000000000000000000000000000000000000..cc406154e9347f4df83b1f7b08c32a961d469f6a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/experimental.py @@ -0,0 +1,43 @@ +"""Contains utilities to flag a feature as "experimental" in datasets.""" + +import warnings +from functools import wraps +from typing import Callable + + +def experimental(fn: Callable) -> Callable: + """Decorator to flag a feature as experimental. + + An experimental feature trigger a warning when used as it might be subject to breaking changes in the future. + + Args: + fn (`Callable`): + The function to flag as experimental. + + Returns: + `Callable`: The decorated function. + + Example: + + ```python + >>> from datasets.utils import experimental + + >>> @experimental + ... def my_function(): + ... print("Hello world!") + + >>> my_function() + UserWarning: 'my_function' is experimental and might be subject to breaking changes in the future. + Hello world! + ``` + """ + + @wraps(fn) + def _inner_fn(*args, **kwargs): + warnings.warn( + (f"'{fn.__name__}' is experimental and might be subject to breaking changes in the future."), + UserWarning, + ) + return fn(*args, **kwargs) + + return _inner_fn diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/extract.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/extract.py new file mode 100644 index 0000000000000000000000000000000000000000..c9d96284e5adbd2372aab38717ec6016a9311ffd --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/extract.py @@ -0,0 +1,444 @@ +import bz2 +import gzip +import lzma +import os +import shutil +import struct +import tarfile +import warnings +import zipfile +from abc import ABC, abstractmethod +from pathlib import Path +from typing import TYPE_CHECKING, Optional, Union + +from .. import config +from ._filelock import FileLock +from .logging import get_logger + + +if TYPE_CHECKING: + import py7zr + import rarfile + + +logger = get_logger(__name__) + + +class ExtractManager: + def __init__(self, cache_dir: Optional[str] = None): + self.extract_dir = ( + os.path.join(cache_dir, config.EXTRACTED_DATASETS_DIR) if cache_dir else config.EXTRACTED_DATASETS_PATH + ) + self.extractor = Extractor + + def _get_output_path(self, path: str) -> str: + from .file_utils import hash_url_to_filename + + # Path where we extract compressed archives + # We extract in the cache dir, and get the extracted path name by hashing the original path" + abs_path = os.path.abspath(path) + return os.path.join(self.extract_dir, hash_url_to_filename(abs_path)) + + def _do_extract(self, output_path: str, force_extract: bool) -> bool: + return force_extract or ( + not os.path.isfile(output_path) and not (os.path.isdir(output_path) and os.listdir(output_path)) + ) + + def extract(self, input_path: str, force_extract: bool = False) -> str: + extractor_format = self.extractor.infer_extractor_format(input_path) + if not extractor_format: + return input_path + output_path = self._get_output_path(input_path) + if self._do_extract(output_path, force_extract): + self.extractor.extract(input_path, output_path, extractor_format) + return output_path + + +class BaseExtractor(ABC): + @classmethod + @abstractmethod + def is_extractable(cls, path: Union[Path, str], **kwargs) -> bool: ... + + @staticmethod + @abstractmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: ... + + +class MagicNumberBaseExtractor(BaseExtractor, ABC): + magic_numbers: list[bytes] = [] + + @staticmethod + def read_magic_number(path: Union[Path, str], magic_number_length: int): + with open(path, "rb") as f: + return f.read(magic_number_length) + + @classmethod + def is_extractable(cls, path: Union[Path, str], magic_number: bytes = b"") -> bool: + if not magic_number: + magic_number_length = max(len(cls_magic_number) for cls_magic_number in cls.magic_numbers) + try: + magic_number = cls.read_magic_number(path, magic_number_length) + except OSError: + return False + return any(magic_number.startswith(cls_magic_number) for cls_magic_number in cls.magic_numbers) + + +class TarExtractor(BaseExtractor): + @classmethod + def is_extractable(cls, path: Union[Path, str], **kwargs) -> bool: + return tarfile.is_tarfile(path) + + @staticmethod + def safemembers(members: tarfile.TarFile, output_path: Union[Path, str]): + """ + Fix for CVE-2007-4559 + Desc: + Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile + module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot) + sequence in filenames in a TAR archive, a related issue to CVE-2001-1267. + See: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-4559 + From: https://stackoverflow.com/a/10077309 + """ + + def resolved(path: Union[Path, str]) -> str: + return os.path.realpath(os.path.abspath(path)) + + def badpath(path: str, base: str) -> bool: + # joinpath will ignore base if path is absolute + return not resolved(os.path.join(base, path)).startswith(base) + + def badlink(info: tarfile.TarInfo, base: str) -> bool: + # Links are interpreted relative to the directory containing the link + tip = resolved(os.path.join(base, os.path.dirname(info.name))) + return badpath(info.linkname, base=tip) + + base = resolved(output_path) + + for finfo in members: + if badpath(finfo.name, base): + logger.error(f"Extraction of {finfo.name} is blocked (illegal path)") + elif finfo.issym() and badlink(finfo, base): + logger.error(f"Extraction of {finfo.name} is blocked: Symlink to {finfo.linkname}") + elif finfo.islnk() and badlink(finfo, base): + logger.error(f"Extraction of {finfo.name} is blocked: Hard link to {finfo.linkname}") + else: + yield finfo + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + os.makedirs(output_path, exist_ok=True) + tar_file = tarfile.open(input_path) + tar_file.extractall(output_path, members=TarExtractor.safemembers(tar_file, output_path)) + tar_file.close() + + +class GzipExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x1f\x8b"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + with gzip.open(input_path, "rb") as gzip_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(gzip_file, extracted_file) + + +class ZipExtractor(MagicNumberBaseExtractor): + magic_numbers = [ + b"PK\x03\x04", + b"PK\x05\x06", # empty archive + b"PK\x07\x08", # spanned archive + ] + + @classmethod + def is_extractable(cls, path: Union[Path, str], magic_number: bytes = b"") -> bool: + if super().is_extractable(path, magic_number=magic_number): + return True + try: + # Alternative version of zipfile.is_zipfile that has less false positives, but misses executable zip archives. + # From: https://github.com/python/cpython/pull/5053 + from zipfile import ( + _CD_SIGNATURE, + _ECD_DISK_NUMBER, + _ECD_DISK_START, + _ECD_ENTRIES_TOTAL, + _ECD_OFFSET, + _ECD_SIZE, + _EndRecData, + sizeCentralDir, + stringCentralDir, + structCentralDir, + ) + + with open(path, "rb") as fp: + endrec = _EndRecData(fp) + if endrec: + if endrec[_ECD_ENTRIES_TOTAL] == 0 and endrec[_ECD_SIZE] == 0 and endrec[_ECD_OFFSET] == 0: + return True # Empty zipfiles are still zipfiles + elif endrec[_ECD_DISK_NUMBER] == endrec[_ECD_DISK_START]: + fp.seek(endrec[_ECD_OFFSET]) # Central directory is on the same disk + if fp.tell() == endrec[_ECD_OFFSET] and endrec[_ECD_SIZE] >= sizeCentralDir: + data = fp.read(sizeCentralDir) # CD is where we expect it to be + if len(data) == sizeCentralDir: + centdir = struct.unpack(structCentralDir, data) # CD is the right size + if centdir[_CD_SIGNATURE] == stringCentralDir: + return True # First central directory entry has correct magic number + return False + except Exception: # catch all errors in case future python versions change the zipfile internals + return False + + @staticmethod + def safemembers(members: list[zipfile.ZipInfo], output_path: Union[Path, str]): + """ + Fix for CVE-2007-4559 + Desc: + Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile + module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot) + sequence in filenames in a TAR archive, a related issue to CVE-2001-1267. + See: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-4559 + From: https://stackoverflow.com/a/10077309 + + This additional mitigation is applied for zipfile as well. + """ + + def resolved(path: Union[Path, str]) -> str: + return os.path.realpath(os.path.abspath(path)) + + def badpath(path: str, base: str) -> bool: + # joinpath will ignore base if path is absolute + return not resolved(os.path.join(base, path)).startswith(base) + + base = resolved(output_path) + + for finfo in members: + if badpath(finfo.filename, base): + logger.error(f"Extraction of {finfo.filename} is blocked (illegal path)") + # zipfile doesn't support symlinks + # elif finfo.is_symlink and badlink(finfo, base): + # logger.error(f"Extraction of {finfo.name} is blocked: Symlink to {finfo.linkname}") + else: + yield finfo + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + os.makedirs(output_path, exist_ok=True) + with zipfile.ZipFile(input_path, "r") as zip_file: + zip_file.extractall(output_path, members=ZipExtractor.safemembers(zip_file.filelist, output_path)) + zip_file.close() + + +class XzExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\xfd\x37\x7a\x58\x5a\x00"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + with lzma.open(input_path) as compressed_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(compressed_file, extracted_file) + + +class RarExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"Rar!\x1a\x07\x00", b"Rar!\x1a\x07\x01\x00"] # RAR_ID # RAR5_ID + + @staticmethod + def safemembers(members: list["rarfile.RarInfo"], output_path: Union[Path, str]): + """ + Fix for CVE-2007-4559 + Desc: + Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile + module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot) + sequence in filenames in a TAR archive, a related issue to CVE-2001-1267. + See: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-4559 + From: https://stackoverflow.com/a/10077309 + + This additional mitigation is applied for rarfile as well. + """ + + def resolved(path: Union[Path, str]) -> str: + return os.path.realpath(os.path.abspath(path)) + + def badpath(path: str, base: str) -> bool: + # joinpath will ignore base if path is absolute + return not resolved(os.path.join(base, path)).startswith(base) + + def badlink(info: "rarfile.RarInfo", base: str) -> bool: + # Links are interpreted relative to the directory containing the link + tip = resolved(os.path.join(base, os.path.dirname(info.filename))) + redir_type, redir_flags, link_name = info.file_redir + return badpath(link_name, base=tip) + + base = resolved(output_path) + + for finfo in members: + if badpath(finfo.filename, base): + logger.error(f"Extraction of {finfo.filename} is blocked (illegal path)") + elif finfo.is_symlink() and badlink(finfo, base): + logger.error(f"Extraction of {finfo.filename} is blocked: Symlink to {finfo.file_redir}") + else: + yield finfo + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.RARFILE_AVAILABLE: + raise ImportError("Please pip install rarfile") + import rarfile + + os.makedirs(output_path, exist_ok=True) + rf = rarfile.RarFile(input_path) + rf.extractall(output_path, members=RarExtractor.safemembers(rf.infolist(), output_path)) + rf.close() + + +class ZstdExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x28\xb5\x2f\xfd"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.ZSTANDARD_AVAILABLE: + raise ImportError("Please pip install zstandard") + import zstandard as zstd + + dctx = zstd.ZstdDecompressor() + with open(input_path, "rb") as ifh, open(output_path, "wb") as ofh: + dctx.copy_stream(ifh, ofh) + + +class Bzip2Extractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x42\x5a\x68"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + with bz2.open(input_path, "rb") as compressed_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(compressed_file, extracted_file) + + +class SevenZipExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x37\x7a\xbc\xaf\x27\x1c"] + + @staticmethod + def safemembers(members: list["py7zr.FileInfo"], output_path: Union[Path, str]): + """ + Fix for CVE-2007-4559 + Desc: + Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile + module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot) + sequence in filenames in a TAR archive, a related issue to CVE-2001-1267. + See: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-4559 + From: https://stackoverflow.com/a/10077309 + + This additional mitigation is applied for py7zr as well. + """ + + def resolved(path: Union[Path, str]) -> str: + return os.path.realpath(os.path.abspath(path)) + + def badpath(path: str, base: str) -> bool: + # joinpath will ignore base if path is absolute + return not resolved(os.path.join(base, path)).startswith(base) + + def badlink(info: "py7zr.FileInfo", base: str) -> bool: + # Links are interpreted relative to the directory containing the link + tip = resolved(os.path.join(base, os.path.dirname(info.filename))) + return badpath(os.path.basename(info.filename), base=tip) + + base = resolved(output_path) + + for finfo in members: + if badpath(finfo.filename, base): + logger.error(f"Extraction of {finfo.filename} is blocked (illegal path)") + # py7zr already checks symlinks validity + # elif finfo.is_symlink and badlink(finfo, base): + # logger.error(f"Extraction of {finfo.name} is blocked: Symlink to {finfo.linkname}") + else: + yield finfo + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.PY7ZR_AVAILABLE: + raise ImportError("Please pip install py7zr") + import py7zr + + os.makedirs(output_path, exist_ok=True) + with py7zr.SevenZipFile(input_path, "r") as archive: + targets = [finfo.filename for finfo in SevenZipExtractor.safemembers(archive.list(), output_path)] + archive.extract(output_path, targets=targets) + + +class Lz4Extractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x04\x22\x4d\x18"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.LZ4_AVAILABLE: + raise ImportError("Please pip install lz4") + import lz4.frame + + with lz4.frame.open(input_path, "rb") as compressed_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(compressed_file, extracted_file) + + +class Extractor: + # Put zip file to the last, b/c it is possible wrongly detected as zip (I guess it means: as tar or gzip) + extractors: dict[str, type[BaseExtractor]] = { + "tar": TarExtractor, + "gzip": GzipExtractor, + "zip": ZipExtractor, + "xz": XzExtractor, + "rar": RarExtractor, + "zstd": ZstdExtractor, + "bz2": Bzip2Extractor, + "7z": SevenZipExtractor, # + "lz4": Lz4Extractor, # + } + + @classmethod + def _get_magic_number_max_length(cls): + return max( + len(extractor_magic_number) + for extractor in cls.extractors.values() + if issubclass(extractor, MagicNumberBaseExtractor) + for extractor_magic_number in extractor.magic_numbers + ) + + @staticmethod + def _read_magic_number(path: Union[Path, str], magic_number_length: int): + try: + return MagicNumberBaseExtractor.read_magic_number(path, magic_number_length=magic_number_length) + except OSError: + return b"" + + @classmethod + def is_extractable(cls, path: Union[Path, str], return_extractor: bool = False) -> bool: + warnings.warn( + "Method 'is_extractable' was deprecated in version 2.4.0 and will be removed in 3.0.0. " + "Use 'infer_extractor_format' instead.", + category=FutureWarning, + ) + extractor_format = cls.infer_extractor_format(path) + if extractor_format: + return True if not return_extractor else (True, cls.extractors[extractor_format]) + return False if not return_extractor else (False, None) + + @classmethod + def infer_extractor_format(cls, path: Union[Path, str]) -> Optional[str]: # + magic_number_max_length = cls._get_magic_number_max_length() + magic_number = cls._read_magic_number(path, magic_number_max_length) + for extractor_format, extractor in cls.extractors.items(): + if extractor.is_extractable(path, magic_number=magic_number): + return extractor_format + + @classmethod + def extract( + cls, + input_path: Union[Path, str], + output_path: Union[Path, str], + extractor_format: str, + ) -> None: + os.makedirs(os.path.dirname(output_path), exist_ok=True) + # Prevent parallel extractions + lock_path = str(Path(output_path).with_suffix(".lock")) + with FileLock(lock_path): + shutil.rmtree(output_path, ignore_errors=True) + extractor = cls.extractors[extractor_format] + return extractor.extract(input_path, output_path) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/file_utils.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..94493d527f9337c5a0061b473cfb93dd6dc769eb --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/file_utils.py @@ -0,0 +1,1420 @@ +""" +Utilities for working with the local dataset cache. +This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp +Copyright by the AllenNLP authors. +""" + +import asyncio +import glob +import io +import json +import multiprocessing +import os +import posixpath +import re +import shutil +import tarfile +import time +import xml.dom.minidom +import zipfile +from collections.abc import Generator +from io import BytesIO +from itertools import chain +from pathlib import Path, PurePosixPath +from typing import Any, Optional, TypeVar, Union +from unittest.mock import patch +from urllib.parse import urlparse +from xml.etree import ElementTree as ET + +import fsspec +import httpx +import huggingface_hub +import huggingface_hub.errors +import requests +from fsspec.core import strip_protocol, url_to_fs +from fsspec.utils import can_be_local +from huggingface_hub.utils import get_session, insecure_hashlib +from packaging import version + +from .. import __version__, config +from ..download.download_config import DownloadConfig +from ..filesystems import COMPRESSION_FILESYSTEMS +from . import _tqdm, logging +from ._filelock import FileLock +from .extract import ExtractManager +from .track import TrackedIterableFromGenerator + + +try: + from aiohttp.client_exceptions import ClientError as _AiohttpClientError +except ImportError: + # aiohttp is not available; synthesize an exception type + # that will never be raised by any actual code for use in the `except` + # clause only. + class _AiohttpClientError(Exception): + pass + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +INCOMPLETE_SUFFIX = ".incomplete" + +T = TypeVar("T", str, Path) + +CONNECTION_ERRORS_TO_RETRY = ( + _AiohttpClientError, + asyncio.TimeoutError, + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + httpx.RequestError, +) +SERVER_UNAVAILABLE_CODE = 504 +RATE_LIMIT_CODE = 429 + + +def is_remote_url(url_or_filename: str) -> bool: + return urlparse(url_or_filename).scheme != "" and not os.path.ismount(urlparse(url_or_filename).scheme + ":/") + + +def is_local_path(url_or_filename: str) -> bool: + # On unix the scheme of a local path is empty (for both absolute and relative), + # while on windows the scheme is the drive name (ex: "c") for absolute paths. + # for details on the windows behavior, see https://bugs.python.org/issue42215 + return urlparse(url_or_filename).scheme == "" or os.path.ismount(urlparse(url_or_filename).scheme + ":/") + + +def is_relative_path(url_or_filename: str) -> bool: + return urlparse(url_or_filename).scheme == "" and not os.path.isabs(url_or_filename) + + +def relative_to_absolute_path(path: T) -> T: + """Convert relative path to absolute path.""" + abs_path_str = os.path.abspath(os.path.expanduser(os.path.expandvars(str(path)))) + return Path(abs_path_str) if isinstance(path, Path) else abs_path_str + + +def url_or_path_join(base_name: str, *pathnames: str) -> str: + if is_remote_url(base_name): + return posixpath.join(base_name, *(str(pathname).replace(os.sep, "/").lstrip("/") for pathname in pathnames)) + else: + return Path(base_name, *pathnames).as_posix() + + +def url_or_path_parent(url_or_path: str) -> str: + if is_remote_url(url_or_path): + return url_or_path[: url_or_path.rindex("/")] + else: + return os.path.dirname(url_or_path) + + +def hash_url_to_filename(url, etag=None): + """ + Convert `url` into a hashed filename in a repeatable way. + If `etag` is specified, append its hash to the url's, delimited + by a period. + If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name + so that TF 2.0 can identify it as a HDF5 file + (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) + """ + url_bytes = url.encode("utf-8") + url_hash = insecure_hashlib.sha256(url_bytes) + filename = url_hash.hexdigest() + + if etag: + etag_bytes = etag.encode("utf-8") + etag_hash = insecure_hashlib.sha256(etag_bytes) + filename += "." + etag_hash.hexdigest() + + if url.endswith(".py"): + filename += ".py" + + return filename + + +def cached_path( + url_or_filename, + download_config=None, + **download_kwargs, +) -> str: + """ + Given something that might be a URL (or might be a local path), + determine which. If it's a URL, download the file and cache it, and + return the path to the cached file. If it's already a local path, + make sure the file exists and then return the path. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + ValueError: if it couldn't parse the url or filename correctly + httpx.NetworkError or requests.exceptions.ConnectionError: in case of internet connection issue + """ + if download_config is None: + download_config = DownloadConfig(**download_kwargs) + + cache_dir = download_config.cache_dir or config.DOWNLOADED_DATASETS_PATH + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + if isinstance(url_or_filename, Path): + url_or_filename = str(url_or_filename) + + # Convert fsspec URL in the format "file://local/path" to "local/path" + if can_be_local(url_or_filename): + url_or_filename = strip_protocol(url_or_filename) + + if is_remote_url(url_or_filename): + # URL, so get it from the cache (downloading if necessary) + url_or_filename, storage_options = _prepare_path_and_storage_options( + url_or_filename, download_config=download_config + ) + # Download files from Hugging Face. + # Note: no need to check for https://huggingface.co file URLs since _prepare_path_and_storage_options + # prepares Hugging Face HTTP URLs as hf:// paths already + if url_or_filename.startswith("hf://") and not url_or_filename.startswith("hf://buckets/"): + resolved_path = huggingface_hub.HfFileSystem( + endpoint=config.HF_ENDPOINT, token=download_config.token + ).resolve_path(url_or_filename) + try: + output_path = huggingface_hub.HfApi( + endpoint=config.HF_ENDPOINT, + token=download_config.token, + library_name="datasets", + library_version=__version__, + user_agent=get_datasets_user_agent(download_config.user_agent), + ).hf_hub_download( + repo_id=resolved_path.repo_id, + repo_type=resolved_path.repo_type, + revision=resolved_path.revision, + filename=resolved_path.path_in_repo, + force_download=download_config.force_download, + proxies=download_config.proxies, + ) + except ( + huggingface_hub.utils.RepositoryNotFoundError, + huggingface_hub.utils.EntryNotFoundError, + huggingface_hub.utils.RevisionNotFoundError, + huggingface_hub.utils.GatedRepoError, + ) as e: + raise FileNotFoundError(str(e)) from e + # Download external files + else: + output_path = get_from_cache( + url_or_filename, + cache_dir=cache_dir, + force_download=download_config.force_download, + user_agent=download_config.user_agent, + use_etag=download_config.use_etag, + token=download_config.token, + storage_options=storage_options, + download_desc=download_config.download_desc, + disable_tqdm=download_config.disable_tqdm, + ) + elif os.path.exists(url_or_filename): + # File, and it exists. + output_path = url_or_filename + elif is_local_path(url_or_filename): + # File, but it doesn't exist. + raise FileNotFoundError(f"Local file {url_or_filename} doesn't exist") + else: + # Something unknown + raise ValueError(f"unable to parse {url_or_filename} as a URL or as a local path") + + if output_path is None: + return output_path + + if download_config.extract_compressed_file: + if download_config.extract_on_the_fly: + # Add a compression prefix to the compressed file so that it can be extracted + # as it's being read using xopen. + protocol = _get_extraction_protocol(output_path, download_config=download_config) + extension = _get_path_extension(url_or_filename.split("::")[0]) + if ( + protocol + and extension not in ["tgz", "tar"] + and not url_or_filename.split("::")[0].endswith((".tar.gz", ".tar.bz2", ".tar.xz")) + ): + output_path = relative_to_absolute_path(output_path) + if protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS: + # there is one single file which is the uncompressed file + inner_file = os.path.basename(output_path) + inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file + output_path = f"{protocol}://{inner_file}::{output_path}" + else: + output_path = f"{protocol}://::{output_path}" + return output_path + + # Eager extraction + output_path = ExtractManager(cache_dir=download_config.cache_dir).extract( + output_path, force_extract=download_config.force_extract + ) + return relative_to_absolute_path(output_path) + + +def get_datasets_user_agent(user_agent: Optional[Union[str, dict]] = None) -> str: + ua = f"datasets/{__version__}" + ua += f"; python/{config.PY_VERSION}" + ua += f"; hf_hub/{huggingface_hub.__version__}" + ua += f"; pyarrow/{config.PYARROW_VERSION}" + if config.TORCH_AVAILABLE: + ua += f"; torch/{config.TORCH_VERSION}" + if config.TF_AVAILABLE: + ua += f"; tensorflow/{config.TF_VERSION}" + if config.JAX_AVAILABLE: + ua += f"; jax/{config.JAX_VERSION}" + if isinstance(user_agent, dict): + ua += f"; {'; '.join(f'{k}/{v}' for k, v in user_agent.items())}" + elif isinstance(user_agent, str): + ua += "; " + user_agent + return ua + + +def get_authentication_headers_for_url(url: str, token: Optional[Union[str, bool]] = None) -> dict: + """Handle the HF authentication""" + if url.startswith(config.HF_ENDPOINT): + return huggingface_hub.utils.build_hf_headers( + token=token, library_name="datasets", library_version=__version__ + ) + else: + return {} + + +def _raise_if_offline_mode_is_enabled(msg: Optional[str] = None): + """Raise an OfflineModeIsEnabled error (subclass of ConnectionError) if HF_HUB_OFFLINE is True.""" + if config.HF_HUB_OFFLINE: + raise huggingface_hub.errors.OfflineModeIsEnabled( + "Offline mode is enabled." if msg is None else "Offline mode is enabled. " + str(msg) + ) + + +def fsspec_head(url, storage_options=None): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + fs, path = url_to_fs(url, **(storage_options or {})) + return fs.info(path) + + +def stack_multiprocessing_download_progress_bars(): + # Stack downloads progress bars automatically using HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS=1 + # We use environment variables since the download may happen in a subprocess + return patch.dict(os.environ, {"HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS": "1"}) + + +class TqdmCallback(fsspec.callbacks.TqdmCallback): + def __init__(self, tqdm_kwargs=None, *args, **kwargs): + if config.FSSPEC_VERSION < version.parse("2024.2.0"): + super().__init__(tqdm_kwargs, *args, **kwargs) + self._tqdm = _tqdm # replace tqdm module by datasets.utils.tqdm module + else: + kwargs["tqdm_cls"] = _tqdm.tqdm + super().__init__(tqdm_kwargs, *args, **kwargs) + + +def fsspec_get(url, temp_file, storage_options=None, desc=None, disable_tqdm=False): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + fs, path = url_to_fs(url, **(storage_options or {})) + callback = TqdmCallback( + tqdm_kwargs={ + "desc": desc or "Downloading", + "unit": "B", + "unit_scale": True, + "position": multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses + if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1" + and multiprocessing.current_process()._identity + else None, + "disable": disable_tqdm, + } + ) + fs.get_file(path, temp_file.name, callback=callback) + + +def get_from_cache( + url, + cache_dir=None, + force_download=False, + user_agent=None, + use_etag=True, + token=None, + storage_options=None, + download_desc=None, + disable_tqdm=False, +) -> str: + """ + Given a URL, look for the corresponding file in the local cache. + If it's not there, download it. Then return the path to the cached file. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + """ + if storage_options is None: + storage_options = {} + if cache_dir is None: + cache_dir = config.HF_DATASETS_CACHE + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + os.makedirs(cache_dir, exist_ok=True) + + response = None + etag = None + + # Try a first time to file the file on the local file system without eTag (None) + # if we don't ask for 'force_download' then we spare a request + filename = hash_url_to_filename(url, etag=None) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download and not use_etag: + return cache_path + + # Prepare headers for authentication + headers = get_authentication_headers_for_url(url, token=token) + if user_agent is not None: + headers["user-agent"] = user_agent + + response = fsspec_head(url, storage_options=storage_options) + etag = (response.get("ETag", None) or response.get("etag", None)) if use_etag else None + + # Try a second time + filename = hash_url_to_filename(url, etag) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download: + return cache_path + + # Prevent parallel downloads of the same file with a lock. + lock_path = cache_path + ".lock" + with FileLock(lock_path): + # Retry in case previously locked processes just enter after the precedent process releases the lock + if os.path.exists(cache_path) and not force_download: + return cache_path + + incomplete_path = cache_path + ".incomplete" + + # Download to temporary file, then copy to cache path once finished. + # Otherwise, you get corrupt cache entries if the download gets interrupted. + with open(incomplete_path, "w+b") as temp_file: + logger.info(f"{url} not found in cache or force_download set to True, downloading to {temp_file.name}") + # GET file object + fsspec_get(url, temp_file, storage_options=storage_options, desc=download_desc, disable_tqdm=disable_tqdm) + + logger.info(f"storing {url} in cache at {cache_path}") + shutil.move(temp_file.name, cache_path) + + logger.info(f"creating metadata file for {cache_path}") + meta = {"url": url, "etag": etag} + meta_path = cache_path + ".json" + with open(meta_path, "w", encoding="utf-8") as meta_file: + json.dump(meta, meta_file) + + return cache_path + + +def add_start_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = "".join(docstr) + "\n\n" + (fn.__doc__ if fn.__doc__ is not None else "") + return fn + + return docstring_decorator + + +def add_end_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = (fn.__doc__ if fn.__doc__ is not None else "") + "\n\n" + "".join(docstr) + return fn + + return docstring_decorator + + +def estimate_dataset_size(paths): + return sum(path.stat().st_size for path in paths) + + +def readline(f: io.RawIOBase): + # From: https://github.com/python/cpython/blob/d27e2f4d118e7a9909b6a3e5da06c5ff95806a85/Lib/_pyio.py#L525 + res = bytearray() + while True: + b = f.read(1) + if not b: + break + res += b + if res.endswith(b"\n"): + break + return bytes(res) + + +####################### +# Streaming utilities # +####################### + +BASE_KNOWN_EXTENSIONS = [ + "txt", + "csv", + "json", + "jsonl", + "tsv", + "conll", + "conllu", + "orig", + "parquet", + "pkl", + "pickle", + "rel", + "xml", + "arrow", +] +COMPRESSION_EXTENSION_TO_PROTOCOL = { + # single file compression + **{ + extension.lstrip("."): fs_class.protocol + for fs_class in COMPRESSION_FILESYSTEMS + for extension in fs_class.extensions + }, + # archive compression + "zip": "zip", + "eval": "zip", +} +SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL = { + extension.lstrip("."): fs_class.protocol + for fs_class in COMPRESSION_FILESYSTEMS + for extension in fs_class.extensions +} +SINGLE_FILE_COMPRESSION_PROTOCOLS = {fs_class.protocol for fs_class in COMPRESSION_FILESYSTEMS} +SINGLE_SLASH_AFTER_PROTOCOL_PATTERN = re.compile(r"(? str: + # Get extension: https://foo.bar/train.json.gz -> gz + extension = path.split(".")[-1] + # Remove query params ("dl=1", "raw=true"): gz?dl=1 -> gz + # Remove shards infos (".txt_1", ".txt-00000-of-00100"): txt_1 -> txt + for symb in "?-_": + extension = extension.split(symb)[0] + return extension + + +def _get_extraction_protocol_with_magic_number(f) -> Optional[str]: + """read the magic number from a file-like object and return the compression protocol""" + # Check if the file object is seekable even before reading the magic number (to avoid https://bugs.python.org/issue26440) + try: + f.seek(0) + except (AttributeError, io.UnsupportedOperation): + return None + magic_number = f.read(MAGIC_NUMBER_MAX_LENGTH) + f.seek(0) + for i in range(MAGIC_NUMBER_MAX_LENGTH): + compression = MAGIC_NUMBER_TO_COMPRESSION_PROTOCOL.get(magic_number[: MAGIC_NUMBER_MAX_LENGTH - i]) + if compression is not None: + return compression + compression = MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOL.get(magic_number[: MAGIC_NUMBER_MAX_LENGTH - i]) + if compression is not None: + raise NotImplementedError(f"Compression protocol '{compression}' not implemented.") + + +def _get_extraction_protocol(urlpath: str, download_config: Optional[DownloadConfig] = None) -> Optional[str]: + # get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz + urlpath = str(urlpath) + path = urlpath.split("::")[0] + extension = _get_path_extension(path) + if ( + extension in BASE_KNOWN_EXTENSIONS + or extension in ["tgz", "tar"] + or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz")) + ): + return None + elif extension in COMPRESSION_EXTENSION_TO_PROTOCOL: + return COMPRESSION_EXTENSION_TO_PROTOCOL[extension] + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + try: + with fsspec.open(urlpath, **(storage_options or {})) as f: + return _get_extraction_protocol_with_magic_number(f) + except FileNotFoundError: + if urlpath.startswith(config.HF_ENDPOINT): + raise FileNotFoundError( + urlpath + "\nIf the repo is private or gated, make sure to log in with `huggingface-cli login`." + ) from None + else: + raise + + +def xjoin(a, *p): + """ + This function extends os.path.join to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xjoin function allows you to apply the join on the first path of the chain. + + Example:: + + >>> xjoin("zip://folder1::https://host.com/archive.zip", "file.txt") + zip://folder1/file.txt::https://host.com/archive.zip + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.join(a, *p) + else: + a = posixpath.join(a, *p) + return "::".join([a] + b) + + +def xdirname(a): + """ + This function extends os.path.dirname to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xdirname function allows you to apply the dirname on the first path of the chain. + + Example:: + + >>> xdirname("zip://folder1/file.txt::https://host.com/archive.zip") + zip://folder1::https://host.com/archive.zip + """ + a, *b = str(a).split("::") + if is_local_path(a): + a = os.path.dirname(Path(a).as_posix()) + else: + a = posixpath.dirname(a) + # if we end up at the root of the protocol, we get for example a = 'http:' + # so we have to fix it by adding the '//' that was removed: + if a.endswith(":"): + a += "//" + return "::".join([a] + b) + + +def xexists(urlpath: str, download_config: Optional[DownloadConfig] = None): + """Extend `os.path.exists` function to support both local and remote files. + + Args: + urlpath (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + + main_hop, *rest_hops = _as_str(urlpath).split("::") + if is_local_path(main_hop): + return os.path.exists(main_hop) + else: + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + main_hop, *rest_hops = urlpath.split("::") + fs, *_ = url_to_fs(urlpath, **storage_options) + return fs.exists(main_hop) + + +def xbasename(a): + """ + This function extends os.path.basename to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xbasename function allows you to apply the basename on the first path of the chain. + + Example:: + + >>> xbasename("zip://folder1/file.txt::https://host.com/archive.zip") + file.txt + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.basename(Path(a).as_posix()) + else: + return posixpath.basename(a) + + +def xsplit(a): + """ + This function extends os.path.split to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xsplit function allows you to apply the xsplit on the first path of the chain. + + Example:: + + >>> xsplit("zip://folder1/file.txt::https://host.com/archive.zip") + ('zip://folder1::https://host.com/archive.zip', 'file.txt') + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.split(Path(a).as_posix()) + else: + a, tail = posixpath.split(a) + return "::".join([a + "//" if a.endswith(":") else a] + b), tail + + +def xsplitext(a): + """ + This function extends os.path.splitext to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xsplitext function allows you to apply the splitext on the first path of the chain. + + Example:: + + >>> xsplitext("zip://folder1/file.txt::https://host.com/archive.zip") + ('zip://folder1/file::https://host.com/archive.zip', '.txt') + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.splitext(Path(a).as_posix()) + else: + a, ext = posixpath.splitext(a) + return "::".join([a] + b), ext + + +def xisfile(path, download_config: Optional[DownloadConfig] = None) -> bool: + """Extend `os.path.isfile` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.isfile(path) + else: + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = url_to_fs(path, **storage_options) + return fs.isfile(main_hop) + + +def xgetsize(path, download_config: Optional[DownloadConfig] = None) -> int: + """Extend `os.path.getsize` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `int`: optional + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.getsize(path) + else: + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = fs, *_ = url_to_fs(path, **storage_options) + try: + size = fs.size(main_hop) + except huggingface_hub.utils.EntryNotFoundError: + raise FileNotFoundError(f"No such file: {path}") + if size is None: + # use xopen instead of fs.open to make data fetching more robust + with xopen(path, download_config=download_config) as f: + size = len(f.read()) + return size + + +def xisdir(path, download_config: Optional[DownloadConfig] = None) -> bool: + """Extend `os.path.isdir` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.isdir(path) + else: + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = fs, *_ = url_to_fs(path, **storage_options) + inner_path = main_hop.split("://")[-1] + if not inner_path.strip("/"): + return True + return fs.isdir(inner_path) + + +def xrelpath(path, start=None): + """Extend `os.path.relpath` function to support remote files. + + Args: + path (`str`): URL path. + start (`str`): Start URL directory path. + + Returns: + `str` + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.relpath(main_hop, start=start) if start else os.path.relpath(main_hop) + else: + return posixpath.relpath(main_hop, start=str(start).split("::")[0]) if start else os.path.relpath(main_hop) + + +class _OverridableIOWrapper(io.RawIOBase): + def __init__(self, f): + self._overrides = {} + self.f = f + + def __getattribute__(self, attr): + if attr == "_overrides": + return object.__getattribute__(self, attr) + elif attr in self._overrides: + return self._overrides[attr] + return getattr(self.f, attr) + + def __setattr__(self, attr, value): + if attr == "_overrides": + object.__setattr__(self, attr, value) + else: + self._overrides[attr] = value + + +def _add_retries_to_file_obj_read_method(file_obj): + read = file_obj.read + max_retries = config.STREAMING_READ_MAX_RETRIES + + def read_with_retries(*args, **kwargs): + disconnect_err = None + for retry in range(1, max_retries + 1): + try: + out = read(*args, **kwargs) + break + except CONNECTION_ERRORS_TO_RETRY as err: + disconnect_err = err + logger.warning( + f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RETRY_INTERVAL}sec [{retry}/{max_retries}]" + ) + time.sleep(config.STREAMING_READ_RETRY_INTERVAL) + except huggingface_hub.errors.HfHubHTTPError as err: + if err.response is not None and err.response.status_code == SERVER_UNAVAILABLE_CODE: + disconnect_err = err + logger.warning( + f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_SERVER_UNAVAILABLE_RETRY_INTERVAL}sec [{retry}/{max_retries}]" + ) + time.sleep(config.STREAMING_READ_SERVER_UNAVAILABLE_RETRY_INTERVAL) + elif err.response is not None and err.response.status_code == RATE_LIMIT_CODE: + disconnect_err = err + logger.warning(str(err)) + logger.warning( + f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RATE_LIMIT_RETRY_INTERVAL}sec [{retry}/{max_retries}]" + ) + time.sleep(config.STREAMING_READ_RATE_LIMIT_RETRY_INTERVAL) + else: + raise + else: + raise ConnectionError("Server Disconnected") from disconnect_err + return out + + try: + file_obj.read = read_with_retries + except AttributeError: # read-only attribute + file_obj = _OverridableIOWrapper(file_obj) + file_obj.read = read_with_retries + return file_obj + + +def _prepare_path_and_storage_options( + urlpath: str, download_config: Optional[DownloadConfig] = None +) -> tuple[str, dict[str, dict[str, Any]]]: + prepared_urlpath = [] + prepared_storage_options = {} + for hop in urlpath.split("::"): + hop, storage_options = _prepare_single_hop_path_and_storage_options(hop, download_config=download_config) + prepared_urlpath.append(hop) + prepared_storage_options.update(storage_options) + return "::".join(prepared_urlpath), storage_options + + +def _prepare_single_hop_path_and_storage_options( + urlpath: str, download_config: Optional[DownloadConfig] = None +) -> tuple[str, dict[str, dict[str, Any]]]: + """ + Prepare the URL and the kwargs that must be passed to the HttpFileSystem or HfFileSystem + + In particular it resolves google drive URLs + It also adds the authentication headers for the Hugging Face Hub, for both https:// and hf:// paths. + + Storage options are formatted in the form {protocol: storage_options_for_protocol} + """ + token = None if download_config is None else download_config.token + if urlpath.startswith(config.HF_ENDPOINT) and "/resolve/" in urlpath: + urlpath = "hf://" + urlpath[len(config.HF_ENDPOINT) + 1 :].replace("/resolve/", "@", 1) + protocol = urlpath.split("://")[0] if "://" in urlpath else "file" + if download_config is not None and protocol in download_config.storage_options: + storage_options = download_config.storage_options[protocol].copy() + elif download_config is not None and protocol not in download_config.storage_options: + storage_options = { + option_name: option_value + for option_name, option_value in download_config.storage_options.items() + if option_name not in fsspec.available_protocols() + } + else: + storage_options = {} + if protocol in {"http", "https"}: + client_kwargs = storage_options.pop("client_kwargs", {}) + storage_options["client_kwargs"] = {"trust_env": True, **client_kwargs} # Enable reading proxy env variables + if "drive.google.com" in urlpath: + response = get_session().head(urlpath, timeout=10) + for k, v in response.cookies.items(): + if k.startswith("download_warning"): + urlpath += "&confirm=" + v + cookies = response.cookies + storage_options = {"cookies": cookies, **storage_options} + # Fix Google Drive URL to avoid Virus scan warning + if "confirm=" not in urlpath: + urlpath += "&confirm=t" + if urlpath.startswith("https://raw.githubusercontent.com/"): + # Workaround for served data with gzip content-encoding: https://github.com/fsspec/filesystem_spec/issues/389 + headers = storage_options.pop("headers", {}) + storage_options["headers"] = {"Accept-Encoding": "identity", **headers} + elif protocol == "hf": + storage_options = { + "endpoint": config.HF_ENDPOINT, + "token": token, + **storage_options, + } + if storage_options: + storage_options = {protocol: storage_options} + return urlpath, storage_options + + +def xopen(file: str, mode="r", *args, download_config: Optional[DownloadConfig] = None, **kwargs): + """Extend `open` function to support remote files using `fsspec`. + + It also has a retry mechanism in case connection fails. + The `args` and `kwargs` are passed to `fsspec.open`, except `token` which is used for queries to private repos on huggingface.co + + Args: + file (`str`): Path name of the file to be opened. + mode (`str`, *optional*, default "r"): Mode in which the file is opened. + *args: Arguments to be passed to `fsspec.open`. + download_config : mainly use token or storage_options to support different platforms and auth types. + **kwargs: Keyword arguments to be passed to `fsspec.open`. + + Returns: + file object + """ + # This works as well for `xopen(str(Path(...)))` + file_str = _as_str(file) + main_hop, *rest_hops = file_str.split("::") + if is_local_path(main_hop): + # ignore fsspec-specific kwargs + kwargs.pop("block_size", None) + return open(main_hop, mode, *args, **kwargs) + # add headers and cookies for authentication on the HF Hub and for Google Drive + file, storage_options = _prepare_path_and_storage_options(file_str, download_config=download_config) + kwargs = {**kwargs, **(storage_options or {})} + + max_retries = config.STREAMING_OPEN_MAX_RETRIES + + disconnect_err = None + for retry in range(1, max_retries + 1): + try: + file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open() + break + except CONNECTION_ERRORS_TO_RETRY as err: + disconnect_err = err + logger.warning( + f"Failed to connect to remote data host. Retrying in {config.STREAMING_OPEN_RETRY_INTERVAL}sec [{retry}/{max_retries}]" + ) + time.sleep(config.STREAMING_OPEN_RETRY_INTERVAL) + except ValueError as e: + if str(e) == "Cannot seek streaming HTTP file": + raise NonStreamableDatasetError( + "Streaming is not possible for this dataset because data host server doesn't support HTTP range " + "requests. You can still load this dataset in non-streaming mode by passing `streaming=False` (default)" + ) from e + else: + raise + except FileNotFoundError: + if file.startswith(config.HF_ENDPOINT): + raise FileNotFoundError( + file + "\nIf the repo is private or gated, make sure to log in with `huggingface-cli login`." + ) from None + else: + raise + else: + raise ConnectionError("Server Disconnected") from disconnect_err + file_obj = _add_retries_to_file_obj_read_method(file_obj) + return file_obj + + +def xlistdir(path: str, download_config: Optional[DownloadConfig] = None) -> list[str]: + """Extend `os.listdir` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `list` of `str` + """ + main_hop, *rest_hops = _as_str(path).split("::") + if is_local_path(main_hop): + return os.listdir(path) + else: + # globbing inside a zip in a private repo requires authentication + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = url_to_fs(path, **storage_options) + inner_path = main_hop.split("://")[-1] + if inner_path.strip("/") and not fs.isdir(inner_path): + raise FileNotFoundError(f"Directory doesn't exist: {path}") + paths = fs.listdir(inner_path, detail=False) + return [os.path.basename(path.rstrip("/")) for path in paths] + + +def xglob(urlpath, *, recursive=False, download_config: Optional[DownloadConfig] = None): + """Extend `glob.glob` function to support remote files. + + Args: + urlpath (`str`): URL path with shell-style wildcard patterns. + recursive (`bool`, default `False`): Whether to match the "**" pattern recursively to zero or more + directories or subdirectories. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `list` of `str` + """ + main_hop, *rest_hops = _as_str(urlpath).split("::") + if is_local_path(main_hop): + return glob.glob(main_hop, recursive=recursive) + else: + # globbing inside a zip in a private repo requires authentication + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + main_hop, *rest_hops = urlpath.split("::") + fs, *_ = url_to_fs(urlpath, **storage_options) + inner_path = main_hop.split("://")[1] + globbed_paths = fs.glob(inner_path) + protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[-1] + return ["::".join([f"{protocol}://{globbed_path}"] + rest_hops) for globbed_path in globbed_paths] + + +def xwalk(urlpath, download_config: Optional[DownloadConfig] = None, **kwargs): + """Extend `os.walk` function to support remote files. + + Args: + urlpath (`str`): URL root path. + download_config : mainly use token or storage_options to support different platforms and auth types. + **kwargs: Additional keyword arguments forwarded to the underlying filesystem. + + + Yields: + `tuple`: 3-tuple (dirpath, dirnames, filenames). + """ + main_hop, *rest_hops = _as_str(urlpath).split("::") + if is_local_path(main_hop): + yield from os.walk(main_hop, **kwargs) + else: + # walking inside a zip in a private repo requires authentication + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + main_hop, *rest_hops = urlpath.split("::") + fs, *_ = url_to_fs(urlpath, **storage_options) + inner_path = main_hop.split("://")[-1] + if inner_path.strip("/") and not fs.isdir(inner_path): + return [] + protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[-1] + for dirpath, dirnames, filenames in fs.walk(inner_path, **kwargs): + yield "::".join([f"{protocol}://{dirpath}"] + rest_hops), dirnames, filenames + + +class xPath(type(Path())): + """Extension of `pathlib.Path` to support both local paths and remote URLs.""" + + def __str__(self): + path_str = super().__str__() + main_hop, *rest_hops = path_str.split("::") + if is_local_path(main_hop): + return main_hop + path_as_posix = path_str.replace("\\", "/") + path_as_posix = SINGLE_SLASH_AFTER_PROTOCOL_PATTERN.sub("://", path_as_posix) + path_as_posix += "//" if path_as_posix.endswith(":") else "" # Add slashes to root of the protocol + return path_as_posix + + def exists(self, download_config: Optional[DownloadConfig] = None): + """Extend `pathlib.Path.exists` method to support both local and remote files. + + Args: + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + return xexists(str(self), download_config=download_config) + + def glob(self, pattern, download_config: Optional[DownloadConfig] = None): + """Glob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Args: + pattern (`str`): Pattern that resulting paths must match. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Yields: + [`xPath`] + """ + posix_path = self.as_posix() + main_hop, *rest_hops = posix_path.split("::") + if is_local_path(main_hop): + yield from Path(main_hop).glob(pattern) + else: + # globbing inside a zip in a private repo requires authentication + if rest_hops: + urlpath = rest_hops[0] + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + storage_options = {urlpath.split("://")[0]: storage_options} + posix_path = "::".join([main_hop, urlpath, *rest_hops[1:]]) + else: + storage_options = None + fs, *_ = url_to_fs(xjoin(posix_path, pattern), **(storage_options or {})) + globbed_paths = fs.glob(xjoin(main_hop, pattern)) + for globbed_path in globbed_paths: + yield type(self)("::".join([f"{fs.protocol}://{globbed_path}"] + rest_hops)) + + def rglob(self, pattern, **kwargs): + """Rglob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Args: + pattern (`str`): Pattern that resulting paths must match. + + Yields: + [`xPath`] + """ + return self.glob("**/" + pattern, **kwargs) + + @property + def parent(self) -> "xPath": + """Name function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + [`xPath`] + """ + return type(self)(xdirname(self.as_posix())) + + @property + def name(self) -> str: + """Name function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + `str` + """ + return PurePosixPath(self.as_posix().split("::")[0]).name + + @property + def stem(self) -> str: + """Stem function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + `str` + """ + return PurePosixPath(self.as_posix().split("::")[0]).stem + + @property + def suffix(self) -> str: + """Suffix function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + `str` + """ + return PurePosixPath(self.as_posix().split("::")[0]).suffix + + def open(self, *args, **kwargs): + """Extend :func:`xopen` to support argument of type :obj:`~pathlib.Path`. + + Args: + **args: Arguments passed to :func:`fsspec.open`. + **kwargs: Keyword arguments passed to :func:`fsspec.open`. + + Returns: + `io.FileIO`: File-like object. + """ + return xopen(str(self), *args, **kwargs) + + def joinpath(self, *p: tuple[str, ...]) -> "xPath": + """Extend :func:`xjoin` to support argument of type :obj:`~pathlib.Path`. + + Args: + *p (`tuple` of `str`): Other path components. + + Returns: + [`xPath`] + """ + return type(self)(xjoin(self.as_posix(), *p)) + + def __truediv__(self, p: str) -> "xPath": + return self.joinpath(p) + + def with_suffix(self, suffix): + main_hop, *rest_hops = str(self).split("::") + if is_local_path(main_hop): + return type(self)(str(super().with_suffix(suffix))) + return type(self)("::".join([type(self)(PurePosixPath(main_hop).with_suffix(suffix)).as_posix()] + rest_hops)) + + +def _as_str(path: Union[str, Path, xPath]): + return str(path) if isinstance(path, xPath) else str(xPath(str(path))) + + +def xgzip_open(filepath_or_buffer, *args, download_config: Optional[DownloadConfig] = None, **kwargs): + import gzip + + if hasattr(filepath_or_buffer, "read"): + return gzip.open(filepath_or_buffer, *args, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + return gzip.open(xopen(filepath_or_buffer, "rb", download_config=download_config), *args, **kwargs) + + +def xnumpy_load(filepath_or_buffer, *args, download_config: Optional[DownloadConfig] = None, **kwargs): + import numpy as np + + if hasattr(filepath_or_buffer, "read"): + return np.load(filepath_or_buffer, *args, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + return np.load(xopen(filepath_or_buffer, "rb", download_config=download_config), *args, **kwargs) + + +def xpandas_read_csv(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import pandas as pd + + if hasattr(filepath_or_buffer, "read"): + return pd.read_csv(filepath_or_buffer, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + if kwargs.get("compression", "infer") == "infer": + kwargs["compression"] = _get_extraction_protocol(filepath_or_buffer, download_config=download_config) + return pd.read_csv(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs) + + +def xpandas_read_excel(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import pandas as pd + + if hasattr(filepath_or_buffer, "read"): + try: + return pd.read_excel(filepath_or_buffer, **kwargs) + except ValueError: # Cannot seek streaming HTTP file + return pd.read_excel(BytesIO(filepath_or_buffer.read()), **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + try: + return pd.read_excel(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs) + except ValueError: # Cannot seek streaming HTTP file + return pd.read_excel( + BytesIO(xopen(filepath_or_buffer, "rb", download_config=download_config).read()), **kwargs + ) + + +def xpyarrow_parquet_read_table(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import pyarrow.parquet as pq + + if hasattr(filepath_or_buffer, "read"): + return pq.read_table(filepath_or_buffer, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + return pq.read_table(xopen(filepath_or_buffer, mode="rb", download_config=download_config), **kwargs) + + +def xsio_loadmat(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import scipy.io as sio + + if hasattr(filepath_or_buffer, "read"): + return sio.loadmat(filepath_or_buffer, **kwargs) + else: + return sio.loadmat(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs) + + +def xet_parse(source, parser=None, download_config: Optional[DownloadConfig] = None): + """Extend `xml.etree.ElementTree.parse` function to support remote files. + + Args: + source: File path or file object. + parser (`XMLParser`, *optional*, default `XMLParser`): Parser instance. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `xml.etree.ElementTree.Element`: Root element of the given source document. + """ + if hasattr(source, "read"): + return ET.parse(source, parser=parser) + else: + with xopen(source, "rb", download_config=download_config) as f: + return ET.parse(f, parser=parser) + + +def xxml_dom_minidom_parse(filename_or_file, download_config: Optional[DownloadConfig] = None, **kwargs): + """Extend `xml.dom.minidom.parse` function to support remote files. + + Args: + filename_or_file (`str` or file): File path or file object. + download_config : mainly use token or storage_options to support different platforms and auth types. + **kwargs (optional): Additional keyword arguments passed to `xml.dom.minidom.parse`. + + Returns: + :obj:`xml.dom.minidom.Document`: Parsed document. + """ + if hasattr(filename_or_file, "read"): + return xml.dom.minidom.parse(filename_or_file, **kwargs) + else: + with xopen(filename_or_file, "rb", download_config=download_config) as f: + return xml.dom.minidom.parse(f, **kwargs) + + +class ArchiveIterable(TrackedIterableFromGenerator): + """An iterable of (path, fileobj) from a TAR archive, used by `iter_archive`""" + + @staticmethod + def _iter_tar(f): + stream = tarfile.open(fileobj=f, mode="r|*") + for tarinfo in stream: + file_path = tarinfo.name + if not tarinfo.isreg(): + continue + if file_path is None: + continue + if os.path.basename(file_path).startswith((".", "__")): + # skipping hidden files + continue + file_obj = stream.extractfile(tarinfo) + yield file_path, file_obj + stream.members = [] + del stream + + @staticmethod + def _iter_zip(f): + zipf = zipfile.ZipFile(f) + for member in zipf.infolist(): + file_path = member.filename + if member.is_dir(): + continue + if file_path is None: + continue + if os.path.basename(file_path).startswith((".", "__")): + # skipping hidden files + continue + file_obj = zipf.open(member) + yield file_path, file_obj + + @classmethod + def _iter_from_fileobj(cls, f) -> Generator[tuple, None, None]: + compression = _get_extraction_protocol_with_magic_number(f) + if compression == "zip": + yield from cls._iter_zip(f) + else: + yield from cls._iter_tar(f) + + @classmethod + def _iter_from_urlpath( + cls, urlpath: str, download_config: Optional[DownloadConfig] = None + ) -> Generator[tuple, None, None]: + compression = _get_extraction_protocol(urlpath, download_config=download_config) + # Set block_size=0 to get faster streaming + # (e.g. for hf:// and https:// it uses streaming Requests file-like instances) + with xopen(urlpath, "rb", download_config=download_config, block_size=0) as f: + if compression == "zip": + yield from cls._iter_zip(f) + else: + yield from cls._iter_tar(f) + + @classmethod + def from_buf(cls, fileobj) -> "ArchiveIterable": + return cls(cls._iter_from_fileobj, fileobj) + + @classmethod + def from_urlpath(cls, urlpath_or_buf, download_config: Optional[DownloadConfig] = None) -> "ArchiveIterable": + return cls(cls._iter_from_urlpath, urlpath_or_buf, download_config) + + +class FilesIterable(TrackedIterableFromGenerator): + """An iterable of paths from a list of directories or files""" + + @classmethod + def _iter_from_urlpaths( + cls, urlpaths: Union[str, list[str]], download_config: Optional[DownloadConfig] = None + ) -> Generator[str, None, None]: + if not isinstance(urlpaths, list): + urlpaths = [urlpaths] + for urlpath in urlpaths: + if xisfile(urlpath, download_config=download_config): + yield urlpath + elif xisdir(urlpath, download_config=download_config): + for dirpath, dirnames, filenames in xwalk(urlpath, download_config=download_config): + # in-place modification to prune the search + dirnames[:] = sorted([dirname for dirname in dirnames if not dirname.startswith((".", "__"))]) + if xbasename(dirpath).startswith((".", "__")): + # skipping hidden directories + continue + for filename in sorted(filenames): + if filename.startswith((".", "__")): + # skipping hidden files + continue + yield xjoin(dirpath, filename) + else: + raise FileNotFoundError(urlpath) + + @classmethod + def from_urlpaths(cls, urlpaths, download_config: Optional[DownloadConfig] = None) -> "FilesIterable": + return cls(cls._iter_from_urlpaths, urlpaths, download_config) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/filelock.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/filelock.py new file mode 100644 index 0000000000000000000000000000000000000000..df0728efe644d8eb32f0e578a85e39ba366e9743 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/filelock.py @@ -0,0 +1,11 @@ +# deprecated, please use the `filelock` package instead + +from filelock import ( # noqa: F401 # imported for backward compatibility TODO: remove in 3.0.0 + BaseFileLock, + SoftFileLock, + Timeout, + UnixFileLock, + WindowsFileLock, +) + +from ._filelock import FileLock # noqa: F401 # imported for backward compatibility. TODO: remove in 3.0.0 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/hub.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/hub.py new file mode 100644 index 0000000000000000000000000000000000000000..6d784333b23328c113ec7be444d0a77410f1b857 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/hub.py @@ -0,0 +1,6 @@ +from functools import partial + +from huggingface_hub import hf_hub_url + + +hf_dataset_url = partial(hf_hub_url, repo_type="dataset") diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/info_utils.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/info_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c3e246f850ffd58c3b572c5a5038b88e863be4fb --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/info_utils.py @@ -0,0 +1,105 @@ +import enum +import os +from typing import Optional + +from huggingface_hub.utils import insecure_hashlib + +from .. import config +from ..exceptions import ( + ExpectedMoreDownloadedFilesError, + ExpectedMoreSplitsError, + NonMatchingChecksumError, + NonMatchingSplitsSizesError, + UnexpectedDownloadedFileError, + UnexpectedSplitsError, +) +from .logging import get_logger + + +logger = get_logger(__name__) + + +class VerificationMode(enum.Enum): + """`Enum` that specifies which verification checks to run. + + The default mode is `BASIC_CHECKS`, which will perform only rudimentary checks to avoid slowdowns + when generating/downloading a dataset for the first time. + + The verification modes: + + | | Verification checks | + |---------------------------|------------------------------------------------------------------------------ | + | `ALL_CHECKS` | Split checks and validity (number of files, checksums) of downloaded files | + | `BASIC_CHECKS` (default) | Same as `ALL_CHECKS` but without checking downloaded files | + | `NO_CHECKS` | None | + + """ + + ALL_CHECKS = "all_checks" + BASIC_CHECKS = "basic_checks" + NO_CHECKS = "no_checks" + + +def verify_checksums(expected_checksums: Optional[dict], recorded_checksums: dict, verification_name=None): + if expected_checksums is None: + logger.info("Unable to verify checksums.") + return + if len(set(expected_checksums) - set(recorded_checksums)) > 0: + raise ExpectedMoreDownloadedFilesError(str(set(expected_checksums) - set(recorded_checksums))) + if len(set(recorded_checksums) - set(expected_checksums)) > 0: + raise UnexpectedDownloadedFileError(str(set(recorded_checksums) - set(expected_checksums))) + bad_urls = [url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]] + for_verification_name = " for " + verification_name if verification_name is not None else "" + if len(bad_urls) > 0: + raise NonMatchingChecksumError( + f"Checksums didn't match{for_verification_name}:\n" + f"{bad_urls}\n" + "Set `verification_mode='no_checks'` to skip checksums verification and ignore this error" + ) + logger.info("All the checksums matched successfully" + for_verification_name) + + +def verify_splits(expected_splits: Optional[dict], recorded_splits: dict): + if expected_splits is None: + logger.info("Unable to verify splits sizes.") + return + if len(set(expected_splits) - set(recorded_splits)) > 0: + raise ExpectedMoreSplitsError(str(set(expected_splits) - set(recorded_splits))) + if len(set(recorded_splits) - set(expected_splits)) > 0: + raise UnexpectedSplitsError(str(set(recorded_splits) - set(expected_splits))) + bad_splits = [ + {"expected": expected_splits[name], "recorded": recorded_splits[name]} + for name in expected_splits + if expected_splits[name].num_examples != recorded_splits[name].num_examples + ] + if len(bad_splits) > 0: + raise NonMatchingSplitsSizesError(str(bad_splits)) + logger.info("All the splits matched successfully.") + + +def get_size_checksum_dict(path: str, record_checksum: bool = False) -> dict: + """Compute the file size and the sha256 checksum of a file""" + if record_checksum: + m = insecure_hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + m.update(chunk) + checksum = m.hexdigest() + else: + checksum = None + return {"num_bytes": os.path.getsize(path), "checksum": checksum} + + +def is_small_dataset(dataset_size): + """Check if `dataset_size` is smaller than `config.IN_MEMORY_MAX_SIZE`. + + Args: + dataset_size (int): Dataset size in bytes. + + Returns: + bool: Whether `dataset_size` is smaller than `config.IN_MEMORY_MAX_SIZE`. + """ + if dataset_size and config.IN_MEMORY_MAX_SIZE: + return dataset_size < config.IN_MEMORY_MAX_SIZE + else: + return False diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/json.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/json.py new file mode 100644 index 0000000000000000000000000000000000000000..216338fd011c91c74f799afaf9ce6f300b0097d3 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/json.py @@ -0,0 +1,134 @@ +from typing import TYPE_CHECKING, Any + +import pandas as pd + + +if TYPE_CHECKING: + from ..features.features import FeatureType + + +def ujson_dumps(*args, **kwargs): + try: + return pd.io.json.ujson_dumps(*args, **kwargs) + except AttributeError: + # Before pandas-2.2.0, ujson_dumps was renamed to dumps: import ujson_dumps as dumps + return pd.io.json.dumps(*args, **kwargs) + + +def ujson_loads(*args, **kwargs): + try: + return pd.io.json.ujson_loads(*args, **kwargs) + except AttributeError: + # Before pandas-2.2.0, ujson_loads was renamed to loads: import ujson_loads as loads + return pd.io.json.loads(*args, **kwargs) + + +def json_encode_field(example: Any, json_field_path: list[str]) -> Any: + if json_field_path: + field, *json_field_path = json_field_path + if example is None: + return None + elif field == 0: + return [json_encode_field(x, json_field_path) for x in example] + else: + return {**example, field: json_encode_field(example.get(field), json_field_path)} + else: + try: + ujson_loads(example) + except Exception: + return ujson_dumps(example) + else: + return example + + +def json_decode_field(example: Any, json_field_path: str) -> Any: + if json_field_path: + field, *json_field_path = json_field_path + if example is None: + return None + elif field == 0: + return [json_decode_field(x, json_field_path) for x in example] + else: + return {**example, field: json_decode_field(example.get(field), json_field_path)} + else: + try: + return ujson_loads(example) + except Exception: + return example + + +def find_mixed_struct_types_field_paths(examples: list, allow_root=False) -> list[list[str]]: + mixed_struct_types_field_paths = [] + examples = [example for example in examples if example is not None] + if not examples: + return [] + paths_and_content_to_check = [([], examples)] + while paths_and_content_to_check: + path, content = paths_and_content_to_check.pop(0) + if all(isinstance(x, dict) for x in content): + if (allow_root or path) and (any(set(x) != set(content[0]) for x in content) or not content[0]): + mixed_struct_types_field_paths.append(path) + else: + for subfield in {field for x in content for field in x}: + examples = [x[subfield] for x in content if subfield in x and x[subfield] is not None] + if not examples: + continue + paths_and_content_to_check.append((path + [subfield], examples)) + elif all(isinstance(x, list) for x in content): + examples = [x for sublist in content for x in sublist if x is not None] + if not examples: + continue + paths_and_content_to_check.append((path + [0], examples)) + elif any(isinstance(x, (dict, list)) for x in content): + mixed_struct_types_field_paths.append(path) + return mixed_struct_types_field_paths + + +def get_json_field_path_from_pyarrow_json_error(err_str: str) -> list[str]: + # e.g. json_field_path_str = "col/subfield_containing_a_list/[]/subsubfield_in_item_in_the_list" + json_field_path_str = err_str.split("Column(", 1)[1].rsplit(") changed from", 1)[0].strip("/") + # e.g. json_field_path = ["col", "subfield_containing_a_list", 0, "subsubfield_in_item_in_the_list"] + json_field_path = [0 if seg == "[]" else seg for seg in json_field_path_str.split("/")] + return json_field_path + + +def insert_json_field_path(json_field_paths: list[list[str]], json_field_path: list[str]) -> None: + # Add to list of json_field_paths and check if other share a common path + for i in range(len(json_field_paths)): + if json_field_paths[i][: len(json_field_path)] == json_field_path: + json_field_paths[i] = json_field_path + break + else: + json_field_paths.append(json_field_path) + + +def json_encode_fields_in_json_lines(original_batch: bytes, json_field_paths: list[list[str]]) -> bytes: + examples = [ujson_loads(line) for line in original_batch.splitlines()] + for json_field_path in json_field_paths: + examples = [json_encode_field(example, json_field_path) for example in examples] + batch = "\n".join([ujson_dumps(example) for example in examples]).encode() + return batch + + +def get_json_field_paths_from_feature(feature: "FeatureType") -> list[list[str]]: + from datasets.features.features import Json, _visit_with_path + + json_field_paths = [] + + def get_json_type_path(_feature, feature_path): + if isinstance(_feature, Json): + json_field_paths.append(feature_path) + return _feature + + _visit_with_path(feature, get_json_type_path) + return json_field_paths + + +def set_json_types_in_feature(feature: "FeatureType", json_field_paths: list[list[str]]) -> None: + from datasets.features.features import Json, _visit_with_path + + def set_json_type(feature, feature_path): + return Json() if feature_path in json_field_paths else feature + + feature = _visit_with_path(feature, set_json_type) + return feature diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/logging.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..851c1a2b73d2f411dbd180e16e37daf91181e13c --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/logging.py @@ -0,0 +1,175 @@ +# Copyright 2020 Optuna, Hugging Face +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Logging utilities.""" + +import logging +import os +from logging import ( + CRITICAL, # NOQA + DEBUG, # NOQA + ERROR, # NOQA + FATAL, # NOQA + INFO, # NOQA + NOTSET, # NOQA + WARN, # NOQA + WARNING, # NOQA +) +from typing import Optional + +from .tqdm import ( # noqa: F401 # imported for backward compatibility + disable_progress_bar, + enable_progress_bar, + is_progress_bar_enabled, + tqdm, +) + + +log_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +_default_log_level = logging.WARNING + + +def _get_default_logging_level(): + """ + If DATASETS_VERBOSITY env var is set to one of the valid choices return that as the new default level. + If it is not - fall back to ``_default_log_level`` + """ + env_level_str = os.getenv("DATASETS_VERBOSITY", None) + if env_level_str: + if env_level_str in log_levels: + return log_levels[env_level_str] + else: + logging.getLogger().warning( + f"Unknown option DATASETS_VERBOSITY={env_level_str}, has to be one of: {', '.join(log_levels.keys())}" + ) + return _default_log_level + + +def _get_library_name() -> str: + return __name__.split(".")[0] + + +def _get_library_root_logger() -> logging.Logger: + return logging.getLogger(_get_library_name()) + + +def _configure_library_root_logger() -> None: + # Apply our default configuration to the library root logger. + library_root_logger = _get_library_root_logger() + library_root_logger.addHandler(logging.StreamHandler()) + library_root_logger.setLevel(_get_default_logging_level()) + + +def _reset_library_root_logger() -> None: + library_root_logger = _get_library_root_logger() + library_root_logger.setLevel(logging.NOTSET) + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """Return a logger with the specified name. + This function can be used in dataset builders. + """ + if name is None: + name = _get_library_name() + return logging.getLogger(name) + + +def get_verbosity() -> int: + """Return the current level for the HuggingFace datasets library's root logger. + Returns: + Logging level, e.g., `datasets.logging.DEBUG` and `datasets.logging.INFO`. + + > [!TIP] + > HuggingFace datasets library has following logging levels: + > - `datasets.logging.CRITICAL`, `datasets.logging.FATAL` + > - `datasets.logging.ERROR` + > - `datasets.logging.WARNING`, `datasets.logging.WARN` + > - `datasets.logging.INFO` + > - `datasets.logging.DEBUG` + """ + return _get_library_root_logger().getEffectiveLevel() + + +def set_verbosity(verbosity: int) -> None: + """Set the level for the Hugging Face Datasets library's root logger. + Args: + verbosity: + Logging level, e.g., `datasets.logging.DEBUG` and `datasets.logging.INFO`. + """ + _get_library_root_logger().setLevel(verbosity) + + +def set_verbosity_info(): + """Set the level for the Hugging Face datasets library's root logger to `INFO`. + + This will display most of the logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.INFO)`. + """ + return set_verbosity(INFO) + + +def set_verbosity_warning(): + """Set the level for the Hugging Face datasets library's root logger to `WARNING`. + + This will display only the warning and errors logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.WARNING)`. + """ + return set_verbosity(WARNING) + + +def set_verbosity_debug(): + """Set the level for the Hugging Face datasets library's root logger to `DEBUG`. + + This will display all the logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.DEBUG)`. + """ + return set_verbosity(DEBUG) + + +def set_verbosity_error(): + """Set the level for the Hugging Face datasets library's root logger to `ERROR`. + + This will display only the errors logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.ERROR)`. + """ + return set_verbosity(ERROR) + + +def disable_propagation() -> None: + """Disable propagation of the library log outputs. + Note that log propagation is disabled by default. + """ + _get_library_root_logger().propagate = False + + +def enable_propagation() -> None: + """Enable propagation of the library log outputs. + Please disable the Hugging Face datasets library's default handler to prevent double logging if the root logger has + been configured. + """ + _get_library_root_logger().propagate = True + + +# Configure the library root logger at the module level (singleton-like) +_configure_library_root_logger() diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/metadata.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..cceecf35d63f518fcb7a62c804b9679e5d0c144e --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/metadata.py @@ -0,0 +1,233 @@ +import re +import textwrap +from collections import Counter +from itertools import groupby +from operator import itemgetter +from typing import Any, ClassVar, Optional + +import yaml +from huggingface_hub import DatasetCardData + +from ..config import METADATA_CONFIGS_FIELD +from ..features import Features +from ..info import DatasetInfo, DatasetInfosDict +from ..naming import _split_re +from ..utils.logging import get_logger + + +logger = get_logger(__name__) + + +class _NoDuplicateSafeLoader(yaml.SafeLoader): + def _check_no_duplicates_on_constructed_node(self, node): + keys = [self.constructed_objects[key_node] for key_node, _ in node.value] + keys = [tuple(key) if isinstance(key, list) else key for key in keys] + counter = Counter(keys) + duplicate_keys = [key for key in counter if counter[key] > 1] + if duplicate_keys: + raise TypeError(f"Got duplicate yaml keys: {duplicate_keys}") + + def construct_mapping(self, node, deep=False): + mapping = super().construct_mapping(node, deep=deep) + self._check_no_duplicates_on_constructed_node(node) + return mapping + + +def _split_yaml_from_readme(readme_content: str) -> tuple[Optional[str], str]: + full_content = list(readme_content.splitlines()) + if full_content and full_content[0] == "---" and "---" in full_content[1:]: + sep_idx = full_content[1:].index("---") + 1 + yamlblock = "\n".join(full_content[1:sep_idx]) + return yamlblock, "\n".join(full_content[sep_idx + 1 :]) + + return None, "\n".join(full_content) + + +class MetadataConfigs(dict[str, dict[str, Any]]): + """Should be in format {config_name: {**config_params}}.""" + + FIELD_NAME: ClassVar[str] = METADATA_CONFIGS_FIELD + + @staticmethod + def _raise_if_data_files_field_not_valid(metadata_config: dict): + yaml_data_files = metadata_config.get("data_files") + if yaml_data_files is not None: + yaml_error_message = textwrap.dedent( + f""" + Expected data_files in YAML to be either a string or a list of strings + or a list of dicts with two keys: 'split' and 'path', but got {yaml_data_files} + Examples of data_files in YAML: + + data_files: data.csv + + data_files: data/*.png + + data_files: + - part0/* + - part1/* + + data_files: + - split: train + path: train/* + - split: test + path: test/* + + data_files: + - split: train + path: + - train/part1/* + - train/part2/* + - split: test + path: test/* + + PS: some symbols like dashes '-' are not allowed in split names + """ + ) + if not isinstance(yaml_data_files, (list, str)): + raise ValueError(yaml_error_message) + if isinstance(yaml_data_files, list): + for yaml_data_files_item in yaml_data_files: + if ( + not isinstance(yaml_data_files_item, (str, dict)) + or isinstance(yaml_data_files_item, dict) + and not ( + len(yaml_data_files_item) == 2 + and "split" in yaml_data_files_item + and re.match(_split_re, yaml_data_files_item["split"]) + and isinstance(yaml_data_files_item.get("path"), (str, list)) + ) + ): + raise ValueError(yaml_error_message) + + @classmethod + def _from_exported_parquet_files_and_dataset_infos( + cls, + parquet_commit_hash: str, + exported_parquet_files: list[dict[str, Any]], + dataset_infos: DatasetInfosDict, + ) -> "MetadataConfigs": + metadata_configs = { + config_name: { + "data_files": [ + { + "split": split_name, + "path": [ + parquet_file["url"].replace("refs%2Fconvert%2Fparquet", parquet_commit_hash) + for parquet_file in parquet_files_for_split + ], + } + for split_name, parquet_files_for_split in groupby(parquet_files_for_config, itemgetter("split")) + ], + "version": str(dataset_infos.get(config_name, DatasetInfo()).version or "0.0.0"), + } + for config_name, parquet_files_for_config in groupby(exported_parquet_files, itemgetter("config")) + } + if dataset_infos: + # Preserve order of configs and splits + metadata_configs = { + config_name: { + "data_files": [ + data_file + for split_name in dataset_info.splits + for data_file in metadata_configs[config_name]["data_files"] + if data_file["split"] == split_name + ], + "version": metadata_configs[config_name]["version"], + } + for config_name, dataset_info in dataset_infos.items() + } + return cls(metadata_configs) + + @classmethod + def from_dataset_card_data(cls, dataset_card_data: DatasetCardData) -> "MetadataConfigs": + if dataset_card_data.get(cls.FIELD_NAME): + metadata_configs = dataset_card_data[cls.FIELD_NAME] + if not isinstance(metadata_configs, list): + raise ValueError(f"Expected {cls.FIELD_NAME} to be a list, but got '{metadata_configs}'") + for metadata_config in metadata_configs: + if "config_name" not in metadata_config: + raise ValueError( + f"Each config must include `config_name` field with a string name of a config, " + f"but got {metadata_config}. " + ) + cls._raise_if_data_files_field_not_valid(metadata_config) + return cls( + { + config.pop("config_name"): { + param: value if param != "features" else Features._from_yaml_list(value) + for param, value in config.items() + } + for metadata_config in metadata_configs + if (config := metadata_config.copy()) + } + ) + return cls() + + def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None: + if self: + for metadata_config in self.values(): + self._raise_if_data_files_field_not_valid(metadata_config) + current_metadata_configs = self.from_dataset_card_data(dataset_card_data) + total_metadata_configs = dict(sorted({**current_metadata_configs, **self}.items())) + for config_name, config_metadata in total_metadata_configs.items(): + config_metadata.pop("config_name", None) + dataset_card_data[self.FIELD_NAME] = [ + {"config_name": config_name, **config_metadata} + for config_name, config_metadata in total_metadata_configs.items() + ] + + def get_default_config_name(self) -> Optional[str]: + default_config_name = None + for config_name, metadata_config in self.items(): + if len(self) == 1 or config_name == "default" or metadata_config.get("default"): + if default_config_name is None: + default_config_name = config_name + else: + raise ValueError( + f"Dataset has several default configs: '{default_config_name}' and '{config_name}'." + ) + return default_config_name + + +# DEPRECATED - just here to support old versions of evaluate like 0.2.2 +# To support new tasks on the Hugging Face Hub, please open a PR for this file: +# https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/pipelines.ts +known_task_ids = { + "image-classification": [], + "translation": [], + "image-segmentation": [], + "fill-mask": [], + "automatic-speech-recognition": [], + "token-classification": [], + "sentence-similarity": [], + "audio-classification": [], + "question-answering": [], + "summarization": [], + "zero-shot-classification": [], + "table-to-text": [], + "feature-extraction": [], + "other": [], + "multiple-choice": [], + "text-classification": [], + "text-to-image": [], + "text2text-generation": [], + "zero-shot-image-classification": [], + "tabular-classification": [], + "tabular-regression": [], + "image-to-image": [], + "tabular-to-text": [], + "unconditional-image-generation": [], + "text-retrieval": [], + "text-to-speech": [], + "object-detection": [], + "audio-to-audio": [], + "text-generation": [], + "conversational": [], + "table-question-answering": [], + "visual-question-answering": [], + "image-to-text": [], + "reinforcement-learning": [], + "voice-activity-detection": [], + "time-series-forecasting": [], + "document-question-answering": [], +} diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/patching.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/patching.py new file mode 100644 index 0000000000000000000000000000000000000000..69563f562e40b33fb9565ba9b6976279c9268b38 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/patching.py @@ -0,0 +1,119 @@ +from importlib import import_module + +from .logging import get_logger + + +logger = get_logger(__name__) + + +class _PatchedModuleObj: + """Set all the modules components as attributes of the _PatchedModuleObj object.""" + + def __init__(self, module, attrs=None): + attrs = attrs or [] + if module is not None: + for key in module.__dict__: + if key in attrs or not key.startswith("__"): + setattr(self, key, getattr(module, key)) + self._original_module = module._original_module if isinstance(module, _PatchedModuleObj) else module + + +class patch_submodule: + """ + Patch a submodule attribute of an object, by keeping all other submodules intact at all levels. + + Example:: + + >>> import importlib + >>> from datasets.load import dataset_module_factory + >>> from datasets.streaming import patch_submodule, xjoin + >>> + >>> dataset_module = dataset_module_factory("stanfordnlp/snli") + >>> snli_module = importlib.import_module(dataset_module.module_path) + >>> patcher = patch_submodule(snli_module, "os.path.join", xjoin) + >>> patcher.start() + >>> assert snli_module.os.path.join is xjoin + """ + + _active_patches = [] + + def __init__(self, obj, target: str, new, attrs=None): + self.obj = obj + self.target = target + self.new = new + self.key = target.split(".")[0] + self.original = {} + self.attrs = attrs or [] + + def __enter__(self): + *submodules, target_attr = self.target.split(".") + + # Patch modules: + # it's used to patch attributes of submodules like "os.path.join"; + # in this case we need to patch "os" and "os.path" + + for i in range(len(submodules)): + try: + submodule = import_module(".".join(submodules[: i + 1])) + except ModuleNotFoundError: + continue + # We iterate over all the globals in self.obj in case we find "os" or "os.path" + for attr in self.obj.__dir__(): + obj_attr = getattr(self.obj, attr) + # We don't check for the name of the global, but rather if its value *is* "os" or "os.path". + # This allows to patch renamed modules like "from os import path as ospath". + if obj_attr is submodule or ( + isinstance(obj_attr, _PatchedModuleObj) and obj_attr._original_module is submodule + ): + self.original[attr] = obj_attr + # patch at top level + setattr(self.obj, attr, _PatchedModuleObj(obj_attr, attrs=self.attrs)) + patched = getattr(self.obj, attr) + # construct lower levels patches + for key in submodules[i + 1 :]: + setattr(patched, key, _PatchedModuleObj(getattr(patched, key, None), attrs=self.attrs)) + patched = getattr(patched, key) + # finally set the target attribute + setattr(patched, target_attr, self.new) + + # Patch attribute itself: + # it's used for builtins like "open", + # and also to patch "os.path.join" we may also need to patch "join" + # itself if it was imported as "from os.path import join". + + if submodules: # if it's an attribute of a submodule like "os.path.join" + try: + attr_value = getattr(import_module(".".join(submodules)), target_attr) + except (AttributeError, ModuleNotFoundError): + return + # We iterate over all the globals in self.obj in case we find "os.path.join" + for attr in self.obj.__dir__(): + # We don't check for the name of the global, but rather if its value *is* "os.path.join". + # This allows to patch renamed attributes like "from os.path import join as pjoin". + if getattr(self.obj, attr) is attr_value: + self.original[attr] = getattr(self.obj, attr) + setattr(self.obj, attr, self.new) + elif target_attr in globals()["__builtins__"]: # if it'a s builtin like "open" + self.original[target_attr] = globals()["__builtins__"][target_attr] + setattr(self.obj, target_attr, self.new) + else: + raise RuntimeError(f"Tried to patch attribute {target_attr} instead of a submodule.") + + def __exit__(self, *exc_info): + for attr in list(self.original): + setattr(self.obj, attr, self.original.pop(attr)) + + def start(self): + """Activate a patch.""" + self.__enter__() + self._active_patches.append(self) + + def stop(self): + """Stop an active patch.""" + try: + self._active_patches.remove(self) + except ValueError: + # If the patch hasn't been started this will fail + return None + + return self.__exit__() diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/py_utils.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/py_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..df6d7b977b554cf772eeb32dcdf8dc5f0b0f2f4c --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/py_utils.py @@ -0,0 +1,640 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Some python utils function and classes.""" + +import copy +import functools +import itertools +import multiprocessing.pool +import os +import queue +import re +import types +import warnings +from collections.abc import Iterable +from contextlib import contextmanager +from dataclasses import fields, is_dataclass +from queue import Empty +from shutil import disk_usage +from typing import Any, Callable, Optional, TypeVar, Union + +import multiprocess +import multiprocess.pool +import numpy as np +from tqdm.auto import tqdm + +from .. import config +from ..parallel import parallel_map +from . import logging +from . import tqdm as hf_tqdm +from ._dill import ( # noqa: F401 # imported for backward compatibility. TODO: remove in 3.0.0 + Pickler, + dump, + dumps, + pklregister, +) + + +try: # pragma: no branch + from typing import Final + + import typing_extensions as _typing_extensions + from typing_extensions import Literal +except ImportError: + _typing_extensions = Literal = Final = None + + +logger = logging.get_logger(__name__) + + +# NOTE: When used on an instance method, the cache is shared across all +# instances and IS NOT per-instance. +# See +# https://stackoverflow.com/questions/14946264/python-lru-cache-decorator-per-instance +# For @property methods, use @memoized_property below. +memoize = functools.lru_cache + + +def size_str(size_in_bytes): + """Returns a human readable size string. + + If size_in_bytes is None, then returns "Unknown size". + + For example `size_str(1.5 * datasets.units.GiB) == "1.50 GiB"`. + + Args: + size_in_bytes: `int` or `None`, the size, in bytes, that we want to + format as a human-readable size string. + """ + if not size_in_bytes: + return "Unknown size" + + _NAME_LIST = [("PiB", 2**50), ("TiB", 2**40), ("GiB", 2**30), ("MiB", 2**20), ("KiB", 2**10)] + + size_in_bytes = float(size_in_bytes) + for name, size_bytes in _NAME_LIST: + value = size_in_bytes / size_bytes + if value >= 1.0: + return f"{value:.2f} {name}" + return f"{int(size_in_bytes)} bytes" + + +def convert_file_size_to_int(size: Union[int, str]) -> int: + """ + Converts a size expressed as a string with digits an unit (like `"50MB"`) to an integer (in bytes). + + Args: + size (`int` or `str`): The size to convert. Will be directly returned if an `int`. + + Example: + + ```py + >>> convert_file_size_to_int("1MiB") + 1048576 + ``` + """ + if isinstance(size, int): + return size + if size.upper().endswith("PIB"): + return int(size[:-3]) * (2**50) + if size.upper().endswith("TIB"): + return int(size[:-3]) * (2**40) + if size.upper().endswith("GIB"): + return int(size[:-3]) * (2**30) + if size.upper().endswith("MIB"): + return int(size[:-3]) * (2**20) + if size.upper().endswith("KIB"): + return int(size[:-3]) * (2**10) + if size.upper().endswith("PB"): + int_size = int(size[:-2]) * (10**15) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("TB"): + int_size = int(size[:-2]) * (10**12) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("GB"): + int_size = int(size[:-2]) * (10**9) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("MB"): + int_size = int(size[:-2]) * (10**6) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("KB"): + int_size = int(size[:-2]) * (10**3) + return int_size // 8 if size.endswith("b") else int_size + raise ValueError(f"`size={size}` is not in a valid format. Use an integer followed by the unit, e.g., '5GB'.") + + +def glob_pattern_to_regex(pattern): + # partially taken from fsspec: + # https://github.com/fsspec/filesystem_spec/blob/697d0f8133d8a5fbc3926e4761d7ecd51337ce50/fsspec/asyn.py#L735 + return ( + pattern.replace("\\", r"\\") + .replace(".", r"\.") + .replace("*", ".*") + .replace("+", r"\+") + .replace("//", "/") + .replace("(", r"\(") + .replace(")", r"\)") + .replace("|", r"\|") + .replace("^", r"\^") + .replace("$", r"\$") + .rstrip("/") + .replace("?", ".") + ) + + +def string_to_dict(string: str, pattern: str) -> Optional[dict[str, str]]: + """Un-format a string using a python f-string pattern. + From https://stackoverflow.com/a/36838374 + + Example:: + + >>> p = 'hello, my name is {name} and I am a {age} year old {what}' + >>> s = p.format(name='cody', age=18, what='quarterback') + >>> s + 'hello, my name is cody and I am a 18 year old quarterback' + >>> string_to_dict(s, p) + {'age': '18', 'name': 'cody', 'what': 'quarterback'} + + Args: + string (str): input string + pattern (str): pattern formatted like a python f-string + This can be a regex - so in case of un-formatting paths you should use posix paths. + Otherwise backslashes for windows paths can cause issues. + + Returns: + Optional[dict[str, str]]: dictionary of variable -> value, retrieved from the input using the pattern, or + `None` if the string does not match the pattern. + """ + pattern = re.sub(r"{([^:}]+)(?::[^}]+)?}", r"{\1}", pattern) # remove format specifiers, e.g. {rank:05d} -> {rank} + regex = re.sub(r"{(.+?)}", r"(?P<_\1>.+)", pattern) + result = re.search(regex, string) + if result is None: + return None + values = list(result.groups()) + keys = re.findall(r"{(.+?)}", pattern) + _dict = dict(zip(keys, values)) + return _dict + + +def asdict(obj): + """Convert an object to its dictionary representation recursively. + + + """ + + # Implementation based on https://docs.python.org/3/library/dataclasses.html#dataclasses.asdict + + def _is_dataclass_instance(obj): + # https://docs.python.org/3/library/dataclasses.html#dataclasses.is_dataclass + return is_dataclass(obj) and not isinstance(obj, type) + + def _asdict_inner(obj): + if _is_dataclass_instance(obj): + result = {} + for f in fields(obj): + value = _asdict_inner(getattr(obj, f.name)) + if not f.init or value != f.default or f.metadata.get("include_in_asdict_even_if_is_default", False): + result[f.name] = value + return result + elif isinstance(obj, tuple) and hasattr(obj, "_fields"): + # obj is a namedtuple + return type(obj)(*[_asdict_inner(v) for v in obj]) + elif isinstance(obj, (list, tuple)): + # Assume we can create an object of this type by passing in a + # generator (which is not true for namedtuples, handled + # above). + return type(obj)(_asdict_inner(v) for v in obj) + elif isinstance(obj, dict): + return {_asdict_inner(k): _asdict_inner(v) for k, v in obj.items()} + else: + return copy.deepcopy(obj) + + if not isinstance(obj, dict) and not _is_dataclass_instance(obj): + raise TypeError(f"{obj} is not a dict or a dataclass") + + return _asdict_inner(obj) + + +@contextmanager +def temporary_assignment(obj, attr, value): + """Temporarily assign obj.attr to value.""" + original = getattr(obj, attr, None) + setattr(obj, attr, value) + try: + yield + finally: + setattr(obj, attr, original) + + +@contextmanager +def temp_seed(seed: int, set_pytorch=False, set_tensorflow=False): + """Temporarily set the random seed. This works for python numpy, pytorch and tensorflow.""" + np_state = np.random.get_state() + np.random.seed(seed) + + if set_pytorch and config.TORCH_AVAILABLE: + import torch + + torch_state = torch.random.get_rng_state() + torch.random.manual_seed(seed) + + if torch.cuda.is_available(): + torch_cuda_states = torch.cuda.get_rng_state_all() + torch.cuda.manual_seed_all(seed) + + if set_tensorflow and config.TF_AVAILABLE: + import tensorflow as tf + from tensorflow.python.eager import context as tfpycontext + + tf_state = tf.random.get_global_generator() + temp_gen = tf.random.Generator.from_seed(seed) + tf.random.set_global_generator(temp_gen) + + if not tf.executing_eagerly(): + raise ValueError("Setting random seed for TensorFlow is only available in eager mode") + + tf_context = tfpycontext.context() # eager mode context + tf_seed = tf_context._seed + tf_rng_initialized = hasattr(tf_context, "_rng") + if tf_rng_initialized: + tf_rng = tf_context._rng + tf_context._set_global_seed(seed) + + try: + yield + finally: + np.random.set_state(np_state) + + if set_pytorch and config.TORCH_AVAILABLE: + torch.random.set_rng_state(torch_state) + if torch.cuda.is_available(): + torch.cuda.set_rng_state_all(torch_cuda_states) + + if set_tensorflow and config.TF_AVAILABLE: + tf.random.set_global_generator(tf_state) + + tf_context._seed = tf_seed + if tf_rng_initialized: + tf_context._rng = tf_rng + else: + delattr(tf_context, "_rng") + + +def unique_values(values): + """Iterate over iterable and return only unique values in order.""" + seen = set() + for value in values: + if value not in seen: + seen.add(value) + yield value + + +def no_op_if_value_is_null(func): + """If the value is None, return None, else call `func`.""" + + def wrapper(value): + return func(value) if value is not None else None + + return wrapper + + +def first_non_null_value(iterable): + """Return the index and the value of the first non-null value in the iterable. If all values are None, return -1 as index.""" + for i, value in enumerate(iterable): + if value is not None: + return i, value + return -1, None + + +def first_non_null_non_empty_value(iterable): + """Return the index and the value of the first non-null non-empty value in the iterable. If all values are None or empty, return -1 as index.""" + for i, value in enumerate(iterable): + if value is not None and not (isinstance(value, (dict, list)) and len(value) == 0): + return i, value + return -1, None + + +def zip_dict(*dicts): + """Iterate over items of dictionaries grouped by their keys.""" + for key in unique_values(itertools.chain(*dicts)): # set merge all keys + # Will raise KeyError if the dict don't have the same keys + yield key, tuple(d[key] for d in dicts) + + +class NonMutableDict(dict): + """Dict where keys can only be added but not modified. + + Will raise an error if the user try to overwrite one key. The error message + can be customized during construction. It will be formatted using {key} for + the overwritten key. + """ + + def __init__(self, *args, **kwargs): + self._error_msg = kwargs.pop( + "error_msg", + "Try to overwrite existing key: {key}", + ) + if kwargs: + raise ValueError("NonMutableDict cannot be initialized with kwargs.") + super().__init__(*args, **kwargs) + + def __setitem__(self, key, value): + if key in self: + raise ValueError(self._error_msg.format(key=key)) + return super().__setitem__(key, value) + + def update(self, other): + if any(k in self for k in other): + raise ValueError(self._error_msg.format(key=set(self) & set(other))) + return super().update(other) + + +class classproperty(property): # pylint: disable=invalid-name + """Descriptor to be used as decorator for @classmethods.""" + + def __get__(self, obj, objtype=None): + return self.fget.__get__(None, objtype)() + + +def _single_map_nested(args): + """Apply a function recursively to each element of a nested data struct.""" + function, data_struct, batched, batch_size, types, rank, disable_tqdm, desc = args + + # Singleton first to spare some computation + if not isinstance(data_struct, dict) and not isinstance(data_struct, types): + if batched: + return function([data_struct])[0] + else: + return function(data_struct) + if ( + batched + and not isinstance(data_struct, dict) + and isinstance(data_struct, types) + and all(not isinstance(v, (dict, types)) for v in data_struct) + ): + return [mapped_item for batch in iter_batched(data_struct, batch_size) for mapped_item in function(batch)] + + # Reduce logging to keep things readable in multiprocessing with tqdm + if rank is not None and logging.get_verbosity() < logging.WARNING: + logging.set_verbosity_warning() + # Print at least one thing to fix tqdm in notebooks in multiprocessing + # see https://github.com/tqdm/tqdm/issues/485#issuecomment-473338308 + if rank is not None and not disable_tqdm and any("notebook" in tqdm_cls.__name__ for tqdm_cls in tqdm.__mro__): + print(" ", end="", flush=True) + + # Loop over single examples or batches and write to buffer/file if examples are to be updated + pbar_iterable = data_struct.items() if isinstance(data_struct, dict) else data_struct + pbar_desc = (desc + " " if desc is not None else "") + "#" + str(rank) if rank is not None else desc + with hf_tqdm(pbar_iterable, disable=disable_tqdm, position=rank, unit="obj", desc=pbar_desc) as pbar: + if isinstance(data_struct, dict): + return { + k: _single_map_nested((function, v, batched, batch_size, types, None, True, None)) for k, v in pbar + } + else: + mapped = [_single_map_nested((function, v, batched, batch_size, types, None, True, None)) for v in pbar] + if isinstance(data_struct, list): + return mapped + elif isinstance(data_struct, tuple): + return tuple(mapped) + else: + return np.array(mapped) + + +def map_nested( + function: Callable[[Any], Any], + data_struct: Any, + dict_only: bool = False, + map_list: bool = True, + map_tuple: bool = False, + map_numpy: bool = False, + num_proc: Optional[int] = None, + parallel_min_length: int = 2, + batched: bool = False, + batch_size: Optional[int] = 1000, + types: Optional[tuple] = None, + disable_tqdm: bool = True, + desc: Optional[str] = None, +) -> Any: + """Apply a function recursively to each element of a nested data struct. + + Use multiprocessing if num_proc > 1 and the length of data_struct is greater than or equal to + `parallel_min_length`. + + + + Before version 2.5.0, multiprocessing was not used if `num_proc` was greater than or equal to ``len(iterable)``. + + Now, if `num_proc` is greater than or equal to ``len(iterable)``, `num_proc` is set to ``len(iterable)`` and + multiprocessing is used. + + + + Args: + function (`Callable`): Function to be applied to `data_struct`. + data_struct (`Any`): Data structure to apply `function` to. + dict_only (`bool`, default `False`): Whether only apply `function` recursively to `dict` values in + `data_struct`. + map_list (`bool`, default `True`): Whether also apply `function` recursively to `list` elements (besides `dict` + values). + map_tuple (`bool`, default `False`): Whether also apply `function` recursively to `tuple` elements (besides + `dict` values). + map_numpy (`bool, default `False`): Whether also apply `function` recursively to `numpy.array` elements (besides + `dict` values). + num_proc (`int`, *optional*): Number of processes. + The level in the data struct used for multiprocessing is the first level that has smaller sub-structs, + starting from the root. + parallel_min_length (`int`, default `2`): Minimum length of `data_struct` required for parallel + processing. + + batched (`bool`, defaults to `False`): + Provide batch of items to `function`. + + batch_size (`int`, *optional*, defaults to `1000`): + Number of items per batch provided to `function` if `batched=True`. + If `batch_size <= 0` or `batch_size == None`, provide the full iterable as a single batch to `function`. + + types (`tuple`, *optional*): Additional types (besides `dict` values) to apply `function` recursively to their + elements. + disable_tqdm (`bool`, default `True`): Whether to disable the tqdm progressbar. + desc (`str`, *optional*): Prefix for the tqdm progressbar. + + Returns: + `Any` + """ + if types is None: + types = [] + if not dict_only: + if map_list: + types.append(list) + if map_tuple: + types.append(tuple) + if map_numpy: + types.append(np.ndarray) + types = tuple(types) + + # Singleton + if not isinstance(data_struct, dict) and not isinstance(data_struct, types): + if batched: + data_struct = [data_struct] + mapped = function(data_struct) + if batched: + mapped = mapped[0] + return mapped + + iterable = list(data_struct.values()) if isinstance(data_struct, dict) else data_struct + + if num_proc is None: + num_proc = 1 + if any(isinstance(v, types) and len(v) > len(iterable) for v in iterable): + mapped = [ + map_nested( + function=function, + data_struct=obj, + num_proc=num_proc, + parallel_min_length=parallel_min_length, + batched=batched, + batch_size=batch_size, + types=types, + ) + for obj in iterable + ] + elif num_proc != -1 and num_proc <= 1 or len(iterable) < parallel_min_length: + if batched: + if batch_size is None or batch_size <= 0: + batch_size = max(len(iterable) // num_proc + int(len(iterable) % num_proc > 0), 1) + iterable = list(iter_batched(iterable, batch_size)) + mapped = [ + _single_map_nested((function, obj, batched, batch_size, types, None, True, None)) + for obj in hf_tqdm(iterable, disable=disable_tqdm, desc=desc) + ] + if batched: + mapped = [mapped_item for mapped_batch in mapped for mapped_item in mapped_batch] + else: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=".* is experimental and might be subject to breaking changes in the future\\.$", + category=UserWarning, + ) + if batched: + if batch_size is None or batch_size <= 0: + batch_size = len(iterable) // num_proc + int(len(iterable) % num_proc > 0) + iterable = list(iter_batched(iterable, batch_size)) + mapped = parallel_map( + function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, _single_map_nested + ) + if batched: + mapped = [mapped_item for mapped_batch in mapped for mapped_item in mapped_batch] + + if isinstance(data_struct, dict): + return dict(zip(data_struct.keys(), mapped)) + else: + if isinstance(data_struct, list): + return mapped + elif isinstance(data_struct, tuple): + return tuple(mapped) + else: + return np.array(mapped) + + +class NestedDataStructure: + def __init__(self, data=None): + self.data = data if data is not None else [] + + def flatten(self, data=None): + data = data if data is not None else self.data + if isinstance(data, dict): + return self.flatten(list(data.values())) + elif isinstance(data, (list, tuple)): + return [flattened for item in data for flattened in self.flatten(item)] + else: + return [data] + + +def has_sufficient_disk_space(needed_bytes, directory="."): + try: + free_bytes = disk_usage(os.path.abspath(directory)).free + except OSError: + return True + return needed_bytes < free_bytes + + +def copyfunc(func): + result = types.FunctionType(func.__code__, func.__globals__, func.__name__, func.__defaults__, func.__closure__) + result.__kwdefaults__ = func.__kwdefaults__ + return result + + +Y = TypeVar("Y") + + +def _write_generator_to_queue(queue: queue.Queue, func: Callable[..., Iterable[Y]], kwargs: dict) -> int: + for i, result in enumerate(func(**kwargs)): + queue.put(result) + return i + + +def _get_pool_pid(pool: Union[multiprocessing.pool.Pool, multiprocess.pool.Pool]) -> set[int]: + return {f.pid for f in pool._pool} + + +def iflatmap_unordered( + pool: Union[multiprocessing.pool.Pool, multiprocess.pool.Pool], + func: Callable[..., Iterable[Y]], + *, + kwargs_iterable: Iterable[dict], +) -> Iterable[Y]: + initial_pool_pid = _get_pool_pid(pool) + pool_changed = False + with pool._ctx.Manager() as manager: + queue = manager.Queue() + async_results = [ + pool.apply_async(_write_generator_to_queue, (queue, func, kwargs)) for kwargs in kwargs_iterable + ] + try: + while True: + try: + yield queue.get(timeout=0.05) + except Empty: + if all(async_result.ready() for async_result in async_results) and queue.empty(): + break + if _get_pool_pid(pool) != initial_pool_pid: + pool_changed = True + # One of the subprocesses has died. We should not wait forever. + raise RuntimeError( + "One of the subprocesses has abruptly died during map operation." + "To debug the error, disable multiprocessing." + ) + finally: + if not pool_changed: + # we get the result in case there's an error to raise + [async_result.get(timeout=0.05) for async_result in async_results] + + +T = TypeVar("T") + + +def iter_batched(iterable: Iterable[T], n: int) -> Iterable[list[T]]: + if n < 1: + raise ValueError(f"Invalid batch size {n}") + batch = [] + for item in iterable: + batch.append(item) + if len(batch) == n: + yield batch + batch = [] + if batch: + yield batch diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/creators.json b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/creators.json new file mode 100644 index 0000000000000000000000000000000000000000..d9e15f0039cc27ed8abd9fdf394423a3fada2c95 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/creators.json @@ -0,0 +1,17 @@ +{ + "language": [ + "found", + "crowdsourced", + "expert-generated", + "machine-generated", + "other" + ], + "annotations": [ + "found", + "crowdsourced", + "expert-generated", + "machine-generated", + "no-annotation", + "other" + ] +} diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/languages.json b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/languages.json new file mode 100644 index 0000000000000000000000000000000000000000..ea7686f956b898af3faf97b86be89b71d88855d4 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/languages.json @@ -0,0 +1,8026 @@ +{ + "code": "Programming language (C++, Java, Javascript, Python, etc.)", + "aa": "Afar", + "aaa": "Ghotuo", + "aab": "Alumu-Tesu", + "aac": "Ari", + "aad": "Amal", + "aae": "Arbëreshë Albanian", + "aaf": "Aranadan", + "aag": "Ambrak", + "aah": "Abu' Arapesh", + "aai": "Arifama-Miniafia", + "aak": "Ankave", + "aal": "Afade", + "aan": "Anambé", + "aao": "Algerian Saharan Arabic", + "aap": "Pará Arára", + "aaq": "Eastern Abnaki", + "aas": "Aasáx", + "aat": "Arvanitika Albanian", + "aau": "Abau", + "aav": "Austro-Asiatic languages", + "aaw": "Solong", + "aax": "Mandobo Atas", + "aaz": "Amarasi", + "ab": "Abkhazian", + "aba": "Abé", + "abb": "Bankon", + "abc": "Ambala Ayta", + "abd": "Manide", + "abe": "Western Abnaki", + "abf": "Abai Sungai", + "abg": "Abaga", + "abh": "Tajiki Arabic", + "abi": "Abidji", + "abj": "Aka-Bea", + "abl": "Lampung Nyo", + "abm": "Abanyom", + "abn": "Abua", + "abo": "Abon", + "abp": "Abellen Ayta", + "abq": "Abaza", + "abr": "Abron", + "abs": "Ambonese Malay", + "abt": "Ambulas", + "abu": "Abure", + "abv": "Baharna Arabic", + "abw": "Pal", + "abx": "Inabaknon", + "aby": "Aneme Wake", + "abz": "Abui", + "aca": "Achagua", + "acb": "Áncá", + "acd": "Gikyode", + "ace": "Achinese", + "acf": "Saint Lucian Creole French", + "ach": "Acoli", + "aci": "Aka-Cari", + "ack": "Aka-Kora", + "acl": "Akar-Bale", + "acm": "Mesopotamian Arabic", + "acn": "Achang", + "acp": "Eastern Acipa", + "acq": "Ta'izzi-Adeni Arabic", + "acr": "Achi", + "acs": "Acroá", + "act": "Achterhoeks", + "acu": "Achuar-Shiwiar", + "acv": "Achumawi", + "acw": "Hijazi Arabic", + "acx": "Omani Arabic", + "acy": "Cypriot Arabic", + "acz": "Acheron", + "ada": "Adangme", + "adb": "Atauran", + "add": "Lidzonka; Dzodinka", + "ade": "Adele", + "adf": "Dhofari Arabic", + "adg": "Andegerebinha", + "adh": "Adhola", + "adi": "Adi", + "adj": "Adioukrou", + "adl": "Galo", + "adn": "Adang", + "ado": "Abu", + "adq": "Adangbe", + "adr": "Adonara", + "ads": "Adamorobe Sign Language", + "adt": "Adnyamathanha", + "adu": "Aduge", + "adw": "Amundava", + "adx": "Amdo Tibetan", + "ady": "Adyghe; Adygei", + "adz": "Adzera", + "ae": "Avestan", + "aea": "Areba", + "aeb": "Tunisian Arabic", + "aec": "Saidi Arabic", + "aed": "Argentine Sign Language", + "aee": "Northeast Pashai; Northeast Pashayi", + "aek": "Haeke", + "ael": "Ambele", + "aem": "Arem", + "aen": "Armenian Sign Language", + "aeq": "Aer", + "aer": "Eastern Arrernte", + "aes": "Alsea", + "aeu": "Akeu", + "aew": "Ambakich", + "aey": "Amele", + "aez": "Aeka", + "af": "Afrikaans", + "afa": "Afro-Asiatic languages", + "afb": "Gulf Arabic", + "afd": "Andai", + "afe": "Putukwam", + "afg": "Afghan Sign Language", + "afh": "Afrihili", + "afi": "Akrukay; Chini", + "afk": "Nanubae", + "afn": "Defaka", + "afo": "Eloyi", + "afp": "Tapei", + "afs": "Afro-Seminole Creole", + "aft": "Afitti", + "afu": "Awutu", + "afz": "Obokuitai", + "aga": "Aguano", + "agb": "Legbo", + "agc": "Agatu", + "agd": "Agarabi", + "age": "Angal", + "agf": "Arguni", + "agg": "Angor", + "agh": "Ngelima", + "agi": "Agariya", + "agj": "Argobba", + "agk": "Isarog Agta", + "agl": "Fembe", + "agm": "Angaataha", + "agn": "Agutaynen", + "ago": "Tainae", + "agq": "Aghem", + "agr": "Aguaruna", + "ags": "Esimbi", + "agt": "Central Cagayan Agta", + "agu": "Aguacateco", + "agv": "Remontado Dumagat", + "agw": "Kahua", + "agx": "Aghul", + "agy": "Southern Alta", + "agz": "Mt. Iriga Agta", + "aha": "Ahanta", + "ahb": "Axamb", + "ahg": "Qimant", + "ahh": "Aghu", + "ahi": "Tiagbamrin Aizi", + "ahk": "Akha", + "ahl": "Igo", + "ahm": "Mobumrin Aizi", + "ahn": "Àhàn", + "aho": "Ahom", + "ahp": "Aproumu Aizi", + "ahr": "Ahirani", + "ahs": "Ashe", + "aht": "Ahtena", + "aia": "Arosi", + "aib": "Ainu (China)", + "aic": "Ainbai", + "aid": "Alngith", + "aie": "Amara", + "aif": "Agi", + "aig": "Antigua and Barbuda Creole English", + "aih": "Ai-Cham", + "aii": "Assyrian Neo-Aramaic", + "aij": "Lishanid Noshan", + "aik": "Ake", + "ail": "Aimele", + "aim": "Aimol", + "ain": "Ainu (Japan)", + "aio": "Aiton", + "aip": "Burumakok", + "aiq": "Aimaq", + "air": "Airoran", + "ait": "Arikem", + "aiw": "Aari", + "aix": "Aighon", + "aiy": "Ali", + "aja": "Aja (South Sudan)", + "ajg": "Aja (Benin)", + "aji": "Ajië", + "ajn": "Andajin", + "ajp": "South Levantine Arabic", + "ajs": "Algerian Jewish Sign Language", + "aju": "Judeo-Moroccan Arabic", + "ajw": "Ajawa", + "ajz": "Amri Karbi", + "ak": "Akan", + "akb": "Batak Angkola", + "akc": "Mpur", + "akd": "Ukpet-Ehom", + "ake": "Akawaio", + "akf": "Akpa", + "akg": "Anakalangu", + "akh": "Angal Heneng", + "aki": "Aiome", + "akj": "Aka-Jeru", + "akk": "Akkadian", + "akl": "Aklanon", + "akm": "Aka-Bo", + "ako": "Akurio", + "akp": "Siwu", + "akq": "Ak", + "akr": "Araki", + "aks": "Akaselem", + "akt": "Akolet", + "aku": "Akum", + "akv": "Akhvakh", + "akw": "Akwa", + "akx": "Aka-Kede", + "aky": "Aka-Kol", + "akz": "Alabama", + "ala": "Alago", + "alc": "Qawasqar", + "ald": "Alladian", + "ale": "Aleut", + "alf": "Alege", + "alg": "Algonquian languages", + "alh": "Alawa", + "ali": "Amaimon", + "alj": "Alangan", + "alk": "Alak", + "all": "Allar", + "alm": "Amblong", + "aln": "Gheg Albanian", + "alo": "Larike-Wakasihu", + "alp": "Alune", + "alq": "Algonquin", + "alr": "Alutor", + "als": "Tosk Albanian", + "alt": "Southern Altai", + "alu": "'Are'are", + "alv": "Atlantic-Congo languages", + "alw": "Alaba-K’abeena; Wanbasana", + "alx": "Amol", + "aly": "Alyawarr", + "alz": "Alur", + "am": "Amharic", + "ama": "Amanayé", + "amb": "Ambo", + "amc": "Amahuaca", + "ame": "Yanesha'", + "amf": "Hamer-Banna", + "amg": "Amurdak", + "ami": "Amis", + "amj": "Amdang", + "amk": "Ambai", + "aml": "War-Jaintia", + "amm": "Ama (Papua New Guinea)", + "amn": "Amanab", + "amo": "Amo", + "amp": "Alamblak", + "amq": "Amahai", + "amr": "Amarakaeri", + "ams": "Southern Amami-Oshima", + "amt": "Amto", + "amu": "Guerrero Amuzgo", + "amv": "Ambelau", + "amw": "Western Neo-Aramaic", + "amx": "Anmatyerre", + "amy": "Ami", + "amz": "Atampaya", + "an": "Aragonese", + "ana": "Andaqui", + "anb": "Andoa", + "anc": "Ngas", + "and": "Ansus", + "ane": "Xârâcùù", + "anf": "Animere", + "ang": "Old English (ca. 450-1100)", + "anh": "Nend", + "ani": "Andi", + "anj": "Anor", + "ank": "Goemai", + "anl": "Anu-Hkongso Chin", + "anm": "Anal", + "ann": "Obolo", + "ano": "Andoque", + "anp": "Angika", + "anq": "Jarawa (India)", + "anr": "Andh", + "ans": "Anserma", + "ant": "Antakarinya; Antikarinya", + "anu": "Anuak", + "anv": "Denya", + "anw": "Anaang", + "anx": "Andra-Hus", + "any": "Anyin", + "anz": "Anem", + "aoa": "Angolar", + "aob": "Abom", + "aoc": "Pemon", + "aod": "Andarum", + "aoe": "Angal Enen", + "aof": "Bragat", + "aog": "Angoram", + "aoi": "Anindilyakwa", + "aoj": "Mufian", + "aok": "Arhö", + "aol": "Alor", + "aom": "Ömie", + "aon": "Bumbita Arapesh", + "aor": "Aore", + "aos": "Taikat", + "aot": "Atong (India); A'tong", + "aou": "A'ou", + "aox": "Atorada", + "aoz": "Uab Meto", + "apa": "Apache languages", + "apb": "Sa'a", + "apc": "North Levantine Arabic", + "apd": "Sudanese Arabic", + "ape": "Bukiyip", + "apf": "Pahanan Agta", + "apg": "Ampanang", + "aph": "Athpariya", + "api": "Apiaká", + "apj": "Jicarilla Apache", + "apk": "Kiowa Apache", + "apl": "Lipan Apache", + "apm": "Mescalero-Chiricahua Apache", + "apn": "Apinayé", + "apo": "Ambul", + "app": "Apma", + "apq": "A-Pucikwar", + "apr": "Arop-Lokep", + "aps": "Arop-Sissano", + "apt": "Apatani", + "apu": "Apurinã", + "apv": "Alapmunte", + "apw": "Western Apache", + "apx": "Aputai", + "apy": "Apalaí", + "apz": "Safeyoka", + "aqa": "Alacalufan languages", + "aqc": "Archi", + "aqd": "Ampari Dogon", + "aqg": "Arigidi", + "aqk": "Aninka", + "aql": "Algic languages", + "aqm": "Atohwaim", + "aqn": "Northern Alta", + "aqp": "Atakapa", + "aqr": "Arhâ", + "aqt": "Angaité", + "aqz": "Akuntsu", + "ar": "Arabic", + "arb": "Standard Arabic", + "arc": "Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)", + "ard": "Arabana", + "are": "Western Arrarnta", + "arh": "Arhuaco", + "ari": "Arikara", + "arj": "Arapaso", + "ark": "Arikapú", + "arl": "Arabela", + "arn": "Mapudungun; Mapuche", + "aro": "Araona", + "arp": "Arapaho", + "arq": "Algerian Arabic", + "arr": "Karo (Brazil)", + "ars": "Najdi Arabic", + "art": "Artificial languages", + "aru": "Aruá (Amazonas State); Arawá", + "arv": "Arbore", + "arw": "Arawak", + "arx": "Aruá (Rodonia State)", + "ary": "Moroccan Arabic", + "arz": "Egyptian Arabic", + "as": "Assamese", + "asa": "Asu (Tanzania)", + "asb": "Assiniboine", + "asc": "Casuarina Coast Asmat", + "ase": "American Sign Language", + "asf": "Auslan; Australian Sign Language", + "asg": "Cishingini", + "ash": "Abishira", + "asi": "Buruwai", + "asj": "Sari", + "ask": "Ashkun", + "asl": "Asilulu", + "asn": "Xingú Asuriní", + "aso": "Dano", + "asp": "Algerian Sign Language", + "asq": "Austrian Sign Language", + "asr": "Asuri", + "ass": "Ipulo", + "ast": "Asturian; Asturleonese; Bable; Leonese", + "asu": "Tocantins Asurini", + "asv": "Asoa", + "asw": "Australian Aborigines Sign Language", + "asx": "Muratayak", + "asy": "Yaosakor Asmat", + "asz": "As", + "ata": "Pele-Ata", + "atb": "Zaiwa", + "atc": "Atsahuaca", + "atd": "Ata Manobo", + "ate": "Atemble", + "atg": "Ivbie North-Okpela-Arhe", + "ath": "Athapascan languages", + "ati": "Attié", + "atj": "Atikamekw", + "atk": "Ati", + "atl": "Mt. Iraya Agta", + "atm": "Ata", + "atn": "Ashtiani", + "ato": "Atong (Cameroon)", + "atp": "Pudtol Atta", + "atq": "Aralle-Tabulahan", + "atr": "Waimiri-Atroari", + "ats": "Gros Ventre", + "att": "Pamplona Atta", + "atu": "Reel", + "atv": "Northern Altai", + "atw": "Atsugewi", + "atx": "Arutani", + "aty": "Aneityum", + "atz": "Arta", + "aua": "Asumboa", + "aub": "Alugu", + "auc": "Waorani", + "aud": "Anuta", + "auf": "Arauan languages", + "aug": "Aguna", + "auh": "Aushi", + "aui": "Anuki", + "auj": "Awjilah", + "auk": "Heyo", + "aul": "Aulua", + "aum": "Asu (Nigeria)", + "aun": "Molmo One", + "auo": "Auyokawa", + "aup": "Makayam", + "auq": "Anus; Korur", + "aur": "Aruek", + "aus": "Australian languages", + "aut": "Austral", + "auu": "Auye", + "auw": "Awyi", + "aux": "Aurá", + "auy": "Awiyaana", + "auz": "Uzbeki Arabic", + "av": "Avaric", + "avb": "Avau", + "avd": "Alviri-Vidari", + "avi": "Avikam", + "avk": "Kotava", + "avl": "Eastern Egyptian Bedawi Arabic", + "avm": "Angkamuthi", + "avn": "Avatime", + "avo": "Agavotaguerra", + "avs": "Aushiri", + "avt": "Au", + "avu": "Avokaya", + "avv": "Avá-Canoeiro", + "awa": "Awadhi", + "awb": "Awa (Papua New Guinea)", + "awc": "Cicipu", + "awd": "Arawakan languages", + "awe": "Awetí", + "awg": "Anguthimri", + "awh": "Awbono", + "awi": "Aekyom", + "awk": "Awabakal", + "awm": "Arawum", + "awn": "Awngi", + "awo": "Awak", + "awr": "Awera", + "aws": "South Awyu", + "awt": "Araweté", + "awu": "Central Awyu", + "awv": "Jair Awyu", + "aww": "Awun", + "awx": "Awara", + "awy": "Edera Awyu", + "axb": "Abipon", + "axe": "Ayerrerenge", + "axg": "Mato Grosso Arára", + "axk": "Yaka (Central African Republic)", + "axl": "Lower Southern Aranda", + "axm": "Middle Armenian", + "axx": "Xârâgurè", + "ay": "Aymara", + "aya": "Awar", + "ayb": "Ayizo Gbe", + "ayc": "Southern Aymara", + "ayd": "Ayabadhu", + "aye": "Ayere", + "ayg": "Ginyanga", + "ayh": "Hadrami Arabic", + "ayi": "Leyigha", + "ayk": "Akuku", + "ayl": "Libyan Arabic", + "ayn": "Sanaani Arabic", + "ayo": "Ayoreo", + "ayp": "North Mesopotamian Arabic", + "ayq": "Ayi (Papua New Guinea)", + "ayr": "Central Aymara", + "ays": "Sorsogon Ayta", + "ayt": "Magbukun Ayta", + "ayu": "Ayu", + "ayz": "Mai Brat", + "az": "Azerbaijani", + "aza": "Azha", + "azb": "South Azerbaijani", + "azc": "Uto-Aztecan languages", + "azd": "Eastern Durango Nahuatl", + "azg": "San Pedro Amuzgos Amuzgo", + "azj": "North Azerbaijani", + "azm": "Ipalapa Amuzgo", + "azn": "Western Durango Nahuatl", + "azo": "Awing", + "azt": "Faire Atta", + "azz": "Highland Puebla Nahuatl", + "ba": "Bashkir", + "baa": "Babatana", + "bab": "Bainouk-Gunyuño", + "bac": "Badui", + "bad": "Banda languages", + "bae": "Baré", + "baf": "Nubaca", + "bag": "Tuki", + "bah": "Bahamas Creole English", + "bai": "Bamileke languages", + "baj": "Barakai", + "bal": "Baluchi", + "ban": "Balinese", + "bao": "Waimaha", + "bap": "Bantawa", + "bar": "Bavarian", + "bas": "Basa (Cameroon)", + "bat": "Baltic languages", + "bau": "Bada (Nigeria)", + "bav": "Vengo", + "baw": "Bambili-Bambui", + "bax": "Bamun", + "bay": "Batuley", + "bba": "Baatonum", + "bbb": "Barai", + "bbc": "Batak Toba", + "bbd": "Bau", + "bbe": "Bangba", + "bbf": "Baibai", + "bbg": "Barama", + "bbh": "Bugan", + "bbi": "Barombi", + "bbj": "Ghomálá'", + "bbk": "Babanki", + "bbl": "Bats", + "bbm": "Babango", + "bbn": "Uneapa", + "bbo": "Northern Bobo Madaré; Konabéré", + "bbp": "West Central Banda", + "bbq": "Bamali", + "bbr": "Girawa", + "bbs": "Bakpinka", + "bbt": "Mburku", + "bbu": "Kulung (Nigeria)", + "bbv": "Karnai", + "bbw": "Baba", + "bbx": "Bubia", + "bby": "Befang", + "bca": "Central Bai", + "bcb": "Bainouk-Samik", + "bcc": "Southern Balochi", + "bcd": "North Babar", + "bce": "Bamenyam", + "bcf": "Bamu", + "bcg": "Baga Pokur", + "bch": "Bariai", + "bci": "Baoulé", + "bcj": "Bardi", + "bck": "Bunuba", + "bcl": "Central Bikol", + "bcm": "Bannoni", + "bcn": "Bali (Nigeria)", + "bco": "Kaluli", + "bcp": "Bali (Democratic Republic of Congo)", + "bcq": "Bench", + "bcr": "Babine", + "bcs": "Kohumono", + "bct": "Bendi", + "bcu": "Awad Bing", + "bcv": "Shoo-Minda-Nye", + "bcw": "Bana", + "bcy": "Bacama", + "bcz": "Bainouk-Gunyaamolo", + "bda": "Bayot", + "bdb": "Basap", + "bdc": "Emberá-Baudó", + "bdd": "Bunama", + "bde": "Bade", + "bdf": "Biage", + "bdg": "Bonggi", + "bdh": "Baka (South Sudan)", + "bdi": "Burun", + "bdj": "Bai (South Sudan); Bai", + "bdk": "Budukh", + "bdl": "Indonesian Bajau", + "bdm": "Buduma", + "bdn": "Baldemu", + "bdo": "Morom", + "bdp": "Bende", + "bdq": "Bahnar", + "bdr": "West Coast Bajau", + "bds": "Burunge", + "bdt": "Bokoto", + "bdu": "Oroko", + "bdv": "Bodo Parja", + "bdw": "Baham", + "bdx": "Budong-Budong", + "bdy": "Bandjalang", + "bdz": "Badeshi", + "be": "Belarusian", + "bea": "Beaver", + "beb": "Bebele", + "bec": "Iceve-Maci", + "bed": "Bedoanas", + "bee": "Byangsi", + "bef": "Benabena", + "beg": "Belait", + "beh": "Biali", + "bei": "Bekati'", + "bej": "Beja; Bedawiyet", + "bek": "Bebeli", + "bem": "Bemba (Zambia)", + "beo": "Beami", + "bep": "Besoa", + "beq": "Beembe", + "ber": "Berber languages", + "bes": "Besme", + "bet": "Guiberoua Béte", + "beu": "Blagar", + "bev": "Daloa Bété", + "bew": "Betawi", + "bex": "Jur Modo", + "bey": "Beli (Papua New Guinea)", + "bez": "Bena (Tanzania)", + "bfa": "Bari", + "bfb": "Pauri Bareli", + "bfc": "Panyi Bai; Northern Bai", + "bfd": "Bafut", + "bfe": "Betaf; Tena", + "bff": "Bofi", + "bfg": "Busang Kayan", + "bfh": "Blafe", + "bfi": "British Sign Language", + "bfj": "Bafanji", + "bfk": "Ban Khor Sign Language", + "bfl": "Banda-Ndélé", + "bfm": "Mmen", + "bfn": "Bunak", + "bfo": "Malba Birifor", + "bfp": "Beba", + "bfq": "Badaga", + "bfr": "Bazigar", + "bfs": "Southern Bai", + "bft": "Balti", + "bfu": "Gahri", + "bfw": "Bondo", + "bfx": "Bantayanon", + "bfy": "Bagheli", + "bfz": "Mahasu Pahari", + "bg": "Bulgarian", + "bga": "Gwamhi-Wuri", + "bgb": "Bobongko", + "bgc": "Haryanvi", + "bgd": "Rathwi Bareli", + "bge": "Bauria", + "bgf": "Bangandu", + "bgg": "Bugun", + "bgi": "Giangan", + "bgj": "Bangolan", + "bgk": "Bit; Buxinhua", + "bgl": "Bo (Laos)", + "bgn": "Western Balochi", + "bgo": "Baga Koga", + "bgp": "Eastern Balochi", + "bgq": "Bagri", + "bgr": "Bawm Chin", + "bgs": "Tagabawa", + "bgt": "Bughotu", + "bgu": "Mbongno", + "bgv": "Warkay-Bipim", + "bgw": "Bhatri", + "bgx": "Balkan Gagauz Turkish", + "bgy": "Benggoi", + "bgz": "Banggai", + "bh": "Bihari languages", + "bha": "Bharia", + "bhb": "Bhili", + "bhc": "Biga", + "bhd": "Bhadrawahi", + "bhe": "Bhaya", + "bhf": "Odiai", + "bhg": "Binandere", + "bhh": "Bukharic", + "bhi": "Bhilali", + "bhj": "Bahing", + "bhl": "Bimin", + "bhm": "Bathari", + "bhn": "Bohtan Neo-Aramaic", + "bho": "Bhojpuri", + "bhp": "Bima", + "bhq": "Tukang Besi South", + "bhr": "Bara Malagasy", + "bhs": "Buwal", + "bht": "Bhattiyali", + "bhu": "Bhunjia", + "bhv": "Bahau", + "bhw": "Biak", + "bhx": "Bhalay", + "bhy": "Bhele", + "bhz": "Bada (Indonesia)", + "bi": "Bislama", + "bia": "Badimaya", + "bib": "Bissa; Bisa", + "bid": "Bidiyo", + "bie": "Bepour", + "bif": "Biafada", + "big": "Biangai", + "bik": "Bikol", + "bil": "Bile", + "bim": "Bimoba", + "bin": "Bini; Edo", + "bio": "Nai", + "bip": "Bila", + "biq": "Bipi", + "bir": "Bisorio", + "bit": "Berinomo", + "biu": "Biete", + "biv": "Southern Birifor", + "biw": "Kol (Cameroon)", + "bix": "Bijori", + "biy": "Birhor", + "biz": "Baloi", + "bja": "Budza", + "bjb": "Banggarla", + "bjc": "Bariji", + "bje": "Biao-Jiao Mien", + "bjf": "Barzani Jewish Neo-Aramaic", + "bjg": "Bidyogo", + "bjh": "Bahinemo", + "bji": "Burji", + "bjj": "Kanauji", + "bjk": "Barok", + "bjl": "Bulu (Papua New Guinea)", + "bjm": "Bajelani", + "bjn": "Banjar", + "bjo": "Mid-Southern Banda", + "bjp": "Fanamaket", + "bjr": "Binumarien", + "bjs": "Bajan", + "bjt": "Balanta-Ganja", + "bju": "Busuu", + "bjv": "Bedjond", + "bjw": "Bakwé", + "bjx": "Banao Itneg", + "bjy": "Bayali", + "bjz": "Baruga", + "bka": "Kyak", + "bkc": "Baka (Cameroon)", + "bkd": "Binukid; Talaandig", + "bkf": "Beeke", + "bkg": "Buraka", + "bkh": "Bakoko", + "bki": "Baki", + "bkj": "Pande", + "bkk": "Brokskat", + "bkl": "Berik", + "bkm": "Kom (Cameroon)", + "bkn": "Bukitan", + "bko": "Kwa'", + "bkp": "Boko (Democratic Republic of Congo)", + "bkq": "Bakairí", + "bkr": "Bakumpai", + "bks": "Northern Sorsoganon", + "bkt": "Boloki", + "bku": "Buhid", + "bkv": "Bekwarra", + "bkw": "Bekwel", + "bkx": "Baikeno", + "bky": "Bokyi", + "bkz": "Bungku", + "bla": "Siksika", + "blb": "Bilua", + "blc": "Bella Coola", + "bld": "Bolango", + "ble": "Balanta-Kentohe", + "blf": "Buol", + "blh": "Kuwaa", + "bli": "Bolia", + "blj": "Bolongan", + "blk": "Pa'o Karen; Pa'O", + "bll": "Biloxi", + "blm": "Beli (South Sudan)", + "bln": "Southern Catanduanes Bikol", + "blo": "Anii", + "blp": "Blablanga", + "blq": "Baluan-Pam", + "blr": "Blang", + "bls": "Balaesang", + "blt": "Tai Dam", + "blv": "Kibala; Bolo", + "blw": "Balangao", + "blx": "Mag-Indi Ayta", + "bly": "Notre", + "blz": "Balantak", + "bm": "Bambara", + "bma": "Lame", + "bmb": "Bembe", + "bmc": "Biem", + "bmd": "Baga Manduri", + "bme": "Limassa", + "bmf": "Bom-Kim", + "bmg": "Bamwe", + "bmh": "Kein", + "bmi": "Bagirmi", + "bmj": "Bote-Majhi", + "bmk": "Ghayavi", + "bml": "Bomboli", + "bmm": "Northern Betsimisaraka Malagasy", + "bmn": "Bina (Papua New Guinea)", + "bmo": "Bambalang", + "bmp": "Bulgebi", + "bmq": "Bomu", + "bmr": "Muinane", + "bms": "Bilma Kanuri", + "bmt": "Biao Mon", + "bmu": "Somba-Siawari", + "bmv": "Bum", + "bmw": "Bomwali", + "bmx": "Baimak", + "bmz": "Baramu", + "bn": "Bengali; Bangla", + "bna": "Bonerate", + "bnb": "Bookan", + "bnc": "Bontok", + "bnd": "Banda (Indonesia)", + "bne": "Bintauna", + "bnf": "Masiwang", + "bng": "Benga", + "bni": "Bangi", + "bnj": "Eastern Tawbuid", + "bnk": "Bierebo", + "bnl": "Boon", + "bnm": "Batanga", + "bnn": "Bunun", + "bno": "Bantoanon", + "bnp": "Bola", + "bnq": "Bantik", + "bnr": "Butmas-Tur", + "bns": "Bundeli", + "bnt": "Bantu languages", + "bnu": "Bentong", + "bnv": "Bonerif; Beneraf; Edwas", + "bnw": "Bisis", + "bnx": "Bangubangu", + "bny": "Bintulu", + "bnz": "Beezen", + "bo": "Tibetan", + "boa": "Bora", + "bob": "Aweer", + "boe": "Mundabli", + "bof": "Bolon", + "bog": "Bamako Sign Language", + "boh": "Boma", + "boi": "Barbareño", + "boj": "Anjam", + "bok": "Bonjo", + "bol": "Bole", + "bom": "Berom", + "bon": "Bine", + "boo": "Tiemacèwè Bozo", + "bop": "Bonkiman", + "boq": "Bogaya", + "bor": "Borôro", + "bot": "Bongo", + "bou": "Bondei", + "bov": "Tuwuli", + "bow": "Rema", + "box": "Buamu", + "boy": "Bodo (Central African Republic)", + "boz": "Tiéyaxo Bozo", + "bpa": "Daakaka", + "bpc": "Mbuk", + "bpd": "Banda-Banda", + "bpe": "Bauni", + "bpg": "Bonggo", + "bph": "Botlikh", + "bpi": "Bagupi", + "bpj": "Binji", + "bpk": "Orowe; 'Ôrôê", + "bpl": "Broome Pearling Lugger Pidgin", + "bpm": "Biyom", + "bpn": "Dzao Min", + "bpo": "Anasi", + "bpp": "Kaure", + "bpq": "Banda Malay", + "bpr": "Koronadal Blaan", + "bps": "Sarangani Blaan", + "bpt": "Barrow Point", + "bpu": "Bongu", + "bpv": "Bian Marind", + "bpw": "Bo (Papua New Guinea)", + "bpx": "Palya Bareli", + "bpy": "Bishnupriya", + "bpz": "Bilba", + "bqa": "Tchumbuli", + "bqb": "Bagusa", + "bqc": "Boko (Benin); Boo", + "bqd": "Bung", + "bqf": "Baga Kaloum", + "bqg": "Bago-Kusuntu", + "bqh": "Baima", + "bqi": "Bakhtiari", + "bqj": "Bandial", + "bqk": "Banda-Mbrès", + "bql": "Bilakura", + "bqm": "Wumboko", + "bqn": "Bulgarian Sign Language", + "bqo": "Balo", + "bqp": "Busa", + "bqq": "Biritai", + "bqr": "Burusu", + "bqs": "Bosngun", + "bqt": "Bamukumbit", + "bqu": "Boguru", + "bqv": "Koro Wachi; Begbere-Ejar", + "bqw": "Buru (Nigeria)", + "bqx": "Baangi", + "bqy": "Bengkala Sign Language", + "bqz": "Bakaka", + "br": "Breton", + "bra": "Braj", + "brb": "Brao; Lave", + "brc": "Berbice Creole Dutch", + "brd": "Baraamu", + "brf": "Bira", + "brg": "Baure", + "brh": "Brahui", + "bri": "Mokpwe", + "brj": "Bieria", + "brk": "Birked", + "brl": "Birwa", + "brm": "Barambu", + "brn": "Boruca", + "bro": "Brokkat", + "brp": "Barapasi", + "brq": "Breri", + "brr": "Birao", + "brs": "Baras", + "brt": "Bitare", + "bru": "Eastern Bru", + "brv": "Western Bru", + "brw": "Bellari", + "brx": "Bodo (India)", + "bry": "Burui", + "brz": "Bilbil", + "bs": "Bosnian", + "bsa": "Abinomn", + "bsb": "Brunei Bisaya", + "bsc": "Bassari; Oniyan", + "bse": "Wushi", + "bsf": "Bauchi", + "bsg": "Bashkardi", + "bsh": "Kati", + "bsi": "Bassossi", + "bsj": "Bangwinji", + "bsk": "Burushaski", + "bsl": "Basa-Gumna", + "bsm": "Busami", + "bsn": "Barasana-Eduria", + "bso": "Buso", + "bsp": "Baga Sitemu", + "bsq": "Bassa", + "bsr": "Bassa-Kontagora", + "bss": "Akoose", + "bst": "Basketo", + "bsu": "Bahonsuai", + "bsv": "Baga Sobané", + "bsw": "Baiso", + "bsx": "Yangkam", + "bsy": "Sabah Bisaya", + "bta": "Bata", + "btc": "Bati (Cameroon)", + "btd": "Batak Dairi", + "bte": "Gamo-Ningi", + "btf": "Birgit", + "btg": "Gagnoa Bété", + "bth": "Biatah Bidayuh", + "bti": "Burate", + "btj": "Bacanese Malay", + "btk": "Batak languages", + "btm": "Batak Mandailing", + "btn": "Ratagnon", + "bto": "Rinconada Bikol", + "btp": "Budibud", + "btq": "Batek", + "btr": "Baetora", + "bts": "Batak Simalungun", + "btt": "Bete-Bendi", + "btu": "Batu", + "btv": "Bateri", + "btw": "Butuanon", + "btx": "Batak Karo", + "bty": "Bobot", + "btz": "Batak Alas-Kluet", + "bua": "Buriat", + "bub": "Bua", + "buc": "Bushi", + "bud": "Ntcham", + "bue": "Beothuk", + "buf": "Bushoong", + "bug": "Buginese", + "buh": "Younuo Bunu", + "bui": "Bongili", + "buj": "Basa-Gurmana", + "buk": "Bugawac", + "bum": "Bulu (Cameroon)", + "bun": "Sherbro", + "buo": "Terei", + "bup": "Busoa", + "buq": "Brem", + "bus": "Bokobaru", + "but": "Bungain", + "buu": "Budu", + "buv": "Bun", + "buw": "Bubi", + "bux": "Boghom", + "buy": "Bullom So", + "buz": "Bukwen", + "bva": "Barein", + "bvb": "Bube", + "bvc": "Baelelea", + "bvd": "Baeggu", + "bve": "Berau Malay", + "bvf": "Boor", + "bvg": "Bonkeng", + "bvh": "Bure", + "bvi": "Belanda Viri", + "bvj": "Baan", + "bvk": "Bukat", + "bvl": "Bolivian Sign Language", + "bvm": "Bamunka", + "bvn": "Buna", + "bvo": "Bolgo", + "bvp": "Bumang", + "bvq": "Birri", + "bvr": "Burarra", + "bvt": "Bati (Indonesia)", + "bvu": "Bukit Malay", + "bvv": "Baniva", + "bvw": "Boga", + "bvx": "Dibole", + "bvy": "Baybayanon", + "bvz": "Bauzi", + "bwa": "Bwatoo", + "bwb": "Namosi-Naitasiri-Serua", + "bwc": "Bwile", + "bwd": "Bwaidoka", + "bwe": "Bwe Karen", + "bwf": "Boselewa", + "bwg": "Barwe", + "bwh": "Bishuo", + "bwi": "Baniwa", + "bwj": "Láá Láá Bwamu", + "bwk": "Bauwaki", + "bwl": "Bwela", + "bwm": "Biwat", + "bwn": "Wunai Bunu", + "bwo": "Boro (Ethiopia); Borna (Ethiopia)", + "bwp": "Mandobo Bawah", + "bwq": "Southern Bobo Madaré", + "bwr": "Bura-Pabir", + "bws": "Bomboma", + "bwt": "Bafaw-Balong", + "bwu": "Buli (Ghana)", + "bww": "Bwa", + "bwx": "Bu-Nao Bunu", + "bwy": "Cwi Bwamu", + "bwz": "Bwisi", + "bxa": "Tairaha", + "bxb": "Belanda Bor", + "bxc": "Molengue", + "bxd": "Pela", + "bxe": "Birale", + "bxf": "Bilur; Minigir", + "bxg": "Bangala", + "bxh": "Buhutu", + "bxi": "Pirlatapa", + "bxj": "Bayungu", + "bxk": "Bukusu; Lubukusu", + "bxl": "Jalkunan", + "bxm": "Mongolia Buriat", + "bxn": "Burduna", + "bxo": "Barikanchi", + "bxp": "Bebil", + "bxq": "Beele", + "bxr": "Russia Buriat", + "bxs": "Busam", + "bxu": "China Buriat", + "bxv": "Berakou", + "bxw": "Bankagooma", + "bxz": "Binahari", + "bya": "Batak", + "byb": "Bikya", + "byc": "Ubaghara", + "byd": "Benyadu'", + "bye": "Pouye", + "byf": "Bete", + "byg": "Baygo", + "byh": "Bhujel", + "byi": "Buyu", + "byj": "Bina (Nigeria)", + "byk": "Biao", + "byl": "Bayono", + "bym": "Bidjara", + "byn": "Bilin; Blin", + "byo": "Biyo", + "byp": "Bumaji", + "byq": "Basay", + "byr": "Baruya; Yipma", + "bys": "Burak", + "byt": "Berti", + "byv": "Medumba", + "byw": "Belhariya", + "byx": "Qaqet", + "byz": "Banaro", + "bza": "Bandi", + "bzb": "Andio", + "bzc": "Southern Betsimisaraka Malagasy", + "bzd": "Bribri", + "bze": "Jenaama Bozo", + "bzf": "Boikin", + "bzg": "Babuza", + "bzh": "Mapos Buang", + "bzi": "Bisu", + "bzj": "Belize Kriol English", + "bzk": "Nicaragua Creole English", + "bzl": "Boano (Sulawesi)", + "bzm": "Bolondo", + "bzn": "Boano (Maluku)", + "bzo": "Bozaba", + "bzp": "Kemberano", + "bzq": "Buli (Indonesia)", + "bzr": "Biri", + "bzs": "Brazilian Sign Language", + "bzt": "Brithenig", + "bzu": "Burmeso", + "bzv": "Naami", + "bzw": "Basa (Nigeria)", + "bzx": "Kɛlɛngaxo Bozo", + "bzy": "Obanliku", + "bzz": "Evant", + "ca": "Catalan; Valencian", + "caa": "Chortí", + "cab": "Garifuna", + "cac": "Chuj", + "cad": "Caddo", + "cae": "Lehar; Laalaa", + "caf": "Southern Carrier", + "cag": "Nivaclé", + "cah": "Cahuarano", + "cai": "Central American Indian languages", + "caj": "Chané", + "cak": "Kaqchikel; Cakchiquel", + "cal": "Carolinian", + "cam": "Cemuhî", + "can": "Chambri", + "cao": "Chácobo", + "cap": "Chipaya", + "caq": "Car Nicobarese", + "car": "Galibi Carib", + "cas": "Tsimané", + "cau": "Caucasian languages", + "cav": "Cavineña", + "caw": "Callawalla", + "cax": "Chiquitano", + "cay": "Cayuga", + "caz": "Canichana", + "cba": "Chibchan languages", + "cbb": "Cabiyarí", + "cbc": "Carapana", + "cbd": "Carijona", + "cbg": "Chimila", + "cbi": "Chachi", + "cbj": "Ede Cabe", + "cbk": "Chavacano", + "cbl": "Bualkhaw Chin", + "cbn": "Nyahkur", + "cbo": "Izora", + "cbq": "Tsucuba; Cuba", + "cbr": "Cashibo-Cacataibo", + "cbs": "Cashinahua", + "cbt": "Chayahuita", + "cbu": "Candoshi-Shapra", + "cbv": "Cacua", + "cbw": "Kinabalian", + "cby": "Carabayo", + "ccc": "Chamicuro", + "ccd": "Cafundo Creole", + "cce": "Chopi", + "ccg": "Samba Daka", + "cch": "Atsam", + "ccj": "Kasanga", + "ccl": "Cutchi-Swahili", + "ccm": "Malaccan Creole Malay", + "ccn": "North Caucasian languages", + "cco": "Comaltepec Chinantec", + "ccp": "Chakma", + "ccr": "Cacaopera", + "ccs": "South Caucasian languages", + "cda": "Choni", + "cdc": "Chadic languages", + "cdd": "Caddoan languages", + "cde": "Chenchu", + "cdf": "Chiru", + "cdh": "Chambeali", + "cdi": "Chodri", + "cdj": "Churahi", + "cdm": "Chepang", + "cdn": "Chaudangsi", + "cdo": "Min Dong Chinese", + "cdr": "Cinda-Regi-Tiyal", + "cds": "Chadian Sign Language", + "cdy": "Chadong", + "cdz": "Koda", + "ce": "Chechen", + "cea": "Lower Chehalis", + "ceb": "Cebuano", + "ceg": "Chamacoco", + "cek": "Eastern Khumi Chin", + "cel": "Celtic languages", + "cen": "Cen", + "cet": "Centúúm", + "cey": "Ekai Chin", + "cfa": "Dijim-Bwilim", + "cfd": "Cara", + "cfg": "Como Karim", + "cfm": "Falam Chin", + "cga": "Changriwa", + "cgc": "Kagayanen", + "cgg": "Chiga", + "cgk": "Chocangacakha", + "ch": "Chamorro", + "chb": "Chibcha", + "chc": "Catawba", + "chd": "Highland Oaxaca Chontal", + "chf": "Tabasco Chontal", + "chg": "Chagatai", + "chh": "Chinook", + "chj": "Ojitlán Chinantec", + "chk": "Chuukese", + "chl": "Cahuilla", + "chm": "Mari (Russia)", + "chn": "Chinook jargon", + "cho": "Choctaw", + "chp": "Chipewyan; Dene Suline", + "chq": "Quiotepec Chinantec", + "chr": "Cherokee", + "cht": "Cholón", + "chw": "Chuwabu", + "chx": "Chantyal", + "chy": "Cheyenne", + "chz": "Ozumacín Chinantec", + "cia": "Cia-Cia", + "cib": "Ci Gbe", + "cic": "Chickasaw", + "cid": "Chimariko", + "cie": "Cineni", + "cih": "Chinali", + "cik": "Chitkuli Kinnauri", + "cim": "Cimbrian", + "cin": "Cinta Larga", + "cip": "Chiapanec", + "cir": "Tiri; Haméa; Méa", + "ciw": "Chippewa", + "ciy": "Chaima", + "cja": "Western Cham", + "cje": "Chru", + "cjh": "Upper Chehalis", + "cji": "Chamalal", + "cjk": "Chokwe", + "cjm": "Eastern Cham", + "cjn": "Chenapian", + "cjo": "Ashéninka Pajonal", + "cjp": "Cabécar", + "cjs": "Shor", + "cjv": "Chuave", + "cjy": "Jinyu Chinese", + "ckb": "Central Kurdish", + "ckh": "Chak", + "ckl": "Cibak", + "ckm": "Chakavian", + "ckn": "Kaang Chin", + "cko": "Anufo", + "ckq": "Kajakse", + "ckr": "Kairak", + "cks": "Tayo", + "ckt": "Chukot", + "cku": "Koasati", + "ckv": "Kavalan", + "ckx": "Caka", + "cky": "Cakfem-Mushere", + "ckz": "Cakchiquel-Quiché Mixed Language", + "cla": "Ron", + "clc": "Chilcotin", + "cld": "Chaldean Neo-Aramaic", + "cle": "Lealao Chinantec", + "clh": "Chilisso", + "cli": "Chakali", + "clj": "Laitu Chin", + "clk": "Idu-Mishmi", + "cll": "Chala", + "clm": "Clallam", + "clo": "Lowland Oaxaca Chontal", + "clt": "Lautu Chin", + "clu": "Caluyanun", + "clw": "Chulym", + "cly": "Eastern Highland Chatino", + "cma": "Maa", + "cmc": "Chamic languages", + "cme": "Cerma", + "cmg": "Classical Mongolian", + "cmi": "Emberá-Chamí", + "cml": "Campalagian", + "cmm": "Michigamea", + "cmn": "Mandarin Chinese", + "cmo": "Central Mnong", + "cmr": "Mro-Khimi Chin", + "cms": "Messapic", + "cmt": "Camtho", + "cna": "Changthang", + "cnb": "Chinbon Chin", + "cnc": "Côông", + "cng": "Northern Qiang", + "cnh": "Hakha Chin; Haka Chin", + "cni": "Asháninka", + "cnk": "Khumi Chin", + "cnl": "Lalana Chinantec", + "cno": "Con", + "cnp": "Northern Ping Chinese; Northern Pinghua", + "cnq": "Chung", + "cnr": "Montenegrin", + "cns": "Central Asmat", + "cnt": "Tepetotutla Chinantec", + "cnu": "Chenoua", + "cnw": "Ngawn Chin", + "cnx": "Middle Cornish", + "co": "Corsican", + "coa": "Cocos Islands Malay", + "cob": "Chicomuceltec", + "coc": "Cocopa", + "cod": "Cocama-Cocamilla", + "coe": "Koreguaje", + "cof": "Colorado", + "cog": "Chong", + "coh": "Chonyi-Dzihana-Kauma; Chichonyi-Chidzihana-Chikauma", + "coj": "Cochimi", + "cok": "Santa Teresa Cora", + "col": "Columbia-Wenatchi", + "com": "Comanche", + "con": "Cofán", + "coo": "Comox", + "cop": "Coptic", + "coq": "Coquille", + "cot": "Caquinte", + "cou": "Wamey", + "cov": "Cao Miao", + "cow": "Cowlitz", + "cox": "Nanti", + "coz": "Chochotec", + "cpa": "Palantla Chinantec", + "cpb": "Ucayali-Yurúa Ashéninka", + "cpc": "Ajyíninka Apurucayali", + "cpe": "English-based creoles and pidgins", + "cpf": "French-based creoles and pidgins", + "cpg": "Cappadocian Greek", + "cpi": "Chinese Pidgin English", + "cpn": "Cherepon", + "cpo": "Kpeego", + "cpp": "Portuguese-based creoles and pidgins", + "cps": "Capiznon", + "cpu": "Pichis Ashéninka", + "cpx": "Pu-Xian Chinese", + "cpy": "South Ucayali Ashéninka", + "cqd": "Chuanqiandian Cluster Miao", + "cr": "Cree", + "cra": "Chara", + "crb": "Island Carib", + "crc": "Lonwolwol", + "crd": "Coeur d'Alene", + "crf": "Caramanta", + "crg": "Michif", + "crh": "Crimean Tatar; Crimean Turkish", + "cri": "Sãotomense", + "crj": "Southern East Cree", + "crk": "Plains Cree", + "crl": "Northern East Cree", + "crm": "Moose Cree", + "crn": "El Nayar Cora", + "cro": "Crow", + "crp": "Creoles and pidgins", + "crq": "Iyo'wujwa Chorote", + "crr": "Carolina Algonquian", + "crs": "Seselwa Creole French", + "crt": "Iyojwa'ja Chorote", + "crv": "Chaura", + "crw": "Chrau", + "crx": "Carrier", + "cry": "Cori", + "crz": "Cruzeño", + "cs": "Czech", + "csa": "Chiltepec Chinantec", + "csb": "Kashubian", + "csc": "Catalan Sign Language; Lengua de señas catalana; Llengua de Signes Catalana", + "csd": "Chiangmai Sign Language", + "cse": "Czech Sign Language", + "csf": "Cuba Sign Language", + "csg": "Chilean Sign Language", + "csh": "Asho Chin", + "csi": "Coast Miwok", + "csj": "Songlai Chin", + "csk": "Jola-Kasa", + "csl": "Chinese Sign Language", + "csm": "Central Sierra Miwok", + "csn": "Colombian Sign Language", + "cso": "Sochiapam Chinantec; Sochiapan Chinantec", + "csp": "Southern Ping Chinese; Southern Pinghua", + "csq": "Croatia Sign Language", + "csr": "Costa Rican Sign Language", + "css": "Southern Ohlone", + "cst": "Northern Ohlone", + "csu": "Central Sudanic languages", + "csv": "Sumtu Chin", + "csw": "Swampy Cree", + "csx": "Cambodian Sign Language", + "csy": "Siyin Chin", + "csz": "Coos", + "cta": "Tataltepec Chatino", + "ctc": "Chetco", + "ctd": "Tedim Chin", + "cte": "Tepinapa Chinantec", + "ctg": "Chittagonian", + "cth": "Thaiphum Chin", + "ctl": "Tlacoatzintepec Chinantec", + "ctm": "Chitimacha", + "ctn": "Chhintange", + "cto": "Emberá-Catío", + "ctp": "Western Highland Chatino", + "cts": "Northern Catanduanes Bikol", + "ctt": "Wayanad Chetti", + "ctu": "Chol", + "cty": "Moundadan Chetty", + "ctz": "Zacatepec Chatino", + "cu": "Church Slavic; Church Slavonic; Old Bulgarian; Old Church Slavonic; Old Slavonic", + "cua": "Cua", + "cub": "Cubeo", + "cuc": "Usila Chinantec", + "cuh": "Chuka; Gichuka", + "cui": "Cuiba", + "cuj": "Mashco Piro", + "cuk": "San Blas Kuna", + "cul": "Culina; Kulina", + "cuo": "Cumanagoto", + "cup": "Cupeño", + "cuq": "Cun", + "cur": "Chhulung", + "cus": "Cushitic languages", + "cut": "Teutila Cuicatec", + "cuu": "Tai Ya", + "cuv": "Cuvok", + "cuw": "Chukwa", + "cux": "Tepeuxila Cuicatec", + "cuy": "Cuitlatec", + "cv": "Chuvash", + "cvg": "Chug", + "cvn": "Valle Nacional Chinantec", + "cwa": "Kabwa", + "cwb": "Maindo", + "cwd": "Woods Cree", + "cwe": "Kwere", + "cwg": "Chewong; Cheq Wong", + "cwt": "Kuwaataay", + "cy": "Welsh", + "cya": "Nopala Chatino", + "cyb": "Cayubaba", + "cyo": "Cuyonon", + "czh": "Huizhou Chinese", + "czk": "Knaanic", + "czn": "Zenzontepec Chatino", + "czo": "Min Zhong Chinese", + "czt": "Zotung Chin", + "da": "Danish", + "daa": "Dangaléat", + "dac": "Dambi", + "dad": "Marik", + "dae": "Duupa", + "dag": "Dagbani", + "dah": "Gwahatike", + "dai": "Day", + "daj": "Dar Fur Daju", + "dak": "Dakota", + "dal": "Dahalo", + "dam": "Damakawa", + "dao": "Daai Chin", + "daq": "Dandami Maria", + "dar": "Dargwa", + "das": "Daho-Doo", + "dau": "Dar Sila Daju", + "dav": "Taita; Dawida", + "daw": "Davawenyo", + "dax": "Dayi", + "day": "Land Dayak languages", + "daz": "Dao", + "dba": "Bangime", + "dbb": "Deno", + "dbd": "Dadiya", + "dbe": "Dabe", + "dbf": "Edopi", + "dbg": "Dogul Dom Dogon", + "dbi": "Doka", + "dbj": "Ida'an", + "dbl": "Dyirbal", + "dbm": "Duguri", + "dbn": "Duriankere", + "dbo": "Dulbu", + "dbp": "Duwai", + "dbq": "Daba", + "dbr": "Dabarre", + "dbt": "Ben Tey Dogon", + "dbu": "Bondum Dom Dogon", + "dbv": "Dungu", + "dbw": "Bankan Tey Dogon", + "dby": "Dibiyaso", + "dcc": "Deccan", + "dcr": "Negerhollands", + "dda": "Dadi Dadi", + "ddd": "Dongotono", + "dde": "Doondo", + "ddg": "Fataluku", + "ddi": "West Goodenough", + "ddj": "Jaru", + "ddn": "Dendi (Benin)", + "ddo": "Dido", + "ddr": "Dhudhuroa", + "dds": "Donno So Dogon", + "ddw": "Dawera-Daweloor", + "de": "German", + "dec": "Dagik", + "ded": "Dedua", + "dee": "Dewoin", + "def": "Dezfuli", + "deg": "Degema", + "deh": "Dehwari", + "dei": "Demisa", + "dek": "Dek", + "del": "Delaware", + "dem": "Dem", + "den": "Slave (Athapascan)", + "dep": "Pidgin Delaware", + "deq": "Dendi (Central African Republic)", + "der": "Deori", + "des": "Desano", + "dev": "Domung", + "dez": "Dengese", + "dga": "Southern Dagaare", + "dgb": "Bunoge Dogon", + "dgc": "Casiguran Dumagat Agta", + "dgd": "Dagaari Dioula", + "dge": "Degenan", + "dgg": "Doga", + "dgh": "Dghwede", + "dgi": "Northern Dagara", + "dgk": "Dagba", + "dgl": "Andaandi; Dongolawi", + "dgn": "Dagoman", + "dgo": "Dogri (individual language)", + "dgr": "Dogrib; Tłı̨chǫ", + "dgs": "Dogoso", + "dgt": "Ndra'ngith", + "dgw": "Daungwurrung", + "dgx": "Doghoro", + "dgz": "Daga", + "dhd": "Dhundari", + "dhg": "Dhangu-Djangu; Dhangu; Djangu", + "dhi": "Dhimal", + "dhl": "Dhalandji", + "dhm": "Zemba", + "dhn": "Dhanki", + "dho": "Dhodia", + "dhr": "Dhargari", + "dhs": "Dhaiso", + "dhu": "Dhurga", + "dhv": "Dehu; Drehu", + "dhw": "Dhanwar (Nepal)", + "dhx": "Dhungaloo", + "dia": "Dia", + "dib": "South Central Dinka", + "dic": "Lakota Dida", + "did": "Didinga", + "dif": "Dieri; Diyari", + "dig": "Digo; Chidigo", + "dih": "Kumiai", + "dii": "Dimbong", + "dij": "Dai", + "dik": "Southwestern Dinka", + "dil": "Dilling", + "dim": "Dime", + "din": "Dinka", + "dio": "Dibo", + "dip": "Northeastern Dinka", + "diq": "Dimli (individual language)", + "dir": "Dirim", + "dis": "Dimasa", + "diu": "Diriku", + "diw": "Northwestern Dinka", + "dix": "Dixon Reef", + "diy": "Diuwe", + "diz": "Ding", + "dja": "Djadjawurrung", + "djb": "Djinba", + "djc": "Dar Daju Daju", + "djd": "Djamindjung; Ngaliwurru", + "dje": "Zarma", + "djf": "Djangun", + "dji": "Djinang", + "djj": "Djeebbana", + "djk": "Eastern Maroon Creole; Businenge Tongo; Nenge", + "djm": "Jamsay Dogon", + "djn": "Jawoyn; Djauan", + "djo": "Jangkang", + "djr": "Djambarrpuyngu", + "dju": "Kapriman", + "djw": "Djawi", + "dka": "Dakpakha", + "dkg": "Kadung", + "dkk": "Dakka", + "dkr": "Kuijau", + "dks": "Southeastern Dinka", + "dkx": "Mazagway", + "dlg": "Dolgan", + "dlk": "Dahalik", + "dlm": "Dalmatian", + "dln": "Darlong", + "dma": "Duma", + "dmb": "Mombo Dogon", + "dmc": "Gavak", + "dmd": "Madhi Madhi", + "dme": "Dugwor", + "dmf": "Medefaidrin", + "dmg": "Upper Kinabatangan", + "dmk": "Domaaki", + "dml": "Dameli", + "dmm": "Dama", + "dmn": "Mande languages", + "dmo": "Kemedzung", + "dmr": "East Damar", + "dms": "Dampelas", + "dmu": "Dubu; Tebi", + "dmv": "Dumpas", + "dmw": "Mudburra", + "dmx": "Dema", + "dmy": "Demta; Sowari", + "dna": "Upper Grand Valley Dani", + "dnd": "Daonda", + "dne": "Ndendeule", + "dng": "Dungan", + "dni": "Lower Grand Valley Dani", + "dnj": "Dan", + "dnk": "Dengka", + "dnn": "Dzùùngoo", + "dno": "Ndrulo; Northern Lendu", + "dnr": "Danaru", + "dnt": "Mid Grand Valley Dani", + "dnu": "Danau", + "dnv": "Danu", + "dnw": "Western Dani", + "dny": "Dení", + "doa": "Dom", + "dob": "Dobu", + "doc": "Northern Dong", + "doe": "Doe", + "dof": "Domu", + "doh": "Dong", + "doi": "Dogri (macrolanguage)", + "dok": "Dondo", + "dol": "Doso", + "don": "Toura (Papua New Guinea)", + "doo": "Dongo", + "dop": "Lukpa", + "doq": "Dominican Sign Language", + "dor": "Dori'o", + "dos": "Dogosé", + "dot": "Dass", + "dov": "Dombe", + "dow": "Doyayo", + "dox": "Bussa", + "doy": "Dompo", + "doz": "Dorze", + "dpp": "Papar", + "dra": "Dravidian languages", + "drb": "Dair", + "drc": "Minderico", + "drd": "Darmiya", + "dre": "Dolpo", + "drg": "Rungus", + "dri": "C'Lela", + "drl": "Paakantyi", + "drn": "West Damar", + "dro": "Daro-Matu Melanau", + "drq": "Dura", + "drs": "Gedeo", + "drt": "Drents", + "dru": "Rukai", + "dry": "Darai", + "dsb": "Lower Sorbian", + "dse": "Dutch Sign Language", + "dsh": "Daasanach", + "dsi": "Disa", + "dsl": "Danish Sign Language", + "dsn": "Dusner", + "dso": "Desiya", + "dsq": "Tadaksahak", + "dsz": "Mardin Sign Language", + "dta": "Daur", + "dtb": "Labuk-Kinabatangan Kadazan", + "dtd": "Ditidaht", + "dth": "Adithinngithigh", + "dti": "Ana Tinga Dogon", + "dtk": "Tene Kan Dogon", + "dtm": "Tomo Kan Dogon", + "dtn": "Daatsʼíin", + "dto": "Tommo So Dogon", + "dtp": "Kadazan Dusun; Central Dusun", + "dtr": "Lotud", + "dts": "Toro So Dogon", + "dtt": "Toro Tegu Dogon", + "dtu": "Tebul Ure Dogon", + "dty": "Dotyali", + "dua": "Duala", + "dub": "Dubli", + "duc": "Duna", + "due": "Umiray Dumaget Agta", + "duf": "Dumbea; Drubea", + "dug": "Duruma; Chiduruma", + "duh": "Dungra Bhil", + "dui": "Dumun", + "duk": "Uyajitaya", + "dul": "Alabat Island Agta", + "dum": "Middle Dutch (ca. 1050-1350)", + "dun": "Dusun Deyah", + "duo": "Dupaninan Agta", + "dup": "Duano", + "duq": "Dusun Malang", + "dur": "Dii", + "dus": "Dumi", + "duu": "Drung", + "duv": "Duvle", + "duw": "Dusun Witu", + "dux": "Duungooma", + "duy": "Dicamay Agta", + "duz": "Duli-Gey", + "dv": "Dhivehi; Divehi; Maldivian", + "dva": "Duau", + "dwa": "Diri", + "dwk": "Dawik Kui", + "dwr": "Dawro", + "dws": "Dutton World Speedwords", + "dwu": "Dhuwal", + "dww": "Dawawa", + "dwy": "Dhuwaya", + "dwz": "Dewas Rai", + "dya": "Dyan", + "dyb": "Dyaberdyaber", + "dyd": "Dyugun", + "dyg": "Villa Viciosa Agta", + "dyi": "Djimini Senoufo", + "dym": "Yanda Dom Dogon", + "dyn": "Dyangadi; Dhanggatti", + "dyo": "Jola-Fonyi", + "dyu": "Dyula", + "dyy": "Djabugay; Dyaabugay", + "dz": "Dzongkha", + "dza": "Tunzu", + "dze": "Djiwarli", + "dzg": "Dazaga", + "dzl": "Dzalakha", + "dzn": "Dzando", + "eaa": "Karenggapa", + "ebc": "Beginci", + "ebg": "Ebughu", + "ebk": "Eastern Bontok", + "ebo": "Teke-Ebo", + "ebr": "Ebrié", + "ebu": "Embu; Kiembu", + "ecr": "Eteocretan", + "ecs": "Ecuadorian Sign Language", + "ecy": "Eteocypriot", + "ee": "Ewe", + "eee": "E", + "efa": "Efai", + "efe": "Efe", + "efi": "Efik", + "ega": "Ega", + "egl": "Emilian", + "egm": "Benamanga", + "ego": "Eggon", + "egx": "Egyptian languages", + "egy": "Egyptian (Ancient)", + "ehs": "Miyakubo Sign Language", + "ehu": "Ehueun", + "eip": "Eipomek", + "eit": "Eitiep", + "eiv": "Askopan", + "eja": "Ejamat", + "eka": "Ekajuk", + "eke": "Ekit", + "ekg": "Ekari", + "eki": "Eki", + "ekk": "Standard Estonian", + "ekl": "Kol (Bangladesh); Kol", + "ekm": "Elip", + "eko": "Koti", + "ekp": "Ekpeye", + "ekr": "Yace", + "eky": "Eastern Kayah", + "el": "Modern Greek (1453-)", + "ele": "Elepi", + "elh": "El Hugeirat", + "eli": "Nding", + "elk": "Elkei", + "elm": "Eleme", + "elo": "El Molo", + "elu": "Elu", + "elx": "Elamite", + "ema": "Emai-Iuleha-Ora", + "emb": "Embaloh", + "eme": "Emerillon", + "emg": "Eastern Meohang", + "emi": "Mussau-Emira", + "emk": "Eastern Maninkakan", + "emm": "Mamulique", + "emn": "Eman", + "emp": "Northern Emberá", + "emq": "Eastern Minyag", + "ems": "Pacific Gulf Yupik", + "emu": "Eastern Muria", + "emw": "Emplawas", + "emx": "Erromintxela", + "emy": "Epigraphic Mayan", + "emz": "Mbessa", + "en": "English", + "ena": "Apali", + "enb": "Markweeta", + "enc": "En", + "end": "Ende", + "enf": "Forest Enets", + "enh": "Tundra Enets", + "enl": "Enlhet", + "enm": "Middle English (1100-1500)", + "enn": "Engenni", + "eno": "Enggano", + "enq": "Enga", + "enr": "Emumu; Emem", + "enu": "Enu", + "env": "Enwan (Edo State)", + "enw": "Enwan (Akwa Ibom State)", + "enx": "Enxet", + "eo": "Esperanto", + "eot": "Beti (Côte d'Ivoire)", + "epi": "Epie", + "era": "Eravallan", + "erg": "Sie", + "erh": "Eruwa", + "eri": "Ogea", + "erk": "South Efate", + "ero": "Horpa", + "err": "Erre", + "ers": "Ersu", + "ert": "Eritai", + "erw": "Erokwanas", + "es": "Spanish; Castilian", + "ese": "Ese Ejja", + "esg": "Aheri Gondi", + "esh": "Eshtehardi", + "esi": "North Alaskan Inupiatun", + "esk": "Northwest Alaska Inupiatun", + "esl": "Egypt Sign Language", + "esm": "Esuma", + "esn": "Salvadoran Sign Language", + "eso": "Estonian Sign Language", + "esq": "Esselen", + "ess": "Central Siberian Yupik", + "esu": "Central Yupik", + "esx": "Eskimo-Aleut languages", + "esy": "Eskayan", + "et": "Estonian", + "etb": "Etebi", + "etc": "Etchemin", + "eth": "Ethiopian Sign Language", + "etn": "Eton (Vanuatu)", + "eto": "Eton (Cameroon)", + "etr": "Edolo", + "ets": "Yekhee", + "ett": "Etruscan", + "etu": "Ejagham", + "etx": "Eten", + "etz": "Semimi", + "eu": "Basque", + "euq": "Basque (family)", + "eve": "Even", + "evh": "Uvbie", + "evn": "Evenki", + "ewo": "Ewondo", + "ext": "Extremaduran", + "eya": "Eyak", + "eyo": "Keiyo", + "eza": "Ezaa", + "eze": "Uzekwe", + "fa": "Persian", + "faa": "Fasu", + "fab": "Fa d'Ambu", + "fad": "Wagi", + "faf": "Fagani", + "fag": "Finongan", + "fah": "Baissa Fali", + "fai": "Faiwol", + "faj": "Faita", + "fak": "Fang (Cameroon)", + "fal": "South Fali", + "fam": "Fam", + "fan": "Fang (Equatorial Guinea)", + "fap": "Paloor", + "far": "Fataleka", + "fat": "Fanti", + "fau": "Fayu", + "fax": "Fala", + "fay": "Southwestern Fars", + "faz": "Northwestern Fars", + "fbl": "West Albay Bikol", + "fcs": "Quebec Sign Language", + "fer": "Feroge", + "ff": "Fulah", + "ffi": "Foia Foia", + "ffm": "Maasina Fulfulde", + "fgr": "Fongoro", + "fi": "Finnish", + "fia": "Nobiin", + "fie": "Fyer", + "fif": "Faifi", + "fil": "Filipino; Pilipino", + "fip": "Fipa", + "fir": "Firan", + "fit": "Tornedalen Finnish; Meänkieli", + "fiu": "Finno-Ugrian languages", + "fiw": "Fiwaga", + "fj": "Fijian", + "fkk": "Kirya-Konzəl", + "fkv": "Kven Finnish", + "fla": "Kalispel-Pend d'Oreille", + "flh": "Foau", + "fli": "Fali", + "fll": "North Fali", + "fln": "Flinders Island", + "flr": "Fuliiru", + "fly": "Flaaitaal; Tsotsitaal", + "fmp": "Fe'fe'", + "fmu": "Far Western Muria", + "fnb": "Fanbak", + "fng": "Fanagalo", + "fni": "Fania", + "fo": "Faroese", + "fod": "Foodo", + "foi": "Foi", + "fom": "Foma", + "fon": "Fon", + "for": "Fore", + "fos": "Siraya", + "fox": "Formosan languages", + "fpe": "Fernando Po Creole English", + "fqs": "Fas", + "fr": "French", + "frc": "Cajun French", + "frd": "Fordata", + "frk": "Frankish", + "frm": "Middle French (ca. 1400-1600)", + "fro": "Old French (842-ca. 1400)", + "frp": "Arpitan; Francoprovençal", + "frq": "Forak", + "frr": "Northern Frisian", + "frs": "Eastern Frisian", + "frt": "Fortsenal", + "fse": "Finnish Sign Language", + "fsl": "French Sign Language", + "fss": "Finland-Swedish Sign Language; finlandssvenskt teckenspråk; suomenruotsalainen viittomakieli", + "fub": "Adamawa Fulfulde", + "fuc": "Pulaar", + "fud": "East Futuna", + "fue": "Borgu Fulfulde", + "fuf": "Pular", + "fuh": "Western Niger Fulfulde", + "fui": "Bagirmi Fulfulde", + "fuj": "Ko", + "fum": "Fum", + "fun": "Fulniô", + "fuq": "Central-Eastern Niger Fulfulde", + "fur": "Friulian", + "fut": "Futuna-Aniwa", + "fuu": "Furu", + "fuv": "Nigerian Fulfulde", + "fuy": "Fuyug", + "fvr": "Fur", + "fwa": "Fwâi", + "fwe": "Fwe", + "fy": "Western Frisian", + "ga": "Irish", + "gaa": "Ga", + "gab": "Gabri", + "gac": "Mixed Great Andamanese", + "gad": "Gaddang", + "gae": "Guarequena", + "gaf": "Gende", + "gag": "Gagauz", + "gah": "Alekano", + "gai": "Borei", + "gaj": "Gadsup", + "gak": "Gamkonora", + "gal": "Galolen", + "gam": "Kandawo", + "gan": "Gan Chinese", + "gao": "Gants", + "gap": "Gal", + "gaq": "Gata'", + "gar": "Galeya", + "gas": "Adiwasi Garasia", + "gat": "Kenati", + "gau": "Mudhili Gadaba", + "gaw": "Nobonob", + "gax": "Borana-Arsi-Guji Oromo", + "gay": "Gayo", + "gaz": "West Central Oromo", + "gba": "Gbaya (Central African Republic)", + "gbb": "Kaytetye", + "gbd": "Karajarri", + "gbe": "Niksek", + "gbf": "Gaikundi", + "gbg": "Gbanziri", + "gbh": "Defi Gbe", + "gbi": "Galela", + "gbj": "Bodo Gadaba", + "gbk": "Gaddi", + "gbl": "Gamit", + "gbm": "Garhwali", + "gbn": "Mo'da", + "gbo": "Northern Grebo", + "gbp": "Gbaya-Bossangoa", + "gbq": "Gbaya-Bozoum", + "gbr": "Gbagyi", + "gbs": "Gbesi Gbe", + "gbu": "Gagadu", + "gbv": "Gbanu", + "gbw": "Gabi-Gabi", + "gbx": "Eastern Xwla Gbe", + "gby": "Gbari", + "gbz": "Zoroastrian Dari", + "gcc": "Mali", + "gcd": "Ganggalida", + "gce": "Galice", + "gcf": "Guadeloupean Creole French", + "gcl": "Grenadian Creole English", + "gcn": "Gaina", + "gcr": "Guianese Creole French", + "gct": "Colonia Tovar German", + "gd": "Scottish Gaelic; Gaelic", + "gda": "Gade Lohar", + "gdb": "Pottangi Ollar Gadaba", + "gdc": "Gugu Badhun", + "gdd": "Gedaged", + "gde": "Gude", + "gdf": "Guduf-Gava", + "gdg": "Ga'dang", + "gdh": "Gadjerawang; Gajirrabeng", + "gdi": "Gundi", + "gdj": "Gurdjar", + "gdk": "Gadang", + "gdl": "Dirasha", + "gdm": "Laal", + "gdn": "Umanakaina", + "gdo": "Ghodoberi", + "gdq": "Mehri", + "gdr": "Wipi", + "gds": "Ghandruk Sign Language", + "gdt": "Kungardutyi", + "gdu": "Gudu", + "gdx": "Godwari", + "gea": "Geruma", + "geb": "Kire", + "gec": "Gboloo Grebo", + "ged": "Gade", + "gef": "Gerai", + "geg": "Gengle", + "geh": "Hutterite German; Hutterisch", + "gei": "Gebe", + "gej": "Gen", + "gek": "Ywom", + "gel": "ut-Ma'in", + "gem": "Germanic languages", + "geq": "Geme", + "ges": "Geser-Gorom", + "gev": "Eviya", + "gew": "Gera", + "gex": "Garre", + "gey": "Enya", + "gez": "Geez", + "gfk": "Patpatar", + "gft": "Gafat", + "gga": "Gao", + "ggb": "Gbii", + "ggd": "Gugadj", + "gge": "Gurr-goni", + "ggg": "Gurgula", + "ggk": "Kungarakany", + "ggl": "Ganglau", + "ggt": "Gitua", + "ggu": "Gagu; Gban", + "ggw": "Gogodala", + "gha": "Ghadamès", + "ghc": "Hiberno-Scottish Gaelic", + "ghe": "Southern Ghale", + "ghh": "Northern Ghale", + "ghk": "Geko Karen", + "ghl": "Ghulfan", + "ghn": "Ghanongga", + "gho": "Ghomara", + "ghr": "Ghera", + "ghs": "Guhu-Samane", + "ght": "Kuke; Kutang Ghale", + "gia": "Kija", + "gib": "Gibanawa", + "gic": "Gail", + "gid": "Gidar", + "gie": "Gaɓogbo; Guébie", + "gig": "Goaria", + "gih": "Githabul", + "gii": "Girirra", + "gil": "Gilbertese", + "gim": "Gimi (Eastern Highlands)", + "gin": "Hinukh", + "gip": "Gimi (West New Britain)", + "giq": "Green Gelao", + "gir": "Red Gelao", + "gis": "North Giziga", + "git": "Gitxsan", + "giu": "Mulao", + "giw": "White Gelao", + "gix": "Gilima", + "giy": "Giyug", + "giz": "South Giziga", + "gjk": "Kachi Koli", + "gjm": "Gunditjmara", + "gjn": "Gonja", + "gjr": "Gurindji Kriol", + "gju": "Gujari", + "gka": "Guya", + "gkd": "Magɨ (Madang Province)", + "gke": "Ndai", + "gkn": "Gokana", + "gko": "Kok-Nar", + "gkp": "Guinea Kpelle", + "gku": "ǂUngkue", + "gl": "Galician", + "glb": "Belning", + "glc": "Bon Gula", + "gld": "Nanai", + "glh": "Northwest Pashai; Northwest Pashayi", + "glj": "Gula Iro", + "glk": "Gilaki", + "gll": "Garlali", + "glo": "Galambu", + "glr": "Glaro-Twabo", + "glu": "Gula (Chad)", + "glw": "Glavda", + "gly": "Gule", + "gma": "Gambera", + "gmb": "Gula'alaa", + "gmd": "Mághdì", + "gme": "East Germanic languages", + "gmg": "Magɨyi", + "gmh": "Middle High German (ca. 1050-1500)", + "gml": "Middle Low German", + "gmm": "Gbaya-Mbodomo", + "gmn": "Gimnime", + "gmq": "North Germanic languages", + "gmr": "Mirning; Mirniny", + "gmu": "Gumalu", + "gmv": "Gamo", + "gmw": "West Germanic languages", + "gmx": "Magoma", + "gmy": "Mycenaean Greek", + "gmz": "Mgbolizhia", + "gn": "Guarani", + "gna": "Kaansa", + "gnb": "Gangte", + "gnc": "Guanche", + "gnd": "Zulgo-Gemzek", + "gne": "Ganang", + "gng": "Ngangam", + "gnh": "Lere", + "gni": "Gooniyandi", + "gnj": "Ngen", + "gnk": "ǁGana", + "gnl": "Gangulu", + "gnm": "Ginuman", + "gnn": "Gumatj", + "gno": "Northern Gondi", + "gnq": "Gana", + "gnr": "Gureng Gureng", + "gnt": "Guntai", + "gnu": "Gnau", + "gnw": "Western Bolivian Guaraní", + "gnz": "Ganzi", + "goa": "Guro", + "gob": "Playero", + "goc": "Gorakor", + "god": "Godié", + "goe": "Gongduk", + "gof": "Gofa", + "gog": "Gogo", + "goh": "Old High German (ca. 750-1050)", + "goi": "Gobasi", + "goj": "Gowlan", + "gok": "Gowli", + "gol": "Gola", + "gom": "Goan Konkani", + "gon": "Gondi", + "goo": "Gone Dau", + "gop": "Yeretuar", + "goq": "Gorap", + "gor": "Gorontalo", + "gos": "Gronings", + "got": "Gothic", + "gou": "Gavar", + "gov": "Goo", + "gow": "Gorowa", + "gox": "Gobu", + "goy": "Goundo", + "goz": "Gozarkhani", + "gpa": "Gupa-Abawa", + "gpe": "Ghanaian Pidgin English", + "gpn": "Taiap", + "gqa": "Ga'anda", + "gqi": "Guiqiong", + "gqn": "Guana (Brazil)", + "gqr": "Gor", + "gqu": "Qau", + "gra": "Rajput Garasia", + "grb": "Grebo", + "grc": "Ancient Greek (to 1453)", + "grd": "Guruntum-Mbaaru", + "grg": "Madi", + "grh": "Gbiri-Niragu", + "gri": "Ghari", + "grj": "Southern Grebo", + "grk": "Greek languages", + "grm": "Kota Marudu Talantang", + "gro": "Groma", + "grq": "Gorovu", + "grr": "Taznatit", + "grs": "Gresi", + "grt": "Garo", + "gru": "Kistane", + "grv": "Central Grebo", + "grw": "Gweda", + "grx": "Guriaso", + "gry": "Barclayville Grebo", + "grz": "Guramalum", + "gse": "Ghanaian Sign Language", + "gsg": "German Sign Language", + "gsl": "Gusilay", + "gsm": "Guatemalan Sign Language", + "gsn": "Nema; Gusan", + "gso": "Southwest Gbaya", + "gsp": "Wasembo", + "gss": "Greek Sign Language", + "gsw": "Swiss German; Alemannic; Alsatian", + "gta": "Guató", + "gtu": "Aghu-Tharnggala", + "gu": "Gujarati", + "gua": "Shiki", + "gub": "Guajajára", + "guc": "Wayuu", + "gud": "Yocoboué Dida", + "gue": "Gurindji", + "guf": "Gupapuyngu", + "gug": "Paraguayan Guaraní", + "guh": "Guahibo", + "gui": "Eastern Bolivian Guaraní", + "guk": "Gumuz", + "gul": "Sea Island Creole English", + "gum": "Guambiano", + "gun": "Mbyá Guaraní", + "guo": "Guayabero", + "gup": "Gunwinggu", + "guq": "Aché", + "gur": "Farefare", + "gus": "Guinean Sign Language", + "gut": "Maléku Jaíka", + "guu": "Yanomamö", + "guw": "Gun", + "gux": "Gourmanchéma", + "guz": "Gusii; Ekegusii", + "gv": "Manx", + "gva": "Guana (Paraguay)", + "gvc": "Guanano", + "gve": "Duwet", + "gvf": "Golin", + "gvj": "Guajá", + "gvl": "Gulay", + "gvm": "Gurmana", + "gvn": "Kuku-Yalanji", + "gvo": "Gavião Do Jiparaná", + "gvp": "Pará Gavião", + "gvr": "Gurung", + "gvs": "Gumawana", + "gvy": "Guyani", + "gwa": "Mbato", + "gwb": "Gwa", + "gwc": "Gawri; Kalami", + "gwd": "Gawwada", + "gwe": "Gweno", + "gwf": "Gowro", + "gwg": "Moo", + "gwi": "Gwichʼin", + "gwj": "ǀGwi", + "gwm": "Awngthim", + "gwn": "Gwandara", + "gwr": "Gwere", + "gwt": "Gawar-Bati", + "gwu": "Guwamu", + "gww": "Kwini", + "gwx": "Gua", + "gxx": "Wè Southern", + "gya": "Northwest Gbaya", + "gyb": "Garus", + "gyd": "Kayardild", + "gye": "Gyem", + "gyf": "Gungabula", + "gyg": "Gbayi", + "gyi": "Gyele", + "gyl": "Gayil", + "gym": "Ngäbere", + "gyn": "Guyanese Creole English", + "gyo": "Gyalsumdo", + "gyr": "Guarayu", + "gyy": "Gunya", + "gyz": "Geji; Gyaazi", + "gza": "Ganza", + "gzi": "Gazi", + "gzn": "Gane", + "ha": "Hausa", + "haa": "Han", + "hab": "Hanoi Sign Language", + "hac": "Gurani", + "had": "Hatam", + "hae": "Eastern Oromo", + "haf": "Haiphong Sign Language", + "hag": "Hanga", + "hah": "Hahon", + "hai": "Haida", + "haj": "Hajong", + "hak": "Hakka Chinese", + "hal": "Halang", + "ham": "Hewa", + "han": "Hangaza", + "hao": "Hakö", + "hap": "Hupla", + "haq": "Ha", + "har": "Harari", + "has": "Haisla", + "hav": "Havu", + "haw": "Hawaiian", + "hax": "Southern Haida", + "hay": "Haya", + "haz": "Hazaragi", + "hba": "Hamba", + "hbb": "Huba", + "hbn": "Heiban", + "hbo": "Ancient Hebrew", + "hbu": "Habu", + "hca": "Andaman Creole Hindi", + "hch": "Huichol", + "hdn": "Northern Haida", + "hds": "Honduras Sign Language", + "hdy": "Hadiyya", + "he": "Hebrew", + "hea": "Northern Qiandong Miao", + "hed": "Herdé", + "heg": "Helong", + "heh": "Hehe", + "hei": "Heiltsuk", + "hem": "Hemba", + "hgm": "Haiǁom", + "hgw": "Haigwai", + "hhi": "Hoia Hoia", + "hhr": "Kerak", + "hhy": "Hoyahoya", + "hi": "Hindi", + "hia": "Lamang", + "hib": "Hibito", + "hid": "Hidatsa", + "hif": "Fiji Hindi", + "hig": "Kamwe", + "hih": "Pamosu", + "hii": "Hinduri", + "hij": "Hijuk", + "hik": "Seit-Kaitetu", + "hil": "Hiligaynon", + "him": "Himachali languages; Western Pahari languages", + "hio": "Tsoa", + "hir": "Himarimã", + "hit": "Hittite", + "hiw": "Hiw", + "hix": "Hixkaryána", + "hji": "Haji", + "hka": "Kahe", + "hke": "Hunde", + "hkh": "Khah; Poguli", + "hkk": "Hunjara-Kaina Ke", + "hkn": "Mel-Khaonh", + "hks": "Hong Kong Sign Language; Heung Kong Sau Yue", + "hla": "Halia", + "hlb": "Halbi", + "hld": "Halang Doan", + "hle": "Hlersu", + "hlt": "Matu Chin", + "hlu": "Hieroglyphic Luwian", + "hma": "Southern Mashan Hmong; Southern Mashan Miao", + "hmb": "Humburi Senni Songhay", + "hmc": "Central Huishui Hmong; Central Huishui Miao", + "hmd": "Large Flowery Miao; A-hmaos; Da-Hua Miao", + "hme": "Eastern Huishui Hmong; Eastern Huishui Miao", + "hmf": "Hmong Don", + "hmg": "Southwestern Guiyang Hmong", + "hmh": "Southwestern Huishui Hmong; Southwestern Huishui Miao", + "hmi": "Northern Huishui Hmong; Northern Huishui Miao", + "hmj": "Ge; Gejia", + "hmk": "Maek", + "hml": "Luopohe Hmong; Luopohe Miao", + "hmm": "Central Mashan Hmong; Central Mashan Miao", + "hmn": "Hmong; Mong", + "hmp": "Northern Mashan Hmong; Northern Mashan Miao", + "hmq": "Eastern Qiandong Miao", + "hmr": "Hmar", + "hms": "Southern Qiandong Miao", + "hmt": "Hamtai", + "hmu": "Hamap", + "hmv": "Hmong Dô", + "hmw": "Western Mashan Hmong; Western Mashan Miao", + "hmx": "Hmong-Mien languages", + "hmy": "Southern Guiyang Hmong; Southern Guiyang Miao", + "hmz": "Hmong Shua; Sinicized Miao", + "hna": "Mina (Cameroon)", + "hnd": "Southern Hindko", + "hne": "Chhattisgarhi", + "hng": "Hungu", + "hnh": "ǁAni", + "hni": "Hani", + "hnj": "Hmong Njua; Mong Leng; Mong Njua", + "hnn": "Hanunoo", + "hno": "Northern Hindko", + "hns": "Caribbean Hindustani", + "hnu": "Hung", + "ho": "Hiri Motu", + "hoa": "Hoava", + "hob": "Mari (Madang Province)", + "hoc": "Ho", + "hod": "Holma", + "hoe": "Horom", + "hoh": "Hobyót", + "hoi": "Holikachuk", + "hoj": "Hadothi; Haroti", + "hok": "Hokan languages", + "hol": "Holu", + "hom": "Homa", + "hoo": "Holoholo", + "hop": "Hopi", + "hor": "Horo", + "hos": "Ho Chi Minh City Sign Language", + "hot": "Hote; Malê", + "hov": "Hovongan", + "how": "Honi", + "hoy": "Holiya", + "hoz": "Hozo", + "hpo": "Hpon", + "hps": "Hawai'i Sign Language (HSL); Hawai'i Pidgin Sign Language", + "hr": "Croatian", + "hra": "Hrangkhol", + "hrc": "Niwer Mil", + "hre": "Hre", + "hrk": "Haruku", + "hrm": "Horned Miao", + "hro": "Haroi", + "hrp": "Nhirrpi", + "hrt": "Hértevin", + "hru": "Hruso", + "hrw": "Warwar Feni", + "hrx": "Hunsrik", + "hrz": "Harzani", + "hsb": "Upper Sorbian", + "hsh": "Hungarian Sign Language", + "hsl": "Hausa Sign Language", + "hsn": "Xiang Chinese", + "hss": "Harsusi", + "ht": "Haitian; Haitian Creole", + "hti": "Hoti", + "hto": "Minica Huitoto", + "hts": "Hadza", + "htu": "Hitu", + "htx": "Middle Hittite", + "hu": "Hungarian", + "hub": "Huambisa", + "huc": "ǂHua; ǂʼAmkhoe", + "hud": "Huaulu", + "hue": "San Francisco Del Mar Huave", + "huf": "Humene", + "hug": "Huachipaeri", + "huh": "Huilliche", + "hui": "Huli", + "huj": "Northern Guiyang Hmong; Northern Guiyang Miao", + "huk": "Hulung", + "hul": "Hula", + "hum": "Hungana", + "huo": "Hu", + "hup": "Hupa", + "huq": "Tsat", + "hur": "Halkomelem", + "hus": "Huastec", + "hut": "Humla", + "huu": "Murui Huitoto", + "huv": "San Mateo Del Mar Huave", + "huw": "Hukumina", + "hux": "Nüpode Huitoto", + "huy": "Hulaulá", + "huz": "Hunzib", + "hvc": "Haitian Vodoun Culture Language", + "hve": "San Dionisio Del Mar Huave", + "hvk": "Haveke", + "hvn": "Sabu", + "hvv": "Santa María Del Mar Huave", + "hwa": "Wané", + "hwc": "Hawai'i Creole English; Hawai'i Pidgin", + "hwo": "Hwana", + "hy": "Armenian", + "hya": "Hya", + "hyw": "Western Armenian", + "hyx": "Armenian (family)", + "hz": "Herero", + "ia": "Interlingua (International Auxiliary Language Association)", + "iai": "Iaai", + "ian": "Iatmul", + "iar": "Purari", + "iba": "Iban", + "ibb": "Ibibio", + "ibd": "Iwaidja", + "ibe": "Akpes", + "ibg": "Ibanag", + "ibh": "Bih", + "ibl": "Ibaloi", + "ibm": "Agoi", + "ibn": "Ibino", + "ibr": "Ibuoro", + "ibu": "Ibu", + "iby": "Ibani", + "ica": "Ede Ica", + "ich": "Etkywan", + "icl": "Icelandic Sign Language", + "icr": "Islander Creole English", + "id": "Indonesian", + "ida": "Idakho-Isukha-Tiriki; Luidakho-Luisukha-Lutirichi", + "idb": "Indo-Portuguese", + "idc": "Idon; Ajiya", + "idd": "Ede Idaca", + "ide": "Idere", + "idi": "Idi", + "idr": "Indri", + "ids": "Idesa", + "idt": "Idaté", + "idu": "Idoma", + "ie": "Interlingue; Occidental", + "ifa": "Amganad Ifugao", + "ifb": "Batad Ifugao; Ayangan Ifugao", + "ife": "Ifè", + "iff": "Ifo", + "ifk": "Tuwali Ifugao", + "ifm": "Teke-Fuumu", + "ifu": "Mayoyao Ifugao", + "ify": "Keley-I Kallahan", + "ig": "Igbo", + "igb": "Ebira", + "ige": "Igede", + "igg": "Igana", + "igl": "Igala", + "igm": "Kanggape", + "ign": "Ignaciano", + "igo": "Isebe", + "igs": "Interglossa", + "igw": "Igwe", + "ihb": "Iha Based Pidgin", + "ihi": "Ihievbe", + "ihp": "Iha", + "ihw": "Bidhawal", + "ii": "Sichuan Yi; Nuosu", + "iin": "Thiin", + "iir": "Indo-Iranian languages", + "ijc": "Izon", + "ije": "Biseni", + "ijj": "Ede Ije", + "ijn": "Kalabari", + "ijo": "Ijo languages", + "ijs": "Southeast Ijo", + "ik": "Inupiaq", + "ike": "Eastern Canadian Inuktitut", + "iki": "Iko", + "ikk": "Ika", + "ikl": "Ikulu", + "iko": "Olulumo-Ikom", + "ikp": "Ikpeshi", + "ikr": "Ikaranggal", + "iks": "Inuit Sign Language", + "ikt": "Inuinnaqtun; Western Canadian Inuktitut", + "ikv": "Iku-Gora-Ankwa", + "ikw": "Ikwere", + "ikx": "Ik", + "ikz": "Ikizu", + "ila": "Ile Ape", + "ilb": "Ila", + "ilg": "Garig-Ilgar", + "ili": "Ili Turki", + "ilk": "Ilongot", + "ilm": "Iranun (Malaysia)", + "ilo": "Iloko", + "ilp": "Iranun (Philippines)", + "ils": "International Sign", + "ilu": "Ili'uun", + "ilv": "Ilue", + "ima": "Mala Malasar", + "imi": "Anamgura", + "iml": "Miluk", + "imn": "Imonda", + "imo": "Imbongu", + "imr": "Imroing", + "ims": "Marsian", + "imt": "Imotong", + "imy": "Milyan", + "inb": "Inga", + "inc": "Indic languages", + "ine": "Indo-European languages", + "ing": "Degexit'an", + "inh": "Ingush", + "inj": "Jungle Inga", + "inl": "Indonesian Sign Language", + "inm": "Minaean", + "inn": "Isinai", + "ino": "Inoke-Yate", + "inp": "Iñapari", + "ins": "Indian Sign Language", + "int": "Intha", + "inz": "Ineseño", + "io": "Ido", + "ior": "Inor", + "iou": "Tuma-Irumu", + "iow": "Iowa-Oto", + "ipi": "Ipili", + "ipo": "Ipiko", + "iqu": "Iquito", + "iqw": "Ikwo", + "ira": "Iranian languages", + "ire": "Iresim", + "irh": "Irarutu", + "iri": "Rigwe; Irigwe", + "irk": "Iraqw", + "irn": "Irántxe", + "iro": "Iroquoian languages", + "irr": "Ir", + "iru": "Irula", + "irx": "Kamberau", + "iry": "Iraya", + "is": "Icelandic", + "isa": "Isabi", + "isc": "Isconahua", + "isd": "Isnag", + "ise": "Italian Sign Language", + "isg": "Irish Sign Language", + "ish": "Esan", + "isi": "Nkem-Nkum", + "isk": "Ishkashimi", + "ism": "Masimasi", + "isn": "Isanzu", + "iso": "Isoko", + "isr": "Israeli Sign Language", + "ist": "Istriot", + "isu": "Isu (Menchum Division)", + "it": "Italian", + "itb": "Binongan Itneg", + "itc": "Italic languages", + "itd": "Southern Tidung", + "ite": "Itene", + "iti": "Inlaod Itneg", + "itk": "Judeo-Italian", + "itl": "Itelmen", + "itm": "Itu Mbon Uzo", + "ito": "Itonama", + "itr": "Iteri", + "its": "Isekiri", + "itt": "Maeng Itneg", + "itv": "Itawit", + "itw": "Ito", + "itx": "Itik", + "ity": "Moyadan Itneg", + "itz": "Itzá", + "iu": "Inuktitut", + "ium": "Iu Mien", + "ivb": "Ibatan", + "ivv": "Ivatan", + "iwk": "I-Wak", + "iwm": "Iwam", + "iwo": "Iwur", + "iws": "Sepik Iwam", + "ixc": "Ixcatec", + "ixl": "Ixil", + "iya": "Iyayu", + "iyo": "Mesaka", + "iyx": "Yaka (Congo)", + "izh": "Ingrian", + "izr": "Izere", + "izz": "Izii", + "ja": "Japanese", + "jaa": "Jamamadí", + "jab": "Hyam", + "jac": "Popti'; Jakalteko", + "jad": "Jahanka", + "jae": "Yabem", + "jaf": "Jara", + "jah": "Jah Hut", + "jaj": "Zazao", + "jak": "Jakun", + "jal": "Yalahatan", + "jam": "Jamaican Creole English", + "jan": "Jandai", + "jao": "Yanyuwa", + "jaq": "Yaqay", + "jas": "New Caledonian Javanese", + "jat": "Jakati", + "jau": "Yaur", + "jax": "Jambi Malay", + "jay": "Yan-nhangu; Nhangu", + "jaz": "Jawe", + "jbe": "Judeo-Berber", + "jbi": "Badjiri", + "jbj": "Arandai", + "jbk": "Barikewa", + "jbm": "Bijim", + "jbn": "Nafusi", + "jbo": "Lojban", + "jbr": "Jofotek-Bromnya", + "jbt": "Jabutí", + "jbu": "Jukun Takum", + "jbw": "Yawijibaya", + "jcs": "Jamaican Country Sign Language", + "jct": "Krymchak", + "jda": "Jad", + "jdg": "Jadgali", + "jdt": "Judeo-Tat", + "jeb": "Jebero", + "jee": "Jerung", + "jeh": "Jeh", + "jei": "Yei", + "jek": "Jeri Kuo", + "jel": "Yelmek", + "jen": "Dza", + "jer": "Jere", + "jet": "Manem", + "jeu": "Jonkor Bourmataguil", + "jgb": "Ngbee", + "jge": "Judeo-Georgian", + "jgk": "Gwak", + "jgo": "Ngomba", + "jhi": "Jehai", + "jhs": "Jhankot Sign Language", + "jia": "Jina", + "jib": "Jibu", + "jic": "Tol", + "jid": "Bu (Kaduna State)", + "jie": "Jilbe", + "jig": "Jingulu; Djingili", + "jih": "sTodsde; Shangzhai", + "jii": "Jiiddu", + "jil": "Jilim", + "jim": "Jimi (Cameroon)", + "jio": "Jiamao", + "jiq": "Guanyinqiao; Lavrung", + "jit": "Jita", + "jiu": "Youle Jinuo", + "jiv": "Shuar", + "jiy": "Buyuan Jinuo", + "jje": "Jejueo", + "jjr": "Bankal", + "jka": "Kaera", + "jkm": "Mobwa Karen", + "jko": "Kubo", + "jkp": "Paku Karen", + "jkr": "Koro (India)", + "jks": "Amami Koniya Sign Language", + "jku": "Labir", + "jle": "Ngile", + "jls": "Jamaican Sign Language", + "jma": "Dima", + "jmb": "Zumbun", + "jmc": "Machame", + "jmd": "Yamdena", + "jmi": "Jimi (Nigeria)", + "jml": "Jumli", + "jmn": "Makuri Naga", + "jmr": "Kamara", + "jms": "Mashi (Nigeria)", + "jmw": "Mouwase", + "jmx": "Western Juxtlahuaca Mixtec", + "jna": "Jangshung", + "jnd": "Jandavra", + "jng": "Yangman", + "jni": "Janji", + "jnj": "Yemsa", + "jnl": "Rawat", + "jns": "Jaunsari", + "job": "Joba", + "jod": "Wojenaka", + "jog": "Jogi", + "jor": "Jorá", + "jos": "Jordanian Sign Language", + "jow": "Jowulu", + "jpa": "Jewish Palestinian Aramaic", + "jpr": "Judeo-Persian", + "jpx": "Japanese (family)", + "jqr": "Jaqaru", + "jra": "Jarai", + "jrb": "Judeo-Arabic", + "jrr": "Jiru", + "jrt": "Jakattoe", + "jru": "Japrería", + "jsl": "Japanese Sign Language", + "jua": "Júma", + "jub": "Wannu", + "juc": "Jurchen", + "jud": "Worodougou", + "juh": "Hõne", + "jui": "Ngadjuri", + "juk": "Wapan", + "jul": "Jirel", + "jum": "Jumjum", + "jun": "Juang", + "juo": "Jiba", + "jup": "Hupdë", + "jur": "Jurúna", + "jus": "Jumla Sign Language", + "jut": "Jutish", + "juu": "Ju", + "juw": "Wãpha", + "juy": "Juray", + "jv": "Javanese", + "jvd": "Javindo", + "jvn": "Caribbean Javanese", + "jwi": "Jwira-Pepesa", + "jya": "Jiarong", + "jye": "Judeo-Yemeni Arabic", + "jyy": "Jaya", + "ka": "Georgian", + "kaa": "Kara-Kalpak; Karakalpak", + "kab": "Kabyle", + "kac": "Kachin; Jingpho", + "kad": "Adara", + "kae": "Ketangalan", + "kaf": "Katso", + "kag": "Kajaman", + "kah": "Kara (Central African Republic)", + "kai": "Karekare", + "kaj": "Jju", + "kak": "Kalanguya; Kayapa Kallahan", + "kam": "Kamba (Kenya)", + "kao": "Xaasongaxango", + "kap": "Bezhta", + "kaq": "Capanahua", + "kar": "Karen languages", + "kav": "Katukína", + "kaw": "Kawi", + "kax": "Kao", + "kay": "Kamayurá", + "kba": "Kalarko", + "kbb": "Kaxuiâna", + "kbc": "Kadiwéu", + "kbd": "Kabardian", + "kbe": "Kanju", + "kbg": "Khamba", + "kbh": "Camsá", + "kbi": "Kaptiau", + "kbj": "Kari", + "kbk": "Grass Koiari", + "kbl": "Kanembu", + "kbm": "Iwal", + "kbn": "Kare (Central African Republic)", + "kbo": "Keliko", + "kbp": "Kabiyè", + "kbq": "Kamano", + "kbr": "Kafa", + "kbs": "Kande", + "kbt": "Abadi", + "kbu": "Kabutra", + "kbv": "Dera (Indonesia)", + "kbw": "Kaiep", + "kbx": "Ap Ma", + "kby": "Manga Kanuri", + "kbz": "Duhwa", + "kca": "Khanty", + "kcb": "Kawacha", + "kcc": "Lubila", + "kcd": "Ngkâlmpw Kanum", + "kce": "Kaivi", + "kcf": "Ukaan", + "kcg": "Tyap", + "kch": "Vono", + "kci": "Kamantan", + "kcj": "Kobiana", + "kck": "Kalanga", + "kcl": "Kela (Papua New Guinea); Kala", + "kcm": "Gula (Central African Republic)", + "kcn": "Nubi", + "kco": "Kinalakna", + "kcp": "Kanga", + "kcq": "Kamo", + "kcr": "Katla", + "kcs": "Koenoem", + "kct": "Kaian", + "kcu": "Kami (Tanzania)", + "kcv": "Kete", + "kcw": "Kabwari", + "kcx": "Kachama-Ganjule", + "kcy": "Korandje", + "kcz": "Konongo", + "kda": "Worimi", + "kdc": "Kutu", + "kdd": "Yankunytjatjara", + "kde": "Makonde", + "kdf": "Mamusi", + "kdg": "Seba", + "kdh": "Tem", + "kdi": "Kumam", + "kdj": "Karamojong", + "kdk": "Numèè; Kwényi", + "kdl": "Tsikimba", + "kdm": "Kagoma", + "kdn": "Kunda", + "kdo": "Kordofanian languages", + "kdp": "Kaningdon-Nindem", + "kdq": "Koch", + "kdr": "Karaim", + "kdt": "Kuy", + "kdu": "Kadaru", + "kdw": "Koneraw", + "kdx": "Kam", + "kdy": "Keder; Keijar", + "kdz": "Kwaja", + "kea": "Kabuverdianu", + "keb": "Kélé", + "kec": "Keiga", + "ked": "Kerewe", + "kee": "Eastern Keres", + "kef": "Kpessi", + "keg": "Tese", + "keh": "Keak", + "kei": "Kei", + "kej": "Kadar", + "kek": "Kekchí", + "kel": "Kela (Democratic Republic of Congo)", + "kem": "Kemak", + "ken": "Kenyang", + "keo": "Kakwa", + "kep": "Kaikadi", + "keq": "Kamar", + "ker": "Kera", + "kes": "Kugbo", + "ket": "Ket", + "keu": "Akebu", + "kev": "Kanikkaran", + "kew": "West Kewa", + "kex": "Kukna", + "key": "Kupia", + "kez": "Kukele", + "kfa": "Kodava", + "kfb": "Northwestern Kolami", + "kfc": "Konda-Dora", + "kfd": "Korra Koraga", + "kfe": "Kota (India)", + "kff": "Koya", + "kfg": "Kudiya", + "kfh": "Kurichiya", + "kfi": "Kannada Kurumba", + "kfj": "Kemiehua", + "kfk": "Kinnauri", + "kfl": "Kung", + "kfm": "Khunsari", + "kfn": "Kuk", + "kfo": "Koro (Côte d'Ivoire)", + "kfp": "Korwa", + "kfq": "Korku", + "kfr": "Kachhi; Kutchi", + "kfs": "Bilaspuri", + "kft": "Kanjari", + "kfu": "Katkari", + "kfv": "Kurmukar", + "kfw": "Kharam Naga", + "kfx": "Kullu Pahari", + "kfy": "Kumaoni", + "kfz": "Koromfé", + "kg": "Kongo", + "kga": "Koyaga", + "kgb": "Kawe", + "kge": "Komering", + "kgf": "Kube", + "kgg": "Kusunda", + "kgi": "Selangor Sign Language", + "kgj": "Gamale Kham", + "kgk": "Kaiwá", + "kgl": "Kunggari", + "kgm": "Karipúna", + "kgn": "Karingani", + "kgo": "Krongo", + "kgp": "Kaingang", + "kgq": "Kamoro", + "kgr": "Abun", + "kgs": "Kumbainggar", + "kgt": "Somyev", + "kgu": "Kobol", + "kgv": "Karas", + "kgw": "Karon Dori", + "kgx": "Kamaru", + "kgy": "Kyerung", + "kha": "Khasi", + "khb": "Lü", + "khc": "Tukang Besi North", + "khd": "Bädi Kanum", + "khe": "Korowai", + "khf": "Khuen", + "khg": "Khams Tibetan", + "khh": "Kehu", + "khi": "Khoisan languages", + "khj": "Kuturmi", + "khk": "Halh Mongolian", + "khl": "Lusi", + "khn": "Khandesi", + "kho": "Khotanese; Sakan", + "khp": "Kapori; Kapauri", + "khq": "Koyra Chiini Songhay", + "khr": "Kharia", + "khs": "Kasua", + "kht": "Khamti", + "khu": "Nkhumbi", + "khv": "Khvarshi", + "khw": "Khowar", + "khx": "Kanu", + "khy": "Kele (Democratic Republic of Congo)", + "khz": "Keapara", + "ki": "Kikuyu; Gikuyu", + "kia": "Kim", + "kib": "Koalib", + "kic": "Kickapoo", + "kid": "Koshin", + "kie": "Kibet", + "kif": "Eastern Parbate Kham", + "kig": "Kimaama; Kimaghima", + "kih": "Kilmeri", + "kii": "Kitsai", + "kij": "Kilivila", + "kil": "Kariya", + "kim": "Karagas", + "kio": "Kiowa", + "kip": "Sheshi Kham", + "kiq": "Kosadle; Kosare", + "kis": "Kis", + "kit": "Agob", + "kiu": "Kirmanjki (individual language)", + "kiv": "Kimbu", + "kiw": "Northeast Kiwai", + "kix": "Khiamniungan Naga", + "kiy": "Kirikiri", + "kiz": "Kisi", + "kj": "Kuanyama; Kwanyama", + "kja": "Mlap", + "kjb": "Q'anjob'al; Kanjobal", + "kjc": "Coastal Konjo", + "kjd": "Southern Kiwai", + "kje": "Kisar", + "kjg": "Khmu", + "kjh": "Khakas", + "kji": "Zabana", + "kjj": "Khinalugh", + "kjk": "Highland Konjo", + "kjl": "Western Parbate Kham", + "kjm": "Kháng", + "kjn": "Kunjen", + "kjo": "Harijan Kinnauri", + "kjp": "Pwo Eastern Karen", + "kjq": "Western Keres", + "kjr": "Kurudu", + "kjs": "East Kewa", + "kjt": "Phrae Pwo Karen", + "kju": "Kashaya", + "kjv": "Kaikavian Literary Language", + "kjx": "Ramopa", + "kjy": "Erave", + "kjz": "Bumthangkha", + "kk": "Kazakh", + "kka": "Kakanda", + "kkb": "Kwerisa", + "kkc": "Odoodee", + "kkd": "Kinuku", + "kke": "Kakabe", + "kkf": "Kalaktang Monpa", + "kkg": "Mabaka Valley Kalinga", + "kkh": "Khün", + "kki": "Kagulu", + "kkj": "Kako", + "kkk": "Kokota", + "kkl": "Kosarek Yale", + "kkm": "Kiong", + "kkn": "Kon Keu", + "kko": "Karko", + "kkp": "Gugubera; Koko-Bera", + "kkq": "Kaeku", + "kkr": "Kir-Balar", + "kks": "Giiwo", + "kkt": "Koi", + "kku": "Tumi", + "kkv": "Kangean", + "kkw": "Teke-Kukuya", + "kkx": "Kohin", + "kky": "Guugu Yimidhirr; Guguyimidjir", + "kkz": "Kaska", + "kl": "Kalaallisut; Greenlandic", + "kla": "Klamath-Modoc", + "klb": "Kiliwa", + "klc": "Kolbila", + "kld": "Gamilaraay", + "kle": "Kulung (Nepal)", + "klf": "Kendeje", + "klg": "Tagakaulo", + "klh": "Weliki", + "kli": "Kalumpang", + "klj": "Khalaj", + "klk": "Kono (Nigeria)", + "kll": "Kagan Kalagan", + "klm": "Migum", + "kln": "Kalenjin", + "klo": "Kapya", + "klp": "Kamasa", + "klq": "Rumu", + "klr": "Khaling", + "kls": "Kalasha", + "klt": "Nukna", + "klu": "Klao", + "klv": "Maskelynes", + "klw": "Tado; Lindu", + "klx": "Koluwawa", + "kly": "Kalao", + "klz": "Kabola", + "km": "Khmer; Central Khmer", + "kma": "Konni", + "kmb": "Kimbundu", + "kmc": "Southern Dong", + "kmd": "Majukayang Kalinga", + "kme": "Bakole", + "kmf": "Kare (Papua New Guinea)", + "kmg": "Kâte", + "kmh": "Kalam", + "kmi": "Kami (Nigeria)", + "kmj": "Kumarbhag Paharia", + "kmk": "Limos Kalinga", + "kml": "Tanudan Kalinga", + "kmm": "Kom (India)", + "kmn": "Awtuw", + "kmo": "Kwoma", + "kmp": "Gimme", + "kmq": "Kwama", + "kmr": "Northern Kurdish", + "kms": "Kamasau", + "kmt": "Kemtuik", + "kmu": "Kanite", + "kmv": "Karipúna Creole French", + "kmw": "Komo (Democratic Republic of Congo)", + "kmx": "Waboda", + "kmy": "Koma", + "kmz": "Khorasani Turkish", + "kn": "Kannada", + "kna": "Dera (Nigeria)", + "knb": "Lubuagan Kalinga", + "knc": "Central Kanuri", + "knd": "Konda", + "kne": "Kankanaey", + "knf": "Mankanya", + "kng": "Koongo", + "kni": "Kanufi", + "knj": "Western Kanjobal", + "knk": "Kuranko", + "knl": "Keninjal", + "knm": "Kanamarí", + "knn": "Konkani (individual language)", + "kno": "Kono (Sierra Leone)", + "knp": "Kwanja", + "knq": "Kintaq", + "knr": "Kaningra", + "kns": "Kensiu", + "knt": "Panoan Katukína", + "knu": "Kono (Guinea)", + "knv": "Tabo", + "knw": "Kung-Ekoka", + "knx": "Kendayan; Salako", + "kny": "Kanyok", + "knz": "Kalamsé", + "ko": "Korean", + "koa": "Konomala", + "koc": "Kpati", + "kod": "Kodi", + "koe": "Kacipo-Bale Suri", + "kof": "Kubi", + "kog": "Cogui; Kogi", + "koh": "Koyo", + "koi": "Komi-Permyak", + "kok": "Konkani (macrolanguage)", + "kol": "Kol (Papua New Guinea)", + "koo": "Konzo", + "kop": "Waube", + "koq": "Kota (Gabon)", + "kos": "Kosraean", + "kot": "Lagwan", + "kou": "Koke", + "kov": "Kudu-Camo", + "kow": "Kugama", + "koy": "Koyukon", + "koz": "Korak", + "kpa": "Kutto", + "kpb": "Mullu Kurumba", + "kpc": "Curripaco", + "kpd": "Koba", + "kpe": "Kpelle", + "kpf": "Komba", + "kpg": "Kapingamarangi", + "kph": "Kplang", + "kpi": "Kofei", + "kpj": "Karajá", + "kpk": "Kpan", + "kpl": "Kpala", + "kpm": "Koho", + "kpn": "Kepkiriwát", + "kpo": "Ikposo", + "kpq": "Korupun-Sela", + "kpr": "Korafe-Yegha", + "kps": "Tehit", + "kpt": "Karata", + "kpu": "Kafoa", + "kpv": "Komi-Zyrian", + "kpw": "Kobon", + "kpx": "Mountain Koiali", + "kpy": "Koryak", + "kpz": "Kupsabiny", + "kqa": "Mum", + "kqb": "Kovai", + "kqc": "Doromu-Koki", + "kqd": "Koy Sanjaq Surat", + "kqe": "Kalagan", + "kqf": "Kakabai", + "kqg": "Khe", + "kqh": "Kisankasa", + "kqi": "Koitabu", + "kqj": "Koromira", + "kqk": "Kotafon Gbe", + "kql": "Kyenele", + "kqm": "Khisa", + "kqn": "Kaonde", + "kqo": "Eastern Krahn", + "kqp": "Kimré", + "kqq": "Krenak", + "kqr": "Kimaragang", + "kqs": "Northern Kissi", + "kqt": "Klias River Kadazan", + "kqu": "Seroa", + "kqv": "Okolod", + "kqw": "Kandas", + "kqx": "Mser", + "kqy": "Koorete", + "kqz": "Korana", + "kr": "Kanuri", + "kra": "Kumhali", + "krb": "Karkin", + "krc": "Karachay-Balkar", + "krd": "Kairui-Midiki", + "kre": "Panará", + "krf": "Koro (Vanuatu)", + "krh": "Kurama", + "kri": "Krio", + "krj": "Kinaray-A", + "krk": "Kerek", + "krl": "Karelian", + "krn": "Sapo", + "kro": "Kru languages", + "krp": "Korop", + "krr": "Krung", + "krs": "Gbaya (Sudan)", + "krt": "Tumari Kanuri", + "kru": "Kurukh", + "krv": "Kavet", + "krw": "Western Krahn", + "krx": "Karon", + "kry": "Kryts", + "krz": "Sota Kanum", + "ks": "Kashmiri", + "ksa": "Shuwa-Zamani", + "ksb": "Shambala", + "ksc": "Southern Kalinga", + "ksd": "Kuanua", + "kse": "Kuni", + "ksf": "Bafia", + "ksg": "Kusaghe", + "ksh": "Kölsch", + "ksi": "Krisa; I'saka", + "ksj": "Uare", + "ksk": "Kansa", + "ksl": "Kumalu", + "ksm": "Kumba", + "ksn": "Kasiguranin", + "kso": "Kofa", + "ksp": "Kaba", + "ksq": "Kwaami", + "ksr": "Borong", + "kss": "Southern Kisi", + "kst": "Winyé", + "ksu": "Khamyang", + "ksv": "Kusu", + "ksw": "S'gaw Karen", + "ksx": "Kedang", + "ksy": "Kharia Thar", + "ksz": "Kodaku", + "kta": "Katua", + "ktb": "Kambaata", + "ktc": "Kholok", + "ktd": "Kokata; Kukatha", + "kte": "Nubri", + "ktf": "Kwami", + "ktg": "Kalkutung", + "kth": "Karanga", + "kti": "North Muyu", + "ktj": "Plapo Krumen", + "ktk": "Kaniet", + "ktl": "Koroshi", + "ktm": "Kurti", + "ktn": "Karitiâna", + "kto": "Kuot", + "ktp": "Kaduo", + "ktq": "Katabaga", + "kts": "South Muyu", + "ktt": "Ketum", + "ktu": "Kituba (Democratic Republic of Congo)", + "ktv": "Eastern Katu", + "ktw": "Kato", + "ktx": "Kaxararí", + "kty": "Kango (Bas-Uélé District)", + "ktz": "Juǀʼhoan; Juǀʼhoansi", + "ku": "Kurdish", + "kub": "Kutep", + "kuc": "Kwinsu", + "kud": "'Auhelawa", + "kue": "Kuman (Papua New Guinea)", + "kuf": "Western Katu", + "kug": "Kupa", + "kuh": "Kushi", + "kui": "Kuikúro-Kalapálo; Kalapalo", + "kuj": "Kuria", + "kuk": "Kepo'", + "kul": "Kulere", + "kum": "Kumyk", + "kun": "Kunama", + "kuo": "Kumukio", + "kup": "Kunimaipa", + "kuq": "Karipuna", + "kus": "Kusaal", + "kut": "Kutenai", + "kuu": "Upper Kuskokwim", + "kuv": "Kur", + "kuw": "Kpagua", + "kux": "Kukatja", + "kuy": "Kuuku-Ya'u", + "kuz": "Kunza", + "kv": "Komi", + "kva": "Bagvalal", + "kvb": "Kubu", + "kvc": "Kove", + "kvd": "Kui (Indonesia)", + "kve": "Kalabakan", + "kvf": "Kabalai", + "kvg": "Kuni-Boazi", + "kvh": "Komodo", + "kvi": "Kwang", + "kvj": "Psikye", + "kvk": "Korean Sign Language", + "kvl": "Kayaw", + "kvm": "Kendem", + "kvn": "Border Kuna", + "kvo": "Dobel", + "kvp": "Kompane", + "kvq": "Geba Karen", + "kvr": "Kerinci", + "kvt": "Lahta Karen; Lahta", + "kvu": "Yinbaw Karen", + "kvv": "Kola", + "kvw": "Wersing", + "kvx": "Parkari Koli", + "kvy": "Yintale Karen; Yintale", + "kvz": "Tsakwambo; Tsaukambo", + "kw": "Cornish", + "kwa": "Dâw", + "kwb": "Kwa", + "kwc": "Likwala", + "kwd": "Kwaio", + "kwe": "Kwerba", + "kwf": "Kwara'ae", + "kwg": "Sara Kaba Deme", + "kwh": "Kowiai", + "kwi": "Awa-Cuaiquer", + "kwj": "Kwanga", + "kwk": "Kwakiutl", + "kwl": "Kofyar", + "kwm": "Kwambi", + "kwn": "Kwangali", + "kwo": "Kwomtari", + "kwp": "Kodia", + "kwr": "Kwer", + "kws": "Kwese", + "kwt": "Kwesten", + "kwu": "Kwakum", + "kwv": "Sara Kaba Náà", + "kww": "Kwinti", + "kwx": "Khirwar", + "kwy": "San Salvador Kongo", + "kwz": "Kwadi", + "kxa": "Kairiru", + "kxb": "Krobu", + "kxc": "Konso; Khonso", + "kxd": "Brunei", + "kxf": "Manumanaw Karen; Manumanaw", + "kxh": "Karo (Ethiopia)", + "kxi": "Keningau Murut", + "kxj": "Kulfa", + "kxk": "Zayein Karen", + "kxm": "Northern Khmer", + "kxn": "Kanowit-Tanjong Melanau", + "kxo": "Kanoé", + "kxp": "Wadiyara Koli", + "kxq": "Smärky Kanum", + "kxr": "Koro (Papua New Guinea)", + "kxs": "Kangjia", + "kxt": "Koiwat", + "kxv": "Kuvi", + "kxw": "Konai", + "kxx": "Likuba", + "kxy": "Kayong", + "kxz": "Kerewo", + "ky": "Kirghiz; Kyrgyz", + "kya": "Kwaya", + "kyb": "Butbut Kalinga", + "kyc": "Kyaka", + "kyd": "Karey", + "kye": "Krache", + "kyf": "Kouya", + "kyg": "Keyagana", + "kyh": "Karok", + "kyi": "Kiput", + "kyj": "Karao", + "kyk": "Kamayo", + "kyl": "Kalapuya", + "kym": "Kpatili", + "kyn": "Northern Binukidnon", + "kyo": "Kelon", + "kyp": "Kang", + "kyq": "Kenga", + "kyr": "Kuruáya", + "kys": "Baram Kayan", + "kyt": "Kayagar", + "kyu": "Western Kayah", + "kyv": "Kayort", + "kyw": "Kudmali", + "kyx": "Rapoisi", + "kyy": "Kambaira", + "kyz": "Kayabí", + "kza": "Western Karaboro", + "kzb": "Kaibobo", + "kzc": "Bondoukou Kulango", + "kzd": "Kadai", + "kze": "Kosena", + "kzf": "Da'a Kaili", + "kzg": "Kikai", + "kzi": "Kelabit", + "kzk": "Kazukuru", + "kzl": "Kayeli", + "kzm": "Kais", + "kzn": "Kokola", + "kzo": "Kaningi", + "kzp": "Kaidipang", + "kzq": "Kaike", + "kzr": "Karang", + "kzs": "Sugut Dusun", + "kzu": "Kayupulau", + "kzv": "Komyandaret", + "kzw": "Karirí-Xocó", + "kzx": "Kamarian", + "kzy": "Kango (Tshopo District)", + "kzz": "Kalabra", + "la": "Latin", + "laa": "Southern Subanen", + "lab": "Linear A", + "lac": "Lacandon", + "lad": "Ladino", + "lae": "Pattani", + "laf": "Lafofa", + "lag": "Langi", + "lah": "Lahnda", + "lai": "Lambya", + "laj": "Lango (Uganda)", + "lal": "Lalia", + "lam": "Lamba", + "lan": "Laru", + "lap": "Laka (Chad)", + "laq": "Qabiao", + "lar": "Larteh", + "las": "Lama (Togo)", + "lau": "Laba", + "law": "Lauje", + "lax": "Tiwa", + "lay": "Lama Bai", + "laz": "Aribwatsa", + "lb": "Luxembourgish; Letzeburgesch", + "lbb": "Label", + "lbc": "Lakkia", + "lbe": "Lak", + "lbf": "Tinani", + "lbg": "Laopang", + "lbi": "La'bi", + "lbj": "Ladakhi", + "lbk": "Central Bontok", + "lbl": "Libon Bikol", + "lbm": "Lodhi", + "lbn": "Rmeet", + "lbo": "Laven", + "lbq": "Wampar", + "lbr": "Lohorung", + "lbs": "Libyan Sign Language", + "lbt": "Lachi", + "lbu": "Labu", + "lbv": "Lavatbura-Lamusong", + "lbw": "Tolaki", + "lbx": "Lawangan", + "lby": "Lamalama; Lamu-Lamu", + "lbz": "Lardil", + "lcc": "Legenyem", + "lcd": "Lola", + "lce": "Loncong; Sekak", + "lcf": "Lubu", + "lch": "Luchazi", + "lcl": "Lisela", + "lcm": "Tungag", + "lcp": "Western Lawa", + "lcq": "Luhu", + "lcs": "Lisabata-Nuniali", + "lda": "Kla-Dan", + "ldb": "Dũya", + "ldd": "Luri", + "ldg": "Lenyima", + "ldh": "Lamja-Dengsa-Tola", + "ldi": "Laari", + "ldj": "Lemoro", + "ldk": "Leelau", + "ldl": "Kaan", + "ldm": "Landoma", + "ldn": "Láadan", + "ldo": "Loo", + "ldp": "Tso", + "ldq": "Lufu", + "lea": "Lega-Shabunda", + "leb": "Lala-Bisa", + "lec": "Leco", + "led": "Lendu", + "lee": "Lyélé", + "lef": "Lelemi", + "leh": "Lenje", + "lei": "Lemio", + "lej": "Lengola", + "lek": "Leipon", + "lel": "Lele (Democratic Republic of Congo)", + "lem": "Nomaande", + "len": "Lenca", + "leo": "Leti (Cameroon)", + "lep": "Lepcha", + "leq": "Lembena", + "ler": "Lenkau", + "les": "Lese", + "let": "Lesing-Gelimi; Amio-Gelimi", + "leu": "Kara (Papua New Guinea)", + "lev": "Lamma", + "lew": "Ledo Kaili", + "lex": "Luang", + "ley": "Lemolang", + "lez": "Lezghian", + "lfa": "Lefa", + "lfn": "Lingua Franca Nova", + "lg": "Ganda; Luganda", + "lga": "Lungga", + "lgb": "Laghu", + "lgg": "Lugbara", + "lgh": "Laghuu", + "lgi": "Lengilu", + "lgk": "Lingarak; Neverver", + "lgl": "Wala", + "lgm": "Lega-Mwenga", + "lgn": "T'apo; Opuuo", + "lgo": "Lango (South Sudan)", + "lgq": "Logba", + "lgr": "Lengo", + "lgt": "Pahi", + "lgu": "Longgu", + "lgz": "Ligenza", + "lha": "Laha (Viet Nam)", + "lhh": "Laha (Indonesia)", + "lhi": "Lahu Shi", + "lhl": "Lahul Lohar", + "lhm": "Lhomi", + "lhn": "Lahanan", + "lhp": "Lhokpu", + "lhs": "Mlahsö", + "lht": "Lo-Toga", + "lhu": "Lahu", + "li": "Limburgan; Limburger; Limburgish", + "lia": "West-Central Limba", + "lib": "Likum", + "lic": "Hlai", + "lid": "Nyindrou", + "lie": "Likila", + "lif": "Limbu", + "lig": "Ligbi", + "lih": "Lihir", + "lij": "Ligurian", + "lik": "Lika", + "lil": "Lillooet", + "lio": "Liki", + "lip": "Sekpele", + "liq": "Libido", + "lir": "Liberian English", + "lis": "Lisu", + "liu": "Logorik", + "liv": "Liv", + "liw": "Col", + "lix": "Liabuku", + "liy": "Banda-Bambari", + "liz": "Libinza", + "lja": "Golpa", + "lje": "Rampi", + "lji": "Laiyolo", + "ljl": "Li'o", + "ljp": "Lampung Api", + "ljw": "Yirandali", + "ljx": "Yuru", + "lka": "Lakalei", + "lkb": "Kabras; Lukabaras", + "lkc": "Kucong", + "lkd": "Lakondê", + "lke": "Kenyi", + "lkh": "Lakha", + "lki": "Laki", + "lkj": "Remun", + "lkl": "Laeko-Libuat", + "lkm": "Kalaamaya", + "lkn": "Lakon; Vure", + "lko": "Khayo; Olukhayo", + "lkr": "Päri", + "lks": "Kisa; Olushisa", + "lkt": "Lakota", + "lku": "Kungkari", + "lky": "Lokoya", + "lla": "Lala-Roba", + "llb": "Lolo", + "llc": "Lele (Guinea)", + "lld": "Ladin", + "lle": "Lele (Papua New Guinea)", + "llf": "Hermit", + "llg": "Lole", + "llh": "Lamu", + "lli": "Teke-Laali", + "llj": "Ladji Ladji", + "llk": "Lelak", + "lll": "Lilau", + "llm": "Lasalimu", + "lln": "Lele (Chad)", + "llp": "North Efate", + "llq": "Lolak", + "lls": "Lithuanian Sign Language", + "llu": "Lau", + "llx": "Lauan", + "lma": "East Limba", + "lmb": "Merei", + "lmc": "Limilngan", + "lmd": "Lumun", + "lme": "Pévé", + "lmf": "South Lembata", + "lmg": "Lamogai", + "lmh": "Lambichhong", + "lmi": "Lombi", + "lmj": "West Lembata", + "lmk": "Lamkang", + "lml": "Hano", + "lmn": "Lambadi", + "lmo": "Lombard", + "lmp": "Limbum", + "lmq": "Lamatuka", + "lmr": "Lamalera", + "lmu": "Lamenu", + "lmv": "Lomaiviti", + "lmw": "Lake Miwok", + "lmx": "Laimbue", + "lmy": "Lamboya", + "ln": "Lingala", + "lna": "Langbashe", + "lnb": "Mbalanhu", + "lnd": "Lundayeh; Lun Bawang", + "lng": "Langobardic", + "lnh": "Lanoh", + "lni": "Daantanai'", + "lnj": "Leningitij", + "lnl": "South Central Banda", + "lnm": "Langam", + "lnn": "Lorediakarkar", + "lns": "Lamnso'", + "lnu": "Longuda", + "lnw": "Lanima", + "lnz": "Lonzo", + "lo": "Lao", + "loa": "Loloda", + "lob": "Lobi", + "loc": "Inonhan", + "loe": "Saluan", + "lof": "Logol", + "log": "Logo", + "loh": "Narim", + "loi": "Loma (Côte d'Ivoire)", + "loj": "Lou", + "lok": "Loko", + "lol": "Mongo", + "lom": "Loma (Liberia)", + "lon": "Malawi Lomwe", + "loo": "Lombo", + "lop": "Lopa", + "loq": "Lobala", + "lor": "Téén", + "los": "Loniu", + "lot": "Otuho", + "lou": "Louisiana Creole", + "lov": "Lopi", + "low": "Tampias Lobu", + "lox": "Loun", + "loy": "Loke", + "loz": "Lozi", + "lpa": "Lelepa", + "lpe": "Lepki", + "lpn": "Long Phuri Naga", + "lpo": "Lipo", + "lpx": "Lopit", + "lqr": "Logir", + "lra": "Rara Bakati'", + "lrc": "Northern Luri", + "lre": "Laurentian", + "lrg": "Laragia", + "lri": "Marachi; Olumarachi", + "lrk": "Loarki", + "lrl": "Lari", + "lrm": "Marama; Olumarama", + "lrn": "Lorang", + "lro": "Laro", + "lrr": "Southern Yamphu", + "lrt": "Larantuka Malay", + "lrv": "Larevat", + "lrz": "Lemerig", + "lsa": "Lasgerdi", + "lsb": "Burundian Sign Language; Langue des Signes Burundaise", + "lsc": "Albarradas Sign Language; Lengua de señas Albarradas", + "lsd": "Lishana Deni", + "lse": "Lusengo", + "lsh": "Lish", + "lsi": "Lashi", + "lsl": "Latvian Sign Language", + "lsm": "Saamia; Olusamia", + "lsn": "Tibetan Sign Language", + "lso": "Laos Sign Language", + "lsp": "Panamanian Sign Language; Lengua de Señas Panameñas", + "lsr": "Aruop", + "lss": "Lasi", + "lst": "Trinidad and Tobago Sign Language", + "lsv": "Sivia Sign Language", + "lsw": "Seychelles Sign Language; Lalang Siny Seselwa; Langue des Signes Seychelloise", + "lsy": "Mauritian Sign Language", + "lt": "Lithuanian", + "ltc": "Late Middle Chinese", + "ltg": "Latgalian", + "lth": "Thur", + "lti": "Leti (Indonesia)", + "ltn": "Latundê", + "lto": "Tsotso; Olutsotso", + "lts": "Tachoni; Lutachoni", + "ltu": "Latu", + "lu": "Luba-Katanga", + "lua": "Luba-Lulua", + "luc": "Aringa", + "lud": "Ludian", + "lue": "Luvale", + "luf": "Laua", + "lui": "Luiseno", + "luj": "Luna", + "luk": "Lunanakha", + "lul": "Olu'bo", + "lum": "Luimbi", + "lun": "Lunda", + "luo": "Luo (Kenya and Tanzania); Dholuo", + "lup": "Lumbu", + "luq": "Lucumi", + "lur": "Laura", + "lus": "Lushai", + "lut": "Lushootseed", + "luu": "Lumba-Yakkha", + "luv": "Luwati", + "luw": "Luo (Cameroon)", + "luy": "Luyia; Oluluyia", + "luz": "Southern Luri", + "lv": "Latvian", + "lva": "Maku'a", + "lvi": "Lavi", + "lvk": "Lavukaleve", + "lvs": "Standard Latvian", + "lvu": "Levuka", + "lwa": "Lwalu", + "lwe": "Lewo Eleng", + "lwg": "Wanga; Oluwanga", + "lwh": "White Lachi", + "lwl": "Eastern Lawa", + "lwm": "Laomian", + "lwo": "Luwo", + "lws": "Malawian Sign Language", + "lwt": "Lewotobi", + "lwu": "Lawu", + "lww": "Lewo", + "lxm": "Lakurumau", + "lya": "Layakha", + "lyg": "Lyngngam", + "lyn": "Luyana", + "lzh": "Literary Chinese", + "lzl": "Litzlitz", + "lzn": "Leinong Naga", + "lzz": "Laz", + "maa": "San Jerónimo Tecóatl Mazatec", + "mab": "Yutanduchi Mixtec", + "mad": "Madurese", + "mae": "Bo-Rukul", + "maf": "Mafa", + "mag": "Magahi", + "mai": "Maithili", + "maj": "Jalapa De Díaz Mazatec", + "mak": "Makasar", + "mam": "Mam", + "man": "Mandingo; Manding", + "map": "Austronesian languages", + "maq": "Chiquihuitlán Mazatec", + "mas": "Masai", + "mat": "San Francisco Matlatzinca", + "mau": "Huautla Mazatec", + "mav": "Sateré-Mawé", + "maw": "Mampruli", + "max": "North Moluccan Malay", + "maz": "Central Mazahua", + "mba": "Higaonon", + "mbb": "Western Bukidnon Manobo", + "mbc": "Macushi", + "mbd": "Dibabawon Manobo", + "mbe": "Molale", + "mbf": "Baba Malay", + "mbh": "Mangseng", + "mbi": "Ilianen Manobo", + "mbj": "Nadëb", + "mbk": "Malol", + "mbl": "Maxakalí", + "mbm": "Ombamba", + "mbn": "Macaguán", + "mbo": "Mbo (Cameroon)", + "mbp": "Malayo", + "mbq": "Maisin", + "mbr": "Nukak Makú", + "mbs": "Sarangani Manobo", + "mbt": "Matigsalug Manobo", + "mbu": "Mbula-Bwazza", + "mbv": "Mbulungish", + "mbw": "Maring", + "mbx": "Mari (East Sepik Province)", + "mby": "Memoni", + "mbz": "Amoltepec Mixtec", + "mca": "Maca", + "mcb": "Machiguenga", + "mcc": "Bitur", + "mcd": "Sharanahua", + "mce": "Itundujia Mixtec", + "mcf": "Matsés", + "mcg": "Mapoyo", + "mch": "Maquiritari", + "mci": "Mese", + "mcj": "Mvanip", + "mck": "Mbunda", + "mcl": "Macaguaje", + "mcm": "Malaccan Creole Portuguese", + "mcn": "Masana", + "mco": "Coatlán Mixe", + "mcp": "Makaa", + "mcq": "Ese", + "mcr": "Menya", + "mcs": "Mambai", + "mct": "Mengisa", + "mcu": "Cameroon Mambila", + "mcv": "Minanibai", + "mcw": "Mawa (Chad)", + "mcx": "Mpiemo", + "mcy": "South Watut", + "mcz": "Mawan", + "mda": "Mada (Nigeria)", + "mdb": "Morigi", + "mdc": "Male (Papua New Guinea)", + "mdd": "Mbum", + "mde": "Maba (Chad)", + "mdf": "Moksha", + "mdg": "Massalat", + "mdh": "Maguindanaon", + "mdi": "Mamvu", + "mdj": "Mangbetu", + "mdk": "Mangbutu", + "mdl": "Maltese Sign Language", + "mdm": "Mayogo", + "mdn": "Mbati", + "mdp": "Mbala", + "mdq": "Mbole", + "mdr": "Mandar", + "mds": "Maria (Papua New Guinea)", + "mdt": "Mbere", + "mdu": "Mboko", + "mdv": "Santa Lucía Monteverde Mixtec", + "mdw": "Mbosi", + "mdx": "Dizin", + "mdy": "Male (Ethiopia)", + "mdz": "Suruí Do Pará", + "mea": "Menka", + "meb": "Ikobi", + "mec": "Marra", + "med": "Melpa", + "mee": "Mengen", + "mef": "Megam", + "meh": "Southwestern Tlaxiaco Mixtec", + "mei": "Midob", + "mej": "Meyah", + "mek": "Mekeo", + "mel": "Central Melanau", + "mem": "Mangala", + "men": "Mende (Sierra Leone)", + "meo": "Kedah Malay", + "mep": "Miriwoong", + "meq": "Merey", + "mer": "Meru", + "mes": "Masmaje", + "met": "Mato", + "meu": "Motu", + "mev": "Mano", + "mew": "Maaka", + "mey": "Hassaniyya", + "mez": "Menominee", + "mfa": "Pattani Malay", + "mfb": "Bangka", + "mfc": "Mba", + "mfd": "Mendankwe-Nkwen", + "mfe": "Morisyen", + "mff": "Naki", + "mfg": "Mogofin", + "mfh": "Matal", + "mfi": "Wandala", + "mfj": "Mefele", + "mfk": "North Mofu", + "mfl": "Putai", + "mfm": "Marghi South", + "mfn": "Cross River Mbembe", + "mfo": "Mbe", + "mfp": "Makassar Malay", + "mfq": "Moba", + "mfr": "Marrithiyel", + "mfs": "Mexican Sign Language", + "mft": "Mokerang", + "mfu": "Mbwela", + "mfv": "Mandjak", + "mfw": "Mulaha", + "mfx": "Melo", + "mfy": "Mayo", + "mfz": "Mabaan", + "mg": "Malagasy", + "mga": "Middle Irish (900-1200)", + "mgb": "Mararit", + "mgc": "Morokodo", + "mgd": "Moru", + "mge": "Mango", + "mgf": "Maklew", + "mgg": "Mpumpong", + "mgh": "Makhuwa-Meetto", + "mgi": "Lijili", + "mgj": "Abureni", + "mgk": "Mawes", + "mgl": "Maleu-Kilenge", + "mgm": "Mambae", + "mgn": "Mbangi", + "mgo": "Meta'", + "mgp": "Eastern Magar", + "mgq": "Malila", + "mgr": "Mambwe-Lungu", + "mgs": "Manda (Tanzania)", + "mgt": "Mongol", + "mgu": "Mailu", + "mgv": "Matengo", + "mgw": "Matumbi", + "mgy": "Mbunga", + "mgz": "Mbugwe", + "mh": "Marshallese", + "mha": "Manda (India)", + "mhb": "Mahongwe", + "mhc": "Mocho", + "mhd": "Mbugu", + "mhe": "Besisi; Mah Meri", + "mhf": "Mamaa", + "mhg": "Margu", + "mhi": "Ma'di", + "mhj": "Mogholi", + "mhk": "Mungaka", + "mhl": "Mauwake", + "mhm": "Makhuwa-Moniga", + "mhn": "Mócheno", + "mho": "Mashi (Zambia)", + "mhp": "Balinese Malay", + "mhq": "Mandan", + "mhr": "Eastern Mari", + "mhs": "Buru (Indonesia)", + "mht": "Mandahuaca", + "mhu": "Digaro-Mishmi; Darang Deng", + "mhw": "Mbukushu", + "mhx": "Maru; Lhaovo", + "mhy": "Ma'anyan", + "mhz": "Mor (Mor Islands)", + "mi": "Maori", + "mia": "Miami", + "mib": "Atatláhuca Mixtec", + "mic": "Mi'kmaq; Micmac", + "mid": "Mandaic", + "mie": "Ocotepec Mixtec", + "mif": "Mofu-Gudur", + "mig": "San Miguel El Grande Mixtec", + "mih": "Chayuco Mixtec", + "mii": "Chigmecatitlán Mixtec", + "mij": "Abar; Mungbam", + "mik": "Mikasuki", + "mil": "Peñoles Mixtec", + "mim": "Alacatlatzala Mixtec", + "min": "Minangkabau", + "mio": "Pinotepa Nacional Mixtec", + "mip": "Apasco-Apoala Mixtec", + "miq": "Mískito", + "mir": "Isthmus Mixe", + "mit": "Southern Puebla Mixtec", + "miu": "Cacaloxtepec Mixtec", + "miw": "Akoye", + "mix": "Mixtepec Mixtec", + "miy": "Ayutla Mixtec", + "miz": "Coatzospan Mixtec", + "mjb": "Makalero", + "mjc": "San Juan Colorado Mixtec", + "mjd": "Northwest Maidu", + "mje": "Muskum", + "mjg": "Tu", + "mjh": "Mwera (Nyasa)", + "mji": "Kim Mun", + "mjj": "Mawak", + "mjk": "Matukar", + "mjl": "Mandeali", + "mjm": "Medebur", + "mjn": "Ma (Papua New Guinea)", + "mjo": "Malankuravan", + "mjp": "Malapandaram", + "mjq": "Malaryan", + "mjr": "Malavedan", + "mjs": "Miship", + "mjt": "Sauria Paharia", + "mju": "Manna-Dora", + "mjv": "Mannan", + "mjw": "Karbi", + "mjx": "Mahali", + "mjy": "Mahican", + "mjz": "Majhi", + "mk": "Macedonian", + "mka": "Mbre", + "mkb": "Mal Paharia", + "mkc": "Siliput", + "mke": "Mawchi", + "mkf": "Miya", + "mkg": "Mak (China)", + "mkh": "Mon-Khmer languages", + "mki": "Dhatki", + "mkj": "Mokilese", + "mkk": "Byep", + "mkl": "Mokole", + "mkm": "Moklen", + "mkn": "Kupang Malay", + "mko": "Mingang Doso", + "mkp": "Moikodi", + "mkq": "Bay Miwok", + "mkr": "Malas", + "mks": "Silacayoapan Mixtec", + "mkt": "Vamale", + "mku": "Konyanka Maninka", + "mkv": "Mafea", + "mkw": "Kituba (Congo)", + "mkx": "Kinamiging Manobo", + "mky": "East Makian", + "mkz": "Makasae", + "ml": "Malayalam", + "mla": "Malo", + "mlb": "Mbule", + "mlc": "Cao Lan", + "mle": "Manambu", + "mlf": "Mal", + "mlh": "Mape", + "mli": "Malimpung", + "mlj": "Miltu", + "mlk": "Ilwana; Kiwilwana", + "mll": "Malua Bay", + "mlm": "Mulam", + "mln": "Malango", + "mlo": "Mlomp", + "mlp": "Bargam", + "mlq": "Western Maninkakan", + "mlr": "Vame", + "mls": "Masalit", + "mlu": "To'abaita", + "mlv": "Motlav; Mwotlap", + "mlw": "Moloko", + "mlx": "Malfaxal; Naha'ai", + "mlz": "Malaynon", + "mma": "Mama", + "mmb": "Momina", + "mmc": "Michoacán Mazahua", + "mmd": "Maonan", + "mme": "Mae", + "mmf": "Mundat", + "mmg": "North Ambrym", + "mmh": "Mehináku", + "mmi": "Musar", + "mmj": "Majhwar", + "mmk": "Mukha-Dora", + "mml": "Man Met", + "mmm": "Maii", + "mmn": "Mamanwa", + "mmo": "Mangga Buang", + "mmp": "Siawi", + "mmq": "Musak", + "mmr": "Western Xiangxi Miao", + "mmt": "Malalamai", + "mmu": "Mmaala", + "mmv": "Miriti", + "mmw": "Emae", + "mmx": "Madak", + "mmy": "Migaama", + "mmz": "Mabaale", + "mn": "Mongolian", + "mna": "Mbula", + "mnb": "Muna", + "mnc": "Manchu", + "mnd": "Mondé", + "mne": "Naba", + "mnf": "Mundani", + "mng": "Eastern Mnong", + "mnh": "Mono (Democratic Republic of Congo)", + "mni": "Manipuri", + "mnj": "Munji", + "mnk": "Mandinka", + "mnl": "Tiale", + "mnm": "Mapena", + "mnn": "Southern Mnong", + "mno": "Manobo languages", + "mnp": "Min Bei Chinese", + "mnq": "Minriq", + "mnr": "Mono (USA)", + "mns": "Mansi", + "mnu": "Mer", + "mnv": "Rennell-Bellona", + "mnw": "Mon", + "mnx": "Manikion", + "mny": "Manyawa", + "mnz": "Moni", + "moa": "Mwan", + "moc": "Mocoví", + "mod": "Mobilian", + "moe": "Innu; Montagnais", + "mog": "Mongondow", + "moh": "Mohawk", + "moi": "Mboi", + "moj": "Monzombo", + "mok": "Morori", + "mom": "Mangue", + "moo": "Monom", + "mop": "Mopán Maya", + "moq": "Mor (Bomberai Peninsula)", + "mor": "Moro", + "mos": "Mossi", + "mot": "Barí", + "mou": "Mogum", + "mov": "Mohave", + "mow": "Moi (Congo)", + "mox": "Molima", + "moy": "Shekkacho", + "moz": "Mukulu; Gergiko", + "mpa": "Mpoto", + "mpb": "Malak Malak; Mullukmulluk", + "mpc": "Mangarrayi", + "mpd": "Machinere", + "mpe": "Majang", + "mpg": "Marba", + "mph": "Maung", + "mpi": "Mpade", + "mpj": "Martu Wangka; Wangkajunga", + "mpk": "Mbara (Chad)", + "mpl": "Middle Watut", + "mpm": "Yosondúa Mixtec", + "mpn": "Mindiri", + "mpo": "Miu", + "mpp": "Migabac", + "mpq": "Matís", + "mpr": "Vangunu", + "mps": "Dadibi", + "mpt": "Mian", + "mpu": "Makuráp", + "mpv": "Mungkip", + "mpw": "Mapidian", + "mpx": "Misima-Panaeati", + "mpy": "Mapia", + "mpz": "Mpi", + "mqa": "Maba (Indonesia)", + "mqb": "Mbuko", + "mqc": "Mangole", + "mqe": "Matepi", + "mqf": "Momuna", + "mqg": "Kota Bangun Kutai Malay", + "mqh": "Tlazoyaltepec Mixtec", + "mqi": "Mariri", + "mqj": "Mamasa", + "mqk": "Rajah Kabunsuwan Manobo", + "mql": "Mbelime", + "mqm": "South Marquesan", + "mqn": "Moronene", + "mqo": "Modole", + "mqp": "Manipa", + "mqq": "Minokok", + "mqr": "Mander", + "mqs": "West Makian", + "mqt": "Mok", + "mqu": "Mandari", + "mqv": "Mosimo", + "mqw": "Murupi", + "mqx": "Mamuju", + "mqy": "Manggarai", + "mqz": "Pano", + "mr": "Marathi", + "mra": "Mlabri", + "mrb": "Marino", + "mrc": "Maricopa", + "mrd": "Western Magar", + "mre": "Martha's Vineyard Sign Language", + "mrf": "Elseng", + "mrg": "Mising", + "mrh": "Mara Chin", + "mrj": "Western Mari", + "mrk": "Hmwaveke", + "mrl": "Mortlockese", + "mrm": "Merlav; Mwerlap", + "mrn": "Cheke Holo", + "mro": "Mru", + "mrp": "Morouas", + "mrq": "North Marquesan", + "mrr": "Maria (India)", + "mrs": "Maragus", + "mrt": "Marghi Central", + "mru": "Mono (Cameroon)", + "mrv": "Mangareva", + "mrw": "Maranao", + "mrx": "Maremgi; Dineor", + "mry": "Mandaya", + "mrz": "Marind", + "ms": "Malay (macrolanguage)", + "msb": "Masbatenyo", + "msc": "Sankaran Maninka", + "msd": "Yucatec Maya Sign Language", + "mse": "Musey", + "msf": "Mekwei", + "msg": "Moraid", + "msh": "Masikoro Malagasy", + "msi": "Sabah Malay", + "msj": "Ma (Democratic Republic of Congo)", + "msk": "Mansaka", + "msl": "Molof; Poule", + "msm": "Agusan Manobo", + "msn": "Vurës", + "mso": "Mombum", + "msp": "Maritsauá", + "msq": "Caac", + "msr": "Mongolian Sign Language", + "mss": "West Masela", + "msu": "Musom", + "msv": "Maslam", + "msw": "Mansoanka", + "msx": "Moresada", + "msy": "Aruamu", + "msz": "Momare", + "mt": "Maltese", + "mta": "Cotabato Manobo", + "mtb": "Anyin Morofo", + "mtc": "Munit", + "mtd": "Mualang", + "mte": "Mono (Solomon Islands)", + "mtf": "Murik (Papua New Guinea)", + "mtg": "Una", + "mth": "Munggui", + "mti": "Maiwa (Papua New Guinea)", + "mtj": "Moskona", + "mtk": "Mbe'", + "mtl": "Montol", + "mtm": "Mator", + "mtn": "Matagalpa", + "mto": "Totontepec Mixe", + "mtp": "Wichí Lhamtés Nocten", + "mtq": "Muong", + "mtr": "Mewari", + "mts": "Yora", + "mtt": "Mota", + "mtu": "Tututepec Mixtec", + "mtv": "Asaro'o", + "mtw": "Southern Binukidnon", + "mtx": "Tidaá Mixtec", + "mty": "Nabi", + "mua": "Mundang", + "mub": "Mubi", + "muc": "Ajumbu", + "mud": "Mednyj Aleut", + "mue": "Media Lengua", + "mug": "Musgu", + "muh": "Mündü", + "mui": "Musi", + "muj": "Mabire", + "muk": "Mugom", + "mum": "Maiwala", + "mun": "Munda languages", + "muo": "Nyong", + "mup": "Malvi", + "muq": "Eastern Xiangxi Miao", + "mur": "Murle", + "mus": "Creek", + "mut": "Western Muria", + "muu": "Yaaku", + "muv": "Muthuvan", + "mux": "Bo-Ung", + "muy": "Muyang", + "muz": "Mursi", + "mva": "Manam", + "mvb": "Mattole", + "mvd": "Mamboru", + "mve": "Marwari (Pakistan)", + "mvf": "Peripheral Mongolian", + "mvg": "Yucuañe Mixtec", + "mvh": "Mulgi", + "mvi": "Miyako", + "mvk": "Mekmek", + "mvl": "Mbara (Australia)", + "mvn": "Minaveha", + "mvo": "Marovo", + "mvp": "Duri", + "mvq": "Moere", + "mvr": "Marau", + "mvs": "Massep", + "mvt": "Mpotovoro", + "mvu": "Marfa", + "mvv": "Tagal Murut", + "mvw": "Machinga", + "mvx": "Meoswar", + "mvy": "Indus Kohistani", + "mvz": "Mesqan", + "mwa": "Mwatebu", + "mwb": "Juwal", + "mwc": "Are", + "mwe": "Mwera (Chimwera)", + "mwf": "Murrinh-Patha", + "mwg": "Aiklep", + "mwh": "Mouk-Aria", + "mwi": "Labo; Ninde", + "mwk": "Kita Maninkakan", + "mwl": "Mirandese", + "mwm": "Sar", + "mwn": "Nyamwanga", + "mwo": "Central Maewo", + "mwp": "Kala Lagaw Ya", + "mwq": "Mün Chin", + "mwr": "Marwari", + "mws": "Mwimbi-Muthambi", + "mwt": "Moken", + "mwu": "Mittu", + "mwv": "Mentawai", + "mww": "Hmong Daw", + "mwz": "Moingi", + "mxa": "Northwest Oaxaca Mixtec", + "mxb": "Tezoatlán Mixtec", + "mxc": "Manyika", + "mxd": "Modang", + "mxe": "Mele-Fila", + "mxf": "Malgbe", + "mxg": "Mbangala", + "mxh": "Mvuba", + "mxi": "Mozarabic", + "mxj": "Miju-Mishmi; Geman Deng", + "mxk": "Monumbo", + "mxl": "Maxi Gbe", + "mxm": "Meramera", + "mxn": "Moi (Indonesia)", + "mxo": "Mbowe", + "mxp": "Tlahuitoltepec Mixe", + "mxq": "Juquila Mixe", + "mxr": "Murik (Malaysia)", + "mxs": "Huitepec Mixtec", + "mxt": "Jamiltepec Mixtec", + "mxu": "Mada (Cameroon)", + "mxv": "Metlatónoc Mixtec", + "mxw": "Namo", + "mxx": "Mahou; Mawukakan", + "mxy": "Southeastern Nochixtlán Mixtec", + "mxz": "Central Masela", + "my": "Burmese", + "myb": "Mbay", + "myc": "Mayeka", + "mye": "Myene", + "myf": "Bambassi", + "myg": "Manta", + "myh": "Makah", + "myj": "Mangayat", + "myk": "Mamara Senoufo", + "myl": "Moma", + "mym": "Me'en", + "myn": "Mayan languages", + "myo": "Anfillo", + "myp": "Pirahã", + "myr": "Muniche", + "mys": "Mesmes", + "myu": "Mundurukú", + "myv": "Erzya", + "myw": "Muyuw", + "myx": "Masaaba", + "myy": "Macuna", + "myz": "Classical Mandaic", + "mza": "Santa María Zacatepec Mixtec", + "mzb": "Tumzabt", + "mzc": "Madagascar Sign Language", + "mzd": "Malimba", + "mze": "Morawa", + "mzg": "Monastic Sign Language", + "mzh": "Wichí Lhamtés Güisnay", + "mzi": "Ixcatlán Mazatec", + "mzj": "Manya", + "mzk": "Nigeria Mambila", + "mzl": "Mazatlán Mixe", + "mzm": "Mumuye", + "mzn": "Mazanderani", + "mzo": "Matipuhy", + "mzp": "Movima", + "mzq": "Mori Atas", + "mzr": "Marúbo", + "mzs": "Macanese", + "mzt": "Mintil", + "mzu": "Inapang", + "mzv": "Manza", + "mzw": "Deg", + "mzx": "Mawayana", + "mzy": "Mozambican Sign Language", + "mzz": "Maiadomu", + "na": "Nauru", + "naa": "Namla", + "nab": "Southern Nambikuára", + "nac": "Narak", + "nae": "Naka'ela", + "naf": "Nabak", + "nag": "Naga Pidgin", + "nah": "Nahuatl languages", + "nai": "North American Indian languages", + "naj": "Nalu", + "nak": "Nakanai", + "nal": "Nalik", + "nam": "Ngan'gityemerri", + "nan": "Min Nan Chinese", + "nao": "Naaba", + "nap": "Neapolitan", + "naq": "Khoekhoe; Nama (Namibia)", + "nar": "Iguta", + "nas": "Naasioi", + "nat": "Ca̱hungwa̱rya̱; Hungworo", + "naw": "Nawuri", + "nax": "Nakwi", + "nay": "Ngarrindjeri", + "naz": "Coatepec Nahuatl", + "nb": "Norwegian Bokmål", + "nba": "Nyemba", + "nbb": "Ndoe", + "nbc": "Chang Naga", + "nbd": "Ngbinda", + "nbe": "Konyak Naga", + "nbg": "Nagarchal", + "nbh": "Ngamo", + "nbi": "Mao Naga", + "nbj": "Ngarinyman", + "nbk": "Nake", + "nbm": "Ngbaka Ma'bo", + "nbn": "Kuri", + "nbo": "Nkukoli", + "nbp": "Nnam", + "nbq": "Nggem", + "nbr": "Numana", + "nbs": "Namibian Sign Language", + "nbt": "Na", + "nbu": "Rongmei Naga", + "nbv": "Ngamambo", + "nbw": "Southern Ngbandi", + "nby": "Ningera", + "nca": "Iyo", + "ncb": "Central Nicobarese", + "ncc": "Ponam", + "ncd": "Nachering", + "nce": "Yale", + "ncf": "Notsi", + "ncg": "Nisga'a", + "nch": "Central Huasteca Nahuatl", + "nci": "Classical Nahuatl", + "ncj": "Northern Puebla Nahuatl", + "nck": "Na-kara", + "ncl": "Michoacán Nahuatl", + "ncm": "Nambo", + "ncn": "Nauna", + "nco": "Sibe", + "ncq": "Northern Katang", + "ncr": "Ncane", + "ncs": "Nicaraguan Sign Language", + "nct": "Chothe Naga", + "ncu": "Chumburung", + "ncx": "Central Puebla Nahuatl", + "ncz": "Natchez", + "nd": "North Ndebele", + "nda": "Ndasa", + "ndb": "Kenswei Nsei", + "ndc": "Ndau", + "ndd": "Nde-Nsele-Nta", + "ndf": "Nadruvian", + "ndg": "Ndengereko", + "ndh": "Ndali", + "ndi": "Samba Leko", + "ndj": "Ndamba", + "ndk": "Ndaka", + "ndl": "Ndolo", + "ndm": "Ndam", + "ndn": "Ngundi", + "ndp": "Ndo", + "ndq": "Ndombe", + "ndr": "Ndoola", + "nds": "Low German; Low Saxon", + "ndt": "Ndunga", + "ndu": "Dugun", + "ndv": "Ndut", + "ndw": "Ndobo", + "ndx": "Nduga", + "ndy": "Lutos", + "ndz": "Ndogo", + "ne": "Nepali (macrolanguage)", + "nea": "Eastern Ngad'a", + "neb": "Toura (Côte d'Ivoire)", + "nec": "Nedebang", + "ned": "Nde-Gbite", + "nee": "Nêlêmwa-Nixumwak", + "nef": "Nefamese", + "neg": "Negidal", + "neh": "Nyenkha", + "nei": "Neo-Hittite", + "nej": "Neko", + "nek": "Neku", + "nem": "Nemi", + "nen": "Nengone", + "neo": "Ná-Meo", + "neq": "North Central Mixe", + "ner": "Yahadian", + "nes": "Bhoti Kinnauri", + "net": "Nete", + "neu": "Neo", + "nev": "Nyaheun", + "new": "Newari; Nepal Bhasa", + "nex": "Neme", + "ney": "Neyo", + "nez": "Nez Perce", + "nfa": "Dhao", + "nfd": "Ahwai", + "nfl": "Ayiwo; Äiwoo", + "nfr": "Nafaanra", + "nfu": "Mfumte", + "ng": "Ndonga", + "nga": "Ngbaka", + "ngb": "Northern Ngbandi", + "ngc": "Ngombe (Democratic Republic of Congo)", + "ngd": "Ngando (Central African Republic)", + "nge": "Ngemba", + "ngf": "Trans-New Guinea languages", + "ngg": "Ngbaka Manza", + "ngh": "Nǁng", + "ngi": "Ngizim", + "ngj": "Ngie", + "ngk": "Dalabon", + "ngl": "Lomwe", + "ngm": "Ngatik Men's Creole", + "ngn": "Ngwo", + "ngp": "Ngulu", + "ngq": "Ngurimi; Ngoreme", + "ngr": "Engdewu", + "ngs": "Gvoko", + "ngt": "Kriang; Ngeq", + "ngu": "Guerrero Nahuatl", + "ngv": "Nagumi", + "ngw": "Ngwaba", + "ngx": "Nggwahyi", + "ngy": "Tibea", + "ngz": "Ngungwel", + "nha": "Nhanda", + "nhb": "Beng", + "nhc": "Tabasco Nahuatl", + "nhd": "Chiripá; Ava Guaraní", + "nhe": "Eastern Huasteca Nahuatl", + "nhf": "Nhuwala", + "nhg": "Tetelcingo Nahuatl", + "nhh": "Nahari", + "nhi": "Zacatlán-Ahuacatlán-Tepetzintla Nahuatl", + "nhk": "Isthmus-Cosoleacaque Nahuatl", + "nhm": "Morelos Nahuatl", + "nhn": "Central Nahuatl", + "nho": "Takuu", + "nhp": "Isthmus-Pajapan Nahuatl", + "nhq": "Huaxcaleca Nahuatl", + "nhr": "Naro", + "nht": "Ometepec Nahuatl", + "nhu": "Noone", + "nhv": "Temascaltepec Nahuatl", + "nhw": "Western Huasteca Nahuatl", + "nhx": "Isthmus-Mecayapan Nahuatl", + "nhy": "Northern Oaxaca Nahuatl", + "nhz": "Santa María La Alta Nahuatl", + "nia": "Nias", + "nib": "Nakame", + "nic": "Niger-Kordofanian languages", + "nid": "Ngandi", + "nie": "Niellim", + "nif": "Nek", + "nig": "Ngalakgan", + "nih": "Nyiha (Tanzania)", + "nii": "Nii", + "nij": "Ngaju", + "nik": "Southern Nicobarese", + "nil": "Nila", + "nim": "Nilamba", + "nin": "Ninzo", + "nio": "Nganasan", + "niq": "Nandi", + "nir": "Nimboran", + "nis": "Nimi", + "nit": "Southeastern Kolami", + "niu": "Niuean", + "niv": "Gilyak", + "niw": "Nimo", + "nix": "Hema", + "niy": "Ngiti", + "niz": "Ningil", + "nja": "Nzanyi", + "njb": "Nocte Naga", + "njd": "Ndonde Hamba", + "njh": "Lotha Naga", + "nji": "Gudanji", + "njj": "Njen", + "njl": "Njalgulgule", + "njm": "Angami Naga", + "njn": "Liangmai Naga", + "njo": "Ao Naga", + "njr": "Njerep", + "njs": "Nisa", + "njt": "Ndyuka-Trio Pidgin", + "nju": "Ngadjunmaya", + "njx": "Kunyi", + "njy": "Njyem", + "njz": "Nyishi", + "nka": "Nkoya", + "nkb": "Khoibu Naga", + "nkc": "Nkongho", + "nkd": "Koireng", + "nke": "Duke", + "nkf": "Inpui Naga", + "nkg": "Nekgini", + "nkh": "Khezha Naga", + "nki": "Thangal Naga", + "nkj": "Nakai", + "nkk": "Nokuku", + "nkm": "Namat", + "nkn": "Nkangala", + "nko": "Nkonya", + "nkp": "Niuatoputapu", + "nkq": "Nkami", + "nkr": "Nukuoro", + "nks": "North Asmat", + "nkt": "Nyika (Tanzania)", + "nku": "Bouna Kulango", + "nkv": "Nyika (Malawi and Zambia)", + "nkw": "Nkutu", + "nkx": "Nkoroo", + "nkz": "Nkari", + "nl": "Dutch; Flemish", + "nla": "Ngombale", + "nlc": "Nalca", + "nle": "East Nyala", + "nlg": "Gela", + "nli": "Grangali", + "nlj": "Nyali", + "nlk": "Ninia Yali", + "nll": "Nihali", + "nlm": "Mankiyali", + "nlo": "Ngul", + "nlq": "Lao Naga", + "nlu": "Nchumbulu", + "nlv": "Orizaba Nahuatl", + "nlw": "Walangama", + "nlx": "Nahali", + "nly": "Nyamal", + "nlz": "Nalögo", + "nma": "Maram Naga", + "nmb": "Big Nambas; V'ënen Taut", + "nmc": "Ngam", + "nmd": "Ndumu", + "nme": "Mzieme Naga", + "nmf": "Tangkhul Naga (India)", + "nmg": "Kwasio", + "nmh": "Monsang Naga", + "nmi": "Nyam", + "nmj": "Ngombe (Central African Republic)", + "nmk": "Namakura", + "nml": "Ndemli", + "nmm": "Manangba", + "nmn": "ǃXóõ", + "nmo": "Moyon Naga", + "nmp": "Nimanbur", + "nmq": "Nambya", + "nmr": "Nimbari", + "nms": "Letemboi", + "nmt": "Namonuito", + "nmu": "Northeast Maidu", + "nmv": "Ngamini", + "nmw": "Nimoa; Rifao", + "nmx": "Nama (Papua New Guinea)", + "nmy": "Namuyi", + "nmz": "Nawdm", + "nn": "Norwegian Nynorsk", + "nna": "Nyangumarta", + "nnb": "Nande", + "nnc": "Nancere", + "nnd": "West Ambae", + "nne": "Ngandyera", + "nnf": "Ngaing", + "nng": "Maring Naga", + "nnh": "Ngiemboon", + "nni": "North Nuaulu", + "nnj": "Nyangatom", + "nnk": "Nankina", + "nnl": "Northern Rengma Naga", + "nnm": "Namia", + "nnn": "Ngete", + "nnp": "Wancho Naga", + "nnq": "Ngindo", + "nnr": "Narungga", + "nnt": "Nanticoke", + "nnu": "Dwang", + "nnv": "Nugunu (Australia)", + "nnw": "Southern Nuni", + "nny": "Nyangga", + "nnz": "Nda'nda'", + "no": "Norwegian", + "noa": "Woun Meu", + "noc": "Nuk", + "nod": "Northern Thai", + "noe": "Nimadi", + "nof": "Nomane", + "nog": "Nogai", + "noh": "Nomu", + "noi": "Noiri", + "noj": "Nonuya", + "nok": "Nooksack", + "nol": "Nomlaki", + "nom": "Nocamán", + "non": "Old Norse", + "nop": "Numanggang", + "noq": "Ngongo", + "nos": "Eastern Nisu", + "not": "Nomatsiguenga", + "nou": "Ewage-Notu", + "nov": "Novial", + "now": "Nyambo", + "noy": "Noy", + "noz": "Nayi", + "npa": "Nar Phu", + "npb": "Nupbikha", + "npg": "Ponyo-Gongwang Naga", + "nph": "Phom Naga", + "npi": "Nepali (individual language)", + "npl": "Southeastern Puebla Nahuatl", + "npn": "Mondropolon", + "npo": "Pochuri Naga", + "nps": "Nipsan", + "npu": "Puimei Naga", + "npx": "Noipx", + "npy": "Napu", + "nqg": "Southern Nago", + "nqk": "Kura Ede Nago", + "nql": "Ngendelengo", + "nqm": "Ndom", + "nqn": "Nen", + "nqo": "N'Ko; N’Ko", + "nqq": "Kyan-Karyaw Naga", + "nqt": "Nteng", + "nqy": "Akyaung Ari Naga", + "nr": "South Ndebele", + "nra": "Ngom", + "nrb": "Nara", + "nrc": "Noric", + "nre": "Southern Rengma Naga", + "nrf": "Jèrriais; Guernésiais", + "nrg": "Narango", + "nri": "Chokri Naga", + "nrk": "Ngarla", + "nrl": "Ngarluma", + "nrm": "Narom", + "nrn": "Norn", + "nrp": "North Picene", + "nrr": "Norra; Nora", + "nrt": "Northern Kalapuya", + "nru": "Narua", + "nrx": "Ngurmbur", + "nrz": "Lala", + "nsa": "Sangtam Naga", + "nsb": "Lower Nossob", + "nsc": "Nshi", + "nsd": "Southern Nisu", + "nse": "Nsenga", + "nsf": "Northwestern Nisu", + "nsg": "Ngasa", + "nsh": "Ngoshie", + "nsi": "Nigerian Sign Language", + "nsk": "Naskapi", + "nsl": "Norwegian Sign Language", + "nsm": "Sumi Naga", + "nsn": "Nehan", + "nso": "Pedi; Northern Sotho; Sepedi", + "nsp": "Nepalese Sign Language", + "nsq": "Northern Sierra Miwok", + "nsr": "Maritime Sign Language", + "nss": "Nali", + "nst": "Tase Naga", + "nsu": "Sierra Negra Nahuatl", + "nsv": "Southwestern Nisu", + "nsw": "Navut", + "nsx": "Nsongo", + "nsy": "Nasal", + "nsz": "Nisenan", + "ntd": "Northern Tidung", + "nte": "Nathembo", + "ntg": "Ngantangarra", + "nti": "Natioro", + "ntj": "Ngaanyatjarra", + "ntk": "Ikoma-Nata-Isenye", + "ntm": "Nateni", + "nto": "Ntomba", + "ntp": "Northern Tepehuan", + "ntr": "Delo", + "ntu": "Natügu", + "ntw": "Nottoway", + "ntx": "Tangkhul Naga (Myanmar)", + "nty": "Mantsi", + "ntz": "Natanzi", + "nua": "Yuanga", + "nub": "Nubian languages", + "nuc": "Nukuini", + "nud": "Ngala", + "nue": "Ngundu", + "nuf": "Nusu", + "nug": "Nungali", + "nuh": "Ndunda", + "nui": "Ngumbi", + "nuj": "Nyole", + "nuk": "Nuu-chah-nulth; Nuuchahnulth", + "nul": "Nusa Laut", + "num": "Niuafo'ou", + "nun": "Anong", + "nuo": "Nguôn", + "nup": "Nupe-Nupe-Tako", + "nuq": "Nukumanu", + "nur": "Nukuria", + "nus": "Nuer", + "nut": "Nung (Viet Nam)", + "nuu": "Ngbundu", + "nuv": "Northern Nuni", + "nuw": "Nguluwan", + "nux": "Mehek", + "nuy": "Nunggubuyu", + "nuz": "Tlamacazapa Nahuatl", + "nv": "Navajo; Navaho", + "nvh": "Nasarian", + "nvm": "Namiae", + "nvo": "Nyokon", + "nwa": "Nawathinehena", + "nwb": "Nyabwa", + "nwc": "Classical Newari; Classical Nepal Bhasa; Old Newari", + "nwe": "Ngwe", + "nwg": "Ngayawung", + "nwi": "Southwest Tanna", + "nwm": "Nyamusa-Molo", + "nwo": "Nauo", + "nwr": "Nawaru", + "nww": "Ndwewe", + "nwx": "Middle Newar", + "nwy": "Nottoway-Meherrin", + "nxa": "Nauete", + "nxd": "Ngando (Democratic Republic of Congo)", + "nxe": "Nage", + "nxg": "Ngad'a", + "nxi": "Nindi", + "nxk": "Koki Naga", + "nxl": "South Nuaulu", + "nxm": "Numidian", + "nxn": "Ngawun", + "nxo": "Ndambomo", + "nxq": "Naxi", + "nxr": "Ninggerum", + "nxx": "Nafri", + "ny": "Nyanja; Chewa; Chichewa", + "nyb": "Nyangbo", + "nyc": "Nyanga-li", + "nyd": "Nyore; Olunyole", + "nye": "Nyengo", + "nyf": "Giryama; Kigiryama", + "nyg": "Nyindu", + "nyh": "Nyikina", + "nyi": "Ama (Sudan)", + "nyj": "Nyanga", + "nyk": "Nyaneka", + "nyl": "Nyeu", + "nym": "Nyamwezi", + "nyn": "Nyankole", + "nyo": "Nyoro", + "nyp": "Nyang'i", + "nyq": "Nayini", + "nyr": "Nyiha (Malawi)", + "nys": "Nyungar", + "nyt": "Nyawaygi", + "nyu": "Nyungwe", + "nyv": "Nyulnyul", + "nyw": "Nyaw", + "nyx": "Nganyaywana", + "nyy": "Nyakyusa-Ngonde", + "nza": "Tigon Mbembe", + "nzb": "Njebi", + "nzd": "Nzadi", + "nzi": "Nzima", + "nzk": "Nzakara", + "nzm": "Zeme Naga", + "nzs": "New Zealand Sign Language", + "nzu": "Teke-Nzikou", + "nzy": "Nzakambay", + "nzz": "Nanga Dama Dogon", + "oaa": "Orok", + "oac": "Oroch", + "oar": "Old Aramaic (up to 700 BCE); Ancient Aramaic (up to 700 BCE)", + "oav": "Old Avar", + "obi": "Obispeño", + "obk": "Southern Bontok", + "obl": "Oblo", + "obm": "Moabite", + "obo": "Obo Manobo", + "obr": "Old Burmese", + "obt": "Old Breton", + "obu": "Obulom", + "oc": "Occitan (post 1500)", + "oca": "Ocaina", + "och": "Old Chinese", + "ocm": "Old Cham", + "oco": "Old Cornish", + "ocu": "Atzingo Matlatzinca", + "oda": "Odut", + "odk": "Od", + "odt": "Old Dutch", + "odu": "Odual", + "ofo": "Ofo", + "ofs": "Old Frisian", + "ofu": "Efutop", + "ogb": "Ogbia", + "ogc": "Ogbah", + "oge": "Old Georgian", + "ogg": "Ogbogolo", + "ogo": "Khana", + "ogu": "Ogbronuagum", + "oht": "Old Hittite", + "ohu": "Old Hungarian", + "oia": "Oirata", + "oie": "Okolie", + "oin": "Inebu One", + "oj": "Ojibwa", + "ojb": "Northwestern Ojibwa", + "ojc": "Central Ojibwa", + "ojg": "Eastern Ojibwa", + "ojp": "Old Japanese", + "ojs": "Severn Ojibwa", + "ojv": "Ontong Java", + "ojw": "Western Ojibwa", + "oka": "Okanagan", + "okb": "Okobo", + "okc": "Kobo", + "okd": "Okodia", + "oke": "Okpe (Southwestern Edo)", + "okg": "Koko Babangk", + "okh": "Koresh-e Rostam", + "oki": "Okiek", + "okj": "Oko-Juwoi", + "okk": "Kwamtim One", + "okl": "Old Kentish Sign Language", + "okm": "Middle Korean (10th-16th cent.)", + "okn": "Oki-No-Erabu", + "oko": "Old Korean (3rd-9th cent.)", + "okr": "Kirike", + "oks": "Oko-Eni-Osayen", + "oku": "Oku", + "okv": "Orokaiva", + "okx": "Okpe (Northwestern Edo)", + "okz": "Old Khmer", + "ola": "Walungge", + "old": "Mochi", + "ole": "Olekha", + "olk": "Olkol", + "olm": "Oloma", + "olo": "Livvi", + "olr": "Olrat", + "olt": "Old Lithuanian", + "olu": "Kuvale", + "om": "Oromo", + "oma": "Omaha-Ponca", + "omb": "East Ambae", + "omc": "Mochica", + "omg": "Omagua", + "omi": "Omi", + "omk": "Omok", + "oml": "Ombo", + "omn": "Minoan", + "omo": "Utarmbung", + "omp": "Old Manipuri", + "omq": "Oto-Manguean languages", + "omr": "Old Marathi", + "omt": "Omotik", + "omu": "Omurano", + "omv": "Omotic languages", + "omw": "South Tairora", + "omx": "Old Mon", + "omy": "Old Malay", + "ona": "Ona", + "onb": "Lingao", + "one": "Oneida", + "ong": "Olo", + "oni": "Onin", + "onj": "Onjob", + "onk": "Kabore One", + "onn": "Onobasulu", + "ono": "Onondaga", + "onp": "Sartang", + "onr": "Northern One", + "ons": "Ono", + "ont": "Ontenu", + "onu": "Unua", + "onw": "Old Nubian", + "onx": "Onin Based Pidgin", + "ood": "Tohono O'odham", + "oog": "Ong", + "oon": "Önge", + "oor": "Oorlams", + "oos": "Old Ossetic", + "opa": "Okpamheri", + "opk": "Kopkaka", + "opm": "Oksapmin", + "opo": "Opao", + "opt": "Opata", + "opy": "Ofayé", + "or": "Oriya (macrolanguage); Odia (macrolanguage)", + "ora": "Oroha", + "orc": "Orma", + "ore": "Orejón", + "org": "Oring", + "orh": "Oroqen", + "orn": "Orang Kanaq", + "oro": "Orokolo", + "orr": "Oruma", + "ors": "Orang Seletar", + "ort": "Adivasi Oriya", + "oru": "Ormuri", + "orv": "Old Russian", + "orw": "Oro Win", + "orx": "Oro", + "ory": "Odia (individual language); Oriya (individual language)", + "orz": "Ormu", + "os": "Ossetian; Ossetic", + "osa": "Osage", + "osc": "Oscan", + "osi": "Osing", + "osn": "Old Sundanese", + "oso": "Ososo", + "osp": "Old Spanish", + "ost": "Osatu", + "osu": "Southern One", + "osx": "Old Saxon", + "ota": "Ottoman Turkish (1500-1928)", + "otb": "Old Tibetan", + "otd": "Ot Danum", + "ote": "Mezquital Otomi", + "oti": "Oti", + "otk": "Old Turkish", + "otl": "Tilapa Otomi", + "otm": "Eastern Highland Otomi", + "otn": "Tenango Otomi", + "oto": "Otomian languages", + "otq": "Querétaro Otomi", + "otr": "Otoro", + "ots": "Estado de México Otomi", + "ott": "Temoaya Otomi", + "otu": "Otuke", + "otw": "Ottawa", + "otx": "Texcatepec Otomi", + "oty": "Old Tamil", + "otz": "Ixtenco Otomi", + "oua": "Tagargrent", + "oub": "Glio-Oubi", + "oue": "Oune", + "oui": "Old Uighur", + "oum": "Ouma", + "ovd": "Elfdalian; Övdalian", + "owi": "Owiniga", + "owl": "Old Welsh", + "oyb": "Oy", + "oyd": "Oyda", + "oym": "Wayampi", + "oyy": "Oya'oya", + "ozm": "Koonzime", + "pa": "Panjabi; Punjabi", + "paa": "Papuan languages", + "pab": "Parecís", + "pac": "Pacoh", + "pad": "Paumarí", + "pae": "Pagibete", + "paf": "Paranawát", + "pag": "Pangasinan", + "pah": "Tenharim", + "pai": "Pe", + "pak": "Parakanã", + "pal": "Pahlavi", + "pam": "Pampanga; Kapampangan", + "pao": "Northern Paiute", + "pap": "Papiamento", + "paq": "Parya", + "par": "Panamint; Timbisha", + "pas": "Papasena", + "pau": "Palauan", + "pav": "Pakaásnovos", + "paw": "Pawnee", + "pax": "Pankararé", + "pay": "Pech", + "paz": "Pankararú", + "pbb": "Páez", + "pbc": "Patamona", + "pbe": "Mezontla Popoloca", + "pbf": "Coyotepec Popoloca", + "pbg": "Paraujano", + "pbh": "E'ñapa Woromaipu", + "pbi": "Parkwa", + "pbl": "Mak (Nigeria)", + "pbm": "Puebla Mazatec", + "pbn": "Kpasam", + "pbo": "Papel", + "pbp": "Badyara", + "pbr": "Pangwa", + "pbs": "Central Pame", + "pbt": "Southern Pashto", + "pbu": "Northern Pashto", + "pbv": "Pnar", + "pby": "Pyu (Papua New Guinea)", + "pca": "Santa Inés Ahuatempan Popoloca", + "pcb": "Pear", + "pcc": "Bouyei", + "pcd": "Picard", + "pce": "Ruching Palaung", + "pcf": "Paliyan", + "pcg": "Paniya", + "pch": "Pardhan", + "pci": "Duruwa", + "pcj": "Parenga", + "pck": "Paite Chin", + "pcl": "Pardhi", + "pcm": "Nigerian Pidgin", + "pcn": "Piti", + "pcp": "Pacahuara", + "pcw": "Pyapun", + "pda": "Anam", + "pdc": "Pennsylvania German", + "pdi": "Pa Di", + "pdn": "Podena; Fedan", + "pdo": "Padoe", + "pdt": "Plautdietsch", + "pdu": "Kayan", + "pea": "Peranakan Indonesian", + "peb": "Eastern Pomo", + "ped": "Mala (Papua New Guinea)", + "pee": "Taje", + "pef": "Northeastern Pomo", + "peg": "Pengo", + "peh": "Bonan", + "pei": "Chichimeca-Jonaz", + "pej": "Northern Pomo", + "pek": "Penchal", + "pel": "Pekal", + "pem": "Phende", + "peo": "Old Persian (ca. 600-400 B.C.)", + "pep": "Kunja", + "peq": "Southern Pomo", + "pes": "Iranian Persian", + "pev": "Pémono", + "pex": "Petats", + "pey": "Petjo", + "pez": "Eastern Penan", + "pfa": "Pááfang", + "pfe": "Pere", + "pfl": "Pfaelzisch", + "pga": "Sudanese Creole Arabic", + "pgd": "Gāndhārī", + "pgg": "Pangwali", + "pgi": "Pagi", + "pgk": "Rerep", + "pgl": "Primitive Irish", + "pgn": "Paelignian", + "pgs": "Pangseng", + "pgu": "Pagu", + "pgz": "Papua New Guinean Sign Language", + "pha": "Pa-Hng", + "phd": "Phudagi", + "phg": "Phuong", + "phh": "Phukha", + "phi": "Philippine languages", + "phj": "Pahari", + "phk": "Phake", + "phl": "Phalura; Palula", + "phm": "Phimbi", + "phn": "Phoenician", + "pho": "Phunoi", + "phq": "Phana'", + "phr": "Pahari-Potwari", + "pht": "Phu Thai", + "phu": "Phuan", + "phv": "Pahlavani", + "phw": "Phangduwali", + "pi": "Pali", + "pia": "Pima Bajo", + "pib": "Yine", + "pic": "Pinji", + "pid": "Piaroa", + "pie": "Piro", + "pif": "Pingelapese", + "pig": "Pisabo", + "pih": "Pitcairn-Norfolk", + "pij": "Pijao", + "pil": "Yom", + "pim": "Powhatan", + "pin": "Piame", + "pio": "Piapoco", + "pip": "Pero", + "pir": "Piratapuyo", + "pis": "Pijin", + "pit": "Pitta Pitta", + "piu": "Pintupi-Luritja", + "piv": "Pileni; Vaeakau-Taumako", + "piw": "Pimbwe", + "pix": "Piu", + "piy": "Piya-Kwonci", + "piz": "Pije", + "pjt": "Pitjantjatjara", + "pka": "Ardhamāgadhī Prākrit", + "pkb": "Pokomo; Kipfokomo", + "pkc": "Paekche", + "pkg": "Pak-Tong", + "pkh": "Pankhu", + "pkn": "Pakanha", + "pko": "Pökoot", + "pkp": "Pukapuka", + "pkr": "Attapady Kurumba", + "pks": "Pakistan Sign Language", + "pkt": "Maleng", + "pku": "Paku", + "pl": "Polish", + "pla": "Miani", + "plb": "Polonombauk", + "plc": "Central Palawano", + "pld": "Polari", + "ple": "Palu'e", + "plf": "Central Malayo-Polynesian languages", + "plg": "Pilagá", + "plh": "Paulohi", + "plj": "Polci", + "plk": "Kohistani Shina", + "pll": "Shwe Palaung", + "pln": "Palenquero", + "plo": "Oluta Popoluca", + "plq": "Palaic", + "plr": "Palaka Senoufo", + "pls": "San Marcos Tlacoyalco Popoloca; San Marcos Tlalcoyalco Popoloca", + "plt": "Plateau Malagasy", + "plu": "Palikúr", + "plv": "Southwest Palawano", + "plw": "Brooke's Point Palawano", + "ply": "Bolyu", + "plz": "Paluan", + "pma": "Paama", + "pmb": "Pambia", + "pmd": "Pallanganmiddang", + "pme": "Pwaamei", + "pmf": "Pamona", + "pmh": "Māhārāṣṭri Prākrit", + "pmi": "Northern Pumi", + "pmj": "Southern Pumi", + "pmk": "Pamlico", + "pml": "Lingua Franca", + "pmm": "Pomo", + "pmn": "Pam", + "pmo": "Pom", + "pmq": "Northern Pame", + "pmr": "Paynamar", + "pms": "Piemontese", + "pmt": "Tuamotuan", + "pmw": "Plains Miwok", + "pmx": "Poumei Naga", + "pmy": "Papuan Malay", + "pmz": "Southern Pame", + "pna": "Punan Bah-Biau", + "pnb": "Western Panjabi", + "pnc": "Pannei", + "pnd": "Mpinda", + "pne": "Western Penan", + "png": "Pangu; Pongu", + "pnh": "Penrhyn", + "pni": "Aoheng", + "pnj": "Pinjarup", + "pnk": "Paunaka", + "pnl": "Paleni", + "pnm": "Punan Batu 1", + "pnn": "Pinai-Hagahai", + "pno": "Panobo", + "pnp": "Pancana", + "pnq": "Pana (Burkina Faso)", + "pnr": "Panim", + "pns": "Ponosakan", + "pnt": "Pontic", + "pnu": "Jiongnai Bunu", + "pnv": "Pinigura", + "pnw": "Banyjima; Panytyima", + "pnx": "Phong-Kniang", + "pny": "Pinyin", + "pnz": "Pana (Central African Republic)", + "poc": "Poqomam", + "poe": "San Juan Atzingo Popoloca", + "pof": "Poke", + "pog": "Potiguára", + "poh": "Poqomchi'", + "poi": "Highland Popoluca", + "pok": "Pokangá", + "pom": "Southeastern Pomo", + "pon": "Pohnpeian", + "poo": "Central Pomo", + "pop": "Pwapwâ", + "poq": "Texistepec Popoluca", + "pos": "Sayula Popoluca", + "pot": "Potawatomi", + "pov": "Upper Guinea Crioulo", + "pow": "San Felipe Otlaltepec Popoloca", + "pox": "Polabian", + "poy": "Pogolo", + "poz": "Malayo-Polynesian languages", + "ppe": "Papi", + "ppi": "Paipai", + "ppk": "Uma", + "ppl": "Pipil; Nicarao", + "ppm": "Papuma", + "ppn": "Papapana", + "ppo": "Folopa", + "ppp": "Pelende", + "ppq": "Pei", + "pps": "San Luís Temalacayuca Popoloca", + "ppt": "Pare", + "ppu": "Papora", + "pqa": "Pa'a", + "pqe": "Eastern Malayo-Polynesian languages", + "pqm": "Malecite-Passamaquoddy", + "pqw": "Western Malayo-Polynesian languages", + "pra": "Prakrit languages", + "prc": "Parachi", + "prd": "Parsi-Dari", + "pre": "Principense", + "prf": "Paranan", + "prg": "Prussian", + "prh": "Porohanon", + "pri": "Paicî", + "prk": "Parauk", + "prl": "Peruvian Sign Language", + "prm": "Kibiri", + "prn": "Prasuni", + "pro": "Old Provençal (to 1500); Old Occitan (to 1500)", + "prp": "Parsi", + "prq": "Ashéninka Perené", + "prr": "Puri", + "prs": "Dari; Afghan Persian", + "prt": "Phai", + "pru": "Puragi", + "prw": "Parawen", + "prx": "Purik", + "prz": "Providencia Sign Language", + "ps": "Pushto; Pashto", + "psa": "Asue Awyu", + "psc": "Iranian Sign Language; Persian Sign Language", + "psd": "Plains Indian Sign Language", + "pse": "Central Malay", + "psg": "Penang Sign Language", + "psh": "Southwest Pashai; Southwest Pashayi", + "psi": "Southeast Pashai; Southeast Pashayi", + "psl": "Puerto Rican Sign Language", + "psm": "Pauserna", + "psn": "Panasuan", + "pso": "Polish Sign Language", + "psp": "Philippine Sign Language", + "psq": "Pasi", + "psr": "Portuguese Sign Language", + "pss": "Kaulong", + "pst": "Central Pashto", + "psu": "Sauraseni Prākrit", + "psw": "Port Sandwich", + "psy": "Piscataway", + "pt": "Portuguese", + "pta": "Pai Tavytera", + "pth": "Pataxó Hã-Ha-Hãe", + "pti": "Pindiini; Wangkatha", + "ptn": "Patani", + "pto": "Zo'é", + "ptp": "Patep", + "ptq": "Pattapu", + "ptr": "Piamatsina", + "ptt": "Enrekang", + "ptu": "Bambam", + "ptv": "Port Vato", + "ptw": "Pentlatch", + "pty": "Pathiya", + "pua": "Western Highland Purepecha", + "pub": "Purum", + "puc": "Punan Merap", + "pud": "Punan Aput", + "pue": "Puelche", + "puf": "Punan Merah", + "pug": "Phuie", + "pui": "Puinave", + "puj": "Punan Tubu", + "pum": "Puma", + "puo": "Puoc", + "pup": "Pulabu", + "puq": "Puquina", + "pur": "Puruborá", + "put": "Putoh", + "puu": "Punu", + "puw": "Puluwatese", + "pux": "Puare", + "puy": "Purisimeño", + "pwa": "Pawaia", + "pwb": "Panawa", + "pwg": "Gapapaiwa", + "pwi": "Patwin", + "pwm": "Molbog", + "pwn": "Paiwan", + "pwo": "Pwo Western Karen", + "pwr": "Powari", + "pww": "Pwo Northern Karen", + "pxm": "Quetzaltepec Mixe", + "pye": "Pye Krumen", + "pym": "Fyam", + "pyn": "Poyanáwa", + "pys": "Paraguayan Sign Language; Lengua de Señas del Paraguay", + "pyu": "Puyuma", + "pyx": "Pyu (Myanmar)", + "pyy": "Pyen", + "pzh": "Pazeh", + "pzn": "Jejara Naga; Para Naga", + "qu": "Quechua", + "qua": "Quapaw", + "qub": "Huallaga Huánuco Quechua", + "quc": "K'iche'; Quiché", + "qud": "Calderón Highland Quichua", + "quf": "Lambayeque Quechua", + "qug": "Chimborazo Highland Quichua", + "quh": "South Bolivian Quechua", + "qui": "Quileute", + "quk": "Chachapoyas Quechua", + "qul": "North Bolivian Quechua", + "qum": "Sipacapense", + "qun": "Quinault", + "qup": "Southern Pastaza Quechua", + "quq": "Quinqui", + "qur": "Yanahuanca Pasco Quechua", + "qus": "Santiago del Estero Quichua", + "quv": "Sacapulteco", + "quw": "Tena Lowland Quichua", + "qux": "Yauyos Quechua", + "quy": "Ayacucho Quechua", + "quz": "Cusco Quechua", + "qva": "Ambo-Pasco Quechua", + "qvc": "Cajamarca Quechua", + "qve": "Eastern Apurímac Quechua", + "qvh": "Huamalíes-Dos de Mayo Huánuco Quechua", + "qvi": "Imbabura Highland Quichua", + "qvj": "Loja Highland Quichua", + "qvl": "Cajatambo North Lima Quechua", + "qvm": "Margos-Yarowilca-Lauricocha Quechua", + "qvn": "North Junín Quechua", + "qvo": "Napo Lowland Quechua", + "qvp": "Pacaraos Quechua", + "qvs": "San Martín Quechua", + "qvw": "Huaylla Wanca Quechua", + "qvy": "Queyu", + "qvz": "Northern Pastaza Quichua", + "qwa": "Corongo Ancash Quechua", + "qwc": "Classical Quechua", + "qwe": "Quechuan (family)", + "qwh": "Huaylas Ancash Quechua", + "qwm": "Kuman (Russia)", + "qws": "Sihuas Ancash Quechua", + "qwt": "Kwalhioqua-Tlatskanai", + "qxa": "Chiquián Ancash Quechua", + "qxc": "Chincha Quechua", + "qxh": "Panao Huánuco Quechua", + "qxl": "Salasaca Highland Quichua", + "qxn": "Northern Conchucos Ancash Quechua", + "qxo": "Southern Conchucos Ancash Quechua", + "qxp": "Puno Quechua", + "qxq": "Qashqa'i", + "qxr": "Cañar Highland Quichua", + "qxs": "Southern Qiang", + "qxt": "Santa Ana de Tusi Pasco Quechua", + "qxu": "Arequipa-La Unión Quechua", + "qxw": "Jauja Wanca Quechua", + "qya": "Quenya", + "qyp": "Quiripi", + "raa": "Dungmali", + "rab": "Camling", + "rac": "Rasawa", + "rad": "Rade", + "raf": "Western Meohang", + "rag": "Logooli; Lulogooli", + "rah": "Rabha", + "rai": "Ramoaaina", + "raj": "Rajasthani", + "rak": "Tulu-Bohuai", + "ral": "Ralte", + "ram": "Canela", + "ran": "Riantana", + "rao": "Rao", + "rap": "Rapanui", + "raq": "Saam", + "rar": "Rarotongan; Cook Islands Maori", + "ras": "Tegali", + "rat": "Razajerdi", + "rau": "Raute", + "rav": "Sampang", + "raw": "Rawang", + "rax": "Rang", + "ray": "Rapa", + "raz": "Rahambuu", + "rbb": "Rumai Palaung", + "rbk": "Northern Bontok", + "rbl": "Miraya Bikol", + "rbp": "Barababaraba", + "rcf": "Réunion Creole French", + "rdb": "Rudbari", + "rea": "Rerau", + "reb": "Rembong", + "ree": "Rejang Kayan", + "reg": "Kara (Tanzania)", + "rei": "Reli", + "rej": "Rejang", + "rel": "Rendille", + "rem": "Remo", + "ren": "Rengao", + "rer": "Rer Bare", + "res": "Reshe", + "ret": "Retta", + "rey": "Reyesano", + "rga": "Roria", + "rge": "Romano-Greek", + "rgk": "Rangkas", + "rgn": "Romagnol", + "rgr": "Resígaro", + "rgs": "Southern Roglai", + "rgu": "Ringgou", + "rhg": "Rohingya", + "rhp": "Yahang", + "ria": "Riang (India)", + "rib": "Bribri Sign Language", + "rif": "Tarifit", + "ril": "Riang Lang; Riang (Myanmar)", + "rim": "Nyaturu", + "rin": "Nungu", + "rir": "Ribun", + "rit": "Ritharrngu", + "riu": "Riung", + "rjg": "Rajong", + "rji": "Raji", + "rjs": "Rajbanshi", + "rka": "Kraol", + "rkb": "Rikbaktsa", + "rkh": "Rakahanga-Manihiki", + "rki": "Rakhine", + "rkm": "Marka", + "rkt": "Rangpuri; Kamta", + "rkw": "Arakwal", + "rm": "Romansh", + "rma": "Rama", + "rmb": "Rembarrnga", + "rmc": "Carpathian Romani", + "rmd": "Traveller Danish", + "rme": "Angloromani", + "rmf": "Kalo Finnish Romani", + "rmg": "Traveller Norwegian", + "rmh": "Murkim", + "rmi": "Lomavren", + "rmk": "Romkun", + "rml": "Baltic Romani", + "rmm": "Roma", + "rmn": "Balkan Romani", + "rmo": "Sinte Romani", + "rmp": "Rempi", + "rmq": "Caló", + "rms": "Romanian Sign Language", + "rmt": "Domari", + "rmu": "Tavringer Romani", + "rmv": "Romanova", + "rmw": "Welsh Romani", + "rmx": "Romam", + "rmy": "Vlax Romani", + "rmz": "Marma", + "rn": "Rundi", + "rnb": "Brunca Sign Language", + "rnd": "Ruund", + "rng": "Ronga", + "rnl": "Ranglong", + "rnn": "Roon", + "rnp": "Rongpo", + "rnr": "Nari Nari", + "rnw": "Rungwa", + "ro": "Romanian; Moldavian; Moldovan", + "roa": "Romance languages", + "rob": "Tae'", + "roc": "Cacgia Roglai", + "rod": "Rogo", + "roe": "Ronji", + "rof": "Rombo", + "rog": "Northern Roglai", + "rol": "Romblomanon", + "rom": "Romany", + "roo": "Rotokas", + "rop": "Kriol", + "ror": "Rongga", + "rou": "Runga", + "row": "Dela-Oenale", + "rpn": "Repanbitip", + "rpt": "Rapting", + "rri": "Ririo", + "rro": "Waima", + "rrt": "Arritinngithigh", + "rsb": "Romano-Serbian", + "rsk": "Ruthenian; Rusyn", + "rsl": "Russian Sign Language", + "rsm": "Miriwoong Sign Language", + "rsn": "Rwandan Sign Language", + "rtc": "Rungtu Chin", + "rth": "Ratahan", + "rtm": "Rotuman", + "rts": "Yurats", + "rtw": "Rathawi", + "ru": "Russian", + "rub": "Gungu", + "ruc": "Ruuli", + "rue": "Rusyn", + "ruf": "Luguru", + "rug": "Roviana", + "ruh": "Ruga", + "rui": "Rufiji", + "ruk": "Che", + "ruo": "Istro Romanian", + "rup": "Macedo-Romanian; Aromanian; Arumanian", + "ruq": "Megleno Romanian", + "rut": "Rutul", + "ruu": "Lanas Lobu", + "ruy": "Mala (Nigeria)", + "ruz": "Ruma", + "rw": "Kinyarwanda", + "rwa": "Rawo", + "rwk": "Rwa", + "rwl": "Ruwila", + "rwm": "Amba (Uganda)", + "rwo": "Rawa", + "rwr": "Marwari (India)", + "rxd": "Ngardi", + "rxw": "Karuwali; Garuwali", + "ryn": "Northern Amami-Oshima", + "rys": "Yaeyama", + "ryu": "Central Okinawan", + "rzh": "Rāziḥī", + "sa": "Sanskrit", + "saa": "Saba", + "sab": "Buglere", + "sac": "Meskwaki", + "sad": "Sandawe", + "sae": "Sabanê", + "saf": "Safaliba", + "sah": "Yakut", + "sai": "South American Indian languages", + "saj": "Sahu", + "sak": "Sake", + "sal": "Salishan languages", + "sam": "Samaritan Aramaic", + "sao": "Sause", + "saq": "Samburu", + "sar": "Saraveca", + "sas": "Sasak", + "sat": "Santali", + "sau": "Saleman", + "sav": "Saafi-Saafi", + "saw": "Sawi", + "sax": "Sa", + "say": "Saya", + "saz": "Saurashtra", + "sba": "Ngambay", + "sbb": "Simbo", + "sbc": "Kele (Papua New Guinea)", + "sbd": "Southern Samo", + "sbe": "Saliba", + "sbf": "Chabu; Shabo", + "sbg": "Seget", + "sbh": "Sori-Harengan", + "sbi": "Seti", + "sbj": "Surbakhal", + "sbk": "Safwa", + "sbl": "Botolan Sambal", + "sbm": "Sagala", + "sbn": "Sindhi Bhil", + "sbo": "Sabüm", + "sbp": "Sangu (Tanzania)", + "sbq": "Sileibi", + "sbr": "Sembakung Murut", + "sbs": "Subiya", + "sbt": "Kimki", + "sbu": "Stod Bhoti", + "sbv": "Sabine", + "sbw": "Simba", + "sbx": "Seberuang", + "sby": "Soli", + "sbz": "Sara Kaba", + "sc": "Sardinian", + "scb": "Chut", + "sce": "Dongxiang", + "scf": "San Miguel Creole French", + "scg": "Sanggau", + "sch": "Sakachep", + "sci": "Sri Lankan Creole Malay", + "sck": "Sadri", + "scl": "Shina", + "scn": "Sicilian", + "sco": "Scots", + "scp": "Hyolmo; Helambu Sherpa", + "scq": "Sa'och", + "scs": "North Slavey", + "sct": "Southern Katang", + "scu": "Shumcho", + "scv": "Sheni", + "scw": "Sha", + "scx": "Sicel", + "sd": "Sindhi", + "sda": "Toraja-Sa'dan", + "sdb": "Shabak", + "sdc": "Sassarese Sardinian", + "sde": "Surubu", + "sdf": "Sarli", + "sdg": "Savi", + "sdh": "Southern Kurdish", + "sdj": "Suundi", + "sdk": "Sos Kundi", + "sdl": "Saudi Arabian Sign Language", + "sdn": "Gallurese Sardinian", + "sdo": "Bukar-Sadung Bidayuh", + "sdp": "Sherdukpen", + "sdq": "Semandang", + "sdr": "Oraon Sadri", + "sds": "Sened", + "sdt": "Shuadit", + "sdu": "Sarudu", + "sdv": "Eastern Sudanic languages", + "sdx": "Sibu Melanau", + "sdz": "Sallands", + "se": "Northern Sami", + "sea": "Semai", + "seb": "Shempire Senoufo", + "sec": "Sechelt", + "sed": "Sedang", + "see": "Seneca", + "sef": "Cebaara Senoufo", + "seg": "Segeju", + "seh": "Sena", + "sei": "Seri", + "sej": "Sene", + "sek": "Sekani", + "sel": "Selkup", + "sem": "Semitic languages", + "sen": "Nanerigé Sénoufo", + "seo": "Suarmin", + "sep": "Sìcìté Sénoufo", + "seq": "Senara Sénoufo", + "ser": "Serrano", + "ses": "Koyraboro Senni Songhai", + "set": "Sentani", + "seu": "Serui-Laut", + "sev": "Nyarafolo Senoufo", + "sew": "Sewa Bay", + "sey": "Secoya", + "sez": "Senthang Chin", + "sfb": "Langue des signes de Belgique Francophone; French Belgian Sign Language", + "sfe": "Eastern Subanen", + "sfm": "Small Flowery Miao", + "sfs": "South African Sign Language", + "sfw": "Sehwi", + "sg": "Sango", + "sga": "Old Irish (to 900)", + "sgb": "Mag-antsi Ayta", + "sgc": "Kipsigis", + "sgd": "Surigaonon", + "sge": "Segai", + "sgg": "Swiss-German Sign Language", + "sgh": "Shughni", + "sgi": "Suga", + "sgj": "Surgujia", + "sgk": "Sangkong", + "sgm": "Singa", + "sgn": "Sign languages", + "sgp": "Singpho", + "sgr": "Sangisari", + "sgs": "Samogitian", + "sgt": "Brokpake", + "sgu": "Salas", + "sgw": "Sebat Bet Gurage", + "sgx": "Sierra Leone Sign Language", + "sgy": "Sanglechi", + "sgz": "Sursurunga", + "sh": "Serbo-Croatian", + "sha": "Shall-Zwall", + "shb": "Ninam", + "shc": "Sonde", + "shd": "Kundal Shahi", + "she": "Sheko", + "shg": "Shua", + "shh": "Shoshoni", + "shi": "Tachelhit", + "shj": "Shatt", + "shk": "Shilluk", + "shl": "Shendu", + "shm": "Shahrudi", + "shn": "Shan", + "sho": "Shanga", + "shp": "Shipibo-Conibo", + "shq": "Sala", + "shr": "Shi", + "shs": "Shuswap", + "sht": "Shasta", + "shu": "Chadian Arabic", + "shv": "Shehri", + "shw": "Shwai", + "shx": "She", + "shy": "Tachawit", + "shz": "Syenara Senoufo", + "si": "Sinhala; Sinhalese", + "sia": "Akkala Sami", + "sib": "Sebop", + "sid": "Sidamo", + "sie": "Simaa", + "sif": "Siamou", + "sig": "Paasaal", + "sih": "Zire; Sîshëë", + "sii": "Shom Peng", + "sij": "Numbami", + "sik": "Sikiana", + "sil": "Tumulung Sisaala", + "sim": "Mende (Papua New Guinea)", + "sio": "Siouan languages", + "sip": "Sikkimese", + "siq": "Sonia", + "sir": "Siri", + "sis": "Siuslaw", + "sit": "Sino-Tibetan languages", + "siu": "Sinagen", + "siv": "Sumariup", + "siw": "Siwai", + "six": "Sumau", + "siy": "Sivandi", + "siz": "Siwi", + "sja": "Epena", + "sjb": "Sajau Basap", + "sjd": "Kildin Sami", + "sje": "Pite Sami", + "sjg": "Assangori", + "sjk": "Kemi Sami", + "sjl": "Sajalong; Miji", + "sjm": "Mapun", + "sjn": "Sindarin", + "sjo": "Xibe", + "sjp": "Surjapuri", + "sjr": "Siar-Lak", + "sjs": "Senhaja De Srair", + "sjt": "Ter Sami", + "sju": "Ume Sami", + "sjw": "Shawnee", + "sk": "Slovak", + "ska": "Skagit", + "skb": "Saek", + "skc": "Ma Manda", + "skd": "Southern Sierra Miwok", + "ske": "Seke (Vanuatu)", + "skf": "Sakirabiá", + "skg": "Sakalava Malagasy", + "skh": "Sikule", + "ski": "Sika", + "skj": "Seke (Nepal)", + "skm": "Kutong", + "skn": "Kolibugan Subanon", + "sko": "Seko Tengah", + "skp": "Sekapan", + "skq": "Sininkere", + "skr": "Saraiki; Seraiki", + "sks": "Maia", + "skt": "Sakata", + "sku": "Sakao", + "skv": "Skou", + "skw": "Skepi Creole Dutch", + "skx": "Seko Padang", + "sky": "Sikaiana", + "skz": "Sekar", + "sl": "Slovenian", + "sla": "Slavic languages", + "slc": "Sáliba", + "sld": "Sissala", + "sle": "Sholaga", + "slf": "Swiss-Italian Sign Language", + "slg": "Selungai Murut", + "slh": "Southern Puget Sound Salish", + "sli": "Lower Silesian", + "slj": "Salumá", + "sll": "Salt-Yui", + "slm": "Pangutaran Sama", + "sln": "Salinan", + "slp": "Lamaholot", + "slq": "Salchuq", + "slr": "Salar", + "sls": "Singapore Sign Language", + "slt": "Sila", + "slu": "Selaru", + "slw": "Sialum", + "slx": "Salampasu", + "sly": "Selayar", + "slz": "Ma'ya", + "sm": "Samoan", + "sma": "Southern Sami", + "smb": "Simbari", + "smc": "Som", + "smf": "Auwe", + "smg": "Simbali", + "smh": "Samei", + "smi": "Sami languages", + "smj": "Lule Sami", + "smk": "Bolinao", + "sml": "Central Sama", + "smm": "Musasa", + "smn": "Inari Sami", + "smp": "Samaritan", + "smq": "Samo", + "smr": "Simeulue", + "sms": "Skolt Sami", + "smt": "Simte", + "smu": "Somray", + "smv": "Samvedi", + "smw": "Sumbawa", + "smx": "Samba", + "smy": "Semnani", + "smz": "Simeku", + "sn": "Shona", + "snc": "Sinaugoro", + "sne": "Bau Bidayuh", + "snf": "Noon", + "sng": "Sanga (Democratic Republic of Congo)", + "sni": "Sensi", + "snj": "Riverain Sango", + "snk": "Soninke", + "snl": "Sangil", + "snm": "Southern Ma'di", + "snn": "Siona", + "sno": "Snohomish", + "snp": "Siane", + "snq": "Sangu (Gabon)", + "snr": "Sihan", + "sns": "South West Bay; Nahavaq", + "snu": "Senggi; Viid", + "snv": "Sa'ban", + "snw": "Selee", + "snx": "Sam", + "sny": "Saniyo-Hiyewe", + "snz": "Kou", + "so": "Somali", + "soa": "Thai Song", + "sob": "Sobei", + "soc": "So (Democratic Republic of Congo)", + "sod": "Songoora", + "soe": "Songomeno", + "sog": "Sogdian", + "soh": "Aka", + "soi": "Sonha", + "soj": "Soi", + "sok": "Sokoro", + "sol": "Solos", + "son": "Songhai languages", + "soo": "Songo", + "sop": "Songe", + "soq": "Kanasi", + "sor": "Somrai", + "sos": "Seeku", + "sou": "Southern Thai", + "sov": "Sonsorol", + "sow": "Sowanda", + "sox": "Swo", + "soy": "Miyobe", + "soz": "Temi", + "spb": "Sepa (Indonesia)", + "spc": "Sapé", + "spd": "Saep", + "spe": "Sepa (Papua New Guinea)", + "spg": "Sian", + "spi": "Saponi", + "spk": "Sengo", + "spl": "Selepet", + "spm": "Akukem", + "spn": "Sanapaná", + "spo": "Spokane", + "spp": "Supyire Senoufo", + "spq": "Loreto-Ucayali Spanish", + "spr": "Saparua", + "sps": "Saposa", + "spt": "Spiti Bhoti", + "spu": "Sapuan", + "spv": "Sambalpuri; Kosli", + "spx": "South Picene", + "spy": "Sabaot", + "sq": "Albanian", + "sqa": "Shama-Sambuga", + "sqh": "Shau", + "sqj": "Albanian languages", + "sqk": "Albanian Sign Language", + "sqm": "Suma", + "sqn": "Susquehannock", + "sqo": "Sorkhei", + "sqq": "Sou", + "sqr": "Siculo Arabic", + "sqs": "Sri Lankan Sign Language", + "sqt": "Soqotri", + "squ": "Squamish", + "sqx": "Kufr Qassem Sign Language (KQSL)", + "sr": "Serbian", + "sra": "Saruga", + "srb": "Sora", + "src": "Logudorese Sardinian", + "sre": "Sara", + "srf": "Nafi", + "srg": "Sulod", + "srh": "Sarikoli", + "sri": "Siriano", + "srk": "Serudung Murut", + "srl": "Isirawa", + "srm": "Saramaccan", + "srn": "Sranan Tongo", + "sro": "Campidanese Sardinian", + "srq": "Sirionó", + "srr": "Serer", + "srs": "Sarsi", + "srt": "Sauri", + "sru": "Suruí", + "srv": "Southern Sorsoganon", + "srw": "Serua", + "srx": "Sirmauri", + "sry": "Sera", + "srz": "Shahmirzadi", + "ss": "Swati", + "ssa": "Nilo-Saharan languages", + "ssb": "Southern Sama", + "ssc": "Suba-Simbiti", + "ssd": "Siroi", + "sse": "Balangingi; Bangingih Sama", + "ssf": "Thao", + "ssg": "Seimat", + "ssh": "Shihhi Arabic", + "ssi": "Sansi", + "ssj": "Sausi", + "ssk": "Sunam", + "ssl": "Western Sisaala", + "ssm": "Semnam", + "ssn": "Waata", + "sso": "Sissano", + "ssp": "Spanish Sign Language", + "ssq": "So'a", + "ssr": "Swiss-French Sign Language", + "sss": "Sô", + "sst": "Sinasina", + "ssu": "Susuami", + "ssv": "Shark Bay", + "ssx": "Samberigi", + "ssy": "Saho", + "ssz": "Sengseng", + "st": "Southern Sotho", + "sta": "Settla", + "stb": "Northern Subanen", + "std": "Sentinel", + "ste": "Liana-Seti", + "stf": "Seta", + "stg": "Trieng", + "sth": "Shelta", + "sti": "Bulo Stieng", + "stj": "Matya Samo", + "stk": "Arammba", + "stl": "Stellingwerfs", + "stm": "Setaman", + "stn": "Owa", + "sto": "Stoney", + "stp": "Southeastern Tepehuan", + "stq": "Saterfriesisch", + "str": "Straits Salish", + "sts": "Shumashti", + "stt": "Budeh Stieng", + "stu": "Samtao", + "stv": "Silt'e", + "stw": "Satawalese", + "sty": "Siberian Tatar", + "su": "Sundanese", + "sua": "Sulka", + "sub": "Suku", + "suc": "Western Subanon", + "sue": "Suena", + "sug": "Suganga", + "sui": "Suki", + "suj": "Shubi", + "suk": "Sukuma", + "suo": "Bouni", + "suq": "Tirmaga-Chai Suri; Suri", + "sur": "Mwaghavul", + "sus": "Susu", + "sut": "Subtiaba", + "suv": "Puroik", + "suw": "Sumbwa", + "sux": "Sumerian", + "suy": "Suyá", + "suz": "Sunwar", + "sv": "Swedish", + "sva": "Svan", + "svb": "Ulau-Suain", + "svc": "Vincentian Creole English", + "sve": "Serili", + "svk": "Slovakian Sign Language", + "svm": "Slavomolisano", + "svs": "Savosavo", + "svx": "Skalvian", + "sw": "Swahili (macrolanguage)", + "swb": "Maore Comorian", + "swc": "Congo Swahili", + "swf": "Sere", + "swg": "Swabian", + "swh": "Swahili (individual language); Kiswahili", + "swi": "Sui", + "swj": "Sira", + "swk": "Malawi Sena", + "swl": "Swedish Sign Language", + "swm": "Samosa", + "swn": "Sawknah", + "swo": "Shanenawa", + "swp": "Suau", + "swq": "Sharwa", + "swr": "Saweru", + "sws": "Seluwasan", + "swt": "Sawila", + "swu": "Suwawa", + "swv": "Shekhawati", + "sww": "Sowa", + "swx": "Suruahá", + "swy": "Sarua", + "sxb": "Suba", + "sxc": "Sicanian", + "sxe": "Sighu", + "sxg": "Shuhi; Shixing", + "sxk": "Southern Kalapuya", + "sxl": "Selian", + "sxm": "Samre", + "sxn": "Sangir", + "sxo": "Sorothaptic", + "sxr": "Saaroa", + "sxs": "Sasaru", + "sxu": "Upper Saxon", + "sxw": "Saxwe Gbe", + "sya": "Siang", + "syb": "Central Subanen", + "syc": "Classical Syriac", + "syd": "Samoyedic languages", + "syi": "Seki", + "syk": "Sukur", + "syl": "Sylheti", + "sym": "Maya Samo", + "syn": "Senaya", + "syo": "Suoy", + "syr": "Syriac", + "sys": "Sinyar", + "syw": "Kagate", + "syx": "Samay", + "syy": "Al-Sayyid Bedouin Sign Language", + "sza": "Semelai", + "szb": "Ngalum", + "szc": "Semaq Beri", + "szd": "Seru", + "sze": "Seze", + "szg": "Sengele", + "szl": "Silesian", + "szn": "Sula", + "szp": "Suabo", + "szs": "Solomon Islands Sign Language", + "szv": "Isu (Fako Division)", + "szw": "Sawai", + "szy": "Sakizaya", + "ta": "Tamil", + "taa": "Lower Tanana", + "tab": "Tabassaran", + "tac": "Lowland Tarahumara", + "tad": "Tause", + "tae": "Tariana", + "taf": "Tapirapé", + "tag": "Tagoi", + "tai": "Tai languages", + "taj": "Eastern Tamang", + "tak": "Tala", + "tal": "Tal", + "tan": "Tangale", + "tao": "Yami", + "tap": "Taabwa", + "taq": "Tamasheq", + "tar": "Central Tarahumara", + "tas": "Tay Boi", + "tau": "Upper Tanana", + "tav": "Tatuyo", + "taw": "Tai", + "tax": "Tamki", + "tay": "Atayal", + "taz": "Tocho", + "tba": "Aikanã", + "tbc": "Takia", + "tbd": "Kaki Ae", + "tbe": "Tanimbili", + "tbf": "Mandara", + "tbg": "North Tairora", + "tbh": "Dharawal; Thurawal", + "tbi": "Gaam", + "tbj": "Tiang", + "tbk": "Calamian Tagbanwa", + "tbl": "Tboli", + "tbm": "Tagbu", + "tbn": "Barro Negro Tunebo", + "tbo": "Tawala", + "tbp": "Taworta; Diebroud", + "tbq": "Tibeto-Burman languages", + "tbr": "Tumtum", + "tbs": "Tanguat", + "tbt": "Tembo (Kitembo)", + "tbu": "Tubar", + "tbv": "Tobo", + "tbw": "Tagbanwa", + "tbx": "Kapin", + "tby": "Tabaru", + "tbz": "Ditammari", + "tca": "Ticuna", + "tcb": "Tanacross", + "tcc": "Datooga", + "tcd": "Tafi", + "tce": "Southern Tutchone", + "tcf": "Malinaltepec Me'phaa; Malinaltepec Tlapanec", + "tcg": "Tamagario", + "tch": "Turks And Caicos Creole English", + "tci": "Wára", + "tck": "Tchitchege", + "tcl": "Taman (Myanmar)", + "tcm": "Tanahmerah", + "tcn": "Tichurong", + "tco": "Taungyo", + "tcp": "Tawr Chin", + "tcq": "Kaiy", + "tcs": "Torres Strait Creole; Yumplatok", + "tct": "T'en", + "tcu": "Southeastern Tarahumara", + "tcw": "Tecpatlán Totonac", + "tcx": "Toda", + "tcy": "Tulu", + "tcz": "Thado Chin", + "tda": "Tagdal", + "tdb": "Panchpargania", + "tdc": "Emberá-Tadó", + "tdd": "Tai Nüa", + "tde": "Tiranige Diga Dogon", + "tdf": "Talieng", + "tdg": "Western Tamang", + "tdh": "Thulung", + "tdi": "Tomadino", + "tdj": "Tajio", + "tdk": "Tambas", + "tdl": "Sur", + "tdm": "Taruma", + "tdn": "Tondano", + "tdo": "Teme", + "tdq": "Tita", + "tdr": "Todrah", + "tds": "Doutai", + "tdt": "Tetun Dili", + "tdv": "Toro", + "tdx": "Tandroy-Mahafaly Malagasy", + "tdy": "Tadyawan", + "te": "Telugu", + "tea": "Temiar", + "teb": "Tetete", + "tec": "Terik", + "ted": "Tepo Krumen", + "tee": "Huehuetla Tepehua", + "tef": "Teressa", + "teg": "Teke-Tege", + "teh": "Tehuelche", + "tei": "Torricelli", + "tek": "Ibali Teke", + "tem": "Timne", + "ten": "Tama (Colombia)", + "teo": "Teso", + "tep": "Tepecano", + "teq": "Temein", + "ter": "Tereno", + "tes": "Tengger", + "tet": "Tetum", + "teu": "Soo", + "tev": "Teor", + "tew": "Tewa (USA)", + "tex": "Tennet", + "tey": "Tulishi", + "tez": "Tetserret", + "tfi": "Tofin Gbe", + "tfn": "Tanaina", + "tfo": "Tefaro", + "tfr": "Teribe", + "tft": "Ternate", + "tg": "Tajik", + "tga": "Sagalla", + "tgb": "Tobilung", + "tgc": "Tigak", + "tgd": "Ciwogai", + "tge": "Eastern Gorkha Tamang", + "tgf": "Chalikha", + "tgh": "Tobagonian Creole English", + "tgi": "Lawunuia", + "tgj": "Tagin", + "tgn": "Tandaganon", + "tgo": "Sudest", + "tgp": "Tangoa", + "tgq": "Tring", + "tgr": "Tareng", + "tgs": "Nume", + "tgt": "Central Tagbanwa", + "tgu": "Tanggu", + "tgv": "Tingui-Boto", + "tgw": "Tagwana Senoufo", + "tgx": "Tagish", + "tgy": "Togoyo", + "tgz": "Tagalaka", + "th": "Thai", + "thd": "Kuuk Thaayorre; Thayore", + "the": "Chitwania Tharu", + "thf": "Thangmi", + "thh": "Northern Tarahumara", + "thi": "Tai Long", + "thk": "Tharaka; Kitharaka", + "thl": "Dangaura Tharu", + "thm": "Aheu", + "thn": "Thachanadan", + "thp": "Thompson", + "thq": "Kochila Tharu", + "thr": "Rana Tharu", + "ths": "Thakali", + "tht": "Tahltan", + "thu": "Thuri", + "thv": "Tahaggart Tamahaq", + "thy": "Tha", + "thz": "Tayart Tamajeq", + "ti": "Tigrinya", + "tia": "Tidikelt Tamazight", + "tic": "Tira", + "tif": "Tifal", + "tig": "Tigre", + "tih": "Timugon Murut", + "tii": "Tiene", + "tij": "Tilung", + "tik": "Tikar", + "til": "Tillamook", + "tim": "Timbe", + "tin": "Tindi", + "tio": "Teop", + "tip": "Trimuris", + "tiq": "Tiéfo", + "tis": "Masadiit Itneg", + "tit": "Tinigua", + "tiu": "Adasen", + "tiv": "Tiv", + "tiw": "Tiwi", + "tix": "Southern Tiwa", + "tiy": "Tiruray", + "tiz": "Tai Hongjin", + "tja": "Tajuasohn", + "tjg": "Tunjung", + "tji": "Northern Tujia", + "tjj": "Tjungundji", + "tjl": "Tai Laing", + "tjm": "Timucua", + "tjn": "Tonjon", + "tjo": "Temacine Tamazight", + "tjp": "Tjupany", + "tjs": "Southern Tujia", + "tju": "Tjurruru", + "tjw": "Djabwurrung", + "tk": "Turkmen", + "tka": "Truká", + "tkb": "Buksa", + "tkd": "Tukudede", + "tke": "Takwane", + "tkf": "Tukumanféd", + "tkg": "Tesaka Malagasy", + "tkl": "Tokelau", + "tkm": "Takelma", + "tkn": "Toku-No-Shima", + "tkp": "Tikopia", + "tkq": "Tee", + "tkr": "Tsakhur", + "tks": "Takestani", + "tkt": "Kathoriya Tharu", + "tku": "Upper Necaxa Totonac", + "tkv": "Mur Pano", + "tkw": "Teanu", + "tkx": "Tangko", + "tkz": "Takua", + "tl": "Tagalog", + "tla": "Southwestern Tepehuan", + "tlb": "Tobelo", + "tlc": "Yecuatla Totonac", + "tld": "Talaud", + "tlf": "Telefol", + "tlg": "Tofanma", + "tlh": "Klingon; tlhIngan Hol", + "tli": "Tlingit", + "tlj": "Talinga-Bwisi", + "tlk": "Taloki", + "tll": "Tetela", + "tlm": "Tolomako", + "tln": "Talondo'", + "tlo": "Talodi", + "tlp": "Filomena Mata-Coahuitlán Totonac", + "tlq": "Tai Loi", + "tlr": "Talise", + "tls": "Tambotalo", + "tlt": "Sou Nama; Teluti", + "tlu": "Tulehu", + "tlv": "Taliabu", + "tlx": "Khehek", + "tly": "Talysh", + "tma": "Tama (Chad)", + "tmb": "Katbol; Avava", + "tmc": "Tumak", + "tmd": "Haruai", + "tme": "Tremembé", + "tmf": "Toba-Maskoy", + "tmg": "Ternateño", + "tmh": "Tamashek", + "tmi": "Tutuba", + "tmj": "Samarokena", + "tmk": "Northwestern Tamang", + "tml": "Tamnim Citak", + "tmm": "Tai Thanh", + "tmn": "Taman (Indonesia)", + "tmo": "Temoq", + "tmq": "Tumleo", + "tmr": "Jewish Babylonian Aramaic (ca. 200-1200 CE)", + "tms": "Tima", + "tmt": "Tasmate", + "tmu": "Iau", + "tmv": "Tembo (Motembo)", + "tmw": "Temuan", + "tmy": "Tami", + "tmz": "Tamanaku", + "tn": "Tswana", + "tna": "Tacana", + "tnb": "Western Tunebo", + "tnc": "Tanimuca-Retuarã", + "tnd": "Angosturas Tunebo", + "tng": "Tobanga", + "tnh": "Maiani", + "tni": "Tandia", + "tnk": "Kwamera", + "tnl": "Lenakel", + "tnm": "Tabla", + "tnn": "North Tanna", + "tno": "Toromono", + "tnp": "Whitesands", + "tnq": "Taino", + "tnr": "Ménik", + "tns": "Tenis", + "tnt": "Tontemboan", + "tnu": "Tay Khang", + "tnv": "Tangchangya", + "tnw": "Tonsawang", + "tnx": "Tanema", + "tny": "Tongwe", + "tnz": "Ten'edn", + "to": "Tonga (Tonga Islands)", + "tob": "Toba", + "toc": "Coyutla Totonac", + "tod": "Toma", + "tof": "Gizrra", + "tog": "Tonga (Nyasa)", + "toh": "Gitonga", + "toi": "Tonga (Zambia)", + "toj": "Tojolabal", + "tok": "Toki Pona", + "tol": "Tolowa", + "tom": "Tombulu", + "too": "Xicotepec De Juárez Totonac", + "top": "Papantla Totonac", + "toq": "Toposa", + "tor": "Togbo-Vara Banda", + "tos": "Highland Totonac", + "tou": "Tho", + "tov": "Upper Taromi", + "tow": "Jemez", + "tox": "Tobian", + "toy": "Topoiyo", + "toz": "To", + "tpa": "Taupota", + "tpc": "Azoyú Me'phaa; Azoyú Tlapanec", + "tpe": "Tippera", + "tpf": "Tarpia", + "tpg": "Kula", + "tpi": "Tok Pisin", + "tpj": "Tapieté", + "tpk": "Tupinikin", + "tpl": "Tlacoapa Me'phaa; Tlacoapa Tlapanec", + "tpm": "Tampulma", + "tpn": "Tupinambá", + "tpo": "Tai Pao", + "tpp": "Pisaflores Tepehua", + "tpq": "Tukpa", + "tpr": "Tuparí", + "tpt": "Tlachichilco Tepehua", + "tpu": "Tampuan", + "tpv": "Tanapag", + "tpw": "Tupí", + "tpx": "Acatepec Me'phaa; Acatepec Tlapanec", + "tpy": "Trumai", + "tpz": "Tinputz", + "tqb": "Tembé", + "tql": "Lehali", + "tqm": "Turumsa", + "tqn": "Tenino", + "tqo": "Toaripi", + "tqp": "Tomoip", + "tqq": "Tunni", + "tqr": "Torona", + "tqt": "Western Totonac", + "tqu": "Touo", + "tqw": "Tonkawa", + "tr": "Turkish", + "tra": "Tirahi", + "trb": "Terebu", + "trc": "Copala Triqui", + "trd": "Turi", + "tre": "East Tarangan", + "trf": "Trinidadian Creole English", + "trg": "Lishán Didán", + "trh": "Turaka", + "tri": "Trió", + "trj": "Toram", + "trk": "Turkic languages", + "trl": "Traveller Scottish", + "trm": "Tregami", + "trn": "Trinitario", + "tro": "Tarao Naga", + "trp": "Kok Borok", + "trq": "San Martín Itunyoso Triqui", + "trr": "Taushiro", + "trs": "Chicahuaxtla Triqui", + "trt": "Tunggare", + "tru": "Turoyo; Surayt", + "trv": "Sediq; Seediq; Taroko", + "trw": "Torwali", + "trx": "Tringgus-Sembaan Bidayuh", + "try": "Turung", + "trz": "Torá", + "ts": "Tsonga", + "tsa": "Tsaangi", + "tsb": "Tsamai", + "tsc": "Tswa", + "tsd": "Tsakonian", + "tse": "Tunisian Sign Language", + "tsg": "Tausug", + "tsh": "Tsuvan", + "tsi": "Tsimshian", + "tsj": "Tshangla", + "tsk": "Tseku", + "tsl": "Ts'ün-Lao", + "tsm": "Turkish Sign Language; Türk İşaret Dili", + "tsp": "Northern Toussian", + "tsq": "Thai Sign Language", + "tsr": "Akei", + "tss": "Taiwan Sign Language", + "tst": "Tondi Songway Kiini", + "tsu": "Tsou", + "tsv": "Tsogo", + "tsw": "Tsishingini", + "tsx": "Mubami", + "tsy": "Tebul Sign Language", + "tsz": "Purepecha", + "tt": "Tatar", + "tta": "Tutelo", + "ttb": "Gaa", + "ttc": "Tektiteko", + "ttd": "Tauade", + "tte": "Bwanabwana", + "ttf": "Tuotomb", + "ttg": "Tutong", + "tth": "Upper Ta'oih", + "tti": "Tobati", + "ttj": "Tooro", + "ttk": "Totoro", + "ttl": "Totela", + "ttm": "Northern Tutchone", + "ttn": "Towei", + "tto": "Lower Ta'oih", + "ttp": "Tombelala", + "ttq": "Tawallammat Tamajaq", + "ttr": "Tera", + "tts": "Northeastern Thai", + "ttt": "Muslim Tat", + "ttu": "Torau", + "ttv": "Titan", + "ttw": "Long Wat", + "tty": "Sikaritai", + "ttz": "Tsum", + "tua": "Wiarumus", + "tub": "Tübatulabal", + "tuc": "Mutu", + "tud": "Tuxá", + "tue": "Tuyuca", + "tuf": "Central Tunebo", + "tug": "Tunia", + "tuh": "Taulil", + "tui": "Tupuri", + "tuj": "Tugutil", + "tul": "Tula", + "tum": "Tumbuka", + "tun": "Tunica", + "tuo": "Tucano", + "tup": "Tupi languages", + "tuq": "Tedaga", + "tus": "Tuscarora", + "tut": "Altaic languages", + "tuu": "Tututni", + "tuv": "Turkana", + "tuw": "Tungus languages", + "tux": "Tuxináwa", + "tuy": "Tugen", + "tuz": "Turka", + "tva": "Vaghua", + "tvd": "Tsuvadi", + "tve": "Te'un", + "tvk": "Southeast Ambrym", + "tvl": "Tuvalu", + "tvm": "Tela-Masbuar", + "tvn": "Tavoyan", + "tvo": "Tidore", + "tvs": "Taveta", + "tvt": "Tutsa Naga", + "tvu": "Tunen", + "tvw": "Sedoa", + "tvx": "Taivoan", + "tvy": "Timor Pidgin", + "tw": "Twi", + "twa": "Twana", + "twb": "Western Tawbuid", + "twc": "Teshenawa", + "twd": "Twents", + "twe": "Tewa (Indonesia)", + "twf": "Northern Tiwa", + "twg": "Tereweng", + "twh": "Tai Dón", + "twl": "Tawara", + "twm": "Tawang Monpa", + "twn": "Twendi", + "two": "Tswapong", + "twp": "Ere", + "twq": "Tasawaq", + "twr": "Southwestern Tarahumara", + "twt": "Turiwára", + "twu": "Termanu", + "tww": "Tuwari", + "twx": "Tewe", + "twy": "Tawoyan", + "txa": "Tombonuo", + "txb": "Tokharian B", + "txc": "Tsetsaut", + "txe": "Totoli", + "txg": "Tangut", + "txh": "Thracian", + "txi": "Ikpeng", + "txj": "Tarjumo", + "txm": "Tomini", + "txn": "West Tarangan", + "txo": "Toto", + "txq": "Tii", + "txr": "Tartessian", + "txs": "Tonsea", + "txt": "Citak", + "txu": "Kayapó", + "txx": "Tatana", + "txy": "Tanosy Malagasy", + "ty": "Tahitian", + "tya": "Tauya", + "tye": "Kyanga", + "tyh": "O'du", + "tyi": "Teke-Tsaayi", + "tyj": "Tai Do; Tai Yo", + "tyl": "Thu Lao", + "tyn": "Kombai", + "typ": "Thaypan", + "tyr": "Tai Daeng", + "tys": "Tày Sa Pa", + "tyt": "Tày Tac", + "tyu": "Kua", + "tyv": "Tuvinian", + "tyx": "Teke-Tyee", + "tyy": "Tiyaa", + "tyz": "Tày", + "tza": "Tanzanian Sign Language", + "tzh": "Tzeltal", + "tzj": "Tz'utujil", + "tzl": "Talossan", + "tzm": "Central Atlas Tamazight", + "tzn": "Tugun", + "tzo": "Tzotzil", + "tzx": "Tabriak", + "uam": "Uamué", + "uan": "Kuan", + "uar": "Tairuma", + "uba": "Ubang", + "ubi": "Ubi", + "ubl": "Buhi'non Bikol", + "ubr": "Ubir", + "ubu": "Umbu-Ungu", + "uby": "Ubykh", + "uda": "Uda", + "ude": "Udihe", + "udg": "Muduga", + "udi": "Udi", + "udj": "Ujir", + "udl": "Wuzlam", + "udm": "Udmurt", + "udu": "Uduk", + "ues": "Kioko", + "ufi": "Ufim", + "ug": "Uighur; Uyghur", + "uga": "Ugaritic", + "ugb": "Kuku-Ugbanh", + "uge": "Ughele", + "ugh": "Kubachi", + "ugn": "Ugandan Sign Language", + "ugo": "Ugong", + "ugy": "Uruguayan Sign Language", + "uha": "Uhami", + "uhn": "Damal", + "uis": "Uisai", + "uiv": "Iyive", + "uji": "Tanjijili", + "uk": "Ukrainian", + "uka": "Kaburi", + "ukg": "Ukuriguma", + "ukh": "Ukhwejo", + "uki": "Kui (India)", + "ukk": "Muak Sa-aak", + "ukl": "Ukrainian Sign Language", + "ukp": "Ukpe-Bayobiri", + "ukq": "Ukwa", + "uks": "Urubú-Kaapor Sign Language; Kaapor Sign Language", + "uku": "Ukue", + "ukv": "Kuku", + "ukw": "Ukwuani-Aboh-Ndoni", + "uky": "Kuuk-Yak", + "ula": "Fungwa", + "ulb": "Ulukwumi", + "ulc": "Ulch", + "ule": "Lule", + "ulf": "Usku; Afra", + "uli": "Ulithian", + "ulk": "Meriam Mir", + "ull": "Ullatan", + "ulm": "Ulumanda'", + "uln": "Unserdeutsch", + "ulu": "Uma' Lung", + "ulw": "Ulwa", + "uma": "Umatilla", + "umb": "Umbundu", + "umc": "Marrucinian", + "umd": "Umbindhamu", + "umg": "Morrobalama; Umbuygamu", + "umi": "Ukit", + "umm": "Umon", + "umn": "Makyan Naga", + "umo": "Umotína", + "ump": "Umpila", + "umr": "Umbugarla", + "ums": "Pendau", + "umu": "Munsee", + "una": "North Watut", + "und": "Undetermined", + "une": "Uneme", + "ung": "Ngarinyin", + "uni": "Uni", + "unk": "Enawené-Nawé", + "unm": "Unami", + "unn": "Kurnai", + "unr": "Mundari", + "unu": "Unubahe", + "unx": "Munda", + "unz": "Unde Kaili", + "uon": "Kulon", + "upi": "Umeda", + "upv": "Uripiv-Wala-Rano-Atchin", + "ur": "Urdu", + "ura": "Urarina", + "urb": "Urubú-Kaapor; Kaapor", + "urc": "Urningangg", + "ure": "Uru", + "urf": "Uradhi", + "urg": "Urigina", + "urh": "Urhobo", + "uri": "Urim", + "urj": "Uralic languages", + "urk": "Urak Lawoi'", + "url": "Urali", + "urm": "Urapmin", + "urn": "Uruangnirin", + "uro": "Ura (Papua New Guinea)", + "urp": "Uru-Pa-In", + "urr": "Lehalurup; Löyöp", + "urt": "Urat", + "uru": "Urumi", + "urv": "Uruava", + "urw": "Sop", + "urx": "Urimo", + "ury": "Orya", + "urz": "Uru-Eu-Wau-Wau", + "usa": "Usarufa", + "ush": "Ushojo", + "usi": "Usui", + "usk": "Usaghade", + "usp": "Uspanteco", + "uss": "us-Saare", + "usu": "Uya", + "uta": "Otank", + "ute": "Ute-Southern Paiute", + "uth": "ut-Hun", + "utp": "Amba (Solomon Islands)", + "utr": "Etulo", + "utu": "Utu", + "uum": "Urum", + "uur": "Ura (Vanuatu)", + "uuu": "U", + "uve": "West Uvean; Fagauvea", + "uvh": "Uri", + "uvl": "Lote", + "uwa": "Kuku-Uwanh", + "uya": "Doko-Uyanga", + "uz": "Uzbek", + "uzn": "Northern Uzbek", + "uzs": "Southern Uzbek", + "vaa": "Vaagri Booli", + "vae": "Vale", + "vaf": "Vafsi", + "vag": "Vagla", + "vah": "Varhadi-Nagpuri", + "vai": "Vai", + "vaj": "Sekele; Northwestern ǃKung; Vasekele", + "val": "Vehes", + "vam": "Vanimo", + "van": "Valman", + "vao": "Vao", + "vap": "Vaiphei", + "var": "Huarijio", + "vas": "Vasavi", + "vau": "Vanuma", + "vav": "Varli", + "vay": "Wayu", + "vbb": "Southeast Babar", + "vbk": "Southwestern Bontok", + "ve": "Venda", + "vec": "Venetian", + "ved": "Veddah", + "vel": "Veluws", + "vem": "Vemgo-Mabas", + "veo": "Ventureño", + "vep": "Veps", + "ver": "Mom Jango", + "vgr": "Vaghri", + "vgt": "Vlaamse Gebarentaal; Flemish Sign Language", + "vi": "Vietnamese", + "vic": "Virgin Islands Creole English", + "vid": "Vidunda", + "vif": "Vili", + "vig": "Viemo", + "vil": "Vilela", + "vin": "Vinza", + "vis": "Vishavan", + "vit": "Viti", + "viv": "Iduna", + "vka": "Kariyarra", + "vkj": "Kujarge", + "vkk": "Kaur", + "vkl": "Kulisusu", + "vkm": "Kamakan", + "vkn": "Koro Nulu", + "vko": "Kodeoha", + "vkp": "Korlai Creole Portuguese", + "vkt": "Tenggarong Kutai Malay", + "vku": "Kurrama", + "vkz": "Koro Zuba", + "vlp": "Valpei", + "vls": "Vlaams", + "vma": "Martuyhunira", + "vmb": "Barbaram", + "vmc": "Juxtlahuaca Mixtec", + "vmd": "Mudu Koraga", + "vme": "East Masela", + "vmf": "Mainfränkisch", + "vmg": "Lungalunga", + "vmh": "Maraghei", + "vmi": "Miwa", + "vmj": "Ixtayutla Mixtec", + "vmk": "Makhuwa-Shirima", + "vml": "Malgana", + "vmm": "Mitlatongo Mixtec", + "vmp": "Soyaltepec Mazatec", + "vmq": "Soyaltepec Mixtec", + "vmr": "Marenje", + "vms": "Moksela", + "vmu": "Muluridyi", + "vmv": "Valley Maidu", + "vmw": "Makhuwa", + "vmx": "Tamazola Mixtec", + "vmy": "Ayautla Mazatec", + "vmz": "Mazatlán Mazatec", + "vnk": "Vano; Lovono", + "vnm": "Vinmavis; Neve'ei", + "vnp": "Vunapu", + "vo": "Volapük", + "vor": "Voro", + "vot": "Votic", + "vra": "Vera'a", + "vro": "Võro", + "vrs": "Varisi", + "vrt": "Burmbar; Banam Bay", + "vsi": "Moldova Sign Language", + "vsl": "Venezuelan Sign Language", + "vsv": "Valencian Sign Language; Llengua de signes valenciana", + "vto": "Vitou", + "vum": "Vumbu", + "vun": "Vunjo", + "vut": "Vute", + "vwa": "Awa (China)", + "wa": "Walloon", + "waa": "Walla Walla", + "wab": "Wab", + "wac": "Wasco-Wishram", + "wad": "Wamesa; Wondama", + "wae": "Walser", + "waf": "Wakoná", + "wag": "Wa'ema", + "wah": "Watubela", + "wai": "Wares", + "waj": "Waffa", + "wak": "Wakashan languages", + "wal": "Wolaytta; Wolaitta", + "wam": "Wampanoag", + "wan": "Wan", + "wao": "Wappo", + "wap": "Wapishana", + "waq": "Wagiman", + "war": "Waray (Philippines)", + "was": "Washo", + "wat": "Kaninuwa", + "wau": "Waurá", + "wav": "Waka", + "waw": "Waiwai", + "wax": "Watam; Marangis", + "way": "Wayana", + "waz": "Wampur", + "wba": "Warao", + "wbb": "Wabo", + "wbe": "Waritai", + "wbf": "Wara", + "wbh": "Wanda", + "wbi": "Vwanji", + "wbj": "Alagwa", + "wbk": "Waigali", + "wbl": "Wakhi", + "wbm": "Wa", + "wbp": "Warlpiri", + "wbq": "Waddar", + "wbr": "Wagdi", + "wbs": "West Bengal Sign Language", + "wbt": "Warnman", + "wbv": "Wajarri", + "wbw": "Woi", + "wca": "Yanomámi", + "wci": "Waci Gbe", + "wdd": "Wandji", + "wdg": "Wadaginam", + "wdj": "Wadjiginy", + "wdk": "Wadikali", + "wdt": "Wendat", + "wdu": "Wadjigu", + "wdy": "Wadjabangayi", + "wea": "Wewaw", + "wec": "Wè Western", + "wed": "Wedau", + "weg": "Wergaia", + "weh": "Weh", + "wei": "Kiunum", + "wem": "Weme Gbe", + "wen": "Sorbian languages", + "weo": "Wemale", + "wep": "Westphalien", + "wer": "Weri", + "wes": "Cameroon Pidgin", + "wet": "Perai", + "weu": "Rawngtu Chin", + "wew": "Wejewa", + "wfg": "Yafi; Zorop", + "wga": "Wagaya", + "wgb": "Wagawaga", + "wgg": "Wangkangurru; Wangganguru", + "wgi": "Wahgi", + "wgo": "Waigeo", + "wgu": "Wirangu", + "wgy": "Warrgamay", + "wha": "Sou Upaa; Manusela", + "whg": "North Wahgi", + "whk": "Wahau Kenyah", + "whu": "Wahau Kayan", + "wib": "Southern Toussian", + "wic": "Wichita", + "wie": "Wik-Epa", + "wif": "Wik-Keyangan", + "wig": "Wik Ngathan", + "wih": "Wik-Me'anha", + "wii": "Minidien", + "wij": "Wik-Iiyanh", + "wik": "Wikalkan", + "wil": "Wilawila", + "wim": "Wik-Mungkan", + "win": "Ho-Chunk", + "wir": "Wiraféd", + "wiu": "Wiru", + "wiv": "Vitu", + "wiy": "Wiyot", + "wja": "Waja", + "wji": "Warji", + "wka": "Kw'adza", + "wkb": "Kumbaran", + "wkd": "Wakde; Mo", + "wkl": "Kalanadi", + "wkr": "Keerray-Woorroong", + "wku": "Kunduvadi", + "wkw": "Wakawaka", + "wky": "Wangkayutyuru", + "wla": "Walio", + "wlc": "Mwali Comorian", + "wle": "Wolane", + "wlg": "Kunbarlang", + "wlh": "Welaun", + "wli": "Waioli", + "wlk": "Wailaki", + "wll": "Wali (Sudan)", + "wlm": "Middle Welsh", + "wlo": "Wolio", + "wlr": "Wailapa", + "wls": "Wallisian", + "wlu": "Wuliwuli", + "wlv": "Wichí Lhamtés Vejoz", + "wlw": "Walak", + "wlx": "Wali (Ghana)", + "wly": "Waling", + "wma": "Mawa (Nigeria)", + "wmb": "Wambaya", + "wmc": "Wamas", + "wmd": "Mamaindé", + "wme": "Wambule", + "wmg": "Western Minyag", + "wmh": "Waima'a", + "wmi": "Wamin", + "wmm": "Maiwa (Indonesia)", + "wmn": "Waamwang", + "wmo": "Wom (Papua New Guinea)", + "wms": "Wambon", + "wmt": "Walmajarri", + "wmw": "Mwani", + "wmx": "Womo", + "wnb": "Wanambre", + "wnc": "Wantoat", + "wnd": "Wandarang", + "wne": "Waneci", + "wng": "Wanggom", + "wni": "Ndzwani Comorian", + "wnk": "Wanukaka", + "wnm": "Wanggamala", + "wnn": "Wunumara", + "wno": "Wano", + "wnp": "Wanap", + "wnu": "Usan", + "wnw": "Wintu", + "wny": "Wanyi; Waanyi", + "wo": "Wolof", + "woa": "Kuwema; Tyaraity", + "wob": "Wè Northern", + "woc": "Wogeo", + "wod": "Wolani", + "woe": "Woleaian", + "wof": "Gambian Wolof", + "wog": "Wogamusin", + "woi": "Kamang", + "wok": "Longto", + "wom": "Wom (Nigeria)", + "won": "Wongo", + "woo": "Manombai", + "wor": "Woria", + "wos": "Hanga Hundi", + "wow": "Wawonii", + "woy": "Weyto", + "wpc": "Maco", + "wrb": "Waluwarra; Warluwara", + "wrg": "Warungu; Gudjal", + "wrh": "Wiradjuri", + "wri": "Wariyangga", + "wrk": "Garrwa", + "wrl": "Warlmanpa", + "wrm": "Warumungu", + "wrn": "Warnang", + "wro": "Worrorra", + "wrp": "Waropen", + "wrr": "Wardaman", + "wrs": "Waris", + "wru": "Waru", + "wrv": "Waruna", + "wrw": "Gugu Warra", + "wrx": "Wae Rana", + "wry": "Merwari", + "wrz": "Waray (Australia)", + "wsa": "Warembori", + "wsg": "Adilabad Gondi", + "wsi": "Wusi", + "wsk": "Waskia", + "wsr": "Owenia", + "wss": "Wasa", + "wsu": "Wasu", + "wsv": "Wotapuri-Katarqalai", + "wtf": "Watiwa", + "wth": "Wathawurrung", + "wti": "Berta", + "wtk": "Watakataui", + "wtm": "Mewati", + "wtw": "Wotu", + "wua": "Wikngenchera", + "wub": "Wunambal", + "wud": "Wudu", + "wuh": "Wutunhua", + "wul": "Silimo", + "wum": "Wumbvu", + "wun": "Bungu", + "wur": "Wurrugu", + "wut": "Wutung", + "wuu": "Wu Chinese", + "wuv": "Wuvulu-Aua", + "wux": "Wulna", + "wuy": "Wauyai", + "wwa": "Waama", + "wwb": "Wakabunga", + "wwo": "Wetamut; Dorig", + "wwr": "Warrwa", + "www": "Wawa", + "wxa": "Waxianghua", + "wxw": "Wardandi", + "wyb": "Wangaaybuwan-Ngiyambaa", + "wyi": "Woiwurrung", + "wym": "Wymysorys", + "wyn": "Wyandot", + "wyr": "Wayoró", + "wyy": "Western Fijian", + "xaa": "Andalusian Arabic", + "xab": "Sambe", + "xac": "Kachari", + "xad": "Adai", + "xae": "Aequian", + "xag": "Aghwan", + "xai": "Kaimbé", + "xaj": "Ararandewára", + "xak": "Máku", + "xal": "Kalmyk; Oirat", + "xam": "ǀXam", + "xan": "Xamtanga", + "xao": "Khao", + "xap": "Apalachee", + "xaq": "Aquitanian", + "xar": "Karami", + "xas": "Kamas", + "xat": "Katawixi", + "xau": "Kauwera", + "xav": "Xavánte", + "xaw": "Kawaiisu", + "xay": "Kayan Mahakam", + "xbb": "Lower Burdekin", + "xbc": "Bactrian", + "xbd": "Bindal", + "xbe": "Bigambal", + "xbg": "Bunganditj", + "xbi": "Kombio", + "xbj": "Birrpayi", + "xbm": "Middle Breton", + "xbn": "Kenaboi", + "xbo": "Bolgarian", + "xbp": "Bibbulman", + "xbr": "Kambera", + "xbw": "Kambiwá", + "xby": "Batjala; Batyala", + "xcb": "Cumbric", + "xcc": "Camunic", + "xce": "Celtiberian", + "xcg": "Cisalpine Gaulish", + "xch": "Chemakum; Chimakum", + "xcl": "Classical Armenian", + "xcm": "Comecrudo", + "xcn": "Cotoname", + "xco": "Chorasmian", + "xcr": "Carian", + "xct": "Classical Tibetan", + "xcu": "Curonian", + "xcv": "Chuvantsy", + "xcw": "Coahuilteco", + "xcy": "Cayuse", + "xda": "Darkinyung", + "xdc": "Dacian", + "xdk": "Dharuk", + "xdm": "Edomite", + "xdo": "Kwandu", + "xdq": "Kaitag", + "xdy": "Malayic Dayak", + "xeb": "Eblan", + "xed": "Hdi", + "xeg": "ǁXegwi", + "xel": "Kelo", + "xem": "Kembayan", + "xep": "Epi-Olmec", + "xer": "Xerénte", + "xes": "Kesawai", + "xet": "Xetá", + "xeu": "Keoru-Ahia", + "xfa": "Faliscan", + "xga": "Galatian", + "xgb": "Gbin", + "xgd": "Gudang", + "xgf": "Gabrielino-Fernandeño", + "xgg": "Goreng", + "xgi": "Garingbal", + "xgl": "Galindan", + "xgm": "Dharumbal; Guwinmal", + "xgn": "Mongolian languages", + "xgr": "Garza", + "xgu": "Unggumi", + "xgw": "Guwa", + "xh": "Xhosa", + "xha": "Harami", + "xhc": "Hunnic", + "xhd": "Hadrami", + "xhe": "Khetrani", + "xhm": "Middle Khmer (1400 to 1850 CE)", + "xhr": "Hernican", + "xht": "Hattic", + "xhu": "Hurrian", + "xhv": "Khua", + "xib": "Iberian", + "xii": "Xiri", + "xil": "Illyrian", + "xin": "Xinca", + "xir": "Xiriâna", + "xis": "Kisan", + "xiv": "Indus Valley Language", + "xiy": "Xipaya", + "xjb": "Minjungbal", + "xjt": "Jaitmatang", + "xka": "Kalkoti", + "xkb": "Northern Nago", + "xkc": "Kho'ini", + "xkd": "Mendalam Kayan", + "xke": "Kereho", + "xkf": "Khengkha", + "xkg": "Kagoro", + "xki": "Kenyan Sign Language", + "xkj": "Kajali", + "xkk": "Kachok; Kaco'", + "xkl": "Mainstream Kenyah", + "xkn": "Kayan River Kayan", + "xko": "Kiorr", + "xkp": "Kabatei", + "xkq": "Koroni", + "xkr": "Xakriabá", + "xks": "Kumbewaha", + "xkt": "Kantosi", + "xku": "Kaamba", + "xkv": "Kgalagadi", + "xkw": "Kembra", + "xkx": "Karore", + "xky": "Uma' Lasan", + "xkz": "Kurtokha", + "xla": "Kamula", + "xlb": "Loup B", + "xlc": "Lycian", + "xld": "Lydian", + "xle": "Lemnian", + "xlg": "Ligurian (Ancient)", + "xli": "Liburnian", + "xln": "Alanic", + "xlo": "Loup A", + "xlp": "Lepontic", + "xls": "Lusitanian", + "xlu": "Cuneiform Luwian", + "xly": "Elymian", + "xma": "Mushungulu", + "xmb": "Mbonga", + "xmc": "Makhuwa-Marrevone", + "xmd": "Mbudum", + "xme": "Median", + "xmf": "Mingrelian", + "xmg": "Mengaka", + "xmh": "Kugu-Muminh", + "xmj": "Majera", + "xmk": "Ancient Macedonian", + "xml": "Malaysian Sign Language", + "xmm": "Manado Malay", + "xmn": "Manichaean Middle Persian", + "xmo": "Morerebi", + "xmp": "Kuku-Mu'inh", + "xmq": "Kuku-Mangk", + "xmr": "Meroitic", + "xms": "Moroccan Sign Language", + "xmt": "Matbat", + "xmu": "Kamu", + "xmv": "Antankarana Malagasy; Tankarana Malagasy", + "xmw": "Tsimihety Malagasy", + "xmx": "Salawati; Maden", + "xmy": "Mayaguduna", + "xmz": "Mori Bawah", + "xna": "Ancient North Arabian", + "xnb": "Kanakanabu", + "xnd": "Na-Dene languages", + "xng": "Middle Mongolian", + "xnh": "Kuanhua", + "xni": "Ngarigu", + "xnj": "Ngoni (Tanzania)", + "xnk": "Nganakarti", + "xnm": "Ngumbarl", + "xnn": "Northern Kankanay", + "xno": "Anglo-Norman", + "xnq": "Ngoni (Mozambique)", + "xnr": "Kangri", + "xns": "Kanashi", + "xnt": "Narragansett", + "xnu": "Nukunul", + "xny": "Nyiyaparli", + "xnz": "Kenzi; Mattoki", + "xoc": "O'chi'chi'", + "xod": "Kokoda", + "xog": "Soga", + "xoi": "Kominimung", + "xok": "Xokleng", + "xom": "Komo (Sudan)", + "xon": "Konkomba", + "xoo": "Xukurú", + "xop": "Kopar", + "xor": "Korubo", + "xow": "Kowaki", + "xpa": "Pirriya", + "xpb": "Northeastern Tasmanian; Pyemmairrener", + "xpc": "Pecheneg", + "xpd": "Oyster Bay Tasmanian", + "xpe": "Liberia Kpelle", + "xpf": "Southeast Tasmanian; Nuenonne", + "xpg": "Phrygian", + "xph": "North Midlands Tasmanian; Tyerrenoterpanner", + "xpi": "Pictish", + "xpj": "Mpalitjanh", + "xpk": "Kulina Pano", + "xpl": "Port Sorell Tasmanian", + "xpm": "Pumpokol", + "xpn": "Kapinawá", + "xpo": "Pochutec", + "xpp": "Puyo-Paekche", + "xpq": "Mohegan-Pequot", + "xpr": "Parthian", + "xps": "Pisidian", + "xpt": "Punthamara", + "xpu": "Punic", + "xpv": "Northern Tasmanian; Tommeginne", + "xpw": "Northwestern Tasmanian; Peerapper", + "xpx": "Southwestern Tasmanian; Toogee", + "xpy": "Puyo", + "xpz": "Bruny Island Tasmanian", + "xqa": "Karakhanid", + "xqt": "Qatabanian", + "xra": "Krahô", + "xrb": "Eastern Karaboro", + "xrd": "Gundungurra", + "xre": "Kreye", + "xrg": "Minang", + "xri": "Krikati-Timbira", + "xrm": "Armazic", + "xrn": "Arin", + "xrr": "Raetic", + "xrt": "Aranama-Tamique", + "xru": "Marriammu", + "xrw": "Karawa", + "xsa": "Sabaean", + "xsb": "Sambal", + "xsc": "Scythian", + "xsd": "Sidetic", + "xse": "Sempan", + "xsh": "Shamang", + "xsi": "Sio", + "xsj": "Subi", + "xsl": "South Slavey", + "xsm": "Kasem", + "xsn": "Sanga (Nigeria)", + "xso": "Solano", + "xsp": "Silopi", + "xsq": "Makhuwa-Saka", + "xsr": "Sherpa", + "xss": "Assan", + "xsu": "Sanumá", + "xsv": "Sudovian", + "xsy": "Saisiyat", + "xta": "Alcozauca Mixtec", + "xtb": "Chazumba Mixtec", + "xtc": "Katcha-Kadugli-Miri", + "xtd": "Diuxi-Tilantongo Mixtec", + "xte": "Ketengban", + "xtg": "Transalpine Gaulish", + "xth": "Yitha Yitha", + "xti": "Sinicahua Mixtec", + "xtj": "San Juan Teita Mixtec", + "xtl": "Tijaltepec Mixtec", + "xtm": "Magdalena Peñasco Mixtec", + "xtn": "Northern Tlaxiaco Mixtec", + "xto": "Tokharian A", + "xtp": "San Miguel Piedras Mixtec", + "xtq": "Tumshuqese", + "xtr": "Early Tripuri", + "xts": "Sindihui Mixtec", + "xtt": "Tacahua Mixtec", + "xtu": "Cuyamecalco Mixtec", + "xtv": "Thawa", + "xtw": "Tawandê", + "xty": "Yoloxochitl Mixtec", + "xua": "Alu Kurumba", + "xub": "Betta Kurumba", + "xud": "Umiida", + "xug": "Kunigami", + "xuj": "Jennu Kurumba", + "xul": "Ngunawal; Nunukul", + "xum": "Umbrian", + "xun": "Unggaranggu", + "xuo": "Kuo", + "xup": "Upper Umpqua", + "xur": "Urartian", + "xut": "Kuthant", + "xuu": "Kxoe; Khwedam", + "xve": "Venetic", + "xvi": "Kamviri", + "xvn": "Vandalic", + "xvo": "Volscian", + "xvs": "Vestinian", + "xwa": "Kwaza", + "xwc": "Woccon", + "xwd": "Wadi Wadi", + "xwe": "Xwela Gbe", + "xwg": "Kwegu", + "xwj": "Wajuk", + "xwk": "Wangkumara", + "xwl": "Western Xwla Gbe", + "xwo": "Written Oirat", + "xwr": "Kwerba Mamberamo", + "xwt": "Wotjobaluk", + "xww": "Wemba Wemba", + "xxb": "Boro (Ghana)", + "xxk": "Ke'o", + "xxm": "Minkin", + "xxr": "Koropó", + "xxt": "Tambora", + "xya": "Yaygir", + "xyb": "Yandjibara", + "xyj": "Mayi-Yapi", + "xyk": "Mayi-Kulan", + "xyl": "Yalakalore", + "xyt": "Mayi-Thakurti", + "xyy": "Yorta Yorta", + "xzh": "Zhang-Zhung", + "xzm": "Zemgalian", + "xzp": "Ancient Zapotec", + "yaa": "Yaminahua", + "yab": "Yuhup", + "yac": "Pass Valley Yali", + "yad": "Yagua", + "yae": "Pumé", + "yaf": "Yaka (Democratic Republic of Congo)", + "yag": "Yámana", + "yah": "Yazgulyam", + "yai": "Yagnobi", + "yaj": "Banda-Yangere", + "yak": "Yakama", + "yal": "Yalunka", + "yam": "Yamba", + "yan": "Mayangna", + "yao": "Yao", + "yap": "Yapese", + "yaq": "Yaqui", + "yar": "Yabarana", + "yas": "Nugunu (Cameroon)", + "yat": "Yambeta", + "yau": "Yuwana", + "yav": "Yangben", + "yaw": "Yawalapití", + "yax": "Yauma", + "yay": "Agwagwune", + "yaz": "Lokaa", + "yba": "Yala", + "ybb": "Yemba", + "ybe": "West Yugur", + "ybh": "Yakha", + "ybi": "Yamphu", + "ybj": "Hasha", + "ybk": "Bokha", + "ybl": "Yukuben", + "ybm": "Yaben", + "ybn": "Yabaâna", + "ybo": "Yabong", + "ybx": "Yawiyo", + "yby": "Yaweyuha", + "ych": "Chesu", + "ycl": "Lolopo", + "ycn": "Yucuna", + "ycp": "Chepya", + "yda": "Yanda", + "ydd": "Eastern Yiddish", + "yde": "Yangum Dey", + "ydg": "Yidgha", + "ydk": "Yoidik", + "yea": "Ravula", + "yec": "Yeniche", + "yee": "Yimas", + "yei": "Yeni", + "yej": "Yevanic", + "yel": "Yela", + "yer": "Tarok", + "yes": "Nyankpa", + "yet": "Yetfa", + "yeu": "Yerukula", + "yev": "Yapunda", + "yey": "Yeyi", + "yga": "Malyangapa", + "ygi": "Yiningayi", + "ygl": "Yangum Gel", + "ygm": "Yagomi", + "ygp": "Gepo", + "ygr": "Yagaria", + "ygs": "Yolŋu Sign Language", + "ygu": "Yugul", + "ygw": "Yagwoia", + "yha": "Baha Buyang", + "yhd": "Judeo-Iraqi Arabic", + "yhl": "Hlepho Phowa", + "yhs": "Yan-nhaŋu Sign Language", + "yi": "Yiddish", + "yia": "Yinggarda", + "yif": "Ache", + "yig": "Wusa Nasu", + "yih": "Western Yiddish", + "yii": "Yidiny", + "yij": "Yindjibarndi", + "yik": "Dongshanba Lalo", + "yil": "Yindjilandji", + "yim": "Yimchungru Naga", + "yin": "Riang Lai; Yinchia", + "yip": "Pholo", + "yiq": "Miqie", + "yir": "North Awyu", + "yis": "Yis", + "yit": "Eastern Lalu", + "yiu": "Awu", + "yiv": "Northern Nisu", + "yix": "Axi Yi", + "yiz": "Azhe", + "yka": "Yakan", + "ykg": "Northern Yukaghir", + "yki": "Yoke", + "ykk": "Yakaikeke", + "ykl": "Khlula", + "ykm": "Kap", + "ykn": "Kua-nsi", + "yko": "Yasa", + "ykr": "Yekora", + "ykt": "Kathu", + "yku": "Kuamasi", + "yky": "Yakoma", + "yla": "Yaul", + "ylb": "Yaleba", + "yle": "Yele", + "ylg": "Yelogu", + "yli": "Angguruk Yali", + "yll": "Yil", + "ylm": "Limi", + "yln": "Langnian Buyang", + "ylo": "Naluo Yi", + "ylr": "Yalarnnga", + "ylu": "Aribwaung", + "yly": "Nyâlayu; Nyelâyu", + "ymb": "Yambes", + "ymc": "Southern Muji", + "ymd": "Muda", + "yme": "Yameo", + "ymg": "Yamongeri", + "ymh": "Mili", + "ymi": "Moji", + "ymk": "Makwe", + "yml": "Iamalele", + "ymm": "Maay", + "ymn": "Yamna; Sunum", + "ymo": "Yangum Mon", + "ymp": "Yamap", + "ymq": "Qila Muji", + "ymr": "Malasar", + "yms": "Mysian", + "ymx": "Northern Muji", + "ymz": "Muzi", + "yna": "Aluo", + "ynd": "Yandruwandha", + "yne": "Lang'e", + "yng": "Yango", + "ynk": "Naukan Yupik", + "ynl": "Yangulam", + "ynn": "Yana", + "yno": "Yong", + "ynq": "Yendang", + "yns": "Yansi", + "ynu": "Yahuna", + "yo": "Yoruba", + "yob": "Yoba", + "yog": "Yogad", + "yoi": "Yonaguni", + "yok": "Yokuts", + "yol": "Yola", + "yom": "Yombe", + "yon": "Yongkom", + "yot": "Yotti", + "yox": "Yoron", + "yoy": "Yoy", + "ypa": "Phala", + "ypb": "Labo Phowa", + "ypg": "Phola", + "yph": "Phupha", + "ypk": "Yupik languages", + "ypm": "Phuma", + "ypn": "Ani Phowa", + "ypo": "Alo Phola", + "ypp": "Phupa", + "ypz": "Phuza", + "yra": "Yerakai", + "yrb": "Yareba", + "yre": "Yaouré", + "yrk": "Nenets", + "yrl": "Nhengatu", + "yrm": "Yirrk-Mel", + "yrn": "Yerong", + "yro": "Yaroamë", + "yrs": "Yarsun", + "yrw": "Yarawata", + "yry": "Yarluyandi", + "ysc": "Yassic", + "ysd": "Samatao", + "ysg": "Sonaga", + "ysl": "Yugoslavian Sign Language", + "ysm": "Myanmar Sign Language", + "ysn": "Sani", + "yso": "Nisi (China)", + "ysp": "Southern Lolopo", + "ysr": "Sirenik Yupik", + "yss": "Yessan-Mayo", + "ysy": "Sanie", + "yta": "Talu", + "ytl": "Tanglang", + "ytp": "Thopho", + "ytw": "Yout Wam", + "yty": "Yatay", + "yua": "Yucateco; Yucatec Maya", + "yub": "Yugambal", + "yuc": "Yuchi", + "yud": "Judeo-Tripolitanian Arabic", + "yue": "Yue Chinese; Cantonese", + "yuf": "Havasupai-Walapai-Yavapai", + "yug": "Yug", + "yui": "Yurutí", + "yuj": "Karkar-Yuri", + "yuk": "Yuki", + "yul": "Yulu", + "yum": "Quechan", + "yun": "Bena (Nigeria)", + "yup": "Yukpa", + "yuq": "Yuqui", + "yur": "Yurok", + "yut": "Yopno", + "yuw": "Yau (Morobe Province)", + "yux": "Southern Yukaghir", + "yuy": "East Yugur", + "yuz": "Yuracare", + "yva": "Yawa", + "yvt": "Yavitero", + "ywa": "Kalou", + "ywg": "Yinhawangka", + "ywl": "Western Lalu", + "ywn": "Yawanawa", + "ywq": "Wuding-Luquan Yi", + "ywr": "Yawuru", + "ywt": "Xishanba Lalo; Central Lalo", + "ywu": "Wumeng Nasu", + "yww": "Yawarawarga", + "yxa": "Mayawali", + "yxg": "Yagara", + "yxl": "Yardliyawarra", + "yxm": "Yinwum", + "yxu": "Yuyu", + "yxy": "Yabula Yabula", + "yyr": "Yir Yoront", + "yyu": "Yau (Sandaun Province)", + "yyz": "Ayizi", + "yzg": "E'ma Buyang", + "yzk": "Zokhuo", + "za": "Zhuang; Chuang", + "zaa": "Sierra de Juárez Zapotec", + "zab": "Western Tlacolula Valley Zapotec; San Juan Guelavía Zapotec", + "zac": "Ocotlán Zapotec", + "zad": "Cajonos Zapotec", + "zae": "Yareni Zapotec", + "zaf": "Ayoquesco Zapotec", + "zag": "Zaghawa", + "zah": "Zangwal", + "zai": "Isthmus Zapotec", + "zaj": "Zaramo", + "zak": "Zanaki", + "zal": "Zauzou", + "zam": "Miahuatlán Zapotec", + "zao": "Ozolotepec Zapotec", + "zap": "Zapotec", + "zaq": "Aloápam Zapotec", + "zar": "Rincón Zapotec", + "zas": "Santo Domingo Albarradas Zapotec", + "zat": "Tabaa Zapotec", + "zau": "Zangskari", + "zav": "Yatzachi Zapotec", + "zaw": "Mitla Zapotec", + "zax": "Xadani Zapotec", + "zay": "Zayse-Zergulla; Zaysete", + "zaz": "Zari", + "zba": "Balaibalan", + "zbc": "Central Berawan", + "zbe": "East Berawan", + "zbl": "Blissymbols; Bliss; Blissymbolics", + "zbt": "Batui", + "zbu": "Bu (Bauchi State)", + "zbw": "West Berawan", + "zca": "Coatecas Altas Zapotec", + "zcd": "Las Delicias Zapotec", + "zch": "Central Hongshuihe Zhuang", + "zdj": "Ngazidja Comorian", + "zea": "Zeeuws", + "zeg": "Zenag", + "zeh": "Eastern Hongshuihe Zhuang", + "zen": "Zenaga", + "zga": "Kinga", + "zgb": "Guibei Zhuang", + "zgh": "Standard Moroccan Tamazight", + "zgm": "Minz Zhuang", + "zgn": "Guibian Zhuang", + "zgr": "Magori", + "zh": "Chinese", + "zhb": "Zhaba", + "zhd": "Dai Zhuang", + "zhi": "Zhire", + "zhn": "Nong Zhuang", + "zhw": "Zhoa", + "zhx": "Chinese (family)", + "zia": "Zia", + "zib": "Zimbabwe Sign Language", + "zik": "Zimakani", + "zil": "Zialo", + "zim": "Mesme", + "zin": "Zinza", + "ziw": "Zigula", + "ziz": "Zizilivakan", + "zka": "Kaimbulawa", + "zkb": "Koibal", + "zkd": "Kadu", + "zkg": "Koguryo", + "zkh": "Khorezmian", + "zkk": "Karankawa", + "zkn": "Kanan", + "zko": "Kott", + "zkp": "São Paulo Kaingáng", + "zkr": "Zakhring", + "zkt": "Kitan", + "zku": "Kaurna", + "zkv": "Krevinian", + "zkz": "Khazar", + "zla": "Zula", + "zle": "East Slavic languages", + "zlj": "Liujiang Zhuang", + "zlm": "Malay (individual language)", + "zln": "Lianshan Zhuang", + "zlq": "Liuqian Zhuang", + "zls": "South Slavic languages", + "zlw": "West Slavic languages", + "zma": "Manda (Australia)", + "zmb": "Zimba", + "zmc": "Margany", + "zmd": "Maridan", + "zme": "Mangerr", + "zmf": "Mfinu", + "zmg": "Marti Ke", + "zmh": "Makolkol", + "zmi": "Negeri Sembilan Malay", + "zmj": "Maridjabin", + "zmk": "Mandandanyi", + "zml": "Matngala", + "zmm": "Marimanindji; Marramaninyshi", + "zmn": "Mbangwe", + "zmo": "Molo", + "zmp": "Mpuono", + "zmq": "Mituku", + "zmr": "Maranunggu", + "zms": "Mbesa", + "zmt": "Maringarr", + "zmu": "Muruwari", + "zmv": "Mbariman-Gudhinma", + "zmw": "Mbo (Democratic Republic of Congo)", + "zmx": "Bomitaba", + "zmy": "Mariyedi", + "zmz": "Mbandja", + "zna": "Zan Gula", + "znd": "Zande languages", + "zne": "Zande (individual language)", + "zng": "Mang", + "znk": "Manangkari", + "zns": "Mangas", + "zoc": "Copainalá Zoque", + "zoh": "Chimalapa Zoque", + "zom": "Zou", + "zoo": "Asunción Mixtepec Zapotec", + "zoq": "Tabasco Zoque", + "zor": "Rayón Zoque", + "zos": "Francisco León Zoque", + "zpa": "Lachiguiri Zapotec", + "zpb": "Yautepec Zapotec", + "zpc": "Choapan Zapotec", + "zpd": "Southeastern Ixtlán Zapotec", + "zpe": "Petapa Zapotec", + "zpf": "San Pedro Quiatoni Zapotec", + "zpg": "Guevea De Humboldt Zapotec", + "zph": "Totomachapan Zapotec", + "zpi": "Santa María Quiegolani Zapotec", + "zpj": "Quiavicuzas Zapotec", + "zpk": "Tlacolulita Zapotec", + "zpl": "Lachixío Zapotec", + "zpm": "Mixtepec Zapotec", + "zpn": "Santa Inés Yatzechi Zapotec", + "zpo": "Amatlán Zapotec", + "zpp": "El Alto Zapotec", + "zpq": "Zoogocho Zapotec", + "zpr": "Santiago Xanica Zapotec", + "zps": "Coatlán Zapotec", + "zpt": "San Vicente Coatlán Zapotec", + "zpu": "Yalálag Zapotec", + "zpv": "Chichicapan Zapotec", + "zpw": "Zaniza Zapotec", + "zpx": "San Baltazar Loxicha Zapotec", + "zpy": "Mazaltepec Zapotec", + "zpz": "Texmelucan Zapotec", + "zqe": "Qiubei Zhuang", + "zra": "Kara (Korea)", + "zrg": "Mirgan", + "zrn": "Zerenkel", + "zro": "Záparo", + "zrp": "Zarphatic", + "zrs": "Mairasi", + "zsa": "Sarasira", + "zsk": "Kaskean", + "zsl": "Zambian Sign Language", + "zsm": "Standard Malay", + "zsr": "Southern Rincon Zapotec", + "zsu": "Sukurum", + "zte": "Elotepec Zapotec", + "ztg": "Xanaguía Zapotec", + "ztl": "Lapaguía-Guivini Zapotec", + "ztm": "San Agustín Mixtepec Zapotec", + "ztn": "Santa Catarina Albarradas Zapotec", + "ztp": "Loxicha Zapotec", + "ztq": "Quioquitani-Quierí Zapotec", + "zts": "Tilquiapan Zapotec", + "ztt": "Tejalapan Zapotec", + "ztu": "Güilá Zapotec", + "ztx": "Zaachila Zapotec", + "zty": "Yatee Zapotec", + "zu": "Zulu", + "zua": "Zeem", + "zuh": "Tokano", + "zum": "Kumzari", + "zun": "Zuni", + "zuy": "Zumaya", + "zwa": "Zay", + "zyb": "Yongbei Zhuang", + "zyg": "Yang Zhuang", + "zyj": "Youjiang Zhuang", + "zyn": "Yongnan Zhuang", + "zyp": "Zyphe Chin", + "zza": "Zaza; Dimili; Dimli (macrolanguage); Kirdki; Kirmanjki (macrolanguage); Zazaki", + "zzj": "Zuojiang Zhuang" +} \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/multilingualities.json b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/multilingualities.json new file mode 100644 index 0000000000000000000000000000000000000000..a35c79f03dfcf7c8a116b7fc8ee1b383ab5022fa --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/multilingualities.json @@ -0,0 +1,6 @@ +{ + "monolingual": "contains a single language", + "multilingual": "contains multiple languages", + "translation": "contains translated or aligned text", + "other": "other type of language distribution" +} diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/readme_structure.yaml b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/readme_structure.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b781e11e8258a446874ebf96104f642d0c190cf --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/readme_structure.yaml @@ -0,0 +1,116 @@ +name: "" # Filename comes here +allow_empty: false +allow_empty_text: true +subsections: + - name: "Dataset Card for X" # First-level markdown heading + allow_empty: false + allow_empty_text: true + subsections: + - name: "Table of Contents" + allow_empty: false + allow_empty_text: false + subsections: null # meaning it should not be checked. + - name: "Dataset Description" + allow_empty: false + allow_empty_text: false + subsections: + - name: "Dataset Summary" + allow_empty: false + allow_empty_text: false + subsections: null + - name: "Supported Tasks and Leaderboards" + allow_empty: true + allow_empty_text: true + subsections: null + - name: Languages + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Dataset Structure" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Data Instances" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Data Fields" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Data Splits" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Dataset Creation" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Curation Rationale" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Source Data" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Initial Data Collection and Normalization" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Who are the source language producers?" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Annotations" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Annotation process" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Who are the annotators?" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Personal and Sensitive Information" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Considerations for Using the Data" + allow_empty: true + allow_empty_text: true + subsections: + - name: "Social Impact of Dataset" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Discussion of Biases" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Other Known Limitations" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Additional Information" + allow_empty: true + allow_empty_text: true + subsections: + - name: "Dataset Curators" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Licensing Information" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Citation Information" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Contributions" + allow_empty: false + allow_empty_text: false + subsections: null diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/size_categories.json b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/size_categories.json new file mode 100644 index 0000000000000000000000000000000000000000..983ce0c10dbb2e2245f90ae47e9de4c1025d5bb1 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/resources/size_categories.json @@ -0,0 +1,14 @@ +[ + "unknown", + "n<1K", + "1K1T" +] diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/sharding.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/sharding.py new file mode 100644 index 0000000000000000000000000000000000000000..753bdf60c2a5b0b23f70893de5ba57a51462d126 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/sharding.py @@ -0,0 +1,92 @@ +import numpy as np + + +def _number_of_shards_in_gen_kwargs(gen_kwargs: dict) -> int: + """Return the number of possible shards according to the input gen_kwargs""" + # Having lists of different sizes makes sharding ambigious, raise an error in this case + # until we decide how to define sharding without ambiguity for users + lists_lengths = {key: len(value) for key, value in gen_kwargs.items() if isinstance(value, list)} + if len(set(lists_lengths.values())) > 1: + raise RuntimeError( + "Sharding is ambiguous for this dataset: " + + "we found several data sources lists of different lengths, and we don't know over which list we should parallelize:\n" + + "\n".join(f"\t- key {key} has length {length}" for key, length in lists_lengths.items()) + + "\nTo fix this, check the 'gen_kwargs' and make sure to use lists only for data sources, " + + "and use tuples otherwise. In the end there should only be one single list, or several lists with the same length." + ) + max_length = max(lists_lengths.values(), default=0) + return max(1, max_length) + + +def _distribute_shards(num_shards: int, max_num_jobs: int) -> list[range]: + """ + Get the range of shard indices per job. + If num_shards>> _distribute_shards(2, max_num_jobs=4) + [range(0, 1), range(1, 2)] + >>> _distribute_shards(10, max_num_jobs=3) + [range(0, 4), range(4, 7), range(7, 10)] + ``` + """ + shards_indices_per_group = [] + for group_idx in range(max_num_jobs): + num_shards_to_add = num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs)) + if num_shards_to_add == 0: + break + start = shards_indices_per_group[-1].stop if shards_indices_per_group else 0 + shard_indices = range(start, start + num_shards_to_add) + shards_indices_per_group.append(shard_indices) + return shards_indices_per_group + + +def _split_gen_kwargs(gen_kwargs: dict, max_num_jobs: int) -> list[dict]: + """Split the gen_kwargs into `max_num_job` gen_kwargs""" + # Having lists of different sizes makes sharding ambigious, raise an error in this case + num_shards = _number_of_shards_in_gen_kwargs(gen_kwargs) + if num_shards == 1: + return [dict(gen_kwargs)] + else: + shard_indices_per_group = _distribute_shards(num_shards=num_shards, max_num_jobs=max_num_jobs) + return [ + { + key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]] + if isinstance(value, list) + else value + for key, value in gen_kwargs.items() + } + for group_idx in range(len(shard_indices_per_group)) + ] + + +def _merge_gen_kwargs(gen_kwargs_list: list[dict]) -> dict: + return { + key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]] + if isinstance(gen_kwargs_list[0][key], list) + else gen_kwargs_list[0][key] + for key in gen_kwargs_list[0] + } + + +def _shuffle_gen_kwargs(rng: np.random.Generator, gen_kwargs: dict) -> dict: + """Return a shuffled copy of the input gen_kwargs""" + # We must shuffle all the lists, and lists of the same size must have the same shuffling. + # This way entangled lists of (shard, shard_metadata) are still in the right order. + + # First, let's generate the shuffled indices per list size + list_sizes = {len(value) for value in gen_kwargs.values() if isinstance(value, list)} + indices_per_size = {} + for size in list_sizes: + indices_per_size[size] = list(range(size)) + rng.shuffle(indices_per_size[size]) + # Now let's copy the gen_kwargs and shuffle the lists based on their sizes + shuffled_kwargs = dict(gen_kwargs) + for key, value in shuffled_kwargs.items(): + if isinstance(value, list): + shuffled_kwargs[key] = [value[i] for i in indices_per_size[len(value)]] + return shuffled_kwargs diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/stratify.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/stratify.py new file mode 100644 index 0000000000000000000000000000000000000000..6dab0469e8168148a81eb2c13f7c103b98d061e7 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/stratify.py @@ -0,0 +1,107 @@ +import numpy as np + + +def approximate_mode(class_counts, n_draws, rng): + """Computes approximate mode of multivariate hypergeometric. + This is an approximation to the mode of the multivariate + hypergeometric given by class_counts and n_draws. + It shouldn't be off by more than one. + It is the mostly likely outcome of drawing n_draws many + samples from the population given by class_counts. + Args + ---------- + class_counts : ndarray of int + Population per class. + n_draws : int + Number of draws (samples to draw) from the overall population. + rng : random state + Used to break ties. + Returns + ------- + sampled_classes : ndarray of int + Number of samples drawn from each class. + np.sum(sampled_classes) == n_draws + + """ + # this computes a bad approximation to the mode of the + # multivariate hypergeometric given by class_counts and n_draws + continuous = n_draws * class_counts / class_counts.sum() + # floored means we don't overshoot n_samples, but probably undershoot + floored = np.floor(continuous) + # we add samples according to how much "left over" probability + # they had, until we arrive at n_samples + need_to_add = int(n_draws - floored.sum()) + if need_to_add > 0: + remainder = continuous - floored + values = np.sort(np.unique(remainder))[::-1] + # add according to remainder, but break ties + # randomly to avoid biases + for value in values: + (inds,) = np.where(remainder == value) + # if we need_to_add less than what's in inds + # we draw randomly from them. + # if we need to add more, we add them all and + # go to the next value + add_now = min(len(inds), need_to_add) + inds = rng.choice(inds, size=add_now, replace=False) + floored[inds] += 1 + need_to_add -= add_now + if need_to_add == 0: + break + return floored.astype(np.int64) + + +def stratified_shuffle_split_generate_indices(y, n_train, n_test, rng, n_splits=10): + """ + + Provides train/test indices to split data in train/test sets. + It's reference is taken from StratifiedShuffleSplit implementation + of scikit-learn library. + + Args + ---------- + + n_train : int, + represents the absolute number of train samples. + + n_test : int, + represents the absolute number of test samples. + + random_state : int or RandomState instance, default=None + Controls the randomness of the training and testing indices produced. + Pass an int for reproducible output across multiple function calls. + + n_splits : int, default=10 + Number of re-shuffling & splitting iterations. + """ + classes, y_indices = np.unique(y, return_inverse=True) + n_classes = classes.shape[0] + class_counts = np.bincount(y_indices) + if np.min(class_counts) < 2: + raise ValueError("Minimum class count error") + if n_train < n_classes: + raise ValueError( + "The train_size = %d should be greater or equal to the number of classes = %d" % (n_train, n_classes) + ) + if n_test < n_classes: + raise ValueError( + "The test_size = %d should be greater or equal to the number of classes = %d" % (n_test, n_classes) + ) + class_indices = np.split(np.argsort(y_indices, kind="mergesort"), np.cumsum(class_counts)[:-1]) + for _ in range(n_splits): + n_i = approximate_mode(class_counts, n_train, rng) + class_counts_remaining = class_counts - n_i + t_i = approximate_mode(class_counts_remaining, n_test, rng) + + train = [] + test = [] + + for i in range(n_classes): + permutation = rng.permutation(class_counts[i]) + perm_indices_class_i = class_indices[i].take(permutation, mode="clip") + train.extend(perm_indices_class_i[: n_i[i]]) + test.extend(perm_indices_class_i[n_i[i] : n_i[i] + t_i[i]]) + train = rng.permutation(train) + test = rng.permutation(test) + + yield train, test diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/tf_utils.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/tf_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..10ae3ef64ae5e14339bf0e855c010253b92a7e02 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/tf_utils.py @@ -0,0 +1,564 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TF-specific utils import.""" + +import os +import warnings +from functools import partial +from math import ceil +from uuid import uuid4 + +import numpy as np +import pyarrow as pa +from multiprocess import get_context + + +try: + from multiprocess.shared_memory import SharedMemory +except ImportError: + SharedMemory = None # Version checks should prevent this being called on older Python versions + +from .. import config + + +def minimal_tf_collate_fn(features): + if isinstance(features, dict): # case batch_size=None: nothing to collate + return features + elif config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + first = features[0] + batch = {} + for k, v in first.items(): + if isinstance(v, np.ndarray): + batch[k] = np.stack([f[k] for f in features]) + elif isinstance(v, tf.Tensor): + batch[k] = tf.stack([f[k] for f in features]) + else: + batch[k] = np.array([f[k] for f in features]) + return batch + + +def minimal_tf_collate_fn_with_renaming(features): + batch = minimal_tf_collate_fn(features) + if "label" in batch: + batch["labels"] = batch["label"] + del batch["label"] + return batch + + +def is_numeric_pa_type(pa_type): + if pa.types.is_list(pa_type): + return is_numeric_pa_type(pa_type.value_type) + return pa.types.is_integer(pa_type) or pa.types.is_floating(pa_type) or pa.types.is_decimal(pa_type) + + +def np_get_batch( + indices, dataset, cols_to_retain, collate_fn, collate_fn_args, columns_to_np_types, return_dict=False +): + if not isinstance(indices, np.ndarray): + indices = indices.numpy() + + is_batched = True + # Optimization - if we're loading a sequential batch, do it with slicing instead of a list of indices + if isinstance(indices, np.integer): + batch = dataset[indices.item()] + is_batched = False + elif np.all(np.diff(indices) == 1): + batch = dataset[indices[0] : indices[-1] + 1] + elif isinstance(indices, np.ndarray): + batch = dataset[indices] + else: + raise RuntimeError(f"Unexpected type for indices: {type(indices)}") + + if cols_to_retain is not None: + batch = { + key: value + for key, value in batch.items() + if key in cols_to_retain or key in ("label", "label_ids", "labels") + } + + if is_batched: + actual_size = len(list(batch.values())[0]) # Get the length of one of the arrays, assume all same + # Our collators expect a list of dicts, not a dict of lists/arrays, so we invert + batch = [{key: value[i] for key, value in batch.items()} for i in range(actual_size)] + batch = collate_fn(batch, **collate_fn_args) + + if return_dict: + out_batch = {} + for col, cast_dtype in columns_to_np_types.items(): + # In case the collate_fn returns something strange + array = np.array(batch[col]) + array = array.astype(cast_dtype) + out_batch[col] = array + else: + out_batch = [] + for col, cast_dtype in columns_to_np_types.items(): + # In case the collate_fn returns something strange + array = np.array(batch[col]) + array = array.astype(cast_dtype) + out_batch.append(array) + return out_batch + + +def dataset_to_tf( + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + output_signature, + shuffle, + batch_size, + drop_remainder, +): + """Create a tf.data.Dataset from the underlying Dataset. This is a single-process method - the multiprocess + equivalent is multiprocess_dataset_to_tf. + + Args: + dataset (`Dataset`): Dataset to wrap with tf.data.Dataset. + cols_to_retain (`List[str]`): Dataset column(s) to load in the + tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and + that do not exist in the original dataset. + collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate + lists of samples into a batch. + collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the + `collate_fn`. Can be empty. + columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes. + output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to + `tf.TensorSpec` objects. + shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for + validation/evaluation. + batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that + the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`. + drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided, + defaults to the same setting as shuffle. + + Returns: + `tf.data.Dataset` + """ + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + # TODO Matt: When our minimum Python version is 3.8 or higher, we can delete all of this and move everything + # to the NumPy multiprocessing path. + if hasattr(tf, "random_index_shuffle"): + random_index_shuffle = tf.random_index_shuffle + elif hasattr(tf.random.experimental, "index_shuffle"): + random_index_shuffle = tf.random.experimental.index_shuffle + else: + if len(dataset) > 10_000_000: + warnings.warn( + "to_tf_dataset() can be memory-inefficient on versions of TensorFlow older than 2.9. " + "If you are iterating over a dataset with a very large number of samples, consider " + "upgrading to TF >= 2.9." + ) + random_index_shuffle = None + + getter_fn = partial( + np_get_batch, + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + return_dict=False, + ) + + # This works because dictionaries always output in the same order + tout = [tf.dtypes.as_dtype(dtype) for dtype in columns_to_np_types.values()] + + @tf.function(input_signature=[tf.TensorSpec(None, tf.int64)]) + def fetch_function(indices): + output = tf.py_function( + getter_fn, + inp=[indices], + Tout=tout, + ) + return {key: output[i] for i, key in enumerate(columns_to_np_types.keys())} + + tf_dataset = tf.data.Dataset.range(len(dataset)) + + if shuffle and random_index_shuffle is not None: + base_seed = tf.fill((3,), value=tf.cast(-1, dtype=tf.int64)) + + def scan_random_index(state, index): + if tf.reduce_all(state == -1): + # This generates a new random seed once per epoch only, + # to ensure that we iterate over each sample exactly once per epoch + state = tf.random.uniform(shape=(3,), maxval=2**62, dtype=tf.int64) + shuffled_index = random_index_shuffle(index=index, seed=state, max_index=len(dataset) - 1) + return state, shuffled_index + + tf_dataset = tf_dataset.scan(base_seed, scan_random_index) + elif shuffle: + tf_dataset = tf_dataset.shuffle(tf_dataset.cardinality()) + + if batch_size is not None: + tf_dataset = tf_dataset.batch(batch_size, drop_remainder=drop_remainder) + + tf_dataset = tf_dataset.map(fetch_function) + + if batch_size is not None: + + def ensure_shapes(input_dict): + return {key: tf.ensure_shape(val, output_signature[key].shape) for key, val in input_dict.items()} + + else: + # Ensure shape but remove batch dimension of output_signature[key].shape + def ensure_shapes(input_dict): + return {key: tf.ensure_shape(val, output_signature[key].shape[1:]) for key, val in input_dict.items()} + + return tf_dataset.map(ensure_shapes) + + +class SharedMemoryContext: + # This is a context manager for creating shared memory that ensures cleanup happens even if a process is interrupted + # The process that creates shared memory is always the one responsible for unlinking it in the end + def __init__(self): + self.created_shms = [] + self.opened_shms = [] + + def get_shm(self, name, size, create): + shm = SharedMemory(size=int(size), name=name, create=create) + if create: + # We only unlink the ones we created in this context + self.created_shms.append(shm) + else: + # If we didn't create it, we only close it when done, we don't unlink it + self.opened_shms.append(shm) + return shm + + def get_array(self, name, shape, dtype, create): + shm = self.get_shm(name=name, size=np.prod(shape) * np.dtype(dtype).itemsize, create=create) + return np.ndarray(shape, dtype=dtype, buffer=shm.buf) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + for shm in self.created_shms: + shm.close() + shm.unlink() + for shm in self.opened_shms: + shm.close() + + +class NumpyMultiprocessingGenerator: + def __init__( + self, + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + output_signature, + shuffle, + batch_size, + drop_remainder, + num_workers, + ): + self.dataset = dataset + self.cols_to_retain = cols_to_retain + self.collate_fn = collate_fn + self.collate_fn_args = collate_fn_args + self.string_columns = [col for col, dtype in columns_to_np_types.items() if dtype is np.str_] + # Strings will be converted to arrays of single unicode chars, so that we can have a constant itemsize + self.columns_to_np_types = { + col: dtype if col not in self.string_columns else np.dtype("U1") + for col, dtype in columns_to_np_types.items() + } + self.output_signature = output_signature + self.shuffle = shuffle + self.batch_size = batch_size + self.drop_remainder = drop_remainder + self.num_workers = num_workers + # Because strings are converted to characters, we need to add one extra dimension to the shape + self.columns_to_ranks = { + col: int(spec.shape.rank) if col not in self.string_columns else int(spec.shape.rank) + 1 + for col, spec in output_signature.items() + } + + def __iter__(self): + # Make sure we only spawn workers if they have work to do + num_workers = min(self.num_workers, int(ceil(len(self.dataset) / self.batch_size))) + # Do the shuffling in iter so that it's done at the start of each epoch + per_worker_batches, final_batch, final_batch_worker = self.distribute_batches( + self.dataset, self.batch_size, self.drop_remainder, num_workers, self.shuffle + ) + ctx = get_context("spawn") + names = [] + shape_arrays = [] + workers = [] + array_ready_events = [ctx.Event() for _ in range(num_workers)] + array_loaded_events = [ctx.Event() for _ in range(num_workers)] + + base_args = { + "dataset": self.dataset, + "cols_to_retain": self.cols_to_retain, + "collate_fn": self.collate_fn, + "collate_fn_args": self.collate_fn_args, + "columns_to_np_types": self.columns_to_np_types, + "columns_to_ranks": self.columns_to_ranks, + "string_columns": self.string_columns, + } + with SharedMemoryContext() as shm_ctx: + for i in range(num_workers): + worker_random_id = str(uuid4()) + worker_name = f"dw_{i}_{worker_random_id}"[:10] + names.append(worker_name) + + worker_shape_arrays = { + col: shm_ctx.get_array(f"{worker_name}_{col}_shape", shape=(rank,), dtype=np.int64, create=True) + for col, rank in self.columns_to_ranks.items() + } + shape_arrays.append(worker_shape_arrays) + + worker_indices = per_worker_batches[i] + if i == final_batch_worker and final_batch is not None: + final_batch_arg = final_batch + else: + final_batch_arg = None + worker_kwargs = { + "worker_name": worker_name, + "indices": worker_indices, + "extra_batch": final_batch_arg, + "array_ready_event": array_ready_events[i], + "array_loaded_event": array_loaded_events[i], + **base_args, + } + worker = ctx.Process(target=self.worker_loop, kwargs=worker_kwargs, daemon=True) + worker.start() + workers.append(worker) + + end_signal_received = False + while not end_signal_received: + for i in range(num_workers): + if not array_ready_events[i].wait(timeout=60): + raise TimeoutError("Data loading worker timed out!") + array_ready_events[i].clear() + array_shapes = shape_arrays[i] + if any(np.any(shape < 0) for shape in array_shapes.values()): + # Child processes send negative array shapes to indicate + # that no more data is going to be sent + end_signal_received = True + break + # Matt: Because array shapes are variable we recreate the shared memory each iteration. + # I suspect repeatedly opening lots of shared memory is the bottleneck for the parent process. + # A future optimization, at the cost of some code complexity, could be to reuse shared memory + # between iterations, but this would require knowing in advance the maximum size, or having + # a system to only create a new memory block when a new maximum size is seen. + # Another potential optimization would be to figure out which memory copies are necessary, + # or whether we can yield objects straight out of shared memory. + with SharedMemoryContext() as batch_shm_ctx: + # This memory context only lasts long enough to copy everything out of the batch + arrays = { + col: batch_shm_ctx.get_array( + f"{names[i]}_{col}", + shape=shape, + dtype=self.columns_to_np_types[col], + create=False, + ) + for col, shape in array_shapes.items() + } + # Copy everything out of shm because the memory + # will be unlinked by the child process at some point + arrays = {col: np.copy(arr) for col, arr in arrays.items()} + # Now we convert any unicode char arrays to strings + for string_col in self.string_columns: + arrays[string_col] = ( + arrays[string_col].view(f"U{arrays[string_col].shape[-1]}").squeeze(-1) + ) + yield arrays + array_loaded_events[i].set() + # Now we just do some cleanup + # Shared memory is cleaned up by the context manager, so we just make sure workers finish + for worker in workers: + worker.join() + + def __call__(self): + return self + + @staticmethod + def worker_loop( + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + columns_to_ranks, + string_columns, + indices, + extra_batch, + worker_name, + array_ready_event, + array_loaded_event, + ): + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" + + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + tf.config.set_visible_devices([], "GPU") # Make sure workers don't try to allocate GPU memory + + def send_batch_to_parent(indices): + batch = np_get_batch( + indices=indices, + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + return_dict=True, + ) + + # Now begins the fun part where we start shovelling shared memory at the parent process + out_arrays = {} + with SharedMemoryContext() as batch_shm_ctx: + # The batch shared memory context exists only as long as it takes for the parent process + # to read everything, after which it cleans everything up again + for col, cast_dtype in columns_to_np_types.items(): + # Everything has to be np.array for this to work, even if the collate_fn is giving us tf.Tensor + array = batch[col] + if col in string_columns: + # We can't send unicode arrays over shared memory, so we convert to single chars ("U1") + # which have a fixed width of 4 bytes. The parent process will convert these back to strings. + array = array.view("U1").reshape(array.shape + (-1,)) + shape_arrays[col][:] = array.shape + out_arrays[col] = batch_shm_ctx.get_array( + f"{worker_name}_{col}", shape=array.shape, dtype=cast_dtype, create=True + ) + out_arrays[col][:] = array + + array_ready_event.set() + array_loaded_event.wait() + array_loaded_event.clear() + + with SharedMemoryContext() as shm_ctx: + shape_arrays = { + col: shm_ctx.get_array(f"{worker_name}_{col}_shape", shape=(rank,), dtype=np.int64, create=False) + for col, rank in columns_to_ranks.items() + } + + for batch in indices: + send_batch_to_parent(batch) + if extra_batch is not None: + send_batch_to_parent(extra_batch) + # Now we send a batsignal to the parent process that we're done + for col, array in shape_arrays.items(): + array[:] = -1 + array_ready_event.set() + + @staticmethod + def distribute_batches(dataset, batch_size, drop_remainder, num_workers, shuffle): + indices = np.arange(len(dataset)) + if shuffle: + np.random.shuffle(indices) + num_samples = len(indices) + # We distribute the batches so that reading from the workers in round-robin order yields the exact + # order specified in indices. This is only important when shuffle is False, but we do it regardless. + incomplete_batch_cutoff = num_samples - (num_samples % batch_size) + indices, last_incomplete_batch = np.split(indices, [incomplete_batch_cutoff]) + if drop_remainder or len(last_incomplete_batch) == 0: + last_incomplete_batch = None + + indices = indices.reshape(-1, batch_size) + num_batches = len(indices) + final_batches_cutoff = num_batches - (num_batches % num_workers) + indices, final_batches = np.split(indices, [final_batches_cutoff]) + indices = indices.reshape(-1, num_workers, batch_size) + + per_worker_indices = np.split(indices, indices.shape[1], axis=1) + per_worker_indices = [np.squeeze(worker_indices, 1) for worker_indices in per_worker_indices] + # Distribute the final batches to the first workers + for i in range(len(final_batches)): + # len(final_batches) can be zero, and is always less than num_workers + per_worker_indices[i] = np.concatenate([per_worker_indices[i], final_batches[i].reshape(1, -1)], axis=0) + # Add the last incomplete batch to the next worker, which might be the first worker + if last_incomplete_batch is not None: + incomplete_batch_worker_idx = len(final_batches) + else: + incomplete_batch_worker_idx = None + return per_worker_indices, last_incomplete_batch, incomplete_batch_worker_idx + + +def multiprocess_dataset_to_tf( + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + output_signature, + shuffle, + batch_size, + drop_remainder, + num_workers, +): + """Create a tf.data.Dataset from the underlying Dataset. This is a multi-process method - the single-process + equivalent is dataset_to_tf. + + Args: + dataset (`Dataset`): Dataset to wrap with tf.data.Dataset. + cols_to_retain (`List[str]`): Dataset column(s) to load in the + tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and + that do not exist in the original dataset. + collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate + lists of samples into a batch. + collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the + `collate_fn`. Can be empty. + columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes. + output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to + `tf.TensorSpec` objects. + shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for + validation/evaluation. + batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that + the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`. + drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided, + defaults to the same setting as shuffle. + num_workers (`int`): Number of workers to use for loading the dataset. Should be >= 1. + + Returns: + `tf.data.Dataset` + """ + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + data_generator = NumpyMultiprocessingGenerator( + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + output_signature=output_signature, + shuffle=shuffle, + batch_size=batch_size, + drop_remainder=drop_remainder, + num_workers=num_workers, + ) + + tf_dataset = tf.data.Dataset.from_generator(data_generator, output_signature=output_signature) + if drop_remainder: + dataset_length = int(len(dataset) // batch_size) + else: + dataset_length = int(ceil(len(dataset) / batch_size)) + return tf_dataset.apply(tf.data.experimental.assert_cardinality(dataset_length)) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/tqdm.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/tqdm.py new file mode 100644 index 0000000000000000000000000000000000000000..88ab7c067abf94290e16c206ef50bdcaa7f70d3b --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/tqdm.py @@ -0,0 +1,135 @@ +"""Utility helpers to handle progress bars in `datasets`. + +Example: + 1. Use `datasets.utils.tqdm` as you would use `tqdm.tqdm` or `tqdm.auto.tqdm`. + 2. To disable progress bars, either use `disable_progress_bars()` helper or set the + environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS` to 1. + 3. To re-enable progress bars, use `enable_progress_bars()`. + 4. To check whether progress bars are disabled, use `are_progress_bars_disabled()`. + +NOTE: Environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS` has the priority. + +Example: + ```py + from datasets.utils import ( + are_progress_bars_disabled, + disable_progress_bars, + enable_progress_bars, + tqdm, + ) + + # Disable progress bars globally + disable_progress_bars() + + # Use as normal `tqdm` + for _ in tqdm(range(5)): + do_something() + + # Still not showing progress bars, as `disable=False` is overwritten to `True`. + for _ in tqdm(range(5), disable=False): + do_something() + + are_progress_bars_disabled() # True + + # Re-enable progress bars globally + enable_progress_bars() + + # Progress bar will be shown ! + for _ in tqdm(range(5)): + do_something() + ``` +""" + +import os +import warnings + +from tqdm.auto import tqdm as old_tqdm + +from ..config import HF_DATASETS_DISABLE_PROGRESS_BARS + + +# `HF_DATASETS_DISABLE_PROGRESS_BARS` is `Optional[bool]` while `_hf_datasets_progress_bars_disabled` +# is a `bool`. If `HF_DATASETS_DISABLE_PROGRESS_BARS` is set to True or False, it has priority. +# If `HF_DATASETS_DISABLE_PROGRESS_BARS` is None, it means the user have not set the +# environment variable and is free to enable/disable progress bars programmatically. +# TL;DR: env variable has priority over code. +# +# By default, progress bars are enabled. +_hf_datasets_progress_bars_disabled: bool = HF_DATASETS_DISABLE_PROGRESS_BARS or False + + +def disable_progress_bars() -> None: + """ + Disable globally progress bars used in `datasets` except if `HF_DATASETS_DISABLE_PROGRESS_BARS` environment + variable has been set. + + Use [`~utils.enable_progress_bars`] to re-enable them. + """ + if HF_DATASETS_DISABLE_PROGRESS_BARS is False: + warnings.warn( + "Cannot disable progress bars: environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS=0` is set and has" + " priority." + ) + return + global _hf_datasets_progress_bars_disabled + _hf_datasets_progress_bars_disabled = True + + +def enable_progress_bars() -> None: + """ + Enable globally progress bars used in `datasets` except if `HF_DATASETS_DISABLE_PROGRESS_BARS` environment + variable has been set. + + Use [`~utils.disable_progress_bars`] to disable them. + """ + if HF_DATASETS_DISABLE_PROGRESS_BARS is True: + warnings.warn( + "Cannot enable progress bars: environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS=1` is set and has" + " priority." + ) + return + global _hf_datasets_progress_bars_disabled + _hf_datasets_progress_bars_disabled = False + + +def are_progress_bars_disabled() -> bool: + """Return whether progress bars are globally disabled or not. + + Progress bars used in `datasets` can be enable or disabled globally using [`~utils.enable_progress_bars`] + and [`~utils.disable_progress_bars`] or by setting `HF_DATASETS_DISABLE_PROGRESS_BARS` as environment variable. + """ + global _hf_datasets_progress_bars_disabled + return _hf_datasets_progress_bars_disabled + + +class tqdm(old_tqdm): + """ + Class to override `disable` argument in case progress bars are globally disabled. + + Taken from https://github.com/tqdm/tqdm/issues/619#issuecomment-619639324. + """ + + def __init__(self, *args, **kwargs): + if are_progress_bars_disabled(): + kwargs["disable"] = True + elif kwargs.get("disable") is None and os.getenv("TQDM_POSITION") == "-1": + # Force-enable progress bars in cloud environments when disable=None + kwargs["disable"] = False + super().__init__(*args, **kwargs) + + def __delattr__(self, attr: str) -> None: + """Fix for https://github.com/huggingface/datasets/issues/6066""" + try: + super().__delattr__(attr) + except AttributeError: + if attr != "_lock": + raise + + +# backward compatibility +enable_progress_bar = enable_progress_bars +disable_progress_bar = disable_progress_bars + + +def is_progress_bar_enabled(): + return not are_progress_bars_disabled() diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/track.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/track.py new file mode 100644 index 0000000000000000000000000000000000000000..ffd3434af790de63fdd046ad8f93c3067c6e6da8 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/track.py @@ -0,0 +1,61 @@ +from collections.abc import Iterable, Iterator + + +class tracked_str(str): + origins = {} + + def set_origin(self, origin: str): + if super().__repr__() not in self.origins: + self.origins[super().__repr__()] = origin + + def get_origin(self): + return self.origins.get(super().__repr__(), str(self)) + + def __repr__(self) -> str: + if super().__repr__() not in self.origins or self.origins[super().__repr__()] == self: + return super().__repr__() + else: + return f"{str(self)} (origin={self.origins[super().__repr__()]})" + + +class tracked_list(list): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.last_item = None + + def __iter__(self) -> Iterator: + for x in super().__iter__(): + self.last_item = x + yield x + self.last_item = None + + def __repr__(self) -> str: + if self.last_item is None: + return super().__repr__() + else: + return f"{self.__class__.__name__}(current={self.last_item})" + + +class TrackedIterableFromGenerator(Iterable): + """Utility class to create an iterable from a generator function, in order to reset the generator when needed.""" + + def __init__(self, generator, *args): + super().__init__() + self.generator = generator + self.args = args + self.last_item = None + + def __iter__(self): + for x in self.generator(*self.args): + self.last_item = x + yield x + self.last_item = None + + def __repr__(self) -> str: + if self.last_item is None: + return super().__repr__() + else: + return f"{self.__class__.__name__}(current={self.last_item})" + + def __reduce__(self): + return (self.__class__, (self.generator, *self.args)) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/typing.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/typing.py new file mode 100644 index 0000000000000000000000000000000000000000..4736d74cbb575bd0e12f625c520202a8c74b3ddc --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/typing.py @@ -0,0 +1,9 @@ +import os +from typing import TypeVar, Union + + +T = TypeVar("T") + +ListLike = Union[list[T], tuple[T, ...]] +NestedDataStructureLike = Union[T, list[T], dict[str, T]] +PathLike = Union[str, bytes, os.PathLike] diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/version.py b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/version.py new file mode 100644 index 0000000000000000000000000000000000000000..75cf4c39d5f9b916ade8f8d88a0f0ca9e5769217 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/datasets/utils/version.py @@ -0,0 +1,106 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Version utils.""" + +import dataclasses +import re +from dataclasses import dataclass +from functools import total_ordering +from typing import Optional, Union + + +_VERSION_REG = re.compile(r"^(?P\d+)" r"\.(?P\d+)" r"\.(?P\d+)$") + + +@total_ordering +@dataclass +class Version: + """Dataset version `MAJOR.MINOR.PATCH`. + + Args: + version_str (`str`): + The dataset version. + description (`str`): + A description of what is new in this version. + major (`str`): + minor (`str`): + patch (`str`): + + Example: + + ```py + >>> VERSION = datasets.Version("1.0.0") + ``` + """ + + version_str: str + description: Optional[str] = None + major: Optional[Union[str, int]] = None + minor: Optional[Union[str, int]] = None + patch: Optional[Union[str, int]] = None + + def __post_init__(self): + self.major, self.minor, self.patch = _str_to_version_tuple(self.version_str) + + def __repr__(self): + return f"{self.tuple[0]}.{self.tuple[1]}.{self.tuple[2]}" + + @property + def tuple(self): + return self.major, self.minor, self.patch + + def _validate_operand(self, other): + if isinstance(other, str): + return Version(other) + elif isinstance(other, Version): + return other + raise TypeError(f"{other} (type {type(other)}) cannot be compared to version.") + + def __eq__(self, other): + try: + other = self._validate_operand(other) + except (TypeError, ValueError): + return False + else: + return self.tuple == other.tuple + + def __lt__(self, other): + other = self._validate_operand(other) + return self.tuple < other.tuple + + def __hash__(self): + return hash(_version_tuple_to_str(self.tuple)) + + @classmethod + def from_dict(cls, dic): + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in dic.items() if k in field_names}) + + def _to_yaml_string(self) -> str: + return self.version_str + + +def _str_to_version_tuple(version_str): + """Return the tuple (major, minor, patch) version extracted from the str.""" + res = _VERSION_REG.match(version_str) + if not res: + raise ValueError(f"Invalid version '{version_str}'. Format should be x.y.z with {{x,y,z}} being digits.") + return tuple(int(v) for v in [res.group("major"), res.group("minor"), res.group("patch")]) + + +def _version_tuple_to_str(version_tuple): + """Return the str version from the version tuple (major, minor, patch).""" + return ".".join(str(v) for v in version_tuple) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/INSTALLER b/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/METADATA b/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..90d70dcdb96d05813eeb977759c773fb8ad30b96 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/METADATA @@ -0,0 +1,38 @@ +Metadata-Version: 2.4 +Name: filelock +Version: 3.29.0 +Summary: A platform independent file lock. +Project-URL: Documentation, https://py-filelock.readthedocs.io +Project-URL: Homepage, https://github.com/tox-dev/py-filelock +Project-URL: Source, https://github.com/tox-dev/py-filelock +Project-URL: Tracker, https://github.com/tox-dev/py-filelock/issues +Maintainer-email: Bernát Gábor +License-Expression: MIT +License-File: LICENSE +Keywords: application,cache,directory,log,user +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Topic :: Internet +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: System +Requires-Python: >=3.10 +Description-Content-Type: text/markdown + +# filelock + +[![PyPI](https://img.shields.io/pypi/v/filelock)](https://pypi.org/project/filelock/) +[![Supported Python versions](https://img.shields.io/pypi/pyversions/filelock.svg)](https://pypi.org/project/filelock/) +[![Documentation status](https://readthedocs.org/projects/py-filelock/badge/?version=latest)](https://py-filelock.readthedocs.io/en/latest/?badge=latest) +[![Downloads](https://static.pepy.tech/badge/filelock/month)](https://pepy.tech/project/filelock) +[![check](https://github.com/tox-dev/py-filelock/actions/workflows/check.yaml/badge.svg)](https://github.com/tox-dev/py-filelock/actions/workflows/check.yaml) + +For more information checkout the [official documentation](https://py-filelock.readthedocs.io/en/latest/index.html). diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/RECORD b/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..0d8ce7b6f9f9ce2c4186473df4e0458c0649a2ca --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/RECORD @@ -0,0 +1,21 @@ +filelock-3.29.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +filelock-3.29.0.dist-info/METADATA,sha256=ITNdGUDKAIMOF1lOgentezkyEbHgPto8QRo1B79Z9Lc,1977 +filelock-3.29.0.dist-info/RECORD,, +filelock-3.29.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +filelock-3.29.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87 +filelock-3.29.0.dist-info/licenses/LICENSE,sha256=YIyJ1QYK6ZIa3M8yNmlbxlSplG4SMj72wCHfoE4pTUg,1088 +filelock/__init__.py,sha256=Ig_QNvsAVtctz2-fkq9EUCgXmW6M5PDudiOC85szmis,2612 +filelock/_api.py,sha256=UxKGuyzWqzb0Bji8_Czq75TVcQWIH1NTNhEHUxyx2js,21347 +filelock/_async_read_write.py,sha256=C_cRi4sZ7_CWcqdEf0pcA5sdJ2m3EI_2tAW6kS_5P6I,7759 +filelock/_error.py,sha256=mnelOh0EVyVeskG3rksL4kW3OArL4TMb2-PwmzQWIFg,788 +filelock/_read_write.py,sha256=5tVPm1TDXvAu0sgva_LQJ4DH-SDo0rVxuYAFLEQGZw4,15323 +filelock/_soft.py,sha256=t2Q3ikXjI5eVvNqNwntXH1i_Tm84_M9XSRBbtUOCrBc,7833 +filelock/_soft_rw/__init__.py,sha256=_ktpGmVObzrvL0K1_EAV3LF7B9JarYxW-mx_eyGPhT0,370 +filelock/_soft_rw/_async.py,sha256=3MdgJN0SBM-9-Msslt4HeikJtz2cc1tbeN1sWK9NV60,8704 +filelock/_soft_rw/_sync.py,sha256=gBbGRLEEI42hh17uFGFB654HyAhkqJ0NJ4XFlvTX0Ac,34936 +filelock/_unix.py,sha256=pbTsIi29Qv8zBtLsXqc3PzlS3wz2db53M1MIqVo9790,4579 +filelock/_util.py,sha256=wPf-LvzmdHWHw4DinDeVJlB0GNVOYGNUeNKSnHBQuUU,1716 +filelock/_windows.py,sha256=kJFOenaffOP6Qza2b3-yxgiPiDcyroWnzz9STGkCrgM,3943 +filelock/asyncio.py,sha256=Hvcow3feO_EPvo24a_bgbsBoHkNeIiIqipUWoHrCWKk,14556 +filelock/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +filelock/version.py,sha256=yPxgPkfVUcMLHpiiTghdwfldZS0DIKnGryve3QX7bOs,522 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/REQUESTED b/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/WHEEL b/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..b1b94fd58e7e9ed0ef3449473bc48de68afcc3fe --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.29.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/licenses/LICENSE b/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..291919c0b6f41d014767f6c877af9f7595fcff99 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/filelock-3.29.0.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Bernát Gábor and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/INSTALLER b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/METADATA b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..30d66530bc93801fc3385006ce179e72b42644df --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/METADATA @@ -0,0 +1,672 @@ +Metadata-Version: 2.4 +Name: frozenlist +Version: 1.8.0 +Summary: A list-like structure which implements collections.abc.MutableSequence +Home-page: https://github.com/aio-libs/frozenlist +Maintainer: aiohttp team +Maintainer-email: team@aiohttp.org +License: Apache-2.0 +Project-URL: Chat: Matrix, https://matrix.to/#/#aio-libs:matrix.org +Project-URL: Chat: Matrix Space, https://matrix.to/#/#aio-libs-space:matrix.org +Project-URL: CI: Github Actions, https://github.com/aio-libs/frozenlist/actions +Project-URL: Code of Conduct, https://github.com/aio-libs/.github/blob/master/CODE_OF_CONDUCT.md +Project-URL: Coverage: codecov, https://codecov.io/github/aio-libs/frozenlist +Project-URL: Docs: Changelog, https://github.com/aio-libs/frozenlist/blob/master/CHANGES.rst#changelog +Project-URL: Docs: RTD, https://frozenlist.aio-libs.org +Project-URL: GitHub: issues, https://github.com/aio-libs/frozenlist/issues +Project-URL: GitHub: repo, https://github.com/aio-libs/frozenlist +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Operating System :: POSIX +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: Microsoft :: Windows +Classifier: Programming Language :: Cython +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Requires-Python: >=3.9 +Description-Content-Type: text/x-rst +License-File: LICENSE +Dynamic: license-file + +frozenlist +========== + +.. image:: https://github.com/aio-libs/frozenlist/workflows/CI/badge.svg + :target: https://github.com/aio-libs/frozenlist/actions + :alt: GitHub status for master branch + +.. image:: https://codecov.io/gh/aio-libs/frozenlist/branch/master/graph/badge.svg?flag=pytest + :target: https://codecov.io/gh/aio-libs/frozenlist?flags[]=pytest + :alt: codecov.io status for master branch + +.. image:: https://img.shields.io/pypi/v/frozenlist.svg?logo=Python&logoColor=white + :target: https://pypi.org/project/frozenlist + :alt: frozenlist @ PyPI + +.. image:: https://readthedocs.org/projects/frozenlist/badge/?version=latest + :target: https://frozenlist.aio-libs.org + :alt: Read The Docs build status badge + +.. image:: https://img.shields.io/matrix/aio-libs:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat + :target: https://matrix.to/#/%23aio-libs:matrix.org + :alt: Matrix Room — #aio-libs:matrix.org + +.. image:: https://img.shields.io/matrix/aio-libs-space:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs-space%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat + :target: https://matrix.to/#/%23aio-libs-space:matrix.org + :alt: Matrix Space — #aio-libs-space:matrix.org + +Introduction +------------ + +``frozenlist.FrozenList`` is a list-like structure which implements +``collections.abc.MutableSequence``. The list is *mutable* until ``FrozenList.freeze`` +is called, after which list modifications raise ``RuntimeError``: + + +>>> from frozenlist import FrozenList +>>> fl = FrozenList([17, 42]) +>>> fl.append('spam') +>>> fl.append('Vikings') +>>> fl + +>>> fl.freeze() +>>> fl + +>>> fl.frozen +True +>>> fl.append("Monty") +Traceback (most recent call last): + File "", line 1, in + File "frozenlist/_frozenlist.pyx", line 97, in frozenlist._frozenlist.FrozenList.append + self._check_frozen() + File "frozenlist/_frozenlist.pyx", line 19, in frozenlist._frozenlist.FrozenList._check_frozen + raise RuntimeError("Cannot modify frozen list.") +RuntimeError: Cannot modify frozen list. + + +FrozenList is also hashable, but only when frozen. Otherwise it also throws a RuntimeError: + + +>>> fl = FrozenList([17, 42, 'spam']) +>>> hash(fl) +Traceback (most recent call last): + File "", line 1, in + File "frozenlist/_frozenlist.pyx", line 111, in frozenlist._frozenlist.FrozenList.__hash__ + raise RuntimeError("Cannot hash unfrozen list.") +RuntimeError: Cannot hash unfrozen list. +>>> fl.freeze() +>>> hash(fl) +3713081631934410656 +>>> dictionary = {fl: 'Vikings'} # frozen fl can be a dict key +>>> dictionary +{: 'Vikings'} + + +Installation +------------ + +:: + + $ pip install frozenlist + + +Documentation +------------- + +https://frozenlist.aio-libs.org + +Communication channels +---------------------- + +We have a *Matrix Space* `#aio-libs-space:matrix.org +`_ which is +also accessible via Gitter. + +License +------- + +``frozenlist`` is offered under the Apache 2 license. + +Source code +----------- + +The project is hosted on GitHub_ + +Please file an issue in the `bug tracker +`_ if you have found a bug +or have some suggestions to improve the library. + +.. _GitHub: https://github.com/aio-libs/frozenlist + +========= +Changelog +========= + +.. + You should *NOT* be adding new change log entries to this file, this + file is managed by towncrier. You *may* edit previous change logs to + fix problems like typo corrections or such. + To add a new change log entry, please see + https://pip.pypa.io/en/latest/development/contributing/#news-entries + we named the news folder "changes". + + WARNING: Don't drop the next directive! + +.. towncrier release notes start + +v1.8.0 +====== + +*(2025-10-05)* + + +Contributor-facing changes +-------------------------- + +- The ``reusable-cibuildwheel.yml`` workflow has been refactored to + be more generic and ``ci-cd.yml`` now holds all the configuration + toggles -- by `@webknjaz `__. + + *Related issues and pull requests on GitHub:* + `#668 `__. + +- When building wheels, the source distribution is now passed directly + to the ``cibuildwheel`` invocation -- by `@webknjaz `__. + + *Related issues and pull requests on GitHub:* + `#669 `__. + +- Builds and tests have been added to + ``ci-cd.yml`` for arm64 Windows wheels -- by `@finnagin `__. + + *Related issues and pull requests on GitHub:* + `#677 `__. + +- Started building wheels for CPython 3.14 -- by `@kumaraditya303 `__. + + *Related issues and pull requests on GitHub:* + `#681 `__, `#682 `__. + +- Removed ``--config-settings=pure-python=false`` from ``requirements/dev.txt``. + Developers on CPython still get accelerated builds by default. To explicitly build + a pure Python wheel, use ``pip install -e . --config-settings=pure-python=true`` + -- by `@bdraco `__. + + *Related issues and pull requests on GitHub:* + `#687 `__. + + +---- + + +v1.7.0 +====== + +*(2025-06-09)* + + +Features +-------- + +- Added deepcopy support to FrozenList -- by `@bdraco `__. + + *Related issues and pull requests on GitHub:* + `#659 `__. + + +Packaging updates and notes for downstreams +------------------------------------------- + +- Fixed an issue where ``frozenlist`` binary wheels would be built with debugging symbols and line tracing enabled, which significantly impacted performance. Line tracing is now disabled by default and can only be enabled explicitly -- by `@bdraco `__. + + This change ensures that production builds are optimized for performance. Developers who need line tracing for debugging purposes can still enable it by: + + 1. Setting the ``FROZENLIST_CYTHON_TRACING`` environment variable + 2. Using the ``--config-setting=with-cython-tracing=true`` option with pip + + *Related issues and pull requests on GitHub:* + `#660 `__. + +- Enabled ``PIP_CONSTRAINT`` environment variable in the build configuration to ensure the pinned Cython version from ``requirements/cython.txt`` is used during wheel builds. + + *Related issues and pull requests on GitHub:* + `#661 `__. + + +---- + + +v1.6.2 +====== + +*(2025-06-03)* + + +No significant changes. + + +---- + + +v1.6.1 +====== + +*(2025-06-02)* + + +Bug fixes +--------- + +- Correctly use ``cimport`` for including ``PyBool_FromLong`` -- by `@lysnikolaou `__. + + *Related issues and pull requests on GitHub:* + `#653 `__. + + +Packaging updates and notes for downstreams +------------------------------------------- + +- Exclude ``_frozenlist.cpp`` from bdists/wheels -- by `@musicinmybrain `__. + + *Related issues and pull requests on GitHub:* + `#649 `__. + +- Updated to use Cython 3.1 universally across the build path -- by `@lysnikolaou `__. + + *Related issues and pull requests on GitHub:* + `#654 `__. + + +---- + + +v1.6.0 +====== + +*(2025-04-17)* + + +Bug fixes +--------- + +- Stopped implicitly allowing the use of Cython pre-release versions when + building the distribution package -- by `@ajsanchezsanz `__ and + `@markgreene74 `__. + + *Related commits on GitHub:* + `41591f2 `__. + + +Features +-------- + +- Implemented support for the free-threaded build of CPython 3.13 -- by `@lysnikolaou `__. + + *Related issues and pull requests on GitHub:* + `#618 `__. + +- Started building armv7l wheels -- by `@bdraco `__. + + *Related issues and pull requests on GitHub:* + `#642 `__. + + +Packaging updates and notes for downstreams +------------------------------------------- + +- Stopped implicitly allowing the use of Cython pre-release versions when + building the distribution package -- by `@ajsanchezsanz `__ and + `@markgreene74 `__. + + *Related commits on GitHub:* + `41591f2 `__. + +- Started building wheels for the free-threaded build of CPython 3.13 -- by `@lysnikolaou `__. + + *Related issues and pull requests on GitHub:* + `#618 `__. + +- The packaging metadata switched to including an SPDX license identifier introduced in `PEP 639 `__ -- by `@cdce8p `__. + + *Related issues and pull requests on GitHub:* + `#639 `__. + + +Contributor-facing changes +-------------------------- + +- GitHub Actions CI/CD is now configured to manage caching pip-ecosystem + dependencies using `re-actors/cache-python-deps`_ -- an action by + `@webknjaz `__ that takes into account ABI stability and the exact + version of Python runtime. + + .. _`re-actors/cache-python-deps`: + https://github.com/marketplace/actions/cache-python-deps + + *Related issues and pull requests on GitHub:* + `#633 `__. + +- Organized dependencies into test and lint dependencies so that no + unnecessary ones are installed during CI runs -- by `@lysnikolaou `__. + + *Related issues and pull requests on GitHub:* + `#636 `__. + + +---- + + +1.5.0 (2024-10-22) +================== + +Bug fixes +--------- + +- An incorrect signature of the ``__class_getitem__`` class method + has been fixed, adding a missing ``class_item`` argument under + Python 3.8 and older. + + This change also improves the code coverage of this method that + was previously missing -- by `@webknjaz `__. + + + *Related issues and pull requests on GitHub:* + `#567 `__, `#571 `__. + + +Improved documentation +---------------------- + +- Rendered issue, PR, and commit links now lead to + ``frozenlist``'s repo instead of ``yarl``'s repo. + + + *Related issues and pull requests on GitHub:* + `#573 `__. + +- On the ``Contributing docs`` page, + a link to the ``Towncrier philosophy`` has been fixed. + + + *Related issues and pull requests on GitHub:* + `#574 `__. + + +Packaging updates and notes for downstreams +------------------------------------------- + +- A name of a temporary building directory now reflects + that it's related to ``frozenlist``, not ``yarl``. + + + *Related issues and pull requests on GitHub:* + `#573 `__. + +- Declared Python 3.13 supported officially in the distribution package metadata. + + + *Related issues and pull requests on GitHub:* + `#595 `__. + + +---- + + +1.4.1 (2023-12-15) +================== + +Packaging updates and notes for downstreams +------------------------------------------- + +- Declared Python 3.12 and PyPy 3.8-3.10 supported officially + in the distribution package metadata. + + + *Related issues and pull requests on GitHub:* + `#553 `__. + +- Replaced the packaging is replaced from an old-fashioned ``setup.py`` to an + in-tree `PEP 517 `__ build backend -- by `@webknjaz `__. + + Whenever the end-users or downstream packagers need to build ``frozenlist`` + from source (a Git checkout or an sdist), they may pass a ``config_settings`` + flag ``pure-python``. If this flag is not set, a C-extension will be built + and included into the distribution. + + Here is how this can be done with ``pip``: + + .. code-block:: console + + $ python3 -m pip install . --config-settings=pure-python= + + This will also work with ``-e | --editable``. + + The same can be achieved via ``pypa/build``: + + .. code-block:: console + + $ python3 -m build --config-setting=pure-python= + + Adding ``-w | --wheel`` can force ``pypa/build`` produce a wheel from source + directly, as opposed to building an ``sdist`` and then building from it. + + + *Related issues and pull requests on GitHub:* + `#560 `__. + + +Contributor-facing changes +-------------------------- + +- It is now possible to request line tracing in Cython builds using the + ``with-cython-tracing`` `PEP 517 `__ config setting + -- `@webknjaz `__. + + This can be used in CI and development environment to measure coverage + on Cython modules, but is not normally useful to the end-users or + downstream packagers. + + Here's a usage example: + + .. code-block:: console + + $ python3 -Im pip install . --config-settings=with-cython-tracing=true + + For editable installs, this setting is on by default. Otherwise, it's + off unless requested explicitly. + + The following produces C-files required for the Cython coverage + plugin to map the measurements back to the PYX-files: + + .. code-block:: console + + $ python -Im pip install -e . + + Alternatively, the ``FROZENLIST_CYTHON_TRACING=1`` environment variable + can be set to do the same as the `PEP 517 `__ config setting. + + + *Related issues and pull requests on GitHub:* + `#560 `__. + +- Coverage collection has been implemented for the Cython modules + -- by `@webknjaz `__. + + It will also be reported to Codecov from any non-release CI jobs. + + + *Related issues and pull requests on GitHub:* + `#561 `__. + +- A step-by-step ``Release Guide`` guide has + been added, describing how to release *frozenlist* -- by `@webknjaz `__. + + This is primarily targeting the maintainers. + + + *Related issues and pull requests on GitHub:* + `#563 `__. + +- Detailed ``Contributing Guidelines`` on + authoring the changelog fragments have been published in the + documentation -- by `@webknjaz `__. + + + *Related issues and pull requests on GitHub:* + `#564 `__. + + +---- + + +1.4.0 (2023-07-12) +================== + +The published source distribution package became buildable +under Python 3.12. + + +---- + + +Bugfixes +-------- + +- Removed an unused ``typing.Tuple`` import + `#411 `_ + + +Deprecations and Removals +------------------------- + +- Dropped Python 3.7 support. + `#413 `_ + + +Misc +---- + +- `#410 `_, `#433 `_ + + +---- + + +1.3.3 (2022-11-08) +================== + +- Fixed CI runs when creating a new release, where new towncrier versions + fail when the current version section is already present. + + +---- + + +1.3.2 (2022-11-08) +================== + +Misc +---- + +- Updated the CI runs to better check for test results and to avoid deprecated syntax. `#327 `_ + + +---- + + +1.3.1 (2022-08-02) +================== + +The published source distribution package became buildable +under Python 3.11. + + +---- + + +1.3.0 (2022-01-18) +================== + +Bugfixes +-------- + +- Do not install C sources with binary distributions. + `#250 `_ + + +Deprecations and Removals +------------------------- + +- Dropped Python 3.6 support + `#274 `_ + + +---- + + +1.2.0 (2021-10-16) +================== + +Features +-------- + +- ``FrozenList`` now supports being used as a generic type as per PEP 585, e.g. ``frozen_int_list: FrozenList[int]`` (requires Python 3.9 or newer). + `#172 `_ +- Added support for Python 3.10. + `#227 `_ +- Started shipping platform-specific wheels with the ``musl`` tag targeting typical Alpine Linux runtimes. + `#227 `_ +- Started shipping platform-specific arm64 wheels for Apple Silicon. + `#227 `_ + + +---- + + +1.1.1 (2020-11-14) +================== + +Bugfixes +-------- + +- Provide x86 Windows wheels. + `#169 `_ + + +---- + + +1.1.0 (2020-10-13) +================== + +Features +-------- + +- Add support for hashing of a frozen list. + `#136 `_ + +- Support Python 3.8 and 3.9. + +- Provide wheels for ``aarch64``, ``i686``, ``ppc64le``, ``s390x`` architectures on + Linux as well as ``x86_64``. + + +---- + + +1.0.0 (2019-11-09) +================== + +Deprecations and Removals +------------------------- + +- Dropped support for Python 3.5; only 3.6, 3.7 and 3.8 are supported going forward. + `#24 `_ diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/RECORD b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..d28cd86b71f7267f1f2c6bd65fefed05bed3b4c0 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/RECORD @@ -0,0 +1,12 @@ +frozenlist-1.8.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +frozenlist-1.8.0.dist-info/METADATA,sha256=lGwi3J9-LHby1pjwCkMG2-Sagg4rPxoOIv5ShsvsNfE,20333 +frozenlist-1.8.0.dist-info/RECORD,, +frozenlist-1.8.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +frozenlist-1.8.0.dist-info/WHEEL,sha256=UtdoUDxc2iNAgyoBBp-d2vg2qqzNe6ydS3XeIOP1_Hw,186 +frozenlist-1.8.0.dist-info/licenses/LICENSE,sha256=b9UkPpLdf5jsacesN3co50kFcJ_1J6W_mNbQJjwE9bY,11332 +frozenlist-1.8.0.dist-info/top_level.txt,sha256=jivtxsPXA3nK3WBWW2LW5Mtu_GHt8UZA13NeCs2cKuA,11 +frozenlist/__init__.py,sha256=xAIE2u9ncAbjATGIPfno_OJfe8AQ-1h7z_uc73dYsEA,2108 +frozenlist/__init__.pyi,sha256=vMEoES1xGegPtVXoCi9XydEeHsyuIq-KdeXwP5PdsaA,1470 +frozenlist/_frozenlist.cpython-314-x86_64-linux-gnu.so,sha256=Wee0Tb8YIQqFAxMZNhzTZ0ZbkHASPn_FNh08EYgwEoQ,749088 +frozenlist/_frozenlist.pyx,sha256=t-aGjuEiVt_MZPBJ0RnraavVmPBK6arz3i48ZvXuYsU,3708 +frozenlist/py.typed,sha256=sow9soTwP9T_gEAQSVh7Gb8855h04Nwmhs2We-JRgZM,7 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/REQUESTED b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/WHEEL b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..ae7b00b985a4c34294b9ae81dfb0f26f6191dd31 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/WHEEL @@ -0,0 +1,7 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.9.0) +Root-Is-Purelib: false +Tag: cp314-cp314-manylinux_2_5_x86_64 +Tag: cp314-cp314-manylinux1_x86_64 +Tag: cp314-cp314-manylinux_2_28_x86_64 + diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/licenses/LICENSE b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..7082a2d5b9047bfc09589f387053e24ea490bc54 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/licenses/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2013-2019 Nikolay Kim and Andrew Svetlov + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/top_level.txt b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..52f13fc459edf8f3def6f792c432f0b64f313176 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/frozenlist-1.8.0.dist-info/top_level.txt @@ -0,0 +1 @@ +frozenlist diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/INSTALLER b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/METADATA b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..1cad428c5702ecbeb8281c8cda3c5b815b878be3 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/METADATA @@ -0,0 +1,87 @@ +Metadata-Version: 2.4 +Name: hf-xet +Version: 1.5.0 +Classifier: Development Status :: 5 - Production/Stable +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: Rust +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Free Threading +Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta +Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence +Requires-Dist: pytest ; extra == 'tests' +Provides-Extra: tests +License-File: LICENSE +Summary: Fast transfer of large files with the Hugging Face Hub. +Maintainer-email: Rajat Arya , Jared Sulzdorf , Di Xiao , Assaf Vayner , Hoyt Koepke +License-Expression: Apache-2.0 +Requires-Python: >=3.8 +Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM +Project-URL: Documentation, https://huggingface.co/docs/hub/xet/index +Project-URL: Homepage, https://github.com/huggingface/xet-core +Project-URL: Issues, https://github.com/huggingface/xet-core/issues +Project-URL: Repository, https://github.com/huggingface/xet-core.git + + +

+ License + GitHub release + Contributor Covenant +

+ +

+

🤗 hf-xet - xet client tech, used in huggingface_hub

+

+ +## Welcome + +`hf-xet` enables `huggingface_hub` to utilize xet storage for uploading and downloading to HF Hub. Xet storage provides chunk-based deduplication, efficient storage/retrieval with local disk caching, and backwards compatibility with Git LFS. This library is not meant to be used directly, and is instead intended to be used from [huggingface_hub](https://pypi.org/project/huggingface-hub). + +## Key features + +♻ **chunk-based deduplication implementation**: avoid transferring and storing chunks that are shared across binary files (models, datasets, etc). + +🤗 **Python bindings**: bindings for [huggingface_hub](https://github.com/huggingface/huggingface_hub/) package. + +↔ **network communications**: concurrent communication to HF Hub Xet backend services (CAS). + +🔖 **local disk caching**: chunk-based cache that sits alongside the existing [huggingface_hub disk cache](https://huggingface.co/docs/huggingface_hub/guides/manage-cache). + +## Installation + +Install the `hf_xet` package with [pip](https://pypi.org/project/hf-xet/): + +```bash +pip install hf_xet +``` + +## Quick Start + +`hf_xet` is not intended to be run independently as it is expected to be used from `huggingface_hub`, so to get started with `huggingface_hub` check out the documentation [here]("https://hf.co/docs/huggingface_hub"). + +## Contributions (feature requests, bugs, etc.) are encouraged & appreciated 💙💚💛💜🧡❤️ + +Please join us in making hf-xet better. We value everyone's contributions. Code is not the only way to help. Answering questions, helping each other, improving documentation, filing issues all help immensely. If you are interested in contributing (please do!), check out the [contribution guide](https://github.com/huggingface/xet-core/blob/main/CONTRIBUTING.md) for this repository. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/RECORD b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..97ceb4a0b85a55de8cad92c06976747a22df38ff --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/RECORD @@ -0,0 +1,9 @@ +hf_xet-1.5.0.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +hf_xet-1.5.0.dist-info/METADATA,sha256=kiRjS5pSbyNKVHLD5pYclp87MWjinAwQrmuPjKJX8yA,4882 +hf_xet-1.5.0.dist-info/RECORD,, +hf_xet-1.5.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +hf_xet-1.5.0.dist-info/WHEEL,sha256=LbLhSGTXlKRmTCKROpRhcqPcmllwkP3mQdJf3GnRbUM,143 +hf_xet-1.5.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357 +hf_xet-1.5.0.dist-info/sboms/hf_xet.cyclonedx.json,sha256=cqFELPUEOVkJUBakT3vgAWYE1N939XMjcP_vAKt_0xw,305769 +hf_xet/__init__.py,sha256=E8UDdyQ8glZ_nve9hHEf22bPang8-RKx4VuApXYeQUo,107 +hf_xet/hf_xet.abi3.so,sha256=6s8jp7y5mQ7kVbKGwgTcVFDzZ8rBdC63i7CPjbm_wEk,11465992 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/REQUESTED b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/WHEEL b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..2598d7e7f7c35b338af38cf8eabcd74b0c04b83f --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: maturin (1.13.1) +Root-Is-Purelib: false +Tag: cp37-abi3-manylinux_2_17_x86_64 +Tag: cp37-abi3-manylinux2014_x86_64 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/licenses/LICENSE b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/licenses/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/sboms/hf_xet.cyclonedx.json b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/sboms/hf_xet.cyclonedx.json new file mode 100644 index 0000000000000000000000000000000000000000..705aa271e651389c4c3ab106d139d34ede189311 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/hf_xet-1.5.0.dist-info/sboms/hf_xet.cyclonedx.json @@ -0,0 +1,9420 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "serialNumber": "urn:uuid:561e6ab6-1b57-4176-abec-4f0b39a90156", + "metadata": { + "timestamp": "2026-05-06T06:03:07.174692068Z", + "tools": [ + { + "vendor": "CycloneDX", + "name": "cargo-cyclonedx", + "version": "0.5.9" + } + ], + "component": { + "type": "library", + "bom-ref": "path+file:///home/runner/work/xet-core/xet-core/hf_xet#1.5.0", + "name": "hf_xet", + "version": "1.5.0", + "scope": "required", + "licenses": [ + { + "expression": "Apache-2.0" + } + ], + "purl": "pkg:cargo/hf_xet@1.5.0?download_url=file://.", + "components": [ + { + "type": "library", + "bom-ref": "path+file:///home/runner/work/xet-core/xet-core/hf_xet#1.5.0 bin-target-0", + "name": "hf_xet", + "version": "1.5.0", + "purl": "pkg:cargo/hf_xet@1.5.0?download_url=file://.#src/lib.rs" + } + ] + }, + "properties": [ + { + "name": "cdx:rustc:sbom:target:all_targets", + "value": "true" + } + ] + }, + "components": [ + { + "type": "library", + "bom-ref": "path+file:///home/runner/work/xet-core/xet-core/xet_client#xet-client@1.5.2", + "name": "xet-client", + "version": "1.5.2", + "description": "Client library for communicating with Hugging Face Xet storage servers. Use through the hf-xet crate.", + "scope": "required", + "licenses": [ + { + "expression": "Apache-2.0" + } + ], + "purl": "pkg:cargo/xet-client@1.5.2?download_url=file:///home/runner/work/xet-core/xet-core/xet_client", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/huggingface/xet-core" + }, + { + "type": "vcs", + "url": "https://github.com/huggingface/xet-core" + } + ] + }, + { + "type": "library", + "bom-ref": "path+file:///home/runner/work/xet-core/xet-core/xet_core_structures#xet-core-structures@1.5.2", + "name": "xet-core-structures", + "version": "1.5.2", + "description": "Core data structures including MerkleHash, metadata shards, and Xorb objects.", + "scope": "required", + "licenses": [ + { + "expression": "Apache-2.0" + } + ], + "purl": "pkg:cargo/xet-core-structures@1.5.2?download_url=file:///home/runner/work/xet-core/xet-core/xet_core_structures", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/huggingface/xet-core" + }, + { + "type": "vcs", + "url": "https://github.com/huggingface/xet-core" + } + ] + }, + { + "type": "library", + "bom-ref": "path+file:///home/runner/work/xet-core/xet-core/xet_data#xet-data@1.5.2", + "name": "xet-data", + "version": "1.5.2", + "description": "Data processing pipeline for chunking, deduplication, and file reconstruction; used in the Hugging Face Xet client tools. Intended to be used through the API in the hf-xet package.", + "scope": "required", + "licenses": [ + { + "expression": "Apache-2.0" + } + ], + "purl": "pkg:cargo/xet-data@1.5.2?download_url=file:///home/runner/work/xet-core/xet-core/xet_data", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/huggingface/xet-core" + }, + { + "type": "vcs", + "url": "https://github.com/huggingface/xet-core" + } + ] + }, + { + "type": "library", + "bom-ref": "path+file:///home/runner/work/xet-core/xet-core/xet_pkg#hf-xet@1.5.2", + "name": "hf-xet", + "version": "1.5.2", + "description": "Client library and tooling for the Hugging Face Xet data storage system.", + "scope": "required", + "licenses": [ + { + "expression": "Apache-2.0" + } + ], + "purl": "pkg:cargo/hf-xet@1.5.2?download_url=file:///home/runner/work/xet-core/xet-core/xet_pkg", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/huggingface/xet-core" + }, + { + "type": "vcs", + "url": "https://github.com/huggingface/xet-core" + } + ] + }, + { + "type": "library", + "bom-ref": "path+file:///home/runner/work/xet-core/xet-core/xet_runtime#xet-runtime@1.5.2", + "name": "xet-runtime", + "version": "1.5.2", + "description": "Async runtime, configuration, logging, and utility infrastructure for the Hugging Face Xet client tools.", + "scope": "required", + "licenses": [ + { + "expression": "Apache-2.0" + } + ], + "purl": "pkg:cargo/xet-runtime@1.5.2?download_url=file:///home/runner/work/xet-core/xet-core/xet_runtime", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/huggingface/xet-core" + }, + { + "type": "vcs", + "url": "https://github.com/huggingface/xet-core" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#aho-corasick@1.1.4", + "author": "Andrew Gallant ", + "name": "aho-corasick", + "version": "1.1.4", + "description": "Fast multiple substring searching.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" + } + ], + "licenses": [ + { + "expression": "Unlicense OR MIT" + } + ], + "purl": "pkg:cargo/aho-corasick@1.1.4", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/BurntSushi/aho-corasick" + }, + { + "type": "vcs", + "url": "https://github.com/BurntSushi/aho-corasick" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#anstream@1.0.0", + "name": "anstream", + "version": "1.0.0", + "description": "IO stream adapters for writing colored text that will gracefully degrade according to your terminal's capabilities.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/anstream@1.0.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/rust-cli/anstyle.git" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#anstyle-parse@1.0.0", + "name": "anstyle-parse", + "version": "1.0.0", + "description": "Parse ANSI Style Escapes", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/anstyle-parse@1.0.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/rust-cli/anstyle.git" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#anstyle-query@1.1.5", + "name": "anstyle-query", + "version": "1.1.5", + "description": "Look up colored console capabilities", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/anstyle-query@1.1.5", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/rust-cli/anstyle.git" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#anstyle@1.0.14", + "name": "anstyle", + "version": "1.0.14", + "description": "ANSI text styling", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/anstyle@1.0.14", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/rust-cli/anstyle.git" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#anyhow@1.0.102", + "author": "David Tolnay ", + "name": "anyhow", + "version": "1.0.102", + "description": "Flexible concrete Error type built on std::error::Error", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/anyhow@1.0.102", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/anyhow" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/anyhow" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#approx@0.5.1", + "author": "Brendan Zabarauskas ", + "name": "approx", + "version": "0.5.1", + "description": "Approximate floating point equality comparisons and assertions.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" + } + ], + "licenses": [ + { + "expression": "Apache-2.0" + } + ], + "purl": "pkg:cargo/approx@0.5.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/approx" + }, + { + "type": "website", + "url": "https://github.com/brendanzab/approx" + }, + { + "type": "vcs", + "url": "https://github.com/brendanzab/approx" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#arrayref@0.3.9", + "author": "David Roundy ", + "name": "arrayref", + "version": "0.3.9", + "description": "Macros to take array references of slices", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + } + ], + "licenses": [ + { + "expression": "BSD-2-Clause" + } + ], + "purl": "pkg:cargo/arrayref@0.3.9", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/arrayref" + }, + { + "type": "vcs", + "url": "https://github.com/droundy/arrayref" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#arrayvec@0.7.6", + "author": "bluss", + "name": "arrayvec", + "version": "0.7.6", + "description": "A vector with fixed capacity, backed by an array (it can be stored on the stack too). Implements fixed capacity ArrayVec and ArrayString.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/arrayvec@0.7.6", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/arrayvec/" + }, + { + "type": "vcs", + "url": "https://github.com/bluss/arrayvec" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#async-trait@0.1.89", + "author": "David Tolnay ", + "name": "async-trait", + "version": "0.1.89", + "description": "Type erasure for async trait methods", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/async-trait@0.1.89", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/async-trait" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/async-trait" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#atomic-waker@1.1.2", + "author": "Stjepan Glavina , Contributors to futures-rs", + "name": "atomic-waker", + "version": "1.1.2", + "description": "A synchronization primitive for task wakeup", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/atomic-waker@1.1.2", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/smol-rs/atomic-waker" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#autocfg@1.5.0", + "author": "Josh Stone ", + "name": "autocfg", + "version": "1.5.0", + "description": "Automatic cfg for Rust compiler features", + "scope": "excluded", + "hashes": [ + { + "alg": "SHA-256", + "content": "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/autocfg@1.5.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/autocfg/" + }, + { + "type": "vcs", + "url": "https://github.com/cuviper/autocfg" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#aws-lc-rs@1.16.2", + "author": "AWS-LibCrypto", + "name": "aws-lc-rs", + "version": "1.16.2", + "description": "aws-lc-rs is a cryptographic library using AWS-LC for its cryptographic operations. This library strives to be API-compatible with the popular Rust library named ring.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" + } + ], + "licenses": [ + { + "expression": "ISC AND (Apache-2.0 OR ISC)" + } + ], + "purl": "pkg:cargo/aws-lc-rs@1.16.2", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/crate/aws-lc-rs" + }, + { + "type": "website", + "url": "https://github.com/aws/aws-lc-rs" + }, + { + "type": "other", + "url": "aws_lc_rs_1_16_2_sys" + }, + { + "type": "vcs", + "url": "https://github.com/aws/aws-lc-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#aws-lc-sys@0.39.0", + "author": "AWS-LC", + "name": "aws-lc-sys", + "version": "0.39.0", + "description": "AWS-LC is a general-purpose cryptographic library maintained by the AWS Cryptography team for AWS and their customers. It іs based on code from the Google BoringSSL project and the OpenSSL project.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" + } + ], + "licenses": [ + { + "expression": "ISC AND (Apache-2.0 OR ISC) AND Apache-2.0 AND MIT AND BSD-3-Clause AND (Apache-2.0 OR ISC OR MIT) AND (Apache-2.0 OR ISC OR MIT-0)" + } + ], + "purl": "pkg:cargo/aws-lc-sys@0.39.0", + "externalReferences": [ + { + "type": "other", + "url": "aws_lc_0_39_0" + }, + { + "type": "vcs", + "url": "https://github.com/aws/aws-lc-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#base64@0.22.1", + "author": "Marshall Pierce ", + "name": "base64", + "version": "0.22.1", + "description": "encodes and decodes base64 as bytes or utf8", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/base64@0.22.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/base64" + }, + { + "type": "vcs", + "url": "https://github.com/marshallpierce/rust-base64" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#bitflags@2.11.0", + "author": "The Rust Project Developers", + "name": "bitflags", + "version": "2.11.0", + "description": "A macro to generate structures which behave like bitflags. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/bitflags@2.11.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/bitflags" + }, + { + "type": "website", + "url": "https://github.com/bitflags/bitflags" + }, + { + "type": "vcs", + "url": "https://github.com/bitflags/bitflags" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#blake3@1.8.3", + "author": "Jack O'Connor , Samuel Neves", + "name": "blake3", + "version": "1.8.3", + "description": "the BLAKE3 hash function", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" + } + ], + "licenses": [ + { + "expression": "CC0-1.0 OR Apache-2.0 OR Apache-2.0 WITH LLVM-exception" + } + ], + "purl": "pkg:cargo/blake3@1.8.3", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/blake3" + }, + { + "type": "vcs", + "url": "https://github.com/BLAKE3-team/BLAKE3" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#block-buffer@0.10.4", + "author": "RustCrypto Developers", + "name": "block-buffer", + "version": "0.10.4", + "description": "Buffer type for block processing of data", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/block-buffer@0.10.4", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/block-buffer" + }, + { + "type": "vcs", + "url": "https://github.com/RustCrypto/utils" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#bstr@1.12.1", + "author": "Andrew Gallant ", + "name": "bstr", + "version": "1.12.1", + "description": "A string type that is not required to be valid UTF-8.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/bstr@1.12.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/bstr" + }, + { + "type": "website", + "url": "https://github.com/BurntSushi/bstr" + }, + { + "type": "vcs", + "url": "https://github.com/BurntSushi/bstr" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#bytemuck@1.25.0", + "author": "Lokathor ", + "name": "bytemuck", + "version": "1.25.0", + "description": "A crate for mucking around with piles of bytes.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + } + ], + "licenses": [ + { + "expression": "Zlib OR Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/bytemuck@1.25.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/Lokathor/bytemuck" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "author": "Carl Lerche , Sean McArthur ", + "name": "bytes", + "version": "1.11.1", + "description": "Types and traits for working with bytes", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/bytes@1.11.1", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/tokio-rs/bytes" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#cc@1.2.57", + "author": "Alex Crichton ", + "name": "cc", + "version": "1.2.57", + "description": "A build-time dependency for Cargo build scripts to assist in invoking the native C compiler to compile native C code into a static archive to be linked into Rust code. ", + "scope": "excluded", + "hashes": [ + { + "alg": "SHA-256", + "content": "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/cc@1.2.57", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/cc" + }, + { + "type": "website", + "url": "https://github.com/rust-lang/cc-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/cc-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#cfg-if@0.1.10", + "author": "Alex Crichton ", + "name": "cfg-if", + "version": "0.1.10", + "description": "A macro to ergonomically define an item depending on a large number of #[cfg] parameters. Structured like an if-else chain, the first matching branch is the item that gets emitted. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/cfg-if@0.1.10", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/cfg-if" + }, + { + "type": "website", + "url": "https://github.com/alexcrichton/cfg-if" + }, + { + "type": "vcs", + "url": "https://github.com/alexcrichton/cfg-if" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.4", + "author": "Alex Crichton ", + "name": "cfg-if", + "version": "1.0.4", + "description": "A macro to ergonomically define an item depending on a large number of #[cfg] parameters. Structured like an if-else chain, the first matching branch is the item that gets emitted. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/cfg-if@1.0.4", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/rust-lang/cfg-if" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#cfg_aliases@0.2.1", + "author": "Zicklag ", + "name": "cfg_aliases", + "version": "0.2.1", + "description": "A tiny utility to help save you a lot of effort with long winded `#[cfg()]` checks.", + "scope": "excluded", + "hashes": [ + { + "alg": "SHA-256", + "content": "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/cfg_aliases@0.2.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/cfg_aliases" + }, + { + "type": "website", + "url": "https://github.com/katharostech/cfg_aliases" + }, + { + "type": "vcs", + "url": "https://github.com/katharostech/cfg_aliases" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#chacha20@0.10.0", + "author": "RustCrypto Developers", + "name": "chacha20", + "version": "0.10.0", + "description": "The ChaCha20 stream cipher (RFC 8439) implemented in pure Rust using traits from the RustCrypto `cipher` crate, with optional architecture-specific hardware acceleration (AVX2, SSE2). Additionally provides the ChaCha8, ChaCha12, XChaCha20, XChaCha12 and XChaCha8 stream ciphers, and also optional rand_core-compatible RNGs based on those ciphers. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/chacha20@0.10.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/chacha20" + }, + { + "type": "vcs", + "url": "https://github.com/RustCrypto/stream-ciphers" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#chrono@0.4.44", + "name": "chrono", + "version": "0.4.44", + "description": "Date and time library for Rust", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/chrono@0.4.44", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/chrono/" + }, + { + "type": "website", + "url": "https://github.com/chronotope/chrono" + }, + { + "type": "vcs", + "url": "https://github.com/chronotope/chrono" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#clap@4.6.0", + "name": "clap", + "version": "4.6.0", + "description": "A simple to use, efficient, and full-featured Command Line Argument Parser", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/clap@4.6.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/clap-rs/clap" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#clap_builder@4.6.0", + "name": "clap_builder", + "version": "4.6.0", + "description": "A simple to use, efficient, and full-featured Command Line Argument Parser", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/clap_builder@4.6.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/clap-rs/clap" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#clap_derive@4.6.0", + "name": "clap_derive", + "version": "4.6.0", + "description": "Parse command line argument by defining a struct, derive crate.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/clap_derive@4.6.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/clap-rs/clap" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#clap_lex@1.1.0", + "name": "clap_lex", + "version": "1.1.0", + "description": "Minimal, flexible command line parser", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/clap_lex@1.1.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/clap-rs/clap" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#cmake@0.1.57", + "author": "Alex Crichton ", + "name": "cmake", + "version": "0.1.57", + "description": "A build dependency for running `cmake` to build a native library ", + "scope": "excluded", + "hashes": [ + { + "alg": "SHA-256", + "content": "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/cmake@0.1.57", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/cmake" + }, + { + "type": "website", + "url": "https://github.com/rust-lang/cmake-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/cmake-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#colorchoice@1.0.5", + "name": "colorchoice", + "version": "1.0.5", + "description": "Global override of color control", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/colorchoice@1.0.5", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/rust-cli/anstyle.git" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#colored@3.1.1", + "author": "Thomas Wickham ", + "name": "colored", + "version": "3.1.1", + "description": "The most simple way to add colors in your terminal", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" + } + ], + "licenses": [ + { + "expression": "MPL-2.0" + } + ], + "purl": "pkg:cargo/colored@3.1.1", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/mackwic/colored" + }, + { + "type": "vcs", + "url": "https://github.com/mackwic/colored" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#const-str@1.1.0", + "author": "Nugine ", + "name": "const-str", + "version": "1.1.0", + "description": "compile-time string operations", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/const-str@1.1.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/Nugine/const-str" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#const_panic@0.2.15", + "author": "rodrimati1992 ", + "name": "const_panic", + "version": "0.2.15", + "description": "const panic with formatting", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" + } + ], + "licenses": [ + { + "expression": "Zlib" + } + ], + "purl": "pkg:cargo/const_panic@0.2.15", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/rodrimati1992/const_panic/" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#constant_time_eq@0.4.2", + "author": "Cesar Eduardo Barros ", + "name": "constant_time_eq", + "version": "0.4.2", + "description": "Compares two equal-sized byte strings in constant time.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + } + ], + "licenses": [ + { + "expression": "CC0-1.0 OR MIT-0 OR Apache-2.0" + } + ], + "purl": "pkg:cargo/constant_time_eq@0.4.2", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/constant_time_eq" + }, + { + "type": "vcs", + "url": "https://github.com/cesarb/constant_time_eq" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#countio@0.3.0", + "author": "Oleh Martsokha ", + "name": "countio", + "version": "0.3.0", + "description": "Byte counting for std::io::{Read, Write, Seek} and its async variants from futures and tokio. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b9702aee5d1d744c01d82f6915644f950f898e014903385464c773b96fefdecb" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/countio@0.3.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/countio" + }, + { + "type": "website", + "url": "https://github.com/spire-rs/countio" + }, + { + "type": "vcs", + "url": "https://github.com/spire-rs/countio" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#cpufeatures@0.2.17", + "author": "RustCrypto Developers", + "name": "cpufeatures", + "version": "0.2.17", + "description": "Lightweight runtime CPU feature detection for aarch64, loongarch64, and x86/x86_64 targets, with no_std support and support for mobile targets including Android and iOS ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/cpufeatures@0.2.17", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/cpufeatures" + }, + { + "type": "vcs", + "url": "https://github.com/RustCrypto/utils" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#cpufeatures@0.3.0", + "author": "RustCrypto Developers", + "name": "cpufeatures", + "version": "0.3.0", + "description": "Lightweight runtime CPU feature detection for aarch64, loongarch64, and x86/x86_64 targets, with no_std support and support for mobile targets including Android and iOS ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/cpufeatures@0.3.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/cpufeatures" + }, + { + "type": "vcs", + "url": "https://github.com/RustCrypto/utils" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#crc32fast@1.5.0", + "author": "Sam Rijs , Alex Crichton ", + "name": "crc32fast", + "version": "1.5.0", + "description": "Fast, SIMD-accelerated CRC32 (IEEE) checksum computation", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/crc32fast@1.5.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/srijs/rust-crc32fast" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#crossbeam-channel@0.5.15", + "name": "crossbeam-channel", + "version": "0.5.15", + "description": "Multi-producer multi-consumer channels for message passing", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/crossbeam-channel@0.5.15", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-channel" + }, + { + "type": "vcs", + "url": "https://github.com/crossbeam-rs/crossbeam" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#crossbeam-utils@0.8.21", + "name": "crossbeam-utils", + "version": "0.8.21", + "description": "Utilities for concurrent programming", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/crossbeam-utils@0.8.21", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/crossbeam-rs/crossbeam/tree/master/crossbeam-utils" + }, + { + "type": "vcs", + "url": "https://github.com/crossbeam-rs/crossbeam" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#crypto-common@0.1.7", + "author": "RustCrypto Developers", + "name": "crypto-common", + "version": "0.1.7", + "description": "Common cryptographic traits", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/crypto-common@0.1.7", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/crypto-common" + }, + { + "type": "vcs", + "url": "https://github.com/RustCrypto/traits" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#csv-core@0.1.13", + "author": "Andrew Gallant ", + "name": "csv-core", + "version": "0.1.13", + "description": "Bare bones CSV parsing with no_std support.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" + } + ], + "licenses": [ + { + "expression": "Unlicense OR MIT" + } + ], + "purl": "pkg:cargo/csv-core@0.1.13", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/csv-core" + }, + { + "type": "website", + "url": "https://github.com/BurntSushi/rust-csv" + }, + { + "type": "vcs", + "url": "https://github.com/BurntSushi/rust-csv" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#csv@1.4.0", + "author": "Andrew Gallant ", + "name": "csv", + "version": "1.4.0", + "description": "Fast CSV parsing with support for serde.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" + } + ], + "licenses": [ + { + "expression": "Unlicense OR MIT" + } + ], + "purl": "pkg:cargo/csv@1.4.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/csv" + }, + { + "type": "website", + "url": "https://github.com/BurntSushi/rust-csv" + }, + { + "type": "vcs", + "url": "https://github.com/BurntSushi/rust-csv" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#ctor-proc-macro@0.0.7", + "author": "Matt Mastracci ", + "name": "ctor-proc-macro", + "version": "0.0.7", + "description": "proc-macro support for the ctor crate", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/ctor-proc-macro@0.0.7", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/mmastrac/rust-ctor" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#ctor@0.6.3", + "author": "Matt Mastracci ", + "name": "ctor", + "version": "0.6.3", + "description": "__attribute__((constructor)) for Rust", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/ctor@0.6.3", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/mmastrac/rust-ctor" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#deranged@0.5.8", + "author": "Jacob Pratt ", + "name": "deranged", + "version": "0.5.8", + "description": "Ranged integers", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/deranged@0.5.8", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/jhpratt/deranged" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#digest@0.10.7", + "author": "RustCrypto Developers", + "name": "digest", + "version": "0.10.7", + "description": "Traits for cryptographic hash functions and message authentication codes", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/digest@0.10.7", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/digest" + }, + { + "type": "vcs", + "url": "https://github.com/RustCrypto/traits" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#dirs-sys@0.5.0", + "author": "Simon Ochsenreither ", + "name": "dirs-sys", + "version": "0.5.0", + "description": "System-level helper functions for the dirs and directories crates.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/dirs-sys@0.5.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/dirs-dev/dirs-sys-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#dirs@6.0.0", + "author": "Simon Ochsenreither ", + "name": "dirs", + "version": "6.0.0", + "description": "A tiny low-level library that provides platform-specific standard locations of directories for config, cache and other data on Linux, Windows, macOS and Redox by leveraging the mechanisms defined by the XDG base/user directory specifications on Linux, the Known Folder API on Windows, and the Standard Directory guidelines on macOS.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/dirs@6.0.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/soc/dirs-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", + "author": "Jane Lusby ", + "name": "displaydoc", + "version": "0.2.5", + "description": "A derive macro for implementing the display Trait via a doc comment and string interpolation ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/displaydoc@0.2.5", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/displaydoc" + }, + { + "type": "website", + "url": "https://github.com/yaahc/displaydoc" + }, + { + "type": "vcs", + "url": "https://github.com/yaahc/displaydoc" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#dtor-proc-macro@0.0.6", + "author": "Matt Mastracci ", + "name": "dtor-proc-macro", + "version": "0.0.6", + "description": "proc-macro support for the dtor crate", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/dtor-proc-macro@0.0.6", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/mmastrac/rust-ctor" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#dtor@0.1.1", + "author": "Matt Mastracci ", + "name": "dtor", + "version": "0.1.1", + "description": "__attribute__((destructor)) for Rust", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/dtor@0.1.1", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/mmastrac/rust-ctor" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#dunce@1.0.5", + "author": "Kornel ", + "name": "dunce", + "version": "1.0.5", + "description": "Normalize Windows paths to the most compatible format, avoiding UNC where possible", + "scope": "excluded", + "hashes": [ + { + "alg": "SHA-256", + "content": "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + } + ], + "licenses": [ + { + "expression": "CC0-1.0 OR MIT-0 OR Apache-2.0" + } + ], + "purl": "pkg:cargo/dunce@1.0.5", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/dunce" + }, + { + "type": "website", + "url": "https://lib.rs/crates/dunce" + }, + { + "type": "vcs", + "url": "https://gitlab.com/kornelski/dunce" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#either@1.15.0", + "author": "bluss", + "name": "either", + "version": "1.15.0", + "description": "The enum `Either` with variants `Left` and `Right` is a general purpose sum type with two cases. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/either@1.15.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/either/1/" + }, + { + "type": "vcs", + "url": "https://github.com/rayon-rs/either" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#errno@0.3.14", + "author": "Chris Wong , Dan Gohman ", + "name": "errno", + "version": "0.3.14", + "description": "Cross-platform interface to the `errno` variable.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/errno@0.3.14", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/errno" + }, + { + "type": "vcs", + "url": "https://github.com/lambda-fairy/rust-errno" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#fastrand@2.3.0", + "author": "Stjepan Glavina ", + "name": "fastrand", + "version": "2.3.0", + "description": "A simple and fast random number generator", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/fastrand@2.3.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/smol-rs/fastrand" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#find-msvc-tools@0.1.9", + "name": "find-msvc-tools", + "version": "0.1.9", + "description": "Find windows-specific tools, read MSVC versions from the registry and from COM interfaces", + "scope": "excluded", + "hashes": [ + { + "alg": "SHA-256", + "content": "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/find-msvc-tools@0.1.9", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/find-msvc-tools" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/cc-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#form_urlencoded@1.2.2", + "author": "The rust-url developers", + "name": "form_urlencoded", + "version": "1.2.2", + "description": "Parser and serializer for the application/x-www-form-urlencoded syntax, as used by HTML forms.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/form_urlencoded@1.2.2", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/servo/rust-url" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#fs_extra@1.3.0", + "author": "Denis Kurilenko ", + "name": "fs_extra", + "version": "1.3.0", + "description": "Expanding std::fs and std::io. Recursively copy folders with information about process and much more.", + "scope": "excluded", + "hashes": [ + { + "alg": "SHA-256", + "content": "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/fs_extra@1.3.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/fs_extra" + }, + { + "type": "website", + "url": "https://github.com/webdesus/fs_extra" + }, + { + "type": "vcs", + "url": "https://github.com/webdesus/fs_extra" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#futures-channel@0.3.32", + "name": "futures-channel", + "version": "0.3.32", + "description": "Channels for asynchronous communication using futures-rs. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/futures-channel@0.3.32", + "externalReferences": [ + { + "type": "website", + "url": "https://rust-lang.github.io/futures-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/futures-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32", + "name": "futures-core", + "version": "0.3.32", + "description": "The core traits and types in for the `futures` library. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/futures-core@0.3.32", + "externalReferences": [ + { + "type": "website", + "url": "https://rust-lang.github.io/futures-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/futures-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#futures-executor@0.3.32", + "name": "futures-executor", + "version": "0.3.32", + "description": "Executors for asynchronous tasks based on the futures-rs library. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/futures-executor@0.3.32", + "externalReferences": [ + { + "type": "website", + "url": "https://rust-lang.github.io/futures-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/futures-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#futures-io@0.3.32", + "name": "futures-io", + "version": "0.3.32", + "description": "The `AsyncRead`, `AsyncWrite`, `AsyncSeek`, and `AsyncBufRead` traits for the futures-rs library. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/futures-io@0.3.32", + "externalReferences": [ + { + "type": "website", + "url": "https://rust-lang.github.io/futures-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/futures-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#futures-macro@0.3.32", + "name": "futures-macro", + "version": "0.3.32", + "description": "The futures-rs procedural macro implementations. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/futures-macro@0.3.32", + "externalReferences": [ + { + "type": "website", + "url": "https://rust-lang.github.io/futures-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/futures-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#futures-sink@0.3.32", + "name": "futures-sink", + "version": "0.3.32", + "description": "The asynchronous `Sink` trait for the futures-rs library. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/futures-sink@0.3.32", + "externalReferences": [ + { + "type": "website", + "url": "https://rust-lang.github.io/futures-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/futures-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#futures-task@0.3.32", + "name": "futures-task", + "version": "0.3.32", + "description": "Tools for working with tasks. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/futures-task@0.3.32", + "externalReferences": [ + { + "type": "website", + "url": "https://rust-lang.github.io/futures-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/futures-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#futures-util@0.3.32", + "name": "futures-util", + "version": "0.3.32", + "description": "Common utilities and extension traits for the futures-rs library. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/futures-util@0.3.32", + "externalReferences": [ + { + "type": "website", + "url": "https://rust-lang.github.io/futures-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/futures-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#futures@0.3.32", + "name": "futures", + "version": "0.3.32", + "description": "An implementation of futures and streams featuring zero allocations, composability, and iterator-like interfaces. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/futures@0.3.32", + "externalReferences": [ + { + "type": "website", + "url": "https://rust-lang.github.io/futures-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/futures-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#gearhash@0.1.3", + "author": "Sam Rijs ", + "name": "gearhash", + "version": "0.1.3", + "description": "Fast, SIMD-accelerated hash function for content-defined chunking", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "c8cf82cf76cd16485e56295a1377c775ce708c9f1a0be6b029076d60a245d213" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/gearhash@0.1.3", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/srijs/rust-gearhash" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#generic-array@0.14.7", + "author": "Bartłomiej Kamiński , Aaron Trent ", + "name": "generic-array", + "version": "0.14.7", + "description": "Generic types implementing functionality of arrays", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/generic-array@0.14.7", + "externalReferences": [ + { + "type": "documentation", + "url": "http://fizyk20.github.io/generic-array/generic_array/" + }, + { + "type": "vcs", + "url": "https://github.com/fizyk20/generic-array.git" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.2.17", + "author": "The Rand Project Developers", + "name": "getrandom", + "version": "0.2.17", + "description": "A small cross-platform library for retrieving random data from system source", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/getrandom@0.2.17", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/getrandom" + }, + { + "type": "vcs", + "url": "https://github.com/rust-random/getrandom" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.3.4", + "author": "The Rand Project Developers", + "name": "getrandom", + "version": "0.3.4", + "description": "A small cross-platform library for retrieving random data from system source", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/getrandom@0.3.4", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/getrandom" + }, + { + "type": "vcs", + "url": "https://github.com/rust-random/getrandom" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.4.2", + "author": "The Rand Project Developers", + "name": "getrandom", + "version": "0.4.2", + "description": "A small cross-platform library for retrieving random data from system source", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/getrandom@0.4.2", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/getrandom" + }, + { + "type": "vcs", + "url": "https://github.com/rust-random/getrandom" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#git-version-macro@0.3.9", + "author": "David Roundy , Maarten de Vries , Mara Bos ", + "name": "git-version-macro", + "version": "0.3.9", + "description": "Internal macro crate for git-version.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" + } + ], + "licenses": [ + { + "expression": "BSD-2-Clause" + } + ], + "purl": "pkg:cargo/git-version-macro@0.3.9", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/fusion-engineering/rust-git-version" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#git-version@0.3.9", + "author": "Mara Bos , Maarten de Vries , David Roundy ", + "name": "git-version", + "version": "0.3.9", + "description": "Compile the git version (tag name, or hash otherwise) and dirty state into your program.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" + } + ], + "licenses": [ + { + "expression": "BSD-2-Clause" + } + ], + "purl": "pkg:cargo/git-version@0.3.9", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/git-version/" + }, + { + "type": "vcs", + "url": "https://github.com/fusion-engineering/rust-git-version" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#heapify@0.2.0", + "name": "heapify", + "version": "0.2.0", + "description": "Convenience functions to turn slices into max-heaps.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/heapify@0.2.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/ethereal-sheep/heapify" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#heck@0.5.0", + "name": "heck", + "version": "0.5.0", + "description": "heck is a case conversion library.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/heck@0.5.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/withoutboats/heck" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#http-body-util@0.1.3", + "author": "Carl Lerche , Lucio Franco , Sean McArthur ", + "name": "http-body-util", + "version": "0.1.3", + "description": "Combinators and adapters for HTTP request or response bodies. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/http-body-util@0.1.3", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/http-body-util" + }, + { + "type": "vcs", + "url": "https://github.com/hyperium/http-body" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#http-body@1.0.1", + "author": "Carl Lerche , Lucio Franco , Sean McArthur ", + "name": "http-body", + "version": "1.0.1", + "description": "Trait representing an asynchronous, streaming, HTTP request or response body. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/http-body@1.0.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/http-body" + }, + { + "type": "vcs", + "url": "https://github.com/hyperium/http-body" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "author": "Alex Crichton , Carl Lerche , Sean McArthur ", + "name": "http", + "version": "1.4.0", + "description": "A set of types for representing HTTP requests and responses. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/http@1.4.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/http" + }, + { + "type": "vcs", + "url": "https://github.com/hyperium/http" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#httparse@1.10.1", + "author": "Sean McArthur ", + "name": "httparse", + "version": "1.10.1", + "description": "A tiny, safe, speedy, zero-copy HTTP/1.x parser.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/httparse@1.10.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/httparse" + }, + { + "type": "vcs", + "url": "https://github.com/seanmonstar/httparse" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#humantime@2.3.0", + "name": "humantime", + "version": "2.3.0", + "description": "A parser and formatter for std::time::{Duration, SystemTime}", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/humantime@2.3.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/humantime" + }, + { + "type": "website", + "url": "https://github.com/chronotope/humantime" + }, + { + "type": "vcs", + "url": "https://github.com/chronotope/humantime" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#hyper-rustls@0.27.7", + "name": "hyper-rustls", + "version": "0.27.7", + "description": "Rustls+hyper integration for pure rust HTTPS", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR ISC OR MIT" + } + ], + "purl": "pkg:cargo/hyper-rustls@0.27.7", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/hyper-rustls/" + }, + { + "type": "website", + "url": "https://github.com/rustls/hyper-rustls" + }, + { + "type": "vcs", + "url": "https://github.com/rustls/hyper-rustls" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#hyper-util@0.1.20", + "author": "Sean McArthur ", + "name": "hyper-util", + "version": "0.1.20", + "description": "hyper utilities", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/hyper-util@0.1.20", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/hyper-util" + }, + { + "type": "website", + "url": "https://hyper.rs" + }, + { + "type": "vcs", + "url": "https://github.com/hyperium/hyper-util" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#hyper@1.8.1", + "author": "Sean McArthur ", + "name": "hyper", + "version": "1.8.1", + "description": "A protective and efficient HTTP library for all.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/hyper@1.8.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/hyper" + }, + { + "type": "website", + "url": "https://hyper.rs" + }, + { + "type": "vcs", + "url": "https://github.com/hyperium/hyper" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#iana-time-zone@0.1.65", + "author": "Andrew Straw , René Kijewski , Ryan Lopopolo ", + "name": "iana-time-zone", + "version": "0.1.65", + "description": "get the IANA time zone for the current system", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/iana-time-zone@0.1.65", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/strawlab/iana-time-zone" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_collections@2.1.1", + "author": "The ICU4X Project Developers", + "name": "icu_collections", + "version": "2.1.1", + "description": "Collection of API for use in ICU libraries.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/icu_collections@2.1.1", + "externalReferences": [ + { + "type": "website", + "url": "https://icu4x.unicode.org" + }, + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_locale_core@2.1.1", + "author": "The ICU4X Project Developers", + "name": "icu_locale_core", + "version": "2.1.1", + "description": "API for managing Unicode Language and Locale Identifiers", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/icu_locale_core@2.1.1", + "externalReferences": [ + { + "type": "website", + "url": "https://icu4x.unicode.org" + }, + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer@2.1.1", + "author": "The ICU4X Project Developers", + "name": "icu_normalizer", + "version": "2.1.1", + "description": "API for normalizing text into Unicode Normalization Forms", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/icu_normalizer@2.1.1", + "externalReferences": [ + { + "type": "website", + "url": "https://icu4x.unicode.org" + }, + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer_data@2.1.1", + "author": "The ICU4X Project Developers", + "name": "icu_normalizer_data", + "version": "2.1.1", + "description": "Data for the icu_normalizer crate", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/icu_normalizer_data@2.1.1", + "externalReferences": [ + { + "type": "website", + "url": "https://icu4x.unicode.org" + }, + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_properties@2.1.2", + "author": "The ICU4X Project Developers", + "name": "icu_properties", + "version": "2.1.2", + "description": "Definitions for Unicode properties", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/icu_properties@2.1.2", + "externalReferences": [ + { + "type": "website", + "url": "https://icu4x.unicode.org" + }, + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_properties_data@2.1.2", + "author": "The ICU4X Project Developers", + "name": "icu_properties_data", + "version": "2.1.2", + "description": "Data for the icu_properties crate", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/icu_properties_data@2.1.2", + "externalReferences": [ + { + "type": "website", + "url": "https://icu4x.unicode.org" + }, + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_provider@2.1.1", + "author": "The ICU4X Project Developers", + "name": "icu_provider", + "version": "2.1.1", + "description": "Trait and struct definitions for the ICU data provider", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/icu_provider@2.1.1", + "externalReferences": [ + { + "type": "website", + "url": "https://icu4x.unicode.org" + }, + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#idna@1.1.0", + "author": "The rust-url developers", + "name": "idna", + "version": "1.1.0", + "description": "IDNA (Internationalizing Domain Names in Applications) and Punycode.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/idna@1.1.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/servo/rust-url/" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#idna_adapter@1.2.1", + "author": "The rust-url developers", + "name": "idna_adapter", + "version": "1.2.1", + "description": "Back end adapter for idna", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/idna_adapter@1.2.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/idna_adapter/latest/idna_adapter/" + }, + { + "type": "website", + "url": "https://docs.rs/crate/idna_adapter/latest" + }, + { + "type": "vcs", + "url": "https://github.com/hsivonen/idna_adapter" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#indoc@2.0.7", + "author": "David Tolnay ", + "name": "indoc", + "version": "2.0.7", + "description": "Indented document literals", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/indoc@2.0.7", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/indoc" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/indoc" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#inventory@0.3.22", + "author": "David Tolnay ", + "name": "inventory", + "version": "0.3.22", + "description": "Typed distributed plugin registration", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "009ae045c87e7082cb72dab0ccd01ae075dd00141ddc108f43a0ea150a9e7227" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/inventory@0.3.22", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/inventory" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/inventory" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#ipnet@2.12.0", + "author": "Kris Price ", + "name": "ipnet", + "version": "2.12.0", + "description": "Provides types and useful methods for working with IPv4 and IPv6 network addresses, commonly called IP prefixes. The new `IpNet`, `Ipv4Net`, and `Ipv6Net` types build on the existing `IpAddr`, `Ipv4Addr`, and `Ipv6Addr` types already provided in Rust's standard library and align to their design to stay consistent. The module also provides useful traits that extend `Ipv4Addr` and `Ipv6Addr` with methods for `Add`, `Sub`, `BitAnd`, and `BitOr` operations. The module only uses stable feature so it is guaranteed to compile using the stable toolchain.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/ipnet@2.12.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/ipnet" + }, + { + "type": "vcs", + "url": "https://github.com/krisprice/ipnet" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#iri-string@0.7.11", + "author": "YOSHIOKA Takuma ", + "name": "iri-string", + "version": "0.7.11", + "description": "IRI as string types", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/iri-string@0.7.11", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/lo48576/iri-string" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#is_terminal_polyfill@1.70.2", + "name": "is_terminal_polyfill", + "version": "1.70.2", + "description": "Polyfill for `is_terminal` stdlib feature for use with older MSRVs", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/is_terminal_polyfill@1.70.2", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/polyfill-rs/is_terminal_polyfill" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#itertools@0.14.0", + "author": "bluss", + "name": "itertools", + "version": "0.14.0", + "description": "Extra iterator adaptors, iterator methods, free functions, and macros.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/itertools@0.14.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/itertools/" + }, + { + "type": "vcs", + "url": "https://github.com/rust-itertools/itertools" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#itoa@1.0.18", + "author": "David Tolnay ", + "name": "itoa", + "version": "1.0.18", + "description": "Fast integer primitive to string conversion", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/itoa@1.0.18", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/itoa" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/itoa" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#jobserver@0.1.34", + "author": "Alex Crichton ", + "name": "jobserver", + "version": "0.1.34", + "description": "An implementation of the GNU Make jobserver for Rust. ", + "scope": "excluded", + "hashes": [ + { + "alg": "SHA-256", + "content": "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/jobserver@0.1.34", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/jobserver" + }, + { + "type": "website", + "url": "https://github.com/rust-lang/jobserver-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/jobserver-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#konst@0.4.3", + "author": "rodrimati1992 ", + "name": "konst", + "version": "0.4.3", + "description": "Const equivalents of std features: comparison, destructuring, iteration, and parsing", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "f660d5f887e3562f9ab6f4a14988795b694099d66b4f5dedc02d197ba9becb1d" + } + ], + "licenses": [ + { + "expression": "Zlib" + } + ], + "purl": "pkg:cargo/konst@0.4.3", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/konst/" + }, + { + "type": "vcs", + "url": "https://github.com/rodrimati1992/konst/" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#konst_proc_macros@0.4.1", + "author": "rodrimati1992 ", + "name": "konst_proc_macros", + "version": "0.4.1", + "description": "Implementation detail of the `konst` crate", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" + } + ], + "licenses": [ + { + "expression": "Zlib" + } + ], + "purl": "pkg:cargo/konst_proc_macros@0.4.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/konst/" + }, + { + "type": "vcs", + "url": "https://github.com/rodrimati1992/konst/" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#lazy_static@1.5.0", + "author": "Marvin Löbel ", + "name": "lazy_static", + "version": "1.5.0", + "description": "A macro for declaring lazily evaluated statics in Rust.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/lazy_static@1.5.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/lazy_static" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang-nursery/lazy-static.rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "author": "The Rust Project Developers", + "name": "libc", + "version": "0.2.183", + "description": "Raw FFI bindings to platform libraries like libc.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/libc@0.2.183", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/rust-lang/libc" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#linux-raw-sys@0.12.1", + "author": "Dan Gohman ", + "name": "linux-raw-sys", + "version": "0.12.1", + "description": "Generated bindings for Linux's userspace API", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/linux-raw-sys@0.12.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/linux-raw-sys" + }, + { + "type": "vcs", + "url": "https://github.com/sunfishcode/linux-raw-sys" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#litemap@0.8.1", + "author": "The ICU4X Project Developers", + "name": "litemap", + "version": "0.8.1", + "description": "A key-value Map implementation based on a flat, sorted Vec.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/litemap@0.8.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/litemap" + }, + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#log@0.4.29", + "author": "The Rust Project Developers", + "name": "log", + "version": "0.4.29", + "description": "A lightweight logging facade for Rust ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/log@0.4.29", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/log" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/log" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#lru-slab@0.1.2", + "author": "Benjamin Saunders ", + "name": "lru-slab", + "version": "0.1.2", + "description": "Pre-allocated storage with constant-time LRU tracking", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0 OR Zlib" + } + ], + "purl": "pkg:cargo/lru-slab@0.1.2", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/Ralith/lru-slab" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#lz4_flex@0.13.0", + "author": "Pascal Seitz , Arthur Silva , ticki ", + "name": "lz4_flex", + "version": "0.13.0", + "description": "Fastest LZ4 implementation in Rust, no unsafe by default.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/lz4_flex@0.13.0", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/pseitz/lz4_flex" + }, + { + "type": "vcs", + "url": "https://github.com/pseitz/lz4_flex" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#matchers@0.2.0", + "author": "Eliza Weisman ", + "name": "matchers", + "version": "0.2.0", + "description": "Regex matching on character and byte streams. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/matchers@0.2.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/matchers/" + }, + { + "type": "website", + "url": "https://github.com/hawkw/matchers" + }, + { + "type": "vcs", + "url": "https://github.com/hawkw/matchers" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0", + "author": "Andrew Gallant , bluss", + "name": "memchr", + "version": "2.8.0", + "description": "Provides extremely fast (uses SIMD on x86_64, aarch64 and wasm32) routines for 1, 2 or 3 byte search and single substring search. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + } + ], + "licenses": [ + { + "expression": "Unlicense OR MIT" + } + ], + "purl": "pkg:cargo/memchr@2.8.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/memchr/" + }, + { + "type": "website", + "url": "https://github.com/BurntSushi/memchr" + }, + { + "type": "vcs", + "url": "https://github.com/BurntSushi/memchr" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#memoffset@0.9.1", + "author": "Gilad Naaman ", + "name": "memoffset", + "version": "0.9.1", + "description": "offset_of functionality for Rust structs.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/memoffset@0.9.1", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/Gilnaa/memoffset" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#mio@1.1.1", + "author": "Carl Lerche , Thomas de Zeeuw , Tokio Contributors ", + "name": "mio", + "version": "1.1.1", + "description": "Lightweight non-blocking I/O.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/mio@1.1.1", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/tokio-rs/mio" + }, + { + "type": "vcs", + "url": "https://github.com/tokio-rs/mio" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#more-asserts@0.3.1", + "author": "Thom Chiovoloni ", + "name": "more-asserts", + "version": "0.3.1", + "description": "Small library providing additional assert_* and debug_assert_* macros.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" + } + ], + "licenses": [ + { + "expression": "Unlicense OR MIT OR Apache-2.0 OR CC0-1.0" + } + ], + "purl": "pkg:cargo/more-asserts@0.3.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/more-asserts" + }, + { + "type": "website", + "url": "https://github.com/thomcc/rust-more-asserts" + }, + { + "type": "vcs", + "url": "https://github.com/thomcc/rust-more-asserts" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#nu-ansi-term@0.50.3", + "author": "ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett , The Nushell Project Developers", + "name": "nu-ansi-term", + "version": "0.50.3", + "description": "Library for ANSI terminal colors and styles (bold, underline)", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/nu-ansi-term@0.50.3", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/nushell/nu-ansi-term" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#num-conv@0.2.0", + "author": "Jacob Pratt ", + "name": "num-conv", + "version": "0.2.0", + "description": "`num_conv` is a crate to convert between integer types without using `as` casts. This provides better certainty when refactoring, makes the exact behavior of code more explicit, and allows using turbofish syntax. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/num-conv@0.2.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/jhpratt/num-conv" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19", + "author": "The Rust Project Developers", + "name": "num-traits", + "version": "0.2.19", + "description": "Numeric traits for generic mathematics", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/num-traits@0.2.19", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/num-traits" + }, + { + "type": "website", + "url": "https://github.com/rust-num/num-traits" + }, + { + "type": "vcs", + "url": "https://github.com/rust-num/num-traits" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.4", + "author": "Aleksey Kladov ", + "name": "once_cell", + "version": "1.21.4", + "description": "Single assignment cells and lazy values.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/once_cell@1.21.4", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/once_cell" + }, + { + "type": "vcs", + "url": "https://github.com/matklad/once_cell" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#oneshot@0.1.13", + "author": "Linus Färnstrand ", + "name": "oneshot", + "version": "0.1.13", + "description": "Oneshot spsc channel with (potentially) lock-free non-blocking send, and a receiver supporting both thread blocking receive operations as well as Future based async polling. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/oneshot@0.1.13", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/faern/oneshot" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#openssl-probe@0.2.1", + "author": "Alex Crichton ", + "name": "openssl-probe", + "version": "0.2.1", + "description": "A library for helping to find system-wide trust anchor (\"root\") certificate locations based on paths typically used by `openssl`. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/openssl-probe@0.2.1", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/rustls/openssl-probe" + }, + { + "type": "vcs", + "url": "https://github.com/rustls/openssl-probe" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#option-ext@0.2.0", + "author": "Simon Ochsenreither ", + "name": "option-ext", + "version": "0.2.0", + "description": "Extends `Option` with additional operations", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + } + ], + "licenses": [ + { + "expression": "MPL-2.0" + } + ], + "purl": "pkg:cargo/option-ext@0.2.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/option-ext/" + }, + { + "type": "website", + "url": "https://github.com/soc/option-ext" + }, + { + "type": "vcs", + "url": "https://github.com/soc/option-ext.git" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#os_str_bytes@6.6.1", + "author": "dylni", + "name": "os_str_bytes", + "version": "6.6.1", + "description": "Convert between byte sequences and platform-native strings ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/os_str_bytes@6.6.1", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/dylni/os_str_bytes" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#percent-encoding@2.3.2", + "author": "The rust-url developers", + "name": "percent-encoding", + "version": "2.3.2", + "description": "Percent encoding and decoding", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/percent-encoding@2.3.2", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/servo/rust-url/" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pin-project-internal@1.1.11", + "name": "pin-project-internal", + "version": "1.1.11", + "description": "Implementation detail of the `pin-project` crate. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/pin-project-internal@1.1.11", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/taiki-e/pin-project" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "name": "pin-project-lite", + "version": "0.2.17", + "description": "A lightweight version of pin-project written with declarative macros. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/pin-project-lite@0.2.17", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/taiki-e/pin-project-lite" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pin-project@1.1.11", + "name": "pin-project", + "version": "1.1.11", + "description": "A crate for safe and ergonomic pin-projection. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/pin-project@1.1.11", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/taiki-e/pin-project" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pin-utils@0.1.0", + "author": "Josef Brandl ", + "name": "pin-utils", + "version": "0.1.0", + "description": "Utilities for pinning ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/pin-utils@0.1.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/pin-utils" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang-nursery/pin-utils" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#potential_utf@0.1.4", + "author": "The ICU4X Project Developers", + "name": "potential_utf", + "version": "0.1.4", + "description": "Unvalidated string and character types", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/potential_utf@0.1.4", + "externalReferences": [ + { + "type": "website", + "url": "https://icu4x.unicode.org" + }, + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#powerfmt@0.2.0", + "author": "Jacob Pratt ", + "name": "powerfmt", + "version": "0.2.0", + "description": " `powerfmt` is a library that provides utilities for formatting values. This crate makes it significantly easier to support filling to a minimum width with alignment, avoid heap allocation, and avoid repetitive calculations. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/powerfmt@0.2.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/jhpratt/powerfmt" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#ppv-lite86@0.2.21", + "author": "The CryptoCorrosion Contributors", + "name": "ppv-lite86", + "version": "0.2.21", + "description": "Cross-platform cryptography-oriented low-level SIMD library.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/ppv-lite86@0.2.21", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/cryptocorrosion/cryptocorrosion" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "author": "David Tolnay , Alex Crichton ", + "name": "proc-macro2", + "version": "1.0.106", + "description": "A substitute implementation of the compiler's `proc_macro` API to decouple token-based libraries from the procedural macro use case.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/proc-macro2@1.0.106", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/proc-macro2" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/proc-macro2" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.26.0", + "author": "PyO3 Project and Contributors ", + "name": "pyo3-build-config", + "version": "0.26.0", + "description": "Build configuration for the PyO3 ecosystem", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/pyo3-build-config@0.26.0", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/pyo3/pyo3" + }, + { + "type": "vcs", + "url": "https://github.com/pyo3/pyo3" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-ffi@0.26.0", + "author": "PyO3 Project and Contributors ", + "name": "pyo3-ffi", + "version": "0.26.0", + "description": "Python-API bindings for the PyO3 ecosystem", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/pyo3-ffi@0.26.0", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/pyo3/pyo3" + }, + { + "type": "other", + "url": "python" + }, + { + "type": "vcs", + "url": "https://github.com/pyo3/pyo3" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros-backend@0.26.0", + "author": "PyO3 Project and Contributors ", + "name": "pyo3-macros-backend", + "version": "0.26.0", + "description": "Code generation for PyO3 package", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/pyo3-macros-backend@0.26.0", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/pyo3/pyo3" + }, + { + "type": "vcs", + "url": "https://github.com/pyo3/pyo3" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros@0.26.0", + "author": "PyO3 Project and Contributors ", + "name": "pyo3-macros", + "version": "0.26.0", + "description": "Proc macros for PyO3 package", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/pyo3-macros@0.26.0", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/pyo3/pyo3" + }, + { + "type": "vcs", + "url": "https://github.com/pyo3/pyo3" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3@0.26.0", + "author": "PyO3 Project and Contributors ", + "name": "pyo3", + "version": "0.26.0", + "description": "Bindings to Python interpreter", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/pyo3@0.26.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/crate/pyo3/" + }, + { + "type": "website", + "url": "https://github.com/pyo3/pyo3" + }, + { + "type": "vcs", + "url": "https://github.com/pyo3/pyo3" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#quinn-proto@0.11.14", + "name": "quinn-proto", + "version": "0.11.14", + "description": "State machine for the QUIC transport protocol", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/quinn-proto@0.11.14", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/quinn-rs/quinn" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#quinn-udp@0.5.14", + "name": "quinn-udp", + "version": "0.5.14", + "description": "UDP sockets with ECN information for the QUIC transport protocol", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/quinn-udp@0.5.14", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/quinn-rs/quinn" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#quinn@0.11.9", + "name": "quinn", + "version": "0.11.9", + "description": "Versatile QUIC transport protocol implementation", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/quinn@0.11.9", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/quinn-rs/quinn" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "author": "David Tolnay ", + "name": "quote", + "version": "1.0.45", + "description": "Quasi-quoting macro quote!(...)", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/quote@1.0.45", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/quote/" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/quote" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rand@0.10.1", + "author": "The Rand Project Developers, The Rust Project Developers", + "name": "rand", + "version": "0.10.1", + "description": "Random number generators and other randomness functionality. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/rand@0.10.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/rand" + }, + { + "type": "website", + "url": "https://rust-random.github.io/book" + }, + { + "type": "vcs", + "url": "https://github.com/rust-random/rand" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rand@0.9.2", + "author": "The Rand Project Developers, The Rust Project Developers", + "name": "rand", + "version": "0.9.2", + "description": "Random number generators and other randomness functionality. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/rand@0.9.2", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/rand" + }, + { + "type": "website", + "url": "https://rust-random.github.io/book" + }, + { + "type": "vcs", + "url": "https://github.com/rust-random/rand" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rand_chacha@0.9.0", + "author": "The Rand Project Developers, The Rust Project Developers, The CryptoCorrosion Contributors", + "name": "rand_chacha", + "version": "0.9.0", + "description": "ChaCha random number generator ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/rand_chacha@0.9.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/rand_chacha" + }, + { + "type": "website", + "url": "https://rust-random.github.io/book" + }, + { + "type": "vcs", + "url": "https://github.com/rust-random/rand" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rand_core@0.10.1", + "author": "The Rand Project Developers", + "name": "rand_core", + "version": "0.10.1", + "description": "Core random number generation traits and tools for implementation.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/rand_core@0.10.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/rand_core" + }, + { + "type": "website", + "url": "https://rust-random.github.io/book" + }, + { + "type": "vcs", + "url": "https://github.com/rust-random/rand_core" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rand_core@0.9.5", + "author": "The Rand Project Developers, The Rust Project Developers", + "name": "rand_core", + "version": "0.9.5", + "description": "Core random number generator traits and tools for implementation. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/rand_core@0.9.5", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/rand_core" + }, + { + "type": "website", + "url": "https://rust-random.github.io/book" + }, + { + "type": "vcs", + "url": "https://github.com/rust-random/rand" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#redb@3.1.1", + "author": "Christopher Berner ", + "name": "redb", + "version": "3.1.1", + "description": "Rust Embedded DataBase", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "ef99362319c782aa4639ad3a306b64c3bb90e12874e99b8df124cb679d988611" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/redb@3.1.1", + "externalReferences": [ + { + "type": "website", + "url": "https://www.redb.org" + }, + { + "type": "vcs", + "url": "https://github.com/cberner/redb" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#regex-automata@0.4.14", + "author": "The Rust Project Developers, Andrew Gallant ", + "name": "regex-automata", + "version": "0.4.14", + "description": "Automata construction and matching using regular expressions.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/regex-automata@0.4.14", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/regex-automata" + }, + { + "type": "website", + "url": "https://github.com/rust-lang/regex/tree/master/regex-automata" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/regex" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#regex-syntax@0.8.10", + "author": "The Rust Project Developers, Andrew Gallant ", + "name": "regex-syntax", + "version": "0.8.10", + "description": "A regular expression parser.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/regex-syntax@0.8.10", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/regex-syntax" + }, + { + "type": "website", + "url": "https://github.com/rust-lang/regex/tree/master/regex-syntax" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/regex" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#regex@1.12.3", + "author": "The Rust Project Developers, Andrew Gallant ", + "name": "regex", + "version": "1.12.3", + "description": "An implementation of regular expressions for Rust. This implementation uses finite automata and guarantees linear time matching on all inputs. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/regex@1.12.3", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/regex" + }, + { + "type": "website", + "url": "https://github.com/rust-lang/regex" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/regex" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#reqwest-middleware@0.5.1", + "author": "Rodrigo Gryzinski ", + "name": "reqwest-middleware", + "version": "0.5.1", + "description": "Wrapper around reqwest to allow for client middleware chains.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "199dda04a536b532d0cc04d7979e39b1c763ea749bf91507017069c00b96056f" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/reqwest-middleware@0.5.1", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/TrueLayer/reqwest-middleware" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#reqwest@0.13.2", + "author": "Sean McArthur ", + "name": "reqwest", + "version": "0.13.2", + "description": "higher level HTTP client library", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/reqwest@0.13.2", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/reqwest" + }, + { + "type": "vcs", + "url": "https://github.com/seanmonstar/reqwest" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#ring@0.17.14", + "name": "ring", + "version": "0.17.14", + "description": "An experiment.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 AND ISC" + } + ], + "purl": "pkg:cargo/ring@0.17.14", + "externalReferences": [ + { + "type": "other", + "url": "ring_core_0_17_14_" + }, + { + "type": "vcs", + "url": "https://github.com/briansmith/ring" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rustc-hash@2.1.1", + "author": "The Rust Project Developers", + "name": "rustc-hash", + "version": "2.1.1", + "description": "A speedy, non-cryptographic hashing algorithm used by rustc", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/rustc-hash@2.1.1", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/rust-lang/rustc-hash" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rustix@1.1.4", + "author": "Dan Gohman , Jakub Konka ", + "name": "rustix", + "version": "1.1.4", + "description": "Safe Rust bindings to POSIX/Unix/Linux/Winsock-like syscalls", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/rustix@1.1.4", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/rustix" + }, + { + "type": "vcs", + "url": "https://github.com/bytecodealliance/rustix" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rustls-native-certs@0.8.3", + "name": "rustls-native-certs", + "version": "0.8.3", + "description": "rustls-native-certs allows rustls to use the platform native certificate store", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR ISC OR MIT" + } + ], + "purl": "pkg:cargo/rustls-native-certs@0.8.3", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/rustls/rustls-native-certs" + }, + { + "type": "vcs", + "url": "https://github.com/rustls/rustls-native-certs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rustls-pki-types@1.14.0", + "name": "rustls-pki-types", + "version": "1.14.0", + "description": "Shared types for the rustls PKI ecosystem", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/rustls-pki-types@1.14.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/rustls-pki-types" + }, + { + "type": "website", + "url": "https://github.com/rustls/pki-types" + }, + { + "type": "vcs", + "url": "https://github.com/rustls/pki-types" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rustls-platform-verifier@0.6.2", + "name": "rustls-platform-verifier", + "version": "0.6.2", + "description": "rustls-platform-verifier supports verifying TLS certificates in rustls with the operating system verifier", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/rustls-platform-verifier@0.6.2", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/rustls/rustls-platform-verifier" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rustls-webpki@0.103.13", + "name": "rustls-webpki", + "version": "0.103.13", + "description": "Web PKI X.509 Certificate Verification.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" + } + ], + "licenses": [ + { + "expression": "ISC" + } + ], + "purl": "pkg:cargo/rustls-webpki@0.103.13", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/rustls/webpki" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rustls@0.23.37", + "name": "rustls", + "version": "0.23.37", + "description": "Rustls is a modern TLS library written in Rust.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR ISC OR MIT" + } + ], + "purl": "pkg:cargo/rustls@0.23.37", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/rustls/rustls" + }, + { + "type": "vcs", + "url": "https://github.com/rustls/rustls" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rustversion@1.0.22", + "author": "David Tolnay ", + "name": "rustversion", + "version": "1.0.22", + "description": "Conditional compilation according to rustc compiler version", + "scope": "excluded", + "hashes": [ + { + "alg": "SHA-256", + "content": "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/rustversion@1.0.22", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/rustversion" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/rustversion" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#ryu@1.0.23", + "author": "David Tolnay ", + "name": "ryu", + "version": "1.0.23", + "description": "Fast floating point to string conversion", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR BSL-1.0" + } + ], + "purl": "pkg:cargo/ryu@1.0.23", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/ryu" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/ryu" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#safe-transmute@0.11.3", + "author": "наб , Eduardo Pinho , Lukas Kalbertodt , Philipp Tessenow , Marijn Suijten ", + "name": "safe-transmute", + "version": "0.11.3", + "description": "A safeguarded transmute() for Rust", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/safe-transmute@0.11.3", + "externalReferences": [ + { + "type": "documentation", + "url": "https://rawcdn.githack.com/nabijaczleweli/safe-transmute-rs/doc/safe_transmute/index.html" + }, + { + "type": "vcs", + "url": "https://github.com/nabijaczleweli/safe-transmute-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#same-file@1.0.6", + "author": "Andrew Gallant ", + "name": "same-file", + "version": "1.0.6", + "description": "A simple crate for determining whether two file paths point to the same file. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" + } + ], + "licenses": [ + { + "expression": "Unlicense OR MIT" + } + ], + "purl": "pkg:cargo/same-file@1.0.6", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/same-file" + }, + { + "type": "website", + "url": "https://github.com/BurntSushi/same-file" + }, + { + "type": "vcs", + "url": "https://github.com/BurntSushi/same-file" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", + "author": "Erick Tryzelaar , David Tolnay ", + "name": "serde", + "version": "1.0.228", + "description": "A generic serialization/deserialization framework", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/serde@1.0.228", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/serde" + }, + { + "type": "website", + "url": "https://serde.rs" + }, + { + "type": "vcs", + "url": "https://github.com/serde-rs/serde" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#serde_core@1.0.228", + "author": "Erick Tryzelaar , David Tolnay ", + "name": "serde_core", + "version": "1.0.228", + "description": "Serde traits only, with no support for derive -- use the `serde` crate instead", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/serde_core@1.0.228", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/serde_core" + }, + { + "type": "website", + "url": "https://serde.rs" + }, + { + "type": "vcs", + "url": "https://github.com/serde-rs/serde" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#serde_derive@1.0.228", + "author": "Erick Tryzelaar , David Tolnay ", + "name": "serde_derive", + "version": "1.0.228", + "description": "Macros 1.1 implementation of #[derive(Serialize, Deserialize)]", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/serde_derive@1.0.228", + "externalReferences": [ + { + "type": "documentation", + "url": "https://serde.rs/derive.html" + }, + { + "type": "website", + "url": "https://serde.rs" + }, + { + "type": "vcs", + "url": "https://github.com/serde-rs/serde" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.149", + "author": "Erick Tryzelaar , David Tolnay ", + "name": "serde_json", + "version": "1.0.149", + "description": "A JSON serialization file format", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/serde_json@1.0.149", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/serde_json" + }, + { + "type": "vcs", + "url": "https://github.com/serde-rs/json" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#serde_repr@0.1.20", + "author": "David Tolnay ", + "name": "serde_repr", + "version": "0.1.20", + "description": "Derive Serialize and Deserialize that delegates to the underlying repr of a C-like enum.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/serde_repr@0.1.20", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/serde_repr" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/serde-repr" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#sha2-asm@0.6.4", + "author": "RustCrypto Developers", + "name": "sha2-asm", + "version": "0.6.4", + "description": "Assembly implementation of SHA-2 compression functions", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/sha2-asm@0.6.4", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/sha2-asm" + }, + { + "type": "vcs", + "url": "https://github.com/RustCrypto/asm-hashes" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#sha2@0.10.9", + "author": "RustCrypto Developers", + "name": "sha2", + "version": "0.10.9", + "description": "Pure Rust implementation of the SHA-2 hash function family including SHA-224, SHA-256, SHA-384, and SHA-512. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/sha2@0.10.9", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/sha2" + }, + { + "type": "vcs", + "url": "https://github.com/RustCrypto/hashes" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#sharded-slab@0.1.7", + "author": "Eliza Weisman ", + "name": "sharded-slab", + "version": "0.1.7", + "description": "A lock-free concurrent slab. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/sharded-slab@0.1.7", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/sharded-slab/" + }, + { + "type": "website", + "url": "https://github.com/hawkw/sharded-slab" + }, + { + "type": "vcs", + "url": "https://github.com/hawkw/sharded-slab" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#shellexpand@3.1.2", + "author": "Vladimir Matveev , Ian Jackson ", + "name": "shellexpand", + "version": "3.1.2", + "description": "Shell-like expansions in strings", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/shellexpand@3.1.2", + "externalReferences": [ + { + "type": "documentation", + "url": "http://docs.rs/shellexpand/" + }, + { + "type": "vcs", + "url": "https://gitlab.com/ijackson/rust-shellexpand" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#shlex@1.3.0", + "author": "comex , Fenhl , Adrian Taylor , Alex Touchet , Daniel Parks , Garrett Berg ", + "name": "shlex", + "version": "1.3.0", + "description": "Split a string into shell words, like Python's shlex.", + "scope": "excluded", + "hashes": [ + { + "alg": "SHA-256", + "content": "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/shlex@1.3.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/comex/rust-shlex" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#signal-hook-registry@1.4.8", + "author": "Michal 'vorner' Vaner , Masaki Hara ", + "name": "signal-hook-registry", + "version": "1.4.8", + "description": "Backend crate for signal-hook", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/signal-hook-registry@1.4.8", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/signal-hook-registry" + }, + { + "type": "vcs", + "url": "https://github.com/vorner/signal-hook" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#signal-hook@0.3.18", + "author": "Michal 'vorner' Vaner , Thomas Himmelstoss ", + "name": "signal-hook", + "version": "0.3.18", + "description": "Unix signal handling", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/signal-hook@0.3.18", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/signal-hook" + }, + { + "type": "vcs", + "url": "https://github.com/vorner/signal-hook" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#slab@0.4.12", + "author": "Carl Lerche ", + "name": "slab", + "version": "0.4.12", + "description": "Pre-allocated storage for a uniform data type", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/slab@0.4.12", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/tokio-rs/slab" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1", + "author": "The Servo Project Developers", + "name": "smallvec", + "version": "1.15.1", + "description": "'Small vector' optimization: store up to a small number of items on the stack", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/smallvec@1.15.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/smallvec/" + }, + { + "type": "vcs", + "url": "https://github.com/servo/rust-smallvec" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#socket2@0.6.3", + "author": "Alex Crichton , Thomas de Zeeuw ", + "name": "socket2", + "version": "0.6.3", + "description": "Utilities for handling networking sockets with a maximal amount of configuration possible intended. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/socket2@0.6.3", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/socket2" + }, + { + "type": "website", + "url": "https://github.com/rust-lang/socket2" + }, + { + "type": "vcs", + "url": "https://github.com/rust-lang/socket2" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#stable_deref_trait@1.2.1", + "author": "Robert Grosse ", + "name": "stable_deref_trait", + "version": "1.2.1", + "description": "An unsafe marker trait for types like Box and Rc that dereference to a stable address even when moved, and hence can be used with libraries such as owning_ref and rental. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/stable_deref_trait@1.2.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/stable_deref_trait/1.2.1/stable_deref_trait" + }, + { + "type": "vcs", + "url": "https://github.com/storyyeller/stable_deref_trait" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#static_assertions@1.1.0", + "author": "Nikolai Vazquez", + "name": "static_assertions", + "version": "1.1.0", + "description": "Compile-time assertions to ensure that invariants are met.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/static_assertions@1.1.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/static_assertions/" + }, + { + "type": "website", + "url": "https://github.com/nvzqz/static-assertions-rs" + }, + { + "type": "vcs", + "url": "https://github.com/nvzqz/static-assertions-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#statrs@0.18.0", + "author": "Michael Ma", + "name": "statrs", + "version": "0.18.0", + "description": "Statistical computing library for Rust", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/statrs@0.18.0", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/statrs-dev/statrs" + }, + { + "type": "vcs", + "url": "https://github.com/statrs-dev/statrs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#strsim@0.11.1", + "author": "Danny Guo , maxbachmann ", + "name": "strsim", + "version": "0.11.1", + "description": "Implementations of string similarity metrics. Includes Hamming, Levenshtein, OSA, Damerau-Levenshtein, Jaro, Jaro-Winkler, and Sørensen-Dice. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/strsim@0.11.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/strsim/" + }, + { + "type": "website", + "url": "https://github.com/rapidfuzz/strsim-rs" + }, + { + "type": "vcs", + "url": "https://github.com/rapidfuzz/strsim-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#subtle@2.6.1", + "author": "Isis Lovecruft , Henry de Valence ", + "name": "subtle", + "version": "2.6.1", + "description": "Pure-Rust traits and utilities for constant-time cryptographic implementations.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + } + ], + "licenses": [ + { + "expression": "BSD-3-Clause" + } + ], + "purl": "pkg:cargo/subtle@2.6.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/subtle" + }, + { + "type": "website", + "url": "https://dalek.rs/" + }, + { + "type": "vcs", + "url": "https://github.com/dalek-cryptography/subtle" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117", + "author": "David Tolnay ", + "name": "syn", + "version": "2.0.117", + "description": "Parser for Rust source code", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/syn@2.0.117", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/syn" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/syn" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#sync_wrapper@1.0.2", + "author": "Actyx AG ", + "name": "sync_wrapper", + "version": "1.0.2", + "description": "A tool for enlisting the compiler's help in proving the absence of concurrency", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + } + ], + "licenses": [ + { + "expression": "Apache-2.0" + } + ], + "purl": "pkg:cargo/sync_wrapper@1.0.2", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/sync_wrapper" + }, + { + "type": "website", + "url": "https://docs.rs/sync_wrapper" + }, + { + "type": "vcs", + "url": "https://github.com/Actyx/sync_wrapper" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#synstructure@0.13.2", + "author": "Nika Layzell ", + "name": "synstructure", + "version": "0.13.2", + "description": "Helper methods and macros for custom derives", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/synstructure@0.13.2", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/synstructure" + }, + { + "type": "vcs", + "url": "https://github.com/mystor/synstructure" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#sysinfo@0.38.4", + "author": "Guillaume Gomez ", + "name": "sysinfo", + "version": "0.38.4", + "description": "Library to get system information such as processes, CPUs, disks, components and networks", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/sysinfo@0.38.4", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/GuillaumeGomez/sysinfo" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#target-lexicon@0.13.5", + "author": "Dan Gohman ", + "name": "target-lexicon", + "version": "0.13.5", + "description": "LLVM target triple types", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 WITH LLVM-exception" + } + ], + "purl": "pkg:cargo/target-lexicon@0.13.5", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/target-lexicon/" + }, + { + "type": "vcs", + "url": "https://github.com/bytecodealliance/target-lexicon" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tempfile@3.27.0", + "author": "Steven Allen , The Rust Project Developers, Ashley Mannix , Jason White ", + "name": "tempfile", + "version": "3.27.0", + "description": "A library for managing temporary files and directories.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/tempfile@3.27.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/tempfile" + }, + { + "type": "website", + "url": "https://stebalien.com/projects/tempfile-rs/" + }, + { + "type": "vcs", + "url": "https://github.com/Stebalien/tempfile" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#thiserror-impl@2.0.18", + "author": "David Tolnay ", + "name": "thiserror-impl", + "version": "2.0.18", + "description": "Implementation detail of the `thiserror` crate", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/thiserror-impl@2.0.18", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/dtolnay/thiserror" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#thiserror@2.0.18", + "author": "David Tolnay ", + "name": "thiserror", + "version": "2.0.18", + "description": "derive(Error)", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/thiserror@2.0.18", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/thiserror" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/thiserror" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#thread_local@1.1.9", + "author": "Amanieu d'Antras ", + "name": "thread_local", + "version": "1.1.9", + "description": "Per-object thread-local storage", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/thread_local@1.1.9", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/thread_local/" + }, + { + "type": "vcs", + "url": "https://github.com/Amanieu/thread_local-rs" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#time-core@0.1.8", + "author": "Jacob Pratt , Time contributors", + "name": "time-core", + "version": "0.1.8", + "description": "This crate is an implementation detail and should not be relied upon directly.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/time-core@0.1.8", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/time-rs/time" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#time-macros@0.2.27", + "author": "Jacob Pratt , Time contributors", + "name": "time-macros", + "version": "0.2.27", + "description": " Procedural macros for the time crate. This crate is an implementation detail and should not be relied upon directly. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/time-macros@0.2.27", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/time-rs/time" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#time@0.3.47", + "author": "Jacob Pratt , Time contributors", + "name": "time", + "version": "0.3.47", + "description": "Date and time library. Fully interoperable with the standard library. Mostly compatible with #![no_std].", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/time@0.3.47", + "externalReferences": [ + { + "type": "website", + "url": "https://time-rs.github.io" + }, + { + "type": "vcs", + "url": "https://github.com/time-rs/time" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tinystr@0.8.2", + "author": "The ICU4X Project Developers", + "name": "tinystr", + "version": "0.8.2", + "description": "A small ASCII-only bounded length string representation.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/tinystr@0.8.2", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tinyvec@1.11.0", + "author": "Lokathor ", + "name": "tinyvec", + "version": "1.11.0", + "description": "`tinyvec` provides 100% safe vec-like data structures.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" + } + ], + "licenses": [ + { + "expression": "Zlib OR Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/tinyvec@1.11.0", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/Lokathor/tinyvec" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tinyvec_macros@0.1.1", + "author": "Soveu ", + "name": "tinyvec_macros", + "version": "0.1.1", + "description": "Some macros for tiny containers", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0 OR Zlib" + } + ], + "purl": "pkg:cargo/tinyvec_macros@0.1.1", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/Soveu/tinyvec_macros" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tokio-macros@2.6.1", + "author": "Tokio Contributors ", + "name": "tokio-macros", + "version": "2.6.1", + "description": "Tokio's proc macros. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tokio-macros@2.6.1", + "externalReferences": [ + { + "type": "website", + "url": "https://tokio.rs" + }, + { + "type": "vcs", + "url": "https://github.com/tokio-rs/tokio" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tokio-retry@0.3.1", + "name": "tokio-retry", + "version": "0.3.1", + "description": "Extensible, asynchronous retry behaviours for futures/tokio", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "40f644c762e9d396831ae2f8935c954b0d758c4532e924bead0f666d0c1c8640" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tokio-retry@0.3.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/tokio-retry" + }, + { + "type": "vcs", + "url": "https://github.com/djc/tokio-retry" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tokio-rustls@0.26.4", + "name": "tokio-rustls", + "version": "0.26.4", + "description": "Asynchronous TLS/SSL streams for Tokio using Rustls.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/tokio-rustls@0.26.4", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/tokio-rustls" + }, + { + "type": "website", + "url": "https://github.com/rustls/tokio-rustls" + }, + { + "type": "vcs", + "url": "https://github.com/rustls/tokio-rustls" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tokio-util@0.7.18", + "author": "Tokio Contributors ", + "name": "tokio-util", + "version": "0.7.18", + "description": "Additional utilities for working with Tokio. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tokio-util@0.7.18", + "externalReferences": [ + { + "type": "website", + "url": "https://tokio.rs" + }, + { + "type": "vcs", + "url": "https://github.com/tokio-rs/tokio" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "author": "Tokio Contributors ", + "name": "tokio", + "version": "1.50.0", + "description": "An event-driven, non-blocking I/O platform for writing asynchronous I/O backed applications. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tokio@1.50.0", + "externalReferences": [ + { + "type": "website", + "url": "https://tokio.rs" + }, + { + "type": "vcs", + "url": "https://github.com/tokio-rs/tokio" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tower-http@0.6.8", + "author": "Tower Maintainers ", + "name": "tower-http", + "version": "0.6.8", + "description": "Tower middleware and utilities for HTTP clients and servers", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tower-http@0.6.8", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/tower-rs/tower-http" + }, + { + "type": "vcs", + "url": "https://github.com/tower-rs/tower-http" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tower-layer@0.3.3", + "author": "Tower Maintainers ", + "name": "tower-layer", + "version": "0.3.3", + "description": "Decorates a `Service` to allow easy composition between `Service`s. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tower-layer@0.3.3", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/tower-layer/0.3.3" + }, + { + "type": "website", + "url": "https://github.com/tower-rs/tower" + }, + { + "type": "vcs", + "url": "https://github.com/tower-rs/tower" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tower-service@0.3.3", + "author": "Tower Maintainers ", + "name": "tower-service", + "version": "0.3.3", + "description": "Trait representing an asynchronous, request / response based, client or server. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tower-service@0.3.3", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/tower-service/0.3.3" + }, + { + "type": "website", + "url": "https://github.com/tower-rs/tower" + }, + { + "type": "vcs", + "url": "https://github.com/tower-rs/tower" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tower@0.5.3", + "author": "Tower Maintainers ", + "name": "tower", + "version": "0.5.3", + "description": "Tower is a library of modular and reusable components for building robust clients and servers. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tower@0.5.3", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/tower-rs/tower" + }, + { + "type": "vcs", + "url": "https://github.com/tower-rs/tower" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-appender@0.2.4", + "author": "Zeki Sherif , Tokio Contributors ", + "name": "tracing-appender", + "version": "0.2.4", + "description": "Provides utilities for file appenders and making non-blocking writers. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tracing-appender@0.2.4", + "externalReferences": [ + { + "type": "website", + "url": "https://tokio.rs" + }, + { + "type": "vcs", + "url": "https://github.com/tokio-rs/tracing" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-attributes@0.1.31", + "author": "Tokio Contributors , Eliza Weisman , David Barsky ", + "name": "tracing-attributes", + "version": "0.1.31", + "description": "Procedural macro attributes for automatically instrumenting functions. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tracing-attributes@0.1.31", + "externalReferences": [ + { + "type": "website", + "url": "https://tokio.rs" + }, + { + "type": "vcs", + "url": "https://github.com/tokio-rs/tracing" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-core@0.1.36", + "author": "Tokio Contributors ", + "name": "tracing-core", + "version": "0.1.36", + "description": "Core primitives for application-level tracing. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tracing-core@0.1.36", + "externalReferences": [ + { + "type": "website", + "url": "https://tokio.rs" + }, + { + "type": "vcs", + "url": "https://github.com/tokio-rs/tracing" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-log@0.2.0", + "author": "Tokio Contributors ", + "name": "tracing-log", + "version": "0.2.0", + "description": "Provides compatibility between `tracing` and the `log` crate. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tracing-log@0.2.0", + "externalReferences": [ + { + "type": "website", + "url": "https://tokio.rs" + }, + { + "type": "vcs", + "url": "https://github.com/tokio-rs/tracing" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-serde@0.2.0", + "author": "Tokio Contributors ", + "name": "tracing-serde", + "version": "0.2.0", + "description": "A compatibility layer for serializing trace data with `serde` ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tracing-serde@0.2.0", + "externalReferences": [ + { + "type": "website", + "url": "https://tokio.rs" + }, + { + "type": "vcs", + "url": "https://github.com/tokio-rs/tracing" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-subscriber@0.3.23", + "author": "Eliza Weisman , David Barsky , Tokio Contributors ", + "name": "tracing-subscriber", + "version": "0.3.23", + "description": "Utilities for implementing and composing `tracing` subscribers. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tracing-subscriber@0.3.23", + "externalReferences": [ + { + "type": "website", + "url": "https://tokio.rs" + }, + { + "type": "vcs", + "url": "https://github.com/tokio-rs/tracing" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44", + "author": "Eliza Weisman , Tokio Contributors ", + "name": "tracing", + "version": "0.1.44", + "description": "Application-level tracing for Rust. ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/tracing@0.1.44", + "externalReferences": [ + { + "type": "website", + "url": "https://tokio.rs" + }, + { + "type": "vcs", + "url": "https://github.com/tokio-rs/tracing" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#try-lock@0.2.5", + "author": "Sean McArthur ", + "name": "try-lock", + "version": "0.2.5", + "description": "A lightweight atomic lock.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/try-lock@0.2.5", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/try-lock" + }, + { + "type": "website", + "url": "https://github.com/seanmonstar/try-lock" + }, + { + "type": "vcs", + "url": "https://github.com/seanmonstar/try-lock" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#twox-hash@2.1.2", + "author": "Jake Goulding ", + "name": "twox-hash", + "version": "2.1.2", + "description": "A Rust implementation of the XXHash and XXH3 algorithms", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/twox-hash@2.1.2", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/twox-hash/" + }, + { + "type": "vcs", + "url": "https://github.com/shepmaster/twox-hash" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#typenum@1.19.0", + "author": "Paho Lurie-Gregg , Andre Bogus ", + "name": "typenum", + "version": "1.19.0", + "description": "Typenum is a Rust library for type-level numbers evaluated at compile time. It currently supports bits, unsigned integers, and signed integers. It also provides a type-level array of type-level numbers, but its implementation is incomplete.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/typenum@1.19.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/typenum" + }, + { + "type": "vcs", + "url": "https://github.com/paholg/typenum" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#typewit@1.14.2", + "author": "rodrimati1992 ", + "name": "typewit", + "version": "1.14.2", + "description": "type-witness-based abstractions, mostly for emulating polymorphism in const fns", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "f8c1ae7cc0fdb8b842d65d127cb981574b0d2b249b74d1c7a2986863dc134f71" + } + ], + "licenses": [ + { + "expression": "Zlib" + } + ], + "purl": "pkg:cargo/typewit@1.14.2", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/typewit/" + }, + { + "type": "vcs", + "url": "https://github.com/rodrimati1992/typewit/" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#unicode-ident@1.0.24", + "author": "David Tolnay ", + "name": "unicode-ident", + "version": "1.0.24", + "description": "Determine whether characters have the XID_Start or XID_Continue properties according to Unicode Standard Annex #31", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + } + ], + "licenses": [ + { + "expression": "(MIT OR Apache-2.0) AND Unicode-3.0" + } + ], + "purl": "pkg:cargo/unicode-ident@1.0.24", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/unicode-ident" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/unicode-ident" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#unindent@0.2.4", + "author": "David Tolnay ", + "name": "unindent", + "version": "0.2.4", + "description": "Remove a column of leading whitespace from a string", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/unindent@0.2.4", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/unindent" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/indoc" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#untrusted@0.9.0", + "author": "Brian Smith ", + "name": "untrusted", + "version": "0.9.0", + "description": "Safe, fast, zero-panic, zero-crashing, zero-allocation parsing of untrusted inputs in Rust.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + } + ], + "licenses": [ + { + "expression": "ISC" + } + ], + "purl": "pkg:cargo/untrusted@0.9.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://briansmith.org/rustdoc/untrusted/" + }, + { + "type": "vcs", + "url": "https://github.com/briansmith/untrusted" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#url@2.5.8", + "author": "The rust-url developers", + "name": "url", + "version": "2.5.8", + "description": "URL library for Rust, based on the WHATWG URL Standard", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/url@2.5.8", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/url" + }, + { + "type": "vcs", + "url": "https://github.com/servo/rust-url" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#urlencoding@2.1.3", + "author": "Kornel , Bertram Truong ", + "name": "urlencoding", + "version": "2.1.3", + "description": "A Rust library for doing URL percentage encoding.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/urlencoding@2.1.3", + "externalReferences": [ + { + "type": "website", + "url": "https://lib.rs/urlencoding" + }, + { + "type": "vcs", + "url": "https://github.com/kornelski/rust_urlencoding" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#utf8_iter@1.0.4", + "author": "Henri Sivonen ", + "name": "utf8_iter", + "version": "1.0.4", + "description": "Iterator by char over potentially-invalid UTF-8 in &[u8]", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/utf8_iter@1.0.4", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/utf8_iter/" + }, + { + "type": "website", + "url": "https://docs.rs/utf8_iter/" + }, + { + "type": "vcs", + "url": "https://github.com/hsivonen/utf8_iter" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#utf8parse@0.2.2", + "author": "Joe Wilm , Christian Duerr ", + "name": "utf8parse", + "version": "0.2.2", + "description": "Table-driven UTF-8 parser", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/utf8parse@0.2.2", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/utf8parse/" + }, + { + "type": "vcs", + "url": "https://github.com/alacritty/vte" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#uuid@1.22.0", + "author": "Ashley Mannix, Dylan DPC, Hunar Roop Kahlon", + "name": "uuid", + "version": "1.22.0", + "description": "A library to generate and parse UUIDs.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/uuid@1.22.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/uuid" + }, + { + "type": "website", + "url": "https://github.com/uuid-rs/uuid" + }, + { + "type": "vcs", + "url": "https://github.com/uuid-rs/uuid" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#version_check@0.9.5", + "author": "Sergio Benitez ", + "name": "version_check", + "version": "0.9.5", + "description": "Tiny crate to check the version of the installed/running rustc.", + "scope": "excluded", + "hashes": [ + { + "alg": "SHA-256", + "content": "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0" + } + ], + "purl": "pkg:cargo/version_check@0.9.5", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/version_check/" + }, + { + "type": "vcs", + "url": "https://github.com/SergioBenitez/version_check" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#walkdir@2.5.0", + "author": "Andrew Gallant ", + "name": "walkdir", + "version": "2.5.0", + "description": "Recursively walk a directory.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" + } + ], + "licenses": [ + { + "expression": "Unlicense OR MIT" + } + ], + "purl": "pkg:cargo/walkdir@2.5.0", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/walkdir/" + }, + { + "type": "website", + "url": "https://github.com/BurntSushi/walkdir" + }, + { + "type": "vcs", + "url": "https://github.com/BurntSushi/walkdir" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#want@0.3.1", + "author": "Sean McArthur ", + "name": "want", + "version": "0.3.1", + "description": "Detect when another Future wants a result.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/want@0.3.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/want" + }, + { + "type": "vcs", + "url": "https://github.com/seanmonstar/want" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#whoami@2.1.1", + "name": "whoami", + "version": "2.1.1", + "description": "Rust library for getting information about the current user and environment", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR BSL-1.0 OR MIT" + } + ], + "purl": "pkg:cargo/whoami@2.1.1", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/whoami" + }, + { + "type": "website", + "url": "https://github.com/ardaku/whoami/releases" + }, + { + "type": "vcs", + "url": "https://github.com/ardaku/whoami" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#writeable@0.6.2", + "author": "The ICU4X Project Developers", + "name": "writeable", + "version": "0.6.2", + "description": "A more efficient alternative to fmt::Display", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/writeable@0.6.2", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#yoke-derive@0.8.1", + "author": "Manish Goregaokar ", + "name": "yoke-derive", + "version": "0.8.1", + "description": "Custom derive for the yoke crate", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/yoke-derive@0.8.1", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#yoke@0.8.1", + "author": "Manish Goregaokar ", + "name": "yoke", + "version": "0.8.1", + "description": "Abstraction allowing borrowed data to be carried along with the backing data it borrows from", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/yoke@0.8.1", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerocopy@0.8.47", + "author": "Joshua Liebow-Feeser , Jack Wrenn ", + "name": "zerocopy", + "version": "0.8.47", + "description": "Zerocopy makes zero-cost memory manipulation effortless. We write \"unsafe\" so you don't have to.", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" + } + ], + "licenses": [ + { + "expression": "BSD-2-Clause OR Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/zerocopy@0.8.47", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/google/zerocopy" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerofrom-derive@0.1.6", + "author": "Manish Goregaokar ", + "name": "zerofrom-derive", + "version": "0.1.6", + "description": "Custom derive for the zerofrom crate", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/zerofrom-derive@0.1.6", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.6", + "author": "Manish Goregaokar ", + "name": "zerofrom", + "version": "0.1.6", + "description": "ZeroFrom trait for constructing", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/zerofrom@0.1.6", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zeroize@1.8.2", + "author": "The RustCrypto Project Developers", + "name": "zeroize", + "version": "1.8.2", + "description": "Securely clear secrets from memory with a simple trait built on stable Rust primitives which guarantee memory is zeroed using an operation will not be 'optimized away' by the compiler. Uses a portable pure Rust implementation that works everywhere, even WASM! ", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + } + ], + "licenses": [ + { + "expression": "Apache-2.0 OR MIT" + } + ], + "purl": "pkg:cargo/zeroize@1.8.2", + "externalReferences": [ + { + "type": "website", + "url": "https://github.com/RustCrypto/utils/tree/master/zeroize" + }, + { + "type": "vcs", + "url": "https://github.com/RustCrypto/utils" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerotrie@0.2.3", + "author": "The ICU4X Project Developers", + "name": "zerotrie", + "version": "0.2.3", + "description": "A data structure that efficiently maps strings to integers", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/zerotrie@0.2.3", + "externalReferences": [ + { + "type": "website", + "url": "https://icu4x.unicode.org" + }, + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerovec-derive@0.11.2", + "author": "Manish Goregaokar ", + "name": "zerovec-derive", + "version": "0.11.2", + "description": "Custom derive for the zerovec crate", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/zerovec-derive@0.11.2", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.11.5", + "author": "The ICU4X Project Developers", + "name": "zerovec", + "version": "0.11.5", + "description": "Zero-copy vector backed by a byte array", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" + } + ], + "licenses": [ + { + "expression": "Unicode-3.0" + } + ], + "purl": "pkg:cargo/zerovec@0.11.5", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/unicode-org/icu4x" + } + ] + }, + { + "type": "library", + "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zmij@1.0.21", + "author": "David Tolnay ", + "name": "zmij", + "version": "1.0.21", + "description": "A double-to-string conversion algorithm based on Schubfach and yy", + "scope": "required", + "hashes": [ + { + "alg": "SHA-256", + "content": "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + } + ], + "licenses": [ + { + "expression": "MIT" + } + ], + "purl": "pkg:cargo/zmij@1.0.21", + "externalReferences": [ + { + "type": "documentation", + "url": "https://docs.rs/zmij" + }, + { + "type": "vcs", + "url": "https://github.com/dtolnay/zmij" + } + ] + } + ], + "dependencies": [ + { + "ref": "path+file:///home/runner/work/xet-core/xet-core/hf_xet#1.5.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#async-trait@0.1.89", + "path+file:///home/runner/work/xet-core/xet-core/xet_pkg#hf-xet@1.5.2", + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#itertools@0.14.0", + "registry+https://github.com/rust-lang/crates.io-index#lazy_static@1.5.0", + "registry+https://github.com/rust-lang/crates.io-index#pyo3@0.26.0", + "registry+https://github.com/rust-lang/crates.io-index#rand@0.10.1", + "registry+https://github.com/rust-lang/crates.io-index#signal-hook@0.3.18", + "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44", + "path+file:///home/runner/work/xet-core/xet-core/xet_client#xet-client@1.5.2", + "path+file:///home/runner/work/xet-core/xet-core/xet_runtime#xet-runtime@1.5.2" + ] + }, + { + "ref": "path+file:///home/runner/work/xet-core/xet-core/xet_client#xet-client@1.5.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#anyhow@1.0.102", + "registry+https://github.com/rust-lang/crates.io-index#async-trait@0.1.89", + "registry+https://github.com/rust-lang/crates.io-index#base64@0.22.1", + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#clap@4.6.0", + "registry+https://github.com/rust-lang/crates.io-index#crc32fast@1.5.0", + "registry+https://github.com/rust-lang/crates.io-index#futures@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#hyper@1.8.1", + "registry+https://github.com/rust-lang/crates.io-index#lazy_static@1.5.0", + "registry+https://github.com/rust-lang/crates.io-index#more-asserts@0.3.1", + "registry+https://github.com/rust-lang/crates.io-index#rand@0.10.1", + "registry+https://github.com/rust-lang/crates.io-index#redb@3.1.1", + "registry+https://github.com/rust-lang/crates.io-index#reqwest@0.13.2", + "registry+https://github.com/rust-lang/crates.io-index#reqwest-middleware@0.5.1", + "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", + "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.149", + "registry+https://github.com/rust-lang/crates.io-index#serde_repr@0.1.20", + "registry+https://github.com/rust-lang/crates.io-index#statrs@0.18.0", + "registry+https://github.com/rust-lang/crates.io-index#tempfile@3.27.0", + "registry+https://github.com/rust-lang/crates.io-index#thiserror@2.0.18", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "registry+https://github.com/rust-lang/crates.io-index#tokio-retry@0.3.1", + "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44", + "registry+https://github.com/rust-lang/crates.io-index#tracing-subscriber@0.3.23", + "registry+https://github.com/rust-lang/crates.io-index#url@2.5.8", + "registry+https://github.com/rust-lang/crates.io-index#urlencoding@2.1.3", + "path+file:///home/runner/work/xet-core/xet-core/xet_core_structures#xet-core-structures@1.5.2", + "path+file:///home/runner/work/xet-core/xet-core/xet_runtime#xet-runtime@1.5.2" + ] + }, + { + "ref": "path+file:///home/runner/work/xet-core/xet-core/xet_core_structures#xet-core-structures@1.5.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#async-trait@0.1.89", + "registry+https://github.com/rust-lang/crates.io-index#base64@0.22.1", + "registry+https://github.com/rust-lang/crates.io-index#blake3@1.8.3", + "registry+https://github.com/rust-lang/crates.io-index#bytemuck@1.25.0", + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#clap@4.6.0", + "registry+https://github.com/rust-lang/crates.io-index#countio@0.3.0", + "registry+https://github.com/rust-lang/crates.io-index#csv@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#futures@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-util@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#heapify@0.2.0", + "registry+https://github.com/rust-lang/crates.io-index#itertools@0.14.0", + "registry+https://github.com/rust-lang/crates.io-index#lazy_static@1.5.0", + "registry+https://github.com/rust-lang/crates.io-index#lz4_flex@0.13.0", + "registry+https://github.com/rust-lang/crates.io-index#more-asserts@0.3.1", + "registry+https://github.com/rust-lang/crates.io-index#rand@0.10.1", + "registry+https://github.com/rust-lang/crates.io-index#regex@1.12.3", + "registry+https://github.com/rust-lang/crates.io-index#safe-transmute@0.11.3", + "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", + "registry+https://github.com/rust-lang/crates.io-index#static_assertions@1.1.0", + "registry+https://github.com/rust-lang/crates.io-index#tempfile@3.27.0", + "registry+https://github.com/rust-lang/crates.io-index#thiserror@2.0.18", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "registry+https://github.com/rust-lang/crates.io-index#tokio-util@0.7.18", + "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44", + "registry+https://github.com/rust-lang/crates.io-index#uuid@1.22.0", + "path+file:///home/runner/work/xet-core/xet-core/xet_runtime#xet-runtime@1.5.2" + ] + }, + { + "ref": "path+file:///home/runner/work/xet-core/xet-core/xet_data#xet-data@1.5.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#anyhow@1.0.102", + "registry+https://github.com/rust-lang/crates.io-index#async-trait@0.1.89", + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#chrono@0.4.44", + "registry+https://github.com/rust-lang/crates.io-index#clap@4.6.0", + "registry+https://github.com/rust-lang/crates.io-index#gearhash@0.1.3", + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#itertools@0.14.0", + "registry+https://github.com/rust-lang/crates.io-index#lazy_static@1.5.0", + "registry+https://github.com/rust-lang/crates.io-index#more-asserts@0.3.1", + "registry+https://github.com/rust-lang/crates.io-index#pyo3@0.26.0", + "registry+https://github.com/rust-lang/crates.io-index#rand@0.10.1", + "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", + "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.149", + "registry+https://github.com/rust-lang/crates.io-index#sha2@0.10.9", + "registry+https://github.com/rust-lang/crates.io-index#tempfile@3.27.0", + "registry+https://github.com/rust-lang/crates.io-index#thiserror@2.0.18", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "registry+https://github.com/rust-lang/crates.io-index#tokio-util@0.7.18", + "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44", + "registry+https://github.com/rust-lang/crates.io-index#url@2.5.8", + "registry+https://github.com/rust-lang/crates.io-index#uuid@1.22.0", + "registry+https://github.com/rust-lang/crates.io-index#walkdir@2.5.0", + "path+file:///home/runner/work/xet-core/xet-core/xet_client#xet-client@1.5.2", + "path+file:///home/runner/work/xet-core/xet-core/xet_core_structures#xet-core-structures@1.5.2", + "path+file:///home/runner/work/xet-core/xet-core/xet_runtime#xet-runtime@1.5.2" + ] + }, + { + "ref": "path+file:///home/runner/work/xet-core/xet-core/xet_pkg#hf-xet@1.5.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#async-trait@0.1.89", + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#more-asserts@0.3.1", + "registry+https://github.com/rust-lang/crates.io-index#pyo3@0.26.0", + "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", + "registry+https://github.com/rust-lang/crates.io-index#thiserror@2.0.18", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "registry+https://github.com/rust-lang/crates.io-index#tokio-util@0.7.18", + "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44", + "registry+https://github.com/rust-lang/crates.io-index#uuid@1.22.0", + "path+file:///home/runner/work/xet-core/xet-core/xet_client#xet-client@1.5.2", + "path+file:///home/runner/work/xet-core/xet-core/xet_core_structures#xet-core-structures@1.5.2", + "path+file:///home/runner/work/xet-core/xet-core/xet_data#xet-data@1.5.2", + "path+file:///home/runner/work/xet-core/xet-core/xet_runtime#xet-runtime@1.5.2" + ] + }, + { + "ref": "path+file:///home/runner/work/xet-core/xet-core/xet_runtime#xet-runtime@1.5.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#anyhow@1.0.102", + "registry+https://github.com/rust-lang/crates.io-index#async-trait@0.1.89", + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#chrono@0.4.44", + "registry+https://github.com/rust-lang/crates.io-index#colored@3.1.1", + "registry+https://github.com/rust-lang/crates.io-index#const-str@1.1.0", + "registry+https://github.com/rust-lang/crates.io-index#ctor@0.6.3", + "registry+https://github.com/rust-lang/crates.io-index#dirs@6.0.0", + "registry+https://github.com/rust-lang/crates.io-index#futures@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#git-version@0.3.9", + "registry+https://github.com/rust-lang/crates.io-index#humantime@2.3.0", + "registry+https://github.com/rust-lang/crates.io-index#konst@0.4.3", + "registry+https://github.com/rust-lang/crates.io-index#lazy_static@1.5.0", + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#more-asserts@0.3.1", + "registry+https://github.com/rust-lang/crates.io-index#oneshot@0.1.13", + "registry+https://github.com/rust-lang/crates.io-index#pin-project@1.1.11", + "registry+https://github.com/rust-lang/crates.io-index#pyo3@0.26.0", + "registry+https://github.com/rust-lang/crates.io-index#rand@0.10.1", + "registry+https://github.com/rust-lang/crates.io-index#reqwest@0.13.2", + "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", + "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.149", + "registry+https://github.com/rust-lang/crates.io-index#shellexpand@3.1.2", + "registry+https://github.com/rust-lang/crates.io-index#sysinfo@0.38.4", + "registry+https://github.com/rust-lang/crates.io-index#thiserror@2.0.18", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "registry+https://github.com/rust-lang/crates.io-index#tokio-util@0.7.18", + "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44", + "registry+https://github.com/rust-lang/crates.io-index#tracing-appender@0.2.4", + "registry+https://github.com/rust-lang/crates.io-index#tracing-subscriber@0.3.23", + "registry+https://github.com/rust-lang/crates.io-index#whoami@2.1.1" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#aho-corasick@1.1.4", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#anstream@1.0.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#anstyle@1.0.14", + "registry+https://github.com/rust-lang/crates.io-index#anstyle-parse@1.0.0", + "registry+https://github.com/rust-lang/crates.io-index#anstyle-query@1.1.5", + "registry+https://github.com/rust-lang/crates.io-index#colorchoice@1.0.5", + "registry+https://github.com/rust-lang/crates.io-index#is_terminal_polyfill@1.70.2", + "registry+https://github.com/rust-lang/crates.io-index#utf8parse@0.2.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#anstyle-parse@1.0.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#utf8parse@0.2.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#anstyle-query@1.1.5" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#anstyle@1.0.14" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#anyhow@1.0.102" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#approx@0.5.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#arrayref@0.3.9" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#arrayvec@0.7.6" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#async-trait@0.1.89", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#atomic-waker@1.1.2" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#autocfg@1.5.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#aws-lc-rs@1.16.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#aws-lc-sys@0.39.0", + "registry+https://github.com/rust-lang/crates.io-index#zeroize@1.8.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#aws-lc-sys@0.39.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cc@1.2.57", + "registry+https://github.com/rust-lang/crates.io-index#cmake@0.1.57", + "registry+https://github.com/rust-lang/crates.io-index#dunce@1.0.5", + "registry+https://github.com/rust-lang/crates.io-index#fs_extra@1.3.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#base64@0.22.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#bitflags@2.11.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#blake3@1.8.3", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#arrayref@0.3.9", + "registry+https://github.com/rust-lang/crates.io-index#arrayvec@0.7.6", + "registry+https://github.com/rust-lang/crates.io-index#cc@1.2.57", + "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.4", + "registry+https://github.com/rust-lang/crates.io-index#constant_time_eq@0.4.2", + "registry+https://github.com/rust-lang/crates.io-index#cpufeatures@0.2.17" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#block-buffer@0.10.4", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#generic-array@0.14.7" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#bstr@1.12.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0", + "registry+https://github.com/rust-lang/crates.io-index#regex-automata@0.4.14", + "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#bytemuck@1.25.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#cc@1.2.57", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#find-msvc-tools@0.1.9", + "registry+https://github.com/rust-lang/crates.io-index#jobserver@0.1.34", + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#shlex@1.3.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#cfg-if@0.1.10" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.4" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#cfg_aliases@0.2.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#chacha20@0.10.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.4", + "registry+https://github.com/rust-lang/crates.io-index#cpufeatures@0.3.0", + "registry+https://github.com/rust-lang/crates.io-index#rand_core@0.10.1" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#chrono@0.4.44", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#iana-time-zone@0.1.65", + "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#clap@4.6.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#clap_builder@4.6.0", + "registry+https://github.com/rust-lang/crates.io-index#clap_derive@4.6.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#clap_builder@4.6.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#anstream@1.0.0", + "registry+https://github.com/rust-lang/crates.io-index#anstyle@1.0.14", + "registry+https://github.com/rust-lang/crates.io-index#clap_lex@1.1.0", + "registry+https://github.com/rust-lang/crates.io-index#strsim@0.11.1" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#clap_derive@4.6.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#heck@0.5.0", + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#clap_lex@1.1.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#cmake@0.1.57", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cc@1.2.57" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#colorchoice@1.0.5" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#colored@3.1.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#const-str@1.1.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#const_panic@0.2.15", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#typewit@1.14.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#constant_time_eq@0.4.2" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#countio@0.3.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#futures-io@0.3.32" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#cpufeatures@0.2.17" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#cpufeatures@0.3.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#crc32fast@1.5.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.4" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#crossbeam-channel@0.5.15", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#crossbeam-utils@0.8.21" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#crossbeam-utils@0.8.21" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#crypto-common@0.1.7", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#generic-array@0.14.7", + "registry+https://github.com/rust-lang/crates.io-index#typenum@1.19.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#csv-core@0.1.13", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#csv@1.4.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#csv-core@0.1.13", + "registry+https://github.com/rust-lang/crates.io-index#itoa@1.0.18", + "registry+https://github.com/rust-lang/crates.io-index#ryu@1.0.23", + "registry+https://github.com/rust-lang/crates.io-index#serde_core@1.0.228" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#ctor-proc-macro@0.0.7" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#ctor@0.6.3", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#ctor-proc-macro@0.0.7", + "registry+https://github.com/rust-lang/crates.io-index#dtor@0.1.1" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#deranged@0.5.8", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#powerfmt@0.2.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#digest@0.10.7", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#block-buffer@0.10.4", + "registry+https://github.com/rust-lang/crates.io-index#crypto-common@0.1.7" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#dirs-sys@0.5.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#option-ext@0.2.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#dirs@6.0.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#dirs-sys@0.5.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#dtor-proc-macro@0.0.6" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#dtor@0.1.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#dtor-proc-macro@0.0.6" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#dunce@1.0.5" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#either@1.15.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#errno@0.3.14", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#fastrand@2.3.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#find-msvc-tools@0.1.9" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#form_urlencoded@1.2.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#percent-encoding@2.3.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#fs_extra@1.3.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#futures-channel@0.3.32", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-sink@0.3.32" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#futures-executor@0.3.32", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-task@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-util@0.3.32" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#futures-io@0.3.32" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#futures-macro@0.3.32", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#futures-sink@0.3.32" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#futures-task@0.3.32" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#futures-util@0.3.32", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#futures-channel@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-io@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-macro@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-sink@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-task@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0", + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#slab@0.4.12" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#futures@0.3.32", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#futures-channel@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-executor@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-io@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-sink@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-task@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-util@0.3.32" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#gearhash@0.1.3", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cfg-if@0.1.10" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#generic-array@0.14.7", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#typenum@1.19.0", + "registry+https://github.com/rust-lang/crates.io-index#version_check@0.9.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.2.17", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.4", + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.3.4", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.4", + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.4.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.4", + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#rand_core@0.10.1" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#git-version-macro@0.3.9", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#git-version@0.3.9", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#git-version-macro@0.3.9" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#heapify@0.2.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#heck@0.5.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#http-body-util@0.1.3", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#http-body@1.0.1", + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#http-body@1.0.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#itoa@1.0.18" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#httparse@1.10.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#humantime@2.3.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#hyper-rustls@0.27.7", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#hyper@1.8.1", + "registry+https://github.com/rust-lang/crates.io-index#hyper-util@0.1.20", + "registry+https://github.com/rust-lang/crates.io-index#rustls@0.23.37", + "registry+https://github.com/rust-lang/crates.io-index#rustls-pki-types@1.14.0", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "registry+https://github.com/rust-lang/crates.io-index#tokio-rustls@0.26.4", + "registry+https://github.com/rust-lang/crates.io-index#tower-service@0.3.3" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#hyper-util@0.1.20", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#base64@0.22.1", + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#futures-channel@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-util@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#http-body@1.0.1", + "registry+https://github.com/rust-lang/crates.io-index#hyper@1.8.1", + "registry+https://github.com/rust-lang/crates.io-index#ipnet@2.12.0", + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#percent-encoding@2.3.2", + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#socket2@0.6.3", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "registry+https://github.com/rust-lang/crates.io-index#tower-service@0.3.3", + "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#hyper@1.8.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#atomic-waker@1.1.2", + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#futures-channel@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#http-body@1.0.1", + "registry+https://github.com/rust-lang/crates.io-index#httparse@1.10.1", + "registry+https://github.com/rust-lang/crates.io-index#itoa@1.0.18", + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#pin-utils@0.1.0", + "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "registry+https://github.com/rust-lang/crates.io-index#want@0.3.1" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#iana-time-zone@0.1.65" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_collections@2.1.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", + "registry+https://github.com/rust-lang/crates.io-index#potential_utf@0.1.4", + "registry+https://github.com/rust-lang/crates.io-index#yoke@0.8.1", + "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.6", + "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.11.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_locale_core@2.1.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", + "registry+https://github.com/rust-lang/crates.io-index#litemap@0.8.1", + "registry+https://github.com/rust-lang/crates.io-index#tinystr@0.8.2", + "registry+https://github.com/rust-lang/crates.io-index#writeable@0.6.2", + "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.11.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer@2.1.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#icu_collections@2.1.1", + "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer_data@2.1.1", + "registry+https://github.com/rust-lang/crates.io-index#icu_properties@2.1.2", + "registry+https://github.com/rust-lang/crates.io-index#icu_provider@2.1.1", + "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1", + "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.11.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer_data@2.1.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_properties@2.1.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#icu_collections@2.1.1", + "registry+https://github.com/rust-lang/crates.io-index#icu_locale_core@2.1.1", + "registry+https://github.com/rust-lang/crates.io-index#icu_properties_data@2.1.2", + "registry+https://github.com/rust-lang/crates.io-index#icu_provider@2.1.1", + "registry+https://github.com/rust-lang/crates.io-index#zerotrie@0.2.3", + "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.11.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_properties_data@2.1.2" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_provider@2.1.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", + "registry+https://github.com/rust-lang/crates.io-index#icu_locale_core@2.1.1", + "registry+https://github.com/rust-lang/crates.io-index#writeable@0.6.2", + "registry+https://github.com/rust-lang/crates.io-index#yoke@0.8.1", + "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.6", + "registry+https://github.com/rust-lang/crates.io-index#zerotrie@0.2.3", + "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.11.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#idna@1.1.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#idna_adapter@1.2.1", + "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1", + "registry+https://github.com/rust-lang/crates.io-index#utf8_iter@1.0.4" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#idna_adapter@1.2.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer@2.1.1", + "registry+https://github.com/rust-lang/crates.io-index#icu_properties@2.1.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#indoc@2.0.7", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#rustversion@1.0.22" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#inventory@0.3.22" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#ipnet@2.12.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#iri-string@0.7.11", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0", + "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#is_terminal_polyfill@1.70.2" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#itertools@0.14.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#either@1.15.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#itoa@1.0.18" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#jobserver@0.1.34", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#konst@0.4.3", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#const_panic@0.2.15", + "registry+https://github.com/rust-lang/crates.io-index#konst_proc_macros@0.4.1", + "registry+https://github.com/rust-lang/crates.io-index#typewit@1.14.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#konst_proc_macros@0.4.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#lazy_static@1.5.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#linux-raw-sys@0.12.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#litemap@0.8.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#log@0.4.29" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#lru-slab@0.1.2" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#lz4_flex@0.13.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#twox-hash@2.1.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#matchers@0.2.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#regex-automata@0.4.14" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#memoffset@0.9.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#autocfg@1.5.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#mio@1.1.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#more-asserts@0.3.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#nu-ansi-term@0.50.3" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#num-conv@0.2.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#autocfg@1.5.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.4" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#oneshot@0.1.13" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#openssl-probe@0.2.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#option-ext@0.2.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#os_str_bytes@6.6.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#percent-encoding@2.3.2" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#pin-project-internal@1.1.11", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#pin-project@1.1.11", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#pin-project-internal@1.1.11" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#pin-utils@0.1.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#potential_utf@0.1.4", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.11.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#powerfmt@0.2.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#ppv-lite86@0.2.21", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#zerocopy@0.8.47" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#unicode-ident@1.0.24" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.26.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#target-lexicon@0.13.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-ffi@0.26.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.26.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros-backend@0.26.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#heck@0.5.0", + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.26.0", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros@0.26.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros-backend@0.26.0", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3@0.26.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#indoc@2.0.7", + "registry+https://github.com/rust-lang/crates.io-index#inventory@0.3.22", + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#memoffset@0.9.1", + "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.4", + "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.26.0", + "registry+https://github.com/rust-lang/crates.io-index#pyo3-ffi@0.26.0", + "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros@0.26.0", + "registry+https://github.com/rust-lang/crates.io-index#unindent@0.2.4" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#quinn-proto@0.11.14", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#aws-lc-rs@1.16.2", + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#lru-slab@0.1.2", + "registry+https://github.com/rust-lang/crates.io-index#rand@0.9.2", + "registry+https://github.com/rust-lang/crates.io-index#rustc-hash@2.1.1", + "registry+https://github.com/rust-lang/crates.io-index#rustls@0.23.37", + "registry+https://github.com/rust-lang/crates.io-index#slab@0.4.12", + "registry+https://github.com/rust-lang/crates.io-index#thiserror@2.0.18", + "registry+https://github.com/rust-lang/crates.io-index#tinyvec@1.11.0", + "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#quinn-udp@0.5.14", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cfg_aliases@0.2.1", + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#socket2@0.6.3", + "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#quinn@0.11.9", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#cfg_aliases@0.2.1", + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#quinn-proto@0.11.14", + "registry+https://github.com/rust-lang/crates.io-index#quinn-udp@0.5.14", + "registry+https://github.com/rust-lang/crates.io-index#rustc-hash@2.1.1", + "registry+https://github.com/rust-lang/crates.io-index#rustls@0.23.37", + "registry+https://github.com/rust-lang/crates.io-index#socket2@0.6.3", + "registry+https://github.com/rust-lang/crates.io-index#thiserror@2.0.18", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rand@0.10.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#chacha20@0.10.0", + "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.4.2", + "registry+https://github.com/rust-lang/crates.io-index#rand_core@0.10.1" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rand@0.9.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#rand_chacha@0.9.0", + "registry+https://github.com/rust-lang/crates.io-index#rand_core@0.9.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rand_chacha@0.9.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#ppv-lite86@0.2.21", + "registry+https://github.com/rust-lang/crates.io-index#rand_core@0.9.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rand_core@0.10.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rand_core@0.9.5", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.3.4" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#redb@3.1.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#regex-automata@0.4.14", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#aho-corasick@1.1.4", + "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0", + "registry+https://github.com/rust-lang/crates.io-index#regex-syntax@0.8.10" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#regex-syntax@0.8.10" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#regex@1.12.3", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#aho-corasick@1.1.4", + "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0", + "registry+https://github.com/rust-lang/crates.io-index#regex-automata@0.4.14", + "registry+https://github.com/rust-lang/crates.io-index#regex-syntax@0.8.10" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#reqwest-middleware@0.5.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#anyhow@1.0.102", + "registry+https://github.com/rust-lang/crates.io-index#async-trait@0.1.89", + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#reqwest@0.13.2", + "registry+https://github.com/rust-lang/crates.io-index#thiserror@2.0.18", + "registry+https://github.com/rust-lang/crates.io-index#tower-service@0.3.3" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#reqwest@0.13.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#base64@0.22.1", + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-util@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#http-body@1.0.1", + "registry+https://github.com/rust-lang/crates.io-index#http-body-util@0.1.3", + "registry+https://github.com/rust-lang/crates.io-index#hyper@1.8.1", + "registry+https://github.com/rust-lang/crates.io-index#hyper-rustls@0.27.7", + "registry+https://github.com/rust-lang/crates.io-index#hyper-util@0.1.20", + "registry+https://github.com/rust-lang/crates.io-index#log@0.4.29", + "registry+https://github.com/rust-lang/crates.io-index#percent-encoding@2.3.2", + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#quinn@0.11.9", + "registry+https://github.com/rust-lang/crates.io-index#rustls@0.23.37", + "registry+https://github.com/rust-lang/crates.io-index#rustls-pki-types@1.14.0", + "registry+https://github.com/rust-lang/crates.io-index#rustls-platform-verifier@0.6.2", + "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", + "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.149", + "registry+https://github.com/rust-lang/crates.io-index#sync_wrapper@1.0.2", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "registry+https://github.com/rust-lang/crates.io-index#tokio-rustls@0.26.4", + "registry+https://github.com/rust-lang/crates.io-index#tokio-util@0.7.18", + "registry+https://github.com/rust-lang/crates.io-index#tower@0.5.3", + "registry+https://github.com/rust-lang/crates.io-index#tower-http@0.6.8", + "registry+https://github.com/rust-lang/crates.io-index#tower-service@0.3.3", + "registry+https://github.com/rust-lang/crates.io-index#url@2.5.8" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#ring@0.17.14", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cc@1.2.57", + "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.4", + "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#untrusted@0.9.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rustc-hash@2.1.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rustix@1.1.4", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#bitflags@2.11.0", + "registry+https://github.com/rust-lang/crates.io-index#errno@0.3.14", + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#linux-raw-sys@0.12.1" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rustls-native-certs@0.8.3", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#openssl-probe@0.2.1", + "registry+https://github.com/rust-lang/crates.io-index#rustls-pki-types@1.14.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rustls-pki-types@1.14.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#zeroize@1.8.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rustls-platform-verifier@0.6.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#log@0.4.29", + "registry+https://github.com/rust-lang/crates.io-index#rustls@0.23.37", + "registry+https://github.com/rust-lang/crates.io-index#rustls-native-certs@0.8.3", + "registry+https://github.com/rust-lang/crates.io-index#rustls-webpki@0.103.13" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rustls-webpki@0.103.13", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#aws-lc-rs@1.16.2", + "registry+https://github.com/rust-lang/crates.io-index#ring@0.17.14", + "registry+https://github.com/rust-lang/crates.io-index#rustls-pki-types@1.14.0", + "registry+https://github.com/rust-lang/crates.io-index#untrusted@0.9.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rustls@0.23.37", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#aws-lc-rs@1.16.2", + "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.4", + "registry+https://github.com/rust-lang/crates.io-index#rustls-pki-types@1.14.0", + "registry+https://github.com/rust-lang/crates.io-index#rustls-webpki@0.103.13", + "registry+https://github.com/rust-lang/crates.io-index#subtle@2.6.1", + "registry+https://github.com/rust-lang/crates.io-index#zeroize@1.8.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#rustversion@1.0.22" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#ryu@1.0.23" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#safe-transmute@0.11.3" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#same-file@1.0.6" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#serde_core@1.0.228", + "registry+https://github.com/rust-lang/crates.io-index#serde_derive@1.0.228" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#serde_core@1.0.228" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#serde_derive@1.0.228", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.149", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#itoa@1.0.18", + "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0", + "registry+https://github.com/rust-lang/crates.io-index#serde_core@1.0.228", + "registry+https://github.com/rust-lang/crates.io-index#zmij@1.0.21" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#serde_repr@0.1.20", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#sha2-asm@0.6.4", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cc@1.2.57" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#sha2@0.10.9", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.4", + "registry+https://github.com/rust-lang/crates.io-index#cpufeatures@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#digest@0.10.7", + "registry+https://github.com/rust-lang/crates.io-index#sha2-asm@0.6.4" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#sharded-slab@0.1.7", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#lazy_static@1.5.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#shellexpand@3.1.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#bstr@1.12.1", + "registry+https://github.com/rust-lang/crates.io-index#dirs@6.0.0", + "registry+https://github.com/rust-lang/crates.io-index#os_str_bytes@6.6.1" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#shlex@1.3.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#signal-hook-registry@1.4.8", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#errno@0.3.14", + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#signal-hook@0.3.18", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#signal-hook-registry@1.4.8" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#slab@0.4.12" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#socket2@0.6.3", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#stable_deref_trait@1.2.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#static_assertions@1.1.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#statrs@0.18.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#approx@0.5.1", + "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#strsim@0.11.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#subtle@2.6.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#unicode-ident@1.0.24" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#sync_wrapper@1.0.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#synstructure@0.13.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#sysinfo@0.38.4", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#memchr@2.8.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#target-lexicon@0.13.5" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tempfile@3.27.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#fastrand@2.3.0", + "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.4.2", + "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.4", + "registry+https://github.com/rust-lang/crates.io-index#rustix@1.1.4" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#thiserror-impl@2.0.18", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#thiserror@2.0.18", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#thiserror-impl@2.0.18" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#thread_local@1.1.9", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.4" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#time-core@0.1.8" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#time-macros@0.2.27", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#num-conv@0.2.0", + "registry+https://github.com/rust-lang/crates.io-index#time-core@0.1.8" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#time@0.3.47", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#deranged@0.5.8", + "registry+https://github.com/rust-lang/crates.io-index#itoa@1.0.18", + "registry+https://github.com/rust-lang/crates.io-index#num-conv@0.2.0", + "registry+https://github.com/rust-lang/crates.io-index#powerfmt@0.2.0", + "registry+https://github.com/rust-lang/crates.io-index#serde_core@1.0.228", + "registry+https://github.com/rust-lang/crates.io-index#time-core@0.1.8", + "registry+https://github.com/rust-lang/crates.io-index#time-macros@0.2.27" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tinystr@0.8.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", + "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.11.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tinyvec@1.11.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#tinyvec_macros@0.1.1" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tinyvec_macros@0.1.1" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tokio-macros@2.6.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tokio-retry@0.3.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#rand@0.10.1", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tokio-rustls@0.26.4", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#rustls@0.23.37", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tokio-util@0.7.18", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-sink@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183", + "registry+https://github.com/rust-lang/crates.io-index#mio@1.1.1", + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#socket2@0.6.3", + "registry+https://github.com/rust-lang/crates.io-index#tokio-macros@2.6.1" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tower-http@0.6.8", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#bitflags@2.11.0", + "registry+https://github.com/rust-lang/crates.io-index#bytes@1.11.1", + "registry+https://github.com/rust-lang/crates.io-index#futures-util@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#http@1.4.0", + "registry+https://github.com/rust-lang/crates.io-index#http-body@1.0.1", + "registry+https://github.com/rust-lang/crates.io-index#iri-string@0.7.11", + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#tower@0.5.3", + "registry+https://github.com/rust-lang/crates.io-index#tower-layer@0.3.3", + "registry+https://github.com/rust-lang/crates.io-index#tower-service@0.3.3" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tower-layer@0.3.3" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tower-service@0.3.3" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tower@0.5.3", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#futures-core@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#futures-util@0.3.32", + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#sync_wrapper@1.0.2", + "registry+https://github.com/rust-lang/crates.io-index#tokio@1.50.0", + "registry+https://github.com/rust-lang/crates.io-index#tower-layer@0.3.3", + "registry+https://github.com/rust-lang/crates.io-index#tower-service@0.3.3" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-appender@0.2.4", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#crossbeam-channel@0.5.15", + "registry+https://github.com/rust-lang/crates.io-index#thiserror@2.0.18", + "registry+https://github.com/rust-lang/crates.io-index#time@0.3.47", + "registry+https://github.com/rust-lang/crates.io-index#tracing-subscriber@0.3.23" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-attributes@0.1.31", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-core@0.1.36", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.4" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-log@0.2.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#log@0.4.29", + "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.4", + "registry+https://github.com/rust-lang/crates.io-index#tracing-core@0.1.36" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-serde@0.2.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", + "registry+https://github.com/rust-lang/crates.io-index#tracing-core@0.1.36" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tracing-subscriber@0.3.23", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#matchers@0.2.0", + "registry+https://github.com/rust-lang/crates.io-index#nu-ansi-term@0.50.3", + "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.4", + "registry+https://github.com/rust-lang/crates.io-index#regex-automata@0.4.14", + "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", + "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.149", + "registry+https://github.com/rust-lang/crates.io-index#sharded-slab@0.1.7", + "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1", + "registry+https://github.com/rust-lang/crates.io-index#thread_local@1.1.9", + "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44", + "registry+https://github.com/rust-lang/crates.io-index#tracing-core@0.1.36", + "registry+https://github.com/rust-lang/crates.io-index#tracing-log@0.2.0", + "registry+https://github.com/rust-lang/crates.io-index#tracing-serde@0.2.0" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#tracing@0.1.44", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#pin-project-lite@0.2.17", + "registry+https://github.com/rust-lang/crates.io-index#tracing-attributes@0.1.31", + "registry+https://github.com/rust-lang/crates.io-index#tracing-core@0.1.36" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#try-lock@0.2.5" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#twox-hash@2.1.2" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#typenum@1.19.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#typewit@1.14.2" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#unicode-ident@1.0.24" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#unindent@0.2.4" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#untrusted@0.9.0" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#url@2.5.8", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#form_urlencoded@1.2.2", + "registry+https://github.com/rust-lang/crates.io-index#idna@1.1.0", + "registry+https://github.com/rust-lang/crates.io-index#percent-encoding@2.3.2", + "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#urlencoding@2.1.3" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#utf8_iter@1.0.4" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#utf8parse@0.2.2" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#uuid@1.22.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.4.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#version_check@0.9.5" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#walkdir@2.5.0", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#same-file@1.0.6" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#want@0.3.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#try-lock@0.2.5" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#whoami@2.1.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.183" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#writeable@0.6.2" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#yoke-derive@0.8.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117", + "registry+https://github.com/rust-lang/crates.io-index#synstructure@0.13.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#yoke@0.8.1", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#stable_deref_trait@1.2.1", + "registry+https://github.com/rust-lang/crates.io-index#yoke-derive@0.8.1", + "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.6" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#zerocopy@0.8.47" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#zerofrom-derive@0.1.6", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117", + "registry+https://github.com/rust-lang/crates.io-index#synstructure@0.13.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.6", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#zerofrom-derive@0.1.6" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#zeroize@1.8.2" + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#zerotrie@0.2.3", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", + "registry+https://github.com/rust-lang/crates.io-index#yoke@0.8.1", + "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.6" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#zerovec-derive@0.11.2", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.106", + "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.45", + "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.117" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.11.5", + "dependsOn": [ + "registry+https://github.com/rust-lang/crates.io-index#yoke@0.8.1", + "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.6", + "registry+https://github.com/rust-lang/crates.io-index#zerovec-derive@0.11.2" + ] + }, + { + "ref": "registry+https://github.com/rust-lang/crates.io-index#zmij@1.0.21" + } + ] +} \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/INSTALLER b/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/METADATA b/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..8056834e6714b089ef49847820064a1ae4b041fd --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/METADATA @@ -0,0 +1,625 @@ +Metadata-Version: 2.4 +Name: httpcore +Version: 1.0.9 +Summary: A minimal low-level HTTP client. +Project-URL: Documentation, https://www.encode.io/httpcore +Project-URL: Homepage, https://www.encode.io/httpcore/ +Project-URL: Source, https://github.com/encode/httpcore +Author-email: Tom Christie +License-Expression: BSD-3-Clause +License-File: LICENSE.md +Classifier: Development Status :: 3 - Alpha +Classifier: Environment :: Web Environment +Classifier: Framework :: AsyncIO +Classifier: Framework :: Trio +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Topic :: Internet :: WWW/HTTP +Requires-Python: >=3.8 +Requires-Dist: certifi +Requires-Dist: h11>=0.16 +Provides-Extra: asyncio +Requires-Dist: anyio<5.0,>=4.0; extra == 'asyncio' +Provides-Extra: http2 +Requires-Dist: h2<5,>=3; extra == 'http2' +Provides-Extra: socks +Requires-Dist: socksio==1.*; extra == 'socks' +Provides-Extra: trio +Requires-Dist: trio<1.0,>=0.22.0; extra == 'trio' +Description-Content-Type: text/markdown + +# HTTP Core + +[![Test Suite](https://github.com/encode/httpcore/workflows/Test%20Suite/badge.svg)](https://github.com/encode/httpcore/actions) +[![Package version](https://badge.fury.io/py/httpcore.svg)](https://pypi.org/project/httpcore/) + +> *Do one thing, and do it well.* + +The HTTP Core package provides a minimal low-level HTTP client, which does +one thing only. Sending HTTP requests. + +It does not provide any high level model abstractions over the API, +does not handle redirects, multipart uploads, building authentication headers, +transparent HTTP caching, URL parsing, session cookie handling, +content or charset decoding, handling JSON, environment based configuration +defaults, or any of that Jazz. + +Some things HTTP Core does do: + +* Sending HTTP requests. +* Thread-safe / task-safe connection pooling. +* HTTP(S) proxy & SOCKS proxy support. +* Supports HTTP/1.1 and HTTP/2. +* Provides both sync and async interfaces. +* Async backend support for `asyncio` and `trio`. + +## Requirements + +Python 3.8+ + +## Installation + +For HTTP/1.1 only support, install with: + +```shell +$ pip install httpcore +``` + +There are also a number of optional extras available... + +```shell +$ pip install httpcore['asyncio,trio,http2,socks'] +``` + +## Sending requests + +Send an HTTP request: + +```python +import httpcore + +response = httpcore.request("GET", "https://www.example.com/") + +print(response) +# +print(response.status) +# 200 +print(response.headers) +# [(b'Accept-Ranges', b'bytes'), (b'Age', b'557328'), (b'Cache-Control', b'max-age=604800'), ...] +print(response.content) +# b'\n\n\nExample Domain\n\n\n ...' +``` + +The top-level `httpcore.request()` function is provided for convenience. In practice whenever you're working with `httpcore` you'll want to use the connection pooling functionality that it provides. + +```python +import httpcore + +http = httpcore.ConnectionPool() +response = http.request("GET", "https://www.example.com/") +``` + +Once you're ready to get going, [head over to the documentation](https://www.encode.io/httpcore/). + +## Motivation + +You *probably* don't want to be using HTTP Core directly. It might make sense if +you're writing something like a proxy service in Python, and you just want +something at the lowest possible level, but more typically you'll want to use +a higher level client library, such as `httpx`. + +The motivation for `httpcore` is: + +* To provide a reusable low-level client library, that other packages can then build on top of. +* To provide a *really clear interface split* between the networking code and client logic, + so that each is easier to understand and reason about in isolation. + +## Dependencies + +The `httpcore` package has the following dependencies... + +* `h11` +* `certifi` + +And the following optional extras... + +* `anyio` - Required by `pip install httpcore['asyncio']`. +* `trio` - Required by `pip install httpcore['trio']`. +* `h2` - Required by `pip install httpcore['http2']`. +* `socksio` - Required by `pip install httpcore['socks']`. + +## Versioning + +We use [SEMVER for our versioning policy](https://semver.org/). + +For changes between package versions please see our [project changelog](CHANGELOG.md). + +We recommend pinning your requirements either the most current major version, or a more specific version range: + +```python +pip install 'httpcore==1.*' +``` +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## Version 1.0.9 (April 24th, 2025) + +- Resolve https://github.com/advisories/GHSA-vqfr-h8mv-ghfj with h11 dependency update. (#1008) + +## Version 1.0.8 (April 11th, 2025) + +- Fix `AttributeError` when importing on Python 3.14. (#1005) + +## Version 1.0.7 (November 15th, 2024) + +- Support `proxy=…` configuration on `ConnectionPool()`. (#974) + +## Version 1.0.6 (October 1st, 2024) + +- Relax `trio` dependency pinning. (#956) +- Handle `trio` raising `NotImplementedError` on unsupported platforms. (#955) +- Handle mapping `ssl.SSLError` to `httpcore.ConnectError`. (#918) + +## 1.0.5 (March 27th, 2024) + +- Handle `EndOfStream` exception for anyio backend. (#899) +- Allow trio `0.25.*` series in package dependancies. (#903) + +## 1.0.4 (February 21st, 2024) + +- Add `target` request extension. (#888) +- Fix support for connection `Upgrade` and `CONNECT` when some data in the stream has been read. (#882) + +## 1.0.3 (February 13th, 2024) + +- Fix support for async cancellations. (#880) +- Fix trace extension when used with socks proxy. (#849) +- Fix SSL context for connections using the "wss" scheme (#869) + +## 1.0.2 (November 10th, 2023) + +- Fix `float("inf")` timeouts in `Event.wait` function. (#846) + +## 1.0.1 (November 3rd, 2023) + +- Fix pool timeout to account for the total time spent retrying. (#823) +- Raise a neater RuntimeError when the correct async deps are not installed. (#826) +- Add support for synchronous TLS-in-TLS streams. (#840) + +## 1.0.0 (October 6th, 2023) + +From version 1.0 our async support is now optional, as the package has minimal dependencies by default. + +For async support use either `pip install 'httpcore[asyncio]'` or `pip install 'httpcore[trio]'`. + +The project versioning policy is now explicitly governed by SEMVER. See https://semver.org/. + +- Async support becomes fully optional. (#809) +- Add support for Python 3.12. (#807) + +## 0.18.0 (September 8th, 2023) + +- Add support for HTTPS proxies. (#745, #786) +- Drop Python 3.7 support. (#727) +- Handle `sni_hostname` extension with SOCKS proxy. (#774) +- Handle HTTP/1.1 half-closed connections gracefully. (#641) +- Change the type of `Extensions` from `Mapping[Str, Any]` to `MutableMapping[Str, Any]`. (#762) + +## 0.17.3 (July 5th, 2023) + +- Support async cancellations, ensuring that the connection pool is left in a clean state when cancellations occur. (#726) +- The networking backend interface has [been added to the public API](https://www.encode.io/httpcore/network-backends). Some classes which were previously private implementation detail are now part of the top-level public API. (#699) +- Graceful handling of HTTP/2 GoAway frames, with requests being transparently retried on a new connection. (#730) +- Add exceptions when a synchronous `trace callback` is passed to an asynchronous request or an asynchronous `trace callback` is passed to a synchronous request. (#717) +- Drop Python 3.7 support. (#727) + +## 0.17.2 (May 23th, 2023) + +- Add `socket_options` argument to `ConnectionPool` and `HTTProxy` classes. (#668) +- Improve logging with per-module logger names. (#690) +- Add `sni_hostname` request extension. (#696) +- Resolve race condition during import of `anyio` package. (#692) +- Enable TCP_NODELAY for all synchronous sockets. (#651) + +## 0.17.1 (May 17th, 2023) + +- If 'retries' is set, then allow retries if an SSL handshake error occurs. (#669) +- Improve correctness of tracebacks on network exceptions, by raising properly chained exceptions. (#678) +- Prevent connection-hanging behaviour when HTTP/2 connections are closed by a server-sent 'GoAway' frame. (#679) +- Fix edge-case exception when removing requests from the connection pool. (#680) +- Fix pool timeout edge-case. (#688) + +## 0.17.0 (March 16th, 2023) + +- Add DEBUG level logging. (#648) +- Respect HTTP/2 max concurrent streams when settings updates are sent by server. (#652) +- Increase the allowable HTTP header size to 100kB. (#647) +- Add `retries` option to SOCKS proxy classes. (#643) + +## 0.16.3 (December 20th, 2022) + +- Allow `ws` and `wss` schemes. Allows us to properly support websocket upgrade connections. (#625) +- Forwarding HTTP proxies use a connection-per-remote-host. Required by some proxy implementations. (#637) +- Don't raise `RuntimeError` when closing a connection pool with active connections. Removes some error cases when cancellations are used. (#631) +- Lazy import `anyio`, so that it's no longer a hard dependancy, and isn't imported if unused. (#639) + +## 0.16.2 (November 25th, 2022) + +- Revert 'Fix async cancellation behaviour', which introduced race conditions. (#627) +- Raise `RuntimeError` if attempting to us UNIX domain sockets on Windows. (#619) + +## 0.16.1 (November 17th, 2022) + +- Fix HTTP/1.1 interim informational responses, such as "100 Continue". (#605) + +## 0.16.0 (October 11th, 2022) + +- Support HTTP/1.1 informational responses. (#581) +- Fix async cancellation behaviour. (#580) +- Support `h11` 0.14. (#579) + +## 0.15.0 (May 17th, 2022) + +- Drop Python 3.6 support (#535) +- Ensure HTTP proxy CONNECT requests include `timeout` configuration. (#506) +- Switch to explicit `typing.Optional` for type hints. (#513) +- For `trio` map OSError exceptions to `ConnectError`. (#543) + +## 0.14.7 (February 4th, 2022) + +- Requests which raise a PoolTimeout need to be removed from the pool queue. (#502) +- Fix AttributeError that happened when Socks5Connection were terminated. (#501) + +## 0.14.6 (February 1st, 2022) + +- Fix SOCKS support for `http://` URLs. (#492) +- Resolve race condition around exceptions during streaming a response. (#491) + +## 0.14.5 (January 18th, 2022) + +- SOCKS proxy support. (#478) +- Add proxy_auth argument to HTTPProxy. (#481) +- Improve error message on 'RemoteProtocolError' exception when server disconnects without sending a response. (#479) + +## 0.14.4 (January 5th, 2022) + +- Support HTTP/2 on HTTPS tunnelling proxies. (#468) +- Fix proxy headers missing on HTTP forwarding. (#456) +- Only instantiate SSL context if required. (#457) +- More robust HTTP/2 handling. (#253, #439, #440, #441) + +## 0.14.3 (November 17th, 2021) + +- Fix race condition when removing closed connections from the pool. (#437) + +## 0.14.2 (November 16th, 2021) + +- Failed connections no longer remain in the pool. (Pull #433) + +## 0.14.1 (November 12th, 2021) + +- `max_connections` becomes optional. (Pull #429) +- `certifi` is now included in the install dependancies. (Pull #428) +- `h2` is now strictly optional. (Pull #428) + +## 0.14.0 (November 11th, 2021) + +The 0.14 release is a complete reworking of `httpcore`, comprehensively addressing some underlying issues in the connection pooling, as well as substantially redesigning the API to be more user friendly. + +Some of the lower-level API design also makes the components more easily testable in isolation, and the package now has 100% test coverage. + +See [discussion #419](https://github.com/encode/httpcore/discussions/419) for a little more background. + +There's some other neat bits in there too, such as the "trace" extension, which gives a hook into inspecting the internal events that occur during the request/response cycle. This extension is needed for the HTTPX cli, in order to... + +* Log the point at which the connection is established, and the IP/port on which it is made. +* Determine if the outgoing request should log as HTTP/1.1 or HTTP/2, rather than having to assume it's HTTP/2 if the --http2 flag was passed. (Which may not actually be true.) +* Log SSL version info / certificate info. + +Note that `curio` support is not currently available in 0.14.0. If you're using `httpcore` with `curio` please get in touch, so we can assess if we ought to prioritize it as a feature or not. + +## 0.13.7 (September 13th, 2021) + +- Fix broken error messaging when URL scheme is missing, or a non HTTP(S) scheme is used. (Pull #403) + +## 0.13.6 (June 15th, 2021) + +### Fixed + +- Close sockets when read or write timeouts occur. (Pull #365) + +## 0.13.5 (June 14th, 2021) + +### Fixed + +- Resolved niggles with AnyIO EOF behaviours. (Pull #358, #362) + +## 0.13.4 (June 9th, 2021) + +### Added + +- Improved error messaging when URL scheme is missing, or a non HTTP(S) scheme is used. (Pull #354) + +### Fixed + +- Switched to `anyio` as the default backend implementation when running with `asyncio`. Resolves some awkward [TLS timeout issues](https://github.com/encode/httpx/discussions/1511). + +## 0.13.3 (May 6th, 2021) + +### Added + +- Support HTTP/2 prior knowledge, using `httpcore.SyncConnectionPool(http1=False)`. (Pull #333) + +### Fixed + +- Handle cases where environment does not provide `select.poll` support. (Pull #331) + +## 0.13.2 (April 29th, 2021) + +### Added + +- Improve error message for specific case of `RemoteProtocolError` where server disconnects without sending a response. (Pull #313) + +## 0.13.1 (April 28th, 2021) + +### Fixed + +- More resiliant testing for closed connections. (Pull #311) +- Don't raise exceptions on ungraceful connection closes. (Pull #310) + +## 0.13.0 (April 21st, 2021) + +The 0.13 release updates the core API in order to match the HTTPX Transport API, +introduced in HTTPX 0.18 onwards. + +An example of making requests with the new interface is: + +```python +with httpcore.SyncConnectionPool() as http: + status_code, headers, stream, extensions = http.handle_request( + method=b'GET', + url=(b'https', b'example.org', 443, b'/'), + headers=[(b'host', b'example.org'), (b'user-agent', b'httpcore')] + stream=httpcore.ByteStream(b''), + extensions={} + ) + body = stream.read() + print(status_code, body) +``` + +### Changed + +- The `.request()` method is now `handle_request()`. (Pull #296) +- The `.arequest()` method is now `.handle_async_request()`. (Pull #296) +- The `headers` argument is no longer optional. (Pull #296) +- The `stream` argument is no longer optional. (Pull #296) +- The `ext` argument is now named `extensions`, and is no longer optional. (Pull #296) +- The `"reason"` extension keyword is now named `"reason_phrase"`. (Pull #296) +- The `"reason_phrase"` and `"http_version"` extensions now use byte strings for their values. (Pull #296) +- The `httpcore.PlainByteStream()` class becomes `httpcore.ByteStream()`. (Pull #296) + +### Added + +- Streams now support a `.read()` interface. (Pull #296) + +### Fixed + +- Task cancellation no longer leaks connections from the connection pool. (Pull #305) + +## 0.12.3 (December 7th, 2020) + +### Fixed + +- Abort SSL connections on close rather than waiting for remote EOF when using `asyncio`. (Pull #167) +- Fix exception raised in case of connect timeouts when using the `anyio` backend. (Pull #236) +- Fix `Host` header precedence for `:authority` in HTTP/2. (Pull #241, #243) +- Handle extra edge case when detecting for socket readability when using `asyncio`. (Pull #242, #244) +- Fix `asyncio` SSL warning when using proxy tunneling. (Pull #249) + +## 0.12.2 (November 20th, 2020) + +### Fixed + +- Properly wrap connect errors on the asyncio backend. (Pull #235) +- Fix `ImportError` occurring on Python 3.9 when using the HTTP/1.1 sync client in a multithreaded context. (Pull #237) + +## 0.12.1 (November 7th, 2020) + +### Added + +- Add connect retries. (Pull #221) + +### Fixed + +- Tweak detection of dropped connections, resolving an issue with open files limits on Linux. (Pull #185) +- Avoid leaking connections when establishing an HTTP tunnel to a proxy has failed. (Pull #223) +- Properly wrap OS errors when using `trio`. (Pull #225) + +## 0.12.0 (October 6th, 2020) + +### Changed + +- HTTP header casing is now preserved, rather than always sent in lowercase. (#216 and python-hyper/h11#104) + +### Added + +- Add Python 3.9 to officially supported versions. + +### Fixed + +- Gracefully handle a stdlib asyncio bug when a connection is closed while it is in a paused-for-reading state. (#201) + +## 0.11.1 (September 28nd, 2020) + +### Fixed + +- Add await to async semaphore release() coroutine (#197) +- Drop incorrect curio classifier (#192) + +## 0.11.0 (September 22nd, 2020) + +The Transport API with 0.11.0 has a couple of significant changes. + +Firstly we've moved changed the request interface in order to allow extensions, which will later enable us to support features +such as trailing headers, HTTP/2 server push, and CONNECT/Upgrade connections. + +The interface changes from: + +```python +def request(method, url, headers, stream, timeout): + return (http_version, status_code, reason, headers, stream) +``` + +To instead including an optional dictionary of extensions on the request and response: + +```python +def request(method, url, headers, stream, ext): + return (status_code, headers, stream, ext) +``` + +Having an open-ended extensions point will allow us to add later support for various optional features, that wouldn't otherwise be supported without these API changes. + +In particular: + +* Trailing headers support. +* HTTP/2 Server Push +* sendfile. +* Exposing raw connection on CONNECT, Upgrade, HTTP/2 bi-di streaming. +* Exposing debug information out of the API, including template name, template context. + +Currently extensions are limited to: + +* request: `timeout` - Optional. Timeout dictionary. +* response: `http_version` - Optional. Include the HTTP version used on the response. +* response: `reason` - Optional. Include the reason phrase used on the response. Only valid with HTTP/1.*. + +See https://github.com/encode/httpx/issues/1274#issuecomment-694884553 for the history behind this. + +Secondly, the async version of `request` is now namespaced as `arequest`. + +This allows concrete transports to support both sync and async implementations on the same class. + +### Added + +- Add curio support. (Pull #168) +- Add anyio support, with `backend="anyio"`. (Pull #169) + +### Changed + +- Update the Transport API to use 'ext' for optional extensions. (Pull #190) +- Update the Transport API to use `.request` and `.arequest` so implementations can support both sync and async. (Pull #189) + +## 0.10.2 (August 20th, 2020) + +### Added + +- Added Unix Domain Socket support. (Pull #139) + +### Fixed + +- Always include the port on proxy CONNECT requests. (Pull #154) +- Fix `max_keepalive_connections` configuration. (Pull #153) +- Fixes behaviour in HTTP/1.1 where server disconnects can be used to signal the end of the response body. (Pull #164) + +## 0.10.1 (August 7th, 2020) + +- Include `max_keepalive_connections` on `AsyncHTTPProxy`/`SyncHTTPProxy` classes. + +## 0.10.0 (August 7th, 2020) + +The most notable change in the 0.10.0 release is that HTTP/2 support is now fully optional. + +Use either `pip install httpcore` for HTTP/1.1 support only, or `pip install httpcore[http2]` for HTTP/1.1 and HTTP/2 support. + +### Added + +- HTTP/2 support becomes optional. (Pull #121, #130) +- Add `local_address=...` support. (Pull #100, #134) +- Add `PlainByteStream`, `IteratorByteStream`, `AsyncIteratorByteStream`. The `AsyncByteSteam` and `SyncByteStream` classes are now pure interface classes. (#133) +- Add `LocalProtocolError`, `RemoteProtocolError` exceptions. (Pull #129) +- Add `UnsupportedProtocol` exception. (Pull #128) +- Add `.get_connection_info()` method. (Pull #102, #137) +- Add better TRACE logs. (Pull #101) + +### Changed + +- `max_keepalive` is deprecated in favour of `max_keepalive_connections`. (Pull #140) + +### Fixed + +- Improve handling of server disconnects. (Pull #112) + +## 0.9.1 (May 27th, 2020) + +### Fixed + +- Proper host resolution for sync case, including IPv6 support. (Pull #97) +- Close outstanding connections when connection pool is closed. (Pull #98) + +## 0.9.0 (May 21th, 2020) + +### Changed + +- URL port becomes an `Optional[int]` instead of `int`. (Pull #92) + +### Fixed + +- Honor HTTP/2 max concurrent streams settings. (Pull #89, #90) +- Remove incorrect debug log. (Pull #83) + +## 0.8.4 (May 11th, 2020) + +### Added + +- Logging via HTTPCORE_LOG_LEVEL and HTTPX_LOG_LEVEL environment variables +and TRACE level logging. (Pull #79) + +### Fixed + +- Reuse of connections on HTTP/2 in close concurrency situations. (Pull #81) + +## 0.8.3 (May 6rd, 2020) + +### Fixed + +- Include `Host` and `Accept` headers on proxy "CONNECT" requests. +- De-duplicate any headers also contained in proxy_headers. +- HTTP/2 flag not being passed down to proxy connections. + +## 0.8.2 (May 3rd, 2020) + +### Fixed + +- Fix connections using proxy forwarding requests not being added to the +connection pool properly. (Pull #70) + +## 0.8.1 (April 30th, 2020) + +### Changed + +- Allow inherintance of both `httpcore.AsyncByteStream`, `httpcore.SyncByteStream` without type conflicts. + +## 0.8.0 (April 30th, 2020) + +### Fixed + +- Fixed tunnel proxy support. + +### Added + +- New `TimeoutException` base class. + +## 0.7.0 (March 5th, 2020) + +- First integration with HTTPX. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/RECORD b/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..338e883792a7051faba2181798210f2552aa7838 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/RECORD @@ -0,0 +1,38 @@ +httpcore-1.0.9.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +httpcore-1.0.9.dist-info/METADATA,sha256=_i1P2mGZEol4d54M8n88BFxTGGP83Zh-rMdPOhjUHCE,21529 +httpcore-1.0.9.dist-info/RECORD,, +httpcore-1.0.9.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +httpcore-1.0.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87 +httpcore-1.0.9.dist-info/licenses/LICENSE.md,sha256=_ctZFUx0y6uhahEkL3dAvqnyPW_rVUeRfYxflKgDkqU,1518 +httpcore/__init__.py,sha256=9kT_kqChCCJUTHww24ZmR_ezcdbpRYWksD-gYNzkZP8,3445 +httpcore/_api.py,sha256=unZmeDschBWCGCPCwkS3Wot9euK6bg_kKxLtGTxw214,3146 +httpcore/_async/__init__.py,sha256=EWdl2v4thnAHzJpqjU4h2a8DUiGAvNiWrkii9pfhTf0,1221 +httpcore/_async/connection.py,sha256=6OcPXqMEfc0BU38_-iHUNDd1vKSTc2UVT09XqNb_BOk,8449 +httpcore/_async/connection_pool.py,sha256=DOIQ2s2ZCf9qfwxhzMprTPLqCL8OxGXiKF6qRHxvVyY,17307 +httpcore/_async/http11.py,sha256=-qM9bV7PjSQF5vxs37-eUXOIFwbIjPcZbNliuX9TtBw,13880 +httpcore/_async/http2.py,sha256=azX1fcmtXaIwjputFlZ4vd92J8xwjGOa9ax9QIv4394,23936 +httpcore/_async/http_proxy.py,sha256=2zVkrlv-Ds-rWGaqaXlrhEJiAQFPo23BT3Gq_sWoBXU,14701 +httpcore/_async/interfaces.py,sha256=jTiaWL83pgpGC9ziv90ZfwaKNMmHwmOalzaKiuTxATo,4455 +httpcore/_async/socks_proxy.py,sha256=lLKgLlggPfhFlqi0ODeBkOWvt9CghBBUyqsnsU1tx6Q,13841 +httpcore/_backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +httpcore/_backends/anyio.py,sha256=x8PgEhXRC8bVqsdzk_YJx8Y6d9Tub06CuUSwnbmtqoY,5252 +httpcore/_backends/auto.py,sha256=zO136PKZmsaTDK-HRk84eA-MUg8_2wJf4NvmK432Aio,1662 +httpcore/_backends/base.py,sha256=aShgRdZnMmRhFWHetjumlM73f8Kz1YOAyCUP_4kHslA,3042 +httpcore/_backends/mock.py,sha256=er9T436uSe7NLrfiLa4x6Nuqg5ivQ693CxWYCWsgbH4,4077 +httpcore/_backends/sync.py,sha256=bhE4d9iK9Umxdsdsgm2EfKnXaBms2WggGYU-7jmUujU,7977 +httpcore/_backends/trio.py,sha256=LHu4_Mr5MswQmmT3yE4oLgf9b_JJfeVS4BjDxeJc7Ro,5996 +httpcore/_exceptions.py,sha256=looCKga3_YVYu3s-d3L9RMPRJyhsY7fiuuGxvkOD0c0,1184 +httpcore/_models.py,sha256=IO2CcXcdpovRcLTdGFGB6RyBZdEm2h_TOmoCc4rEKho,17623 +httpcore/_ssl.py,sha256=srqmSNU4iOUvWF-SrJvb8G_YEbHFELOXQOwdDIBTS9c,187 +httpcore/_sync/__init__.py,sha256=JBDIgXt5la1LCJ1sLQeKhjKFpLnpNr8Svs6z2ni3fgg,1141 +httpcore/_sync/connection.py,sha256=9exGOb3PB-Mp2T1-sckSeL2t-tJ_9-NXomV8ihmWCgU,8238 +httpcore/_sync/connection_pool.py,sha256=a-T8LTsUxc7r0Ww1atfHSDoWPjQ0fA8Ul7S3-F0Mj70,16955 +httpcore/_sync/http11.py,sha256=IFobD1Md5JFlJGKWnh1_Q3epikUryI8qo09v8MiJIEA,13476 +httpcore/_sync/http2.py,sha256=AxU4yhcq68Bn5vqdJYtiXKYUj7nvhYbxz3v4rT4xnvA,23400 +httpcore/_sync/http_proxy.py,sha256=_al_6crKuEZu2wyvu493RZImJdBJnj5oGKNjLOJL2Zo,14463 +httpcore/_sync/interfaces.py,sha256=snXON42vUDHO5JBJvo8D4VWk2Wat44z2OXXHDrjbl94,4344 +httpcore/_sync/socks_proxy.py,sha256=zegZW9Snqj2_992DFJa8_CppOVBkVL4AgwduRkStakQ,13614 +httpcore/_synchronization.py,sha256=zSi13mAColBnknjZBknUC6hKNDQT4C6ijnezZ-r0T2s,9434 +httpcore/_trace.py,sha256=ck6ZoIzYTkdNAIfq5MGeKqBXDtqjOX-qfYwmZFbrGco,3952 +httpcore/_utils.py,sha256=_RLgXYOAYC350ikALV59GZ68IJrdocRZxPs9PjmzdFY,1537 +httpcore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/REQUESTED b/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/WHEEL b/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..12228d414b6cfed7c39d3781c85c63256a1d7fb5 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.27.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/licenses/LICENSE.md b/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/licenses/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..311b2b56c53f678ab95fc0def708c675d521a807 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpcore-1.0.9.dist-info/licenses/LICENSE.md @@ -0,0 +1,27 @@ +Copyright © 2020, [Encode OSS Ltd](https://www.encode.io/). +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e9addde071f81758baf350c4ab6bde2556340131 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/__init__.py @@ -0,0 +1,105 @@ +from .__version__ import __description__, __title__, __version__ +from ._api import * +from ._auth import * +from ._client import * +from ._config import * +from ._content import * +from ._exceptions import * +from ._models import * +from ._status_codes import * +from ._transports import * +from ._types import * +from ._urls import * + +try: + from ._main import main +except ImportError: # pragma: no cover + + def main() -> None: # type: ignore + import sys + + print( + "The httpx command line client could not run because the required " + "dependencies were not installed.\nMake sure you've installed " + "everything with: pip install 'httpx[cli]'" + ) + sys.exit(1) + + +__all__ = [ + "__description__", + "__title__", + "__version__", + "ASGITransport", + "AsyncBaseTransport", + "AsyncByteStream", + "AsyncClient", + "AsyncHTTPTransport", + "Auth", + "BaseTransport", + "BasicAuth", + "ByteStream", + "Client", + "CloseError", + "codes", + "ConnectError", + "ConnectTimeout", + "CookieConflict", + "Cookies", + "create_ssl_context", + "DecodingError", + "delete", + "DigestAuth", + "get", + "head", + "Headers", + "HTTPError", + "HTTPStatusError", + "HTTPTransport", + "InvalidURL", + "Limits", + "LocalProtocolError", + "main", + "MockTransport", + "NetRCAuth", + "NetworkError", + "options", + "patch", + "PoolTimeout", + "post", + "ProtocolError", + "Proxy", + "ProxyError", + "put", + "QueryParams", + "ReadError", + "ReadTimeout", + "RemoteProtocolError", + "request", + "Request", + "RequestError", + "RequestNotRead", + "Response", + "ResponseNotRead", + "stream", + "StreamClosed", + "StreamConsumed", + "StreamError", + "SyncByteStream", + "Timeout", + "TimeoutException", + "TooManyRedirects", + "TransportError", + "UnsupportedProtocol", + "URL", + "USE_CLIENT_DEFAULT", + "WriteError", + "WriteTimeout", + "WSGITransport", +] + + +__locals = locals() +for __name in __all__: + if not __name.startswith("__"): + setattr(__locals[__name], "__module__", "httpx") # noqa diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/__version__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/__version__.py new file mode 100644 index 0000000000000000000000000000000000000000..801bfacf671017cfbebf1ac26ec385daa02ed260 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/__version__.py @@ -0,0 +1,3 @@ +__title__ = "httpx" +__description__ = "A next generation HTTP client, for Python 3." +__version__ = "0.28.1" diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_api.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_api.py new file mode 100644 index 0000000000000000000000000000000000000000..c3cda1ecda8629edbdca2e3bc04bc51dba5e1430 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_api.py @@ -0,0 +1,438 @@ +from __future__ import annotations + +import typing +from contextlib import contextmanager + +from ._client import Client +from ._config import DEFAULT_TIMEOUT_CONFIG +from ._models import Response +from ._types import ( + AuthTypes, + CookieTypes, + HeaderTypes, + ProxyTypes, + QueryParamTypes, + RequestContent, + RequestData, + RequestFiles, + TimeoutTypes, +) +from ._urls import URL + +if typing.TYPE_CHECKING: + import ssl # pragma: no cover + + +__all__ = [ + "delete", + "get", + "head", + "options", + "patch", + "post", + "put", + "request", + "stream", +] + + +def request( + method: str, + url: URL | str, + *, + params: QueryParamTypes | None = None, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + trust_env: bool = True, +) -> Response: + """ + Sends an HTTP request. + + **Parameters:** + + * **method** - HTTP method for the new `Request` object: `GET`, `OPTIONS`, + `HEAD`, `POST`, `PUT`, `PATCH`, or `DELETE`. + * **url** - URL for the new `Request` object. + * **params** - *(optional)* Query parameters to include in the URL, as a + string, dictionary, or sequence of two-tuples. + * **content** - *(optional)* Binary content to include in the body of the + request, as bytes or a byte iterator. + * **data** - *(optional)* Form data to include in the body of the request, + as a dictionary. + * **files** - *(optional)* A dictionary of upload files to include in the + body of the request. + * **json** - *(optional)* A JSON serializable object to include in the body + of the request. + * **headers** - *(optional)* Dictionary of HTTP headers to include in the + request. + * **cookies** - *(optional)* Dictionary of Cookie items to include in the + request. + * **auth** - *(optional)* An authentication class to use when sending the + request. + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. + * **timeout** - *(optional)* The timeout configuration to use when sending + the request. + * **follow_redirects** - *(optional)* Enables or disables HTTP redirects. + * **verify** - *(optional)* Either `True` to use an SSL context with the + default CA bundle, `False` to disable verification, or an instance of + `ssl.SSLContext` to use a custom context. + * **trust_env** - *(optional)* Enables or disables usage of environment + variables for configuration. + + **Returns:** `Response` + + Usage: + + ``` + >>> import httpx + >>> response = httpx.request('GET', 'https://httpbin.org/get') + >>> response + + ``` + """ + with Client( + cookies=cookies, + proxy=proxy, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) as client: + return client.request( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + auth=auth, + follow_redirects=follow_redirects, + ) + + +@contextmanager +def stream( + method: str, + url: URL | str, + *, + params: QueryParamTypes | None = None, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + trust_env: bool = True, +) -> typing.Iterator[Response]: + """ + Alternative to `httpx.request()` that streams the response body + instead of loading it into memory at once. + + **Parameters**: See `httpx.request`. + + See also: [Streaming Responses][0] + + [0]: /quickstart#streaming-responses + """ + with Client( + cookies=cookies, + proxy=proxy, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) as client: + with client.stream( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + auth=auth, + follow_redirects=follow_redirects, + ) as response: + yield response + + +def get( + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends a `GET` request. + + **Parameters**: See `httpx.request`. + + Note that the `data`, `files`, `json` and `content` parameters are not available + on this function, as `GET` requests should not include a request body. + """ + return request( + "GET", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def options( + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends an `OPTIONS` request. + + **Parameters**: See `httpx.request`. + + Note that the `data`, `files`, `json` and `content` parameters are not available + on this function, as `OPTIONS` requests should not include a request body. + """ + return request( + "OPTIONS", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def head( + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends a `HEAD` request. + + **Parameters**: See `httpx.request`. + + Note that the `data`, `files`, `json` and `content` parameters are not available + on this function, as `HEAD` requests should not include a request body. + """ + return request( + "HEAD", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def post( + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends a `POST` request. + + **Parameters**: See `httpx.request`. + """ + return request( + "POST", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def put( + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends a `PUT` request. + + **Parameters**: See `httpx.request`. + """ + return request( + "PUT", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def patch( + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends a `PATCH` request. + + **Parameters**: See `httpx.request`. + """ + return request( + "PATCH", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def delete( + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + verify: ssl.SSLContext | str | bool = True, + trust_env: bool = True, +) -> Response: + """ + Sends a `DELETE` request. + + **Parameters**: See `httpx.request`. + + Note that the `data`, `files`, `json` and `content` parameters are not available + on this function, as `DELETE` requests should not include a request body. + """ + return request( + "DELETE", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_auth.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_auth.py new file mode 100644 index 0000000000000000000000000000000000000000..b03971ab4b311d60790dc22ca24d9966426ec0a4 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_auth.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +import hashlib +import os +import re +import time +import typing +from base64 import b64encode +from urllib.request import parse_http_list + +from ._exceptions import ProtocolError +from ._models import Cookies, Request, Response +from ._utils import to_bytes, to_str, unquote + +if typing.TYPE_CHECKING: # pragma: no cover + from hashlib import _Hash + + +__all__ = ["Auth", "BasicAuth", "DigestAuth", "NetRCAuth"] + + +class Auth: + """ + Base class for all authentication schemes. + + To implement a custom authentication scheme, subclass `Auth` and override + the `.auth_flow()` method. + + If the authentication scheme does I/O such as disk access or network calls, or uses + synchronization primitives such as locks, you should override `.sync_auth_flow()` + and/or `.async_auth_flow()` instead of `.auth_flow()` to provide specialized + implementations that will be used by `Client` and `AsyncClient` respectively. + """ + + requires_request_body = False + requires_response_body = False + + def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + """ + Execute the authentication flow. + + To dispatch a request, `yield` it: + + ``` + yield request + ``` + + The client will `.send()` the response back into the flow generator. You can + access it like so: + + ``` + response = yield request + ``` + + A `return` (or reaching the end of the generator) will result in the + client returning the last response obtained from the server. + + You can dispatch as many requests as is necessary. + """ + yield request + + def sync_auth_flow( + self, request: Request + ) -> typing.Generator[Request, Response, None]: + """ + Execute the authentication flow synchronously. + + By default, this defers to `.auth_flow()`. You should override this method + when the authentication scheme does I/O and/or uses concurrency primitives. + """ + if self.requires_request_body: + request.read() + + flow = self.auth_flow(request) + request = next(flow) + + while True: + response = yield request + if self.requires_response_body: + response.read() + + try: + request = flow.send(response) + except StopIteration: + break + + async def async_auth_flow( + self, request: Request + ) -> typing.AsyncGenerator[Request, Response]: + """ + Execute the authentication flow asynchronously. + + By default, this defers to `.auth_flow()`. You should override this method + when the authentication scheme does I/O and/or uses concurrency primitives. + """ + if self.requires_request_body: + await request.aread() + + flow = self.auth_flow(request) + request = next(flow) + + while True: + response = yield request + if self.requires_response_body: + await response.aread() + + try: + request = flow.send(response) + except StopIteration: + break + + +class FunctionAuth(Auth): + """ + Allows the 'auth' argument to be passed as a simple callable function, + that takes the request, and returns a new, modified request. + """ + + def __init__(self, func: typing.Callable[[Request], Request]) -> None: + self._func = func + + def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + yield self._func(request) + + +class BasicAuth(Auth): + """ + Allows the 'auth' argument to be passed as a (username, password) pair, + and uses HTTP Basic authentication. + """ + + def __init__(self, username: str | bytes, password: str | bytes) -> None: + self._auth_header = self._build_auth_header(username, password) + + def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + request.headers["Authorization"] = self._auth_header + yield request + + def _build_auth_header(self, username: str | bytes, password: str | bytes) -> str: + userpass = b":".join((to_bytes(username), to_bytes(password))) + token = b64encode(userpass).decode() + return f"Basic {token}" + + +class NetRCAuth(Auth): + """ + Use a 'netrc' file to lookup basic auth credentials based on the url host. + """ + + def __init__(self, file: str | None = None) -> None: + # Lazily import 'netrc'. + # There's no need for us to load this module unless 'NetRCAuth' is being used. + import netrc + + self._netrc_info = netrc.netrc(file) + + def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + auth_info = self._netrc_info.authenticators(request.url.host) + if auth_info is None or not auth_info[2]: + # The netrc file did not have authentication credentials for this host. + yield request + else: + # Build a basic auth header with credentials from the netrc file. + request.headers["Authorization"] = self._build_auth_header( + username=auth_info[0], password=auth_info[2] + ) + yield request + + def _build_auth_header(self, username: str | bytes, password: str | bytes) -> str: + userpass = b":".join((to_bytes(username), to_bytes(password))) + token = b64encode(userpass).decode() + return f"Basic {token}" + + +class DigestAuth(Auth): + _ALGORITHM_TO_HASH_FUNCTION: dict[str, typing.Callable[[bytes], _Hash]] = { + "MD5": hashlib.md5, + "MD5-SESS": hashlib.md5, + "SHA": hashlib.sha1, + "SHA-SESS": hashlib.sha1, + "SHA-256": hashlib.sha256, + "SHA-256-SESS": hashlib.sha256, + "SHA-512": hashlib.sha512, + "SHA-512-SESS": hashlib.sha512, + } + + def __init__(self, username: str | bytes, password: str | bytes) -> None: + self._username = to_bytes(username) + self._password = to_bytes(password) + self._last_challenge: _DigestAuthChallenge | None = None + self._nonce_count = 1 + + def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + if self._last_challenge: + request.headers["Authorization"] = self._build_auth_header( + request, self._last_challenge + ) + + response = yield request + + if response.status_code != 401 or "www-authenticate" not in response.headers: + # If the response is not a 401 then we don't + # need to build an authenticated request. + return + + for auth_header in response.headers.get_list("www-authenticate"): + if auth_header.lower().startswith("digest "): + break + else: + # If the response does not include a 'WWW-Authenticate: Digest ...' + # header, then we don't need to build an authenticated request. + return + + self._last_challenge = self._parse_challenge(request, response, auth_header) + self._nonce_count = 1 + + request.headers["Authorization"] = self._build_auth_header( + request, self._last_challenge + ) + if response.cookies: + Cookies(response.cookies).set_cookie_header(request=request) + yield request + + def _parse_challenge( + self, request: Request, response: Response, auth_header: str + ) -> _DigestAuthChallenge: + """ + Returns a challenge from a Digest WWW-Authenticate header. + These take the form of: + `Digest realm="realm@host.com",qop="auth,auth-int",nonce="abc",opaque="xyz"` + """ + scheme, _, fields = auth_header.partition(" ") + + # This method should only ever have been called with a Digest auth header. + assert scheme.lower() == "digest" + + header_dict: dict[str, str] = {} + for field in parse_http_list(fields): + key, value = field.strip().split("=", 1) + header_dict[key] = unquote(value) + + try: + realm = header_dict["realm"].encode() + nonce = header_dict["nonce"].encode() + algorithm = header_dict.get("algorithm", "MD5") + opaque = header_dict["opaque"].encode() if "opaque" in header_dict else None + qop = header_dict["qop"].encode() if "qop" in header_dict else None + return _DigestAuthChallenge( + realm=realm, nonce=nonce, algorithm=algorithm, opaque=opaque, qop=qop + ) + except KeyError as exc: + message = "Malformed Digest WWW-Authenticate header" + raise ProtocolError(message, request=request) from exc + + def _build_auth_header( + self, request: Request, challenge: _DigestAuthChallenge + ) -> str: + hash_func = self._ALGORITHM_TO_HASH_FUNCTION[challenge.algorithm.upper()] + + def digest(data: bytes) -> bytes: + return hash_func(data).hexdigest().encode() + + A1 = b":".join((self._username, challenge.realm, self._password)) + + path = request.url.raw_path + A2 = b":".join((request.method.encode(), path)) + # TODO: implement auth-int + HA2 = digest(A2) + + nc_value = b"%08x" % self._nonce_count + cnonce = self._get_client_nonce(self._nonce_count, challenge.nonce) + self._nonce_count += 1 + + HA1 = digest(A1) + if challenge.algorithm.lower().endswith("-sess"): + HA1 = digest(b":".join((HA1, challenge.nonce, cnonce))) + + qop = self._resolve_qop(challenge.qop, request=request) + if qop is None: + # Following RFC 2069 + digest_data = [HA1, challenge.nonce, HA2] + else: + # Following RFC 2617/7616 + digest_data = [HA1, challenge.nonce, nc_value, cnonce, qop, HA2] + + format_args = { + "username": self._username, + "realm": challenge.realm, + "nonce": challenge.nonce, + "uri": path, + "response": digest(b":".join(digest_data)), + "algorithm": challenge.algorithm.encode(), + } + if challenge.opaque: + format_args["opaque"] = challenge.opaque + if qop: + format_args["qop"] = b"auth" + format_args["nc"] = nc_value + format_args["cnonce"] = cnonce + + return "Digest " + self._get_header_value(format_args) + + def _get_client_nonce(self, nonce_count: int, nonce: bytes) -> bytes: + s = str(nonce_count).encode() + s += nonce + s += time.ctime().encode() + s += os.urandom(8) + + return hashlib.sha1(s).hexdigest()[:16].encode() + + def _get_header_value(self, header_fields: dict[str, bytes]) -> str: + NON_QUOTED_FIELDS = ("algorithm", "qop", "nc") + QUOTED_TEMPLATE = '{}="{}"' + NON_QUOTED_TEMPLATE = "{}={}" + + header_value = "" + for i, (field, value) in enumerate(header_fields.items()): + if i > 0: + header_value += ", " + template = ( + QUOTED_TEMPLATE + if field not in NON_QUOTED_FIELDS + else NON_QUOTED_TEMPLATE + ) + header_value += template.format(field, to_str(value)) + + return header_value + + def _resolve_qop(self, qop: bytes | None, request: Request) -> bytes | None: + if qop is None: + return None + qops = re.split(b", ?", qop) + if b"auth" in qops: + return b"auth" + + if qops == [b"auth-int"]: + raise NotImplementedError("Digest auth-int support is not yet implemented") + + message = f'Unexpected qop value "{qop!r}" in digest auth' + raise ProtocolError(message, request=request) + + +class _DigestAuthChallenge(typing.NamedTuple): + realm: bytes + nonce: bytes + algorithm: str + opaque: bytes | None + qop: bytes | None diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_client.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_client.py new file mode 100644 index 0000000000000000000000000000000000000000..2249231f8c3b912c731ff160344d3672e2f11738 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_client.py @@ -0,0 +1,2019 @@ +from __future__ import annotations + +import datetime +import enum +import logging +import time +import typing +import warnings +from contextlib import asynccontextmanager, contextmanager +from types import TracebackType + +from .__version__ import __version__ +from ._auth import Auth, BasicAuth, FunctionAuth +from ._config import ( + DEFAULT_LIMITS, + DEFAULT_MAX_REDIRECTS, + DEFAULT_TIMEOUT_CONFIG, + Limits, + Proxy, + Timeout, +) +from ._decoders import SUPPORTED_DECODERS +from ._exceptions import ( + InvalidURL, + RemoteProtocolError, + TooManyRedirects, + request_context, +) +from ._models import Cookies, Headers, Request, Response +from ._status_codes import codes +from ._transports.base import AsyncBaseTransport, BaseTransport +from ._transports.default import AsyncHTTPTransport, HTTPTransport +from ._types import ( + AsyncByteStream, + AuthTypes, + CertTypes, + CookieTypes, + HeaderTypes, + ProxyTypes, + QueryParamTypes, + RequestContent, + RequestData, + RequestExtensions, + RequestFiles, + SyncByteStream, + TimeoutTypes, +) +from ._urls import URL, QueryParams +from ._utils import URLPattern, get_environment_proxies + +if typing.TYPE_CHECKING: + import ssl # pragma: no cover + +__all__ = ["USE_CLIENT_DEFAULT", "AsyncClient", "Client"] + +# The type annotation for @classmethod and context managers here follows PEP 484 +# https://www.python.org/dev/peps/pep-0484/#annotating-instance-and-class-methods +T = typing.TypeVar("T", bound="Client") +U = typing.TypeVar("U", bound="AsyncClient") + + +def _is_https_redirect(url: URL, location: URL) -> bool: + """ + Return 'True' if 'location' is a HTTPS upgrade of 'url' + """ + if url.host != location.host: + return False + + return ( + url.scheme == "http" + and _port_or_default(url) == 80 + and location.scheme == "https" + and _port_or_default(location) == 443 + ) + + +def _port_or_default(url: URL) -> int | None: + if url.port is not None: + return url.port + return {"http": 80, "https": 443}.get(url.scheme) + + +def _same_origin(url: URL, other: URL) -> bool: + """ + Return 'True' if the given URLs share the same origin. + """ + return ( + url.scheme == other.scheme + and url.host == other.host + and _port_or_default(url) == _port_or_default(other) + ) + + +class UseClientDefault: + """ + For some parameters such as `auth=...` and `timeout=...` we need to be able + to indicate the default "unset" state, in a way that is distinctly different + to using `None`. + + The default "unset" state indicates that whatever default is set on the + client should be used. This is different to setting `None`, which + explicitly disables the parameter, possibly overriding a client default. + + For example we use `timeout=USE_CLIENT_DEFAULT` in the `request()` signature. + Omitting the `timeout` parameter will send a request using whatever default + timeout has been configured on the client. Including `timeout=None` will + ensure no timeout is used. + + Note that user code shouldn't need to use the `USE_CLIENT_DEFAULT` constant, + but it is used internally when a parameter is not included. + """ + + +USE_CLIENT_DEFAULT = UseClientDefault() + + +logger = logging.getLogger("httpx") + +USER_AGENT = f"python-httpx/{__version__}" +ACCEPT_ENCODING = ", ".join( + [key for key in SUPPORTED_DECODERS.keys() if key != "identity"] +) + + +class ClientState(enum.Enum): + # UNOPENED: + # The client has been instantiated, but has not been used to send a request, + # or been opened by entering the context of a `with` block. + UNOPENED = 1 + # OPENED: + # The client has either sent a request, or is within a `with` block. + OPENED = 2 + # CLOSED: + # The client has either exited the `with` block, or `close()` has + # been called explicitly. + CLOSED = 3 + + +class BoundSyncStream(SyncByteStream): + """ + A byte stream that is bound to a given response instance, and that + ensures the `response.elapsed` is set once the response is closed. + """ + + def __init__( + self, stream: SyncByteStream, response: Response, start: float + ) -> None: + self._stream = stream + self._response = response + self._start = start + + def __iter__(self) -> typing.Iterator[bytes]: + for chunk in self._stream: + yield chunk + + def close(self) -> None: + elapsed = time.perf_counter() - self._start + self._response.elapsed = datetime.timedelta(seconds=elapsed) + self._stream.close() + + +class BoundAsyncStream(AsyncByteStream): + """ + An async byte stream that is bound to a given response instance, and that + ensures the `response.elapsed` is set once the response is closed. + """ + + def __init__( + self, stream: AsyncByteStream, response: Response, start: float + ) -> None: + self._stream = stream + self._response = response + self._start = start + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + async for chunk in self._stream: + yield chunk + + async def aclose(self) -> None: + elapsed = time.perf_counter() - self._start + self._response.elapsed = datetime.timedelta(seconds=elapsed) + await self._stream.aclose() + + +EventHook = typing.Callable[..., typing.Any] + + +class BaseClient: + def __init__( + self, + *, + auth: AuthTypes | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + follow_redirects: bool = False, + max_redirects: int = DEFAULT_MAX_REDIRECTS, + event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, + base_url: URL | str = "", + trust_env: bool = True, + default_encoding: str | typing.Callable[[bytes], str] = "utf-8", + ) -> None: + event_hooks = {} if event_hooks is None else event_hooks + + self._base_url = self._enforce_trailing_slash(URL(base_url)) + + self._auth = self._build_auth(auth) + self._params = QueryParams(params) + self.headers = Headers(headers) + self._cookies = Cookies(cookies) + self._timeout = Timeout(timeout) + self.follow_redirects = follow_redirects + self.max_redirects = max_redirects + self._event_hooks = { + "request": list(event_hooks.get("request", [])), + "response": list(event_hooks.get("response", [])), + } + self._trust_env = trust_env + self._default_encoding = default_encoding + self._state = ClientState.UNOPENED + + @property + def is_closed(self) -> bool: + """ + Check if the client being closed + """ + return self._state == ClientState.CLOSED + + @property + def trust_env(self) -> bool: + return self._trust_env + + def _enforce_trailing_slash(self, url: URL) -> URL: + if url.raw_path.endswith(b"/"): + return url + return url.copy_with(raw_path=url.raw_path + b"/") + + def _get_proxy_map( + self, proxy: ProxyTypes | None, allow_env_proxies: bool + ) -> dict[str, Proxy | None]: + if proxy is None: + if allow_env_proxies: + return { + key: None if url is None else Proxy(url=url) + for key, url in get_environment_proxies().items() + } + return {} + else: + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + return {"all://": proxy} + + @property + def timeout(self) -> Timeout: + return self._timeout + + @timeout.setter + def timeout(self, timeout: TimeoutTypes) -> None: + self._timeout = Timeout(timeout) + + @property + def event_hooks(self) -> dict[str, list[EventHook]]: + return self._event_hooks + + @event_hooks.setter + def event_hooks(self, event_hooks: dict[str, list[EventHook]]) -> None: + self._event_hooks = { + "request": list(event_hooks.get("request", [])), + "response": list(event_hooks.get("response", [])), + } + + @property + def auth(self) -> Auth | None: + """ + Authentication class used when none is passed at the request-level. + + See also [Authentication][0]. + + [0]: /quickstart/#authentication + """ + return self._auth + + @auth.setter + def auth(self, auth: AuthTypes) -> None: + self._auth = self._build_auth(auth) + + @property + def base_url(self) -> URL: + """ + Base URL to use when sending requests with relative URLs. + """ + return self._base_url + + @base_url.setter + def base_url(self, url: URL | str) -> None: + self._base_url = self._enforce_trailing_slash(URL(url)) + + @property + def headers(self) -> Headers: + """ + HTTP headers to include when sending requests. + """ + return self._headers + + @headers.setter + def headers(self, headers: HeaderTypes) -> None: + client_headers = Headers( + { + b"Accept": b"*/*", + b"Accept-Encoding": ACCEPT_ENCODING.encode("ascii"), + b"Connection": b"keep-alive", + b"User-Agent": USER_AGENT.encode("ascii"), + } + ) + client_headers.update(headers) + self._headers = client_headers + + @property + def cookies(self) -> Cookies: + """ + Cookie values to include when sending requests. + """ + return self._cookies + + @cookies.setter + def cookies(self, cookies: CookieTypes) -> None: + self._cookies = Cookies(cookies) + + @property + def params(self) -> QueryParams: + """ + Query parameters to include in the URL when sending requests. + """ + return self._params + + @params.setter + def params(self, params: QueryParamTypes) -> None: + self._params = QueryParams(params) + + def build_request( + self, + method: str, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Request: + """ + Build and return a request instance. + + * The `params`, `headers` and `cookies` arguments + are merged with any values set on the client. + * The `url` argument is merged with any `base_url` set on the client. + + See also: [Request instances][0] + + [0]: /advanced/clients/#request-instances + """ + url = self._merge_url(url) + headers = self._merge_headers(headers) + cookies = self._merge_cookies(cookies) + params = self._merge_queryparams(params) + extensions = {} if extensions is None else extensions + if "timeout" not in extensions: + timeout = ( + self.timeout + if isinstance(timeout, UseClientDefault) + else Timeout(timeout) + ) + extensions = dict(**extensions, timeout=timeout.as_dict()) + return Request( + method, + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + extensions=extensions, + ) + + def _merge_url(self, url: URL | str) -> URL: + """ + Merge a URL argument together with any 'base_url' on the client, + to create the URL used for the outgoing request. + """ + merge_url = URL(url) + if merge_url.is_relative_url: + # To merge URLs we always append to the base URL. To get this + # behaviour correct we always ensure the base URL ends in a '/' + # separator, and strip any leading '/' from the merge URL. + # + # So, eg... + # + # >>> client = Client(base_url="https://www.example.com/subpath") + # >>> client.base_url + # URL('https://www.example.com/subpath/') + # >>> client.build_request("GET", "/path").url + # URL('https://www.example.com/subpath/path') + merge_raw_path = self.base_url.raw_path + merge_url.raw_path.lstrip(b"/") + return self.base_url.copy_with(raw_path=merge_raw_path) + return merge_url + + def _merge_cookies(self, cookies: CookieTypes | None = None) -> CookieTypes | None: + """ + Merge a cookies argument together with any cookies on the client, + to create the cookies used for the outgoing request. + """ + if cookies or self.cookies: + merged_cookies = Cookies(self.cookies) + merged_cookies.update(cookies) + return merged_cookies + return cookies + + def _merge_headers(self, headers: HeaderTypes | None = None) -> HeaderTypes | None: + """ + Merge a headers argument together with any headers on the client, + to create the headers used for the outgoing request. + """ + merged_headers = Headers(self.headers) + merged_headers.update(headers) + return merged_headers + + def _merge_queryparams( + self, params: QueryParamTypes | None = None + ) -> QueryParamTypes | None: + """ + Merge a queryparams argument together with any queryparams on the client, + to create the queryparams used for the outgoing request. + """ + if params or self.params: + merged_queryparams = QueryParams(self.params) + return merged_queryparams.merge(params) + return params + + def _build_auth(self, auth: AuthTypes | None) -> Auth | None: + if auth is None: + return None + elif isinstance(auth, tuple): + return BasicAuth(username=auth[0], password=auth[1]) + elif isinstance(auth, Auth): + return auth + elif callable(auth): + return FunctionAuth(func=auth) + else: + raise TypeError(f'Invalid "auth" argument: {auth!r}') + + def _build_request_auth( + self, + request: Request, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + ) -> Auth: + auth = ( + self._auth if isinstance(auth, UseClientDefault) else self._build_auth(auth) + ) + + if auth is not None: + return auth + + username, password = request.url.username, request.url.password + if username or password: + return BasicAuth(username=username, password=password) + + return Auth() + + def _build_redirect_request(self, request: Request, response: Response) -> Request: + """ + Given a request and a redirect response, return a new request that + should be used to effect the redirect. + """ + method = self._redirect_method(request, response) + url = self._redirect_url(request, response) + headers = self._redirect_headers(request, url, method) + stream = self._redirect_stream(request, method) + cookies = Cookies(self.cookies) + return Request( + method=method, + url=url, + headers=headers, + cookies=cookies, + stream=stream, + extensions=request.extensions, + ) + + def _redirect_method(self, request: Request, response: Response) -> str: + """ + When being redirected we may want to change the method of the request + based on certain specs or browser behavior. + """ + method = request.method + + # https://tools.ietf.org/html/rfc7231#section-6.4.4 + if response.status_code == codes.SEE_OTHER and method != "HEAD": + method = "GET" + + # Do what the browsers do, despite standards... + # Turn 302s into GETs. + if response.status_code == codes.FOUND and method != "HEAD": + method = "GET" + + # If a POST is responded to with a 301, turn it into a GET. + # This bizarre behaviour is explained in 'requests' issue 1704. + if response.status_code == codes.MOVED_PERMANENTLY and method == "POST": + method = "GET" + + return method + + def _redirect_url(self, request: Request, response: Response) -> URL: + """ + Return the URL for the redirect to follow. + """ + location = response.headers["Location"] + + try: + url = URL(location) + except InvalidURL as exc: + raise RemoteProtocolError( + f"Invalid URL in location header: {exc}.", request=request + ) from None + + # Handle malformed 'Location' headers that are "absolute" form, have no host. + # See: https://github.com/encode/httpx/issues/771 + if url.scheme and not url.host: + url = url.copy_with(host=request.url.host) + + # Facilitate relative 'Location' headers, as allowed by RFC 7231. + # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') + if url.is_relative_url: + url = request.url.join(url) + + # Attach previous fragment if needed (RFC 7231 7.1.2) + if request.url.fragment and not url.fragment: + url = url.copy_with(fragment=request.url.fragment) + + return url + + def _redirect_headers(self, request: Request, url: URL, method: str) -> Headers: + """ + Return the headers that should be used for the redirect request. + """ + headers = Headers(request.headers) + + if not _same_origin(url, request.url): + if not _is_https_redirect(request.url, url): + # Strip Authorization headers when responses are redirected + # away from the origin. (Except for direct HTTP to HTTPS redirects.) + headers.pop("Authorization", None) + + # Update the Host header. + headers["Host"] = url.netloc.decode("ascii") + + if method != request.method and method == "GET": + # If we've switch to a 'GET' request, then strip any headers which + # are only relevant to the request body. + headers.pop("Content-Length", None) + headers.pop("Transfer-Encoding", None) + + # We should use the client cookie store to determine any cookie header, + # rather than whatever was on the original outgoing request. + headers.pop("Cookie", None) + + return headers + + def _redirect_stream( + self, request: Request, method: str + ) -> SyncByteStream | AsyncByteStream | None: + """ + Return the body that should be used for the redirect request. + """ + if method != request.method and method == "GET": + return None + + return request.stream + + def _set_timeout(self, request: Request) -> None: + if "timeout" not in request.extensions: + timeout = ( + self.timeout + if isinstance(self.timeout, UseClientDefault) + else Timeout(self.timeout) + ) + request.extensions = dict(**request.extensions, timeout=timeout.as_dict()) + + +class Client(BaseClient): + """ + An HTTP client, with connection pooling, HTTP/2, redirects, cookie persistence, etc. + + It can be shared between threads. + + Usage: + + ```python + >>> client = httpx.Client() + >>> response = client.get('https://example.org') + ``` + + **Parameters:** + + * **auth** - *(optional)* An authentication class to use when sending + requests. + * **params** - *(optional)* Query parameters to include in request URLs, as + a string, dictionary, or sequence of two-tuples. + * **headers** - *(optional)* Dictionary of HTTP headers to include when + sending requests. + * **cookies** - *(optional)* Dictionary of Cookie items to include when + sending requests. + * **verify** - *(optional)* Either `True` to use an SSL context with the + default CA bundle, `False` to disable verification, or an instance of + `ssl.SSLContext` to use a custom context. + * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be + enabled. Defaults to `False`. + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. + * **timeout** - *(optional)* The timeout configuration to use when sending + requests. + * **limits** - *(optional)* The limits configuration to use. + * **max_redirects** - *(optional)* The maximum number of redirect responses + that should be followed. + * **base_url** - *(optional)* A URL to use as the base when building + request URLs. + * **transport** - *(optional)* A transport class to use for sending requests + over the network. + * **trust_env** - *(optional)* Enables or disables usage of environment + variables for configuration. + * **default_encoding** - *(optional)* The default encoding to use for decoding + response text, if no charset information is included in a response Content-Type + header. Set to a callable for automatic character set detection. Default: "utf-8". + """ + + def __init__( + self, + *, + auth: AuthTypes | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + proxy: ProxyTypes | None = None, + mounts: None | (typing.Mapping[str, BaseTransport | None]) = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + follow_redirects: bool = False, + limits: Limits = DEFAULT_LIMITS, + max_redirects: int = DEFAULT_MAX_REDIRECTS, + event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, + base_url: URL | str = "", + transport: BaseTransport | None = None, + default_encoding: str | typing.Callable[[bytes], str] = "utf-8", + ) -> None: + super().__init__( + auth=auth, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + follow_redirects=follow_redirects, + max_redirects=max_redirects, + event_hooks=event_hooks, + base_url=base_url, + trust_env=trust_env, + default_encoding=default_encoding, + ) + + if http2: + try: + import h2 # noqa + except ImportError: # pragma: no cover + raise ImportError( + "Using http2=True, but the 'h2' package is not installed. " + "Make sure to install httpx using `pip install httpx[http2]`." + ) from None + + allow_env_proxies = trust_env and transport is None + proxy_map = self._get_proxy_map(proxy, allow_env_proxies) + + self._transport = self._init_transport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + transport=transport, + ) + self._mounts: dict[URLPattern, BaseTransport | None] = { + URLPattern(key): None + if proxy is None + else self._init_proxy_transport( + proxy, + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + ) + for key, proxy in proxy_map.items() + } + if mounts is not None: + self._mounts.update( + {URLPattern(key): transport for key, transport in mounts.items()} + ) + + self._mounts = dict(sorted(self._mounts.items())) + + def _init_transport( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + transport: BaseTransport | None = None, + ) -> BaseTransport: + if transport is not None: + return transport + + return HTTPTransport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + ) + + def _init_proxy_transport( + self, + proxy: Proxy, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + ) -> BaseTransport: + return HTTPTransport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + proxy=proxy, + ) + + def _transport_for_url(self, url: URL) -> BaseTransport: + """ + Returns the transport instance that should be used for a given URL. + This will either be the standard connection pool, or a proxy. + """ + for pattern, transport in self._mounts.items(): + if pattern.matches(url): + return self._transport if transport is None else transport + + return self._transport + + def request( + self, + method: str, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Build and send a request. + + Equivalent to: + + ```python + request = client.build_request(...) + response = client.send(request, ...) + ``` + + See `Client.build_request()`, `Client.send()` and + [Merging of configuration][0] for how the various parameters + are merged with client-level configuration. + + [0]: /advanced/clients/#merging-of-configuration + """ + if cookies is not None: + message = ( + "Setting per-request cookies=<...> is being deprecated, because " + "the expected behaviour on cookie persistence is ambiguous. Set " + "cookies directly on the client instance instead." + ) + warnings.warn(message, DeprecationWarning, stacklevel=2) + + request = self.build_request( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + extensions=extensions, + ) + return self.send(request, auth=auth, follow_redirects=follow_redirects) + + @contextmanager + def stream( + self, + method: str, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> typing.Iterator[Response]: + """ + Alternative to `httpx.request()` that streams the response body + instead of loading it into memory at once. + + **Parameters**: See `httpx.request`. + + See also: [Streaming Responses][0] + + [0]: /quickstart#streaming-responses + """ + request = self.build_request( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + extensions=extensions, + ) + response = self.send( + request=request, + auth=auth, + follow_redirects=follow_redirects, + stream=True, + ) + try: + yield response + finally: + response.close() + + def send( + self, + request: Request, + *, + stream: bool = False, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + ) -> Response: + """ + Send a request. + + The request is sent as-is, unmodified. + + Typically you'll want to build one with `Client.build_request()` + so that any client-level configuration is merged into the request, + but passing an explicit `httpx.Request()` is supported as well. + + See also: [Request instances][0] + + [0]: /advanced/clients/#request-instances + """ + if self._state == ClientState.CLOSED: + raise RuntimeError("Cannot send a request, as the client has been closed.") + + self._state = ClientState.OPENED + follow_redirects = ( + self.follow_redirects + if isinstance(follow_redirects, UseClientDefault) + else follow_redirects + ) + + self._set_timeout(request) + + auth = self._build_request_auth(request, auth) + + response = self._send_handling_auth( + request, + auth=auth, + follow_redirects=follow_redirects, + history=[], + ) + try: + if not stream: + response.read() + + return response + + except BaseException as exc: + response.close() + raise exc + + def _send_handling_auth( + self, + request: Request, + auth: Auth, + follow_redirects: bool, + history: list[Response], + ) -> Response: + auth_flow = auth.sync_auth_flow(request) + try: + request = next(auth_flow) + + while True: + response = self._send_handling_redirects( + request, + follow_redirects=follow_redirects, + history=history, + ) + try: + try: + next_request = auth_flow.send(response) + except StopIteration: + return response + + response.history = list(history) + response.read() + request = next_request + history.append(response) + + except BaseException as exc: + response.close() + raise exc + finally: + auth_flow.close() + + def _send_handling_redirects( + self, + request: Request, + follow_redirects: bool, + history: list[Response], + ) -> Response: + while True: + if len(history) > self.max_redirects: + raise TooManyRedirects( + "Exceeded maximum allowed redirects.", request=request + ) + + for hook in self._event_hooks["request"]: + hook(request) + + response = self._send_single_request(request) + try: + for hook in self._event_hooks["response"]: + hook(response) + response.history = list(history) + + if not response.has_redirect_location: + return response + + request = self._build_redirect_request(request, response) + history = history + [response] + + if follow_redirects: + response.read() + else: + response.next_request = request + return response + + except BaseException as exc: + response.close() + raise exc + + def _send_single_request(self, request: Request) -> Response: + """ + Sends a single request, without handling any redirections. + """ + transport = self._transport_for_url(request.url) + start = time.perf_counter() + + if not isinstance(request.stream, SyncByteStream): + raise RuntimeError( + "Attempted to send an async request with a sync Client instance." + ) + + with request_context(request=request): + response = transport.handle_request(request) + + assert isinstance(response.stream, SyncByteStream) + + response.request = request + response.stream = BoundSyncStream( + response.stream, response=response, start=start + ) + self.cookies.extract_cookies(response) + response.default_encoding = self._default_encoding + + logger.info( + 'HTTP Request: %s %s "%s %d %s"', + request.method, + request.url, + response.http_version, + response.status_code, + response.reason_phrase, + ) + + return response + + def get( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `GET` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "GET", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def options( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send an `OPTIONS` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "OPTIONS", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def head( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `HEAD` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "HEAD", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def post( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `POST` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "POST", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def put( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `PUT` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "PUT", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def patch( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `PATCH` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "PATCH", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def delete( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `DELETE` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "DELETE", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def close(self) -> None: + """ + Close transport and proxies. + """ + if self._state != ClientState.CLOSED: + self._state = ClientState.CLOSED + + self._transport.close() + for transport in self._mounts.values(): + if transport is not None: + transport.close() + + def __enter__(self: T) -> T: + if self._state != ClientState.UNOPENED: + msg = { + ClientState.OPENED: "Cannot open a client instance more than once.", + ClientState.CLOSED: ( + "Cannot reopen a client instance, once it has been closed." + ), + }[self._state] + raise RuntimeError(msg) + + self._state = ClientState.OPENED + + self._transport.__enter__() + for transport in self._mounts.values(): + if transport is not None: + transport.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + self._state = ClientState.CLOSED + + self._transport.__exit__(exc_type, exc_value, traceback) + for transport in self._mounts.values(): + if transport is not None: + transport.__exit__(exc_type, exc_value, traceback) + + +class AsyncClient(BaseClient): + """ + An asynchronous HTTP client, with connection pooling, HTTP/2, redirects, + cookie persistence, etc. + + It can be shared between tasks. + + Usage: + + ```python + >>> async with httpx.AsyncClient() as client: + >>> response = await client.get('https://example.org') + ``` + + **Parameters:** + + * **auth** - *(optional)* An authentication class to use when sending + requests. + * **params** - *(optional)* Query parameters to include in request URLs, as + a string, dictionary, or sequence of two-tuples. + * **headers** - *(optional)* Dictionary of HTTP headers to include when + sending requests. + * **cookies** - *(optional)* Dictionary of Cookie items to include when + sending requests. + * **verify** - *(optional)* Either `True` to use an SSL context with the + default CA bundle, `False` to disable verification, or an instance of + `ssl.SSLContext` to use a custom context. + * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be + enabled. Defaults to `False`. + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. + * **timeout** - *(optional)* The timeout configuration to use when sending + requests. + * **limits** - *(optional)* The limits configuration to use. + * **max_redirects** - *(optional)* The maximum number of redirect responses + that should be followed. + * **base_url** - *(optional)* A URL to use as the base when building + request URLs. + * **transport** - *(optional)* A transport class to use for sending requests + over the network. + * **trust_env** - *(optional)* Enables or disables usage of environment + variables for configuration. + * **default_encoding** - *(optional)* The default encoding to use for decoding + response text, if no charset information is included in a response Content-Type + header. Set to a callable for automatic character set detection. Default: "utf-8". + """ + + def __init__( + self, + *, + auth: AuthTypes | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + http1: bool = True, + http2: bool = False, + proxy: ProxyTypes | None = None, + mounts: None | (typing.Mapping[str, AsyncBaseTransport | None]) = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + follow_redirects: bool = False, + limits: Limits = DEFAULT_LIMITS, + max_redirects: int = DEFAULT_MAX_REDIRECTS, + event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, + base_url: URL | str = "", + transport: AsyncBaseTransport | None = None, + trust_env: bool = True, + default_encoding: str | typing.Callable[[bytes], str] = "utf-8", + ) -> None: + super().__init__( + auth=auth, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + follow_redirects=follow_redirects, + max_redirects=max_redirects, + event_hooks=event_hooks, + base_url=base_url, + trust_env=trust_env, + default_encoding=default_encoding, + ) + + if http2: + try: + import h2 # noqa + except ImportError: # pragma: no cover + raise ImportError( + "Using http2=True, but the 'h2' package is not installed. " + "Make sure to install httpx using `pip install httpx[http2]`." + ) from None + + allow_env_proxies = trust_env and transport is None + proxy_map = self._get_proxy_map(proxy, allow_env_proxies) + + self._transport = self._init_transport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + transport=transport, + ) + + self._mounts: dict[URLPattern, AsyncBaseTransport | None] = { + URLPattern(key): None + if proxy is None + else self._init_proxy_transport( + proxy, + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + ) + for key, proxy in proxy_map.items() + } + if mounts is not None: + self._mounts.update( + {URLPattern(key): transport for key, transport in mounts.items()} + ) + self._mounts = dict(sorted(self._mounts.items())) + + def _init_transport( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + transport: AsyncBaseTransport | None = None, + ) -> AsyncBaseTransport: + if transport is not None: + return transport + + return AsyncHTTPTransport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + ) + + def _init_proxy_transport( + self, + proxy: Proxy, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + ) -> AsyncBaseTransport: + return AsyncHTTPTransport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + proxy=proxy, + ) + + def _transport_for_url(self, url: URL) -> AsyncBaseTransport: + """ + Returns the transport instance that should be used for a given URL. + This will either be the standard connection pool, or a proxy. + """ + for pattern, transport in self._mounts.items(): + if pattern.matches(url): + return self._transport if transport is None else transport + + return self._transport + + async def request( + self, + method: str, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Build and send a request. + + Equivalent to: + + ```python + request = client.build_request(...) + response = await client.send(request, ...) + ``` + + See `AsyncClient.build_request()`, `AsyncClient.send()` + and [Merging of configuration][0] for how the various parameters + are merged with client-level configuration. + + [0]: /advanced/clients/#merging-of-configuration + """ + + if cookies is not None: # pragma: no cover + message = ( + "Setting per-request cookies=<...> is being deprecated, because " + "the expected behaviour on cookie persistence is ambiguous. Set " + "cookies directly on the client instance instead." + ) + warnings.warn(message, DeprecationWarning, stacklevel=2) + + request = self.build_request( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + extensions=extensions, + ) + return await self.send(request, auth=auth, follow_redirects=follow_redirects) + + @asynccontextmanager + async def stream( + self, + method: str, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> typing.AsyncIterator[Response]: + """ + Alternative to `httpx.request()` that streams the response body + instead of loading it into memory at once. + + **Parameters**: See `httpx.request`. + + See also: [Streaming Responses][0] + + [0]: /quickstart#streaming-responses + """ + request = self.build_request( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + extensions=extensions, + ) + response = await self.send( + request=request, + auth=auth, + follow_redirects=follow_redirects, + stream=True, + ) + try: + yield response + finally: + await response.aclose() + + async def send( + self, + request: Request, + *, + stream: bool = False, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + ) -> Response: + """ + Send a request. + + The request is sent as-is, unmodified. + + Typically you'll want to build one with `AsyncClient.build_request()` + so that any client-level configuration is merged into the request, + but passing an explicit `httpx.Request()` is supported as well. + + See also: [Request instances][0] + + [0]: /advanced/clients/#request-instances + """ + if self._state == ClientState.CLOSED: + raise RuntimeError("Cannot send a request, as the client has been closed.") + + self._state = ClientState.OPENED + follow_redirects = ( + self.follow_redirects + if isinstance(follow_redirects, UseClientDefault) + else follow_redirects + ) + + self._set_timeout(request) + + auth = self._build_request_auth(request, auth) + + response = await self._send_handling_auth( + request, + auth=auth, + follow_redirects=follow_redirects, + history=[], + ) + try: + if not stream: + await response.aread() + + return response + + except BaseException as exc: + await response.aclose() + raise exc + + async def _send_handling_auth( + self, + request: Request, + auth: Auth, + follow_redirects: bool, + history: list[Response], + ) -> Response: + auth_flow = auth.async_auth_flow(request) + try: + request = await auth_flow.__anext__() + + while True: + response = await self._send_handling_redirects( + request, + follow_redirects=follow_redirects, + history=history, + ) + try: + try: + next_request = await auth_flow.asend(response) + except StopAsyncIteration: + return response + + response.history = list(history) + await response.aread() + request = next_request + history.append(response) + + except BaseException as exc: + await response.aclose() + raise exc + finally: + await auth_flow.aclose() + + async def _send_handling_redirects( + self, + request: Request, + follow_redirects: bool, + history: list[Response], + ) -> Response: + while True: + if len(history) > self.max_redirects: + raise TooManyRedirects( + "Exceeded maximum allowed redirects.", request=request + ) + + for hook in self._event_hooks["request"]: + await hook(request) + + response = await self._send_single_request(request) + try: + for hook in self._event_hooks["response"]: + await hook(response) + + response.history = list(history) + + if not response.has_redirect_location: + return response + + request = self._build_redirect_request(request, response) + history = history + [response] + + if follow_redirects: + await response.aread() + else: + response.next_request = request + return response + + except BaseException as exc: + await response.aclose() + raise exc + + async def _send_single_request(self, request: Request) -> Response: + """ + Sends a single request, without handling any redirections. + """ + transport = self._transport_for_url(request.url) + start = time.perf_counter() + + if not isinstance(request.stream, AsyncByteStream): + raise RuntimeError( + "Attempted to send an sync request with an AsyncClient instance." + ) + + with request_context(request=request): + response = await transport.handle_async_request(request) + + assert isinstance(response.stream, AsyncByteStream) + response.request = request + response.stream = BoundAsyncStream( + response.stream, response=response, start=start + ) + self.cookies.extract_cookies(response) + response.default_encoding = self._default_encoding + + logger.info( + 'HTTP Request: %s %s "%s %d %s"', + request.method, + request.url, + response.http_version, + response.status_code, + response.reason_phrase, + ) + + return response + + async def get( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `GET` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "GET", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def options( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send an `OPTIONS` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "OPTIONS", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def head( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `HEAD` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "HEAD", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def post( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `POST` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "POST", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def put( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `PUT` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "PUT", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def patch( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `PATCH` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "PATCH", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def delete( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `DELETE` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "DELETE", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def aclose(self) -> None: + """ + Close transport and proxies. + """ + if self._state != ClientState.CLOSED: + self._state = ClientState.CLOSED + + await self._transport.aclose() + for proxy in self._mounts.values(): + if proxy is not None: + await proxy.aclose() + + async def __aenter__(self: U) -> U: + if self._state != ClientState.UNOPENED: + msg = { + ClientState.OPENED: "Cannot open a client instance more than once.", + ClientState.CLOSED: ( + "Cannot reopen a client instance, once it has been closed." + ), + }[self._state] + raise RuntimeError(msg) + + self._state = ClientState.OPENED + + await self._transport.__aenter__() + for proxy in self._mounts.values(): + if proxy is not None: + await proxy.__aenter__() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + self._state = ClientState.CLOSED + + await self._transport.__aexit__(exc_type, exc_value, traceback) + for proxy in self._mounts.values(): + if proxy is not None: + await proxy.__aexit__(exc_type, exc_value, traceback) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_config.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_config.py new file mode 100644 index 0000000000000000000000000000000000000000..467a6c90ae269babe3af7963d9d7c78b9f012268 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_config.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import os +import typing + +from ._models import Headers +from ._types import CertTypes, HeaderTypes, TimeoutTypes +from ._urls import URL + +if typing.TYPE_CHECKING: + import ssl # pragma: no cover + +__all__ = ["Limits", "Proxy", "Timeout", "create_ssl_context"] + + +class UnsetType: + pass # pragma: no cover + + +UNSET = UnsetType() + + +def create_ssl_context( + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, +) -> ssl.SSLContext: + import ssl + import warnings + + import certifi + + if verify is True: + if trust_env and os.environ.get("SSL_CERT_FILE"): # pragma: nocover + ctx = ssl.create_default_context(cafile=os.environ["SSL_CERT_FILE"]) + elif trust_env and os.environ.get("SSL_CERT_DIR"): # pragma: nocover + ctx = ssl.create_default_context(capath=os.environ["SSL_CERT_DIR"]) + else: + # Default case... + ctx = ssl.create_default_context(cafile=certifi.where()) + elif verify is False: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + elif isinstance(verify, str): # pragma: nocover + message = ( + "`verify=` is deprecated. " + "Use `verify=ssl.create_default_context(cafile=...)` " + "or `verify=ssl.create_default_context(capath=...)` instead." + ) + warnings.warn(message, DeprecationWarning) + if os.path.isdir(verify): + return ssl.create_default_context(capath=verify) + return ssl.create_default_context(cafile=verify) + else: + ctx = verify + + if cert: # pragma: nocover + message = ( + "`cert=...` is deprecated. Use `verify=` instead," + "with `.load_cert_chain()` to configure the certificate chain." + ) + warnings.warn(message, DeprecationWarning) + if isinstance(cert, str): + ctx.load_cert_chain(cert) + else: + ctx.load_cert_chain(*cert) + + return ctx + + +class Timeout: + """ + Timeout configuration. + + **Usage**: + + Timeout(None) # No timeouts. + Timeout(5.0) # 5s timeout on all operations. + Timeout(None, connect=5.0) # 5s timeout on connect, no other timeouts. + Timeout(5.0, connect=10.0) # 10s timeout on connect. 5s timeout elsewhere. + Timeout(5.0, pool=None) # No timeout on acquiring connection from pool. + # 5s timeout elsewhere. + """ + + def __init__( + self, + timeout: TimeoutTypes | UnsetType = UNSET, + *, + connect: None | float | UnsetType = UNSET, + read: None | float | UnsetType = UNSET, + write: None | float | UnsetType = UNSET, + pool: None | float | UnsetType = UNSET, + ) -> None: + if isinstance(timeout, Timeout): + # Passed as a single explicit Timeout. + assert connect is UNSET + assert read is UNSET + assert write is UNSET + assert pool is UNSET + self.connect = timeout.connect # type: typing.Optional[float] + self.read = timeout.read # type: typing.Optional[float] + self.write = timeout.write # type: typing.Optional[float] + self.pool = timeout.pool # type: typing.Optional[float] + elif isinstance(timeout, tuple): + # Passed as a tuple. + self.connect = timeout[0] + self.read = timeout[1] + self.write = None if len(timeout) < 3 else timeout[2] + self.pool = None if len(timeout) < 4 else timeout[3] + elif not ( + isinstance(connect, UnsetType) + or isinstance(read, UnsetType) + or isinstance(write, UnsetType) + or isinstance(pool, UnsetType) + ): + self.connect = connect + self.read = read + self.write = write + self.pool = pool + else: + if isinstance(timeout, UnsetType): + raise ValueError( + "httpx.Timeout must either include a default, or set all " + "four parameters explicitly." + ) + self.connect = timeout if isinstance(connect, UnsetType) else connect + self.read = timeout if isinstance(read, UnsetType) else read + self.write = timeout if isinstance(write, UnsetType) else write + self.pool = timeout if isinstance(pool, UnsetType) else pool + + def as_dict(self) -> dict[str, float | None]: + return { + "connect": self.connect, + "read": self.read, + "write": self.write, + "pool": self.pool, + } + + def __eq__(self, other: typing.Any) -> bool: + return ( + isinstance(other, self.__class__) + and self.connect == other.connect + and self.read == other.read + and self.write == other.write + and self.pool == other.pool + ) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + if len({self.connect, self.read, self.write, self.pool}) == 1: + return f"{class_name}(timeout={self.connect})" + return ( + f"{class_name}(connect={self.connect}, " + f"read={self.read}, write={self.write}, pool={self.pool})" + ) + + +class Limits: + """ + Configuration for limits to various client behaviors. + + **Parameters:** + + * **max_connections** - The maximum number of concurrent connections that may be + established. + * **max_keepalive_connections** - Allow the connection pool to maintain + keep-alive connections below this point. Should be less than or equal + to `max_connections`. + * **keepalive_expiry** - Time limit on idle keep-alive connections in seconds. + """ + + def __init__( + self, + *, + max_connections: int | None = None, + max_keepalive_connections: int | None = None, + keepalive_expiry: float | None = 5.0, + ) -> None: + self.max_connections = max_connections + self.max_keepalive_connections = max_keepalive_connections + self.keepalive_expiry = keepalive_expiry + + def __eq__(self, other: typing.Any) -> bool: + return ( + isinstance(other, self.__class__) + and self.max_connections == other.max_connections + and self.max_keepalive_connections == other.max_keepalive_connections + and self.keepalive_expiry == other.keepalive_expiry + ) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + return ( + f"{class_name}(max_connections={self.max_connections}, " + f"max_keepalive_connections={self.max_keepalive_connections}, " + f"keepalive_expiry={self.keepalive_expiry})" + ) + + +class Proxy: + def __init__( + self, + url: URL | str, + *, + ssl_context: ssl.SSLContext | None = None, + auth: tuple[str, str] | None = None, + headers: HeaderTypes | None = None, + ) -> None: + url = URL(url) + headers = Headers(headers) + + if url.scheme not in ("http", "https", "socks5", "socks5h"): + raise ValueError(f"Unknown scheme for proxy URL {url!r}") + + if url.username or url.password: + # Remove any auth credentials from the URL. + auth = (url.username, url.password) + url = url.copy_with(username=None, password=None) + + self.url = url + self.auth = auth + self.headers = headers + self.ssl_context = ssl_context + + @property + def raw_auth(self) -> tuple[bytes, bytes] | None: + # The proxy authentication as raw bytes. + return ( + None + if self.auth is None + else (self.auth[0].encode("utf-8"), self.auth[1].encode("utf-8")) + ) + + def __repr__(self) -> str: + # The authentication is represented with the password component masked. + auth = (self.auth[0], "********") if self.auth else None + + # Build a nice concise representation. + url_str = f"{str(self.url)!r}" + auth_str = f", auth={auth!r}" if auth else "" + headers_str = f", headers={dict(self.headers)!r}" if self.headers else "" + return f"Proxy({url_str}{auth_str}{headers_str})" + + +DEFAULT_TIMEOUT_CONFIG = Timeout(timeout=5.0) +DEFAULT_LIMITS = Limits(max_connections=100, max_keepalive_connections=20) +DEFAULT_MAX_REDIRECTS = 20 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_content.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_content.py new file mode 100644 index 0000000000000000000000000000000000000000..6f479a0885f723b7395843d41164a87041820776 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_content.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import inspect +import warnings +from json import dumps as json_dumps +from typing import ( + Any, + AsyncIterable, + AsyncIterator, + Iterable, + Iterator, + Mapping, +) +from urllib.parse import urlencode + +from ._exceptions import StreamClosed, StreamConsumed +from ._multipart import MultipartStream +from ._types import ( + AsyncByteStream, + RequestContent, + RequestData, + RequestFiles, + ResponseContent, + SyncByteStream, +) +from ._utils import peek_filelike_length, primitive_value_to_str + +__all__ = ["ByteStream"] + + +class ByteStream(AsyncByteStream, SyncByteStream): + def __init__(self, stream: bytes) -> None: + self._stream = stream + + def __iter__(self) -> Iterator[bytes]: + yield self._stream + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield self._stream + + +class IteratorByteStream(SyncByteStream): + CHUNK_SIZE = 65_536 + + def __init__(self, stream: Iterable[bytes]) -> None: + self._stream = stream + self._is_stream_consumed = False + self._is_generator = inspect.isgenerator(stream) + + def __iter__(self) -> Iterator[bytes]: + if self._is_stream_consumed and self._is_generator: + raise StreamConsumed() + + self._is_stream_consumed = True + if hasattr(self._stream, "read"): + # File-like interfaces should use 'read' directly. + chunk = self._stream.read(self.CHUNK_SIZE) + while chunk: + yield chunk + chunk = self._stream.read(self.CHUNK_SIZE) + else: + # Otherwise iterate. + for part in self._stream: + yield part + + +class AsyncIteratorByteStream(AsyncByteStream): + CHUNK_SIZE = 65_536 + + def __init__(self, stream: AsyncIterable[bytes]) -> None: + self._stream = stream + self._is_stream_consumed = False + self._is_generator = inspect.isasyncgen(stream) + + async def __aiter__(self) -> AsyncIterator[bytes]: + if self._is_stream_consumed and self._is_generator: + raise StreamConsumed() + + self._is_stream_consumed = True + if hasattr(self._stream, "aread"): + # File-like interfaces should use 'aread' directly. + chunk = await self._stream.aread(self.CHUNK_SIZE) + while chunk: + yield chunk + chunk = await self._stream.aread(self.CHUNK_SIZE) + else: + # Otherwise iterate. + async for part in self._stream: + yield part + + +class UnattachedStream(AsyncByteStream, SyncByteStream): + """ + If a request or response is serialized using pickle, then it is no longer + attached to a stream for I/O purposes. Any stream operations should result + in `httpx.StreamClosed`. + """ + + def __iter__(self) -> Iterator[bytes]: + raise StreamClosed() + + async def __aiter__(self) -> AsyncIterator[bytes]: + raise StreamClosed() + yield b"" # pragma: no cover + + +def encode_content( + content: str | bytes | Iterable[bytes] | AsyncIterable[bytes], +) -> tuple[dict[str, str], SyncByteStream | AsyncByteStream]: + if isinstance(content, (bytes, str)): + body = content.encode("utf-8") if isinstance(content, str) else content + content_length = len(body) + headers = {"Content-Length": str(content_length)} if body else {} + return headers, ByteStream(body) + + elif isinstance(content, Iterable) and not isinstance(content, dict): + # `not isinstance(content, dict)` is a bit oddly specific, but it + # catches a case that's easy for users to make in error, and would + # otherwise pass through here, like any other bytes-iterable, + # because `dict` happens to be iterable. See issue #2491. + content_length_or_none = peek_filelike_length(content) + + if content_length_or_none is None: + headers = {"Transfer-Encoding": "chunked"} + else: + headers = {"Content-Length": str(content_length_or_none)} + return headers, IteratorByteStream(content) # type: ignore + + elif isinstance(content, AsyncIterable): + headers = {"Transfer-Encoding": "chunked"} + return headers, AsyncIteratorByteStream(content) + + raise TypeError(f"Unexpected type for 'content', {type(content)!r}") + + +def encode_urlencoded_data( + data: RequestData, +) -> tuple[dict[str, str], ByteStream]: + plain_data = [] + for key, value in data.items(): + if isinstance(value, (list, tuple)): + plain_data.extend([(key, primitive_value_to_str(item)) for item in value]) + else: + plain_data.append((key, primitive_value_to_str(value))) + body = urlencode(plain_data, doseq=True).encode("utf-8") + content_length = str(len(body)) + content_type = "application/x-www-form-urlencoded" + headers = {"Content-Length": content_length, "Content-Type": content_type} + return headers, ByteStream(body) + + +def encode_multipart_data( + data: RequestData, files: RequestFiles, boundary: bytes | None +) -> tuple[dict[str, str], MultipartStream]: + multipart = MultipartStream(data=data, files=files, boundary=boundary) + headers = multipart.get_headers() + return headers, multipart + + +def encode_text(text: str) -> tuple[dict[str, str], ByteStream]: + body = text.encode("utf-8") + content_length = str(len(body)) + content_type = "text/plain; charset=utf-8" + headers = {"Content-Length": content_length, "Content-Type": content_type} + return headers, ByteStream(body) + + +def encode_html(html: str) -> tuple[dict[str, str], ByteStream]: + body = html.encode("utf-8") + content_length = str(len(body)) + content_type = "text/html; charset=utf-8" + headers = {"Content-Length": content_length, "Content-Type": content_type} + return headers, ByteStream(body) + + +def encode_json(json: Any) -> tuple[dict[str, str], ByteStream]: + body = json_dumps( + json, ensure_ascii=False, separators=(",", ":"), allow_nan=False + ).encode("utf-8") + content_length = str(len(body)) + content_type = "application/json" + headers = {"Content-Length": content_length, "Content-Type": content_type} + return headers, ByteStream(body) + + +def encode_request( + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: Any | None = None, + boundary: bytes | None = None, +) -> tuple[dict[str, str], SyncByteStream | AsyncByteStream]: + """ + Handles encoding the given `content`, `data`, `files`, and `json`, + returning a two-tuple of (, ). + """ + if data is not None and not isinstance(data, Mapping): + # We prefer to separate `content=` + # for raw request content, and `data=
` for url encoded or + # multipart form content. + # + # However for compat with requests, we *do* still support + # `data=` usages. We deal with that case here, treating it + # as if `content=<...>` had been supplied instead. + message = "Use 'content=<...>' to upload raw bytes/text content." + warnings.warn(message, DeprecationWarning, stacklevel=2) + return encode_content(data) + + if content is not None: + return encode_content(content) + elif files: + return encode_multipart_data(data or {}, files, boundary) + elif data: + return encode_urlencoded_data(data) + elif json is not None: + return encode_json(json) + + return {}, ByteStream(b"") + + +def encode_response( + content: ResponseContent | None = None, + text: str | None = None, + html: str | None = None, + json: Any | None = None, +) -> tuple[dict[str, str], SyncByteStream | AsyncByteStream]: + """ + Handles encoding the given `content`, returning a two-tuple of + (, ). + """ + if content is not None: + return encode_content(content) + elif text is not None: + return encode_text(text) + elif html is not None: + return encode_html(html) + elif json is not None: + return encode_json(json) + + return {}, ByteStream(b"") diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_decoders.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_decoders.py new file mode 100644 index 0000000000000000000000000000000000000000..899dfada878e1181fca6d3c75a79526a076abb9e --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_decoders.py @@ -0,0 +1,393 @@ +""" +Handlers for Content-Encoding. + +See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding +""" + +from __future__ import annotations + +import codecs +import io +import typing +import zlib + +from ._exceptions import DecodingError + +# Brotli support is optional +try: + # The C bindings in `brotli` are recommended for CPython. + import brotli +except ImportError: # pragma: no cover + try: + # The CFFI bindings in `brotlicffi` are recommended for PyPy + # and other environments. + import brotlicffi as brotli + except ImportError: + brotli = None + + +# Zstandard support is optional +try: + import zstandard +except ImportError: # pragma: no cover + zstandard = None # type: ignore + + +class ContentDecoder: + def decode(self, data: bytes) -> bytes: + raise NotImplementedError() # pragma: no cover + + def flush(self) -> bytes: + raise NotImplementedError() # pragma: no cover + + +class IdentityDecoder(ContentDecoder): + """ + Handle unencoded data. + """ + + def decode(self, data: bytes) -> bytes: + return data + + def flush(self) -> bytes: + return b"" + + +class DeflateDecoder(ContentDecoder): + """ + Handle 'deflate' decoding. + + See: https://stackoverflow.com/questions/1838699 + """ + + def __init__(self) -> None: + self.first_attempt = True + self.decompressor = zlib.decompressobj() + + def decode(self, data: bytes) -> bytes: + was_first_attempt = self.first_attempt + self.first_attempt = False + try: + return self.decompressor.decompress(data) + except zlib.error as exc: + if was_first_attempt: + self.decompressor = zlib.decompressobj(-zlib.MAX_WBITS) + return self.decode(data) + raise DecodingError(str(exc)) from exc + + def flush(self) -> bytes: + try: + return self.decompressor.flush() + except zlib.error as exc: # pragma: no cover + raise DecodingError(str(exc)) from exc + + +class GZipDecoder(ContentDecoder): + """ + Handle 'gzip' decoding. + + See: https://stackoverflow.com/questions/1838699 + """ + + def __init__(self) -> None: + self.decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16) + + def decode(self, data: bytes) -> bytes: + try: + return self.decompressor.decompress(data) + except zlib.error as exc: + raise DecodingError(str(exc)) from exc + + def flush(self) -> bytes: + try: + return self.decompressor.flush() + except zlib.error as exc: # pragma: no cover + raise DecodingError(str(exc)) from exc + + +class BrotliDecoder(ContentDecoder): + """ + Handle 'brotli' decoding. + + Requires `pip install brotlipy`. See: https://brotlipy.readthedocs.io/ + or `pip install brotli`. See https://github.com/google/brotli + Supports both 'brotlipy' and 'Brotli' packages since they share an import + name. The top branches are for 'brotlipy' and bottom branches for 'Brotli' + """ + + def __init__(self) -> None: + if brotli is None: # pragma: no cover + raise ImportError( + "Using 'BrotliDecoder', but neither of the 'brotlicffi' or 'brotli' " + "packages have been installed. " + "Make sure to install httpx using `pip install httpx[brotli]`." + ) from None + + self.decompressor = brotli.Decompressor() + self.seen_data = False + self._decompress: typing.Callable[[bytes], bytes] + if hasattr(self.decompressor, "decompress"): + # The 'brotlicffi' package. + self._decompress = self.decompressor.decompress # pragma: no cover + else: + # The 'brotli' package. + self._decompress = self.decompressor.process # pragma: no cover + + def decode(self, data: bytes) -> bytes: + if not data: + return b"" + self.seen_data = True + try: + return self._decompress(data) + except brotli.error as exc: + raise DecodingError(str(exc)) from exc + + def flush(self) -> bytes: + if not self.seen_data: + return b"" + try: + if hasattr(self.decompressor, "finish"): + # Only available in the 'brotlicffi' package. + + # As the decompressor decompresses eagerly, this + # will never actually emit any data. However, it will potentially throw + # errors if a truncated or damaged data stream has been used. + self.decompressor.finish() # pragma: no cover + return b"" + except brotli.error as exc: # pragma: no cover + raise DecodingError(str(exc)) from exc + + +class ZStandardDecoder(ContentDecoder): + """ + Handle 'zstd' RFC 8878 decoding. + + Requires `pip install zstandard`. + Can be installed as a dependency of httpx using `pip install httpx[zstd]`. + """ + + # inspired by the ZstdDecoder implementation in urllib3 + def __init__(self) -> None: + if zstandard is None: # pragma: no cover + raise ImportError( + "Using 'ZStandardDecoder', ..." + "Make sure to install httpx using `pip install httpx[zstd]`." + ) from None + + self.decompressor = zstandard.ZstdDecompressor().decompressobj() + self.seen_data = False + + def decode(self, data: bytes) -> bytes: + assert zstandard is not None + self.seen_data = True + output = io.BytesIO() + try: + output.write(self.decompressor.decompress(data)) + while self.decompressor.eof and self.decompressor.unused_data: + unused_data = self.decompressor.unused_data + self.decompressor = zstandard.ZstdDecompressor().decompressobj() + output.write(self.decompressor.decompress(unused_data)) + except zstandard.ZstdError as exc: + raise DecodingError(str(exc)) from exc + return output.getvalue() + + def flush(self) -> bytes: + if not self.seen_data: + return b"" + ret = self.decompressor.flush() # note: this is a no-op + if not self.decompressor.eof: + raise DecodingError("Zstandard data is incomplete") # pragma: no cover + return bytes(ret) + + +class MultiDecoder(ContentDecoder): + """ + Handle the case where multiple encodings have been applied. + """ + + def __init__(self, children: typing.Sequence[ContentDecoder]) -> None: + """ + 'children' should be a sequence of decoders in the order in which + each was applied. + """ + # Note that we reverse the order for decoding. + self.children = list(reversed(children)) + + def decode(self, data: bytes) -> bytes: + for child in self.children: + data = child.decode(data) + return data + + def flush(self) -> bytes: + data = b"" + for child in self.children: + data = child.decode(data) + child.flush() + return data + + +class ByteChunker: + """ + Handles returning byte content in fixed-size chunks. + """ + + def __init__(self, chunk_size: int | None = None) -> None: + self._buffer = io.BytesIO() + self._chunk_size = chunk_size + + def decode(self, content: bytes) -> list[bytes]: + if self._chunk_size is None: + return [content] if content else [] + + self._buffer.write(content) + if self._buffer.tell() >= self._chunk_size: + value = self._buffer.getvalue() + chunks = [ + value[i : i + self._chunk_size] + for i in range(0, len(value), self._chunk_size) + ] + if len(chunks[-1]) == self._chunk_size: + self._buffer.seek(0) + self._buffer.truncate() + return chunks + else: + self._buffer.seek(0) + self._buffer.write(chunks[-1]) + self._buffer.truncate() + return chunks[:-1] + else: + return [] + + def flush(self) -> list[bytes]: + value = self._buffer.getvalue() + self._buffer.seek(0) + self._buffer.truncate() + return [value] if value else [] + + +class TextChunker: + """ + Handles returning text content in fixed-size chunks. + """ + + def __init__(self, chunk_size: int | None = None) -> None: + self._buffer = io.StringIO() + self._chunk_size = chunk_size + + def decode(self, content: str) -> list[str]: + if self._chunk_size is None: + return [content] if content else [] + + self._buffer.write(content) + if self._buffer.tell() >= self._chunk_size: + value = self._buffer.getvalue() + chunks = [ + value[i : i + self._chunk_size] + for i in range(0, len(value), self._chunk_size) + ] + if len(chunks[-1]) == self._chunk_size: + self._buffer.seek(0) + self._buffer.truncate() + return chunks + else: + self._buffer.seek(0) + self._buffer.write(chunks[-1]) + self._buffer.truncate() + return chunks[:-1] + else: + return [] + + def flush(self) -> list[str]: + value = self._buffer.getvalue() + self._buffer.seek(0) + self._buffer.truncate() + return [value] if value else [] + + +class TextDecoder: + """ + Handles incrementally decoding bytes into text + """ + + def __init__(self, encoding: str = "utf-8") -> None: + self.decoder = codecs.getincrementaldecoder(encoding)(errors="replace") + + def decode(self, data: bytes) -> str: + return self.decoder.decode(data) + + def flush(self) -> str: + return self.decoder.decode(b"", True) + + +class LineDecoder: + """ + Handles incrementally reading lines from text. + + Has the same behaviour as the stdllib splitlines, + but handling the input iteratively. + """ + + def __init__(self) -> None: + self.buffer: list[str] = [] + self.trailing_cr: bool = False + + def decode(self, text: str) -> list[str]: + # See https://docs.python.org/3/library/stdtypes.html#str.splitlines + NEWLINE_CHARS = "\n\r\x0b\x0c\x1c\x1d\x1e\x85\u2028\u2029" + + # We always push a trailing `\r` into the next decode iteration. + if self.trailing_cr: + text = "\r" + text + self.trailing_cr = False + if text.endswith("\r"): + self.trailing_cr = True + text = text[:-1] + + if not text: + # NOTE: the edge case input of empty text doesn't occur in practice, + # because other httpx internals filter out this value + return [] # pragma: no cover + + trailing_newline = text[-1] in NEWLINE_CHARS + lines = text.splitlines() + + if len(lines) == 1 and not trailing_newline: + # No new lines, buffer the input and continue. + self.buffer.append(lines[0]) + return [] + + if self.buffer: + # Include any existing buffer in the first portion of the + # splitlines result. + lines = ["".join(self.buffer) + lines[0]] + lines[1:] + self.buffer = [] + + if not trailing_newline: + # If the last segment of splitlines is not newline terminated, + # then drop it from our output and start a new buffer. + self.buffer = [lines.pop()] + + return lines + + def flush(self) -> list[str]: + if not self.buffer and not self.trailing_cr: + return [] + + lines = ["".join(self.buffer)] + self.buffer = [] + self.trailing_cr = False + return lines + + +SUPPORTED_DECODERS = { + "identity": IdentityDecoder, + "gzip": GZipDecoder, + "deflate": DeflateDecoder, + "br": BrotliDecoder, + "zstd": ZStandardDecoder, +} + + +if brotli is None: + SUPPORTED_DECODERS.pop("br") # pragma: no cover +if zstandard is None: + SUPPORTED_DECODERS.pop("zstd") # pragma: no cover diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_exceptions.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..77f45a6d3986d15626fc8a5fd459d6a3e0fbe466 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_exceptions.py @@ -0,0 +1,379 @@ +""" +Our exception hierarchy: + +* HTTPError + x RequestError + + TransportError + - TimeoutException + · ConnectTimeout + · ReadTimeout + · WriteTimeout + · PoolTimeout + - NetworkError + · ConnectError + · ReadError + · WriteError + · CloseError + - ProtocolError + · LocalProtocolError + · RemoteProtocolError + - ProxyError + - UnsupportedProtocol + + DecodingError + + TooManyRedirects + x HTTPStatusError +* InvalidURL +* CookieConflict +* StreamError + x StreamConsumed + x StreamClosed + x ResponseNotRead + x RequestNotRead +""" + +from __future__ import annotations + +import contextlib +import typing + +if typing.TYPE_CHECKING: + from ._models import Request, Response # pragma: no cover + +__all__ = [ + "CloseError", + "ConnectError", + "ConnectTimeout", + "CookieConflict", + "DecodingError", + "HTTPError", + "HTTPStatusError", + "InvalidURL", + "LocalProtocolError", + "NetworkError", + "PoolTimeout", + "ProtocolError", + "ProxyError", + "ReadError", + "ReadTimeout", + "RemoteProtocolError", + "RequestError", + "RequestNotRead", + "ResponseNotRead", + "StreamClosed", + "StreamConsumed", + "StreamError", + "TimeoutException", + "TooManyRedirects", + "TransportError", + "UnsupportedProtocol", + "WriteError", + "WriteTimeout", +] + + +class HTTPError(Exception): + """ + Base class for `RequestError` and `HTTPStatusError`. + + Useful for `try...except` blocks when issuing a request, + and then calling `.raise_for_status()`. + + For example: + + ``` + try: + response = httpx.get("https://www.example.com") + response.raise_for_status() + except httpx.HTTPError as exc: + print(f"HTTP Exception for {exc.request.url} - {exc}") + ``` + """ + + def __init__(self, message: str) -> None: + super().__init__(message) + self._request: Request | None = None + + @property + def request(self) -> Request: + if self._request is None: + raise RuntimeError("The .request property has not been set.") + return self._request + + @request.setter + def request(self, request: Request) -> None: + self._request = request + + +class RequestError(HTTPError): + """ + Base class for all exceptions that may occur when issuing a `.request()`. + """ + + def __init__(self, message: str, *, request: Request | None = None) -> None: + super().__init__(message) + # At the point an exception is raised we won't typically have a request + # instance to associate it with. + # + # The 'request_context' context manager is used within the Client and + # Response methods in order to ensure that any raised exceptions + # have a `.request` property set on them. + self._request = request + + +class TransportError(RequestError): + """ + Base class for all exceptions that occur at the level of the Transport API. + """ + + +# Timeout exceptions... + + +class TimeoutException(TransportError): + """ + The base class for timeout errors. + + An operation has timed out. + """ + + +class ConnectTimeout(TimeoutException): + """ + Timed out while connecting to the host. + """ + + +class ReadTimeout(TimeoutException): + """ + Timed out while receiving data from the host. + """ + + +class WriteTimeout(TimeoutException): + """ + Timed out while sending data to the host. + """ + + +class PoolTimeout(TimeoutException): + """ + Timed out waiting to acquire a connection from the pool. + """ + + +# Core networking exceptions... + + +class NetworkError(TransportError): + """ + The base class for network-related errors. + + An error occurred while interacting with the network. + """ + + +class ReadError(NetworkError): + """ + Failed to receive data from the network. + """ + + +class WriteError(NetworkError): + """ + Failed to send data through the network. + """ + + +class ConnectError(NetworkError): + """ + Failed to establish a connection. + """ + + +class CloseError(NetworkError): + """ + Failed to close a connection. + """ + + +# Other transport exceptions... + + +class ProxyError(TransportError): + """ + An error occurred while establishing a proxy connection. + """ + + +class UnsupportedProtocol(TransportError): + """ + Attempted to make a request to an unsupported protocol. + + For example issuing a request to `ftp://www.example.com`. + """ + + +class ProtocolError(TransportError): + """ + The protocol was violated. + """ + + +class LocalProtocolError(ProtocolError): + """ + A protocol was violated by the client. + + For example if the user instantiated a `Request` instance explicitly, + failed to include the mandatory `Host:` header, and then issued it directly + using `client.send()`. + """ + + +class RemoteProtocolError(ProtocolError): + """ + The protocol was violated by the server. + + For example, returning malformed HTTP. + """ + + +# Other request exceptions... + + +class DecodingError(RequestError): + """ + Decoding of the response failed, due to a malformed encoding. + """ + + +class TooManyRedirects(RequestError): + """ + Too many redirects. + """ + + +# Client errors + + +class HTTPStatusError(HTTPError): + """ + The response had an error HTTP status of 4xx or 5xx. + + May be raised when calling `response.raise_for_status()` + """ + + def __init__(self, message: str, *, request: Request, response: Response) -> None: + super().__init__(message) + self.request = request + self.response = response + + +class InvalidURL(Exception): + """ + URL is improperly formed or cannot be parsed. + """ + + def __init__(self, message: str) -> None: + super().__init__(message) + + +class CookieConflict(Exception): + """ + Attempted to lookup a cookie by name, but multiple cookies existed. + + Can occur when calling `response.cookies.get(...)`. + """ + + def __init__(self, message: str) -> None: + super().__init__(message) + + +# Stream exceptions... + +# These may occur as the result of a programming error, by accessing +# the request/response stream in an invalid manner. + + +class StreamError(RuntimeError): + """ + The base class for stream exceptions. + + The developer made an error in accessing the request stream in + an invalid way. + """ + + def __init__(self, message: str) -> None: + super().__init__(message) + + +class StreamConsumed(StreamError): + """ + Attempted to read or stream content, but the content has already + been streamed. + """ + + def __init__(self) -> None: + message = ( + "Attempted to read or stream some content, but the content has " + "already been streamed. For requests, this could be due to passing " + "a generator as request content, and then receiving a redirect " + "response or a secondary request as part of an authentication flow." + "For responses, this could be due to attempting to stream the response " + "content more than once." + ) + super().__init__(message) + + +class StreamClosed(StreamError): + """ + Attempted to read or stream response content, but the request has been + closed. + """ + + def __init__(self) -> None: + message = ( + "Attempted to read or stream content, but the stream has " "been closed." + ) + super().__init__(message) + + +class ResponseNotRead(StreamError): + """ + Attempted to access streaming response content, without having called `read()`. + """ + + def __init__(self) -> None: + message = ( + "Attempted to access streaming response content," + " without having called `read()`." + ) + super().__init__(message) + + +class RequestNotRead(StreamError): + """ + Attempted to access streaming request content, without having called `read()`. + """ + + def __init__(self) -> None: + message = ( + "Attempted to access streaming request content," + " without having called `read()`." + ) + super().__init__(message) + + +@contextlib.contextmanager +def request_context( + request: Request | None = None, +) -> typing.Iterator[None]: + """ + A context manager that can be used to attach the given request context + to any `RequestError` exceptions that are raised within the block. + """ + try: + yield + except RequestError as exc: + if request is not None: + exc.request = request + raise exc diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_main.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_main.py new file mode 100644 index 0000000000000000000000000000000000000000..cffa4bb7db0f930f4db56653a061c4d7400ba4e6 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_main.py @@ -0,0 +1,506 @@ +from __future__ import annotations + +import functools +import json +import sys +import typing + +import click +import pygments.lexers +import pygments.util +import rich.console +import rich.markup +import rich.progress +import rich.syntax +import rich.table + +from ._client import Client +from ._exceptions import RequestError +from ._models import Response +from ._status_codes import codes + +if typing.TYPE_CHECKING: + import httpcore # pragma: no cover + + +def print_help() -> None: + console = rich.console.Console() + + console.print("[bold]HTTPX :butterfly:", justify="center") + console.print() + console.print("A next generation HTTP client.", justify="center") + console.print() + console.print( + "Usage: [bold]httpx[/bold] [cyan] [OPTIONS][/cyan] ", justify="left" + ) + console.print() + + table = rich.table.Table.grid(padding=1, pad_edge=True) + table.add_column("Parameter", no_wrap=True, justify="left", style="bold") + table.add_column("Description") + table.add_row( + "-m, --method [cyan]METHOD", + "Request method, such as GET, POST, PUT, PATCH, DELETE, OPTIONS, HEAD.\n" + "[Default: GET, or POST if a request body is included]", + ) + table.add_row( + "-p, --params [cyan] ...", + "Query parameters to include in the request URL.", + ) + table.add_row( + "-c, --content [cyan]TEXT", "Byte content to include in the request body." + ) + table.add_row( + "-d, --data [cyan] ...", "Form data to include in the request body." + ) + table.add_row( + "-f, --files [cyan] ...", + "Form files to include in the request body.", + ) + table.add_row("-j, --json [cyan]TEXT", "JSON data to include in the request body.") + table.add_row( + "-h, --headers [cyan] ...", + "Include additional HTTP headers in the request.", + ) + table.add_row( + "--cookies [cyan] ...", "Cookies to include in the request." + ) + table.add_row( + "--auth [cyan]", + "Username and password to include in the request. Specify '-' for the password" + " to use a password prompt. Note that using --verbose/-v will expose" + " the Authorization header, including the password encoding" + " in a trivially reversible format.", + ) + + table.add_row( + "--proxy [cyan]URL", + "Send the request via a proxy. Should be the URL giving the proxy address.", + ) + + table.add_row( + "--timeout [cyan]FLOAT", + "Timeout value to use for network operations, such as establishing the" + " connection, reading some data, etc... [Default: 5.0]", + ) + + table.add_row("--follow-redirects", "Automatically follow redirects.") + table.add_row("--no-verify", "Disable SSL verification.") + table.add_row( + "--http2", "Send the request using HTTP/2, if the remote server supports it." + ) + + table.add_row( + "--download [cyan]FILE", + "Save the response content as a file, rather than displaying it.", + ) + + table.add_row("-v, --verbose", "Verbose output. Show request as well as response.") + table.add_row("--help", "Show this message and exit.") + console.print(table) + + +def get_lexer_for_response(response: Response) -> str: + content_type = response.headers.get("Content-Type") + if content_type is not None: + mime_type, _, _ = content_type.partition(";") + try: + return typing.cast( + str, pygments.lexers.get_lexer_for_mimetype(mime_type.strip()).name + ) + except pygments.util.ClassNotFound: # pragma: no cover + pass + return "" # pragma: no cover + + +def format_request_headers(request: httpcore.Request, http2: bool = False) -> str: + version = "HTTP/2" if http2 else "HTTP/1.1" + headers = [ + (name.lower() if http2 else name, value) for name, value in request.headers + ] + method = request.method.decode("ascii") + target = request.url.target.decode("ascii") + lines = [f"{method} {target} {version}"] + [ + f"{name.decode('ascii')}: {value.decode('ascii')}" for name, value in headers + ] + return "\n".join(lines) + + +def format_response_headers( + http_version: bytes, + status: int, + reason_phrase: bytes | None, + headers: list[tuple[bytes, bytes]], +) -> str: + version = http_version.decode("ascii") + reason = ( + codes.get_reason_phrase(status) + if reason_phrase is None + else reason_phrase.decode("ascii") + ) + lines = [f"{version} {status} {reason}"] + [ + f"{name.decode('ascii')}: {value.decode('ascii')}" for name, value in headers + ] + return "\n".join(lines) + + +def print_request_headers(request: httpcore.Request, http2: bool = False) -> None: + console = rich.console.Console() + http_text = format_request_headers(request, http2=http2) + syntax = rich.syntax.Syntax(http_text, "http", theme="ansi_dark", word_wrap=True) + console.print(syntax) + syntax = rich.syntax.Syntax("", "http", theme="ansi_dark", word_wrap=True) + console.print(syntax) + + +def print_response_headers( + http_version: bytes, + status: int, + reason_phrase: bytes | None, + headers: list[tuple[bytes, bytes]], +) -> None: + console = rich.console.Console() + http_text = format_response_headers(http_version, status, reason_phrase, headers) + syntax = rich.syntax.Syntax(http_text, "http", theme="ansi_dark", word_wrap=True) + console.print(syntax) + syntax = rich.syntax.Syntax("", "http", theme="ansi_dark", word_wrap=True) + console.print(syntax) + + +def print_response(response: Response) -> None: + console = rich.console.Console() + lexer_name = get_lexer_for_response(response) + if lexer_name: + if lexer_name.lower() == "json": + try: + data = response.json() + text = json.dumps(data, indent=4) + except ValueError: # pragma: no cover + text = response.text + else: + text = response.text + + syntax = rich.syntax.Syntax(text, lexer_name, theme="ansi_dark", word_wrap=True) + console.print(syntax) + else: + console.print(f"<{len(response.content)} bytes of binary data>") + + +_PCTRTT = typing.Tuple[typing.Tuple[str, str], ...] +_PCTRTTT = typing.Tuple[_PCTRTT, ...] +_PeerCertRetDictType = typing.Dict[str, typing.Union[str, _PCTRTTT, _PCTRTT]] + + +def format_certificate(cert: _PeerCertRetDictType) -> str: # pragma: no cover + lines = [] + for key, value in cert.items(): + if isinstance(value, (list, tuple)): + lines.append(f"* {key}:") + for item in value: + if key in ("subject", "issuer"): + for sub_item in item: + lines.append(f"* {sub_item[0]}: {sub_item[1]!r}") + elif isinstance(item, tuple) and len(item) == 2: + lines.append(f"* {item[0]}: {item[1]!r}") + else: + lines.append(f"* {item!r}") + else: + lines.append(f"* {key}: {value!r}") + return "\n".join(lines) + + +def trace( + name: str, info: typing.Mapping[str, typing.Any], verbose: bool = False +) -> None: + console = rich.console.Console() + if name == "connection.connect_tcp.started" and verbose: + host = info["host"] + console.print(f"* Connecting to {host!r}") + elif name == "connection.connect_tcp.complete" and verbose: + stream = info["return_value"] + server_addr = stream.get_extra_info("server_addr") + console.print(f"* Connected to {server_addr[0]!r} on port {server_addr[1]}") + elif name == "connection.start_tls.complete" and verbose: # pragma: no cover + stream = info["return_value"] + ssl_object = stream.get_extra_info("ssl_object") + version = ssl_object.version() + cipher = ssl_object.cipher() + server_cert = ssl_object.getpeercert() + alpn = ssl_object.selected_alpn_protocol() + console.print(f"* SSL established using {version!r} / {cipher[0]!r}") + console.print(f"* Selected ALPN protocol: {alpn!r}") + if server_cert: + console.print("* Server certificate:") + console.print(format_certificate(server_cert)) + elif name == "http11.send_request_headers.started" and verbose: + request = info["request"] + print_request_headers(request, http2=False) + elif name == "http2.send_request_headers.started" and verbose: # pragma: no cover + request = info["request"] + print_request_headers(request, http2=True) + elif name == "http11.receive_response_headers.complete": + http_version, status, reason_phrase, headers = info["return_value"] + print_response_headers(http_version, status, reason_phrase, headers) + elif name == "http2.receive_response_headers.complete": # pragma: no cover + status, headers = info["return_value"] + http_version = b"HTTP/2" + reason_phrase = None + print_response_headers(http_version, status, reason_phrase, headers) + + +def download_response(response: Response, download: typing.BinaryIO) -> None: + console = rich.console.Console() + console.print() + content_length = response.headers.get("Content-Length") + with rich.progress.Progress( + "[progress.description]{task.description}", + "[progress.percentage]{task.percentage:>3.0f}%", + rich.progress.BarColumn(bar_width=None), + rich.progress.DownloadColumn(), + rich.progress.TransferSpeedColumn(), + ) as progress: + description = f"Downloading [bold]{rich.markup.escape(download.name)}" + download_task = progress.add_task( + description, + total=int(content_length or 0), + start=content_length is not None, + ) + for chunk in response.iter_bytes(): + download.write(chunk) + progress.update(download_task, completed=response.num_bytes_downloaded) + + +def validate_json( + ctx: click.Context, + param: click.Option | click.Parameter, + value: typing.Any, +) -> typing.Any: + if value is None: + return None + + try: + return json.loads(value) + except json.JSONDecodeError: # pragma: no cover + raise click.BadParameter("Not valid JSON") + + +def validate_auth( + ctx: click.Context, + param: click.Option | click.Parameter, + value: typing.Any, +) -> typing.Any: + if value == (None, None): + return None + + username, password = value + if password == "-": # pragma: no cover + password = click.prompt("Password", hide_input=True) + return (username, password) + + +def handle_help( + ctx: click.Context, + param: click.Option | click.Parameter, + value: typing.Any, +) -> None: + if not value or ctx.resilient_parsing: + return + + print_help() + ctx.exit() + + +@click.command(add_help_option=False) +@click.argument("url", type=str) +@click.option( + "--method", + "-m", + "method", + type=str, + help=( + "Request method, such as GET, POST, PUT, PATCH, DELETE, OPTIONS, HEAD. " + "[Default: GET, or POST if a request body is included]" + ), +) +@click.option( + "--params", + "-p", + "params", + type=(str, str), + multiple=True, + help="Query parameters to include in the request URL.", +) +@click.option( + "--content", + "-c", + "content", + type=str, + help="Byte content to include in the request body.", +) +@click.option( + "--data", + "-d", + "data", + type=(str, str), + multiple=True, + help="Form data to include in the request body.", +) +@click.option( + "--files", + "-f", + "files", + type=(str, click.File(mode="rb")), + multiple=True, + help="Form files to include in the request body.", +) +@click.option( + "--json", + "-j", + "json", + type=str, + callback=validate_json, + help="JSON data to include in the request body.", +) +@click.option( + "--headers", + "-h", + "headers", + type=(str, str), + multiple=True, + help="Include additional HTTP headers in the request.", +) +@click.option( + "--cookies", + "cookies", + type=(str, str), + multiple=True, + help="Cookies to include in the request.", +) +@click.option( + "--auth", + "auth", + type=(str, str), + default=(None, None), + callback=validate_auth, + help=( + "Username and password to include in the request. " + "Specify '-' for the password to use a password prompt. " + "Note that using --verbose/-v will expose the Authorization header, " + "including the password encoding in a trivially reversible format." + ), +) +@click.option( + "--proxy", + "proxy", + type=str, + default=None, + help="Send the request via a proxy. Should be the URL giving the proxy address.", +) +@click.option( + "--timeout", + "timeout", + type=float, + default=5.0, + help=( + "Timeout value to use for network operations, such as establishing the " + "connection, reading some data, etc... [Default: 5.0]" + ), +) +@click.option( + "--follow-redirects", + "follow_redirects", + is_flag=True, + default=False, + help="Automatically follow redirects.", +) +@click.option( + "--no-verify", + "verify", + is_flag=True, + default=True, + help="Disable SSL verification.", +) +@click.option( + "--http2", + "http2", + type=bool, + is_flag=True, + default=False, + help="Send the request using HTTP/2, if the remote server supports it.", +) +@click.option( + "--download", + type=click.File("wb"), + help="Save the response content as a file, rather than displaying it.", +) +@click.option( + "--verbose", + "-v", + type=bool, + is_flag=True, + default=False, + help="Verbose. Show request as well as response.", +) +@click.option( + "--help", + is_flag=True, + is_eager=True, + expose_value=False, + callback=handle_help, + help="Show this message and exit.", +) +def main( + url: str, + method: str, + params: list[tuple[str, str]], + content: str, + data: list[tuple[str, str]], + files: list[tuple[str, click.File]], + json: str, + headers: list[tuple[str, str]], + cookies: list[tuple[str, str]], + auth: tuple[str, str] | None, + proxy: str, + timeout: float, + follow_redirects: bool, + verify: bool, + http2: bool, + download: typing.BinaryIO | None, + verbose: bool, +) -> None: + """ + An HTTP command line client. + Sends a request and displays the response. + """ + if not method: + method = "POST" if content or data or files or json else "GET" + + try: + with Client(proxy=proxy, timeout=timeout, http2=http2, verify=verify) as client: + with client.stream( + method, + url, + params=list(params), + content=content, + data=dict(data), + files=files, # type: ignore + json=json, + headers=headers, + cookies=dict(cookies), + auth=auth, + follow_redirects=follow_redirects, + extensions={"trace": functools.partial(trace, verbose=verbose)}, + ) as response: + if download is not None: + download_response(response, download) + else: + response.read() + if response.content: + print_response(response) + + except RequestError as exc: + console = rich.console.Console() + console.print(f"[red]{type(exc).__name__}[/red]: {exc}") + sys.exit(1) + + sys.exit(0 if response.is_success else 1) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_models.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_models.py new file mode 100644 index 0000000000000000000000000000000000000000..67d74bf86bfc80e22d9a4a3153572845accd9039 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_models.py @@ -0,0 +1,1277 @@ +from __future__ import annotations + +import codecs +import datetime +import email.message +import json as jsonlib +import re +import typing +import urllib.request +from collections.abc import Mapping +from http.cookiejar import Cookie, CookieJar + +from ._content import ByteStream, UnattachedStream, encode_request, encode_response +from ._decoders import ( + SUPPORTED_DECODERS, + ByteChunker, + ContentDecoder, + IdentityDecoder, + LineDecoder, + MultiDecoder, + TextChunker, + TextDecoder, +) +from ._exceptions import ( + CookieConflict, + HTTPStatusError, + RequestNotRead, + ResponseNotRead, + StreamClosed, + StreamConsumed, + request_context, +) +from ._multipart import get_multipart_boundary_from_content_type +from ._status_codes import codes +from ._types import ( + AsyncByteStream, + CookieTypes, + HeaderTypes, + QueryParamTypes, + RequestContent, + RequestData, + RequestExtensions, + RequestFiles, + ResponseContent, + ResponseExtensions, + SyncByteStream, +) +from ._urls import URL +from ._utils import to_bytes_or_str, to_str + +__all__ = ["Cookies", "Headers", "Request", "Response"] + +SENSITIVE_HEADERS = {"authorization", "proxy-authorization"} + + +def _is_known_encoding(encoding: str) -> bool: + """ + Return `True` if `encoding` is a known codec. + """ + try: + codecs.lookup(encoding) + except LookupError: + return False + return True + + +def _normalize_header_key(key: str | bytes, encoding: str | None = None) -> bytes: + """ + Coerce str/bytes into a strictly byte-wise HTTP header key. + """ + return key if isinstance(key, bytes) else key.encode(encoding or "ascii") + + +def _normalize_header_value(value: str | bytes, encoding: str | None = None) -> bytes: + """ + Coerce str/bytes into a strictly byte-wise HTTP header value. + """ + if isinstance(value, bytes): + return value + if not isinstance(value, str): + raise TypeError(f"Header value must be str or bytes, not {type(value)}") + return value.encode(encoding or "ascii") + + +def _parse_content_type_charset(content_type: str) -> str | None: + # We used to use `cgi.parse_header()` here, but `cgi` became a dead battery. + # See: https://peps.python.org/pep-0594/#cgi + msg = email.message.Message() + msg["content-type"] = content_type + return msg.get_content_charset(failobj=None) + + +def _parse_header_links(value: str) -> list[dict[str, str]]: + """ + Returns a list of parsed link headers, for more info see: + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link + The generic syntax of those is: + Link: < uri-reference >; param1=value1; param2="value2" + So for instance: + Link; '; type="image/jpeg",;' + would return + [ + {"url": "http:/.../front.jpeg", "type": "image/jpeg"}, + {"url": "http://.../back.jpeg"}, + ] + :param value: HTTP Link entity-header field + :return: list of parsed link headers + """ + links: list[dict[str, str]] = [] + replace_chars = " '\"" + value = value.strip(replace_chars) + if not value: + return links + for val in re.split(", *<", value): + try: + url, params = val.split(";", 1) + except ValueError: + url, params = val, "" + link = {"url": url.strip("<> '\"")} + for param in params.split(";"): + try: + key, value = param.split("=") + except ValueError: + break + link[key.strip(replace_chars)] = value.strip(replace_chars) + links.append(link) + return links + + +def _obfuscate_sensitive_headers( + items: typing.Iterable[tuple[typing.AnyStr, typing.AnyStr]], +) -> typing.Iterator[tuple[typing.AnyStr, typing.AnyStr]]: + for k, v in items: + if to_str(k.lower()) in SENSITIVE_HEADERS: + v = to_bytes_or_str("[secure]", match_type_of=v) + yield k, v + + +class Headers(typing.MutableMapping[str, str]): + """ + HTTP headers, as a case-insensitive multi-dict. + """ + + def __init__( + self, + headers: HeaderTypes | None = None, + encoding: str | None = None, + ) -> None: + self._list = [] # type: typing.List[typing.Tuple[bytes, bytes, bytes]] + + if isinstance(headers, Headers): + self._list = list(headers._list) + elif isinstance(headers, Mapping): + for k, v in headers.items(): + bytes_key = _normalize_header_key(k, encoding) + bytes_value = _normalize_header_value(v, encoding) + self._list.append((bytes_key, bytes_key.lower(), bytes_value)) + elif headers is not None: + for k, v in headers: + bytes_key = _normalize_header_key(k, encoding) + bytes_value = _normalize_header_value(v, encoding) + self._list.append((bytes_key, bytes_key.lower(), bytes_value)) + + self._encoding = encoding + + @property + def encoding(self) -> str: + """ + Header encoding is mandated as ascii, but we allow fallbacks to utf-8 + or iso-8859-1. + """ + if self._encoding is None: + for encoding in ["ascii", "utf-8"]: + for key, value in self.raw: + try: + key.decode(encoding) + value.decode(encoding) + except UnicodeDecodeError: + break + else: + # The else block runs if 'break' did not occur, meaning + # all values fitted the encoding. + self._encoding = encoding + break + else: + # The ISO-8859-1 encoding covers all 256 code points in a byte, + # so will never raise decode errors. + self._encoding = "iso-8859-1" + return self._encoding + + @encoding.setter + def encoding(self, value: str) -> None: + self._encoding = value + + @property + def raw(self) -> list[tuple[bytes, bytes]]: + """ + Returns a list of the raw header items, as byte pairs. + """ + return [(raw_key, value) for raw_key, _, value in self._list] + + def keys(self) -> typing.KeysView[str]: + return {key.decode(self.encoding): None for _, key, value in self._list}.keys() + + def values(self) -> typing.ValuesView[str]: + values_dict: dict[str, str] = {} + for _, key, value in self._list: + str_key = key.decode(self.encoding) + str_value = value.decode(self.encoding) + if str_key in values_dict: + values_dict[str_key] += f", {str_value}" + else: + values_dict[str_key] = str_value + return values_dict.values() + + def items(self) -> typing.ItemsView[str, str]: + """ + Return `(key, value)` items of headers. Concatenate headers + into a single comma separated value when a key occurs multiple times. + """ + values_dict: dict[str, str] = {} + for _, key, value in self._list: + str_key = key.decode(self.encoding) + str_value = value.decode(self.encoding) + if str_key in values_dict: + values_dict[str_key] += f", {str_value}" + else: + values_dict[str_key] = str_value + return values_dict.items() + + def multi_items(self) -> list[tuple[str, str]]: + """ + Return a list of `(key, value)` pairs of headers. Allow multiple + occurrences of the same key without concatenating into a single + comma separated value. + """ + return [ + (key.decode(self.encoding), value.decode(self.encoding)) + for _, key, value in self._list + ] + + def get(self, key: str, default: typing.Any = None) -> typing.Any: + """ + Return a header value. If multiple occurrences of the header occur + then concatenate them together with commas. + """ + try: + return self[key] + except KeyError: + return default + + def get_list(self, key: str, split_commas: bool = False) -> list[str]: + """ + Return a list of all header values for a given key. + If `split_commas=True` is passed, then any comma separated header + values are split into multiple return strings. + """ + get_header_key = key.lower().encode(self.encoding) + + values = [ + item_value.decode(self.encoding) + for _, item_key, item_value in self._list + if item_key.lower() == get_header_key + ] + + if not split_commas: + return values + + split_values = [] + for value in values: + split_values.extend([item.strip() for item in value.split(",")]) + return split_values + + def update(self, headers: HeaderTypes | None = None) -> None: # type: ignore + headers = Headers(headers) + for key in headers.keys(): + if key in self: + self.pop(key) + self._list.extend(headers._list) + + def copy(self) -> Headers: + return Headers(self, encoding=self.encoding) + + def __getitem__(self, key: str) -> str: + """ + Return a single header value. + + If there are multiple headers with the same key, then we concatenate + them with commas. See: https://tools.ietf.org/html/rfc7230#section-3.2.2 + """ + normalized_key = key.lower().encode(self.encoding) + + items = [ + header_value.decode(self.encoding) + for _, header_key, header_value in self._list + if header_key == normalized_key + ] + + if items: + return ", ".join(items) + + raise KeyError(key) + + def __setitem__(self, key: str, value: str) -> None: + """ + Set the header `key` to `value`, removing any duplicate entries. + Retains insertion order. + """ + set_key = key.encode(self._encoding or "utf-8") + set_value = value.encode(self._encoding or "utf-8") + lookup_key = set_key.lower() + + found_indexes = [ + idx + for idx, (_, item_key, _) in enumerate(self._list) + if item_key == lookup_key + ] + + for idx in reversed(found_indexes[1:]): + del self._list[idx] + + if found_indexes: + idx = found_indexes[0] + self._list[idx] = (set_key, lookup_key, set_value) + else: + self._list.append((set_key, lookup_key, set_value)) + + def __delitem__(self, key: str) -> None: + """ + Remove the header `key`. + """ + del_key = key.lower().encode(self.encoding) + + pop_indexes = [ + idx + for idx, (_, item_key, _) in enumerate(self._list) + if item_key.lower() == del_key + ] + + if not pop_indexes: + raise KeyError(key) + + for idx in reversed(pop_indexes): + del self._list[idx] + + def __contains__(self, key: typing.Any) -> bool: + header_key = key.lower().encode(self.encoding) + return header_key in [key for _, key, _ in self._list] + + def __iter__(self) -> typing.Iterator[typing.Any]: + return iter(self.keys()) + + def __len__(self) -> int: + return len(self._list) + + def __eq__(self, other: typing.Any) -> bool: + try: + other_headers = Headers(other) + except ValueError: + return False + + self_list = [(key, value) for _, key, value in self._list] + other_list = [(key, value) for _, key, value in other_headers._list] + return sorted(self_list) == sorted(other_list) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + + encoding_str = "" + if self.encoding != "ascii": + encoding_str = f", encoding={self.encoding!r}" + + as_list = list(_obfuscate_sensitive_headers(self.multi_items())) + as_dict = dict(as_list) + + no_duplicate_keys = len(as_dict) == len(as_list) + if no_duplicate_keys: + return f"{class_name}({as_dict!r}{encoding_str})" + return f"{class_name}({as_list!r}{encoding_str})" + + +class Request: + def __init__( + self, + method: str, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + stream: SyncByteStream | AsyncByteStream | None = None, + extensions: RequestExtensions | None = None, + ) -> None: + self.method = method.upper() + self.url = URL(url) if params is None else URL(url, params=params) + self.headers = Headers(headers) + self.extensions = {} if extensions is None else dict(extensions) + + if cookies: + Cookies(cookies).set_cookie_header(self) + + if stream is None: + content_type: str | None = self.headers.get("content-type") + headers, stream = encode_request( + content=content, + data=data, + files=files, + json=json, + boundary=get_multipart_boundary_from_content_type( + content_type=content_type.encode(self.headers.encoding) + if content_type + else None + ), + ) + self._prepare(headers) + self.stream = stream + # Load the request body, except for streaming content. + if isinstance(stream, ByteStream): + self.read() + else: + # There's an important distinction between `Request(content=...)`, + # and `Request(stream=...)`. + # + # Using `content=...` implies automatically populated `Host` and content + # headers, of either `Content-Length: ...` or `Transfer-Encoding: chunked`. + # + # Using `stream=...` will not automatically include *any* + # auto-populated headers. + # + # As an end-user you don't really need `stream=...`. It's only + # useful when: + # + # * Preserving the request stream when copying requests, eg for redirects. + # * Creating request instances on the *server-side* of the transport API. + self.stream = stream + + def _prepare(self, default_headers: dict[str, str]) -> None: + for key, value in default_headers.items(): + # Ignore Transfer-Encoding if the Content-Length has been set explicitly. + if key.lower() == "transfer-encoding" and "Content-Length" in self.headers: + continue + self.headers.setdefault(key, value) + + auto_headers: list[tuple[bytes, bytes]] = [] + + has_host = "Host" in self.headers + has_content_length = ( + "Content-Length" in self.headers or "Transfer-Encoding" in self.headers + ) + + if not has_host and self.url.host: + auto_headers.append((b"Host", self.url.netloc)) + if not has_content_length and self.method in ("POST", "PUT", "PATCH"): + auto_headers.append((b"Content-Length", b"0")) + + self.headers = Headers(auto_headers + self.headers.raw) + + @property + def content(self) -> bytes: + if not hasattr(self, "_content"): + raise RequestNotRead() + return self._content + + def read(self) -> bytes: + """ + Read and return the request content. + """ + if not hasattr(self, "_content"): + assert isinstance(self.stream, typing.Iterable) + self._content = b"".join(self.stream) + if not isinstance(self.stream, ByteStream): + # If a streaming request has been read entirely into memory, then + # we can replace the stream with a raw bytes implementation, + # to ensure that any non-replayable streams can still be used. + self.stream = ByteStream(self._content) + return self._content + + async def aread(self) -> bytes: + """ + Read and return the request content. + """ + if not hasattr(self, "_content"): + assert isinstance(self.stream, typing.AsyncIterable) + self._content = b"".join([part async for part in self.stream]) + if not isinstance(self.stream, ByteStream): + # If a streaming request has been read entirely into memory, then + # we can replace the stream with a raw bytes implementation, + # to ensure that any non-replayable streams can still be used. + self.stream = ByteStream(self._content) + return self._content + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + url = str(self.url) + return f"<{class_name}({self.method!r}, {url!r})>" + + def __getstate__(self) -> dict[str, typing.Any]: + return { + name: value + for name, value in self.__dict__.items() + if name not in ["extensions", "stream"] + } + + def __setstate__(self, state: dict[str, typing.Any]) -> None: + for name, value in state.items(): + setattr(self, name, value) + self.extensions = {} + self.stream = UnattachedStream() + + +class Response: + def __init__( + self, + status_code: int, + *, + headers: HeaderTypes | None = None, + content: ResponseContent | None = None, + text: str | None = None, + html: str | None = None, + json: typing.Any = None, + stream: SyncByteStream | AsyncByteStream | None = None, + request: Request | None = None, + extensions: ResponseExtensions | None = None, + history: list[Response] | None = None, + default_encoding: str | typing.Callable[[bytes], str] = "utf-8", + ) -> None: + self.status_code = status_code + self.headers = Headers(headers) + + self._request: Request | None = request + + # When follow_redirects=False and a redirect is received, + # the client will set `response.next_request`. + self.next_request: Request | None = None + + self.extensions = {} if extensions is None else dict(extensions) + self.history = [] if history is None else list(history) + + self.is_closed = False + self.is_stream_consumed = False + + self.default_encoding = default_encoding + + if stream is None: + headers, stream = encode_response(content, text, html, json) + self._prepare(headers) + self.stream = stream + if isinstance(stream, ByteStream): + # Load the response body, except for streaming content. + self.read() + else: + # There's an important distinction between `Response(content=...)`, + # and `Response(stream=...)`. + # + # Using `content=...` implies automatically populated content headers, + # of either `Content-Length: ...` or `Transfer-Encoding: chunked`. + # + # Using `stream=...` will not automatically include any content headers. + # + # As an end-user you don't really need `stream=...`. It's only + # useful when creating response instances having received a stream + # from the transport API. + self.stream = stream + + self._num_bytes_downloaded = 0 + + def _prepare(self, default_headers: dict[str, str]) -> None: + for key, value in default_headers.items(): + # Ignore Transfer-Encoding if the Content-Length has been set explicitly. + if key.lower() == "transfer-encoding" and "content-length" in self.headers: + continue + self.headers.setdefault(key, value) + + @property + def elapsed(self) -> datetime.timedelta: + """ + Returns the time taken for the complete request/response + cycle to complete. + """ + if not hasattr(self, "_elapsed"): + raise RuntimeError( + "'.elapsed' may only be accessed after the response " + "has been read or closed." + ) + return self._elapsed + + @elapsed.setter + def elapsed(self, elapsed: datetime.timedelta) -> None: + self._elapsed = elapsed + + @property + def request(self) -> Request: + """ + Returns the request instance associated to the current response. + """ + if self._request is None: + raise RuntimeError( + "The request instance has not been set on this response." + ) + return self._request + + @request.setter + def request(self, value: Request) -> None: + self._request = value + + @property + def http_version(self) -> str: + try: + http_version: bytes = self.extensions["http_version"] + except KeyError: + return "HTTP/1.1" + else: + return http_version.decode("ascii", errors="ignore") + + @property + def reason_phrase(self) -> str: + try: + reason_phrase: bytes = self.extensions["reason_phrase"] + except KeyError: + return codes.get_reason_phrase(self.status_code) + else: + return reason_phrase.decode("ascii", errors="ignore") + + @property + def url(self) -> URL: + """ + Returns the URL for which the request was made. + """ + return self.request.url + + @property + def content(self) -> bytes: + if not hasattr(self, "_content"): + raise ResponseNotRead() + return self._content + + @property + def text(self) -> str: + if not hasattr(self, "_text"): + content = self.content + if not content: + self._text = "" + else: + decoder = TextDecoder(encoding=self.encoding or "utf-8") + self._text = "".join([decoder.decode(self.content), decoder.flush()]) + return self._text + + @property + def encoding(self) -> str | None: + """ + Return an encoding to use for decoding the byte content into text. + The priority for determining this is given by... + + * `.encoding = <>` has been set explicitly. + * The encoding as specified by the charset parameter in the Content-Type header. + * The encoding as determined by `default_encoding`, which may either be + a string like "utf-8" indicating the encoding to use, or may be a callable + which enables charset autodetection. + """ + if not hasattr(self, "_encoding"): + encoding = self.charset_encoding + if encoding is None or not _is_known_encoding(encoding): + if isinstance(self.default_encoding, str): + encoding = self.default_encoding + elif hasattr(self, "_content"): + encoding = self.default_encoding(self._content) + self._encoding = encoding or "utf-8" + return self._encoding + + @encoding.setter + def encoding(self, value: str) -> None: + """ + Set the encoding to use for decoding the byte content into text. + + If the `text` attribute has been accessed, attempting to set the + encoding will throw a ValueError. + """ + if hasattr(self, "_text"): + raise ValueError( + "Setting encoding after `text` has been accessed is not allowed." + ) + self._encoding = value + + @property + def charset_encoding(self) -> str | None: + """ + Return the encoding, as specified by the Content-Type header. + """ + content_type = self.headers.get("Content-Type") + if content_type is None: + return None + + return _parse_content_type_charset(content_type) + + def _get_content_decoder(self) -> ContentDecoder: + """ + Returns a decoder instance which can be used to decode the raw byte + content, depending on the Content-Encoding used in the response. + """ + if not hasattr(self, "_decoder"): + decoders: list[ContentDecoder] = [] + values = self.headers.get_list("content-encoding", split_commas=True) + for value in values: + value = value.strip().lower() + try: + decoder_cls = SUPPORTED_DECODERS[value] + decoders.append(decoder_cls()) + except KeyError: + continue + + if len(decoders) == 1: + self._decoder = decoders[0] + elif len(decoders) > 1: + self._decoder = MultiDecoder(children=decoders) + else: + self._decoder = IdentityDecoder() + + return self._decoder + + @property + def is_informational(self) -> bool: + """ + A property which is `True` for 1xx status codes, `False` otherwise. + """ + return codes.is_informational(self.status_code) + + @property + def is_success(self) -> bool: + """ + A property which is `True` for 2xx status codes, `False` otherwise. + """ + return codes.is_success(self.status_code) + + @property + def is_redirect(self) -> bool: + """ + A property which is `True` for 3xx status codes, `False` otherwise. + + Note that not all responses with a 3xx status code indicate a URL redirect. + + Use `response.has_redirect_location` to determine responses with a properly + formed URL redirection. + """ + return codes.is_redirect(self.status_code) + + @property + def is_client_error(self) -> bool: + """ + A property which is `True` for 4xx status codes, `False` otherwise. + """ + return codes.is_client_error(self.status_code) + + @property + def is_server_error(self) -> bool: + """ + A property which is `True` for 5xx status codes, `False` otherwise. + """ + return codes.is_server_error(self.status_code) + + @property + def is_error(self) -> bool: + """ + A property which is `True` for 4xx and 5xx status codes, `False` otherwise. + """ + return codes.is_error(self.status_code) + + @property + def has_redirect_location(self) -> bool: + """ + Returns True for 3xx responses with a properly formed URL redirection, + `False` otherwise. + """ + return ( + self.status_code + in ( + # 301 (Cacheable redirect. Method may change to GET.) + codes.MOVED_PERMANENTLY, + # 302 (Uncacheable redirect. Method may change to GET.) + codes.FOUND, + # 303 (Client should make a GET or HEAD request.) + codes.SEE_OTHER, + # 307 (Equiv. 302, but retain method) + codes.TEMPORARY_REDIRECT, + # 308 (Equiv. 301, but retain method) + codes.PERMANENT_REDIRECT, + ) + and "Location" in self.headers + ) + + def raise_for_status(self) -> Response: + """ + Raise the `HTTPStatusError` if one occurred. + """ + request = self._request + if request is None: + raise RuntimeError( + "Cannot call `raise_for_status` as the request " + "instance has not been set on this response." + ) + + if self.is_success: + return self + + if self.has_redirect_location: + message = ( + "{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n" + "Redirect location: '{0.headers[location]}'\n" + "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/{0.status_code}" + ) + else: + message = ( + "{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n" + "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/{0.status_code}" + ) + + status_class = self.status_code // 100 + error_types = { + 1: "Informational response", + 3: "Redirect response", + 4: "Client error", + 5: "Server error", + } + error_type = error_types.get(status_class, "Invalid status code") + message = message.format(self, error_type=error_type) + raise HTTPStatusError(message, request=request, response=self) + + def json(self, **kwargs: typing.Any) -> typing.Any: + return jsonlib.loads(self.content, **kwargs) + + @property + def cookies(self) -> Cookies: + if not hasattr(self, "_cookies"): + self._cookies = Cookies() + self._cookies.extract_cookies(self) + return self._cookies + + @property + def links(self) -> dict[str | None, dict[str, str]]: + """ + Returns the parsed header links of the response, if any + """ + header = self.headers.get("link") + if header is None: + return {} + + return { + (link.get("rel") or link.get("url")): link + for link in _parse_header_links(header) + } + + @property + def num_bytes_downloaded(self) -> int: + return self._num_bytes_downloaded + + def __repr__(self) -> str: + return f"" + + def __getstate__(self) -> dict[str, typing.Any]: + return { + name: value + for name, value in self.__dict__.items() + if name not in ["extensions", "stream", "is_closed", "_decoder"] + } + + def __setstate__(self, state: dict[str, typing.Any]) -> None: + for name, value in state.items(): + setattr(self, name, value) + self.is_closed = True + self.extensions = {} + self.stream = UnattachedStream() + + def read(self) -> bytes: + """ + Read and return the response content. + """ + if not hasattr(self, "_content"): + self._content = b"".join(self.iter_bytes()) + return self._content + + def iter_bytes(self, chunk_size: int | None = None) -> typing.Iterator[bytes]: + """ + A byte-iterator over the decoded response content. + This allows us to handle gzip, deflate, brotli, and zstd encoded responses. + """ + if hasattr(self, "_content"): + chunk_size = len(self._content) if chunk_size is None else chunk_size + for i in range(0, len(self._content), max(chunk_size, 1)): + yield self._content[i : i + chunk_size] + else: + decoder = self._get_content_decoder() + chunker = ByteChunker(chunk_size=chunk_size) + with request_context(request=self._request): + for raw_bytes in self.iter_raw(): + decoded = decoder.decode(raw_bytes) + for chunk in chunker.decode(decoded): + yield chunk + decoded = decoder.flush() + for chunk in chunker.decode(decoded): + yield chunk # pragma: no cover + for chunk in chunker.flush(): + yield chunk + + def iter_text(self, chunk_size: int | None = None) -> typing.Iterator[str]: + """ + A str-iterator over the decoded response content + that handles both gzip, deflate, etc but also detects the content's + string encoding. + """ + decoder = TextDecoder(encoding=self.encoding or "utf-8") + chunker = TextChunker(chunk_size=chunk_size) + with request_context(request=self._request): + for byte_content in self.iter_bytes(): + text_content = decoder.decode(byte_content) + for chunk in chunker.decode(text_content): + yield chunk + text_content = decoder.flush() + for chunk in chunker.decode(text_content): + yield chunk # pragma: no cover + for chunk in chunker.flush(): + yield chunk + + def iter_lines(self) -> typing.Iterator[str]: + decoder = LineDecoder() + with request_context(request=self._request): + for text in self.iter_text(): + for line in decoder.decode(text): + yield line + for line in decoder.flush(): + yield line + + def iter_raw(self, chunk_size: int | None = None) -> typing.Iterator[bytes]: + """ + A byte-iterator over the raw response content. + """ + if self.is_stream_consumed: + raise StreamConsumed() + if self.is_closed: + raise StreamClosed() + if not isinstance(self.stream, SyncByteStream): + raise RuntimeError("Attempted to call a sync iterator on an async stream.") + + self.is_stream_consumed = True + self._num_bytes_downloaded = 0 + chunker = ByteChunker(chunk_size=chunk_size) + + with request_context(request=self._request): + for raw_stream_bytes in self.stream: + self._num_bytes_downloaded += len(raw_stream_bytes) + for chunk in chunker.decode(raw_stream_bytes): + yield chunk + + for chunk in chunker.flush(): + yield chunk + + self.close() + + def close(self) -> None: + """ + Close the response and release the connection. + Automatically called if the response body is read to completion. + """ + if not isinstance(self.stream, SyncByteStream): + raise RuntimeError("Attempted to call an sync close on an async stream.") + + if not self.is_closed: + self.is_closed = True + with request_context(request=self._request): + self.stream.close() + + async def aread(self) -> bytes: + """ + Read and return the response content. + """ + if not hasattr(self, "_content"): + self._content = b"".join([part async for part in self.aiter_bytes()]) + return self._content + + async def aiter_bytes( + self, chunk_size: int | None = None + ) -> typing.AsyncIterator[bytes]: + """ + A byte-iterator over the decoded response content. + This allows us to handle gzip, deflate, brotli, and zstd encoded responses. + """ + if hasattr(self, "_content"): + chunk_size = len(self._content) if chunk_size is None else chunk_size + for i in range(0, len(self._content), max(chunk_size, 1)): + yield self._content[i : i + chunk_size] + else: + decoder = self._get_content_decoder() + chunker = ByteChunker(chunk_size=chunk_size) + with request_context(request=self._request): + async for raw_bytes in self.aiter_raw(): + decoded = decoder.decode(raw_bytes) + for chunk in chunker.decode(decoded): + yield chunk + decoded = decoder.flush() + for chunk in chunker.decode(decoded): + yield chunk # pragma: no cover + for chunk in chunker.flush(): + yield chunk + + async def aiter_text( + self, chunk_size: int | None = None + ) -> typing.AsyncIterator[str]: + """ + A str-iterator over the decoded response content + that handles both gzip, deflate, etc but also detects the content's + string encoding. + """ + decoder = TextDecoder(encoding=self.encoding or "utf-8") + chunker = TextChunker(chunk_size=chunk_size) + with request_context(request=self._request): + async for byte_content in self.aiter_bytes(): + text_content = decoder.decode(byte_content) + for chunk in chunker.decode(text_content): + yield chunk + text_content = decoder.flush() + for chunk in chunker.decode(text_content): + yield chunk # pragma: no cover + for chunk in chunker.flush(): + yield chunk + + async def aiter_lines(self) -> typing.AsyncIterator[str]: + decoder = LineDecoder() + with request_context(request=self._request): + async for text in self.aiter_text(): + for line in decoder.decode(text): + yield line + for line in decoder.flush(): + yield line + + async def aiter_raw( + self, chunk_size: int | None = None + ) -> typing.AsyncIterator[bytes]: + """ + A byte-iterator over the raw response content. + """ + if self.is_stream_consumed: + raise StreamConsumed() + if self.is_closed: + raise StreamClosed() + if not isinstance(self.stream, AsyncByteStream): + raise RuntimeError("Attempted to call an async iterator on an sync stream.") + + self.is_stream_consumed = True + self._num_bytes_downloaded = 0 + chunker = ByteChunker(chunk_size=chunk_size) + + with request_context(request=self._request): + async for raw_stream_bytes in self.stream: + self._num_bytes_downloaded += len(raw_stream_bytes) + for chunk in chunker.decode(raw_stream_bytes): + yield chunk + + for chunk in chunker.flush(): + yield chunk + + await self.aclose() + + async def aclose(self) -> None: + """ + Close the response and release the connection. + Automatically called if the response body is read to completion. + """ + if not isinstance(self.stream, AsyncByteStream): + raise RuntimeError("Attempted to call an async close on an sync stream.") + + if not self.is_closed: + self.is_closed = True + with request_context(request=self._request): + await self.stream.aclose() + + +class Cookies(typing.MutableMapping[str, str]): + """ + HTTP Cookies, as a mutable mapping. + """ + + def __init__(self, cookies: CookieTypes | None = None) -> None: + if cookies is None or isinstance(cookies, dict): + self.jar = CookieJar() + if isinstance(cookies, dict): + for key, value in cookies.items(): + self.set(key, value) + elif isinstance(cookies, list): + self.jar = CookieJar() + for key, value in cookies: + self.set(key, value) + elif isinstance(cookies, Cookies): + self.jar = CookieJar() + for cookie in cookies.jar: + self.jar.set_cookie(cookie) + else: + self.jar = cookies + + def extract_cookies(self, response: Response) -> None: + """ + Loads any cookies based on the response `Set-Cookie` headers. + """ + urllib_response = self._CookieCompatResponse(response) + urllib_request = self._CookieCompatRequest(response.request) + + self.jar.extract_cookies(urllib_response, urllib_request) # type: ignore + + def set_cookie_header(self, request: Request) -> None: + """ + Sets an appropriate 'Cookie:' HTTP header on the `Request`. + """ + urllib_request = self._CookieCompatRequest(request) + self.jar.add_cookie_header(urllib_request) + + def set(self, name: str, value: str, domain: str = "", path: str = "/") -> None: + """ + Set a cookie value by name. May optionally include domain and path. + """ + kwargs = { + "version": 0, + "name": name, + "value": value, + "port": None, + "port_specified": False, + "domain": domain, + "domain_specified": bool(domain), + "domain_initial_dot": domain.startswith("."), + "path": path, + "path_specified": bool(path), + "secure": False, + "expires": None, + "discard": True, + "comment": None, + "comment_url": None, + "rest": {"HttpOnly": None}, + "rfc2109": False, + } + cookie = Cookie(**kwargs) # type: ignore + self.jar.set_cookie(cookie) + + def get( # type: ignore + self, + name: str, + default: str | None = None, + domain: str | None = None, + path: str | None = None, + ) -> str | None: + """ + Get a cookie by name. May optionally include domain and path + in order to specify exactly which cookie to retrieve. + """ + value = None + for cookie in self.jar: + if cookie.name == name: + if domain is None or cookie.domain == domain: + if path is None or cookie.path == path: + if value is not None: + message = f"Multiple cookies exist with name={name}" + raise CookieConflict(message) + value = cookie.value + + if value is None: + return default + return value + + def delete( + self, + name: str, + domain: str | None = None, + path: str | None = None, + ) -> None: + """ + Delete a cookie by name. May optionally include domain and path + in order to specify exactly which cookie to delete. + """ + if domain is not None and path is not None: + return self.jar.clear(domain, path, name) + + remove = [ + cookie + for cookie in self.jar + if cookie.name == name + and (domain is None or cookie.domain == domain) + and (path is None or cookie.path == path) + ] + + for cookie in remove: + self.jar.clear(cookie.domain, cookie.path, cookie.name) + + def clear(self, domain: str | None = None, path: str | None = None) -> None: + """ + Delete all cookies. Optionally include a domain and path in + order to only delete a subset of all the cookies. + """ + args = [] + if domain is not None: + args.append(domain) + if path is not None: + assert domain is not None + args.append(path) + self.jar.clear(*args) + + def update(self, cookies: CookieTypes | None = None) -> None: # type: ignore + cookies = Cookies(cookies) + for cookie in cookies.jar: + self.jar.set_cookie(cookie) + + def __setitem__(self, name: str, value: str) -> None: + return self.set(name, value) + + def __getitem__(self, name: str) -> str: + value = self.get(name) + if value is None: + raise KeyError(name) + return value + + def __delitem__(self, name: str) -> None: + return self.delete(name) + + def __len__(self) -> int: + return len(self.jar) + + def __iter__(self) -> typing.Iterator[str]: + return (cookie.name for cookie in self.jar) + + def __bool__(self) -> bool: + for _ in self.jar: + return True + return False + + def __repr__(self) -> str: + cookies_repr = ", ".join( + [ + f"" + for cookie in self.jar + ] + ) + + return f"" + + class _CookieCompatRequest(urllib.request.Request): + """ + Wraps a `Request` instance up in a compatibility interface suitable + for use with `CookieJar` operations. + """ + + def __init__(self, request: Request) -> None: + super().__init__( + url=str(request.url), + headers=dict(request.headers), + method=request.method, + ) + self.request = request + + def add_unredirected_header(self, key: str, value: str) -> None: + super().add_unredirected_header(key, value) + self.request.headers[key] = value + + class _CookieCompatResponse: + """ + Wraps a `Request` instance up in a compatibility interface suitable + for use with `CookieJar` operations. + """ + + def __init__(self, response: Response) -> None: + self.response = response + + def info(self) -> email.message.Message: + info = email.message.Message() + for key, value in self.response.headers.multi_items(): + # Note that setting `info[key]` here is an "append" operation, + # not a "replace" operation. + # https://docs.python.org/3/library/email.compat32-message.html#email.message.Message.__setitem__ + info[key] = value + return info diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_multipart.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_multipart.py new file mode 100644 index 0000000000000000000000000000000000000000..b4761af9b2cf384de5189269927d781a700dbe46 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_multipart.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +import io +import mimetypes +import os +import re +import typing +from pathlib import Path + +from ._types import ( + AsyncByteStream, + FileContent, + FileTypes, + RequestData, + RequestFiles, + SyncByteStream, +) +from ._utils import ( + peek_filelike_length, + primitive_value_to_str, + to_bytes, +) + +_HTML5_FORM_ENCODING_REPLACEMENTS = {'"': "%22", "\\": "\\\\"} +_HTML5_FORM_ENCODING_REPLACEMENTS.update( + {chr(c): "%{:02X}".format(c) for c in range(0x1F + 1) if c != 0x1B} +) +_HTML5_FORM_ENCODING_RE = re.compile( + r"|".join([re.escape(c) for c in _HTML5_FORM_ENCODING_REPLACEMENTS.keys()]) +) + + +def _format_form_param(name: str, value: str) -> bytes: + """ + Encode a name/value pair within a multipart form. + """ + + def replacer(match: typing.Match[str]) -> str: + return _HTML5_FORM_ENCODING_REPLACEMENTS[match.group(0)] + + value = _HTML5_FORM_ENCODING_RE.sub(replacer, value) + return f'{name}="{value}"'.encode() + + +def _guess_content_type(filename: str | None) -> str | None: + """ + Guesses the mimetype based on a filename. Defaults to `application/octet-stream`. + + Returns `None` if `filename` is `None` or empty. + """ + if filename: + return mimetypes.guess_type(filename)[0] or "application/octet-stream" + return None + + +def get_multipart_boundary_from_content_type( + content_type: bytes | None, +) -> bytes | None: + if not content_type or not content_type.startswith(b"multipart/form-data"): + return None + # parse boundary according to + # https://www.rfc-editor.org/rfc/rfc2046#section-5.1.1 + if b";" in content_type: + for section in content_type.split(b";"): + if section.strip().lower().startswith(b"boundary="): + return section.strip()[len(b"boundary=") :].strip(b'"') + return None + + +class DataField: + """ + A single form field item, within a multipart form field. + """ + + def __init__(self, name: str, value: str | bytes | int | float | None) -> None: + if not isinstance(name, str): + raise TypeError( + f"Invalid type for name. Expected str, got {type(name)}: {name!r}" + ) + if value is not None and not isinstance(value, (str, bytes, int, float)): + raise TypeError( + "Invalid type for value. Expected primitive type," + f" got {type(value)}: {value!r}" + ) + self.name = name + self.value: str | bytes = ( + value if isinstance(value, bytes) else primitive_value_to_str(value) + ) + + def render_headers(self) -> bytes: + if not hasattr(self, "_headers"): + name = _format_form_param("name", self.name) + self._headers = b"".join( + [b"Content-Disposition: form-data; ", name, b"\r\n\r\n"] + ) + + return self._headers + + def render_data(self) -> bytes: + if not hasattr(self, "_data"): + self._data = to_bytes(self.value) + + return self._data + + def get_length(self) -> int: + headers = self.render_headers() + data = self.render_data() + return len(headers) + len(data) + + def render(self) -> typing.Iterator[bytes]: + yield self.render_headers() + yield self.render_data() + + +class FileField: + """ + A single file field item, within a multipart form field. + """ + + CHUNK_SIZE = 64 * 1024 + + def __init__(self, name: str, value: FileTypes) -> None: + self.name = name + + fileobj: FileContent + + headers: dict[str, str] = {} + content_type: str | None = None + + # This large tuple based API largely mirror's requests' API + # It would be good to think of better APIs for this that we could + # include in httpx 2.0 since variable length tuples(especially of 4 elements) + # are quite unwieldly + if isinstance(value, tuple): + if len(value) == 2: + # neither the 3rd parameter (content_type) nor the 4th (headers) + # was included + filename, fileobj = value + elif len(value) == 3: + filename, fileobj, content_type = value + else: + # all 4 parameters included + filename, fileobj, content_type, headers = value # type: ignore + else: + filename = Path(str(getattr(value, "name", "upload"))).name + fileobj = value + + if content_type is None: + content_type = _guess_content_type(filename) + + has_content_type_header = any("content-type" in key.lower() for key in headers) + if content_type is not None and not has_content_type_header: + # note that unlike requests, we ignore the content_type provided in the 3rd + # tuple element if it is also included in the headers requests does + # the opposite (it overwrites the headerwith the 3rd tuple element) + headers["Content-Type"] = content_type + + if isinstance(fileobj, io.StringIO): + raise TypeError( + "Multipart file uploads require 'io.BytesIO', not 'io.StringIO'." + ) + if isinstance(fileobj, io.TextIOBase): + raise TypeError( + "Multipart file uploads must be opened in binary mode, not text mode." + ) + + self.filename = filename + self.file = fileobj + self.headers = headers + + def get_length(self) -> int | None: + headers = self.render_headers() + + if isinstance(self.file, (str, bytes)): + return len(headers) + len(to_bytes(self.file)) + + file_length = peek_filelike_length(self.file) + + # If we can't determine the filesize without reading it into memory, + # then return `None` here, to indicate an unknown file length. + if file_length is None: + return None + + return len(headers) + file_length + + def render_headers(self) -> bytes: + if not hasattr(self, "_headers"): + parts = [ + b"Content-Disposition: form-data; ", + _format_form_param("name", self.name), + ] + if self.filename: + filename = _format_form_param("filename", self.filename) + parts.extend([b"; ", filename]) + for header_name, header_value in self.headers.items(): + key, val = f"\r\n{header_name}: ".encode(), header_value.encode() + parts.extend([key, val]) + parts.append(b"\r\n\r\n") + self._headers = b"".join(parts) + + return self._headers + + def render_data(self) -> typing.Iterator[bytes]: + if isinstance(self.file, (str, bytes)): + yield to_bytes(self.file) + return + + if hasattr(self.file, "seek"): + try: + self.file.seek(0) + except io.UnsupportedOperation: + pass + + chunk = self.file.read(self.CHUNK_SIZE) + while chunk: + yield to_bytes(chunk) + chunk = self.file.read(self.CHUNK_SIZE) + + def render(self) -> typing.Iterator[bytes]: + yield self.render_headers() + yield from self.render_data() + + +class MultipartStream(SyncByteStream, AsyncByteStream): + """ + Request content as streaming multipart encoded form data. + """ + + def __init__( + self, + data: RequestData, + files: RequestFiles, + boundary: bytes | None = None, + ) -> None: + if boundary is None: + boundary = os.urandom(16).hex().encode("ascii") + + self.boundary = boundary + self.content_type = "multipart/form-data; boundary=%s" % boundary.decode( + "ascii" + ) + self.fields = list(self._iter_fields(data, files)) + + def _iter_fields( + self, data: RequestData, files: RequestFiles + ) -> typing.Iterator[FileField | DataField]: + for name, value in data.items(): + if isinstance(value, (tuple, list)): + for item in value: + yield DataField(name=name, value=item) + else: + yield DataField(name=name, value=value) + + file_items = files.items() if isinstance(files, typing.Mapping) else files + for name, value in file_items: + yield FileField(name=name, value=value) + + def iter_chunks(self) -> typing.Iterator[bytes]: + for field in self.fields: + yield b"--%s\r\n" % self.boundary + yield from field.render() + yield b"\r\n" + yield b"--%s--\r\n" % self.boundary + + def get_content_length(self) -> int | None: + """ + Return the length of the multipart encoded content, or `None` if + any of the files have a length that cannot be determined upfront. + """ + boundary_length = len(self.boundary) + length = 0 + + for field in self.fields: + field_length = field.get_length() + if field_length is None: + return None + + length += 2 + boundary_length + 2 # b"--{boundary}\r\n" + length += field_length + length += 2 # b"\r\n" + + length += 2 + boundary_length + 4 # b"--{boundary}--\r\n" + return length + + # Content stream interface. + + def get_headers(self) -> dict[str, str]: + content_length = self.get_content_length() + content_type = self.content_type + if content_length is None: + return {"Transfer-Encoding": "chunked", "Content-Type": content_type} + return {"Content-Length": str(content_length), "Content-Type": content_type} + + def __iter__(self) -> typing.Iterator[bytes]: + for chunk in self.iter_chunks(): + yield chunk + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + for chunk in self.iter_chunks(): + yield chunk diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_status_codes.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_status_codes.py new file mode 100644 index 0000000000000000000000000000000000000000..133a6231a5b53fd2f073799ca1bd07c50abe40ae --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_status_codes.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from enum import IntEnum + +__all__ = ["codes"] + + +class codes(IntEnum): + """HTTP status codes and reason phrases + + Status codes from the following RFCs are all observed: + + * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 + * RFC 6585: Additional HTTP Status Codes + * RFC 3229: Delta encoding in HTTP + * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 + * RFC 5842: Binding Extensions to WebDAV + * RFC 7238: Permanent Redirect + * RFC 2295: Transparent Content Negotiation in HTTP + * RFC 2774: An HTTP Extension Framework + * RFC 7540: Hypertext Transfer Protocol Version 2 (HTTP/2) + * RFC 2324: Hyper Text Coffee Pot Control Protocol (HTCPCP/1.0) + * RFC 7725: An HTTP Status Code to Report Legal Obstacles + * RFC 8297: An HTTP Status Code for Indicating Hints + * RFC 8470: Using Early Data in HTTP + """ + + def __new__(cls, value: int, phrase: str = "") -> codes: + obj = int.__new__(cls, value) + obj._value_ = value + + obj.phrase = phrase # type: ignore[attr-defined] + return obj + + def __str__(self) -> str: + return str(self.value) + + @classmethod + def get_reason_phrase(cls, value: int) -> str: + try: + return codes(value).phrase # type: ignore + except ValueError: + return "" + + @classmethod + def is_informational(cls, value: int) -> bool: + """ + Returns `True` for 1xx status codes, `False` otherwise. + """ + return 100 <= value <= 199 + + @classmethod + def is_success(cls, value: int) -> bool: + """ + Returns `True` for 2xx status codes, `False` otherwise. + """ + return 200 <= value <= 299 + + @classmethod + def is_redirect(cls, value: int) -> bool: + """ + Returns `True` for 3xx status codes, `False` otherwise. + """ + return 300 <= value <= 399 + + @classmethod + def is_client_error(cls, value: int) -> bool: + """ + Returns `True` for 4xx status codes, `False` otherwise. + """ + return 400 <= value <= 499 + + @classmethod + def is_server_error(cls, value: int) -> bool: + """ + Returns `True` for 5xx status codes, `False` otherwise. + """ + return 500 <= value <= 599 + + @classmethod + def is_error(cls, value: int) -> bool: + """ + Returns `True` for 4xx or 5xx status codes, `False` otherwise. + """ + return 400 <= value <= 599 + + # informational + CONTINUE = 100, "Continue" + SWITCHING_PROTOCOLS = 101, "Switching Protocols" + PROCESSING = 102, "Processing" + EARLY_HINTS = 103, "Early Hints" + + # success + OK = 200, "OK" + CREATED = 201, "Created" + ACCEPTED = 202, "Accepted" + NON_AUTHORITATIVE_INFORMATION = 203, "Non-Authoritative Information" + NO_CONTENT = 204, "No Content" + RESET_CONTENT = 205, "Reset Content" + PARTIAL_CONTENT = 206, "Partial Content" + MULTI_STATUS = 207, "Multi-Status" + ALREADY_REPORTED = 208, "Already Reported" + IM_USED = 226, "IM Used" + + # redirection + MULTIPLE_CHOICES = 300, "Multiple Choices" + MOVED_PERMANENTLY = 301, "Moved Permanently" + FOUND = 302, "Found" + SEE_OTHER = 303, "See Other" + NOT_MODIFIED = 304, "Not Modified" + USE_PROXY = 305, "Use Proxy" + TEMPORARY_REDIRECT = 307, "Temporary Redirect" + PERMANENT_REDIRECT = 308, "Permanent Redirect" + + # client error + BAD_REQUEST = 400, "Bad Request" + UNAUTHORIZED = 401, "Unauthorized" + PAYMENT_REQUIRED = 402, "Payment Required" + FORBIDDEN = 403, "Forbidden" + NOT_FOUND = 404, "Not Found" + METHOD_NOT_ALLOWED = 405, "Method Not Allowed" + NOT_ACCEPTABLE = 406, "Not Acceptable" + PROXY_AUTHENTICATION_REQUIRED = 407, "Proxy Authentication Required" + REQUEST_TIMEOUT = 408, "Request Timeout" + CONFLICT = 409, "Conflict" + GONE = 410, "Gone" + LENGTH_REQUIRED = 411, "Length Required" + PRECONDITION_FAILED = 412, "Precondition Failed" + REQUEST_ENTITY_TOO_LARGE = 413, "Request Entity Too Large" + REQUEST_URI_TOO_LONG = 414, "Request-URI Too Long" + UNSUPPORTED_MEDIA_TYPE = 415, "Unsupported Media Type" + REQUESTED_RANGE_NOT_SATISFIABLE = 416, "Requested Range Not Satisfiable" + EXPECTATION_FAILED = 417, "Expectation Failed" + IM_A_TEAPOT = 418, "I'm a teapot" + MISDIRECTED_REQUEST = 421, "Misdirected Request" + UNPROCESSABLE_ENTITY = 422, "Unprocessable Entity" + LOCKED = 423, "Locked" + FAILED_DEPENDENCY = 424, "Failed Dependency" + TOO_EARLY = 425, "Too Early" + UPGRADE_REQUIRED = 426, "Upgrade Required" + PRECONDITION_REQUIRED = 428, "Precondition Required" + TOO_MANY_REQUESTS = 429, "Too Many Requests" + REQUEST_HEADER_FIELDS_TOO_LARGE = 431, "Request Header Fields Too Large" + UNAVAILABLE_FOR_LEGAL_REASONS = 451, "Unavailable For Legal Reasons" + + # server errors + INTERNAL_SERVER_ERROR = 500, "Internal Server Error" + NOT_IMPLEMENTED = 501, "Not Implemented" + BAD_GATEWAY = 502, "Bad Gateway" + SERVICE_UNAVAILABLE = 503, "Service Unavailable" + GATEWAY_TIMEOUT = 504, "Gateway Timeout" + HTTP_VERSION_NOT_SUPPORTED = 505, "HTTP Version Not Supported" + VARIANT_ALSO_NEGOTIATES = 506, "Variant Also Negotiates" + INSUFFICIENT_STORAGE = 507, "Insufficient Storage" + LOOP_DETECTED = 508, "Loop Detected" + NOT_EXTENDED = 510, "Not Extended" + NETWORK_AUTHENTICATION_REQUIRED = 511, "Network Authentication Required" + + +# Include lower-case styles for `requests` compatibility. +for code in codes: + setattr(codes, code._name_.lower(), int(code)) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7a321053b29bcd48698cf2bd74a1d19c8556aefb --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/__init__.py @@ -0,0 +1,15 @@ +from .asgi import * +from .base import * +from .default import * +from .mock import * +from .wsgi import * + +__all__ = [ + "ASGITransport", + "AsyncBaseTransport", + "BaseTransport", + "AsyncHTTPTransport", + "HTTPTransport", + "MockTransport", + "WSGITransport", +] diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/asgi.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/asgi.py new file mode 100644 index 0000000000000000000000000000000000000000..2bc4efae0e1b14620f75f712eb15ecf500d14eef --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/asgi.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import typing + +from .._models import Request, Response +from .._types import AsyncByteStream +from .base import AsyncBaseTransport + +if typing.TYPE_CHECKING: # pragma: no cover + import asyncio + + import trio + + Event = typing.Union[asyncio.Event, trio.Event] + + +_Message = typing.MutableMapping[str, typing.Any] +_Receive = typing.Callable[[], typing.Awaitable[_Message]] +_Send = typing.Callable[ + [typing.MutableMapping[str, typing.Any]], typing.Awaitable[None] +] +_ASGIApp = typing.Callable[ + [typing.MutableMapping[str, typing.Any], _Receive, _Send], typing.Awaitable[None] +] + +__all__ = ["ASGITransport"] + + +def is_running_trio() -> bool: + try: + # sniffio is a dependency of trio. + + # See https://github.com/python-trio/trio/issues/2802 + import sniffio + + if sniffio.current_async_library() == "trio": + return True + except ImportError: # pragma: nocover + pass + + return False + + +def create_event() -> Event: + if is_running_trio(): + import trio + + return trio.Event() + + import asyncio + + return asyncio.Event() + + +class ASGIResponseStream(AsyncByteStream): + def __init__(self, body: list[bytes]) -> None: + self._body = body + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + yield b"".join(self._body) + + +class ASGITransport(AsyncBaseTransport): + """ + A custom AsyncTransport that handles sending requests directly to an ASGI app. + + ```python + transport = httpx.ASGITransport( + app=app, + root_path="/submount", + client=("1.2.3.4", 123) + ) + client = httpx.AsyncClient(transport=transport) + ``` + + Arguments: + + * `app` - The ASGI application. + * `raise_app_exceptions` - Boolean indicating if exceptions in the application + should be raised. Default to `True`. Can be set to `False` for use cases + such as testing the content of a client 500 response. + * `root_path` - The root path on which the ASGI application should be mounted. + * `client` - A two-tuple indicating the client IP and port of incoming requests. + ``` + """ + + def __init__( + self, + app: _ASGIApp, + raise_app_exceptions: bool = True, + root_path: str = "", + client: tuple[str, int] = ("127.0.0.1", 123), + ) -> None: + self.app = app + self.raise_app_exceptions = raise_app_exceptions + self.root_path = root_path + self.client = client + + async def handle_async_request( + self, + request: Request, + ) -> Response: + assert isinstance(request.stream, AsyncByteStream) + + # ASGI scope. + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": request.method, + "headers": [(k.lower(), v) for (k, v) in request.headers.raw], + "scheme": request.url.scheme, + "path": request.url.path, + "raw_path": request.url.raw_path.split(b"?")[0], + "query_string": request.url.query, + "server": (request.url.host, request.url.port), + "client": self.client, + "root_path": self.root_path, + } + + # Request. + request_body_chunks = request.stream.__aiter__() + request_complete = False + + # Response. + status_code = None + response_headers = None + body_parts = [] + response_started = False + response_complete = create_event() + + # ASGI callables. + + async def receive() -> dict[str, typing.Any]: + nonlocal request_complete + + if request_complete: + await response_complete.wait() + return {"type": "http.disconnect"} + + try: + body = await request_body_chunks.__anext__() + except StopAsyncIteration: + request_complete = True + return {"type": "http.request", "body": b"", "more_body": False} + return {"type": "http.request", "body": body, "more_body": True} + + async def send(message: typing.MutableMapping[str, typing.Any]) -> None: + nonlocal status_code, response_headers, response_started + + if message["type"] == "http.response.start": + assert not response_started + + status_code = message["status"] + response_headers = message.get("headers", []) + response_started = True + + elif message["type"] == "http.response.body": + assert not response_complete.is_set() + body = message.get("body", b"") + more_body = message.get("more_body", False) + + if body and request.method != "HEAD": + body_parts.append(body) + + if not more_body: + response_complete.set() + + try: + await self.app(scope, receive, send) + except Exception: # noqa: PIE-786 + if self.raise_app_exceptions: + raise + + response_complete.set() + if status_code is None: + status_code = 500 + if response_headers is None: + response_headers = {} + + assert response_complete.is_set() + assert status_code is not None + assert response_headers is not None + + stream = ASGIResponseStream(body_parts) + + return Response(status_code, headers=response_headers, stream=stream) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/base.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/base.py new file mode 100644 index 0000000000000000000000000000000000000000..66fd99d702480b555c06694fe14715ea6df3dfc3 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/base.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import typing +from types import TracebackType + +from .._models import Request, Response + +T = typing.TypeVar("T", bound="BaseTransport") +A = typing.TypeVar("A", bound="AsyncBaseTransport") + +__all__ = ["AsyncBaseTransport", "BaseTransport"] + + +class BaseTransport: + def __enter__(self: T) -> T: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + self.close() + + def handle_request(self, request: Request) -> Response: + """ + Send a single HTTP request and return a response. + + Developers shouldn't typically ever need to call into this API directly, + since the Client class provides all the higher level user-facing API + niceties. + + In order to properly release any network resources, the response + stream should *either* be consumed immediately, with a call to + `response.stream.read()`, or else the `handle_request` call should + be followed with a try/finally block to ensuring the stream is + always closed. + + Example usage: + + with httpx.HTTPTransport() as transport: + req = httpx.Request( + method=b"GET", + url=(b"https", b"www.example.com", 443, b"/"), + headers=[(b"Host", b"www.example.com")], + ) + resp = transport.handle_request(req) + body = resp.stream.read() + print(resp.status_code, resp.headers, body) + + + Takes a `Request` instance as the only argument. + + Returns a `Response` instance. + """ + raise NotImplementedError( + "The 'handle_request' method must be implemented." + ) # pragma: no cover + + def close(self) -> None: + pass + + +class AsyncBaseTransport: + async def __aenter__(self: A) -> A: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + await self.aclose() + + async def handle_async_request( + self, + request: Request, + ) -> Response: + raise NotImplementedError( + "The 'handle_async_request' method must be implemented." + ) # pragma: no cover + + async def aclose(self) -> None: + pass diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/default.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/default.py new file mode 100644 index 0000000000000000000000000000000000000000..d5aa05ff234fd3fbf4fee88c4a7d3e3c151a538f --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/default.py @@ -0,0 +1,406 @@ +""" +Custom transports, with nicely configured defaults. + +The following additional keyword arguments are currently supported by httpcore... + +* uds: str +* local_address: str +* retries: int + +Example usages... + +# Disable HTTP/2 on a single specific domain. +mounts = { + "all://": httpx.HTTPTransport(http2=True), + "all://*example.org": httpx.HTTPTransport() +} + +# Using advanced httpcore configuration, with connection retries. +transport = httpx.HTTPTransport(retries=1) +client = httpx.Client(transport=transport) + +# Using advanced httpcore configuration, with unix domain sockets. +transport = httpx.HTTPTransport(uds="socket.uds") +client = httpx.Client(transport=transport) +""" + +from __future__ import annotations + +import contextlib +import typing +from types import TracebackType + +if typing.TYPE_CHECKING: + import ssl # pragma: no cover + + import httpx # pragma: no cover + +from .._config import DEFAULT_LIMITS, Limits, Proxy, create_ssl_context +from .._exceptions import ( + ConnectError, + ConnectTimeout, + LocalProtocolError, + NetworkError, + PoolTimeout, + ProtocolError, + ProxyError, + ReadError, + ReadTimeout, + RemoteProtocolError, + TimeoutException, + UnsupportedProtocol, + WriteError, + WriteTimeout, +) +from .._models import Request, Response +from .._types import AsyncByteStream, CertTypes, ProxyTypes, SyncByteStream +from .._urls import URL +from .base import AsyncBaseTransport, BaseTransport + +T = typing.TypeVar("T", bound="HTTPTransport") +A = typing.TypeVar("A", bound="AsyncHTTPTransport") + +SOCKET_OPTION = typing.Union[ + typing.Tuple[int, int, int], + typing.Tuple[int, int, typing.Union[bytes, bytearray]], + typing.Tuple[int, int, None, int], +] + +__all__ = ["AsyncHTTPTransport", "HTTPTransport"] + +HTTPCORE_EXC_MAP: dict[type[Exception], type[httpx.HTTPError]] = {} + + +def _load_httpcore_exceptions() -> dict[type[Exception], type[httpx.HTTPError]]: + import httpcore + + return { + httpcore.TimeoutException: TimeoutException, + httpcore.ConnectTimeout: ConnectTimeout, + httpcore.ReadTimeout: ReadTimeout, + httpcore.WriteTimeout: WriteTimeout, + httpcore.PoolTimeout: PoolTimeout, + httpcore.NetworkError: NetworkError, + httpcore.ConnectError: ConnectError, + httpcore.ReadError: ReadError, + httpcore.WriteError: WriteError, + httpcore.ProxyError: ProxyError, + httpcore.UnsupportedProtocol: UnsupportedProtocol, + httpcore.ProtocolError: ProtocolError, + httpcore.LocalProtocolError: LocalProtocolError, + httpcore.RemoteProtocolError: RemoteProtocolError, + } + + +@contextlib.contextmanager +def map_httpcore_exceptions() -> typing.Iterator[None]: + global HTTPCORE_EXC_MAP + if len(HTTPCORE_EXC_MAP) == 0: + HTTPCORE_EXC_MAP = _load_httpcore_exceptions() + try: + yield + except Exception as exc: + mapped_exc = None + + for from_exc, to_exc in HTTPCORE_EXC_MAP.items(): + if not isinstance(exc, from_exc): + continue + # We want to map to the most specific exception we can find. + # Eg if `exc` is an `httpcore.ReadTimeout`, we want to map to + # `httpx.ReadTimeout`, not just `httpx.TimeoutException`. + if mapped_exc is None or issubclass(to_exc, mapped_exc): + mapped_exc = to_exc + + if mapped_exc is None: # pragma: no cover + raise + + message = str(exc) + raise mapped_exc(message) from exc + + +class ResponseStream(SyncByteStream): + def __init__(self, httpcore_stream: typing.Iterable[bytes]) -> None: + self._httpcore_stream = httpcore_stream + + def __iter__(self) -> typing.Iterator[bytes]: + with map_httpcore_exceptions(): + for part in self._httpcore_stream: + yield part + + def close(self) -> None: + if hasattr(self._httpcore_stream, "close"): + self._httpcore_stream.close() + + +class HTTPTransport(BaseTransport): + def __init__( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + proxy: ProxyTypes | None = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + import httpcore + + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + if proxy is None: + self._pool = httpcore.ConnectionPool( + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + uds=uds, + local_address=local_address, + retries=retries, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("http", "https"): + self._pool = httpcore.HTTPProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + ssl_context=ssl_context, + proxy_ssl_context=proxy.ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("socks5", "socks5h"): + try: + import socksio # noqa + except ImportError: # pragma: no cover + raise ImportError( + "Using SOCKS proxy, but the 'socksio' package is not installed. " + "Make sure to install httpx using `pip install httpx[socks]`." + ) from None + + self._pool = httpcore.SOCKSProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + ) + else: # pragma: no cover + raise ValueError( + "Proxy protocol must be either 'http', 'https', 'socks5', or 'socks5h'," + f" but got {proxy.url.scheme!r}." + ) + + def __enter__(self: T) -> T: # Use generics for subclass support. + self._pool.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + with map_httpcore_exceptions(): + self._pool.__exit__(exc_type, exc_value, traceback) + + def handle_request( + self, + request: Request, + ) -> Response: + assert isinstance(request.stream, SyncByteStream) + import httpcore + + req = httpcore.Request( + method=request.method, + url=httpcore.URL( + scheme=request.url.raw_scheme, + host=request.url.raw_host, + port=request.url.port, + target=request.url.raw_path, + ), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + with map_httpcore_exceptions(): + resp = self._pool.handle_request(req) + + assert isinstance(resp.stream, typing.Iterable) + + return Response( + status_code=resp.status, + headers=resp.headers, + stream=ResponseStream(resp.stream), + extensions=resp.extensions, + ) + + def close(self) -> None: + self._pool.close() + + +class AsyncResponseStream(AsyncByteStream): + def __init__(self, httpcore_stream: typing.AsyncIterable[bytes]) -> None: + self._httpcore_stream = httpcore_stream + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + with map_httpcore_exceptions(): + async for part in self._httpcore_stream: + yield part + + async def aclose(self) -> None: + if hasattr(self._httpcore_stream, "aclose"): + await self._httpcore_stream.aclose() + + +class AsyncHTTPTransport(AsyncBaseTransport): + def __init__( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + proxy: ProxyTypes | None = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + import httpcore + + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + if proxy is None: + self._pool = httpcore.AsyncConnectionPool( + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + uds=uds, + local_address=local_address, + retries=retries, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("http", "https"): + self._pool = httpcore.AsyncHTTPProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + proxy_ssl_context=proxy.ssl_context, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("socks5", "socks5h"): + try: + import socksio # noqa + except ImportError: # pragma: no cover + raise ImportError( + "Using SOCKS proxy, but the 'socksio' package is not installed. " + "Make sure to install httpx using `pip install httpx[socks]`." + ) from None + + self._pool = httpcore.AsyncSOCKSProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + ) + else: # pragma: no cover + raise ValueError( + "Proxy protocol must be either 'http', 'https', 'socks5', or 'socks5h'," + " but got {proxy.url.scheme!r}." + ) + + async def __aenter__(self: A) -> A: # Use generics for subclass support. + await self._pool.__aenter__() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + with map_httpcore_exceptions(): + await self._pool.__aexit__(exc_type, exc_value, traceback) + + async def handle_async_request( + self, + request: Request, + ) -> Response: + assert isinstance(request.stream, AsyncByteStream) + import httpcore + + req = httpcore.Request( + method=request.method, + url=httpcore.URL( + scheme=request.url.raw_scheme, + host=request.url.raw_host, + port=request.url.port, + target=request.url.raw_path, + ), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + with map_httpcore_exceptions(): + resp = await self._pool.handle_async_request(req) + + assert isinstance(resp.stream, typing.AsyncIterable) + + return Response( + status_code=resp.status, + headers=resp.headers, + stream=AsyncResponseStream(resp.stream), + extensions=resp.extensions, + ) + + async def aclose(self) -> None: + await self._pool.aclose() diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/mock.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/mock.py new file mode 100644 index 0000000000000000000000000000000000000000..8c418f59e06cae43abdbb626ec21cafc7e8c6277 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/mock.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import typing + +from .._models import Request, Response +from .base import AsyncBaseTransport, BaseTransport + +SyncHandler = typing.Callable[[Request], Response] +AsyncHandler = typing.Callable[[Request], typing.Coroutine[None, None, Response]] + + +__all__ = ["MockTransport"] + + +class MockTransport(AsyncBaseTransport, BaseTransport): + def __init__(self, handler: SyncHandler | AsyncHandler) -> None: + self.handler = handler + + def handle_request( + self, + request: Request, + ) -> Response: + request.read() + response = self.handler(request) + if not isinstance(response, Response): # pragma: no cover + raise TypeError("Cannot use an async handler in a sync Client") + return response + + async def handle_async_request( + self, + request: Request, + ) -> Response: + await request.aread() + response = self.handler(request) + + # Allow handler to *optionally* be an `async` function. + # If it is, then the `response` variable need to be awaited to actually + # return the result. + + if not isinstance(response, Response): + response = await response + + return response diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/wsgi.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/wsgi.py new file mode 100644 index 0000000000000000000000000000000000000000..8592ffe017a87367cc7578184540096a9682908d --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_transports/wsgi.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import io +import itertools +import sys +import typing + +from .._models import Request, Response +from .._types import SyncByteStream +from .base import BaseTransport + +if typing.TYPE_CHECKING: + from _typeshed import OptExcInfo # pragma: no cover + from _typeshed.wsgi import WSGIApplication # pragma: no cover + +_T = typing.TypeVar("_T") + + +__all__ = ["WSGITransport"] + + +def _skip_leading_empty_chunks(body: typing.Iterable[_T]) -> typing.Iterable[_T]: + body = iter(body) + for chunk in body: + if chunk: + return itertools.chain([chunk], body) + return [] + + +class WSGIByteStream(SyncByteStream): + def __init__(self, result: typing.Iterable[bytes]) -> None: + self._close = getattr(result, "close", None) + self._result = _skip_leading_empty_chunks(result) + + def __iter__(self) -> typing.Iterator[bytes]: + for part in self._result: + yield part + + def close(self) -> None: + if self._close is not None: + self._close() + + +class WSGITransport(BaseTransport): + """ + A custom transport that handles sending requests directly to an WSGI app. + The simplest way to use this functionality is to use the `app` argument. + + ``` + client = httpx.Client(app=app) + ``` + + Alternatively, you can setup the transport instance explicitly. + This allows you to include any additional configuration arguments specific + to the WSGITransport class: + + ``` + transport = httpx.WSGITransport( + app=app, + script_name="/submount", + remote_addr="1.2.3.4" + ) + client = httpx.Client(transport=transport) + ``` + + Arguments: + + * `app` - The WSGI application. + * `raise_app_exceptions` - Boolean indicating if exceptions in the application + should be raised. Default to `True`. Can be set to `False` for use cases + such as testing the content of a client 500 response. + * `script_name` - The root path on which the WSGI application should be mounted. + * `remote_addr` - A string indicating the client IP of incoming requests. + ``` + """ + + def __init__( + self, + app: WSGIApplication, + raise_app_exceptions: bool = True, + script_name: str = "", + remote_addr: str = "127.0.0.1", + wsgi_errors: typing.TextIO | None = None, + ) -> None: + self.app = app + self.raise_app_exceptions = raise_app_exceptions + self.script_name = script_name + self.remote_addr = remote_addr + self.wsgi_errors = wsgi_errors + + def handle_request(self, request: Request) -> Response: + request.read() + wsgi_input = io.BytesIO(request.content) + + port = request.url.port or {"http": 80, "https": 443}[request.url.scheme] + environ = { + "wsgi.version": (1, 0), + "wsgi.url_scheme": request.url.scheme, + "wsgi.input": wsgi_input, + "wsgi.errors": self.wsgi_errors or sys.stderr, + "wsgi.multithread": True, + "wsgi.multiprocess": False, + "wsgi.run_once": False, + "REQUEST_METHOD": request.method, + "SCRIPT_NAME": self.script_name, + "PATH_INFO": request.url.path, + "QUERY_STRING": request.url.query.decode("ascii"), + "SERVER_NAME": request.url.host, + "SERVER_PORT": str(port), + "SERVER_PROTOCOL": "HTTP/1.1", + "REMOTE_ADDR": self.remote_addr, + } + for header_key, header_value in request.headers.raw: + key = header_key.decode("ascii").upper().replace("-", "_") + if key not in ("CONTENT_TYPE", "CONTENT_LENGTH"): + key = "HTTP_" + key + environ[key] = header_value.decode("ascii") + + seen_status = None + seen_response_headers = None + seen_exc_info = None + + def start_response( + status: str, + response_headers: list[tuple[str, str]], + exc_info: OptExcInfo | None = None, + ) -> typing.Callable[[bytes], typing.Any]: + nonlocal seen_status, seen_response_headers, seen_exc_info + seen_status = status + seen_response_headers = response_headers + seen_exc_info = exc_info + return lambda _: None + + result = self.app(environ, start_response) + + stream = WSGIByteStream(result) + + assert seen_status is not None + assert seen_response_headers is not None + if seen_exc_info and seen_exc_info[0] and self.raise_app_exceptions: + raise seen_exc_info[1] + + status_code = int(seen_status.split()[0]) + headers = [ + (key.encode("ascii"), value.encode("ascii")) + for key, value in seen_response_headers + ] + + return Response(status_code, headers=headers, stream=stream) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_types.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_types.py new file mode 100644 index 0000000000000000000000000000000000000000..704dfdffc8ba61eb913fa918072381e410b23c00 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_types.py @@ -0,0 +1,114 @@ +""" +Type definitions for type checking purposes. +""" + +from http.cookiejar import CookieJar +from typing import ( + IO, + TYPE_CHECKING, + Any, + AsyncIterable, + AsyncIterator, + Callable, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Sequence, + Tuple, + Union, +) + +if TYPE_CHECKING: # pragma: no cover + from ._auth import Auth # noqa: F401 + from ._config import Proxy, Timeout # noqa: F401 + from ._models import Cookies, Headers, Request # noqa: F401 + from ._urls import URL, QueryParams # noqa: F401 + + +PrimitiveData = Optional[Union[str, int, float, bool]] + +URLTypes = Union["URL", str] + +QueryParamTypes = Union[ + "QueryParams", + Mapping[str, Union[PrimitiveData, Sequence[PrimitiveData]]], + List[Tuple[str, PrimitiveData]], + Tuple[Tuple[str, PrimitiveData], ...], + str, + bytes, +] + +HeaderTypes = Union[ + "Headers", + Mapping[str, str], + Mapping[bytes, bytes], + Sequence[Tuple[str, str]], + Sequence[Tuple[bytes, bytes]], +] + +CookieTypes = Union["Cookies", CookieJar, Dict[str, str], List[Tuple[str, str]]] + +TimeoutTypes = Union[ + Optional[float], + Tuple[Optional[float], Optional[float], Optional[float], Optional[float]], + "Timeout", +] +ProxyTypes = Union["URL", str, "Proxy"] +CertTypes = Union[str, Tuple[str, str], Tuple[str, str, str]] + +AuthTypes = Union[ + Tuple[Union[str, bytes], Union[str, bytes]], + Callable[["Request"], "Request"], + "Auth", +] + +RequestContent = Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] +ResponseContent = Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] +ResponseExtensions = Mapping[str, Any] + +RequestData = Mapping[str, Any] + +FileContent = Union[IO[bytes], bytes, str] +FileTypes = Union[ + # file (or bytes) + FileContent, + # (filename, file (or bytes)) + Tuple[Optional[str], FileContent], + # (filename, file (or bytes), content_type) + Tuple[Optional[str], FileContent, Optional[str]], + # (filename, file (or bytes), content_type, headers) + Tuple[Optional[str], FileContent, Optional[str], Mapping[str, str]], +] +RequestFiles = Union[Mapping[str, FileTypes], Sequence[Tuple[str, FileTypes]]] + +RequestExtensions = Mapping[str, Any] + +__all__ = ["AsyncByteStream", "SyncByteStream"] + + +class SyncByteStream: + def __iter__(self) -> Iterator[bytes]: + raise NotImplementedError( + "The '__iter__' method must be implemented." + ) # pragma: no cover + yield b"" # pragma: no cover + + def close(self) -> None: + """ + Subclasses can override this method to release any network resources + after a request/response cycle is complete. + """ + + +class AsyncByteStream: + async def __aiter__(self) -> AsyncIterator[bytes]: + raise NotImplementedError( + "The '__aiter__' method must be implemented." + ) # pragma: no cover + yield b"" # pragma: no cover + + async def aclose(self) -> None: + pass diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_urlparse.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_urlparse.py new file mode 100644 index 0000000000000000000000000000000000000000..bf190fd560ee4fc8a11af371a15fc5f1dc284d34 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_urlparse.py @@ -0,0 +1,527 @@ +""" +An implementation of `urlparse` that provides URL validation and normalization +as described by RFC3986. + +We rely on this implementation rather than the one in Python's stdlib, because: + +* It provides more complete URL validation. +* It properly differentiates between an empty querystring and an absent querystring, + to distinguish URLs with a trailing '?'. +* It handles scheme, hostname, port, and path normalization. +* It supports IDNA hostnames, normalizing them to their encoded form. +* The API supports passing individual components, as well as the complete URL string. + +Previously we relied on the excellent `rfc3986` package to handle URL parsing and +validation, but this module provides a simpler alternative, with less indirection +required. +""" + +from __future__ import annotations + +import ipaddress +import re +import typing + +import idna + +from ._exceptions import InvalidURL + +MAX_URL_LENGTH = 65536 + +# https://datatracker.ietf.org/doc/html/rfc3986.html#section-2.3 +UNRESERVED_CHARACTERS = ( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" +) +SUB_DELIMS = "!$&'()*+,;=" + +PERCENT_ENCODED_REGEX = re.compile("%[A-Fa-f0-9]{2}") + +# https://url.spec.whatwg.org/#percent-encoded-bytes + +# The fragment percent-encode set is the C0 control percent-encode set +# and U+0020 SPACE, U+0022 ("), U+003C (<), U+003E (>), and U+0060 (`). +FRAG_SAFE = "".join( + [chr(i) for i in range(0x20, 0x7F) if i not in (0x20, 0x22, 0x3C, 0x3E, 0x60)] +) + +# The query percent-encode set is the C0 control percent-encode set +# and U+0020 SPACE, U+0022 ("), U+0023 (#), U+003C (<), and U+003E (>). +QUERY_SAFE = "".join( + [chr(i) for i in range(0x20, 0x7F) if i not in (0x20, 0x22, 0x23, 0x3C, 0x3E)] +) + +# The path percent-encode set is the query percent-encode set +# and U+003F (?), U+0060 (`), U+007B ({), and U+007D (}). +PATH_SAFE = "".join( + [ + chr(i) + for i in range(0x20, 0x7F) + if i not in (0x20, 0x22, 0x23, 0x3C, 0x3E) + (0x3F, 0x60, 0x7B, 0x7D) + ] +) + +# The userinfo percent-encode set is the path percent-encode set +# and U+002F (/), U+003A (:), U+003B (;), U+003D (=), U+0040 (@), +# U+005B ([) to U+005E (^), inclusive, and U+007C (|). +USERNAME_SAFE = "".join( + [ + chr(i) + for i in range(0x20, 0x7F) + if i + not in (0x20, 0x22, 0x23, 0x3C, 0x3E) + + (0x3F, 0x60, 0x7B, 0x7D) + + (0x2F, 0x3A, 0x3B, 0x3D, 0x40, 0x5B, 0x5C, 0x5D, 0x5E, 0x7C) + ] +) +PASSWORD_SAFE = "".join( + [ + chr(i) + for i in range(0x20, 0x7F) + if i + not in (0x20, 0x22, 0x23, 0x3C, 0x3E) + + (0x3F, 0x60, 0x7B, 0x7D) + + (0x2F, 0x3A, 0x3B, 0x3D, 0x40, 0x5B, 0x5C, 0x5D, 0x5E, 0x7C) + ] +) +# Note... The terminology 'userinfo' percent-encode set in the WHATWG document +# is used for the username and password quoting. For the joint userinfo component +# we remove U+003A (:) from the safe set. +USERINFO_SAFE = "".join( + [ + chr(i) + for i in range(0x20, 0x7F) + if i + not in (0x20, 0x22, 0x23, 0x3C, 0x3E) + + (0x3F, 0x60, 0x7B, 0x7D) + + (0x2F, 0x3B, 0x3D, 0x40, 0x5B, 0x5C, 0x5D, 0x5E, 0x7C) + ] +) + + +# {scheme}: (optional) +# //{authority} (optional) +# {path} +# ?{query} (optional) +# #{fragment} (optional) +URL_REGEX = re.compile( + ( + r"(?:(?P{scheme}):)?" + r"(?://(?P{authority}))?" + r"(?P{path})" + r"(?:\?(?P{query}))?" + r"(?:#(?P{fragment}))?" + ).format( + scheme="([a-zA-Z][a-zA-Z0-9+.-]*)?", + authority="[^/?#]*", + path="[^?#]*", + query="[^#]*", + fragment=".*", + ) +) + +# {userinfo}@ (optional) +# {host} +# :{port} (optional) +AUTHORITY_REGEX = re.compile( + ( + r"(?:(?P{userinfo})@)?" r"(?P{host})" r":?(?P{port})?" + ).format( + userinfo=".*", # Any character sequence. + host="(\\[.*\\]|[^:@]*)", # Either any character sequence excluding ':' or '@', + # or an IPv6 address enclosed within square brackets. + port=".*", # Any character sequence. + ) +) + + +# If we call urlparse with an individual component, then we need to regex +# validate that component individually. +# Note that we're duplicating the same strings as above. Shock! Horror!! +COMPONENT_REGEX = { + "scheme": re.compile("([a-zA-Z][a-zA-Z0-9+.-]*)?"), + "authority": re.compile("[^/?#]*"), + "path": re.compile("[^?#]*"), + "query": re.compile("[^#]*"), + "fragment": re.compile(".*"), + "userinfo": re.compile("[^@]*"), + "host": re.compile("(\\[.*\\]|[^:]*)"), + "port": re.compile(".*"), +} + + +# We use these simple regexs as a first pass before handing off to +# the stdlib 'ipaddress' module for IP address validation. +IPv4_STYLE_HOSTNAME = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$") +IPv6_STYLE_HOSTNAME = re.compile(r"^\[.*\]$") + + +class ParseResult(typing.NamedTuple): + scheme: str + userinfo: str + host: str + port: int | None + path: str + query: str | None + fragment: str | None + + @property + def authority(self) -> str: + return "".join( + [ + f"{self.userinfo}@" if self.userinfo else "", + f"[{self.host}]" if ":" in self.host else self.host, + f":{self.port}" if self.port is not None else "", + ] + ) + + @property + def netloc(self) -> str: + return "".join( + [ + f"[{self.host}]" if ":" in self.host else self.host, + f":{self.port}" if self.port is not None else "", + ] + ) + + def copy_with(self, **kwargs: str | None) -> ParseResult: + if not kwargs: + return self + + defaults = { + "scheme": self.scheme, + "authority": self.authority, + "path": self.path, + "query": self.query, + "fragment": self.fragment, + } + defaults.update(kwargs) + return urlparse("", **defaults) + + def __str__(self) -> str: + authority = self.authority + return "".join( + [ + f"{self.scheme}:" if self.scheme else "", + f"//{authority}" if authority else "", + self.path, + f"?{self.query}" if self.query is not None else "", + f"#{self.fragment}" if self.fragment is not None else "", + ] + ) + + +def urlparse(url: str = "", **kwargs: str | None) -> ParseResult: + # Initial basic checks on allowable URLs. + # --------------------------------------- + + # Hard limit the maximum allowable URL length. + if len(url) > MAX_URL_LENGTH: + raise InvalidURL("URL too long") + + # If a URL includes any ASCII control characters including \t, \r, \n, + # then treat it as invalid. + if any(char.isascii() and not char.isprintable() for char in url): + char = next(char for char in url if char.isascii() and not char.isprintable()) + idx = url.find(char) + error = ( + f"Invalid non-printable ASCII character in URL, {char!r} at position {idx}." + ) + raise InvalidURL(error) + + # Some keyword arguments require special handling. + # ------------------------------------------------ + + # Coerce "port" to a string, if it is provided as an integer. + if "port" in kwargs: + port = kwargs["port"] + kwargs["port"] = str(port) if isinstance(port, int) else port + + # Replace "netloc" with "host and "port". + if "netloc" in kwargs: + netloc = kwargs.pop("netloc") or "" + kwargs["host"], _, kwargs["port"] = netloc.partition(":") + + # Replace "username" and/or "password" with "userinfo". + if "username" in kwargs or "password" in kwargs: + username = quote(kwargs.pop("username", "") or "", safe=USERNAME_SAFE) + password = quote(kwargs.pop("password", "") or "", safe=PASSWORD_SAFE) + kwargs["userinfo"] = f"{username}:{password}" if password else username + + # Replace "raw_path" with "path" and "query". + if "raw_path" in kwargs: + raw_path = kwargs.pop("raw_path") or "" + kwargs["path"], seperator, kwargs["query"] = raw_path.partition("?") + if not seperator: + kwargs["query"] = None + + # Ensure that IPv6 "host" addresses are always escaped with "[...]". + if "host" in kwargs: + host = kwargs.get("host") or "" + if ":" in host and not (host.startswith("[") and host.endswith("]")): + kwargs["host"] = f"[{host}]" + + # If any keyword arguments are provided, ensure they are valid. + # ------------------------------------------------------------- + + for key, value in kwargs.items(): + if value is not None: + if len(value) > MAX_URL_LENGTH: + raise InvalidURL(f"URL component '{key}' too long") + + # If a component includes any ASCII control characters including \t, \r, \n, + # then treat it as invalid. + if any(char.isascii() and not char.isprintable() for char in value): + char = next( + char for char in value if char.isascii() and not char.isprintable() + ) + idx = value.find(char) + error = ( + f"Invalid non-printable ASCII character in URL {key} component, " + f"{char!r} at position {idx}." + ) + raise InvalidURL(error) + + # Ensure that keyword arguments match as a valid regex. + if not COMPONENT_REGEX[key].fullmatch(value): + raise InvalidURL(f"Invalid URL component '{key}'") + + # The URL_REGEX will always match, but may have empty components. + url_match = URL_REGEX.match(url) + assert url_match is not None + url_dict = url_match.groupdict() + + # * 'scheme', 'authority', and 'path' may be empty strings. + # * 'query' may be 'None', indicating no trailing "?" portion. + # Any string including the empty string, indicates a trailing "?". + # * 'fragment' may be 'None', indicating no trailing "#" portion. + # Any string including the empty string, indicates a trailing "#". + scheme = kwargs.get("scheme", url_dict["scheme"]) or "" + authority = kwargs.get("authority", url_dict["authority"]) or "" + path = kwargs.get("path", url_dict["path"]) or "" + query = kwargs.get("query", url_dict["query"]) + frag = kwargs.get("fragment", url_dict["fragment"]) + + # The AUTHORITY_REGEX will always match, but may have empty components. + authority_match = AUTHORITY_REGEX.match(authority) + assert authority_match is not None + authority_dict = authority_match.groupdict() + + # * 'userinfo' and 'host' may be empty strings. + # * 'port' may be 'None'. + userinfo = kwargs.get("userinfo", authority_dict["userinfo"]) or "" + host = kwargs.get("host", authority_dict["host"]) or "" + port = kwargs.get("port", authority_dict["port"]) + + # Normalize and validate each component. + # We end up with a parsed representation of the URL, + # with components that are plain ASCII bytestrings. + parsed_scheme: str = scheme.lower() + parsed_userinfo: str = quote(userinfo, safe=USERINFO_SAFE) + parsed_host: str = encode_host(host) + parsed_port: int | None = normalize_port(port, scheme) + + has_scheme = parsed_scheme != "" + has_authority = ( + parsed_userinfo != "" or parsed_host != "" or parsed_port is not None + ) + validate_path(path, has_scheme=has_scheme, has_authority=has_authority) + if has_scheme or has_authority: + path = normalize_path(path) + + parsed_path: str = quote(path, safe=PATH_SAFE) + parsed_query: str | None = None if query is None else quote(query, safe=QUERY_SAFE) + parsed_frag: str | None = None if frag is None else quote(frag, safe=FRAG_SAFE) + + # The parsed ASCII bytestrings are our canonical form. + # All properties of the URL are derived from these. + return ParseResult( + parsed_scheme, + parsed_userinfo, + parsed_host, + parsed_port, + parsed_path, + parsed_query, + parsed_frag, + ) + + +def encode_host(host: str) -> str: + if not host: + return "" + + elif IPv4_STYLE_HOSTNAME.match(host): + # Validate IPv4 hostnames like #.#.#.# + # + # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2 + # + # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet + try: + ipaddress.IPv4Address(host) + except ipaddress.AddressValueError: + raise InvalidURL(f"Invalid IPv4 address: {host!r}") + return host + + elif IPv6_STYLE_HOSTNAME.match(host): + # Validate IPv6 hostnames like [...] + # + # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2 + # + # "A host identified by an Internet Protocol literal address, version 6 + # [RFC3513] or later, is distinguished by enclosing the IP literal + # within square brackets ("[" and "]"). This is the only place where + # square bracket characters are allowed in the URI syntax." + try: + ipaddress.IPv6Address(host[1:-1]) + except ipaddress.AddressValueError: + raise InvalidURL(f"Invalid IPv6 address: {host!r}") + return host[1:-1] + + elif host.isascii(): + # Regular ASCII hostnames + # + # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2 + # + # reg-name = *( unreserved / pct-encoded / sub-delims ) + WHATWG_SAFE = '"`{}%|\\' + return quote(host.lower(), safe=SUB_DELIMS + WHATWG_SAFE) + + # IDNA hostnames + try: + return idna.encode(host.lower()).decode("ascii") + except idna.IDNAError: + raise InvalidURL(f"Invalid IDNA hostname: {host!r}") + + +def normalize_port(port: str | int | None, scheme: str) -> int | None: + # From https://tools.ietf.org/html/rfc3986#section-3.2.3 + # + # "A scheme may define a default port. For example, the "http" scheme + # defines a default port of "80", corresponding to its reserved TCP + # port number. The type of port designated by the port number (e.g., + # TCP, UDP, SCTP) is defined by the URI scheme. URI producers and + # normalizers should omit the port component and its ":" delimiter if + # port is empty or if its value would be the same as that of the + # scheme's default." + if port is None or port == "": + return None + + try: + port_as_int = int(port) + except ValueError: + raise InvalidURL(f"Invalid port: {port!r}") + + # See https://url.spec.whatwg.org/#url-miscellaneous + default_port = {"ftp": 21, "http": 80, "https": 443, "ws": 80, "wss": 443}.get( + scheme + ) + if port_as_int == default_port: + return None + return port_as_int + + +def validate_path(path: str, has_scheme: bool, has_authority: bool) -> None: + """ + Path validation rules that depend on if the URL contains + a scheme or authority component. + + See https://datatracker.ietf.org/doc/html/rfc3986.html#section-3.3 + """ + if has_authority: + # If a URI contains an authority component, then the path component + # must either be empty or begin with a slash ("/") character." + if path and not path.startswith("/"): + raise InvalidURL("For absolute URLs, path must be empty or begin with '/'") + + if not has_scheme and not has_authority: + # If a URI does not contain an authority component, then the path cannot begin + # with two slash characters ("//"). + if path.startswith("//"): + raise InvalidURL("Relative URLs cannot have a path starting with '//'") + + # In addition, a URI reference (Section 4.1) may be a relative-path reference, + # in which case the first path segment cannot contain a colon (":") character. + if path.startswith(":"): + raise InvalidURL("Relative URLs cannot have a path starting with ':'") + + +def normalize_path(path: str) -> str: + """ + Drop "." and ".." segments from a URL path. + + For example: + + normalize_path("/path/./to/somewhere/..") == "/path/to" + """ + # Fast return when no '.' characters in the path. + if "." not in path: + return path + + components = path.split("/") + + # Fast return when no '.' or '..' components in the path. + if "." not in components and ".." not in components: + return path + + # https://datatracker.ietf.org/doc/html/rfc3986#section-5.2.4 + output: list[str] = [] + for component in components: + if component == ".": + pass + elif component == "..": + if output and output != [""]: + output.pop() + else: + output.append(component) + return "/".join(output) + + +def PERCENT(string: str) -> str: + return "".join([f"%{byte:02X}" for byte in string.encode("utf-8")]) + + +def percent_encoded(string: str, safe: str) -> str: + """ + Use percent-encoding to quote a string. + """ + NON_ESCAPED_CHARS = UNRESERVED_CHARACTERS + safe + + # Fast path for strings that don't need escaping. + if not string.rstrip(NON_ESCAPED_CHARS): + return string + + return "".join( + [char if char in NON_ESCAPED_CHARS else PERCENT(char) for char in string] + ) + + +def quote(string: str, safe: str) -> str: + """ + Use percent-encoding to quote a string, omitting existing '%xx' escape sequences. + + See: https://www.rfc-editor.org/rfc/rfc3986#section-2.1 + + * `string`: The string to be percent-escaped. + * `safe`: A string containing characters that may be treated as safe, and do not + need to be escaped. Unreserved characters are always treated as safe. + See: https://www.rfc-editor.org/rfc/rfc3986#section-2.3 + """ + parts = [] + current_position = 0 + for match in re.finditer(PERCENT_ENCODED_REGEX, string): + start_position, end_position = match.start(), match.end() + matched_text = match.group(0) + # Add any text up to the '%xx' escape sequence. + if start_position != current_position: + leading_text = string[current_position:start_position] + parts.append(percent_encoded(leading_text, safe=safe)) + + # Add the '%xx' escape sequence. + parts.append(matched_text) + current_position = end_position + + # Add any text after the final '%xx' escape sequence. + if current_position != len(string): + trailing_text = string[current_position:] + parts.append(percent_encoded(trailing_text, safe=safe)) + + return "".join(parts) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_urls.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_urls.py new file mode 100644 index 0000000000000000000000000000000000000000..147a8fa333acaf31618d37ba2896e3a5bf5e4d02 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_urls.py @@ -0,0 +1,641 @@ +from __future__ import annotations + +import typing +from urllib.parse import parse_qs, unquote, urlencode + +import idna + +from ._types import QueryParamTypes +from ._urlparse import urlparse +from ._utils import primitive_value_to_str + +__all__ = ["URL", "QueryParams"] + + +class URL: + """ + url = httpx.URL("HTTPS://jo%40email.com:a%20secret@müller.de:1234/pa%20th?search=ab#anchorlink") + + assert url.scheme == "https" + assert url.username == "jo@email.com" + assert url.password == "a secret" + assert url.userinfo == b"jo%40email.com:a%20secret" + assert url.host == "müller.de" + assert url.raw_host == b"xn--mller-kva.de" + assert url.port == 1234 + assert url.netloc == b"xn--mller-kva.de:1234" + assert url.path == "/pa th" + assert url.query == b"?search=ab" + assert url.raw_path == b"/pa%20th?search=ab" + assert url.fragment == "anchorlink" + + The components of a URL are broken down like this: + + https://jo%40email.com:a%20secret@müller.de:1234/pa%20th?search=ab#anchorlink + [scheme] [ username ] [password] [ host ][port][ path ] [ query ] [fragment] + [ userinfo ] [ netloc ][ raw_path ] + + Note that: + + * `url.scheme` is normalized to always be lowercased. + + * `url.host` is normalized to always be lowercased. Internationalized domain + names are represented in unicode, without IDNA encoding applied. For instance: + + url = httpx.URL("http://中国.icom.museum") + assert url.host == "中国.icom.museum" + url = httpx.URL("http://xn--fiqs8s.icom.museum") + assert url.host == "中国.icom.museum" + + * `url.raw_host` is normalized to always be lowercased, and is IDNA encoded. + + url = httpx.URL("http://中国.icom.museum") + assert url.raw_host == b"xn--fiqs8s.icom.museum" + url = httpx.URL("http://xn--fiqs8s.icom.museum") + assert url.raw_host == b"xn--fiqs8s.icom.museum" + + * `url.port` is either None or an integer. URLs that include the default port for + "http", "https", "ws", "wss", and "ftp" schemes have their port + normalized to `None`. + + assert httpx.URL("http://example.com") == httpx.URL("http://example.com:80") + assert httpx.URL("http://example.com").port is None + assert httpx.URL("http://example.com:80").port is None + + * `url.userinfo` is raw bytes, without URL escaping. Usually you'll want to work + with `url.username` and `url.password` instead, which handle the URL escaping. + + * `url.raw_path` is raw bytes of both the path and query, without URL escaping. + This portion is used as the target when constructing HTTP requests. Usually you'll + want to work with `url.path` instead. + + * `url.query` is raw bytes, without URL escaping. A URL query string portion can + only be properly URL escaped when decoding the parameter names and values + themselves. + """ + + def __init__(self, url: URL | str = "", **kwargs: typing.Any) -> None: + if kwargs: + allowed = { + "scheme": str, + "username": str, + "password": str, + "userinfo": bytes, + "host": str, + "port": int, + "netloc": bytes, + "path": str, + "query": bytes, + "raw_path": bytes, + "fragment": str, + "params": object, + } + + # Perform type checking for all supported keyword arguments. + for key, value in kwargs.items(): + if key not in allowed: + message = f"{key!r} is an invalid keyword argument for URL()" + raise TypeError(message) + if value is not None and not isinstance(value, allowed[key]): + expected = allowed[key].__name__ + seen = type(value).__name__ + message = f"Argument {key!r} must be {expected} but got {seen}" + raise TypeError(message) + if isinstance(value, bytes): + kwargs[key] = value.decode("ascii") + + if "params" in kwargs: + # Replace any "params" keyword with the raw "query" instead. + # + # Ensure that empty params use `kwargs["query"] = None` rather + # than `kwargs["query"] = ""`, so that generated URLs do not + # include an empty trailing "?". + params = kwargs.pop("params") + kwargs["query"] = None if not params else str(QueryParams(params)) + + if isinstance(url, str): + self._uri_reference = urlparse(url, **kwargs) + elif isinstance(url, URL): + self._uri_reference = url._uri_reference.copy_with(**kwargs) + else: + raise TypeError( + "Invalid type for url. Expected str or httpx.URL," + f" got {type(url)}: {url!r}" + ) + + @property + def scheme(self) -> str: + """ + The URL scheme, such as "http", "https". + Always normalised to lowercase. + """ + return self._uri_reference.scheme + + @property + def raw_scheme(self) -> bytes: + """ + The raw bytes representation of the URL scheme, such as b"http", b"https". + Always normalised to lowercase. + """ + return self._uri_reference.scheme.encode("ascii") + + @property + def userinfo(self) -> bytes: + """ + The URL userinfo as a raw bytestring. + For example: b"jo%40email.com:a%20secret". + """ + return self._uri_reference.userinfo.encode("ascii") + + @property + def username(self) -> str: + """ + The URL username as a string, with URL decoding applied. + For example: "jo@email.com" + """ + userinfo = self._uri_reference.userinfo + return unquote(userinfo.partition(":")[0]) + + @property + def password(self) -> str: + """ + The URL password as a string, with URL decoding applied. + For example: "a secret" + """ + userinfo = self._uri_reference.userinfo + return unquote(userinfo.partition(":")[2]) + + @property + def host(self) -> str: + """ + The URL host as a string. + Always normalized to lowercase, with IDNA hosts decoded into unicode. + + Examples: + + url = httpx.URL("http://www.EXAMPLE.org") + assert url.host == "www.example.org" + + url = httpx.URL("http://中国.icom.museum") + assert url.host == "中国.icom.museum" + + url = httpx.URL("http://xn--fiqs8s.icom.museum") + assert url.host == "中国.icom.museum" + + url = httpx.URL("https://[::ffff:192.168.0.1]") + assert url.host == "::ffff:192.168.0.1" + """ + host: str = self._uri_reference.host + + if host.startswith("xn--"): + host = idna.decode(host) + + return host + + @property + def raw_host(self) -> bytes: + """ + The raw bytes representation of the URL host. + Always normalized to lowercase, and IDNA encoded. + + Examples: + + url = httpx.URL("http://www.EXAMPLE.org") + assert url.raw_host == b"www.example.org" + + url = httpx.URL("http://中国.icom.museum") + assert url.raw_host == b"xn--fiqs8s.icom.museum" + + url = httpx.URL("http://xn--fiqs8s.icom.museum") + assert url.raw_host == b"xn--fiqs8s.icom.museum" + + url = httpx.URL("https://[::ffff:192.168.0.1]") + assert url.raw_host == b"::ffff:192.168.0.1" + """ + return self._uri_reference.host.encode("ascii") + + @property + def port(self) -> int | None: + """ + The URL port as an integer. + + Note that the URL class performs port normalization as per the WHATWG spec. + Default ports for "http", "https", "ws", "wss", and "ftp" schemes are always + treated as `None`. + + For example: + + assert httpx.URL("http://www.example.com") == httpx.URL("http://www.example.com:80") + assert httpx.URL("http://www.example.com:80").port is None + """ + return self._uri_reference.port + + @property + def netloc(self) -> bytes: + """ + Either `` or `:` as bytes. + Always normalized to lowercase, and IDNA encoded. + + This property may be used for generating the value of a request + "Host" header. + """ + return self._uri_reference.netloc.encode("ascii") + + @property + def path(self) -> str: + """ + The URL path as a string. Excluding the query string, and URL decoded. + + For example: + + url = httpx.URL("https://example.com/pa%20th") + assert url.path == "/pa th" + """ + path = self._uri_reference.path or "/" + return unquote(path) + + @property + def query(self) -> bytes: + """ + The URL query string, as raw bytes, excluding the leading b"?". + + This is necessarily a bytewise interface, because we cannot + perform URL decoding of this representation until we've parsed + the keys and values into a QueryParams instance. + + For example: + + url = httpx.URL("https://example.com/?filter=some%20search%20terms") + assert url.query == b"filter=some%20search%20terms" + """ + query = self._uri_reference.query or "" + return query.encode("ascii") + + @property + def params(self) -> QueryParams: + """ + The URL query parameters, neatly parsed and packaged into an immutable + multidict representation. + """ + return QueryParams(self._uri_reference.query) + + @property + def raw_path(self) -> bytes: + """ + The complete URL path and query string as raw bytes. + Used as the target when constructing HTTP requests. + + For example: + + GET /users?search=some%20text HTTP/1.1 + Host: www.example.org + Connection: close + """ + path = self._uri_reference.path or "/" + if self._uri_reference.query is not None: + path += "?" + self._uri_reference.query + return path.encode("ascii") + + @property + def fragment(self) -> str: + """ + The URL fragments, as used in HTML anchors. + As a string, without the leading '#'. + """ + return unquote(self._uri_reference.fragment or "") + + @property + def is_absolute_url(self) -> bool: + """ + Return `True` for absolute URLs such as 'http://example.com/path', + and `False` for relative URLs such as '/path'. + """ + # We don't use `.is_absolute` from `rfc3986` because it treats + # URLs with a fragment portion as not absolute. + # What we actually care about is if the URL provides + # a scheme and hostname to which connections should be made. + return bool(self._uri_reference.scheme and self._uri_reference.host) + + @property + def is_relative_url(self) -> bool: + """ + Return `False` for absolute URLs such as 'http://example.com/path', + and `True` for relative URLs such as '/path'. + """ + return not self.is_absolute_url + + def copy_with(self, **kwargs: typing.Any) -> URL: + """ + Copy this URL, returning a new URL with some components altered. + Accepts the same set of parameters as the components that are made + available via properties on the `URL` class. + + For example: + + url = httpx.URL("https://www.example.com").copy_with( + username="jo@gmail.com", password="a secret" + ) + assert url == "https://jo%40email.com:a%20secret@www.example.com" + """ + return URL(self, **kwargs) + + def copy_set_param(self, key: str, value: typing.Any = None) -> URL: + return self.copy_with(params=self.params.set(key, value)) + + def copy_add_param(self, key: str, value: typing.Any = None) -> URL: + return self.copy_with(params=self.params.add(key, value)) + + def copy_remove_param(self, key: str) -> URL: + return self.copy_with(params=self.params.remove(key)) + + def copy_merge_params(self, params: QueryParamTypes) -> URL: + return self.copy_with(params=self.params.merge(params)) + + def join(self, url: URL | str) -> URL: + """ + Return an absolute URL, using this URL as the base. + + Eg. + + url = httpx.URL("https://www.example.com/test") + url = url.join("/new/path") + assert url == "https://www.example.com/new/path" + """ + from urllib.parse import urljoin + + return URL(urljoin(str(self), str(URL(url)))) + + def __hash__(self) -> int: + return hash(str(self)) + + def __eq__(self, other: typing.Any) -> bool: + return isinstance(other, (URL, str)) and str(self) == str(URL(other)) + + def __str__(self) -> str: + return str(self._uri_reference) + + def __repr__(self) -> str: + scheme, userinfo, host, port, path, query, fragment = self._uri_reference + + if ":" in userinfo: + # Mask any password component. + userinfo = f'{userinfo.split(":")[0]}:[secure]' + + authority = "".join( + [ + f"{userinfo}@" if userinfo else "", + f"[{host}]" if ":" in host else host, + f":{port}" if port is not None else "", + ] + ) + url = "".join( + [ + f"{self.scheme}:" if scheme else "", + f"//{authority}" if authority else "", + path, + f"?{query}" if query is not None else "", + f"#{fragment}" if fragment is not None else "", + ] + ) + + return f"{self.__class__.__name__}({url!r})" + + @property + def raw(self) -> tuple[bytes, bytes, int, bytes]: # pragma: nocover + import collections + import warnings + + warnings.warn("URL.raw is deprecated.") + RawURL = collections.namedtuple( + "RawURL", ["raw_scheme", "raw_host", "port", "raw_path"] + ) + return RawURL( + raw_scheme=self.raw_scheme, + raw_host=self.raw_host, + port=self.port, + raw_path=self.raw_path, + ) + + +class QueryParams(typing.Mapping[str, str]): + """ + URL query parameters, as a multi-dict. + """ + + def __init__(self, *args: QueryParamTypes | None, **kwargs: typing.Any) -> None: + assert len(args) < 2, "Too many arguments." + assert not (args and kwargs), "Cannot mix named and unnamed arguments." + + value = args[0] if args else kwargs + + if value is None or isinstance(value, (str, bytes)): + value = value.decode("ascii") if isinstance(value, bytes) else value + self._dict = parse_qs(value, keep_blank_values=True) + elif isinstance(value, QueryParams): + self._dict = {k: list(v) for k, v in value._dict.items()} + else: + dict_value: dict[typing.Any, list[typing.Any]] = {} + if isinstance(value, (list, tuple)): + # Convert list inputs like: + # [("a", "123"), ("a", "456"), ("b", "789")] + # To a dict representation, like: + # {"a": ["123", "456"], "b": ["789"]} + for item in value: + dict_value.setdefault(item[0], []).append(item[1]) + else: + # Convert dict inputs like: + # {"a": "123", "b": ["456", "789"]} + # To dict inputs where values are always lists, like: + # {"a": ["123"], "b": ["456", "789"]} + dict_value = { + k: list(v) if isinstance(v, (list, tuple)) else [v] + for k, v in value.items() + } + + # Ensure that keys and values are neatly coerced to strings. + # We coerce values `True` and `False` to JSON-like "true" and "false" + # representations, and coerce `None` values to the empty string. + self._dict = { + str(k): [primitive_value_to_str(item) for item in v] + for k, v in dict_value.items() + } + + def keys(self) -> typing.KeysView[str]: + """ + Return all the keys in the query params. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert list(q.keys()) == ["a", "b"] + """ + return self._dict.keys() + + def values(self) -> typing.ValuesView[str]: + """ + Return all the values in the query params. If a key occurs more than once + only the first item for that key is returned. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert list(q.values()) == ["123", "789"] + """ + return {k: v[0] for k, v in self._dict.items()}.values() + + def items(self) -> typing.ItemsView[str, str]: + """ + Return all items in the query params. If a key occurs more than once + only the first item for that key is returned. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert list(q.items()) == [("a", "123"), ("b", "789")] + """ + return {k: v[0] for k, v in self._dict.items()}.items() + + def multi_items(self) -> list[tuple[str, str]]: + """ + Return all items in the query params. Allow duplicate keys to occur. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert list(q.multi_items()) == [("a", "123"), ("a", "456"), ("b", "789")] + """ + multi_items: list[tuple[str, str]] = [] + for k, v in self._dict.items(): + multi_items.extend([(k, i) for i in v]) + return multi_items + + def get(self, key: typing.Any, default: typing.Any = None) -> typing.Any: + """ + Get a value from the query param for a given key. If the key occurs + more than once, then only the first value is returned. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert q.get("a") == "123" + """ + if key in self._dict: + return self._dict[str(key)][0] + return default + + def get_list(self, key: str) -> list[str]: + """ + Get all values from the query param for a given key. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert q.get_list("a") == ["123", "456"] + """ + return list(self._dict.get(str(key), [])) + + def set(self, key: str, value: typing.Any = None) -> QueryParams: + """ + Return a new QueryParams instance, setting the value of a key. + + Usage: + + q = httpx.QueryParams("a=123") + q = q.set("a", "456") + assert q == httpx.QueryParams("a=456") + """ + q = QueryParams() + q._dict = dict(self._dict) + q._dict[str(key)] = [primitive_value_to_str(value)] + return q + + def add(self, key: str, value: typing.Any = None) -> QueryParams: + """ + Return a new QueryParams instance, setting or appending the value of a key. + + Usage: + + q = httpx.QueryParams("a=123") + q = q.add("a", "456") + assert q == httpx.QueryParams("a=123&a=456") + """ + q = QueryParams() + q._dict = dict(self._dict) + q._dict[str(key)] = q.get_list(key) + [primitive_value_to_str(value)] + return q + + def remove(self, key: str) -> QueryParams: + """ + Return a new QueryParams instance, removing the value of a key. + + Usage: + + q = httpx.QueryParams("a=123") + q = q.remove("a") + assert q == httpx.QueryParams("") + """ + q = QueryParams() + q._dict = dict(self._dict) + q._dict.pop(str(key), None) + return q + + def merge(self, params: QueryParamTypes | None = None) -> QueryParams: + """ + Return a new QueryParams instance, updated with. + + Usage: + + q = httpx.QueryParams("a=123") + q = q.merge({"b": "456"}) + assert q == httpx.QueryParams("a=123&b=456") + + q = httpx.QueryParams("a=123") + q = q.merge({"a": "456", "b": "789"}) + assert q == httpx.QueryParams("a=456&b=789") + """ + q = QueryParams(params) + q._dict = {**self._dict, **q._dict} + return q + + def __getitem__(self, key: typing.Any) -> str: + return self._dict[key][0] + + def __contains__(self, key: typing.Any) -> bool: + return key in self._dict + + def __iter__(self) -> typing.Iterator[typing.Any]: + return iter(self.keys()) + + def __len__(self) -> int: + return len(self._dict) + + def __bool__(self) -> bool: + return bool(self._dict) + + def __hash__(self) -> int: + return hash(str(self)) + + def __eq__(self, other: typing.Any) -> bool: + if not isinstance(other, self.__class__): + return False + return sorted(self.multi_items()) == sorted(other.multi_items()) + + def __str__(self) -> str: + return urlencode(self.multi_items()) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + query_string = str(self) + return f"{class_name}({query_string!r})" + + def update(self, params: QueryParamTypes | None = None) -> None: + raise RuntimeError( + "QueryParams are immutable since 0.18.0. " + "Use `q = q.merge(...)` to create an updated copy." + ) + + def __setitem__(self, key: str, value: str) -> None: + raise RuntimeError( + "QueryParams are immutable since 0.18.0. " + "Use `q = q.set(key, value)` to create an updated copy." + ) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_utils.py b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7fe827da4d071b32ea6da44328629699d6fc88ce --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/_utils.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import ipaddress +import os +import re +import typing +from urllib.request import getproxies + +from ._types import PrimitiveData + +if typing.TYPE_CHECKING: # pragma: no cover + from ._urls import URL + + +def primitive_value_to_str(value: PrimitiveData) -> str: + """ + Coerce a primitive data type into a string value. + + Note that we prefer JSON-style 'true'/'false' for boolean values here. + """ + if value is True: + return "true" + elif value is False: + return "false" + elif value is None: + return "" + return str(value) + + +def get_environment_proxies() -> dict[str, str | None]: + """Gets proxy information from the environment""" + + # urllib.request.getproxies() falls back on System + # Registry and Config for proxies on Windows and macOS. + # We don't want to propagate non-HTTP proxies into + # our configuration such as 'TRAVIS_APT_PROXY'. + proxy_info = getproxies() + mounts: dict[str, str | None] = {} + + for scheme in ("http", "https", "all"): + if proxy_info.get(scheme): + hostname = proxy_info[scheme] + mounts[f"{scheme}://"] = ( + hostname if "://" in hostname else f"http://{hostname}" + ) + + no_proxy_hosts = [host.strip() for host in proxy_info.get("no", "").split(",")] + for hostname in no_proxy_hosts: + # See https://curl.haxx.se/libcurl/c/CURLOPT_NOPROXY.html for details + # on how names in `NO_PROXY` are handled. + if hostname == "*": + # If NO_PROXY=* is used or if "*" occurs as any one of the comma + # separated hostnames, then we should just bypass any information + # from HTTP_PROXY, HTTPS_PROXY, ALL_PROXY, and always ignore + # proxies. + return {} + elif hostname: + # NO_PROXY=.google.com is marked as "all://*.google.com, + # which disables "www.google.com" but not "google.com" + # NO_PROXY=google.com is marked as "all://*google.com, + # which disables "www.google.com" and "google.com". + # (But not "wwwgoogle.com") + # NO_PROXY can include domains, IPv6, IPv4 addresses and "localhost" + # NO_PROXY=example.com,::1,localhost,192.168.0.0/16 + if "://" in hostname: + mounts[hostname] = None + elif is_ipv4_hostname(hostname): + mounts[f"all://{hostname}"] = None + elif is_ipv6_hostname(hostname): + mounts[f"all://[{hostname}]"] = None + elif hostname.lower() == "localhost": + mounts[f"all://{hostname}"] = None + else: + mounts[f"all://*{hostname}"] = None + + return mounts + + +def to_bytes(value: str | bytes, encoding: str = "utf-8") -> bytes: + return value.encode(encoding) if isinstance(value, str) else value + + +def to_str(value: str | bytes, encoding: str = "utf-8") -> str: + return value if isinstance(value, str) else value.decode(encoding) + + +def to_bytes_or_str(value: str, match_type_of: typing.AnyStr) -> typing.AnyStr: + return value if isinstance(match_type_of, str) else value.encode() + + +def unquote(value: str) -> str: + return value[1:-1] if value[0] == value[-1] == '"' else value + + +def peek_filelike_length(stream: typing.Any) -> int | None: + """ + Given a file-like stream object, return its length in number of bytes + without reading it into memory. + """ + try: + # Is it an actual file? + fd = stream.fileno() + # Yup, seems to be an actual file. + length = os.fstat(fd).st_size + except (AttributeError, OSError): + # No... Maybe it's something that supports random access, like `io.BytesIO`? + try: + # Assuming so, go to end of stream to figure out its length, + # then put it back in place. + offset = stream.tell() + length = stream.seek(0, os.SEEK_END) + stream.seek(offset) + except (AttributeError, OSError): + # Not even that? Sorry, we're doomed... + return None + + return length + + +class URLPattern: + """ + A utility class currently used for making lookups against proxy keys... + + # Wildcard matching... + >>> pattern = URLPattern("all://") + >>> pattern.matches(httpx.URL("http://example.com")) + True + + # Witch scheme matching... + >>> pattern = URLPattern("https://") + >>> pattern.matches(httpx.URL("https://example.com")) + True + >>> pattern.matches(httpx.URL("http://example.com")) + False + + # With domain matching... + >>> pattern = URLPattern("https://example.com") + >>> pattern.matches(httpx.URL("https://example.com")) + True + >>> pattern.matches(httpx.URL("http://example.com")) + False + >>> pattern.matches(httpx.URL("https://other.com")) + False + + # Wildcard scheme, with domain matching... + >>> pattern = URLPattern("all://example.com") + >>> pattern.matches(httpx.URL("https://example.com")) + True + >>> pattern.matches(httpx.URL("http://example.com")) + True + >>> pattern.matches(httpx.URL("https://other.com")) + False + + # With port matching... + >>> pattern = URLPattern("https://example.com:1234") + >>> pattern.matches(httpx.URL("https://example.com:1234")) + True + >>> pattern.matches(httpx.URL("https://example.com")) + False + """ + + def __init__(self, pattern: str) -> None: + from ._urls import URL + + if pattern and ":" not in pattern: + raise ValueError( + f"Proxy keys should use proper URL forms rather " + f"than plain scheme strings. " + f'Instead of "{pattern}", use "{pattern}://"' + ) + + url = URL(pattern) + self.pattern = pattern + self.scheme = "" if url.scheme == "all" else url.scheme + self.host = "" if url.host == "*" else url.host + self.port = url.port + if not url.host or url.host == "*": + self.host_regex: typing.Pattern[str] | None = None + elif url.host.startswith("*."): + # *.example.com should match "www.example.com", but not "example.com" + domain = re.escape(url.host[2:]) + self.host_regex = re.compile(f"^.+\\.{domain}$") + elif url.host.startswith("*"): + # *example.com should match "www.example.com" and "example.com" + domain = re.escape(url.host[1:]) + self.host_regex = re.compile(f"^(.+\\.)?{domain}$") + else: + # example.com should match "example.com" but not "www.example.com" + domain = re.escape(url.host) + self.host_regex = re.compile(f"^{domain}$") + + def matches(self, other: URL) -> bool: + if self.scheme and self.scheme != other.scheme: + return False + if ( + self.host + and self.host_regex is not None + and not self.host_regex.match(other.host) + ): + return False + if self.port is not None and self.port != other.port: + return False + return True + + @property + def priority(self) -> tuple[int, int, int]: + """ + The priority allows URLPattern instances to be sortable, so that + we can match from most specific to least specific. + """ + # URLs with a port should take priority over URLs without a port. + port_priority = 0 if self.port is not None else 1 + # Longer hostnames should match first. + host_priority = -len(self.host) + # Longer schemes should match first. + scheme_priority = -len(self.scheme) + return (port_priority, host_priority, scheme_priority) + + def __hash__(self) -> int: + return hash(self.pattern) + + def __lt__(self, other: URLPattern) -> bool: + return self.priority < other.priority + + def __eq__(self, other: typing.Any) -> bool: + return isinstance(other, URLPattern) and self.pattern == other.pattern + + +def is_ipv4_hostname(hostname: str) -> bool: + try: + ipaddress.IPv4Address(hostname.split("/")[0]) + except Exception: + return False + return True + + +def is_ipv6_hostname(hostname: str) -> bool: + try: + ipaddress.IPv6Address(hostname.split("/")[0]) + except Exception: + return False + return True diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/py.typed b/bioagent-bench/.venv/lib/python3.14/site-packages/httpx/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/INSTALLER b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..5c69047b2eb8235994febeeae1da4a82365a240a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/INSTALLER @@ -0,0 +1 @@ +uv \ No newline at end of file diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/METADATA b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..d7ca45662842ba99c3ecaf0c6771029f4afef966 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/METADATA @@ -0,0 +1,112 @@ +Metadata-Version: 2.4 +Name: packaging +Version: 26.2 +Summary: Core utilities for Python packages +Author-email: Donald Stufft +Requires-Python: >=3.8 +Description-Content-Type: text/x-rst +License-Expression: Apache-2.0 OR BSD-2-Clause +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Programming Language :: Python :: Free Threading :: 4 - Resilient +Classifier: Typing :: Typed +License-File: LICENSE +License-File: LICENSE.APACHE +License-File: LICENSE.BSD +Project-URL: Documentation, https://packaging.pypa.io/ +Project-URL: Source, https://github.com/pypa/packaging + +packaging +========= + +.. start-intro + +Reusable core utilities for various Python Packaging +`interoperability specifications `_. + +This library provides utilities that implement the interoperability +specifications which have clearly one correct behaviour (eg: :pep:`440`) +or benefit greatly from having a single shared implementation (eg: :pep:`425`). + +.. end-intro + +The ``packaging`` project includes the following: version handling, specifiers, +markers, requirements, tags, metadata, lockfiles, utilities. + +Documentation +------------- + +The `documentation`_ provides information and the API for the following: + +- Version Handling +- Specifiers +- Markers +- Licenses +- Requirements +- Metadata +- Tags +- Lockfiles (pylock) +- Direct URL helpers +- Dependency groups +- Errors +- Utilities + +Installation +------------ + +Use ``pip`` to install these utilities:: + + pip install packaging + +The ``packaging`` library uses calendar-based versioning (``YY.N``). + +Discussion +---------- + +If you run into bugs, you can file them in our `issue tracker`_. + +You can also join discussions on `GitHub Discussions`_ to ask questions or get involved. + +.. _`documentation`: https://packaging.pypa.io/ +.. _`issue tracker`: https://github.com/pypa/packaging/issues +.. _`GitHub Discussions`: https://github.com/pypa/packaging/discussions + + +Code of Conduct +--------------- + +Everyone interacting in the packaging project's codebases, issue trackers, chat +rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. + +.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md + +Contributing +------------ + +The ``CONTRIBUTING.rst`` file outlines how to contribute to this project as +well as how to report a potential security issue. The documentation for this +project also covers information about `project development`_ and `security`_. + +.. _`project development`: https://packaging.pypa.io/en/latest/development/ +.. _`security`: https://packaging.pypa.io/en/latest/security/ + +Project History +--------------- + +Please review the ``CHANGELOG.rst`` file or the `Changelog documentation`_ for +recent changes and project history. + +.. _`Changelog documentation`: https://packaging.pypa.io/en/latest/changelog/ + diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/RECORD b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..e84f245b3893ebefb3f23bf4afce3da25f3fb2f1 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/RECORD @@ -0,0 +1,29 @@ +packaging-26.2.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2 +packaging-26.2.dist-info/METADATA,sha256=T5y815M0FaR5P3dnyYoralEsgj_IHIczeBVwXyMOyr8,3543 +packaging-26.2.dist-info/RECORD,, +packaging-26.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +packaging-26.2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 +packaging-26.2.dist-info/licenses/LICENSE,sha256=ytHvW9NA1z4HS6YU0m996spceUDD2MNIUuZcSQlobEg,197 +packaging-26.2.dist-info/licenses/LICENSE.APACHE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174 +packaging-26.2.dist-info/licenses/LICENSE.BSD,sha256=tw5-m3QvHMb5SLNMFqo5_-zpQZY2S8iP8NIYDwAo-sU,1344 +packaging/__init__.py,sha256=QhMEdPu2XogrJzV3S0KWS6t7l0I9k8EeDRJl4fnw87s,494 +packaging/_elffile.py,sha256=-sKkptYqzYw2-x3QByJa5mB4rfPWu1pxkZHRx1WAFCY,3211 +packaging/_manylinux.py,sha256=Hf6nB0cOrayEs96-p3oIXAgGnFquv20DO5l-o2_Xnv0,9559 +packaging/_musllinux.py,sha256=Z6swjH3MA7XS3qXnmMN7QPhqP3fnoYI0eQ18e9-HgAE,2707 +packaging/_parser.py,sha256=Kf2nsDw4c54X82pY8ba4F02Bve6OygGMAjL-Begqcew,11698 +packaging/_structures.py,sha256=60jRbF78p8z5MKnNd6cAprgOadCJHV0DlmUmRBqFZcs,1109 +packaging/_tokenizer.py,sha256=tFU2Wr-ZZJdAbkXLEJo7qUQDJaIkfft9DqaifiEND7A,5391 +packaging/dependency_groups.py,sha256=XZIAVFK9uHG4RCGprmJn3VInUWMesxha_kytJuMO9eY,10218 +packaging/direct_url.py,sha256=eKmbDiPP1sLV4Mj_kCSZqqknrIyVO9Sr7JpF8KCjp4U,10917 +packaging/errors.py,sha256=6hfEYXAf8v_IF65-lFadJOMIieBP2xIKtyEXjG1nGIs,2680 +packaging/licenses/__init__.py,sha256=_Jx0XRiD_58palsWnyLrLuh59ZpGCPIPXLKdZo9OJvQ,7293 +packaging/licenses/_spdx.py,sha256=WW7DXiyg68up_YND_wpRYlr1SHhiV4FfJLQffghhMxQ,51122 +packaging/markers.py,sha256=8fDIUhAF6YMnCNB5FSiwh9pEIusiFzAF73J-0OB8bTk,17055 +packaging/metadata.py,sha256=crAh0E3GVGVqPlu6EdRFsaG-Y6UYznTUqjuGKRGPv6c,38770 +packaging/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +packaging/pylock.py,sha256=G_1gncTmDbRLY1jo4VDI9Uw-b5IErh_Q9V_BbVJTmD8,33890 +packaging/requirements.py,sha256=dd1c9aa1gp5NI6btF6UFRQjPn1nxQXnE_T34yDDTEpc,4383 +packaging/specifiers.py,sha256=Mfp8avQg0lVot17to9lVKBtZD1FsWBTItoGwFUZ3wtg,71514 +packaging/tags.py,sha256=NQ1weo69_Sjte3xBZ1I_G63CIgCmaN0C24mz-z3hGYo,34224 +packaging/utils.py,sha256=M7-JMKic2sP1YtV_8aW7eVGB-x3ADuKCiSrsVeCd2Uo,9848 +packaging/version.py,sha256=Y1aTtxe3sn2xOMa5BdI85-AcHuybbanOVkEvvSRRC8I,38369 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/REQUESTED b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/WHEEL b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..d8b9936dad9ab2513fa6979f411560d3b6b57e37 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: flit 3.12.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/licenses/LICENSE b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..6f62d44e4ef733c0e713afcd2371fed7f2b3de67 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/licenses/LICENSE @@ -0,0 +1,3 @@ +This software is made available under the terms of *either* of the licenses +found in LICENSE.APACHE or LICENSE.BSD. Contributions to this software is made +under the terms of *both* these licenses. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/licenses/LICENSE.APACHE b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/licenses/LICENSE.APACHE new file mode 100644 index 0000000000000000000000000000000000000000..f433b1a53f5b830a205fd2df78e2b34974656c7b --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/licenses/LICENSE.APACHE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/licenses/LICENSE.BSD b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/licenses/LICENSE.BSD new file mode 100644 index 0000000000000000000000000000000000000000..42ce7b75c92fb01a3f6ed17eea363f756b7da582 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/packaging-26.2.dist-info/licenses/LICENSE.BSD @@ -0,0 +1,23 @@ +Copyright (c) Donald Stufft and individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3a5ba538b05cab44263485e2ba88ef466f230259 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/__init__.py @@ -0,0 +1,82 @@ +""" + Pygments + ~~~~~~~~ + + Pygments is a syntax highlighting package written in Python. + + It is a generic syntax highlighter for general use in all kinds of software + such as forum systems, wikis or other applications that need to prettify + source code. Highlights are: + + * a wide range of common languages and markup formats is supported + * special attention is paid to details, increasing quality by a fair amount + * support for new languages and formats are added easily + * a number of output formats, presently HTML, LaTeX, RTF, SVG, all image + formats that PIL supports, and ANSI sequences + * it is usable as a command-line tool and as a library + * ... and it highlights even Brainfuck! + + The `Pygments master branch`_ is installable with ``easy_install Pygments==dev``. + + .. _Pygments master branch: + https://github.com/pygments/pygments/archive/master.zip#egg=Pygments-dev + + :copyright: Copyright 2006-present by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +from io import StringIO, BytesIO + +__version__ = '2.20.0' +__docformat__ = 'restructuredtext' + +__all__ = ['lex', 'format', 'highlight'] + + +def lex(code, lexer): + """ + Lex `code` with the `lexer` (must be a `Lexer` instance) + and return an iterable of tokens. Currently, this only calls + `lexer.get_tokens()`. + """ + try: + return lexer.get_tokens(code) + except TypeError: + # Heuristic to catch a common mistake. + from pygments.lexer import RegexLexer + if isinstance(lexer, type) and issubclass(lexer, RegexLexer): + raise TypeError('lex() argument must be a lexer instance, ' + 'not a class') + raise + + +def format(tokens, formatter, outfile=None): # pylint: disable=redefined-builtin + """ + Format ``tokens`` (an iterable of tokens) with the formatter ``formatter`` + (a `Formatter` instance). + + If ``outfile`` is given and a valid file object (an object with a + ``write`` method), the result will be written to it, otherwise it + is returned as a string. + """ + try: + if not outfile: + realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO() + formatter.format(tokens, realoutfile) + return realoutfile.getvalue() + else: + formatter.format(tokens, outfile) + except TypeError: + # Heuristic to catch a common mistake. + from pygments.formatter import Formatter + if isinstance(formatter, type) and issubclass(formatter, Formatter): + raise TypeError('format() argument must be a formatter instance, ' + 'not a class') + raise + + +def highlight(code, lexer, formatter, outfile=None): + """ + This is the most high-level highlighting function. It combines `lex` and + `format` in one function. + """ + return format(lex(code, lexer), formatter, outfile) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/__main__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..818bfc3ae6ac4b79e029f714c56101ab2b1076ea --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/__main__.py @@ -0,0 +1,17 @@ +""" + pygments.__main__ + ~~~~~~~~~~~~~~~~~ + + Main entry point for ``python -m pygments``. + + :copyright: Copyright 2006-present by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import sys +import pygments.cmdline + +try: + sys.exit(pygments.cmdline.main(sys.argv)) +except KeyboardInterrupt: + sys.exit(1) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/cmdline.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/cmdline.py new file mode 100644 index 0000000000000000000000000000000000000000..bd4a1fb43926b7f7bdd7ef5028dbe19a9137515a --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/cmdline.py @@ -0,0 +1,668 @@ +""" + pygments.cmdline + ~~~~~~~~~~~~~~~~ + + Command line interface. + + :copyright: Copyright 2006-present by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import os +import sys +import shutil +import argparse +from textwrap import dedent + +from pygments import __version__, highlight +from pygments.util import ClassNotFound, OptionError, docstring_headline, \ + guess_decode, guess_decode_from_terminal, terminal_encoding, \ + UnclosingTextIOWrapper +from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \ + load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename +from pygments.lexers.special import TextLexer +from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter +from pygments.formatters import get_all_formatters, get_formatter_by_name, \ + load_formatter_from_file, get_formatter_for_filename, find_formatter_class +from pygments.formatters.terminal import TerminalFormatter +from pygments.formatters.terminal256 import Terminal256Formatter, TerminalTrueColorFormatter +from pygments.filters import get_all_filters, find_filter_class +from pygments.styles import get_all_styles, get_style_by_name + + +def _parse_options(o_strs): + opts = {} + if not o_strs: + return opts + for o_str in o_strs: + if not o_str.strip(): + continue + o_args = o_str.split(',') + for o_arg in o_args: + o_arg = o_arg.strip() + try: + o_key, o_val = o_arg.split('=', 1) + o_key = o_key.strip() + o_val = o_val.strip() + except ValueError: + opts[o_arg] = True + else: + opts[o_key] = o_val + return opts + + +def _parse_filters(f_strs): + filters = [] + if not f_strs: + return filters + for f_str in f_strs: + if ':' in f_str: + fname, fopts = f_str.split(':', 1) + filters.append((fname, _parse_options([fopts]))) + else: + filters.append((f_str, {})) + return filters + + +def _print_help(what, name): + try: + if what == 'lexer': + cls = get_lexer_by_name(name) + print(f"Help on the {cls.name} lexer:") + print(dedent(cls.__doc__)) + elif what == 'formatter': + cls = find_formatter_class(name) + print(f"Help on the {cls.name} formatter:") + print(dedent(cls.__doc__)) + elif what == 'filter': + cls = find_filter_class(name) + print(f"Help on the {name} filter:") + print(dedent(cls.__doc__)) + return 0 + except (AttributeError, ValueError): + print(f"{what} not found!", file=sys.stderr) + return 1 + + +def _print_list(what): + if what == 'lexer': + print() + print("Lexers:") + print("~~~~~~~") + + info = [] + for fullname, names, exts, _ in get_all_lexers(): + tup = (', '.join(names)+':', fullname, + exts and '(filenames ' + ', '.join(exts) + ')' or '') + info.append(tup) + info.sort() + for i in info: + print(('* {}\n {} {}').format(*i)) + + elif what == 'formatter': + print() + print("Formatters:") + print("~~~~~~~~~~~") + + info = [] + for cls in get_all_formatters(): + doc = docstring_headline(cls) + tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and + '(filenames ' + ', '.join(cls.filenames) + ')' or '') + info.append(tup) + info.sort() + for i in info: + print(('* {}\n {} {}').format(*i)) + + elif what == 'filter': + print() + print("Filters:") + print("~~~~~~~~") + + for name in get_all_filters(): + cls = find_filter_class(name) + print("* " + name + ':') + print(f" {docstring_headline(cls)}") + + elif what == 'style': + print() + print("Styles:") + print("~~~~~~~") + + for name in get_all_styles(): + cls = get_style_by_name(name) + print("* " + name + ':') + print(f" {docstring_headline(cls)}") + + +def _print_list_as_json(requested_items): + import json + result = {} + if 'lexer' in requested_items: + info = {} + for fullname, names, filenames, mimetypes in get_all_lexers(): + info[fullname] = { + 'aliases': names, + 'filenames': filenames, + 'mimetypes': mimetypes + } + result['lexers'] = info + + if 'formatter' in requested_items: + info = {} + for cls in get_all_formatters(): + doc = docstring_headline(cls) + info[cls.name] = { + 'aliases': cls.aliases, + 'filenames': cls.filenames, + 'doc': doc + } + result['formatters'] = info + + if 'filter' in requested_items: + info = {} + for name in get_all_filters(): + cls = find_filter_class(name) + info[name] = { + 'doc': docstring_headline(cls) + } + result['filters'] = info + + if 'style' in requested_items: + info = {} + for name in get_all_styles(): + cls = get_style_by_name(name) + info[name] = { + 'doc': docstring_headline(cls) + } + result['styles'] = info + + json.dump(result, sys.stdout) + +def main_inner(parser, argns): + if argns.help: + parser.print_help() + return 0 + + if argns.V: + print(f'Pygments version {__version__}, (c) 2006-present by Georg Brandl, Matthäus ' + 'Chajdas and contributors.') + return 0 + + def is_only_option(opt): + return not any(v for (k, v) in vars(argns).items() if k != opt) + + # handle ``pygmentize -L`` + if argns.L is not None: + arg_set = set() + for k, v in vars(argns).items(): + if v: + arg_set.add(k) + + arg_set.discard('L') + arg_set.discard('json') + + if arg_set: + parser.print_help(sys.stderr) + return 2 + + # print version + if not argns.json: + main(['', '-V']) + allowed_types = {'lexer', 'formatter', 'filter', 'style'} + largs = [arg.rstrip('s') for arg in argns.L] + if any(arg not in allowed_types for arg in largs): + parser.print_help(sys.stderr) + return 0 + if not largs: + largs = allowed_types + if not argns.json: + for arg in largs: + _print_list(arg) + else: + _print_list_as_json(largs) + return 0 + + # handle ``pygmentize -H`` + if argns.H: + if not is_only_option('H'): + parser.print_help(sys.stderr) + return 2 + what, name = argns.H + if what not in ('lexer', 'formatter', 'filter'): + parser.print_help(sys.stderr) + return 2 + return _print_help(what, name) + + # parse -O options + parsed_opts = _parse_options(argns.O or []) + + # parse -P options + for p_opt in argns.P or []: + try: + name, value = p_opt.split('=', 1) + except ValueError: + parsed_opts[p_opt] = True + else: + parsed_opts[name] = value + + # encodings + inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding')) + outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding')) + + # handle ``pygmentize -N`` + if argns.N: + lexer = find_lexer_class_for_filename(argns.N) + if lexer is None: + lexer = TextLexer + + print(lexer.aliases[0]) + return 0 + + # handle ``pygmentize -C`` + if argns.C: + inp = sys.stdin.buffer.read() + try: + lexer = guess_lexer(inp, inencoding=inencoding) + except ClassNotFound: + lexer = TextLexer + + print(lexer.aliases[0]) + return 0 + + # handle ``pygmentize -S`` + S_opt = argns.S + a_opt = argns.a + if S_opt is not None: + f_opt = argns.f + if not f_opt: + parser.print_help(sys.stderr) + return 2 + if argns.l or argns.INPUTFILE: + parser.print_help(sys.stderr) + return 2 + + try: + parsed_opts['style'] = S_opt + fmter = get_formatter_by_name(f_opt, **parsed_opts) + except ClassNotFound as err: + print(err, file=sys.stderr) + return 1 + + print(fmter.get_style_defs(a_opt or '')) + return 0 + + # if no -S is given, -a is not allowed + if argns.a is not None: + parser.print_help(sys.stderr) + return 2 + + # parse -F options + F_opts = _parse_filters(argns.F or []) + + # -x: allow custom (eXternal) lexers and formatters + allow_custom_lexer_formatter = bool(argns.x) + + # select lexer + lexer = None + + # given by name? + lexername = argns.l + if lexername: + # custom lexer, located relative to user's cwd + if allow_custom_lexer_formatter and '.py' in lexername: + try: + filename = None + name = None + if ':' in lexername: + filename, name = lexername.rsplit(':', 1) + + if '.py' in name: + # This can happen on Windows: If the lexername is + # C:\lexer.py -- return to normal load path in that case + name = None + + if filename and name: + lexer = load_lexer_from_file(filename, name, + **parsed_opts) + else: + lexer = load_lexer_from_file(lexername, **parsed_opts) + except ClassNotFound as err: + print('Error:', err, file=sys.stderr) + return 1 + else: + try: + lexer = get_lexer_by_name(lexername, **parsed_opts) + except (OptionError, ClassNotFound) as err: + print('Error:', err, file=sys.stderr) + return 1 + + # read input code + code = None + + if argns.INPUTFILE: + if argns.s: + print('Error: -s option not usable when input file specified', + file=sys.stderr) + return 2 + + infn = argns.INPUTFILE + try: + with open(infn, 'rb') as infp: + code = infp.read() + except Exception as err: + print('Error: cannot read infile:', err, file=sys.stderr) + return 1 + if not inencoding: + code, inencoding = guess_decode(code) + + # do we have to guess the lexer? + if not lexer: + try: + lexer = get_lexer_for_filename(infn, code, **parsed_opts) + except ClassNotFound as err: + if argns.g: + try: + lexer = guess_lexer(code, **parsed_opts) + except ClassNotFound: + lexer = TextLexer(**parsed_opts) + else: + print('Error:', err, file=sys.stderr) + return 1 + except OptionError as err: + print('Error:', err, file=sys.stderr) + return 1 + + elif not argns.s: # treat stdin as full file (-s support is later) + # read code from terminal, always in binary mode since we want to + # decode ourselves and be tolerant with it + code = sys.stdin.buffer.read() # use .buffer to get a binary stream + if not inencoding: + code, inencoding = guess_decode_from_terminal(code, sys.stdin) + # else the lexer will do the decoding + if not lexer: + try: + lexer = guess_lexer(code, **parsed_opts) + except ClassNotFound: + lexer = TextLexer(**parsed_opts) + + else: # -s option needs a lexer with -l + if not lexer: + print('Error: when using -s a lexer has to be selected with -l', + file=sys.stderr) + return 2 + + # process filters + for fname, fopts in F_opts: + try: + lexer.add_filter(fname, **fopts) + except ClassNotFound as err: + print('Error:', err, file=sys.stderr) + return 1 + + # select formatter + outfn = argns.o + fmter = argns.f + if fmter: + # custom formatter, located relative to user's cwd + if allow_custom_lexer_formatter and '.py' in fmter: + try: + filename = None + name = None + if ':' in fmter: + # Same logic as above for custom lexer + filename, name = fmter.rsplit(':', 1) + + if '.py' in name: + name = None + + if filename and name: + fmter = load_formatter_from_file(filename, name, + **parsed_opts) + else: + fmter = load_formatter_from_file(fmter, **parsed_opts) + except ClassNotFound as err: + print('Error:', err, file=sys.stderr) + return 1 + else: + try: + fmter = get_formatter_by_name(fmter, **parsed_opts) + except (OptionError, ClassNotFound) as err: + print('Error:', err, file=sys.stderr) + return 1 + + if outfn: + if not fmter: + try: + fmter = get_formatter_for_filename(outfn, **parsed_opts) + except (OptionError, ClassNotFound) as err: + print('Error:', err, file=sys.stderr) + return 1 + try: + outfile = open(outfn, 'wb') + except Exception as err: + print('Error: cannot open outfile:', err, file=sys.stderr) + return 1 + else: + if not fmter: + if os.environ.get('COLORTERM','') in ('truecolor', '24bit'): + fmter = TerminalTrueColorFormatter(**parsed_opts) + elif '256' in os.environ.get('TERM', ''): + fmter = Terminal256Formatter(**parsed_opts) + else: + fmter = TerminalFormatter(**parsed_opts) + outfile = sys.stdout.buffer + + # determine output encoding if not explicitly selected + if not outencoding: + if outfn: + # output file? use lexer encoding for now (can still be None) + fmter.encoding = inencoding + else: + # else use terminal encoding + fmter.encoding = terminal_encoding(sys.stdout) + + # provide coloring under Windows, if possible + if not outfn and sys.platform in ('win32', 'cygwin') and \ + fmter.name in ('Terminal', 'Terminal256'): # pragma: no cover + # unfortunately colorama doesn't support binary streams on Py3 + outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding) + fmter.encoding = None + try: + import colorama.initialise + except ImportError: + pass + else: + outfile = colorama.initialise.wrap_stream( + outfile, convert=None, strip=None, autoreset=False, wrap=True) + + # When using the LaTeX formatter and the option `escapeinside` is + # specified, we need a special lexer which collects escaped text + # before running the chosen language lexer. + escapeinside = parsed_opts.get('escapeinside', '') + if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter): + left = escapeinside[0] + right = escapeinside[1] + lexer = LatexEmbeddedLexer(left, right, lexer) + + # ... and do it! + if not argns.s: + # process whole input as per normal... + try: + highlight(code, lexer, fmter, outfile) + finally: + if outfn: + outfile.close() + return 0 + else: + # line by line processing of stdin (eg: for 'tail -f')... + try: + while 1: + line = sys.stdin.buffer.readline() + if not line: + break + if not inencoding: + line = guess_decode_from_terminal(line, sys.stdin)[0] + highlight(line, lexer, fmter, outfile) + if hasattr(outfile, 'flush'): + outfile.flush() + return 0 + except KeyboardInterrupt: # pragma: no cover + return 0 + finally: + if outfn: + outfile.close() + + +class HelpFormatter(argparse.HelpFormatter): + def __init__(self, prog, indent_increment=2, max_help_position=16, width=None): + if width is None: + try: + width = shutil.get_terminal_size().columns - 2 + except Exception: + pass + argparse.HelpFormatter.__init__(self, prog, indent_increment, + max_help_position, width) + + +def main(args=sys.argv): + """ + Main command line entry point. + """ + desc = "Highlight an input file and write the result to an output file." + parser = argparse.ArgumentParser(description=desc, add_help=False, + formatter_class=HelpFormatter) + + operation = parser.add_argument_group('Main operation') + lexersel = operation.add_mutually_exclusive_group() + lexersel.add_argument( + '-l', metavar='LEXER', + help='Specify the lexer to use. (Query names with -L.) If not ' + 'given and -g is not present, the lexer is guessed from the filename.') + lexersel.add_argument( + '-g', action='store_true', + help='Guess the lexer from the file contents, or pass through ' + 'as plain text if nothing can be guessed.') + operation.add_argument( + '-F', metavar='FILTER[:options]', action='append', + help='Add a filter to the token stream. (Query names with -L.) ' + 'Filter options are given after a colon if necessary.') + operation.add_argument( + '-f', metavar='FORMATTER', + help='Specify the formatter to use. (Query names with -L.) ' + 'If not given, the formatter is guessed from the output filename, ' + 'and defaults to the terminal formatter if the output is to the ' + 'terminal or an unknown file extension.') + operation.add_argument( + '-O', metavar='OPTION=value[,OPTION=value,...]', action='append', + help='Give options to the lexer and formatter as a comma-separated ' + 'list of key-value pairs. ' + 'Example: `-O bg=light,python=cool`.') + operation.add_argument( + '-P', metavar='OPTION=value', action='append', + help='Give a single option to the lexer and formatter - with this ' + 'you can pass options whose value contains commas and equal signs. ' + 'Example: `-P "heading=Pygments, the Python highlighter"`.') + operation.add_argument( + '-o', metavar='OUTPUTFILE', + help='Where to write the output. Defaults to standard output.') + + operation.add_argument( + 'INPUTFILE', nargs='?', + help='Where to read the input. Defaults to standard input.') + + flags = parser.add_argument_group('Operation flags') + flags.add_argument( + '-v', action='store_true', + help='Print a detailed traceback on unhandled exceptions, which ' + 'is useful for debugging and bug reports.') + flags.add_argument( + '-s', action='store_true', + help='Process lines one at a time until EOF, rather than waiting to ' + 'process the entire file. This only works for stdin, only for lexers ' + 'with no line-spanning constructs, and is intended for streaming ' + 'input such as you get from `tail -f`. ' + 'Example usage: `tail -f sql.log | pygmentize -s -l sql`.') + flags.add_argument( + '-x', action='store_true', + help='Allow custom lexers and formatters to be loaded from a .py file ' + 'relative to the current working directory. For example, ' + '`-l ./customlexer.py -x`. By default, this option expects a file ' + 'with a class named CustomLexer or CustomFormatter; you can also ' + 'specify your own class name with a colon (`-l ./lexer.py:MyLexer`). ' + 'Users should be very careful not to use this option with untrusted ' + 'files, because it will import and run them.') + flags.add_argument('--json', help='Output as JSON. This can ' + 'be only used in conjunction with -L.', + default=False, + action='store_true') + + special_modes_group = parser.add_argument_group( + 'Special modes - do not do any highlighting') + special_modes = special_modes_group.add_mutually_exclusive_group() + special_modes.add_argument( + '-S', metavar='STYLE -f formatter', + help='Print style definitions for STYLE for a formatter ' + 'given with -f. The argument given by -a is formatter ' + 'dependent.') + special_modes.add_argument( + '-L', nargs='*', metavar='WHAT', + help='List lexers, formatters, styles or filters -- ' + 'give additional arguments for the thing(s) you want to list ' + '(e.g. "styles"), or omit them to list everything.') + special_modes.add_argument( + '-N', metavar='FILENAME', + help='Guess and print out a lexer name based solely on the given ' + 'filename. Does not take input or highlight anything. If no specific ' + 'lexer can be determined, "text" is printed.') + special_modes.add_argument( + '-C', action='store_true', + help='Like -N, but print out a lexer name based solely on ' + 'a given content from standard input.') + special_modes.add_argument( + '-H', action='store', nargs=2, metavar=('NAME', 'TYPE'), + help='Print detailed help for the object of type , ' + 'where is one of "lexer", "formatter" or "filter".') + special_modes.add_argument( + '-V', action='store_true', + help='Print the package version.') + special_modes.add_argument( + '-h', '--help', action='store_true', + help='Print this help.') + special_modes_group.add_argument( + '-a', metavar='ARG', + help='Formatter-specific additional argument for the -S (print ' + 'style sheet) mode.') + + argns = parser.parse_args(args[1:]) + + try: + return main_inner(parser, argns) + except BrokenPipeError: + # someone closed our stdout, e.g. by quitting a pager. + return 0 + except Exception: + if argns.v: + print(file=sys.stderr) + print('*' * 65, file=sys.stderr) + print('An unhandled exception occurred while highlighting.', + file=sys.stderr) + print('Please report the whole traceback to the issue tracker at', + file=sys.stderr) + print('.', + file=sys.stderr) + print('*' * 65, file=sys.stderr) + print(file=sys.stderr) + raise + import traceback + info = traceback.format_exception(*sys.exc_info()) + msg = info[-1].strip() + if len(info) >= 3: + # extract relevant file and position info + msg += '\n (f{})'.format(info[-2].split('\n')[0].strip()[1:]) + print(file=sys.stderr) + print('*** Error while highlighting:', file=sys.stderr) + print(msg, file=sys.stderr) + print('*** If this is a bug you want to report, please rerun with -v.', + file=sys.stderr) + return 1 diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/console.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/console.py new file mode 100644 index 0000000000000000000000000000000000000000..d9009168cadbe25ba0bb1fc5427a5db0f1582830 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/console.py @@ -0,0 +1,70 @@ +""" + pygments.console + ~~~~~~~~~~~~~~~~ + + Format colored console output. + + :copyright: Copyright 2006-present by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +esc = "\x1b[" + +codes = {} +codes[""] = "" +codes["reset"] = esc + "39;49;00m" + +codes["bold"] = esc + "01m" +codes["faint"] = esc + "02m" +codes["standout"] = esc + "03m" +codes["underline"] = esc + "04m" +codes["blink"] = esc + "05m" +codes["overline"] = esc + "06m" + +dark_colors = ["black", "red", "green", "yellow", "blue", + "magenta", "cyan", "gray"] +light_colors = ["brightblack", "brightred", "brightgreen", "brightyellow", "brightblue", + "brightmagenta", "brightcyan", "white"] + +x = 30 +for dark, light in zip(dark_colors, light_colors): + codes[dark] = esc + "%im" % x + codes[light] = esc + "%im" % (60 + x) + x += 1 + +del dark, light, x + +codes["white"] = codes["bold"] + + +def reset_color(): + return codes["reset"] + + +def colorize(color_key, text): + return codes[color_key] + text + codes["reset"] + + +def ansiformat(attr, text): + """ + Format ``text`` with a color and/or some attributes:: + + color normal color + *color* bold color + _color_ underlined color + +color+ blinking color + """ + result = [] + if attr[:1] == attr[-1:] == '+': + result.append(codes['blink']) + attr = attr[1:-1] + if attr[:1] == attr[-1:] == '*': + result.append(codes['bold']) + attr = attr[1:-1] + if attr[:1] == attr[-1:] == '_': + result.append(codes['underline']) + attr = attr[1:-1] + result.append(codes[attr]) + result.append(text) + result.append(codes['reset']) + return ''.join(result) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/filter.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/filter.py new file mode 100644 index 0000000000000000000000000000000000000000..6c926c26cfcf0ca7579f05affcab46c389595b71 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/filter.py @@ -0,0 +1,70 @@ +""" + pygments.filter + ~~~~~~~~~~~~~~~ + + Module that implements the default filter. + + :copyright: Copyright 2006-present by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + + +def apply_filters(stream, filters, lexer=None): + """ + Use this method to apply an iterable of filters to + a stream. If lexer is given it's forwarded to the + filter, otherwise the filter receives `None`. + """ + def _apply(filter_, stream): + yield from filter_.filter(lexer, stream) + for filter_ in filters: + stream = _apply(filter_, stream) + return stream + + +def simplefilter(f): + """ + Decorator that converts a function into a filter:: + + @simplefilter + def lowercase(self, lexer, stream, options): + for ttype, value in stream: + yield ttype, value.lower() + """ + return type(f.__name__, (FunctionFilter,), { + '__module__': getattr(f, '__module__'), + '__doc__': f.__doc__, + 'function': f, + }) + + +class Filter: + """ + Default filter. Subclass this class or use the `simplefilter` + decorator to create own filters. + """ + + def __init__(self, **options): + self.options = options + + def filter(self, lexer, stream): + raise NotImplementedError() + + +class FunctionFilter(Filter): + """ + Abstract class used by `simplefilter` to create simple + function filters on the fly. The `simplefilter` decorator + automatically creates subclasses of this class for + functions passed to it. + """ + function = None + + def __init__(self, **options): + if not hasattr(self, 'function'): + raise TypeError(f'{self.__class__.__name__!r} used without bound function') + Filter.__init__(self, **options) + + def filter(self, lexer, stream): + # pylint: disable=not-callable + yield from self.function(lexer, stream, self.options) diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/filters/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/filters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c7ba6adb5b75ac3bf8e9e94868575e6ce770e6c1 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/filters/__init__.py @@ -0,0 +1,942 @@ +""" + pygments.filters + ~~~~~~~~~~~~~~~~ + + Module containing filter lookup functions and default + filters. + + :copyright: Copyright 2006-present by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \ + string_to_tokentype +from pygments.filter import Filter +from pygments.util import get_list_opt, get_int_opt, get_bool_opt, \ + get_choice_opt, ClassNotFound, OptionError +from pygments.plugin import find_plugin_filters + + +def find_filter_class(filtername): + """Lookup a filter by name. Return None if not found.""" + if filtername in FILTERS: + return FILTERS[filtername] + for name, cls in find_plugin_filters(): + if name == filtername: + return cls + return None + + +def get_filter_by_name(filtername, **options): + """Return an instantiated filter. + + Options are passed to the filter initializer if wanted. + Raise a ClassNotFound if not found. + """ + cls = find_filter_class(filtername) + if cls: + return cls(**options) + else: + raise ClassNotFound(f'filter {filtername!r} not found') + + +def get_all_filters(): + """Return a generator of all filter names.""" + yield from FILTERS + for name, _ in find_plugin_filters(): + yield name + + +def _replace_special(ttype, value, regex, specialttype, + replacefunc=lambda x: x): + last = 0 + for match in regex.finditer(value): + start, end = match.start(), match.end() + if start != last: + yield ttype, value[last:start] + yield specialttype, replacefunc(value[start:end]) + last = end + if last != len(value): + yield ttype, value[last:] + + +class CodeTagFilter(Filter): + """Highlight special code tags in comments and docstrings. + + Options accepted: + + `codetags` : list of strings + A list of strings that are flagged as code tags. The default is to + highlight ``XXX``, ``TODO``, ``FIXME``, ``BUG`` and ``NOTE``. + + .. versionchanged:: 2.13 + Now recognizes ``FIXME`` by default. + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + tags = get_list_opt(options, 'codetags', + ['XXX', 'TODO', 'FIXME', 'BUG', 'NOTE']) + self.tag_re = re.compile(r'\b({})\b'.format('|'.join([ + re.escape(tag) for tag in tags if tag + ]))) + + def filter(self, lexer, stream): + regex = self.tag_re + for ttype, value in stream: + if ttype in String.Doc or \ + ttype in Comment and \ + ttype not in Comment.Preproc: + yield from _replace_special(ttype, value, regex, Comment.Special) + else: + yield ttype, value + + +class SymbolFilter(Filter): + """Convert mathematical symbols into Unicode characters. + + Examples are ``\\`` in Isabelle or + ``\\longrightarrow`` in LaTeX. + + This is mostly useful for HTML or console output when you want to + approximate the source rendering you'd see in an IDE. + + Options accepted: + + `lang` : string + The symbol language. Must be one of ``'isabelle'`` or + ``'latex'``. The default is ``'isabelle'``. + """ + + latex_symbols = { + '\\alpha' : '\U000003b1', + '\\beta' : '\U000003b2', + '\\gamma' : '\U000003b3', + '\\delta' : '\U000003b4', + '\\varepsilon' : '\U000003b5', + '\\zeta' : '\U000003b6', + '\\eta' : '\U000003b7', + '\\vartheta' : '\U000003b8', + '\\iota' : '\U000003b9', + '\\kappa' : '\U000003ba', + '\\lambda' : '\U000003bb', + '\\mu' : '\U000003bc', + '\\nu' : '\U000003bd', + '\\xi' : '\U000003be', + '\\pi' : '\U000003c0', + '\\varrho' : '\U000003c1', + '\\sigma' : '\U000003c3', + '\\tau' : '\U000003c4', + '\\upsilon' : '\U000003c5', + '\\varphi' : '\U000003c6', + '\\chi' : '\U000003c7', + '\\psi' : '\U000003c8', + '\\omega' : '\U000003c9', + '\\Gamma' : '\U00000393', + '\\Delta' : '\U00000394', + '\\Theta' : '\U00000398', + '\\Lambda' : '\U0000039b', + '\\Xi' : '\U0000039e', + '\\Pi' : '\U000003a0', + '\\Sigma' : '\U000003a3', + '\\Upsilon' : '\U000003a5', + '\\Phi' : '\U000003a6', + '\\Psi' : '\U000003a8', + '\\Omega' : '\U000003a9', + '\\leftarrow' : '\U00002190', + '\\longleftarrow' : '\U000027f5', + '\\rightarrow' : '\U00002192', + '\\longrightarrow' : '\U000027f6', + '\\Leftarrow' : '\U000021d0', + '\\Longleftarrow' : '\U000027f8', + '\\Rightarrow' : '\U000021d2', + '\\Longrightarrow' : '\U000027f9', + '\\leftrightarrow' : '\U00002194', + '\\longleftrightarrow' : '\U000027f7', + '\\Leftrightarrow' : '\U000021d4', + '\\Longleftrightarrow' : '\U000027fa', + '\\mapsto' : '\U000021a6', + '\\longmapsto' : '\U000027fc', + '\\relbar' : '\U00002500', + '\\Relbar' : '\U00002550', + '\\hookleftarrow' : '\U000021a9', + '\\hookrightarrow' : '\U000021aa', + '\\leftharpoondown' : '\U000021bd', + '\\rightharpoondown' : '\U000021c1', + '\\leftharpoonup' : '\U000021bc', + '\\rightharpoonup' : '\U000021c0', + '\\rightleftharpoons' : '\U000021cc', + '\\leadsto' : '\U0000219d', + '\\downharpoonleft' : '\U000021c3', + '\\downharpoonright' : '\U000021c2', + '\\upharpoonleft' : '\U000021bf', + '\\upharpoonright' : '\U000021be', + '\\restriction' : '\U000021be', + '\\uparrow' : '\U00002191', + '\\Uparrow' : '\U000021d1', + '\\downarrow' : '\U00002193', + '\\Downarrow' : '\U000021d3', + '\\updownarrow' : '\U00002195', + '\\Updownarrow' : '\U000021d5', + '\\langle' : '\U000027e8', + '\\rangle' : '\U000027e9', + '\\lceil' : '\U00002308', + '\\rceil' : '\U00002309', + '\\lfloor' : '\U0000230a', + '\\rfloor' : '\U0000230b', + '\\flqq' : '\U000000ab', + '\\frqq' : '\U000000bb', + '\\bot' : '\U000022a5', + '\\top' : '\U000022a4', + '\\wedge' : '\U00002227', + '\\bigwedge' : '\U000022c0', + '\\vee' : '\U00002228', + '\\bigvee' : '\U000022c1', + '\\forall' : '\U00002200', + '\\exists' : '\U00002203', + '\\nexists' : '\U00002204', + '\\neg' : '\U000000ac', + '\\Box' : '\U000025a1', + '\\Diamond' : '\U000025c7', + '\\vdash' : '\U000022a2', + '\\models' : '\U000022a8', + '\\dashv' : '\U000022a3', + '\\surd' : '\U0000221a', + '\\le' : '\U00002264', + '\\ge' : '\U00002265', + '\\ll' : '\U0000226a', + '\\gg' : '\U0000226b', + '\\lesssim' : '\U00002272', + '\\gtrsim' : '\U00002273', + '\\lessapprox' : '\U00002a85', + '\\gtrapprox' : '\U00002a86', + '\\in' : '\U00002208', + '\\notin' : '\U00002209', + '\\subset' : '\U00002282', + '\\supset' : '\U00002283', + '\\subseteq' : '\U00002286', + '\\supseteq' : '\U00002287', + '\\sqsubset' : '\U0000228f', + '\\sqsupset' : '\U00002290', + '\\sqsubseteq' : '\U00002291', + '\\sqsupseteq' : '\U00002292', + '\\cap' : '\U00002229', + '\\bigcap' : '\U000022c2', + '\\cup' : '\U0000222a', + '\\bigcup' : '\U000022c3', + '\\sqcup' : '\U00002294', + '\\bigsqcup' : '\U00002a06', + '\\sqcap' : '\U00002293', + '\\Bigsqcap' : '\U00002a05', + '\\setminus' : '\U00002216', + '\\propto' : '\U0000221d', + '\\uplus' : '\U0000228e', + '\\bigplus' : '\U00002a04', + '\\sim' : '\U0000223c', + '\\doteq' : '\U00002250', + '\\simeq' : '\U00002243', + '\\approx' : '\U00002248', + '\\asymp' : '\U0000224d', + '\\cong' : '\U00002245', + '\\equiv' : '\U00002261', + '\\Join' : '\U000022c8', + '\\bowtie' : '\U00002a1d', + '\\prec' : '\U0000227a', + '\\succ' : '\U0000227b', + '\\preceq' : '\U0000227c', + '\\succeq' : '\U0000227d', + '\\parallel' : '\U00002225', + '\\mid' : '\U000000a6', + '\\pm' : '\U000000b1', + '\\mp' : '\U00002213', + '\\times' : '\U000000d7', + '\\div' : '\U000000f7', + '\\cdot' : '\U000022c5', + '\\star' : '\U000022c6', + '\\circ' : '\U00002218', + '\\dagger' : '\U00002020', + '\\ddagger' : '\U00002021', + '\\lhd' : '\U000022b2', + '\\rhd' : '\U000022b3', + '\\unlhd' : '\U000022b4', + '\\unrhd' : '\U000022b5', + '\\triangleleft' : '\U000025c3', + '\\triangleright' : '\U000025b9', + '\\triangle' : '\U000025b3', + '\\triangleq' : '\U0000225c', + '\\oplus' : '\U00002295', + '\\bigoplus' : '\U00002a01', + '\\otimes' : '\U00002297', + '\\bigotimes' : '\U00002a02', + '\\odot' : '\U00002299', + '\\bigodot' : '\U00002a00', + '\\ominus' : '\U00002296', + '\\oslash' : '\U00002298', + '\\dots' : '\U00002026', + '\\cdots' : '\U000022ef', + '\\sum' : '\U00002211', + '\\prod' : '\U0000220f', + '\\coprod' : '\U00002210', + '\\infty' : '\U0000221e', + '\\int' : '\U0000222b', + '\\oint' : '\U0000222e', + '\\clubsuit' : '\U00002663', + '\\diamondsuit' : '\U00002662', + '\\heartsuit' : '\U00002661', + '\\spadesuit' : '\U00002660', + '\\aleph' : '\U00002135', + '\\emptyset' : '\U00002205', + '\\nabla' : '\U00002207', + '\\partial' : '\U00002202', + '\\flat' : '\U0000266d', + '\\natural' : '\U0000266e', + '\\sharp' : '\U0000266f', + '\\angle' : '\U00002220', + '\\copyright' : '\U000000a9', + '\\textregistered' : '\U000000ae', + '\\textonequarter' : '\U000000bc', + '\\textonehalf' : '\U000000bd', + '\\textthreequarters' : '\U000000be', + '\\textordfeminine' : '\U000000aa', + '\\textordmasculine' : '\U000000ba', + '\\euro' : '\U000020ac', + '\\pounds' : '\U000000a3', + '\\yen' : '\U000000a5', + '\\textcent' : '\U000000a2', + '\\textcurrency' : '\U000000a4', + '\\textdegree' : '\U000000b0', + } + + isabelle_symbols = { + '\\' : '\U0001d7ec', + '\\' : '\U0001d7ed', + '\\' : '\U0001d7ee', + '\\' : '\U0001d7ef', + '\\' : '\U0001d7f0', + '\\' : '\U0001d7f1', + '\\' : '\U0001d7f2', + '\\' : '\U0001d7f3', + '\\' : '\U0001d7f4', + '\\' : '\U0001d7f5', + '\\' : '\U0001d49c', + '\\' : '\U0000212c', + '\\' : '\U0001d49e', + '\\' : '\U0001d49f', + '\\' : '\U00002130', + '\\' : '\U00002131', + '\\' : '\U0001d4a2', + '\\' : '\U0000210b', + '\\' : '\U00002110', + '\\' : '\U0001d4a5', + '\\' : '\U0001d4a6', + '\\' : '\U00002112', + '\\' : '\U00002133', + '\\' : '\U0001d4a9', + '\\' : '\U0001d4aa', + '\\

' : '\U0001d5c9', + '\\' : '\U0001d5ca', + '\\' : '\U0001d5cb', + '\\' : '\U0001d5cc', + '\\' : '\U0001d5cd', + '\\' : '\U0001d5ce', + '\\' : '\U0001d5cf', + '\\' : '\U0001d5d0', + '\\' : '\U0001d5d1', + '\\' : '\U0001d5d2', + '\\' : '\U0001d5d3', + '\\' : '\U0001d504', + '\\' : '\U0001d505', + '\\' : '\U0000212d', + '\\

' : '\U0001d507', + '\\' : '\U0001d508', + '\\' : '\U0001d509', + '\\' : '\U0001d50a', + '\\' : '\U0000210c', + '\\' : '\U00002111', + '\\' : '\U0001d50d', + '\\' : '\U0001d50e', + '\\' : '\U0001d50f', + '\\' : '\U0001d510', + '\\' : '\U0001d511', + '\\' : '\U0001d512', + '\\' : '\U0001d513', + '\\' : '\U0001d514', + '\\' : '\U0000211c', + '\\' : '\U0001d516', + '\\' : '\U0001d517', + '\\' : '\U0001d518', + '\\' : '\U0001d519', + '\\' : '\U0001d51a', + '\\' : '\U0001d51b', + '\\' : '\U0001d51c', + '\\' : '\U00002128', + '\\' : '\U0001d51e', + '\\' : '\U0001d51f', + '\\' : '\U0001d520', + '\\
' : '\U0001d521', + '\\' : '\U0001d522', + '\\' : '\U0001d523', + '\\' : '\U0001d524', + '\\' : '\U0001d525', + '\\' : '\U0001d526', + '\\' : '\U0001d527', + '\\' : '\U0001d528', + '\\' : '\U0001d529', + '\\' : '\U0001d52a', + '\\' : '\U0001d52b', + '\\' : '\U0001d52c', + '\\' : '\U0001d52d', + '\\' : '\U0001d52e', + '\\' : '\U0001d52f', + '\\' : '\U0001d530', + '\\' : '\U0001d531', + '\\' : '\U0001d532', + '\\' : '\U0001d533', + '\\' : '\U0001d534', + '\\' : '\U0001d535', + '\\' : '\U0001d536', + '\\' : '\U0001d537', + '\\' : '\U000003b1', + '\\' : '\U000003b2', + '\\' : '\U000003b3', + '\\' : '\U000003b4', + '\\' : '\U000003b5', + '\\' : '\U000003b6', + '\\' : '\U000003b7', + '\\' : '\U000003b8', + '\\' : '\U000003b9', + '\\' : '\U000003ba', + '\\' : '\U000003bb', + '\\' : '\U000003bc', + '\\' : '\U000003bd', + '\\' : '\U000003be', + '\\' : '\U000003c0', + '\\' : '\U000003c1', + '\\' : '\U000003c3', + '\\' : '\U000003c4', + '\\' : '\U000003c5', + '\\' : '\U000003c6', + '\\' : '\U000003c7', + '\\' : '\U000003c8', + '\\' : '\U000003c9', + '\\' : '\U00000393', + '\\' : '\U00000394', + '\\' : '\U00000398', + '\\' : '\U0000039b', + '\\' : '\U0000039e', + '\\' : '\U000003a0', + '\\' : '\U000003a3', + '\\' : '\U000003a5', + '\\' : '\U000003a6', + '\\' : '\U000003a8', + '\\' : '\U000003a9', + '\\' : '\U0001d539', + '\\' : '\U00002102', + '\\' : '\U00002115', + '\\' : '\U0000211a', + '\\' : '\U0000211d', + '\\' : '\U00002124', + '\\' : '\U00002190', + '\\' : '\U000027f5', + '\\' : '\U00002192', + '\\' : '\U000027f6', + '\\' : '\U000021d0', + '\\' : '\U000027f8', + '\\' : '\U000021d2', + '\\' : '\U000027f9', + '\\' : '\U00002194', + '\\' : '\U000027f7', + '\\' : '\U000021d4', + '\\' : '\U000027fa', + '\\' : '\U000021a6', + '\\' : '\U000027fc', + '\\' : '\U00002500', + '\\' : '\U00002550', + '\\' : '\U000021a9', + '\\' : '\U000021aa', + '\\' : '\U000021bd', + '\\' : '\U000021c1', + '\\' : '\U000021bc', + '\\' : '\U000021c0', + '\\' : '\U000021cc', + '\\' : '\U0000219d', + '\\' : '\U000021c3', + '\\' : '\U000021c2', + '\\' : '\U000021bf', + '\\' : '\U000021be', + '\\' : '\U000021be', + '\\' : '\U00002237', + '\\' : '\U00002191', + '\\' : '\U000021d1', + '\\' : '\U00002193', + '\\' : '\U000021d3', + '\\' : '\U00002195', + '\\' : '\U000021d5', + '\\' : '\U000027e8', + '\\' : '\U000027e9', + '\\' : '\U00002308', + '\\' : '\U00002309', + '\\' : '\U0000230a', + '\\' : '\U0000230b', + '\\' : '\U00002987', + '\\' : '\U00002988', + '\\' : '\U000027e6', + '\\' : '\U000027e7', + '\\' : '\U00002983', + '\\' : '\U00002984', + '\\' : '\U000000ab', + '\\' : '\U000000bb', + '\\' : '\U000022a5', + '\\' : '\U000022a4', + '\\' : '\U00002227', + '\\' : '\U000022c0', + '\\' : '\U00002228', + '\\' : '\U000022c1', + '\\' : '\U00002200', + '\\' : '\U00002203', + '\\' : '\U00002204', + '\\' : '\U000000ac', + '\\' : '\U000025a1', + '\\' : '\U000025c7', + '\\' : '\U000022a2', + '\\' : '\U000022a8', + '\\' : '\U000022a9', + '\\' : '\U000022ab', + '\\' : '\U000022a3', + '\\' : '\U0000221a', + '\\' : '\U00002264', + '\\' : '\U00002265', + '\\' : '\U0000226a', + '\\' : '\U0000226b', + '\\' : '\U00002272', + '\\' : '\U00002273', + '\\' : '\U00002a85', + '\\' : '\U00002a86', + '\\' : '\U00002208', + '\\' : '\U00002209', + '\\' : '\U00002282', + '\\' : '\U00002283', + '\\' : '\U00002286', + '\\' : '\U00002287', + '\\' : '\U0000228f', + '\\' : '\U00002290', + '\\' : '\U00002291', + '\\' : '\U00002292', + '\\' : '\U00002229', + '\\' : '\U000022c2', + '\\' : '\U0000222a', + '\\' : '\U000022c3', + '\\' : '\U00002294', + '\\' : '\U00002a06', + '\\' : '\U00002293', + '\\' : '\U00002a05', + '\\' : '\U00002216', + '\\' : '\U0000221d', + '\\' : '\U0000228e', + '\\' : '\U00002a04', + '\\' : '\U00002260', + '\\' : '\U0000223c', + '\\' : '\U00002250', + '\\' : '\U00002243', + '\\' : '\U00002248', + '\\' : '\U0000224d', + '\\' : '\U00002245', + '\\' : '\U00002323', + '\\' : '\U00002261', + '\\' : '\U00002322', + '\\' : '\U000022c8', + '\\' : '\U00002a1d', + '\\' : '\U0000227a', + '\\' : '\U0000227b', + '\\' : '\U0000227c', + '\\' : '\U0000227d', + '\\' : '\U00002225', + '\\' : '\U000000a6', + '\\' : '\U000000b1', + '\\' : '\U00002213', + '\\' : '\U000000d7', + '\\
' : '\U000000f7', + '\\' : '\U000022c5', + '\\' : '\U000022c6', + '\\' : '\U00002219', + '\\' : '\U00002218', + '\\' : '\U00002020', + '\\' : '\U00002021', + '\\' : '\U000022b2', + '\\' : '\U000022b3', + '\\' : '\U000022b4', + '\\' : '\U000022b5', + '\\' : '\U000025c3', + '\\' : '\U000025b9', + '\\' : '\U000025b3', + '\\' : '\U0000225c', + '\\' : '\U00002295', + '\\' : '\U00002a01', + '\\' : '\U00002297', + '\\' : '\U00002a02', + '\\' : '\U00002299', + '\\' : '\U00002a00', + '\\' : '\U00002296', + '\\' : '\U00002298', + '\\' : '\U00002026', + '\\' : '\U000022ef', + '\\' : '\U00002211', + '\\' : '\U0000220f', + '\\' : '\U00002210', + '\\' : '\U0000221e', + '\\' : '\U0000222b', + '\\' : '\U0000222e', + '\\' : '\U00002663', + '\\' : '\U00002662', + '\\' : '\U00002661', + '\\' : '\U00002660', + '\\' : '\U00002135', + '\\' : '\U00002205', + '\\' : '\U00002207', + '\\' : '\U00002202', + '\\' : '\U0000266d', + '\\' : '\U0000266e', + '\\' : '\U0000266f', + '\\' : '\U00002220', + '\\' : '\U000000a9', + '\\' : '\U000000ae', + '\\' : '\U000000ad', + '\\' : '\U000000af', + '\\' : '\U000000bc', + '\\' : '\U000000bd', + '\\' : '\U000000be', + '\\' : '\U000000aa', + '\\' : '\U000000ba', + '\\
' : '\U000000a7', + '\\' : '\U000000b6', + '\\' : '\U000000a1', + '\\' : '\U000000bf', + '\\' : '\U000020ac', + '\\' : '\U000000a3', + '\\' : '\U000000a5', + '\\' : '\U000000a2', + '\\' : '\U000000a4', + '\\' : '\U000000b0', + '\\' : '\U00002a3f', + '\\' : '\U00002127', + '\\' : '\U000025ca', + '\\' : '\U00002118', + '\\' : '\U00002240', + '\\' : '\U000022c4', + '\\' : '\U000000b4', + '\\' : '\U00000131', + '\\' : '\U000000a8', + '\\' : '\U000000b8', + '\\' : '\U000002dd', + '\\' : '\U000003f5', + '\\' : '\U000023ce', + '\\' : '\U00002039', + '\\' : '\U0000203a', + '\\' : '\U00002302', + '\\<^sub>' : '\U000021e9', + '\\<^sup>' : '\U000021e7', + '\\<^bold>' : '\U00002759', + '\\<^bsub>' : '\U000021d8', + '\\<^esub>' : '\U000021d9', + '\\<^bsup>' : '\U000021d7', + '\\<^esup>' : '\U000021d6', + } + + lang_map = {'isabelle' : isabelle_symbols, 'latex' : latex_symbols} + + def __init__(self, **options): + Filter.__init__(self, **options) + lang = get_choice_opt(options, 'lang', + ['isabelle', 'latex'], 'isabelle') + self.symbols = self.lang_map[lang] + + def filter(self, lexer, stream): + for ttype, value in stream: + if value in self.symbols: + yield ttype, self.symbols[value] + else: + yield ttype, value + + +class KeywordCaseFilter(Filter): + """Convert keywords to lowercase or uppercase or capitalize them. + + This means first letter uppercase, rest lowercase. + + This can be useful e.g. if you highlight Pascal code and want to adapt the + code to your styleguide. + + Options accepted: + + `case` : string + The casing to convert keywords to. Must be one of ``'lower'``, + ``'upper'`` or ``'capitalize'``. The default is ``'lower'``. + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + case = get_choice_opt(options, 'case', + ['lower', 'upper', 'capitalize'], 'lower') + self.convert = getattr(str, case) + + def filter(self, lexer, stream): + for ttype, value in stream: + if ttype in Keyword: + yield ttype, self.convert(value) + else: + yield ttype, value + + +class NameHighlightFilter(Filter): + """Highlight a normal Name (and Name.*) token with a different token type. + + Example:: + + filter = NameHighlightFilter( + names=['foo', 'bar', 'baz'], + tokentype=Name.Function, + ) + + This would highlight the names "foo", "bar" and "baz" + as functions. `Name.Function` is the default token type. + + Options accepted: + + `names` : list of strings + A list of names that should be given the different token type. + There is no default. + `tokentype` : TokenType or string + A token type or a string containing a token type name that is + used for highlighting the strings in `names`. The default is + `Name.Function`. + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + self.names = set(get_list_opt(options, 'names', [])) + tokentype = options.get('tokentype') + if tokentype: + self.tokentype = string_to_tokentype(tokentype) + else: + self.tokentype = Name.Function + + def filter(self, lexer, stream): + for ttype, value in stream: + if ttype in Name and value in self.names: + yield self.tokentype, value + else: + yield ttype, value + + +class ErrorToken(Exception): + pass + + +class RaiseOnErrorTokenFilter(Filter): + """Raise an exception when the lexer generates an error token. + + Options accepted: + + `excclass` : Exception class + The exception class to raise. + The default is `pygments.filters.ErrorToken`. + + .. versionadded:: 0.8 + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + self.exception = options.get('excclass', ErrorToken) + try: + # issubclass() will raise TypeError if first argument is not a class + if not issubclass(self.exception, Exception): + raise TypeError + except TypeError: + raise OptionError('excclass option is not an exception class') + + def filter(self, lexer, stream): + for ttype, value in stream: + if ttype is Error: + raise self.exception(value) + yield ttype, value + + +class VisibleWhitespaceFilter(Filter): + """Convert tabs, newlines and/or spaces to visible characters. + + Options accepted: + + `spaces` : string or bool + If this is a one-character string, spaces will be replaces by this string. + If it is another true value, spaces will be replaced by ``·`` (unicode + MIDDLE DOT). If it is a false value, spaces will not be replaced. The + default is ``False``. + `tabs` : string or bool + The same as for `spaces`, but the default replacement character is ``»`` + (unicode RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK). The default value + is ``False``. Note: this will not work if the `tabsize` option for the + lexer is nonzero, as tabs will already have been expanded then. + `tabsize` : int + If tabs are to be replaced by this filter (see the `tabs` option), this + is the total number of characters that a tab should be expanded to. + The default is ``8``. + `newlines` : string or bool + The same as for `spaces`, but the default replacement character is ``¶`` + (unicode PILCROW SIGN). The default value is ``False``. + `wstokentype` : bool + If true, give whitespace the special `Whitespace` token type. This allows + styling the visible whitespace differently (e.g. greyed out), but it can + disrupt background colors. The default is ``True``. + + .. versionadded:: 0.8 + """ + + def __init__(self, **options): + Filter.__init__(self, **options) + for name, default in [('spaces', '·'), + ('tabs', '»'), + ('newlines', '¶')]: + opt = options.get(name, False) + if isinstance(opt, str) and len(opt) == 1: + setattr(self, name, opt) + else: + setattr(self, name, (opt and default or '')) + tabsize = get_int_opt(options, 'tabsize', 8) + if self.tabs: + self.tabs += ' ' * (tabsize - 1) + if self.newlines: + self.newlines += '\n' + self.wstt = get_bool_opt(options, 'wstokentype', True) + + def filter(self, lexer, stream): + if self.wstt: + spaces = self.spaces or ' ' + tabs = self.tabs or '\t' + newlines = self.newlines or '\n' + regex = re.compile(r'\s') + + def replacefunc(wschar): + if wschar == ' ': + return spaces + elif wschar == '\t': + return tabs + elif wschar == '\n': + return newlines + return wschar + + for ttype, value in stream: + yield from _replace_special(ttype, value, regex, Whitespace, + replacefunc) + else: + spaces, tabs, newlines = self.spaces, self.tabs, self.newlines + # simpler processing + for ttype, value in stream: + if spaces: + value = value.replace(' ', spaces) + if tabs: + value = value.replace('\t', tabs) + if newlines: + value = value.replace('\n', newlines) + yield ttype, value + + +class GobbleFilter(Filter): + """Gobble source code lines (eats initial characters). + + This filter drops the first ``n`` characters off every line of code. This + may be useful when the source code fed to the lexer is indented by a fixed + amount of space that isn't desired in the output. + + Options accepted: + + `n` : int + The number of characters to gobble. + + .. versionadded:: 1.2 + """ + def __init__(self, **options): + Filter.__init__(self, **options) + self.n = get_int_opt(options, 'n', 0) + + def gobble(self, value, left): + if left < len(value): + return value[left:], 0 + else: + return '', left - len(value) + + def filter(self, lexer, stream): + n = self.n + left = n # How many characters left to gobble. + for ttype, value in stream: + # Remove ``left`` tokens from first line, ``n`` from all others. + parts = value.split('\n') + (parts[0], left) = self.gobble(parts[0], left) + for i in range(1, len(parts)): + (parts[i], left) = self.gobble(parts[i], n) + value = '\n'.join(parts) + + if value != '': + yield ttype, value + + +class TokenMergeFilter(Filter): + """Merge consecutive tokens with the same token type in the output stream. + + .. versionadded:: 1.2 + """ + def __init__(self, **options): + Filter.__init__(self, **options) + + def filter(self, lexer, stream): + current_type = None + current_value = None + for ttype, value in stream: + if ttype is current_type: + current_value += value + else: + if current_type is not None: + yield current_type, current_value + current_type = ttype + current_value = value + if current_type is not None: + yield current_type, current_value + + +FILTERS = { + 'codetagify': CodeTagFilter, + 'keywordcase': KeywordCaseFilter, + 'highlight': NameHighlightFilter, + 'raiseonerror': RaiseOnErrorTokenFilter, + 'whitespace': VisibleWhitespaceFilter, + 'gobble': GobbleFilter, + 'tokenmerge': TokenMergeFilter, + 'symbols': SymbolFilter, +} diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatter.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..4fa603d551c99fab81f2aa29037c3b8b89d0fcd5 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatter.py @@ -0,0 +1,129 @@ +""" + pygments.formatter + ~~~~~~~~~~~~~~~~~~ + + Base formatter class. + + :copyright: Copyright 2006-present by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import codecs + +from pygments.util import get_bool_opt +from pygments.styles import get_style_by_name + +__all__ = ['Formatter'] + + +def _lookup_style(style): + if isinstance(style, str): + return get_style_by_name(style) + return style + + +class Formatter: + """ + Converts a token stream to text. + + Formatters should have attributes to help selecting them. These + are similar to the corresponding :class:`~pygments.lexer.Lexer` + attributes. + + .. autoattribute:: name + :no-value: + + .. autoattribute:: aliases + :no-value: + + .. autoattribute:: filenames + :no-value: + + You can pass options as keyword arguments to the constructor. + All formatters accept these basic options: + + ``style`` + The style to use, can be a string or a Style subclass + (default: "default"). Not used by e.g. the + TerminalFormatter. + ``full`` + Tells the formatter to output a "full" document, i.e. + a complete self-contained document. This doesn't have + any effect for some formatters (default: false). + ``title`` + If ``full`` is true, the title that should be used to + caption the document (default: ''). + ``encoding`` + If given, must be an encoding name. This will be used to + convert the Unicode token strings to byte strings in the + output. If it is "" or None, Unicode strings will be written + to the output file, which most file-like objects do not + support (default: None). + ``outencoding`` + Overrides ``encoding`` if given. + + """ + + #: Full name for the formatter, in human-readable form. + name = None + + #: A list of short, unique identifiers that can be used to lookup + #: the formatter from a list, e.g. using :func:`.get_formatter_by_name()`. + aliases = [] + + #: A list of fnmatch patterns that match filenames for which this + #: formatter can produce output. The patterns in this list should be unique + #: among all formatters. + filenames = [] + + #: If True, this formatter outputs Unicode strings when no encoding + #: option is given. + unicodeoutput = True + + def __init__(self, **options): + """ + As with lexers, this constructor takes arbitrary optional arguments, + and if you override it, you should first process your own options, then + call the base class implementation. + """ + self.style = _lookup_style(options.get('style', 'default')) + self.full = get_bool_opt(options, 'full', False) + self.title = options.get('title', '') + self.encoding = options.get('encoding', None) or None + if self.encoding in ('guess', 'chardet'): + # can happen for e.g. pygmentize -O encoding=guess + self.encoding = 'utf-8' + self.encoding = options.get('outencoding') or self.encoding + self.options = options + + def get_style_defs(self, arg=''): + """ + This method must return statements or declarations suitable to define + the current style for subsequent highlighted text (e.g. CSS classes + in the `HTMLFormatter`). + + The optional argument `arg` can be used to modify the generation and + is formatter dependent (it is standardized because it can be given on + the command line). + + This method is called by the ``-S`` :doc:`command-line option `, + the `arg` is then given by the ``-a`` option. + """ + return '' + + def format(self, tokensource, outfile): + """ + This method must format the tokens from the `tokensource` iterable and + write the formatted version to the file object `outfile`. + + Formatter options can control how exactly the tokens are converted. + """ + if self.encoding: + # wrap the outfile in a StreamWriter + outfile = codecs.lookup(self.encoding)[3](outfile) + return self.format_unencoded(tokensource, outfile) + + # Allow writing Formatter[str] or Formatter[bytes]. That's equivalent to + # Formatter. This helps when using third-party type stubs from typeshed. + def __class_getitem__(cls, name): + return cls diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/__init__.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..829496dc97e147d159e106ddc12a49e8e416faf1 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/__init__.py @@ -0,0 +1,157 @@ +""" + pygments.formatters + ~~~~~~~~~~~~~~~~~~~ + + Pygments formatters. + + :copyright: Copyright 2006-present by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re +import sys +import types +import fnmatch +from os.path import basename + +from pygments.formatters._mapping import FORMATTERS +from pygments.plugin import find_plugin_formatters +from pygments.util import ClassNotFound + +__all__ = ['get_formatter_by_name', 'get_formatter_for_filename', + 'get_all_formatters', 'load_formatter_from_file'] + list(FORMATTERS) + +_formatter_cache = {} # classes by name +_pattern_cache = {} + + +def _fn_matches(fn, glob): + """Return whether the supplied file name fn matches pattern filename.""" + if glob not in _pattern_cache: + pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob)) + return pattern.match(fn) + return _pattern_cache[glob].match(fn) + + +def _load_formatters(module_name): + """Load a formatter (and all others in the module too).""" + mod = __import__(module_name, None, None, ['__all__']) + for formatter_name in mod.__all__: + cls = getattr(mod, formatter_name) + _formatter_cache[cls.name] = cls + + +def get_all_formatters(): + """Return a generator for all formatter classes.""" + # NB: this returns formatter classes, not info like get_all_lexers(). + for info in FORMATTERS.values(): + if info[1] not in _formatter_cache: + _load_formatters(info[0]) + yield _formatter_cache[info[1]] + for _, formatter in find_plugin_formatters(): + yield formatter + + +def find_formatter_class(alias): + """Lookup a formatter by alias. + + Returns None if not found. + """ + for module_name, name, aliases, _, _ in FORMATTERS.values(): + if alias in aliases: + if name not in _formatter_cache: + _load_formatters(module_name) + return _formatter_cache[name] + for _, cls in find_plugin_formatters(): + if alias in cls.aliases: + return cls + + +def get_formatter_by_name(_alias, **options): + """ + Return an instance of a :class:`.Formatter` subclass that has `alias` in its + aliases list. The formatter is given the `options` at its instantiation. + + Will raise :exc:`pygments.util.ClassNotFound` if no formatter with that + alias is found. + """ + cls = find_formatter_class(_alias) + if cls is None: + raise ClassNotFound(f"no formatter found for name {_alias!r}") + return cls(**options) + + +def load_formatter_from_file(filename, formattername="CustomFormatter", **options): + """ + Return a `Formatter` subclass instance loaded from the provided file, relative + to the current directory. + + The file is expected to contain a Formatter class named ``formattername`` + (by default, CustomFormatter). Users should be very careful with the input, because + this method is equivalent to running ``eval()`` on the input file. The formatter is + given the `options` at its instantiation. + + :exc:`pygments.util.ClassNotFound` is raised if there are any errors loading + the formatter. + + .. versionadded:: 2.2 + """ + try: + # This empty dict will contain the namespace for the exec'd file + custom_namespace = {} + with open(filename, 'rb') as f: + exec(f.read(), custom_namespace) + # Retrieve the class `formattername` from that namespace + if formattername not in custom_namespace: + raise ClassNotFound(f'no valid {formattername} class found in {filename}') + formatter_class = custom_namespace[formattername] + # And finally instantiate it with the options + return formatter_class(**options) + except OSError as err: + raise ClassNotFound(f'cannot read {filename}: {err}') + except ClassNotFound: + raise + except Exception as err: + raise ClassNotFound(f'error when loading custom formatter: {err}') + + +def get_formatter_for_filename(fn, **options): + """ + Return a :class:`.Formatter` subclass instance that has a filename pattern + matching `fn`. The formatter is given the `options` at its instantiation. + + Will raise :exc:`pygments.util.ClassNotFound` if no formatter for that filename + is found. + """ + fn = basename(fn) + for modname, name, _, filenames, _ in FORMATTERS.values(): + for filename in filenames: + if _fn_matches(fn, filename): + if name not in _formatter_cache: + _load_formatters(modname) + return _formatter_cache[name](**options) + for _name, cls in find_plugin_formatters(): + for filename in cls.filenames: + if _fn_matches(fn, filename): + return cls(**options) + raise ClassNotFound(f"no formatter found for file name {fn!r}") + + +class _automodule(types.ModuleType): + """Automatically import formatters.""" + + def __getattr__(self, name): + info = FORMATTERS.get(name) + if info: + _load_formatters(info[0]) + cls = _formatter_cache[info[1]] + setattr(self, name, cls) + return cls + raise AttributeError(name) + + +oldmod = sys.modules[__name__] +newmod = _automodule(__name__) +newmod.__dict__.update(oldmod.__dict__) +sys.modules[__name__] = newmod +del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/_mapping.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/_mapping.py new file mode 100644 index 0000000000000000000000000000000000000000..72ca84040b626183e3328679db600c13472021be --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/_mapping.py @@ -0,0 +1,23 @@ +# Automatically generated by scripts/gen_mapfiles.py. +# DO NOT EDIT BY HAND; run `tox -e mapfiles` instead. + +FORMATTERS = { + 'BBCodeFormatter': ('pygments.formatters.bbcode', 'BBCode', ('bbcode', 'bb'), (), 'Format tokens with BBcodes. These formatting codes are used by many bulletin boards, so you can highlight your sourcecode with pygments before posting it there.'), + 'BmpImageFormatter': ('pygments.formatters.img', 'img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'), + 'GifImageFormatter': ('pygments.formatters.img', 'img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'), + 'GroffFormatter': ('pygments.formatters.groff', 'groff', ('groff', 'troff', 'roff'), (), 'Format tokens with groff escapes to change their color and font style.'), + 'HtmlFormatter': ('pygments.formatters.html', 'HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ```` tags. By default, the content is enclosed in a ``
`` tag, itself wrapped in a ``
`` tag (but see the `nowrap` option). The ``
``'s CSS class can be set by the `cssclass` option."), + 'IRCFormatter': ('pygments.formatters.irc', 'IRC', ('irc', 'IRC'), (), 'Format tokens with IRC color sequences'), + 'ImageFormatter': ('pygments.formatters.img', 'img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'), + 'JpgImageFormatter': ('pygments.formatters.img', 'img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'), + 'LatexFormatter': ('pygments.formatters.latex', 'LaTeX', ('latex', 'tex'), ('*.tex',), 'Format tokens as LaTeX code. This needs the `fancyvrb` and `color` standard packages.'), + 'NullFormatter': ('pygments.formatters.other', 'Text only', ('text', 'null'), ('*.txt',), 'Output the text unchanged without any formatting.'), + 'PangoMarkupFormatter': ('pygments.formatters.pangomarkup', 'Pango Markup', ('pango', 'pangomarkup'), (), 'Format tokens as Pango Markup code. It can then be rendered to an SVG.'), + 'RawTokenFormatter': ('pygments.formatters.other', 'Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'), + 'RtfFormatter': ('pygments.formatters.rtf', 'RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft(R) Word(R) documents.'), + 'SvgFormatter': ('pygments.formatters.svg', 'SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file. This formatter is still experimental. Each line of code is a ```` element with explicit ``x`` and ``y`` coordinates containing ```` elements with the individual token styles.'), + 'Terminal256Formatter': ('pygments.formatters.terminal256', 'Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'), + 'TerminalFormatter': ('pygments.formatters.terminal', 'Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'), + 'TerminalTrueColorFormatter': ('pygments.formatters.terminal256', 'TerminalTrueColor', ('terminal16m', 'console16m', '16m'), (), 'Format tokens with ANSI color sequences, for output in a true-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'), + 'TestcaseFormatter': ('pygments.formatters.other', 'Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.'), +} diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/bbcode.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/bbcode.py new file mode 100644 index 0000000000000000000000000000000000000000..90d8d806f202e9dff22e33c431f3e35cefdd6642 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/bbcode.py @@ -0,0 +1,108 @@ +""" + pygments.formatters.bbcode + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + BBcode formatter. + + :copyright: Copyright 2006-present by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + + +from pygments.formatter import Formatter +from pygments.util import get_bool_opt + +__all__ = ['BBCodeFormatter'] + + +class BBCodeFormatter(Formatter): + """ + Format tokens with BBcodes. These formatting codes are used by many + bulletin boards, so you can highlight your sourcecode with pygments before + posting it there. + + This formatter has no support for background colors and borders, as there + are no common BBcode tags for that. + + Some board systems (e.g. phpBB) don't support colors in their [code] tag, + so you can't use the highlighting together with that tag. + Text in a [code] tag usually is shown with a monospace font (which this + formatter can do with the ``monofont`` option) and no spaces (which you + need for indentation) are removed. + + Additional options accepted: + + `style` + The style to use, can be a string or a Style subclass (default: + ``'default'``). + + `codetag` + If set to true, put the output into ``[code]`` tags (default: + ``false``) + + `monofont` + If set to true, add a tag to show the code with a monospace font + (default: ``false``). + """ + name = 'BBCode' + aliases = ['bbcode', 'bb'] + filenames = [] + + def __init__(self, **options): + Formatter.__init__(self, **options) + self._code = get_bool_opt(options, 'codetag', False) + self._mono = get_bool_opt(options, 'monofont', False) + + self.styles = {} + self._make_styles() + + def _make_styles(self): + for ttype, ndef in self.style: + start = end = '' + if ndef['color']: + start += '[color=#{}]'.format(ndef['color']) + end = '[/color]' + end + if ndef['bold']: + start += '[b]' + end = '[/b]' + end + if ndef['italic']: + start += '[i]' + end = '[/i]' + end + if ndef['underline']: + start += '[u]' + end = '[/u]' + end + # there are no common BBcodes for background-color and border + + self.styles[ttype] = start, end + + def format_unencoded(self, tokensource, outfile): + if self._code: + outfile.write('[code]') + if self._mono: + outfile.write('[font=monospace]') + + lastval = '' + lasttype = None + + for ttype, value in tokensource: + while ttype not in self.styles: + ttype = ttype.parent + if ttype == lasttype: + lastval += value + else: + if lastval: + start, end = self.styles[lasttype] + outfile.write(''.join((start, lastval, end))) + lastval = value + lasttype = ttype + + if lastval: + start, end = self.styles[lasttype] + outfile.write(''.join((start, lastval, end))) + + if self._mono: + outfile.write('[/font]') + if self._code: + outfile.write('[/code]') + if self._code or self._mono: + outfile.write('\n') diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/groff.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/groff.py new file mode 100644 index 0000000000000000000000000000000000000000..f5d3146b45b0103dbad53aa2822757b8102d1c42 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/groff.py @@ -0,0 +1,170 @@ +""" + pygments.formatters.groff + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + Formatter for groff output. + + :copyright: Copyright 2006-present by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import math +from pygments.formatter import Formatter +from pygments.util import get_bool_opt, get_int_opt + +__all__ = ['GroffFormatter'] + + +class GroffFormatter(Formatter): + """ + Format tokens with groff escapes to change their color and font style. + + .. versionadded:: 2.11 + + Additional options accepted: + + `style` + The style to use, can be a string or a Style subclass (default: + ``'default'``). + + `monospaced` + If set to true, monospace font will be used (default: ``true``). + + `linenos` + If set to true, print the line numbers (default: ``false``). + + `wrap` + Wrap lines to the specified number of characters. Disabled if set to 0 + (default: ``0``). + """ + + name = 'groff' + aliases = ['groff','troff','roff'] + filenames = [] + + def __init__(self, **options): + Formatter.__init__(self, **options) + + self.monospaced = get_bool_opt(options, 'monospaced', True) + self.linenos = get_bool_opt(options, 'linenos', False) + self._lineno = 0 + self.wrap = get_int_opt(options, 'wrap', 0) + self._linelen = 0 + + self.styles = {} + self._make_styles() + + + def _make_styles(self): + regular = '\\f[CR]' if self.monospaced else '\\f[R]' + bold = '\\f[CB]' if self.monospaced else '\\f[B]' + italic = '\\f[CI]' if self.monospaced else '\\f[I]' + + for ttype, ndef in self.style: + start = end = '' + if ndef['color']: + start += '\\m[{}]'.format(ndef['color']) + end = '\\m[]' + end + if ndef['bold']: + start += bold + end = regular + end + if ndef['italic']: + start += italic + end = regular + end + if ndef['bgcolor']: + start += '\\M[{}]'.format(ndef['bgcolor']) + end = '\\M[]' + end + + self.styles[ttype] = start, end + + + def _define_colors(self, outfile): + colors = set() + for _, ndef in self.style: + if ndef['color'] is not None: + colors.add(ndef['color']) + + for color in sorted(colors): + outfile.write('.defcolor ' + color + ' rgb #' + color + '\n') + + + def _write_lineno(self, outfile): + self._lineno += 1 + outfile.write("%s% 4d " % (self._lineno != 1 and '\n' or '', self._lineno)) + + + def _wrap_line(self, line): + length = len(line.rstrip('\n')) + space = ' ' if self.linenos else '' + newline = '' + + if length > self.wrap: + for i in range(0, math.floor(length / self.wrap)): + chunk = line[i*self.wrap:i*self.wrap+self.wrap] + newline += (chunk + '\n' + space) + remainder = length % self.wrap + if remainder > 0: + newline += line[-remainder-1:] + self._linelen = remainder + elif self._linelen + length > self.wrap: + newline = ('\n' + space) + line + self._linelen = length + else: + newline = line + self._linelen += length + + return newline + + + def _escape_chars(self, text): + text = text.replace('\\', '\\[u005C]'). \ + replace('.', '\\[char46]'). \ + replace('\'', '\\[u0027]'). \ + replace('`', '\\[u0060]'). \ + replace('~', '\\[u007E]') + copy = text + + for char in copy: + if len(char) != len(char.encode()): + uni = char.encode('unicode_escape') \ + .decode()[1:] \ + .replace('x', 'u00') \ + .upper() + text = text.replace(char, '\\[u' + uni[1:] + ']') + + return text + + + def format_unencoded(self, tokensource, outfile): + self._define_colors(outfile) + + outfile.write('.nf\n\\f[CR]\n') + + if self.linenos: + self._write_lineno(outfile) + + for ttype, value in tokensource: + while ttype not in self.styles: + ttype = ttype.parent + start, end = self.styles[ttype] + + for line in value.splitlines(True): + if self.wrap > 0: + line = self._wrap_line(line) + + if start and end: + text = self._escape_chars(line.rstrip('\n')) + if text != '': + outfile.write(''.join((start, text, end))) + else: + outfile.write(self._escape_chars(line.rstrip('\n'))) + + if line.endswith('\n'): + if self.linenos: + self._write_lineno(outfile) + self._linelen = 0 + else: + outfile.write('\n') + self._linelen = 0 + + outfile.write('\n.fi') diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/html.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/html.py new file mode 100644 index 0000000000000000000000000000000000000000..17e67414520e016127fba1541207f23c19a0d187 --- /dev/null +++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/formatters/html.py @@ -0,0 +1,997 @@ +""" + pygments.formatters.html + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Formatter for HTML output. + + :copyright: Copyright 2006-present by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import functools +import os +import sys +import os.path +from io import StringIO + +from pygments.formatter import Formatter +from pygments.token import Token, Text, STANDARD_TYPES +from pygments.util import get_bool_opt, get_int_opt, get_list_opt + +import html + +try: + import ctags +except ImportError: + ctags = None + +__all__ = ['HtmlFormatter'] + + +_escape_html_table = { + ord('&'): '&', + ord('<'): '<', + ord('>'): '>', + ord('"'): '"', + ord("'"): ''', +} + + +def escape_html(text, table=_escape_html_table): + """Escape &, <, > as well as single and double quotes for HTML.""" + return text.translate(table) + + +def webify(color): + if color.startswith('calc') or color.startswith('var'): + return color + else: + # Check if the color can be shortened from 6 to 3 characters + color = color.upper() + if (len(color) == 6 and + ( color[0] == color[1] + and color[2] == color[3] + and color[4] == color[5])): + return f'#{color[0]}{color[2]}{color[4]}' + else: + return f'#{color}' + + +def _get_ttype_class(ttype): + fname = STANDARD_TYPES.get(ttype) + if fname: + return fname + aname = '' + while fname is None: + aname = '-' + ttype[-1] + aname + ttype = ttype.parent + fname = STANDARD_TYPES.get(ttype) + return fname + aname + + +CSSFILE_TEMPLATE = '''\ +/* +generated by Pygments +Copyright 2006-present by the Pygments team. +Licensed under the BSD license, see LICENSE for details. +*/ +%(styledefs)s +''' + +DOC_HEADER = '''\ + + + + + %(title)s + + + + +

%(title)s

+ +''' + +DOC_HEADER_EXTERNALCSS = '''\ + + + + + %(title)s + + + + +

%(title)s

+ +''' + +DOC_FOOTER = '''\ + + +''' + + +class HtmlFormatter(Formatter): + r""" + Format tokens as HTML 4 ```` tags. By default, the content is enclosed + in a ``
`` tag, itself wrapped in a ``
`` tag (but see the `nowrap` option). + The ``
``'s CSS class can be set by the `cssclass` option. + + If the `linenos` option is set to ``"table"``, the ``
`` is
+    additionally wrapped inside a ```` which has one row and two
+    cells: one containing the line numbers and one containing the code.
+    Example:
+
+    .. sourcecode:: html
+
+        
+
+ + +
+
1
+            2
+
+
def foo(bar):
+              pass
+            
+
+ + (whitespace added to improve clarity). + + A list of lines can be specified using the `hl_lines` option to make these + lines highlighted (as of Pygments 0.11). + + With the `full` option, a complete HTML 4 document is output, including + the style definitions inside a ``$)', _handle_cssblock), + + include('keywords'), + include('inline'), + ], + 'keywords': [ + (words(( + '\\define', '\\end', 'caption', 'created', 'modified', 'tags', + 'title', 'type'), prefix=r'^', suffix=r'\b'), + Keyword), + ], + 'inline': [ + # escape + (r'\\.', Text), + # created or modified date + (r'\d{17}', Number.Integer), + # italics + (r'(\s)(//[^/]+//)((?=\W|\n))', + bygroups(Text, Generic.Emph, Text)), + # superscript + (r'(\s)(\^\^[^\^]+\^\^)', bygroups(Text, Generic.Emph)), + # subscript + (r'(\s)(,,[^,]+,,)', bygroups(Text, Generic.Emph)), + # underscore + (r'(\s)(__[^_]+__)', bygroups(Text, Generic.Strong)), + # bold + (r"(\s)(''[^']+'')((?=\W|\n))", + bygroups(Text, Generic.Strong, Text)), + # strikethrough + (r'(\s)(~~[^~]+~~)((?=\W|\n))', + bygroups(Text, Generic.Deleted, Text)), + # TiddlyWiki variables + (r'<<[^>]+>>', Name.Tag), + (r'\$\$[^$]+\$\$', Name.Tag), + (r'\$\([^)]+\)\$', Name.Tag), + # TiddlyWiki style or class + (r'^@@.*$', Name.Tag), + # HTML tags + (r']+>', Name.Tag), + # inline code + (r'`[^`]+`', String.Backtick), + # HTML escaped symbols + (r'&\S*?;', String.Regex), + # Wiki links + (r'(\[{2})([^]\|]+)(\]{2})', bygroups(Text, Name.Tag, Text)), + # External links + (r'(\[{2})([^]\|]+)(\|)([^]\|]+)(\]{2})', + bygroups(Text, Name.Tag, Text, Name.Attribute, Text)), + # Transclusion + (r'(\{{2})([^}]+)(\}{2})', bygroups(Text, Name.Tag, Text)), + # URLs + (r'(\b.?.?tps?://[^\s"]+)', bygroups(Name.Attribute)), + + # general text, must come last! + (r'[\w]+', Text), + (r'.', Text) + ], + } + + def __init__(self, **options): + self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True) + RegexLexer.__init__(self, **options) + + +class WikitextLexer(RegexLexer): + """ + For MediaWiki Wikitext. + + Parsing Wikitext is tricky, and results vary between different MediaWiki + installations, so we only highlight common syntaxes (built-in or from + popular extensions), and also assume templates produce no unbalanced + syntaxes. + """ + name = 'Wikitext' + url = 'https://www.mediawiki.org/wiki/Wikitext' + aliases = ['wikitext', 'mediawiki'] + filenames = [] + mimetypes = ['text/x-wiki'] + version_added = '2.15' + flags = re.MULTILINE + + def nowiki_tag_rules(tag_name): + return [ + (rf'(?i)()', bygroups(Punctuation, + Name.Tag, Whitespace, Punctuation), '#pop'), + include('entity'), + include('text'), + ] + + def plaintext_tag_rules(tag_name): + return [ + (rf'(?si)(.*?)()', bygroups(Text, + Punctuation, Name.Tag, Whitespace, Punctuation), '#pop'), + ] + + def delegate_tag_rules(tag_name, lexer, **lexer_kwargs): + return [ + (rf'(?i)()', bygroups(Punctuation, + Name.Tag, Whitespace, Punctuation), '#pop'), + (rf'(?si).+?(?=)', using(lexer, **lexer_kwargs)), + ] + + def text_rules(token): + return [ + (r'\w+', token), + (r'[^\S\n]+', token), + (r'(?s).', token), + ] + + def handle_syntaxhighlight(self, match, ctx): + from pygments.lexers import get_lexer_by_name + + attr_content = match.group() + start = 0 + index = 0 + while True: + index = attr_content.find('>', start) + # Exclude comment end (-->) + if attr_content[index-2:index] != '--': + break + start = index + 1 + + if index == -1: + # No tag end + yield from self.get_tokens_unprocessed(attr_content, stack=['root', 'attr']) + return + attr = attr_content[:index] + yield from self.get_tokens_unprocessed(attr, stack=['root', 'attr']) + yield match.start(3) + index, Punctuation, '>' + + lexer = None + content = attr_content[index+1:] + lang_match = re.findall(r'\blang=("|\'|)(\w+)(\1)', attr) + + if len(lang_match) >= 1: + # Pick the last match in case of multiple matches + lang = lang_match[-1][1] + try: + lexer = get_lexer_by_name(lang) + except ClassNotFound: + pass + + if lexer is None: + yield match.start() + index + 1, Text, content + else: + yield from lexer.get_tokens_unprocessed(content) + + def handle_score(self, match, ctx): + attr_content = match.group() + start = 0 + index = 0 + while True: + index = attr_content.find('>', start) + # Exclude comment end (-->) + if attr_content[index-2:index] != '--': + break + start = index + 1 + + if index == -1: + # No tag end + yield from self.get_tokens_unprocessed(attr_content, stack=['root', 'attr']) + return + attr = attr_content[:index] + content = attr_content[index+1:] + yield from self.get_tokens_unprocessed(attr, stack=['root', 'attr']) + yield match.start(3) + index, Punctuation, '>' + + lang_match = re.findall(r'\blang=("|\'|)(\w+)(\1)', attr) + # Pick the last match in case of multiple matches + lang = lang_match[-1][1] if len(lang_match) >= 1 else 'lilypond' + + if lang == 'lilypond': # Case sensitive + yield from LilyPondLexer().get_tokens_unprocessed(content) + else: # ABC + # FIXME: Use ABC lexer in the future + yield match.start() + index + 1, Text, content + + # a-z removed to prevent linter from complaining, REMEMBER to use (?i) + title_char = r' %!"$&\'()*,\-./0-9:;=?@A-Z\\\^_`~+\u0080-\uFFFF' + nbsp_char = r'(?:\t| |&\#0*160;|&\#[Xx]0*[Aa]0;|[ \xA0\u1680\u2000-\u200A\u202F\u205F\u3000])' + link_address = r'(?:[0-9.]+|\[[0-9a-f:.]+\]|[^\x00-\x20"<>\[\]\x7F\xA0\u1680\u2000-\u200A\u202F\u205F\u3000\uFFFD])' + link_char_class = r'[^\x00-\x20"<>\[\]\x7F\xA0\u1680\u2000-\u200A\u202F\u205F\u3000\uFFFD]' + double_slashes_i = { + '__FORCETOC__', '__NOCONTENTCONVERT__', '__NOCC__', '__NOEDITSECTION__', '__NOGALLERY__', + '__NOTITLECONVERT__', '__NOTC__', '__NOTOC__', '__TOC__', + } + double_slashes = { + '__EXPECTUNUSEDCATEGORY__', '__HIDDENCAT__', '__INDEX__', '__NEWSECTIONLINK__', + '__NOINDEX__', '__NONEWSECTIONLINK__', '__STATICREDIRECT__', '__NOGLOBAL__', + '__DISAMBIG__', '__EXPECTED_UNCONNECTED_PAGE__', + } + protocols = { + 'bitcoin:', 'ftp://', 'ftps://', 'geo:', 'git://', 'gopher://', 'http://', 'https://', + 'irc://', 'ircs://', 'magnet:', 'mailto:', 'mms://', 'news:', 'nntp://', 'redis://', + 'sftp://', 'sip:', 'sips:', 'sms:', 'ssh://', 'svn://', 'tel:', 'telnet://', 'urn:', + 'worldwind://', 'xmpp:', '//', + } + non_relative_protocols = protocols - {'//'} + html_tags = { + 'abbr', 'b', 'bdi', 'bdo', 'big', 'blockquote', 'br', 'caption', 'center', 'cite', 'code', + 'data', 'dd', 'del', 'dfn', 'div', 'dl', 'dt', 'em', 'font', 'h1', 'h2', 'h3', 'h4', 'h5', + 'h6', 'hr', 'i', 'ins', 'kbd', 'li', 'link', 'mark', 'meta', 'ol', 'p', 'q', 'rb', 'rp', + 'rt', 'rtc', 'ruby', 's', 'samp', 'small', 'span', 'strike', 'strong', 'sub', 'sup', + 'table', 'td', 'th', 'time', 'tr', 'tt', 'u', 'ul', 'var', 'wbr', + } + parser_tags = { + 'graph', 'charinsert', 'rss', 'chem', 'categorytree', 'nowiki', 'inputbox', 'math', + 'hiero', 'score', 'pre', 'ref', 'translate', 'imagemap', 'templatestyles', 'languages', + 'noinclude', 'mapframe', 'section', 'poem', 'syntaxhighlight', 'includeonly', 'tvar', + 'onlyinclude', 'templatedata', 'langconvert', 'timeline', 'dynamicpagelist', 'gallery', + 'maplink', 'ce', 'references', + } + variant_langs = { + # ZhConverter.php + 'zh', 'zh-hans', 'zh-hant', 'zh-cn', 'zh-hk', 'zh-mo', 'zh-my', 'zh-sg', 'zh-tw', + # WuuConverter.php + 'wuu', 'wuu-hans', 'wuu-hant', + # UzConverter.php + 'uz', 'uz-latn', 'uz-cyrl', + # TlyConverter.php + 'tly', 'tly-cyrl', + # TgConverter.php + 'tg', 'tg-latn', + # SrConverter.php + 'sr', 'sr-ec', 'sr-el', + # ShiConverter.php + 'shi', 'shi-tfng', 'shi-latn', + # ShConverter.php + 'sh-latn', 'sh-cyrl', + # KuConverter.php + 'ku', 'ku-arab', 'ku-latn', + # IuConverter.php + 'iu', 'ike-cans', 'ike-latn', + # GanConverter.php + 'gan', 'gan-hans', 'gan-hant', + # EnConverter.php + 'en', 'en-x-piglatin', + # CrhConverter.php + 'crh', 'crh-cyrl', 'crh-latn', + # BanConverter.php + 'ban', 'ban-bali', 'ban-x-dharma', 'ban-x-palmleaf', 'ban-x-pku', + } + magic_vars_i = { + 'ARTICLEPATH', 'INT', 'PAGEID', 'SCRIPTPATH', 'SERVER', 'SERVERNAME', 'STYLEPATH', + } + magic_vars = { + '!', '=', 'BASEPAGENAME', 'BASEPAGENAMEE', 'CASCADINGSOURCES', 'CONTENTLANGUAGE', + 'CONTENTLANG', 'CURRENTDAY', 'CURRENTDAY2', 'CURRENTDAYNAME', 'CURRENTDOW', 'CURRENTHOUR', + 'CURRENTMONTH', 'CURRENTMONTH2', 'CURRENTMONTH1', 'CURRENTMONTHABBREV', 'CURRENTMONTHNAME', + 'CURRENTMONTHNAMEGEN', 'CURRENTTIME', 'CURRENTTIMESTAMP', 'CURRENTVERSION', 'CURRENTWEEK', + 'CURRENTYEAR', 'DIRECTIONMARK', 'DIRMARK', 'FULLPAGENAME', 'FULLPAGENAMEE', 'LOCALDAY', + 'LOCALDAY2', 'LOCALDAYNAME', 'LOCALDOW', 'LOCALHOUR', 'LOCALMONTH', 'LOCALMONTH2', + 'LOCALMONTH1', 'LOCALMONTHABBREV', 'LOCALMONTHNAME', 'LOCALMONTHNAMEGEN', 'LOCALTIME', + 'LOCALTIMESTAMP', 'LOCALWEEK', 'LOCALYEAR', 'NAMESPACE', 'NAMESPACEE', 'NAMESPACENUMBER', + 'NUMBEROFACTIVEUSERS', 'NUMBEROFADMINS', 'NUMBEROFARTICLES', 'NUMBEROFEDITS', + 'NUMBEROFFILES', 'NUMBEROFPAGES', 'NUMBEROFUSERS', 'PAGELANGUAGE', 'PAGENAME', 'PAGENAMEE', + 'REVISIONDAY', 'REVISIONDAY2', 'REVISIONID', 'REVISIONMONTH', 'REVISIONMONTH1', + 'REVISIONSIZE', 'REVISIONTIMESTAMP', 'REVISIONUSER', 'REVISIONYEAR', 'ROOTPAGENAME', + 'ROOTPAGENAMEE', 'SITENAME', 'SUBJECTPAGENAME', 'ARTICLEPAGENAME', 'SUBJECTPAGENAMEE', + 'ARTICLEPAGENAMEE', 'SUBJECTSPACE', 'ARTICLESPACE', 'SUBJECTSPACEE', 'ARTICLESPACEE', + 'SUBPAGENAME', 'SUBPAGENAMEE', 'TALKPAGENAME', 'TALKPAGENAMEE', 'TALKSPACE', 'TALKSPACEE', + } + parser_functions_i = { + 'ANCHORENCODE', 'BIDI', 'CANONICALURL', 'CANONICALURLE', 'FILEPATH', 'FORMATNUM', + 'FULLURL', 'FULLURLE', 'GENDER', 'GRAMMAR', 'INT', r'\#LANGUAGE', 'LC', 'LCFIRST', 'LOCALURL', + 'LOCALURLE', 'NS', 'NSE', 'PADLEFT', 'PADRIGHT', 'PAGEID', 'PLURAL', 'UC', 'UCFIRST', + 'URLENCODE', + } + parser_functions = { + 'BASEPAGENAME', 'BASEPAGENAMEE', 'CASCADINGSOURCES', 'DEFAULTSORT', 'DEFAULTSORTKEY', + 'DEFAULTCATEGORYSORT', 'FULLPAGENAME', 'FULLPAGENAMEE', 'NAMESPACE', 'NAMESPACEE', + 'NAMESPACENUMBER', 'NUMBERINGROUP', 'NUMINGROUP', 'NUMBEROFACTIVEUSERS', 'NUMBEROFADMINS', + 'NUMBEROFARTICLES', 'NUMBEROFEDITS', 'NUMBEROFFILES', 'NUMBEROFPAGES', 'NUMBEROFUSERS', + 'PAGENAME', 'PAGENAMEE', 'PAGESINCATEGORY', 'PAGESINCAT', 'PAGESIZE', 'PROTECTIONEXPIRY', + 'PROTECTIONLEVEL', 'REVISIONDAY', 'REVISIONDAY2', 'REVISIONID', 'REVISIONMONTH', + 'REVISIONMONTH1', 'REVISIONTIMESTAMP', 'REVISIONUSER', 'REVISIONYEAR', 'ROOTPAGENAME', + 'ROOTPAGENAMEE', 'SUBJECTPAGENAME', 'ARTICLEPAGENAME', 'SUBJECTPAGENAMEE', + 'ARTICLEPAGENAMEE', 'SUBJECTSPACE', 'ARTICLESPACE', 'SUBJECTSPACEE', 'ARTICLESPACEE', + 'SUBPAGENAME', 'SUBPAGENAMEE', 'TALKPAGENAME', 'TALKPAGENAMEE', 'TALKSPACE', 'TALKSPACEE', + 'INT', 'DISPLAYTITLE', 'PAGESINNAMESPACE', 'PAGESINNS', + } + + tokens = { + 'root': [ + # Redirects + (r"""(?xi) + (\A\s*?)(\#REDIRECT:?) # may contain a colon + (\s+)(\[\[) (?=[^\]\n]* \]\]$) + """, + bygroups(Whitespace, Keyword, Whitespace, Punctuation), 'redirect-inner'), + # Subheadings + (r'^(={2,6})(.+?)(\1)(\s*$\n)', + bygroups(Generic.Subheading, Generic.Subheading, Generic.Subheading, Whitespace)), + # Headings + (r'^(=.+?=)(\s*$\n)', + bygroups(Generic.Heading, Whitespace)), + # Double-slashed magic words + (words(double_slashes_i, prefix=r'(?i)'), Name.Function.Magic), + (words(double_slashes), Name.Function.Magic), + # Raw URLs + (r'(?i)\b(?:{}){}{}*'.format('|'.join(protocols), + link_address, link_char_class), Name.Label), + # Magic links + (rf'\b(?:RFC|PMID){nbsp_char}+[0-9]+\b', + Name.Function.Magic), + (r"""(?x) + \bISBN {nbsp_char} + (?: 97[89] {nbsp_dash}? )? + (?: [0-9] {nbsp_dash}? ){{9}} # escape format() + [0-9Xx]\b + """.format(nbsp_char=nbsp_char, nbsp_dash=f'(?:-|{nbsp_char})'), Name.Function.Magic), + include('list'), + include('inline'), + include('text'), + ], + 'redirect-inner': [ + (r'(\]\])(\s*?\n)', bygroups(Punctuation, Whitespace), '#pop'), + (r'(\#)([^#]*?)', bygroups(Punctuation, Name.Label)), + (rf'(?i)[{title_char}]+', Name.Tag), + ], + 'list': [ + # Description lists + (r'^;', Keyword, 'dt'), + # Ordered lists, unordered lists and indents + (r'^[#:*]+', Keyword), + # Horizontal rules + (r'^-{4,}', Keyword), + ], + 'inline': [ + # Signatures + (r'~{3,5}', Keyword), + # Entities + include('entity'), + # Bold & italic + (r"('')(''')(?!')", bygroups(Generic.Emph, + Generic.EmphStrong), 'inline-italic-bold'), + (r"'''(?!')", Generic.Strong, 'inline-bold'), + (r"''(?!')", Generic.Emph, 'inline-italic'), + # Comments & parameters & templates + include('replaceable'), + # Media links + ( + r"""(?xi) + (\[\[) + (File|Image) (:) + ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} | )*) + (?: (\#) ([{}]*?) )? + """.format(title_char, f'{title_char}#'), + bygroups(Punctuation, Name.Namespace, Punctuation, + using(this, state=['wikilink-name']), Punctuation, Name.Label), + 'medialink-inner' + ), + # Wikilinks + ( + r"""(?xi) + (\[\[)(?!{}) # Should not contain URLs + (?: ([{}]*) (:))? + ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} | )*?) + (?: (\#) ([{}]*?) )? + (\]\]) + """.format('|'.join(protocols), title_char.replace('/', ''), + title_char, f'{title_char}#'), + bygroups(Punctuation, Name.Namespace, Punctuation, + using(this, state=['wikilink-name']), Punctuation, Name.Label, Punctuation) + ), + ( + r"""(?xi) + (\[\[)(?!{}) + (?: ([{}]*) (:))? + ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} | )*?) + (?: (\#) ([{}]*?) )? + (\|) + """.format('|'.join(protocols), title_char.replace('/', ''), + title_char, f'{title_char}#'), + bygroups(Punctuation, Name.Namespace, Punctuation, + using(this, state=['wikilink-name']), Punctuation, Name.Label, Punctuation), + 'wikilink-inner' + ), + # External links + ( + r"""(?xi) + (\[) + ((?:{}) {} {}*) + (\s*) + """.format('|'.join(protocols), link_address, link_char_class), + bygroups(Punctuation, Name.Label, Whitespace), + 'extlink-inner' + ), + # Tables + (r'^(:*)(\s*?)(\{\|)([^\n]*)$', bygroups(Keyword, + Whitespace, Punctuation, using(this, state=['root', 'attr'])), 'table'), + # HTML tags + (r'(?i)(<)({})\b'.format('|'.join(html_tags)), + bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'), + (r'(?i)()'.format('|'.join(html_tags)), + bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)), + # + (r'(?i)(<)(nowiki)\b', bygroups(Punctuation, + Name.Tag), ('tag-nowiki', 'tag-inner')), + #
+            (r'(?i)(<)(pre)\b', bygroups(Punctuation,
+             Name.Tag), ('tag-pre', 'tag-inner')),
+            # 
+            (r'(?i)(<)(categorytree)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-categorytree', 'tag-inner')),
+            # 
+            (r'(?i)(<)(hiero)\b', bygroups(Punctuation,
+             Name.Tag), ('tag-hiero', 'tag-inner')),
+            # 
+            (r'(?i)(<)(math)\b', bygroups(Punctuation,
+             Name.Tag), ('tag-math', 'tag-inner')),
+            # 
+            (r'(?i)(<)(chem)\b', bygroups(Punctuation,
+             Name.Tag), ('tag-chem', 'tag-inner')),
+            # 
+            (r'(?i)(<)(ce)\b', bygroups(Punctuation,
+             Name.Tag), ('tag-ce', 'tag-inner')),
+            # 
+            (r'(?i)(<)(charinsert)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-charinsert', 'tag-inner')),
+            # 
+            (r'(?i)(<)(templatedata)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-templatedata', 'tag-inner')),
+            # 
+            (r'(?i)(<)(gallery)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-gallery', 'tag-inner')),
+            # 
+            (r'(?i)(<)(gallery)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-graph', 'tag-inner')),
+            # 
+            (r'(?i)(<)(dynamicpagelist)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-dynamicpagelist', 'tag-inner')),
+            # 
+            (r'(?i)(<)(inputbox)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-inputbox', 'tag-inner')),
+            # 
+            (r'(?i)(<)(rss)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-rss', 'tag-inner')),
+            # 
+            (r'(?i)(<)(imagemap)\b', bygroups(
+                Punctuation, Name.Tag), ('tag-imagemap', 'tag-inner')),
+            # 
+            (r'(?i)()',
+             bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
+            (r'(?si)(<)(syntaxhighlight)\b([^>]*?(?.*?)(?=)',
+             bygroups(Punctuation, Name.Tag, handle_syntaxhighlight)),
+            # : Fallback case for self-closing tags
+            (r'(?i)(<)(syntaxhighlight)\b(\s*?)((?:[^>]|-->)*?)(/\s*?(?)*?)(/\s*?(?)*?)(/\s*?(?|\Z)', Comment.Multiline),
+            # Parameters
+            (
+                r"""(?x)
+                (\{{3})
+                    ([^|]*?)
+                    (?=\}{3}|\|)
+                """,
+                bygroups(Punctuation, Name.Variable),
+                'parameter-inner',
+            ),
+            # Magic variables
+            (r'(?i)(\{{\{{)(\s*)({})(\s*)(\}}\}})'.format('|'.join(magic_vars_i)),
+             bygroups(Punctuation, Whitespace, Name.Function, Whitespace, Punctuation)),
+            (r'(\{{\{{)(\s*)({})(\s*)(\}}\}})'.format('|'.join(magic_vars)),
+                bygroups(Punctuation, Whitespace, Name.Function, Whitespace, Punctuation)),
+            # Parser functions & templates
+            (r'\{\{', Punctuation, 'template-begin-space'),
+            #  legacy syntax
+            (r'(?i)(<)(tvar)\b(\|)([^>]*?)(>)', bygroups(Punctuation,
+             Name.Tag, Punctuation, String, Punctuation)),
+            (r'', Punctuation, '#pop'),
+            # 
+            (r'(?i)(<)(tvar)\b', bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'),
+            (r'(?i)()',
+             bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
+        ],
+        'parameter-inner': [
+            (r'\}{3}', Punctuation, '#pop'),
+            (r'\|', Punctuation),
+            include('inline'),
+            include('text'),
+        ],
+        'template-begin-space': [
+            # Templates allow line breaks at the beginning, and due to how MediaWiki handles
+            # comments, an extra state is required to handle things like {{\n\n name}}
+            (r'|\Z)', Comment.Multiline),
+            (r'\s+', Whitespace),
+            # Parser functions
+            (
+                r'(?i)(\#[{}]*?|{})(:)'.format(title_char,
+                                           '|'.join(parser_functions_i)),
+                bygroups(Name.Function, Punctuation), ('#pop', 'template-inner')
+            ),
+            (
+                r'({})(:)'.format('|'.join(parser_functions)),
+                bygroups(Name.Function, Punctuation), ('#pop', 'template-inner')
+            ),
+            # Templates
+            (
+                rf'(?i)([{title_char}]*?)(:)',
+                bygroups(Name.Namespace, Punctuation), ('#pop', 'template-name')
+            ),
+            default(('#pop', 'template-name'),),
+        ],
+        'template-name': [
+            (r'(\s*?)(\|)', bygroups(Text, Punctuation), ('#pop', 'template-inner')),
+            (r'\}\}', Punctuation, '#pop'),
+            (r'\n', Text, '#pop'),
+            include('replaceable'),
+            *text_rules(Name.Tag),
+        ],
+        'template-inner': [
+            (r'\}\}', Punctuation, '#pop'),
+            (r'\|', Punctuation),
+            (
+                r"""(?x)
+                    (?<=\|)
+                    ( (?: (?! \{\{ | \}\} )[^=\|<])*? ) # Exclude templates and tags
+                    (=)
+                """,
+                bygroups(Name.Label, Operator)
+            ),
+            include('inline'),
+            include('text'),
+        ],
+        'table': [
+            # Use [ \t\n\r\0\x0B] instead of \s to follow PHP trim() behavior
+            # Endings
+            (r'^([ \t\n\r\0\x0B]*?)(\|\})',
+             bygroups(Whitespace, Punctuation), '#pop'),
+            # Table rows
+            (r'^([ \t\n\r\0\x0B]*?)(\|-+)(.*)$', bygroups(Whitespace, Punctuation,
+             using(this, state=['root', 'attr']))),
+            # Captions
+            (
+                r"""(?x)
+                ^([ \t\n\r\0\x0B]*?)(\|\+)
+                # Exclude links, template and tags
+                (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|) )?
+                (.*?)$
+                """,
+                bygroups(Whitespace, Punctuation, using(this, state=[
+                         'root', 'attr']), Punctuation, Generic.Heading),
+            ),
+            # Table data
+            (
+                r"""(?x)
+                ( ^(?:[ \t\n\r\0\x0B]*?)\| | \|\| )
+                (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|)(?!\|) )?
+                """,
+                bygroups(Punctuation, using(this, state=[
+                         'root', 'attr']), Punctuation),
+            ),
+            # Table headers
+            (
+                r"""(?x)
+                ( ^(?:[ \t\n\r\0\x0B]*?)!  )
+                (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|)(?!\|) )?
+                """,
+                bygroups(Punctuation, using(this, state=[
+                         'root', 'attr']), Punctuation),
+                'table-header',
+            ),
+            include('list'),
+            include('inline'),
+            include('text'),
+        ],
+        'table-header': [
+            # Requires another state for || handling inside headers
+            (r'\n', Text, '#pop'),
+            (
+                r"""(?x)
+                (!!|\|\|)
+                (?:
+                    ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )
+                    (\|)(?!\|)
+                )?
+                """,
+                bygroups(Punctuation, using(this, state=[
+                         'root', 'attr']), Punctuation)
+            ),
+            *text_rules(Generic.Subheading),
+        ],
+        'entity': [
+            (r'&\S*?;', Name.Entity),
+        ],
+        'dt': [
+            (r'\n', Text, '#pop'),
+            include('inline'),
+            (r':', Keyword, '#pop'),
+            include('text'),
+        ],
+        'extlink-inner': [
+            (r'\]', Punctuation, '#pop'),
+            include('inline'),
+            include('text'),
+        ],
+        'nowiki-ish': [
+            include('entity'),
+            include('text'),
+        ],
+        'attr': [
+            include('replaceable'),
+            (r'\s+', Whitespace),
+            (r'(=)(\s*)(")', bygroups(Operator, Whitespace, String.Double), 'attr-val-2'),
+            (r"(=)(\s*)(')", bygroups(Operator, Whitespace, String.Single), 'attr-val-1'),
+            (r'(=)(\s*)', bygroups(Operator, Whitespace), 'attr-val-0'),
+            (r'[\w:-]+', Name.Attribute),
+
+        ],
+        'attr-val-0': [
+            (r'\s', Whitespace, '#pop'),
+            include('replaceable'),
+            *text_rules(String),
+        ],
+        'attr-val-1': [
+            (r"'", String.Single, '#pop'),
+            include('replaceable'),
+            *text_rules(String.Single),
+        ],
+        'attr-val-2': [
+            (r'"', String.Double, '#pop'),
+            include('replaceable'),
+            *text_rules(String.Double),
+        ],
+        'tag-inner-ordinary': [
+            (r'/?\s*>', Punctuation, '#pop'),
+            include('tag-attr'),
+        ],
+        'tag-inner': [
+            # Return to root state for self-closing tags
+            (r'/\s*>', Punctuation, '#pop:2'),
+            (r'\s*>', Punctuation, '#pop'),
+            include('tag-attr'),
+        ],
+        # There states below are just like their non-tag variants, the key difference is
+        # they forcibly quit when encountering tag closing markup
+        'tag-attr': [
+            include('replaceable'),
+            (r'\s+', Whitespace),
+            (r'(=)(\s*)(")', bygroups(Operator,
+             Whitespace, String.Double), 'tag-attr-val-2'),
+            (r"(=)(\s*)(')", bygroups(Operator,
+             Whitespace, String.Single), 'tag-attr-val-1'),
+            (r'(=)(\s*)', bygroups(Operator, Whitespace), 'tag-attr-val-0'),
+            (r'[\w:-]+', Name.Attribute),
+
+        ],
+        'tag-attr-val-0': [
+            (r'\s', Whitespace, '#pop'),
+            (r'/?>', Punctuation, '#pop:2'),
+            include('replaceable'),
+            *text_rules(String),
+        ],
+        'tag-attr-val-1': [
+            (r"'", String.Single, '#pop'),
+            (r'/?>', Punctuation, '#pop:2'),
+            include('replaceable'),
+            *text_rules(String.Single),
+        ],
+        'tag-attr-val-2': [
+            (r'"', String.Double, '#pop'),
+            (r'/?>', Punctuation, '#pop:2'),
+            include('replaceable'),
+            *text_rules(String.Double),
+        ],
+        'tag-nowiki': nowiki_tag_rules('nowiki'),
+        'tag-pre': nowiki_tag_rules('pre'),
+        'tag-categorytree': plaintext_tag_rules('categorytree'),
+        'tag-dynamicpagelist': plaintext_tag_rules('dynamicpagelist'),
+        'tag-hiero': plaintext_tag_rules('hiero'),
+        'tag-inputbox': plaintext_tag_rules('inputbox'),
+        'tag-imagemap': plaintext_tag_rules('imagemap'),
+        'tag-charinsert': plaintext_tag_rules('charinsert'),
+        'tag-timeline': plaintext_tag_rules('timeline'),
+        'tag-gallery': plaintext_tag_rules('gallery'),
+        'tag-graph': plaintext_tag_rules('graph'),
+        'tag-rss': plaintext_tag_rules('rss'),
+        'tag-math': delegate_tag_rules('math', TexLexer, state='math'),
+        'tag-chem': delegate_tag_rules('chem', TexLexer, state='math'),
+        'tag-ce': delegate_tag_rules('ce', TexLexer, state='math'),
+        'tag-templatedata': delegate_tag_rules('templatedata', JsonLexer),
+        'text-italic': text_rules(Generic.Emph),
+        'text-bold': text_rules(Generic.Strong),
+        'text-bold-italic': text_rules(Generic.EmphStrong),
+        'text': text_rules(Text),
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/math.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/math.py
new file mode 100644
index 0000000000000000000000000000000000000000..b38c13e1209ae49df646465b00c5fd3df40bf30a
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/math.py
@@ -0,0 +1,21 @@
+"""
+    pygments.lexers.math
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Just export lexers that were contained in this module.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+# ruff: noqa: F401
+from pygments.lexers.python import NumPyLexer
+from pygments.lexers.matlab import MatlabLexer, MatlabSessionLexer, \
+    OctaveLexer, ScilabLexer
+from pygments.lexers.julia import JuliaLexer, JuliaConsoleLexer
+from pygments.lexers.r import RConsoleLexer, SLexer, RdLexer
+from pygments.lexers.modeling import BugsLexer, JagsLexer, StanLexer
+from pygments.lexers.idl import IDLLexer
+from pygments.lexers.algebra import MuPADLexer
+
+__all__ = []
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/matlab.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/matlab.py
new file mode 100644
index 0000000000000000000000000000000000000000..b27f4db63edd75313a76304d7a403f9d62d2b3b2
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/matlab.py
@@ -0,0 +1,3307 @@
+"""
+    pygments.lexers.matlab
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Matlab and related languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, bygroups, default, words, \
+    do_insertions, include
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Generic, Whitespace
+
+from pygments.lexers import _scilab_builtins
+
+__all__ = ['MatlabLexer', 'MatlabSessionLexer', 'OctaveLexer', 'ScilabLexer']
+
+
+class MatlabLexer(RegexLexer):
+    """
+    For Matlab source code.
+    """
+    name = 'Matlab'
+    aliases = ['matlab']
+    filenames = ['*.m']
+    mimetypes = ['text/matlab']
+    url = 'https://www.mathworks.com/products/matlab.html'
+    version_added = '0.10'
+
+    _operators = r'-|==|~=|<=|>=|<|>|&&|&|~|\|\|?|\.\*|\*|\+|\.\^|\^|\.\\|\./|/|\\'
+
+    tokens = {
+        'expressions': [
+            # operators:
+            (_operators, Operator),
+
+            # numbers (must come before punctuation to handle `.5`; cannot use
+            # `\b` due to e.g. `5. + .5`).  The negative lookahead on operators
+            # avoids including the dot in `1./x` (the dot is part of `./`).
+            (rf'(? and then
+            # (equal | open-parenthesis |  | ).
+            (rf'(?:^|(?<=;))(\s*)(\w+)(\s+)(?!=|\(|{_operators}\s|\s)',
+             bygroups(Whitespace, Name, Whitespace), 'commandargs'),
+
+            include('expressions')
+        ],
+        'blockcomment': [
+            (r'^\s*%\}', Comment.Multiline, '#pop'),
+            (r'^.*\n', Comment.Multiline),
+            (r'.', Comment.Multiline),
+        ],
+        'deffunc': [
+            (r'(\s*)(?:(\S+)(\s*)(=)(\s*))?(.+)(\()(.*)(\))(\s*)',
+             bygroups(Whitespace, Text, Whitespace, Punctuation,
+                      Whitespace, Name.Function, Punctuation, Text,
+                      Punctuation, Whitespace), '#pop'),
+            # function with no args
+            (r'(\s*)([a-zA-Z_]\w*)',
+             bygroups(Whitespace, Name.Function), '#pop'),
+        ],
+        'propattrs': [
+            (r'(\w+)(\s*)(=)(\s*)(\d+)',
+             bygroups(Name.Builtin, Whitespace, Punctuation, Whitespace,
+                      Number)),
+            (r'(\w+)(\s*)(=)(\s*)([a-zA-Z]\w*)',
+             bygroups(Name.Builtin, Whitespace, Punctuation, Whitespace,
+                      Keyword)),
+            (r',', Punctuation),
+            (r'\)', Punctuation, '#pop'),
+            (r'\s+', Whitespace),
+            (r'.', Text),
+        ],
+        'defprops': [
+            (r'%\{\s*\n', Comment.Multiline, 'blockcomment'),
+            (r'%.*$', Comment),
+            (r'(?.
+    """
+    name = 'Matlab session'
+    aliases = ['matlabsession']
+    url = 'https://www.mathworks.com/products/matlab.html'
+    version_added = '0.10'
+    _example = "matlabsession/matlabsession_sample.txt"
+
+    def get_tokens_unprocessed(self, text):
+        mlexer = MatlabLexer(**self.options)
+
+        curcode = ''
+        insertions = []
+        continuation = False
+
+        for match in line_re.finditer(text):
+            line = match.group()
+
+            if line.startswith('>> '):
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, line[:3])]))
+                curcode += line[3:]
+
+            elif line.startswith('>>'):
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, line[:2])]))
+                curcode += line[2:]
+
+            elif line.startswith('???'):
+
+                idx = len(curcode)
+
+                # without is showing error on same line as before...?
+                # line = "\n" + line
+                token = (0, Generic.Traceback, line)
+                insertions.append((idx, [token]))
+            elif continuation and insertions:
+                # line_start is the length of the most recent prompt symbol
+                line_start = len(insertions[-1][-1][-1])
+                # Set leading spaces with the length of the prompt to be a generic prompt
+                # This keeps code aligned when prompts are removed, say with some Javascript
+                if line.startswith(' '*line_start):
+                    insertions.append(
+                        (len(curcode), [(0, Generic.Prompt, line[:line_start])]))
+                    curcode += line[line_start:]
+                else:
+                    curcode += line
+            else:
+                if curcode:
+                    yield from do_insertions(
+                        insertions, mlexer.get_tokens_unprocessed(curcode))
+                    curcode = ''
+                    insertions = []
+
+                yield match.start(), Generic.Output, line
+
+            # Does not allow continuation if a comment is included after the ellipses.
+            # Continues any line that ends with ..., even comments (lines that start with %)
+            if line.strip().endswith('...'):
+                continuation = True
+            else:
+                continuation = False
+
+        if curcode:  # or item:
+            yield from do_insertions(
+                insertions, mlexer.get_tokens_unprocessed(curcode))
+
+
+class OctaveLexer(RegexLexer):
+    """
+    For GNU Octave source code.
+    """
+    name = 'Octave'
+    url = 'https://www.gnu.org/software/octave/index'
+    aliases = ['octave']
+    filenames = ['*.m']
+    mimetypes = ['text/octave']
+    version_added = '1.5'
+
+    # These lists are generated automatically.
+    # Run the following in bash shell:
+    #
+    # First dump all of the Octave manual into a plain text file:
+    #
+    #   $ info octave --subnodes -o octave-manual
+    #
+    # Now grep through it:
+
+    # for i in \
+    #     "Built-in Function" "Command" "Function File" \
+    #     "Loadable Function" "Mapping Function";
+    # do
+    #     perl -e '@name = qw('"$i"');
+    #              print lc($name[0]),"_kw = [\n"';
+    #
+    #     perl -n -e 'print "\"$1\",\n" if /-- '"$i"': .* (\w*) \(/;' \
+    #         octave-manual | sort | uniq ;
+    #     echo "]" ;
+    #     echo;
+    # done
+
+    # taken from Octave Mercurial changeset 8cc154f45e37 (30-jan-2011)
+
+    builtin_kw = (
+        "addlistener", "addpath", "addproperty", "all",
+        "and", "any", "argnames", "argv", "assignin",
+        "atexit", "autoload",
+        "available_graphics_toolkits", "beep_on_error",
+        "bitand", "bitmax", "bitor", "bitshift", "bitxor",
+        "cat", "cell", "cellstr", "char", "class", "clc",
+        "columns", "command_line_path",
+        "completion_append_char", "completion_matches",
+        "complex", "confirm_recursive_rmdir", "cputime",
+        "crash_dumps_octave_core", "ctranspose", "cumprod",
+        "cumsum", "debug_on_error", "debug_on_interrupt",
+        "debug_on_warning", "default_save_options",
+        "dellistener", "diag", "diff", "disp",
+        "doc_cache_file", "do_string_escapes", "double",
+        "drawnow", "e", "echo_executing_commands", "eps",
+        "eq", "errno", "errno_list", "error", "eval",
+        "evalin", "exec", "exist", "exit", "eye", "false",
+        "fclear", "fclose", "fcntl", "fdisp", "feof",
+        "ferror", "feval", "fflush", "fgetl", "fgets",
+        "fieldnames", "file_in_loadpath", "file_in_path",
+        "filemarker", "filesep", "find_dir_in_path",
+        "fixed_point_format", "fnmatch", "fopen", "fork",
+        "formula", "fprintf", "fputs", "fread", "freport",
+        "frewind", "fscanf", "fseek", "fskipl", "ftell",
+        "functions", "fwrite", "ge", "genpath", "get",
+        "getegid", "getenv", "geteuid", "getgid",
+        "getpgrp", "getpid", "getppid", "getuid", "glob",
+        "gt", "gui_mode", "history_control",
+        "history_file", "history_size",
+        "history_timestamp_format_string", "home",
+        "horzcat", "hypot", "ifelse",
+        "ignore_function_time_stamp", "inferiorto",
+        "info_file", "info_program", "inline", "input",
+        "intmax", "intmin", "ipermute",
+        "is_absolute_filename", "isargout", "isbool",
+        "iscell", "iscellstr", "ischar", "iscomplex",
+        "isempty", "isfield", "isfloat", "isglobal",
+        "ishandle", "isieee", "isindex", "isinteger",
+        "islogical", "ismatrix", "ismethod", "isnull",
+        "isnumeric", "isobject", "isreal",
+        "is_rooted_relative_filename", "issorted",
+        "isstruct", "isvarname", "kbhit", "keyboard",
+        "kill", "lasterr", "lasterror", "lastwarn",
+        "ldivide", "le", "length", "link", "linspace",
+        "logical", "lstat", "lt", "make_absolute_filename",
+        "makeinfo_program", "max_recursion_depth", "merge",
+        "methods", "mfilename", "minus", "mislocked",
+        "mkdir", "mkfifo", "mkstemp", "mldivide", "mlock",
+        "mouse_wheel_zoom", "mpower", "mrdivide", "mtimes",
+        "munlock", "nargin", "nargout",
+        "native_float_format", "ndims", "ne", "nfields",
+        "nnz", "norm", "not", "numel", "nzmax",
+        "octave_config_info", "octave_core_file_limit",
+        "octave_core_file_name",
+        "octave_core_file_options", "ones", "or",
+        "output_max_field_width", "output_precision",
+        "page_output_immediately", "page_screen_output",
+        "path", "pathsep", "pause", "pclose", "permute",
+        "pi", "pipe", "plus", "popen", "power",
+        "print_empty_dimensions", "printf",
+        "print_struct_array_contents", "prod",
+        "program_invocation_name", "program_name",
+        "putenv", "puts", "pwd", "quit", "rats", "rdivide",
+        "readdir", "readlink", "read_readline_init_file",
+        "realmax", "realmin", "rehash", "rename",
+        "repelems", "re_read_readline_init_file", "reset",
+        "reshape", "resize", "restoredefaultpath",
+        "rethrow", "rmdir", "rmfield", "rmpath", "rows",
+        "save_header_format_string", "save_precision",
+        "saving_history", "scanf", "set", "setenv",
+        "shell_cmd", "sighup_dumps_octave_core",
+        "sigterm_dumps_octave_core", "silent_functions",
+        "single", "size", "size_equal", "sizemax",
+        "sizeof", "sleep", "source", "sparse_auto_mutate",
+        "split_long_rows", "sprintf", "squeeze", "sscanf",
+        "stat", "stderr", "stdin", "stdout", "strcmp",
+        "strcmpi", "string_fill_char", "strncmp",
+        "strncmpi", "struct", "struct_levels_to_print",
+        "strvcat", "subsasgn", "subsref", "sum", "sumsq",
+        "superiorto", "suppress_verbose_help_message",
+        "symlink", "system", "tic", "tilde_expand",
+        "times", "tmpfile", "tmpnam", "toc", "toupper",
+        "transpose", "true", "typeinfo", "umask", "uminus",
+        "uname", "undo_string_escapes", "unlink", "uplus",
+        "upper", "usage", "usleep", "vec", "vectorize",
+        "vertcat", "waitpid", "warning", "warranty",
+        "whos_line_format", "yes_or_no", "zeros",
+        "inf", "Inf", "nan", "NaN")
+
+    command_kw = ("close", "load", "who", "whos")
+
+    function_kw = (
+        "accumarray", "accumdim", "acosd", "acotd",
+        "acscd", "addtodate", "allchild", "ancestor",
+        "anova", "arch_fit", "arch_rnd", "arch_test",
+        "area", "arma_rnd", "arrayfun", "ascii", "asctime",
+        "asecd", "asind", "assert", "atand",
+        "autoreg_matrix", "autumn", "axes", "axis", "bar",
+        "barh", "bartlett", "bartlett_test", "beep",
+        "betacdf", "betainv", "betapdf", "betarnd",
+        "bicgstab", "bicubic", "binary", "binocdf",
+        "binoinv", "binopdf", "binornd", "bitcmp",
+        "bitget", "bitset", "blackman", "blanks",
+        "blkdiag", "bone", "box", "brighten", "calendar",
+        "cast", "cauchy_cdf", "cauchy_inv", "cauchy_pdf",
+        "cauchy_rnd", "caxis", "celldisp", "center", "cgs",
+        "chisquare_test_homogeneity",
+        "chisquare_test_independence", "circshift", "cla",
+        "clabel", "clf", "clock", "cloglog", "closereq",
+        "colon", "colorbar", "colormap", "colperm",
+        "comet", "common_size", "commutation_matrix",
+        "compan", "compare_versions", "compass",
+        "computer", "cond", "condest", "contour",
+        "contourc", "contourf", "contrast", "conv",
+        "convhull", "cool", "copper", "copyfile", "cor",
+        "corrcoef", "cor_test", "cosd", "cotd", "cov",
+        "cplxpair", "cross", "cscd", "cstrcat", "csvread",
+        "csvwrite", "ctime", "cumtrapz", "curl", "cut",
+        "cylinder", "date", "datenum", "datestr",
+        "datetick", "datevec", "dblquad", "deal",
+        "deblank", "deconv", "delaunay", "delaunayn",
+        "delete", "demo", "detrend", "diffpara", "diffuse",
+        "dir", "discrete_cdf", "discrete_inv",
+        "discrete_pdf", "discrete_rnd", "display",
+        "divergence", "dlmwrite", "dos", "dsearch",
+        "dsearchn", "duplication_matrix", "durbinlevinson",
+        "ellipsoid", "empirical_cdf", "empirical_inv",
+        "empirical_pdf", "empirical_rnd", "eomday",
+        "errorbar", "etime", "etreeplot", "example",
+        "expcdf", "expinv", "expm", "exppdf", "exprnd",
+        "ezcontour", "ezcontourf", "ezmesh", "ezmeshc",
+        "ezplot", "ezpolar", "ezsurf", "ezsurfc", "factor",
+        "factorial", "fail", "fcdf", "feather", "fftconv",
+        "fftfilt", "fftshift", "figure", "fileattrib",
+        "fileparts", "fill", "findall", "findobj",
+        "findstr", "finv", "flag", "flipdim", "fliplr",
+        "flipud", "fpdf", "fplot", "fractdiff", "freqz",
+        "freqz_plot", "frnd", "fsolve",
+        "f_test_regression", "ftp", "fullfile", "fzero",
+        "gamcdf", "gaminv", "gampdf", "gamrnd", "gca",
+        "gcbf", "gcbo", "gcf", "genvarname", "geocdf",
+        "geoinv", "geopdf", "geornd", "getfield", "ginput",
+        "glpk", "gls", "gplot", "gradient",
+        "graphics_toolkit", "gray", "grid", "griddata",
+        "griddatan", "gtext", "gunzip", "gzip", "hadamard",
+        "hamming", "hankel", "hanning", "hggroup",
+        "hidden", "hilb", "hist", "histc", "hold", "hot",
+        "hotelling_test", "housh", "hsv", "hurst",
+        "hygecdf", "hygeinv", "hygepdf", "hygernd",
+        "idivide", "ifftshift", "image", "imagesc",
+        "imfinfo", "imread", "imshow", "imwrite", "index",
+        "info", "inpolygon", "inputname", "interpft",
+        "interpn", "intersect", "invhilb", "iqr", "isa",
+        "isdefinite", "isdir", "is_duplicate_entry",
+        "isequal", "isequalwithequalnans", "isfigure",
+        "ishermitian", "ishghandle", "is_leap_year",
+        "isletter", "ismac", "ismember", "ispc", "isprime",
+        "isprop", "isscalar", "issquare", "isstrprop",
+        "issymmetric", "isunix", "is_valid_file_id",
+        "isvector", "jet", "kendall",
+        "kolmogorov_smirnov_cdf",
+        "kolmogorov_smirnov_test", "kruskal_wallis_test",
+        "krylov", "kurtosis", "laplace_cdf", "laplace_inv",
+        "laplace_pdf", "laplace_rnd", "legend", "legendre",
+        "license", "line", "linkprop", "list_primes",
+        "loadaudio", "loadobj", "logistic_cdf",
+        "logistic_inv", "logistic_pdf", "logistic_rnd",
+        "logit", "loglog", "loglogerr", "logm", "logncdf",
+        "logninv", "lognpdf", "lognrnd", "logspace",
+        "lookfor", "ls_command", "lsqnonneg", "magic",
+        "mahalanobis", "manova", "matlabroot",
+        "mcnemar_test", "mean", "meansq", "median", "menu",
+        "mesh", "meshc", "meshgrid", "meshz", "mexext",
+        "mget", "mkpp", "mode", "moment", "movefile",
+        "mpoles", "mput", "namelengthmax", "nargchk",
+        "nargoutchk", "nbincdf", "nbininv", "nbinpdf",
+        "nbinrnd", "nchoosek", "ndgrid", "newplot", "news",
+        "nonzeros", "normcdf", "normest", "norminv",
+        "normpdf", "normrnd", "now", "nthroot", "null",
+        "ocean", "ols", "onenormest", "optimget",
+        "optimset", "orderfields", "orient", "orth",
+        "pack", "pareto", "parseparams", "pascal", "patch",
+        "pathdef", "pcg", "pchip", "pcolor", "pcr",
+        "peaks", "periodogram", "perl", "perms", "pie",
+        "pink", "planerot", "playaudio", "plot",
+        "plotmatrix", "plotyy", "poisscdf", "poissinv",
+        "poisspdf", "poissrnd", "polar", "poly",
+        "polyaffine", "polyarea", "polyderiv", "polyfit",
+        "polygcd", "polyint", "polyout", "polyreduce",
+        "polyval", "polyvalm", "postpad", "powerset",
+        "ppder", "ppint", "ppjumps", "ppplot", "ppval",
+        "pqpnonneg", "prepad", "primes", "print",
+        "print_usage", "prism", "probit", "qp", "qqplot",
+        "quadcc", "quadgk", "quadl", "quadv", "quiver",
+        "qzhess", "rainbow", "randi", "range", "rank",
+        "ranks", "rat", "reallog", "realpow", "realsqrt",
+        "record", "rectangle_lw", "rectangle_sw",
+        "rectint", "refresh", "refreshdata",
+        "regexptranslate", "repmat", "residue", "ribbon",
+        "rindex", "roots", "rose", "rosser", "rotdim",
+        "rref", "run", "run_count", "rundemos", "run_test",
+        "runtests", "saveas", "saveaudio", "saveobj",
+        "savepath", "scatter", "secd", "semilogx",
+        "semilogxerr", "semilogy", "semilogyerr",
+        "setaudio", "setdiff", "setfield", "setxor",
+        "shading", "shift", "shiftdim", "sign_test",
+        "sinc", "sind", "sinetone", "sinewave", "skewness",
+        "slice", "sombrero", "sortrows", "spaugment",
+        "spconvert", "spdiags", "spearman", "spectral_adf",
+        "spectral_xdf", "specular", "speed", "spencer",
+        "speye", "spfun", "sphere", "spinmap", "spline",
+        "spones", "sprand", "sprandn", "sprandsym",
+        "spring", "spstats", "spy", "sqp", "stairs",
+        "statistics", "std", "stdnormal_cdf",
+        "stdnormal_inv", "stdnormal_pdf", "stdnormal_rnd",
+        "stem", "stft", "strcat", "strchr", "strjust",
+        "strmatch", "strread", "strsplit", "strtok",
+        "strtrim", "strtrunc", "structfun", "studentize",
+        "subplot", "subsindex", "subspace", "substr",
+        "substruct", "summer", "surf", "surface", "surfc",
+        "surfl", "surfnorm", "svds", "swapbytes",
+        "sylvester_matrix", "symvar", "synthesis", "table",
+        "tand", "tar", "tcdf", "tempdir", "tempname",
+        "test", "text", "textread", "textscan", "tinv",
+        "title", "toeplitz", "tpdf", "trace", "trapz",
+        "treelayout", "treeplot", "triangle_lw",
+        "triangle_sw", "tril", "trimesh", "triplequad",
+        "triplot", "trisurf", "triu", "trnd", "tsearchn",
+        "t_test", "t_test_regression", "type", "unidcdf",
+        "unidinv", "unidpdf", "unidrnd", "unifcdf",
+        "unifinv", "unifpdf", "unifrnd", "union", "unique",
+        "unix", "unmkpp", "unpack", "untabify", "untar",
+        "unwrap", "unzip", "u_test", "validatestring",
+        "vander", "var", "var_test", "vech", "ver",
+        "version", "view", "voronoi", "voronoin",
+        "waitforbuttonpress", "wavread", "wavwrite",
+        "wblcdf", "wblinv", "wblpdf", "wblrnd", "weekday",
+        "welch_test", "what", "white", "whitebg",
+        "wienrnd", "wilcoxon_test", "wilkinson", "winter",
+        "xlabel", "xlim", "ylabel", "yulewalker", "zip",
+        "zlabel", "z_test")
+
+    loadable_kw = (
+        "airy", "amd", "balance", "besselh", "besseli",
+        "besselj", "besselk", "bessely", "bitpack",
+        "bsxfun", "builtin", "ccolamd", "cellfun",
+        "cellslices", "chol", "choldelete", "cholinsert",
+        "cholinv", "cholshift", "cholupdate", "colamd",
+        "colloc", "convhulln", "convn", "csymamd",
+        "cummax", "cummin", "daspk", "daspk_options",
+        "dasrt", "dasrt_options", "dassl", "dassl_options",
+        "dbclear", "dbdown", "dbstack", "dbstatus",
+        "dbstop", "dbtype", "dbup", "dbwhere", "det",
+        "dlmread", "dmperm", "dot", "eig", "eigs",
+        "endgrent", "endpwent", "etree", "fft", "fftn",
+        "fftw", "filter", "find", "full", "gcd",
+        "getgrent", "getgrgid", "getgrnam", "getpwent",
+        "getpwnam", "getpwuid", "getrusage", "givens",
+        "gmtime", "gnuplot_binary", "hess", "ifft",
+        "ifftn", "inv", "isdebugmode", "issparse", "kron",
+        "localtime", "lookup", "lsode", "lsode_options",
+        "lu", "luinc", "luupdate", "matrix_type", "max",
+        "min", "mktime", "pinv", "qr", "qrdelete",
+        "qrinsert", "qrshift", "qrupdate", "quad",
+        "quad_options", "qz", "rand", "rande", "randg",
+        "randn", "randp", "randperm", "rcond", "regexp",
+        "regexpi", "regexprep", "schur", "setgrent",
+        "setpwent", "sort", "spalloc", "sparse", "spparms",
+        "sprank", "sqrtm", "strfind", "strftime",
+        "strptime", "strrep", "svd", "svd_driver", "syl",
+        "symamd", "symbfact", "symrcm", "time", "tsearch",
+        "typecast", "urlread", "urlwrite")
+
+    mapping_kw = (
+        "abs", "acos", "acosh", "acot", "acoth", "acsc",
+        "acsch", "angle", "arg", "asec", "asech", "asin",
+        "asinh", "atan", "atanh", "beta", "betainc",
+        "betaln", "bincoeff", "cbrt", "ceil", "conj", "cos",
+        "cosh", "cot", "coth", "csc", "csch", "erf", "erfc",
+        "erfcx", "erfinv", "exp", "finite", "fix", "floor",
+        "fmod", "gamma", "gammainc", "gammaln", "imag",
+        "isalnum", "isalpha", "isascii", "iscntrl",
+        "isdigit", "isfinite", "isgraph", "isinf",
+        "islower", "isna", "isnan", "isprint", "ispunct",
+        "isspace", "isupper", "isxdigit", "lcm", "lgamma",
+        "log", "lower", "mod", "real", "rem", "round",
+        "roundb", "sec", "sech", "sign", "sin", "sinh",
+        "sqrt", "tan", "tanh", "toascii", "tolower", "xor")
+
+    builtin_consts = (
+        "EDITOR", "EXEC_PATH", "I", "IMAGE_PATH", "NA",
+        "OCTAVE_HOME", "OCTAVE_VERSION", "PAGER",
+        "PAGER_FLAGS", "SEEK_CUR", "SEEK_END", "SEEK_SET",
+        "SIG", "S_ISBLK", "S_ISCHR", "S_ISDIR", "S_ISFIFO",
+        "S_ISLNK", "S_ISREG", "S_ISSOCK", "WCONTINUE",
+        "WCOREDUMP", "WEXITSTATUS", "WIFCONTINUED",
+        "WIFEXITED", "WIFSIGNALED", "WIFSTOPPED", "WNOHANG",
+        "WSTOPSIG", "WTERMSIG", "WUNTRACED")
+
+    tokens = {
+        'root': [
+            (r'%\{\s*\n', Comment.Multiline, 'percentblockcomment'),
+            (r'#\{\s*\n', Comment.Multiline, 'hashblockcomment'),
+            (r'[%#].*$', Comment),
+            (r'^\s*function\b', Keyword, 'deffunc'),
+
+            # from 'iskeyword' on hg changeset 8cc154f45e37
+            (words((
+                '__FILE__', '__LINE__', 'break', 'case', 'catch', 'classdef',
+                'continue', 'do', 'else', 'elseif', 'end', 'end_try_catch',
+                'end_unwind_protect', 'endclassdef', 'endevents', 'endfor',
+                'endfunction', 'endif', 'endmethods', 'endproperties', 'endswitch',
+                'endwhile', 'events', 'for', 'function', 'get', 'global', 'if',
+                'methods', 'otherwise', 'persistent', 'properties', 'return',
+                'set', 'static', 'switch', 'try', 'until', 'unwind_protect',
+                'unwind_protect_cleanup', 'while'), suffix=r'\b'),
+             Keyword),
+
+            (words(builtin_kw + command_kw + function_kw + loadable_kw + mapping_kw,
+                   suffix=r'\b'),  Name.Builtin),
+
+            (words(builtin_consts, suffix=r'\b'), Name.Constant),
+
+            # operators in Octave but not Matlab:
+            (r'-=|!=|!|/=|--', Operator),
+            # operators:
+            (r'-|==|~=|<|>|<=|>=|&&|&|~|\|\|?', Operator),
+            # operators in Octave but not Matlab requiring escape for re:
+            (r'\*=|\+=|\^=|\/=|\\=|\*\*|\+\+|\.\*\*', Operator),
+            # operators requiring escape for re:
+            (r'\.\*|\*|\+|\.\^|\^|\.\\|\.\/|\/|\\', Operator),
+
+
+            # punctuation:
+            (r'[\[\](){}:@.,]', Punctuation),
+            (r'=|:|;', Punctuation),
+
+            (r'"[^"]*"', String),
+
+            (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float),
+            (r'\d+[eEf][+-]?[0-9]+', Number.Float),
+            (r'\d+', Number.Integer),
+
+            # quote can be transpose, instead of string:
+            # (not great, but handles common cases...)
+            (r'(?<=[\w)\].])\'+', Operator),
+            (r'(?|<=|>=|&&|&|~|\|\|?', Operator),
+            # operators requiring escape for re:
+            (r'\.\*|\*|\+|\.\^|\^|\.\\|\.\/|\/|\\', Operator),
+
+            # punctuation:
+            (r'[\[\](){}@.,=:;]+', Punctuation),
+
+            (r'"[^"]*"', String),
+
+            # quote can be transpose, instead of string:
+            # (not great, but handles common cases...)
+            (r'(?<=[\w)\].])\'+', Operator),
+            (r'(?', r'<', r'|', r'!', r"'")
+
+    operator_words = ('and', 'or', 'not')
+
+    tokens = {
+        'root': [
+            (r'/\*', Comment.Multiline, 'comment'),
+            (r'"(?:[^"\\]|\\.)*"', String),
+            (r'\(|\)|\[|\]|\{|\}', Punctuation),
+            (r'[,;$]', Punctuation),
+            (words (constants), Name.Constant),
+            (words (keywords), Keyword),
+            (words (operators), Operator),
+            (words (operator_words), Operator.Word),
+            (r'''(?x)
+              ((?:[a-zA-Z_#][\w#]*|`[^`]*`)
+              (?:::[a-zA-Z_#][\w#]*|`[^`]*`)*)(\s*)([(])''',
+             bygroups(Name.Function, Text.Whitespace, Punctuation)),
+            (r'''(?x)
+              (?:[a-zA-Z_#%][\w#%]*|`[^`]*`)
+              (?:::[a-zA-Z_#%][\w#%]*|`[^`]*`)*''', Name.Variable),
+            (r'[-+]?(\d*\.\d+([bdefls][-+]?\d+)?|\d+(\.\d*)?[bdefls][-+]?\d+)', Number.Float),
+            (r'[-+]?\d+', Number.Integer),
+            (r'\s+', Text.Whitespace),
+            (r'.', Text)
+        ],
+        'comment': [
+            (r'[^*/]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ]
+    }
+
+    def analyse_text (text):
+        strength = 0.0
+        # Input expression terminator.
+        if re.search (r'\$\s*$', text, re.MULTILINE):
+            strength += 0.05
+        # Function definition operator.
+        if ':=' in text:
+            strength += 0.02
+        return strength
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/meson.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/meson.py
new file mode 100644
index 0000000000000000000000000000000000000000..154a2de5a154a139e337d04a62d990b694bb5379
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/meson.py
@@ -0,0 +1,139 @@
+"""
+    pygments.lexers.meson
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Pygments lexer for the Meson build system
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, include
+from pygments.token import Comment, Name, Number, Punctuation, Operator, \
+    Keyword, String, Whitespace
+
+__all__ = ['MesonLexer']
+
+
+class MesonLexer(RegexLexer):
+    """Meson language lexer.
+
+    The grammar definition use to transcribe the syntax was retrieved from
+    https://mesonbuild.com/Syntax.html#grammar for version 0.58.
+    Some of those definitions are improperly transcribed, so the Meson++
+    implementation was also checked: https://github.com/dcbaker/meson-plus-plus.
+    """
+
+    # TODO String interpolation @VARNAME@ inner matches
+    # TODO keyword_arg: value inner matches
+
+    name = 'Meson'
+    url = 'https://mesonbuild.com/'
+    aliases = ['meson', 'meson.build']
+    filenames = ['meson.build', 'meson.options', 'meson_options.txt']
+    mimetypes = ['text/x-meson']
+    version_added = '2.10'
+
+    tokens = {
+        'root': [
+            (r'#.*?$', Comment),
+            (r"'''.*'''", String.Single),
+            (r'[1-9][0-9]*', Number.Integer),
+            (r'0o[0-7]+', Number.Oct),
+            (r'0x[a-fA-F0-9]+', Number.Hex),
+            include('string'),
+            include('keywords'),
+            include('expr'),
+            (r'[a-zA-Z_][a-zA-Z_0-9]*', Name),
+            (r'\s+', Whitespace),
+        ],
+        'string': [
+            (r"[']{3}([']{0,2}([^\\']|\\(.|\n)))*[']{3}", String),
+            (r"'.*?(?`_.
+    """
+
+    name = "MCFunction"
+    url = "https://minecraft.wiki/w/Commands"
+    aliases = ["mcfunction", "mcf"]
+    filenames = ["*.mcfunction"]
+    mimetypes = ["text/mcfunction"]
+    version_added = '2.12'
+
+    # Used to denotate the start of a block comment, borrowed from Github's mcfunction
+    _block_comment_prefix = "[>!]"
+
+    tokens = {
+        "root": [
+            include("names"),
+            include("comments"),
+            include("literals"),
+            include("whitespace"),
+            include("property"),
+            include("operators"),
+            include("selectors"),
+        ],
+
+        "names": [
+            # The start of a command (either beginning of line OR after the run keyword)
+            #  We don't encode a list of keywords since mods, plugins, or even pre-processors
+            #  may add new commands, so we have a 'close-enough' regex which catches them.
+            (r"^(\s*)([a-z_]+)", bygroups(Whitespace, Name.Builtin)),
+            (r"(?<=run)\s+[a-z_]+", Name.Builtin),
+
+            # UUID
+            (r"\b[0-9a-fA-F]+(?:-[0-9a-fA-F]+){4}\b", Name.Variable),
+            include("resource-name"),
+            # normal command names and scoreboards
+            #  there's no way to know the differences unfortuntely
+            (r"[A-Za-z_][\w.#%$]+", Keyword.Constant),
+            (r"[#%$][\w.#%$]+", Name.Variable.Magic),
+        ],
+
+        "resource-name": [
+            # resource names have to be lowercase
+            (r"#?[a-z_][a-z_.-]*:[a-z0-9_./-]+", Name.Function),
+            # similar to above except optional `:``
+            #  a `/` must be present "somewhere"
+            (r"#?[a-z0-9_\.\-]+\/[a-z0-9_\.\-\/]+", Name.Function),
+        ],
+
+        "whitespace": [
+            (r"\s+", Whitespace),
+        ],
+
+        "comments": [
+            (rf"^\s*(#{_block_comment_prefix})", Comment.Multiline,
+             ("comments.block", "comments.block.emphasized")),
+            (r"#.*$", Comment.Single),
+        ],
+        "comments.block": [
+            (rf"^\s*#{_block_comment_prefix}", Comment.Multiline,
+             "comments.block.emphasized"),
+            (r"^\s*#", Comment.Multiline, "comments.block.normal"),
+            default("#pop"),
+        ],
+        "comments.block.normal": [
+            include("comments.block.special"),
+            (r"\S+", Comment.Multiline),
+            (r"\n", Text, "#pop"),
+            include("whitespace"),
+        ],
+        "comments.block.emphasized": [
+            include("comments.block.special"),
+            (r"\S+", String.Doc),
+            (r"\n", Text, "#pop"),
+            include("whitespace"),
+        ],
+        "comments.block.special": [
+            # Params
+            (r"@\S+", Name.Decorator),
+
+            include("resource-name"),
+
+            # Scoreboard player names
+            (r"[#%$][\w.#%$]+", Name.Variable.Magic),
+        ],
+
+        "operators": [
+            (r"[\-~%^?!+*<>\\/|&=.]", Operator),
+        ],
+
+        "literals": [
+            (r"\.\.", Literal),
+            (r"(true|false)", Keyword.Pseudo),
+
+            # these are like unquoted strings and appear in many places
+            (r"[A-Za-z_]+", Name.Variable.Class),
+
+            (r"[0-7]b", Number.Byte),
+            (r"[+-]?\d*\.?\d+([eE]?[+-]?\d+)?[df]?\b", Number.Float),
+            (r"[+-]?\d+\b", Number.Integer),
+            (r'"', String.Double, "literals.string-double"),
+            (r"'", String.Single, "literals.string-single"),
+        ],
+        "literals.string-double": [
+            (r"\\.", String.Escape),
+            (r'[^\\"\n]+', String.Double),
+            (r'"', String.Double, "#pop"),
+        ],
+        "literals.string-single": [
+            (r"\\.", String.Escape),
+            (r"[^\\'\n]+", String.Single),
+            (r"'", String.Single, "#pop"),
+        ],
+
+        "selectors": [
+            (r"@[a-z]", Name.Variable),
+        ],
+
+
+        ## Generic Property Container
+        # There are several, differing instances where the language accepts
+        #  specific contained keys or contained key, value pairings.
+        #
+        # Property Maps:
+        # - Starts with either `[` or `{`
+        # - Key separated by `:` or `=`
+        # - Deliminated by `,`
+        #
+        # Property Lists:
+        # - Starts with `[`
+        # - Deliminated by `,`
+        #
+        # For simplicity, these patterns match a generic, nestable structure
+        #  which follow a key, value pattern. For normal lists, there's only keys.
+        # This allow some "illegal" structures, but we'll accept those for
+        #  sake of simplicity
+        #
+        # Examples:
+        # - `[facing=up, powered=true]` (blockstate)
+        # - `[name="hello world", nbt={key: 1b}]` (selector + nbt)
+        # - `[{"text": "value"}, "literal"]` (json)
+        ##
+        "property": [
+            # This state gets included in root and also several substates
+            # We do this to shortcut the starting of new properties
+            #  within other properties. Lists can have sublists and compounds
+            #  and values can start a new property (see the `difficult_1.txt`
+            #  snippet).
+            (r"\{", Punctuation, ("property.curly", "property.key")),
+            (r"\[", Punctuation, ("property.square", "property.key")),
+        ],
+        "property.curly": [
+            include("whitespace"),
+            include("property"),
+            (r"\}", Punctuation, "#pop"),
+        ],
+        "property.square": [
+            include("whitespace"),
+            include("property"),
+            (r"\]", Punctuation, "#pop"),
+
+            # lists can have sequences of items
+            (r",", Punctuation),
+        ],
+        "property.key": [
+            include("whitespace"),
+
+            # resource names (for advancements)
+            #  can omit `:` to default `minecraft:`
+            # must check if there is a future equals sign if `:` is in the name
+            (r"#?[a-z_][a-z_\.\-]*\:[a-z0-9_\.\-/]+(?=\s*\=)", Name.Attribute, "property.delimiter"),
+            (r"#?[a-z_][a-z0-9_\.\-/]+", Name.Attribute, "property.delimiter"),
+
+            # unquoted NBT key
+            (r"[A-Za-z_\-\+]+", Name.Attribute, "property.delimiter"),
+
+            # quoted JSON or NBT key
+            (r'"', Name.Attribute, "property.delimiter", "literals.string-double"),
+            (r"'", Name.Attribute, "property.delimiter", "literals.string-single"),
+
+            # index for a list
+            (r"-?\d+", Number.Integer, "property.delimiter"),
+
+            default("#pop"),
+        ],
+        "property.key.string-double": [
+            (r"\\.", String.Escape),
+            (r'[^\\"\n]+', Name.Attribute),
+            (r'"', Name.Attribute, "#pop"),
+        ],
+        "property.key.string-single": [
+            (r"\\.", String.Escape),
+            (r"[^\\'\n]+", Name.Attribute),
+            (r"'", Name.Attribute, "#pop"),
+        ],
+        "property.delimiter": [
+            include("whitespace"),
+
+            (r"[:=]!?", Punctuation, "property.value"),
+            (r",", Punctuation),
+
+            default("#pop"),
+        ],
+        "property.value": [
+            include("whitespace"),
+
+            # unquoted resource names are valid literals here
+            (r"#?[a-z_][a-z_\.\-]*\:[a-z0-9_\.\-/]+", Name.Tag),
+            (r"#?[a-z_][a-z0-9_\.\-/]+", Name.Tag),
+
+            include("literals"),
+            include("property"),
+
+            default("#pop"),
+        ],
+    }
+
+
+class MCSchemaLexer(RegexLexer):
+    """Lexer for Minecraft Add-ons data Schemas, an interface structure standard used in Minecraft
+    """
+
+    name = 'MCSchema'
+    url = 'https://learn.microsoft.com/en-us/minecraft/creator/reference/content/schemasreference/'
+    aliases = ['mcschema']
+    filenames = ['*.mcschema']
+    mimetypes = ['text/mcschema']
+    version_added = '2.14'
+
+    tokens = {
+        'commentsandwhitespace': [
+            (r'\s+', Whitespace),
+            (r'//.*?$', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline)
+        ],
+        'slashstartsregex': [
+            include('commentsandwhitespace'),
+            (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+             r'([gimuysd]+\b|\B)', String.Regex, '#pop'),
+            (r'(?=/)', Text, ('#pop', 'badregex')),
+            default('#pop')
+        ],
+        'badregex': [
+            (r'\n', Whitespace, '#pop')
+        ],
+        'singlestring': [
+            (r'\\.', String.Escape),
+            (r"'", String.Single, '#pop'),
+            (r"[^\\']+", String.Single),
+        ],
+        'doublestring': [
+            (r'\\.', String.Escape),
+            (r'"', String.Double, '#pop'),
+            (r'[^\\"]+', String.Double),
+        ],
+        'root': [
+            (r'^(?=\s|/|', Comment, '#pop'),
+            (r'[^\-]+|-', Comment),
+        ],
+    }
+
+
+class ReasonLexer(RegexLexer):
+    """
+    For the ReasonML language.
+    """
+
+    name = 'ReasonML'
+    url = 'https://reasonml.github.io/'
+    aliases = ['reasonml', 'reason']
+    filenames = ['*.re', '*.rei']
+    mimetypes = ['text/x-reasonml']
+    version_added = '2.6'
+
+    keywords = (
+        'as', 'assert', 'begin', 'class', 'constraint', 'do', 'done', 'downto',
+        'else', 'end', 'exception', 'external', 'false', 'for', 'fun', 'esfun',
+        'function', 'functor', 'if', 'in', 'include', 'inherit', 'initializer', 'lazy',
+        'let', 'switch', 'module', 'pub', 'mutable', 'new', 'nonrec', 'object', 'of',
+        'open', 'pri', 'rec', 'sig', 'struct', 'then', 'to', 'true', 'try',
+        'type', 'val', 'virtual', 'when', 'while', 'with',
+    )
+    keyopts = (
+        '!=', '#', '&', '&&', r'\(', r'\)', r'\*', r'\+', ',', '-',
+        r'-\.', '=>', r'\.', r'\.\.', r'\.\.\.', ':', '::', ':=', ':>', ';', ';;', '<',
+        '<-', '=', '>', '>]', r'>\}', r'\?', r'\?\?', r'\[', r'\[<', r'\[>',
+        r'\[\|', ']', '_', '`', r'\{', r'\{<', r'\|', r'\|\|', r'\|]', r'\}', '~'
+    )
+
+    operators = r'[!$%&*+\./:<=>?@^|~-]'
+    word_operators = ('and', 'asr', 'land', 'lor', 'lsl', 'lsr', 'lxor', 'mod', 'or')
+    prefix_syms = r'[!?~]'
+    infix_syms = r'[=<>@^|&+\*/$%-]'
+    primitives = ('unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array')
+
+    tokens = {
+        'escape-sequence': [
+            (r'\\[\\"\'ntbr]', String.Escape),
+            (r'\\[0-9]{3}', String.Escape),
+            (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        ],
+        'root': [
+            (r'\s+', Text),
+            (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo),
+            (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'),
+            (r'\b([A-Z][\w\']*)', Name.Class),
+            (r'//.*?\n', Comment.Single),
+            (r'\/\*(?!/)', Comment.Multiline, 'comment'),
+            (r'\b({})\b'.format('|'.join(keywords)), Keyword),
+            (r'({})'.format('|'.join(keyopts[::-1])), Operator.Word),
+            (rf'({infix_syms}|{prefix_syms})?{operators}', Operator),
+            (r'\b({})\b'.format('|'.join(word_operators)), Operator.Word),
+            (r'\b({})\b'.format('|'.join(primitives)), Keyword.Type),
+
+            (r"[^\W\d][\w']*", Name),
+
+            (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
+            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+            (r'0[oO][0-7][0-7_]*', Number.Oct),
+            (r'0[bB][01][01_]*', Number.Bin),
+            (r'\d[\d_]*', Number.Integer),
+
+            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+             String.Char),
+            (r"'.'", String.Char),
+            (r"'", Keyword),
+
+            (r'"', String.Double, 'string'),
+
+            (r'[~?][a-z][\w\']*:', Name.Variable),
+        ],
+        'comment': [
+            (r'[^/*]+', Comment.Multiline),
+            (r'\/\*', Comment.Multiline, '#push'),
+            (r'\*\/', Comment.Multiline, '#pop'),
+            (r'\*', Comment.Multiline),
+        ],
+        'string': [
+            (r'[^\\"]+', String.Double),
+            include('escape-sequence'),
+            (r'\\\n', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'dotted': [
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
+            (r'[A-Z][\w\']*', Name.Class, '#pop'),
+            (r'[a-z_][\w\']*', Name, '#pop'),
+            default('#pop'),
+        ],
+    }
+
+
+class FStarLexer(RegexLexer):
+    """
+    For the F* language.
+    """
+
+    name = 'FStar'
+    url = 'https://www.fstar-lang.org/'
+    aliases = ['fstar']
+    filenames = ['*.fst', '*.fsti']
+    mimetypes = ['text/x-fstar']
+    version_added = '2.7'
+
+    keywords = (
+        'abstract', 'attributes', 'noeq', 'unopteq', 'and'
+        'begin', 'by', 'default', 'effect', 'else', 'end', 'ensures',
+        'exception', 'exists', 'false', 'forall', 'fun', 'function', 'if',
+        'in', 'include', 'inline', 'inline_for_extraction', 'irreducible',
+        'logic', 'match', 'module', 'mutable', 'new', 'new_effect', 'noextract',
+        'of', 'open', 'opaque', 'private', 'range_of', 'reifiable',
+        'reify', 'reflectable', 'requires', 'set_range_of', 'sub_effect',
+        'synth', 'then', 'total', 'true', 'try', 'type', 'unfold', 'unfoldable',
+        'val', 'when', 'with', 'not'
+    )
+    decl_keywords = ('let', 'rec')
+    assume_keywords = ('assume', 'admit', 'assert', 'calc')
+    keyopts = (
+        r'~', r'-', r'/\\', r'\\/', r'<:', r'<@', r'\(\|', r'\|\)', r'#', r'u#',
+        r'&', r'\(', r'\)', r'\(\)', r',', r'~>', r'->', r'<-', r'<--', r'<==>',
+        r'==>', r'\.', r'\?', r'\?\.', r'\.\[', r'\.\(', r'\.\(\|', r'\.\[\|',
+        r'\{:pattern', r':', r'::', r':=', r';', r';;', r'=', r'%\[', r'!\{',
+        r'\[', r'\[@', r'\[\|', r'\|>', r'\]', r'\|\]', r'\{', r'\|', r'\}', r'\$'
+    )
+
+    operators = r'[!$%&*+\./:<=>?@^|~-]'
+    prefix_syms = r'[!?~]'
+    infix_syms = r'[=<>@^|&+\*/$%-]'
+    primitives = ('unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array')
+
+    tokens = {
+        'escape-sequence': [
+            (r'\\[\\"\'ntbr]', String.Escape),
+            (r'\\[0-9]{3}', String.Escape),
+            (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        ],
+        'root': [
+            (r'\s+', Text),
+            (r'false|true|False|True|\(\)|\[\]', Name.Builtin.Pseudo),
+            (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'),
+            (r'\b([A-Z][\w\']*)', Name.Class),
+            (r'\(\*(?![)])', Comment, 'comment'),
+            (r'\/\/.+$', Comment),
+            (r'\b({})\b'.format('|'.join(keywords)), Keyword),
+            (r'\b({})\b'.format('|'.join(assume_keywords)), Name.Exception),
+            (r'\b({})\b'.format('|'.join(decl_keywords)), Keyword.Declaration),
+            (r'({})'.format('|'.join(keyopts[::-1])), Operator),
+            (rf'({infix_syms}|{prefix_syms})?{operators}', Operator),
+            (r'\b({})\b'.format('|'.join(primitives)), Keyword.Type),
+
+            (r"[^\W\d][\w']*", Name),
+
+            (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
+            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+            (r'0[oO][0-7][0-7_]*', Number.Oct),
+            (r'0[bB][01][01_]*', Number.Bin),
+            (r'\d[\d_]*', Number.Integer),
+
+            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+             String.Char),
+            (r"'.'", String.Char),
+            (r"'", Keyword),  # a stray quote is another syntax element
+            (r"\`([\w\'.]+)\`", Operator.Word),  # for infix applications
+            (r"\`", Keyword),  # for quoting
+            (r'"', String.Double, 'string'),
+
+            (r'[~?][a-z][\w\']*:', Name.Variable),
+        ],
+        'comment': [
+            (r'[^(*)]+', Comment),
+            (r'\(\*', Comment, '#push'),
+            (r'\*\)', Comment, '#pop'),
+            (r'[(*)]', Comment),
+        ],
+        'string': [
+            (r'[^\\"]+', String.Double),
+            include('escape-sequence'),
+            (r'\\\n', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'dotted': [
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
+            (r'[A-Z][\w\']*', Name.Class, '#pop'),
+            (r'[a-z_][\w\']*', Name, '#pop'),
+            default('#pop'),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/modeling.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/modeling.py
new file mode 100644
index 0000000000000000000000000000000000000000..4c22aeb0043a31b6231fb031134b12cafe1024fc
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/modeling.py
@@ -0,0 +1,368 @@
+"""
+    pygments.lexers.modeling
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for modeling languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+
+from pygments.lexers.html import HtmlLexer
+from pygments.lexers import _stan_builtins
+
+__all__ = ['ModelicaLexer', 'BugsLexer', 'JagsLexer', 'StanLexer']
+
+
+class ModelicaLexer(RegexLexer):
+    """
+    For Modelica source code.
+    """
+    name = 'Modelica'
+    url = 'http://www.modelica.org/'
+    aliases = ['modelica']
+    filenames = ['*.mo']
+    mimetypes = ['text/x-modelica']
+    version_added = '1.1'
+
+    flags = re.DOTALL | re.MULTILINE
+
+    _name = r"(?:'(?:[^\\']|\\.)+'|[a-zA-Z_]\w*)"
+
+    tokens = {
+        'whitespace': [
+            (r'[\s\ufeff]+', Text),
+            (r'//[^\n]*\n?', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline)
+        ],
+        'root': [
+            include('whitespace'),
+            (r'"', String.Double, 'string'),
+            (r'[()\[\]{},;]+', Punctuation),
+            (r'\.?[*^/+-]|\.|<>|[<>:=]=?', Operator),
+            (r'\d+(\.?\d*[eE][-+]?\d+|\.\d*)', Number.Float),
+            (r'\d+', Number.Integer),
+            (r'(abs|acos|actualStream|array|asin|assert|AssertionLevel|atan|'
+             r'atan2|backSample|Boolean|cardinality|cat|ceil|change|Clock|'
+             r'Connections|cos|cosh|cross|delay|diagonal|div|edge|exp|'
+             r'ExternalObject|fill|floor|getInstanceName|hold|homotopy|'
+             r'identity|inStream|integer|Integer|interval|inverse|isPresent|'
+             r'linspace|log|log10|matrix|max|min|mod|ndims|noClock|noEvent|'
+             r'ones|outerProduct|pre|previous|product|Real|reinit|rem|rooted|'
+             r'sample|scalar|semiLinear|shiftSample|sign|sin|sinh|size|skew|'
+             r'smooth|spatialDistribution|sqrt|StateSelect|String|subSample|'
+             r'sum|superSample|symmetric|tan|tanh|terminal|terminate|time|'
+             r'transpose|vector|zeros)\b', Name.Builtin),
+            (r'(algorithm|annotation|break|connect|constant|constrainedby|der|'
+             r'discrete|each|else|elseif|elsewhen|encapsulated|enumeration|'
+             r'equation|exit|expandable|extends|external|firstTick|final|flow|for|if|'
+             r'import|impure|in|initial|inner|input|interval|loop|nondiscrete|outer|'
+             r'output|parameter|partial|protected|public|pure|redeclare|'
+             r'replaceable|return|stream|then|when|while)\b',
+             Keyword.Reserved),
+            (r'(and|not|or)\b', Operator.Word),
+            (r'(block|class|connector|end|function|model|operator|package|'
+             r'record|type)\b', Keyword.Reserved, 'class'),
+            (r'(false|true)\b', Keyword.Constant),
+            (r'within\b', Keyword.Reserved, 'package-prefix'),
+            (_name, Name)
+        ],
+        'class': [
+            include('whitespace'),
+            (r'(function|record)\b', Keyword.Reserved),
+            (r'(if|for|when|while)\b', Keyword.Reserved, '#pop'),
+            (_name, Name.Class, '#pop'),
+            default('#pop')
+        ],
+        'package-prefix': [
+            include('whitespace'),
+            (_name, Name.Namespace, '#pop'),
+            default('#pop')
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (r'\\[\'"?\\abfnrtv]', String.Escape),
+            (r'(?i)<\s*html\s*>([^\\"]|\\.)+?(<\s*/\s*html\s*>|(?="))',
+             using(HtmlLexer)),
+            (r'<|\\?[^"\\<]+', String.Double)
+        ]
+    }
+
+
+class BugsLexer(RegexLexer):
+    """
+    Pygments Lexer for OpenBugs and WinBugs
+    models.
+    """
+
+    name = 'BUGS'
+    aliases = ['bugs', 'winbugs', 'openbugs']
+    filenames = ['*.bug']
+    url = 'https://www.mrc-bsu.cam.ac.uk/software/bugs/openbugs'
+    version_added = '1.6'
+
+    _FUNCTIONS = (
+        # Scalar functions
+        'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh',
+        'cloglog', 'cos', 'cosh', 'cumulative', 'cut', 'density', 'deviance',
+        'equals', 'expr', 'gammap', 'ilogit', 'icloglog', 'integral', 'log',
+        'logfact', 'loggam', 'logit', 'max', 'min', 'phi', 'post.p.value',
+        'pow', 'prior.p.value', 'probit', 'replicate.post', 'replicate.prior',
+        'round', 'sin', 'sinh', 'solution', 'sqrt', 'step', 'tan', 'tanh',
+        'trunc',
+        # Vector functions
+        'inprod', 'interp.lin', 'inverse', 'logdet', 'mean', 'eigen.vals',
+        'ode', 'prod', 'p.valueM', 'rank', 'ranked', 'replicate.postM',
+        'sd', 'sort', 'sum',
+        # Special
+        'D', 'I', 'F', 'T', 'C')
+    """ OpenBUGS built-in functions
+
+    From http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAII
+
+    This also includes
+
+    - T, C, I : Truncation and censoring.
+      ``T`` and ``C`` are in OpenBUGS. ``I`` in WinBUGS.
+    - D : ODE
+    - F : Functional http://www.openbugs.info/Examples/Functionals.html
+
+    """
+
+    _DISTRIBUTIONS = ('dbern', 'dbin', 'dcat', 'dnegbin', 'dpois',
+                      'dhyper', 'dbeta', 'dchisqr', 'ddexp', 'dexp',
+                      'dflat', 'dgamma', 'dgev', 'df', 'dggamma', 'dgpar',
+                      'dloglik', 'dlnorm', 'dlogis', 'dnorm', 'dpar',
+                      'dt', 'dunif', 'dweib', 'dmulti', 'ddirch', 'dmnorm',
+                      'dmt', 'dwish')
+    """ OpenBUGS built-in distributions
+
+    Functions from
+    http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAI
+    """
+
+    tokens = {
+        'whitespace': [
+            (r"\s+", Text),
+        ],
+        'comments': [
+            # Comments
+            (r'#.*$', Comment.Single),
+        ],
+        'root': [
+            # Comments
+            include('comments'),
+            include('whitespace'),
+            # Block start
+            (r'(model)(\s+)(\{)',
+             bygroups(Keyword.Namespace, Text, Punctuation)),
+            # Reserved Words
+            (r'(for|in)(?![\w.])', Keyword.Reserved),
+            # Built-in Functions
+            (r'({})(?=\s*\()'.format(r'|'.join(_FUNCTIONS + _DISTRIBUTIONS)),
+             Name.Builtin),
+            # Regular variable names
+            (r'[A-Za-z][\w.]*', Name),
+            # Number Literals
+            (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number),
+            # Punctuation
+            (r'\[|\]|\(|\)|:|,|;', Punctuation),
+            # Assignment operators
+            # SLexer makes these tokens Operators.
+            (r'<-|~', Operator),
+            # Infix and prefix operators
+            (r'\+|-|\*|/', Operator),
+            # Block
+            (r'[{}]', Punctuation),
+        ]
+    }
+
+    def analyse_text(text):
+        if re.search(r"^\s*model\s*{", text, re.M):
+            return 0.7
+        else:
+            return 0.0
+
+
+class JagsLexer(RegexLexer):
+    """
+    Pygments Lexer for JAGS.
+    """
+
+    name = 'JAGS'
+    aliases = ['jags']
+    filenames = ['*.jag', '*.bug']
+    url = 'https://mcmc-jags.sourceforge.io'
+    version_added = '1.6'
+
+    # JAGS
+    _FUNCTIONS = (
+        'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh',
+        'cos', 'cosh', 'cloglog',
+        'equals', 'exp', 'icloglog', 'ifelse', 'ilogit', 'log', 'logfact',
+        'loggam', 'logit', 'phi', 'pow', 'probit', 'round', 'sin', 'sinh',
+        'sqrt', 'step', 'tan', 'tanh', 'trunc', 'inprod', 'interp.lin',
+        'logdet', 'max', 'mean', 'min', 'prod', 'sum', 'sd', 'inverse',
+        'rank', 'sort', 't', 'acos', 'acosh', 'asin', 'asinh', 'atan',
+        # Truncation/Censoring (should I include)
+        'T', 'I')
+    # Distributions with density, probability and quartile functions
+    _DISTRIBUTIONS = tuple(f'[dpq]{x}' for x in
+                           ('bern', 'beta', 'dchiqsqr', 'ddexp', 'dexp',
+                            'df', 'gamma', 'gen.gamma', 'logis', 'lnorm',
+                            'negbin', 'nchisqr', 'norm', 'par', 'pois', 'weib'))
+    # Other distributions without density and probability
+    _OTHER_DISTRIBUTIONS = (
+        'dt', 'dunif', 'dbetabin', 'dbern', 'dbin', 'dcat', 'dhyper',
+        'ddirch', 'dmnorm', 'dwish', 'dmt', 'dmulti', 'dbinom', 'dchisq',
+        'dnbinom', 'dweibull', 'ddirich')
+
+    tokens = {
+        'whitespace': [
+            (r"\s+", Text),
+        ],
+        'names': [
+            # Regular variable names
+            (r'[a-zA-Z][\w.]*\b', Name),
+        ],
+        'comments': [
+            # do not use stateful comments
+            (r'(?s)/\*.*?\*/', Comment.Multiline),
+            # Comments
+            (r'#.*$', Comment.Single),
+        ],
+        'root': [
+            # Comments
+            include('comments'),
+            include('whitespace'),
+            # Block start
+            (r'(model|data)(\s+)(\{)',
+             bygroups(Keyword.Namespace, Text, Punctuation)),
+            (r'var(?![\w.])', Keyword.Declaration),
+            # Reserved Words
+            (r'(for|in)(?![\w.])', Keyword.Reserved),
+            # Builtins
+            # Need to use lookahead because . is a valid char
+            (r'({})(?=\s*\()'.format(r'|'.join(_FUNCTIONS
+                                          + _DISTRIBUTIONS
+                                          + _OTHER_DISTRIBUTIONS)),
+             Name.Builtin),
+            # Names
+            include('names'),
+            # Number Literals
+            (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number),
+            (r'\[|\]|\(|\)|:|,|;', Punctuation),
+            # Assignment operators
+            (r'<-|~', Operator),
+            # # JAGS includes many more than OpenBUGS
+            (r'\+|-|\*|\/|\|\|[&]{2}|[<>=]=?|\^|%.*?%', Operator),
+            (r'[{}]', Punctuation),
+        ]
+    }
+
+    def analyse_text(text):
+        if re.search(r'^\s*model\s*\{', text, re.M):
+            if re.search(r'^\s*data\s*\{', text, re.M):
+                return 0.9
+            elif re.search(r'^\s*var', text, re.M):
+                return 0.9
+            else:
+                return 0.3
+        else:
+            return 0
+
+
+class StanLexer(RegexLexer):
+    """Pygments Lexer for Stan models.
+
+    The Stan modeling language is specified in the *Stan Modeling Language
+    User's Guide and Reference Manual, v2.17.0*,
+    `pdf `__.
+    """
+
+    name = 'Stan'
+    aliases = ['stan']
+    filenames = ['*.stan']
+    url = 'https://mc-stan.org'
+    version_added = '1.6'
+
+    tokens = {
+        'whitespace': [
+            (r"\s+", Text),
+        ],
+        'comments': [
+            (r'(?s)/\*.*?\*/', Comment.Multiline),
+            # Comments
+            (r'(//|#).*$', Comment.Single),
+        ],
+        'root': [
+            (r'"[^"]*"', String),
+            # Comments
+            include('comments'),
+            # block start
+            include('whitespace'),
+            # Block start
+            (r'({})(\s*)(\{{)'.format(r'|'.join(('functions', 'data', r'transformed\s+?data',
+                        'parameters', r'transformed\s+parameters',
+                        'model', r'generated\s+quantities'))),
+             bygroups(Keyword.Namespace, Text, Punctuation)),
+            # target keyword
+            (r'target\s*\+=', Keyword),
+            # jacobian += statement
+            (r'jacobian\s*\+=', Keyword),
+            # Reserved Words
+            (r'({})\b'.format(r'|'.join(_stan_builtins.KEYWORDS)), Keyword),
+            # Truncation
+            (r'T(?=\s*\[)', Keyword),
+            # Data types
+            (r'({})\b'.format(r'|'.join(_stan_builtins.TYPES)), Keyword.Type),
+             # < should be punctuation, but elsewhere I can't tell if it is in
+             # a range constraint
+            (r'(<)(\s*)(upper|lower|offset|multiplier)(\s*)(=)',
+             bygroups(Operator, Whitespace, Keyword, Whitespace, Punctuation)),
+            (r'(,)(\s*)(upper)(\s*)(=)',
+             bygroups(Punctuation, Whitespace, Keyword, Whitespace, Punctuation)),
+            # Punctuation
+            (r"[;,\[\]()]", Punctuation),
+            # Builtin
+            (r'({})(?=\s*\()'.format('|'.join(_stan_builtins.FUNCTIONS)), Name.Builtin),
+            (r'(~)(\s*)({})(?=\s*\()'.format('|'.join(_stan_builtins.DISTRIBUTIONS)),
+                bygroups(Operator, Whitespace, Name.Builtin)),
+            # Special names ending in __, like lp__
+            (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo),
+            (r'({})\b'.format(r'|'.join(_stan_builtins.RESERVED)), Keyword.Reserved),
+            # user-defined functions
+            (r'[A-Za-z]\w*(?=\s*\()]', Name.Function),
+            # Imaginary Literals
+            (r'[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?i', Number.Float),
+            (r'\.[0-9]+([eE][+-]?[0-9]+)?i', Number.Float),
+            (r'[0-9]+i', Number.Float),
+            # Real Literals
+            (r'[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?', Number.Float),
+            (r'\.[0-9]+([eE][+-]?[0-9]+)?', Number.Float),
+            # Integer Literals
+            (r'[0-9]+', Number.Integer),
+            # Regular variable names
+            (r'[A-Za-z]\w*\b', Name),
+            # Assignment operators
+            (r'<-|(?:\+|-|\.?/|\.?\*|=)?=|~', Operator),
+            # Infix, prefix and postfix operators (and = )
+            (r"\+|-|\.?\*|\.?/|\\|'|\.?\^|!=?|<=?|>=?|\|\||&&|%|\?|:|%/%|!", Operator),
+            # Block delimiters
+            (r'[{}]', Punctuation),
+            # Distribution |
+            (r'\|', Punctuation)
+        ]
+    }
+
+    def analyse_text(text):
+        if re.search(r'^\s*parameters\s*\{', text, re.M):
+            return 1.0
+        else:
+            return 0.0
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/modula2.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/modula2.py
new file mode 100644
index 0000000000000000000000000000000000000000..c0ebd5c14c33767f8374de499baa938b2c943425
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/modula2.py
@@ -0,0 +1,1579 @@
+"""
+    pygments.lexers.modula2
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Multi-Dialect Lexer for Modula-2.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include
+from pygments.util import get_bool_opt, get_list_opt
+from pygments.token import Text, Comment, Operator, Keyword, Name, \
+    String, Number, Punctuation, Error
+
+__all__ = ['Modula2Lexer']
+
+
+# Multi-Dialect Modula-2 Lexer
+class Modula2Lexer(RegexLexer):
+    """
+    For Modula-2 source code.
+
+    The Modula-2 lexer supports several dialects.  By default, it operates in
+    fallback mode, recognising the *combined* literals, punctuation symbols
+    and operators of all supported dialects, and the *combined* reserved words
+    and builtins of PIM Modula-2, ISO Modula-2 and Modula-2 R10, while not
+    differentiating between library defined identifiers.
+
+    To select a specific dialect, a dialect option may be passed
+    or a dialect tag may be embedded into a source file.
+
+    Dialect Options:
+
+    `m2pim`
+        Select PIM Modula-2 dialect.
+    `m2iso`
+        Select ISO Modula-2 dialect.
+    `m2r10`
+        Select Modula-2 R10 dialect.
+    `objm2`
+        Select Objective Modula-2 dialect.
+
+    The PIM and ISO dialect options may be qualified with a language extension.
+
+    Language Extensions:
+
+    `+aglet`
+        Select Aglet Modula-2 extensions, available with m2iso.
+    `+gm2`
+        Select GNU Modula-2 extensions, available with m2pim.
+    `+p1`
+        Select p1 Modula-2 extensions, available with m2iso.
+    `+xds`
+        Select XDS Modula-2 extensions, available with m2iso.
+
+
+    Passing a Dialect Option via Unix Commandline Interface
+
+    Dialect options may be passed to the lexer using the `dialect` key.
+    Only one such option should be passed. If multiple dialect options are
+    passed, the first valid option is used, any subsequent options are ignored.
+
+    Examples:
+
+    `$ pygmentize -O full,dialect=m2iso -f html -o /path/to/output /path/to/input`
+        Use ISO dialect to render input to HTML output
+    `$ pygmentize -O full,dialect=m2iso+p1 -f rtf -o /path/to/output /path/to/input`
+        Use ISO dialect with p1 extensions to render input to RTF output
+
+
+    Embedding a Dialect Option within a source file
+
+    A dialect option may be embedded in a source file in form of a dialect
+    tag, a specially formatted comment that specifies a dialect option.
+
+    Dialect Tag EBNF::
+
+       dialectTag :
+           OpeningCommentDelim Prefix dialectOption ClosingCommentDelim ;
+
+       dialectOption :
+           'm2pim' | 'm2iso' | 'm2r10' | 'objm2' |
+           'm2iso+aglet' | 'm2pim+gm2' | 'm2iso+p1' | 'm2iso+xds' ;
+
+       Prefix : '!' ;
+
+       OpeningCommentDelim : '(*' ;
+
+       ClosingCommentDelim : '*)' ;
+
+    No whitespace is permitted between the tokens of a dialect tag.
+
+    In the event that a source file contains multiple dialect tags, the first
+    tag that contains a valid dialect option will be used and any subsequent
+    dialect tags will be ignored.  Ideally, a dialect tag should be placed
+    at the beginning of a source file.
+
+    An embedded dialect tag overrides a dialect option set via command line.
+
+    Examples:
+
+    ``(*!m2r10*) DEFINITION MODULE Foobar; ...``
+        Use Modula2 R10 dialect to render this source file.
+    ``(*!m2pim+gm2*) DEFINITION MODULE Bazbam; ...``
+        Use PIM dialect with GNU extensions to render this source file.
+
+
+    Algol Publication Mode:
+
+    In Algol publication mode, source text is rendered for publication of
+    algorithms in scientific papers and academic texts, following the format
+    of the Revised Algol-60 Language Report.  It is activated by passing
+    one of two corresponding styles as an option:
+
+    `algol`
+        render reserved words lowercase underline boldface
+        and builtins lowercase boldface italic
+    `algol_nu`
+        render reserved words lowercase boldface (no underlining)
+        and builtins lowercase boldface italic
+
+    The lexer automatically performs the required lowercase conversion when
+    this mode is activated.
+
+    Example:
+
+    ``$ pygmentize -O full,style=algol -f latex -o /path/to/output /path/to/input``
+        Render input file in Algol publication mode to LaTeX output.
+
+
+    Rendering Mode of First Class ADT Identifiers:
+
+    The rendering of standard library first class ADT identifiers is controlled
+    by option flag "treat_stdlib_adts_as_builtins".
+
+    When this option is turned on, standard library ADT identifiers are rendered
+    as builtins.  When it is turned off, they are rendered as ordinary library
+    identifiers.
+
+    `treat_stdlib_adts_as_builtins` (default: On)
+
+    The option is useful for dialects that support ADTs as first class objects
+    and provide ADTs in the standard library that would otherwise be built-in.
+
+    At present, only Modula-2 R10 supports library ADTs as first class objects
+    and therefore, no ADT identifiers are defined for any other dialects.
+
+    Example:
+
+    ``$ pygmentize -O full,dialect=m2r10,treat_stdlib_adts_as_builtins=Off ...``
+        Render standard library ADTs as ordinary library types.
+
+    .. versionchanged:: 2.1
+       Added multi-dialect support.
+    """
+    name = 'Modula-2'
+    url = 'http://www.modula2.org/'
+    aliases = ['modula2', 'm2']
+    filenames = ['*.def', '*.mod']
+    mimetypes = ['text/x-modula2']
+    version_added = '1.3'
+
+    flags = re.MULTILINE | re.DOTALL
+
+    tokens = {
+        'whitespace': [
+            (r'\n+', Text),  # blank lines
+            (r'\s+', Text),  # whitespace
+        ],
+        'dialecttags': [
+            # PIM Dialect Tag
+            (r'\(\*!m2pim\*\)', Comment.Special),
+            # ISO Dialect Tag
+            (r'\(\*!m2iso\*\)', Comment.Special),
+            # M2R10 Dialect Tag
+            (r'\(\*!m2r10\*\)', Comment.Special),
+            # ObjM2 Dialect Tag
+            (r'\(\*!objm2\*\)', Comment.Special),
+            # Aglet Extensions Dialect Tag
+            (r'\(\*!m2iso\+aglet\*\)', Comment.Special),
+            # GNU Extensions Dialect Tag
+            (r'\(\*!m2pim\+gm2\*\)', Comment.Special),
+            # p1 Extensions Dialect Tag
+            (r'\(\*!m2iso\+p1\*\)', Comment.Special),
+            # XDS Extensions Dialect Tag
+            (r'\(\*!m2iso\+xds\*\)', Comment.Special),
+        ],
+        'identifiers': [
+            (r'([a-zA-Z_$][\w$]*)', Name),
+        ],
+        'prefixed_number_literals': [
+            #
+            # Base-2, whole number
+            (r'0b[01]+(\'[01]+)*', Number.Bin),
+            #
+            # Base-16, whole number
+            (r'0[ux][0-9A-F]+(\'[0-9A-F]+)*', Number.Hex),
+        ],
+        'plain_number_literals': [
+            #
+            # Base-10, real number with exponent
+            (r'[0-9]+(\'[0-9]+)*'  # integral part
+             r'\.[0-9]+(\'[0-9]+)*'  # fractional part
+             r'[eE][+-]?[0-9]+(\'[0-9]+)*',  # exponent
+             Number.Float),
+            #
+            # Base-10, real number without exponent
+            (r'[0-9]+(\'[0-9]+)*'  # integral part
+             r'\.[0-9]+(\'[0-9]+)*',  # fractional part
+             Number.Float),
+            #
+            # Base-10, whole number
+            (r'[0-9]+(\'[0-9]+)*', Number.Integer),
+        ],
+        'suffixed_number_literals': [
+            #
+            # Base-8, whole number
+            (r'[0-7]+B', Number.Oct),
+            #
+            # Base-8, character code
+            (r'[0-7]+C', Number.Oct),
+            #
+            # Base-16, number
+            (r'[0-9A-F]+H', Number.Hex),
+        ],
+        'string_literals': [
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+        ],
+        'digraph_operators': [
+            # Dot Product Operator
+            (r'\*\.', Operator),
+            # Array Concatenation Operator
+            (r'\+>', Operator),  # M2R10 + ObjM2
+            # Inequality Operator
+            (r'<>', Operator),  # ISO + PIM
+            # Less-Or-Equal, Subset
+            (r'<=', Operator),
+            # Greater-Or-Equal, Superset
+            (r'>=', Operator),
+            # Identity Operator
+            (r'==', Operator),  # M2R10 + ObjM2
+            # Type Conversion Operator
+            (r'::', Operator),  # M2R10 + ObjM2
+            # Assignment Symbol
+            (r':=', Operator),
+            # Postfix Increment Mutator
+            (r'\+\+', Operator),  # M2R10 + ObjM2
+            # Postfix Decrement Mutator
+            (r'--', Operator),  # M2R10 + ObjM2
+        ],
+        'unigraph_operators': [
+            # Arithmetic Operators
+            (r'[+-]', Operator),
+            (r'[*/]', Operator),
+            # ISO 80000-2 compliant Set Difference Operator
+            (r'\\', Operator),  # M2R10 + ObjM2
+            # Relational Operators
+            (r'[=#<>]', Operator),
+            # Dereferencing Operator
+            (r'\^', Operator),
+            # Dereferencing Operator Synonym
+            (r'@', Operator),  # ISO
+            # Logical AND Operator Synonym
+            (r'&', Operator),  # PIM + ISO
+            # Logical NOT Operator Synonym
+            (r'~', Operator),  # PIM + ISO
+            # Smalltalk Message Prefix
+            (r'`', Operator),  # ObjM2
+        ],
+        'digraph_punctuation': [
+            # Range Constructor
+            (r'\.\.', Punctuation),
+            # Opening Chevron Bracket
+            (r'<<', Punctuation),  # M2R10 + ISO
+            # Closing Chevron Bracket
+            (r'>>', Punctuation),  # M2R10 + ISO
+            # Blueprint Punctuation
+            (r'->', Punctuation),  # M2R10 + ISO
+            # Distinguish |# and # in M2 R10
+            (r'\|#', Punctuation),
+            # Distinguish ## and # in M2 R10
+            (r'##', Punctuation),
+            # Distinguish |* and * in M2 R10
+            (r'\|\*', Punctuation),
+        ],
+        'unigraph_punctuation': [
+            # Common Punctuation
+            (r'[()\[\]{},.:;|]', Punctuation),
+            # Case Label Separator Synonym
+            (r'!', Punctuation),  # ISO
+            # Blueprint Punctuation
+            (r'\?', Punctuation),  # M2R10 + ObjM2
+        ],
+        'comments': [
+            # Single Line Comment
+            (r'^//.*?\n', Comment.Single),  # M2R10 + ObjM2
+            # Block Comment
+            (r'\(\*([^$].*?)\*\)', Comment.Multiline),
+            # Template Block Comment
+            (r'/\*(.*?)\*/', Comment.Multiline),  # M2R10 + ObjM2
+        ],
+        'pragmas': [
+            # ISO Style Pragmas
+            (r'<\*.*?\*>', Comment.Preproc),  # ISO, M2R10 + ObjM2
+            # Pascal Style Pragmas
+            (r'\(\*\$.*?\*\)', Comment.Preproc),  # PIM
+        ],
+        'root': [
+            include('whitespace'),
+            include('dialecttags'),
+            include('pragmas'),
+            include('comments'),
+            include('identifiers'),
+            include('suffixed_number_literals'),  # PIM + ISO
+            include('prefixed_number_literals'),  # M2R10 + ObjM2
+            include('plain_number_literals'),
+            include('string_literals'),
+            include('digraph_punctuation'),
+            include('digraph_operators'),
+            include('unigraph_punctuation'),
+            include('unigraph_operators'),
+        ]
+    }
+
+#  C o m m o n   D a t a s e t s
+
+    # Common Reserved Words Dataset
+    common_reserved_words = (
+        # 37 common reserved words
+        'AND', 'ARRAY', 'BEGIN', 'BY', 'CASE', 'CONST', 'DEFINITION', 'DIV',
+        'DO', 'ELSE', 'ELSIF', 'END', 'EXIT', 'FOR', 'FROM', 'IF',
+        'IMPLEMENTATION', 'IMPORT', 'IN', 'LOOP', 'MOD', 'MODULE', 'NOT',
+        'OF', 'OR', 'POINTER', 'PROCEDURE', 'RECORD', 'REPEAT', 'RETURN',
+        'SET', 'THEN', 'TO', 'TYPE', 'UNTIL', 'VAR', 'WHILE',
+    )
+
+    # Common Builtins Dataset
+    common_builtins = (
+        # 16 common builtins
+        'ABS', 'BOOLEAN', 'CARDINAL', 'CHAR', 'CHR', 'FALSE', 'INTEGER',
+        'LONGINT', 'LONGREAL', 'MAX', 'MIN', 'NIL', 'ODD', 'ORD', 'REAL',
+        'TRUE',
+    )
+
+    # Common Pseudo-Module Builtins Dataset
+    common_pseudo_builtins = (
+        # 4 common pseudo builtins
+        'ADDRESS', 'BYTE', 'WORD', 'ADR'
+    )
+
+#  P I M   M o d u l a - 2   D a t a s e t s
+
+    # Lexemes to Mark as Error Tokens for PIM Modula-2
+    pim_lexemes_to_reject = (
+        '!', '`', '@', '$', '%', '?', '\\', '==', '++', '--', '::', '*.',
+        '+>', '->', '<<', '>>', '|#', '##',
+    )
+
+    # PIM Modula-2 Additional Reserved Words Dataset
+    pim_additional_reserved_words = (
+        # 3 additional reserved words
+        'EXPORT', 'QUALIFIED', 'WITH',
+    )
+
+    # PIM Modula-2 Additional Builtins Dataset
+    pim_additional_builtins = (
+        # 16 additional builtins
+        'BITSET', 'CAP', 'DEC', 'DISPOSE', 'EXCL', 'FLOAT', 'HALT', 'HIGH',
+        'INC', 'INCL', 'NEW', 'NIL', 'PROC', 'SIZE', 'TRUNC', 'VAL',
+    )
+
+    # PIM Modula-2 Additional Pseudo-Module Builtins Dataset
+    pim_additional_pseudo_builtins = (
+        # 5 additional pseudo builtins
+        'SYSTEM', 'PROCESS', 'TSIZE', 'NEWPROCESS', 'TRANSFER',
+    )
+
+#  I S O   M o d u l a - 2   D a t a s e t s
+
+    # Lexemes to Mark as Error Tokens for ISO Modula-2
+    iso_lexemes_to_reject = (
+        '`', '$', '%', '?', '\\', '==', '++', '--', '::', '*.', '+>', '->',
+        '<<', '>>', '|#', '##',
+    )
+
+    # ISO Modula-2 Additional Reserved Words Dataset
+    iso_additional_reserved_words = (
+        # 9 additional reserved words (ISO 10514-1)
+        'EXCEPT', 'EXPORT', 'FINALLY', 'FORWARD', 'PACKEDSET', 'QUALIFIED',
+        'REM', 'RETRY', 'WITH',
+        # 10 additional reserved words (ISO 10514-2 & ISO 10514-3)
+        'ABSTRACT', 'AS', 'CLASS', 'GUARD', 'INHERIT', 'OVERRIDE', 'READONLY',
+        'REVEAL', 'TRACED', 'UNSAFEGUARDED',
+    )
+
+    # ISO Modula-2 Additional Builtins Dataset
+    iso_additional_builtins = (
+        # 26 additional builtins (ISO 10514-1)
+        'BITSET', 'CAP', 'CMPLX', 'COMPLEX', 'DEC', 'DISPOSE', 'EXCL', 'FLOAT',
+        'HALT', 'HIGH', 'IM', 'INC', 'INCL', 'INT', 'INTERRUPTIBLE',  'LENGTH',
+        'LFLOAT', 'LONGCOMPLEX', 'NEW', 'PROC', 'PROTECTION', 'RE', 'SIZE',
+        'TRUNC', 'UNINTERRUBTIBLE', 'VAL',
+        # 5 additional builtins (ISO 10514-2 & ISO 10514-3)
+        'CREATE', 'DESTROY', 'EMPTY', 'ISMEMBER', 'SELF',
+    )
+
+    # ISO Modula-2 Additional Pseudo-Module Builtins Dataset
+    iso_additional_pseudo_builtins = (
+        # 14 additional builtins (SYSTEM)
+        'SYSTEM', 'BITSPERLOC', 'LOCSPERBYTE', 'LOCSPERWORD', 'LOC',
+        'ADDADR', 'SUBADR', 'DIFADR', 'MAKEADR', 'ADR',
+        'ROTATE', 'SHIFT', 'CAST', 'TSIZE',
+        # 13 additional builtins (COROUTINES)
+        'COROUTINES', 'ATTACH', 'COROUTINE', 'CURRENT', 'DETACH', 'HANDLER',
+        'INTERRUPTSOURCE', 'IOTRANSFER', 'IsATTACHED', 'LISTEN',
+        'NEWCOROUTINE', 'PROT', 'TRANSFER',
+        # 9 additional builtins (EXCEPTIONS)
+        'EXCEPTIONS', 'AllocateSource', 'CurrentNumber', 'ExceptionNumber',
+        'ExceptionSource', 'GetMessage', 'IsCurrentSource',
+        'IsExceptionalExecution', 'RAISE',
+        # 3 additional builtins (TERMINATION)
+        'TERMINATION', 'IsTerminating', 'HasHalted',
+        # 4 additional builtins (M2EXCEPTION)
+        'M2EXCEPTION', 'M2Exceptions', 'M2Exception', 'IsM2Exception',
+        'indexException', 'rangeException', 'caseSelectException',
+        'invalidLocation', 'functionException', 'wholeValueException',
+        'wholeDivException', 'realValueException', 'realDivException',
+        'complexValueException', 'complexDivException', 'protException',
+        'sysException', 'coException', 'exException',
+    )
+
+#  M o d u l a - 2   R 1 0   D a t a s e t s
+
+    # Lexemes to Mark as Error Tokens for Modula-2 R10
+    m2r10_lexemes_to_reject = (
+        '!', '`', '@', '$', '%', '&', '<>',
+    )
+
+    # Modula-2 R10 reserved words in addition to the common set
+    m2r10_additional_reserved_words = (
+        # 12 additional reserved words
+        'ALIAS', 'ARGLIST', 'BLUEPRINT', 'COPY', 'GENLIB', 'INDETERMINATE',
+        'NEW', 'NONE', 'OPAQUE', 'REFERENTIAL', 'RELEASE', 'RETAIN',
+        # 2 additional reserved words with symbolic assembly option
+        'ASM', 'REG',
+    )
+
+    # Modula-2 R10 builtins in addition to the common set
+    m2r10_additional_builtins = (
+        # 26 additional builtins
+        'CARDINAL', 'COUNT', 'EMPTY', 'EXISTS', 'INSERT', 'LENGTH', 'LONGCARD',
+        'OCTET', 'PTR', 'PRED', 'READ', 'READNEW', 'REMOVE', 'RETRIEVE', 'SORT',
+        'STORE', 'SUBSET', 'SUCC', 'TLIMIT', 'TMAX', 'TMIN', 'TRUE', 'TSIZE',
+        'UNICHAR', 'WRITE', 'WRITEF',
+    )
+
+    # Modula-2 R10 Additional Pseudo-Module Builtins Dataset
+    m2r10_additional_pseudo_builtins = (
+        # 13 additional builtins (TPROPERTIES)
+        'TPROPERTIES', 'PROPERTY', 'LITERAL', 'TPROPERTY', 'TLITERAL',
+        'TBUILTIN', 'TDYN', 'TREFC', 'TNIL', 'TBASE', 'TPRECISION',
+        'TMAXEXP', 'TMINEXP',
+        # 4 additional builtins (CONVERSION)
+        'CONVERSION', 'TSXFSIZE', 'SXF', 'VAL',
+        # 35 additional builtins (UNSAFE)
+        'UNSAFE', 'CAST', 'INTRINSIC', 'AVAIL', 'ADD', 'SUB', 'ADDC', 'SUBC',
+        'FETCHADD', 'FETCHSUB', 'SHL', 'SHR', 'ASHR', 'ROTL', 'ROTR', 'ROTLC',
+        'ROTRC', 'BWNOT', 'BWAND', 'BWOR', 'BWXOR', 'BWNAND', 'BWNOR',
+        'SETBIT', 'TESTBIT', 'LSBIT', 'MSBIT', 'CSBITS', 'BAIL', 'HALT',
+        'TODO', 'FFI', 'ADDR', 'VARGLIST', 'VARGC',
+        # 11 additional builtins (ATOMIC)
+        'ATOMIC', 'INTRINSIC', 'AVAIL', 'SWAP', 'CAS', 'INC', 'DEC', 'BWAND',
+        'BWNAND', 'BWOR', 'BWXOR',
+        # 7 additional builtins (COMPILER)
+        'COMPILER', 'DEBUG', 'MODNAME', 'PROCNAME', 'LINENUM', 'DEFAULT',
+        'HASH',
+        # 5 additional builtins (ASSEMBLER)
+        'ASSEMBLER', 'REGISTER', 'SETREG', 'GETREG', 'CODE',
+    )
+
+#  O b j e c t i v e   M o d u l a - 2   D a t a s e t s
+
+    # Lexemes to Mark as Error Tokens for Objective Modula-2
+    objm2_lexemes_to_reject = (
+        '!', '$', '%', '&', '<>',
+    )
+
+    # Objective Modula-2 Extensions
+    # reserved words in addition to Modula-2 R10
+    objm2_additional_reserved_words = (
+        # 16 additional reserved words
+        'BYCOPY', 'BYREF', 'CLASS', 'CONTINUE', 'CRITICAL', 'INOUT', 'METHOD',
+        'ON', 'OPTIONAL', 'OUT', 'PRIVATE', 'PROTECTED', 'PROTOCOL', 'PUBLIC',
+        'SUPER', 'TRY',
+    )
+
+    # Objective Modula-2 Extensions
+    # builtins in addition to Modula-2 R10
+    objm2_additional_builtins = (
+        # 3 additional builtins
+        'OBJECT', 'NO', 'YES',
+    )
+
+    # Objective Modula-2 Extensions
+    # pseudo-module builtins in addition to Modula-2 R10
+    objm2_additional_pseudo_builtins = (
+        # None
+    )
+
+#  A g l e t   M o d u l a - 2   D a t a s e t s
+
+    # Aglet Extensions
+    # reserved words in addition to ISO Modula-2
+    aglet_additional_reserved_words = (
+        # None
+    )
+
+    # Aglet Extensions
+    # builtins in addition to ISO Modula-2
+    aglet_additional_builtins = (
+        # 9 additional builtins
+        'BITSET8', 'BITSET16', 'BITSET32', 'CARDINAL8', 'CARDINAL16',
+        'CARDINAL32', 'INTEGER8', 'INTEGER16', 'INTEGER32',
+    )
+
+    # Aglet Modula-2 Extensions
+    # pseudo-module builtins in addition to ISO Modula-2
+    aglet_additional_pseudo_builtins = (
+        # None
+    )
+
+#  G N U   M o d u l a - 2   D a t a s e t s
+
+    # GNU Extensions
+    # reserved words in addition to PIM Modula-2
+    gm2_additional_reserved_words = (
+        # 10 additional reserved words
+        'ASM', '__ATTRIBUTE__', '__BUILTIN__', '__COLUMN__', '__DATE__',
+        '__FILE__', '__FUNCTION__', '__LINE__', '__MODULE__', 'VOLATILE',
+    )
+
+    # GNU Extensions
+    # builtins in addition to PIM Modula-2
+    gm2_additional_builtins = (
+        # 21 additional builtins
+        'BITSET8', 'BITSET16', 'BITSET32', 'CARDINAL8', 'CARDINAL16',
+        'CARDINAL32', 'CARDINAL64', 'COMPLEX32', 'COMPLEX64', 'COMPLEX96',
+        'COMPLEX128', 'INTEGER8', 'INTEGER16', 'INTEGER32', 'INTEGER64',
+        'REAL8', 'REAL16', 'REAL32', 'REAL96', 'REAL128', 'THROW',
+    )
+
+    # GNU Extensions
+    # pseudo-module builtins in addition to PIM Modula-2
+    gm2_additional_pseudo_builtins = (
+        # None
+    )
+
+#  p 1   M o d u l a - 2   D a t a s e t s
+
+    # p1 Extensions
+    # reserved words in addition to ISO Modula-2
+    p1_additional_reserved_words = (
+        # None
+    )
+
+    # p1 Extensions
+    # builtins in addition to ISO Modula-2
+    p1_additional_builtins = (
+        # None
+    )
+
+    # p1 Modula-2 Extensions
+    # pseudo-module builtins in addition to ISO Modula-2
+    p1_additional_pseudo_builtins = (
+        # 1 additional builtin
+        'BCD',
+    )
+
+#  X D S   M o d u l a - 2   D a t a s e t s
+
+    # XDS Extensions
+    # reserved words in addition to ISO Modula-2
+    xds_additional_reserved_words = (
+        # 1 additional reserved word
+        'SEQ',
+    )
+
+    # XDS Extensions
+    # builtins in addition to ISO Modula-2
+    xds_additional_builtins = (
+        # 9 additional builtins
+        'ASH', 'ASSERT', 'DIFFADR_TYPE', 'ENTIER', 'INDEX', 'LEN',
+        'LONGCARD', 'SHORTCARD', 'SHORTINT',
+    )
+
+    # XDS Modula-2 Extensions
+    # pseudo-module builtins in addition to ISO Modula-2
+    xds_additional_pseudo_builtins = (
+        # 22 additional builtins (SYSTEM)
+        'PROCESS', 'NEWPROCESS', 'BOOL8', 'BOOL16', 'BOOL32', 'CARD8',
+        'CARD16', 'CARD32', 'INT8', 'INT16', 'INT32', 'REF', 'MOVE',
+        'FILL', 'GET', 'PUT', 'CC', 'int', 'unsigned', 'size_t', 'void'
+        # 3 additional builtins (COMPILER)
+        'COMPILER', 'OPTION', 'EQUATION'
+    )
+
+#  P I M   S t a n d a r d   L i b r a r y   D a t a s e t s
+
+    # PIM Modula-2 Standard Library Modules Dataset
+    pim_stdlib_module_identifiers = (
+        'Terminal', 'FileSystem', 'InOut', 'RealInOut', 'MathLib0', 'Storage',
+    )
+
+    # PIM Modula-2 Standard Library Types Dataset
+    pim_stdlib_type_identifiers = (
+        'Flag', 'FlagSet', 'Response', 'Command', 'Lock', 'Permission',
+        'MediumType', 'File', 'FileProc', 'DirectoryProc', 'FileCommand',
+        'DirectoryCommand',
+    )
+
+    # PIM Modula-2 Standard Library Procedures Dataset
+    pim_stdlib_proc_identifiers = (
+        'Read', 'BusyRead', 'ReadAgain', 'Write', 'WriteString', 'WriteLn',
+        'Create', 'Lookup', 'Close', 'Delete', 'Rename', 'SetRead', 'SetWrite',
+        'SetModify', 'SetOpen', 'Doio', 'SetPos', 'GetPos', 'Length', 'Reset',
+        'Again', 'ReadWord', 'WriteWord', 'ReadChar', 'WriteChar',
+        'CreateMedium', 'DeleteMedium', 'AssignName', 'DeassignName',
+        'ReadMedium', 'LookupMedium', 'OpenInput', 'OpenOutput', 'CloseInput',
+        'CloseOutput', 'ReadString', 'ReadInt', 'ReadCard', 'ReadWrd',
+        'WriteInt', 'WriteCard', 'WriteOct', 'WriteHex', 'WriteWrd',
+        'ReadReal', 'WriteReal', 'WriteFixPt', 'WriteRealOct', 'sqrt', 'exp',
+        'ln', 'sin', 'cos', 'arctan', 'entier', 'ALLOCATE', 'DEALLOCATE',
+    )
+
+    # PIM Modula-2 Standard Library Variables Dataset
+    pim_stdlib_var_identifiers = (
+        'Done', 'termCH', 'in', 'out'
+    )
+
+    # PIM Modula-2 Standard Library Constants Dataset
+    pim_stdlib_const_identifiers = (
+        'EOL',
+    )
+
+#  I S O   S t a n d a r d   L i b r a r y   D a t a s e t s
+
+    # ISO Modula-2 Standard Library Modules Dataset
+    iso_stdlib_module_identifiers = (
+        # TO DO
+    )
+
+    # ISO Modula-2 Standard Library Types Dataset
+    iso_stdlib_type_identifiers = (
+        # TO DO
+    )
+
+    # ISO Modula-2 Standard Library Procedures Dataset
+    iso_stdlib_proc_identifiers = (
+        # TO DO
+    )
+
+    # ISO Modula-2 Standard Library Variables Dataset
+    iso_stdlib_var_identifiers = (
+        # TO DO
+    )
+
+    # ISO Modula-2 Standard Library Constants Dataset
+    iso_stdlib_const_identifiers = (
+        # TO DO
+    )
+
+#  M 2   R 1 0   S t a n d a r d   L i b r a r y   D a t a s e t s
+
+    # Modula-2 R10 Standard Library ADTs Dataset
+    m2r10_stdlib_adt_identifiers = (
+        'BCD', 'LONGBCD', 'BITSET', 'SHORTBITSET', 'LONGBITSET',
+        'LONGLONGBITSET', 'COMPLEX', 'LONGCOMPLEX', 'SHORTCARD', 'LONGLONGCARD',
+        'SHORTINT', 'LONGLONGINT', 'POSINT', 'SHORTPOSINT', 'LONGPOSINT',
+        'LONGLONGPOSINT', 'BITSET8', 'BITSET16', 'BITSET32', 'BITSET64',
+        'BITSET128', 'BS8', 'BS16', 'BS32', 'BS64', 'BS128', 'CARDINAL8',
+        'CARDINAL16', 'CARDINAL32', 'CARDINAL64', 'CARDINAL128', 'CARD8',
+        'CARD16', 'CARD32', 'CARD64', 'CARD128', 'INTEGER8', 'INTEGER16',
+        'INTEGER32', 'INTEGER64', 'INTEGER128', 'INT8', 'INT16', 'INT32',
+        'INT64', 'INT128', 'STRING', 'UNISTRING',
+    )
+
+    # Modula-2 R10 Standard Library Blueprints Dataset
+    m2r10_stdlib_blueprint_identifiers = (
+        'ProtoRoot', 'ProtoComputational', 'ProtoNumeric', 'ProtoScalar',
+        'ProtoNonScalar', 'ProtoCardinal', 'ProtoInteger', 'ProtoReal',
+        'ProtoComplex', 'ProtoVector', 'ProtoTuple', 'ProtoCompArray',
+        'ProtoCollection', 'ProtoStaticArray', 'ProtoStaticSet',
+        'ProtoStaticString', 'ProtoArray', 'ProtoString', 'ProtoSet',
+        'ProtoMultiSet', 'ProtoDictionary', 'ProtoMultiDict', 'ProtoExtension',
+        'ProtoIO', 'ProtoCardMath', 'ProtoIntMath', 'ProtoRealMath',
+    )
+
+    # Modula-2 R10 Standard Library Modules Dataset
+    m2r10_stdlib_module_identifiers = (
+        'ASCII', 'BooleanIO', 'CharIO', 'UnicharIO', 'OctetIO',
+        'CardinalIO', 'LongCardIO', 'IntegerIO', 'LongIntIO', 'RealIO',
+        'LongRealIO', 'BCDIO', 'LongBCDIO', 'CardMath', 'LongCardMath',
+        'IntMath', 'LongIntMath', 'RealMath', 'LongRealMath', 'BCDMath',
+        'LongBCDMath', 'FileIO', 'FileSystem', 'Storage', 'IOSupport',
+    )
+
+    # Modula-2 R10 Standard Library Types Dataset
+    m2r10_stdlib_type_identifiers = (
+        'File', 'Status',
+        # TO BE COMPLETED
+    )
+
+    # Modula-2 R10 Standard Library Procedures Dataset
+    m2r10_stdlib_proc_identifiers = (
+        'ALLOCATE', 'DEALLOCATE', 'SIZE',
+        # TO BE COMPLETED
+    )
+
+    # Modula-2 R10 Standard Library Variables Dataset
+    m2r10_stdlib_var_identifiers = (
+        'stdIn', 'stdOut', 'stdErr',
+    )
+
+    # Modula-2 R10 Standard Library Constants Dataset
+    m2r10_stdlib_const_identifiers = (
+        'pi', 'tau',
+    )
+
+#  D i a l e c t s
+
+    # Dialect modes
+    dialects = (
+        'unknown',
+        'm2pim', 'm2iso', 'm2r10', 'objm2',
+        'm2iso+aglet', 'm2pim+gm2', 'm2iso+p1', 'm2iso+xds',
+    )
+
+#   D a t a b a s e s
+
+    # Lexemes to Mark as Errors Database
+    lexemes_to_reject_db = {
+        # Lexemes to reject for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Lexemes to reject for PIM Modula-2
+        'm2pim': (
+            pim_lexemes_to_reject,
+        ),
+        # Lexemes to reject for ISO Modula-2
+        'm2iso': (
+            iso_lexemes_to_reject,
+        ),
+        # Lexemes to reject for Modula-2 R10
+        'm2r10': (
+            m2r10_lexemes_to_reject,
+        ),
+        # Lexemes to reject for Objective Modula-2
+        'objm2': (
+            objm2_lexemes_to_reject,
+        ),
+        # Lexemes to reject for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_lexemes_to_reject,
+        ),
+        # Lexemes to reject for GNU Modula-2
+        'm2pim+gm2': (
+            pim_lexemes_to_reject,
+        ),
+        # Lexemes to reject for p1 Modula-2
+        'm2iso+p1': (
+            iso_lexemes_to_reject,
+        ),
+        # Lexemes to reject for XDS Modula-2
+        'm2iso+xds': (
+            iso_lexemes_to_reject,
+        ),
+    }
+
+    # Reserved Words Database
+    reserved_words_db = {
+        # Reserved words for unknown dialect
+        'unknown': (
+            common_reserved_words,
+            pim_additional_reserved_words,
+            iso_additional_reserved_words,
+            m2r10_additional_reserved_words,
+        ),
+
+        # Reserved words for PIM Modula-2
+        'm2pim': (
+            common_reserved_words,
+            pim_additional_reserved_words,
+        ),
+
+        # Reserved words for Modula-2 R10
+        'm2iso': (
+            common_reserved_words,
+            iso_additional_reserved_words,
+        ),
+
+        # Reserved words for ISO Modula-2
+        'm2r10': (
+            common_reserved_words,
+            m2r10_additional_reserved_words,
+        ),
+
+        # Reserved words for Objective Modula-2
+        'objm2': (
+            common_reserved_words,
+            m2r10_additional_reserved_words,
+            objm2_additional_reserved_words,
+        ),
+
+        # Reserved words for Aglet Modula-2 Extensions
+        'm2iso+aglet': (
+            common_reserved_words,
+            iso_additional_reserved_words,
+            aglet_additional_reserved_words,
+        ),
+
+        # Reserved words for GNU Modula-2 Extensions
+        'm2pim+gm2': (
+            common_reserved_words,
+            pim_additional_reserved_words,
+            gm2_additional_reserved_words,
+        ),
+
+        # Reserved words for p1 Modula-2 Extensions
+        'm2iso+p1': (
+            common_reserved_words,
+            iso_additional_reserved_words,
+            p1_additional_reserved_words,
+        ),
+
+        # Reserved words for XDS Modula-2 Extensions
+        'm2iso+xds': (
+            common_reserved_words,
+            iso_additional_reserved_words,
+            xds_additional_reserved_words,
+        ),
+    }
+
+    # Builtins Database
+    builtins_db = {
+        # Builtins for unknown dialect
+        'unknown': (
+            common_builtins,
+            pim_additional_builtins,
+            iso_additional_builtins,
+            m2r10_additional_builtins,
+        ),
+
+        # Builtins for PIM Modula-2
+        'm2pim': (
+            common_builtins,
+            pim_additional_builtins,
+        ),
+
+        # Builtins for ISO Modula-2
+        'm2iso': (
+            common_builtins,
+            iso_additional_builtins,
+        ),
+
+        # Builtins for ISO Modula-2
+        'm2r10': (
+            common_builtins,
+            m2r10_additional_builtins,
+        ),
+
+        # Builtins for Objective Modula-2
+        'objm2': (
+            common_builtins,
+            m2r10_additional_builtins,
+            objm2_additional_builtins,
+        ),
+
+        # Builtins for Aglet Modula-2 Extensions
+        'm2iso+aglet': (
+            common_builtins,
+            iso_additional_builtins,
+            aglet_additional_builtins,
+        ),
+
+        # Builtins for GNU Modula-2 Extensions
+        'm2pim+gm2': (
+            common_builtins,
+            pim_additional_builtins,
+            gm2_additional_builtins,
+        ),
+
+        # Builtins for p1 Modula-2 Extensions
+        'm2iso+p1': (
+            common_builtins,
+            iso_additional_builtins,
+            p1_additional_builtins,
+        ),
+
+        # Builtins for XDS Modula-2 Extensions
+        'm2iso+xds': (
+            common_builtins,
+            iso_additional_builtins,
+            xds_additional_builtins,
+        ),
+    }
+
+    # Pseudo-Module Builtins Database
+    pseudo_builtins_db = {
+        # Builtins for unknown dialect
+        'unknown': (
+            common_pseudo_builtins,
+            pim_additional_pseudo_builtins,
+            iso_additional_pseudo_builtins,
+            m2r10_additional_pseudo_builtins,
+        ),
+
+        # Builtins for PIM Modula-2
+        'm2pim': (
+            common_pseudo_builtins,
+            pim_additional_pseudo_builtins,
+        ),
+
+        # Builtins for ISO Modula-2
+        'm2iso': (
+            common_pseudo_builtins,
+            iso_additional_pseudo_builtins,
+        ),
+
+        # Builtins for ISO Modula-2
+        'm2r10': (
+            common_pseudo_builtins,
+            m2r10_additional_pseudo_builtins,
+        ),
+
+        # Builtins for Objective Modula-2
+        'objm2': (
+            common_pseudo_builtins,
+            m2r10_additional_pseudo_builtins,
+            objm2_additional_pseudo_builtins,
+        ),
+
+        # Builtins for Aglet Modula-2 Extensions
+        'm2iso+aglet': (
+            common_pseudo_builtins,
+            iso_additional_pseudo_builtins,
+            aglet_additional_pseudo_builtins,
+        ),
+
+        # Builtins for GNU Modula-2 Extensions
+        'm2pim+gm2': (
+            common_pseudo_builtins,
+            pim_additional_pseudo_builtins,
+            gm2_additional_pseudo_builtins,
+        ),
+
+        # Builtins for p1 Modula-2 Extensions
+        'm2iso+p1': (
+            common_pseudo_builtins,
+            iso_additional_pseudo_builtins,
+            p1_additional_pseudo_builtins,
+        ),
+
+        # Builtins for XDS Modula-2 Extensions
+        'm2iso+xds': (
+            common_pseudo_builtins,
+            iso_additional_pseudo_builtins,
+            xds_additional_pseudo_builtins,
+        ),
+    }
+
+    # Standard Library ADTs Database
+    stdlib_adts_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library ADTs for PIM Modula-2
+        'm2pim': (
+            # No first class library types
+        ),
+
+        # Standard Library ADTs for ISO Modula-2
+        'm2iso': (
+            # No first class library types
+        ),
+
+        # Standard Library ADTs for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_adt_identifiers,
+        ),
+
+        # Standard Library ADTs for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_adt_identifiers,
+        ),
+
+        # Standard Library ADTs for Aglet Modula-2
+        'm2iso+aglet': (
+            # No first class library types
+        ),
+
+        # Standard Library ADTs for GNU Modula-2
+        'm2pim+gm2': (
+            # No first class library types
+        ),
+
+        # Standard Library ADTs for p1 Modula-2
+        'm2iso+p1': (
+            # No first class library types
+        ),
+
+        # Standard Library ADTs for XDS Modula-2
+        'm2iso+xds': (
+            # No first class library types
+        ),
+    }
+
+    # Standard Library Modules Database
+    stdlib_modules_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library Modules for PIM Modula-2
+        'm2pim': (
+            pim_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for ISO Modula-2
+        'm2iso': (
+            iso_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_blueprint_identifiers,
+            m2r10_stdlib_module_identifiers,
+            m2r10_stdlib_adt_identifiers,
+        ),
+
+        # Standard Library Modules for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_blueprint_identifiers,
+            m2r10_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for GNU Modula-2
+        'm2pim+gm2': (
+            pim_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for p1 Modula-2
+        'm2iso+p1': (
+            iso_stdlib_module_identifiers,
+        ),
+
+        # Standard Library Modules for XDS Modula-2
+        'm2iso+xds': (
+            iso_stdlib_module_identifiers,
+        ),
+    }
+
+    # Standard Library Types Database
+    stdlib_types_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library Types for PIM Modula-2
+        'm2pim': (
+            pim_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for ISO Modula-2
+        'm2iso': (
+            iso_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for GNU Modula-2
+        'm2pim+gm2': (
+            pim_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for p1 Modula-2
+        'm2iso+p1': (
+            iso_stdlib_type_identifiers,
+        ),
+
+        # Standard Library Types for XDS Modula-2
+        'm2iso+xds': (
+            iso_stdlib_type_identifiers,
+        ),
+    }
+
+    # Standard Library Procedures Database
+    stdlib_procedures_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library Procedures for PIM Modula-2
+        'm2pim': (
+            pim_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for ISO Modula-2
+        'm2iso': (
+            iso_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for GNU Modula-2
+        'm2pim+gm2': (
+            pim_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for p1 Modula-2
+        'm2iso+p1': (
+            iso_stdlib_proc_identifiers,
+        ),
+
+        # Standard Library Procedures for XDS Modula-2
+        'm2iso+xds': (
+            iso_stdlib_proc_identifiers,
+        ),
+    }
+
+    # Standard Library Variables Database
+    stdlib_variables_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library Variables for PIM Modula-2
+        'm2pim': (
+            pim_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for ISO Modula-2
+        'm2iso': (
+            iso_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for GNU Modula-2
+        'm2pim+gm2': (
+            pim_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for p1 Modula-2
+        'm2iso+p1': (
+            iso_stdlib_var_identifiers,
+        ),
+
+        # Standard Library Variables for XDS Modula-2
+        'm2iso+xds': (
+            iso_stdlib_var_identifiers,
+        ),
+    }
+
+    # Standard Library Constants Database
+    stdlib_constants_db = {
+        # Empty entry for unknown dialect
+        'unknown': (
+            # LEAVE THIS EMPTY
+        ),
+        # Standard Library Constants for PIM Modula-2
+        'm2pim': (
+            pim_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for ISO Modula-2
+        'm2iso': (
+            iso_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for Modula-2 R10
+        'm2r10': (
+            m2r10_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for Objective Modula-2
+        'objm2': (
+            m2r10_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for Aglet Modula-2
+        'm2iso+aglet': (
+            iso_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for GNU Modula-2
+        'm2pim+gm2': (
+            pim_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for p1 Modula-2
+        'm2iso+p1': (
+            iso_stdlib_const_identifiers,
+        ),
+
+        # Standard Library Constants for XDS Modula-2
+        'm2iso+xds': (
+            iso_stdlib_const_identifiers,
+        ),
+    }
+
+#   M e t h o d s
+
+    # initialise a lexer instance
+    def __init__(self, **options):
+        #
+        # check dialect options
+        #
+        dialects = get_list_opt(options, 'dialect', [])
+        #
+        for dialect_option in dialects:
+            if dialect_option in self.dialects[1:-1]:
+                # valid dialect option found
+                self.set_dialect(dialect_option)
+                break
+        #
+        # Fallback Mode (DEFAULT)
+        else:
+            # no valid dialect option
+            self.set_dialect('unknown')
+        #
+        self.dialect_set_by_tag = False
+        #
+        # check style options
+        #
+        styles = get_list_opt(options, 'style', [])
+        #
+        # use lowercase mode for Algol style
+        if 'algol' in styles or 'algol_nu' in styles:
+            self.algol_publication_mode = True
+        else:
+            self.algol_publication_mode = False
+        #
+        # Check option flags
+        #
+        self.treat_stdlib_adts_as_builtins = get_bool_opt(
+            options, 'treat_stdlib_adts_as_builtins', True)
+        #
+        # call superclass initialiser
+        RegexLexer.__init__(self, **options)
+
+    # Set lexer to a specified dialect
+    def set_dialect(self, dialect_id):
+        #
+        # if __debug__:
+        #    print 'entered set_dialect with arg: ', dialect_id
+        #
+        # check dialect name against known dialects
+        if dialect_id not in self.dialects:
+            dialect = 'unknown'  # default
+        else:
+            dialect = dialect_id
+        #
+        # compose lexemes to reject set
+        lexemes_to_reject_set = set()
+        # add each list of reject lexemes for this dialect
+        for list in self.lexemes_to_reject_db[dialect]:
+            lexemes_to_reject_set.update(set(list))
+        #
+        # compose reserved words set
+        reswords_set = set()
+        # add each list of reserved words for this dialect
+        for list in self.reserved_words_db[dialect]:
+            reswords_set.update(set(list))
+        #
+        # compose builtins set
+        builtins_set = set()
+        # add each list of builtins for this dialect excluding reserved words
+        for list in self.builtins_db[dialect]:
+            builtins_set.update(set(list).difference(reswords_set))
+        #
+        # compose pseudo-builtins set
+        pseudo_builtins_set = set()
+        # add each list of builtins for this dialect excluding reserved words
+        for list in self.pseudo_builtins_db[dialect]:
+            pseudo_builtins_set.update(set(list).difference(reswords_set))
+        #
+        # compose ADTs set
+        adts_set = set()
+        # add each list of ADTs for this dialect excluding reserved words
+        for list in self.stdlib_adts_db[dialect]:
+            adts_set.update(set(list).difference(reswords_set))
+        #
+        # compose modules set
+        modules_set = set()
+        # add each list of builtins for this dialect excluding builtins
+        for list in self.stdlib_modules_db[dialect]:
+            modules_set.update(set(list).difference(builtins_set))
+        #
+        # compose types set
+        types_set = set()
+        # add each list of types for this dialect excluding builtins
+        for list in self.stdlib_types_db[dialect]:
+            types_set.update(set(list).difference(builtins_set))
+        #
+        # compose procedures set
+        procedures_set = set()
+        # add each list of procedures for this dialect excluding builtins
+        for list in self.stdlib_procedures_db[dialect]:
+            procedures_set.update(set(list).difference(builtins_set))
+        #
+        # compose variables set
+        variables_set = set()
+        # add each list of variables for this dialect excluding builtins
+        for list in self.stdlib_variables_db[dialect]:
+            variables_set.update(set(list).difference(builtins_set))
+        #
+        # compose constants set
+        constants_set = set()
+        # add each list of constants for this dialect excluding builtins
+        for list in self.stdlib_constants_db[dialect]:
+            constants_set.update(set(list).difference(builtins_set))
+        #
+        # update lexer state
+        self.dialect = dialect
+        self.lexemes_to_reject = lexemes_to_reject_set
+        self.reserved_words = reswords_set
+        self.builtins = builtins_set
+        self.pseudo_builtins = pseudo_builtins_set
+        self.adts = adts_set
+        self.modules = modules_set
+        self.types = types_set
+        self.procedures = procedures_set
+        self.variables = variables_set
+        self.constants = constants_set
+        #
+        # if __debug__:
+        #    print 'exiting set_dialect'
+        #    print ' self.dialect: ', self.dialect
+        #    print ' self.lexemes_to_reject: ', self.lexemes_to_reject
+        #    print ' self.reserved_words: ', self.reserved_words
+        #    print ' self.builtins: ', self.builtins
+        #    print ' self.pseudo_builtins: ', self.pseudo_builtins
+        #    print ' self.adts: ', self.adts
+        #    print ' self.modules: ', self.modules
+        #    print ' self.types: ', self.types
+        #    print ' self.procedures: ', self.procedures
+        #    print ' self.variables: ', self.variables
+        #    print ' self.types: ', self.types
+        #    print ' self.constants: ', self.constants
+
+    # Extracts a dialect name from a dialect tag comment string  and checks
+    # the extracted name against known dialects.  If a match is found,  the
+    # matching name is returned, otherwise dialect id 'unknown' is returned
+    def get_dialect_from_dialect_tag(self, dialect_tag):
+        #
+        # if __debug__:
+        #    print 'entered get_dialect_from_dialect_tag with arg: ', dialect_tag
+        #
+        # constants
+        left_tag_delim = '(*!'
+        right_tag_delim = '*)'
+        left_tag_delim_len = len(left_tag_delim)
+        right_tag_delim_len = len(right_tag_delim)
+        indicator_start = left_tag_delim_len
+        indicator_end = -(right_tag_delim_len)
+        #
+        # check comment string for dialect indicator
+        if len(dialect_tag) > (left_tag_delim_len + right_tag_delim_len) \
+           and dialect_tag.startswith(left_tag_delim) \
+           and dialect_tag.endswith(right_tag_delim):
+            #
+            # if __debug__:
+            #    print 'dialect tag found'
+            #
+            # extract dialect indicator
+            indicator = dialect_tag[indicator_start:indicator_end]
+            #
+            # if __debug__:
+            #    print 'extracted: ', indicator
+            #
+            # check against known dialects
+            for index in range(1, len(self.dialects)):
+                #
+                # if __debug__:
+                #    print 'dialects[', index, ']: ', self.dialects[index]
+                #
+                if indicator == self.dialects[index]:
+                    #
+                    # if __debug__:
+                    #    print 'matching dialect found'
+                    #
+                    # indicator matches known dialect
+                    return indicator
+            else:
+                # indicator does not match any dialect
+                return 'unknown'  # default
+        else:
+            # invalid indicator string
+            return 'unknown'  # default
+
+    # intercept the token stream, modify token attributes and return them
+    def get_tokens_unprocessed(self, text):
+        for index, token, value in RegexLexer.get_tokens_unprocessed(self, text):
+            #
+            # check for dialect tag if dialect has not been set by tag
+            if not self.dialect_set_by_tag and token == Comment.Special:
+                indicated_dialect = self.get_dialect_from_dialect_tag(value)
+                if indicated_dialect != 'unknown':
+                    # token is a dialect indicator
+                    # reset reserved words and builtins
+                    self.set_dialect(indicated_dialect)
+                    self.dialect_set_by_tag = True
+            #
+            # check for reserved words, predefined and stdlib identifiers
+            if token is Name:
+                if value in self.reserved_words:
+                    token = Keyword.Reserved
+                    if self.algol_publication_mode:
+                        value = value.lower()
+                #
+                elif value in self.builtins:
+                    token = Name.Builtin
+                    if self.algol_publication_mode:
+                        value = value.lower()
+                #
+                elif value in self.pseudo_builtins:
+                    token = Name.Builtin.Pseudo
+                    if self.algol_publication_mode:
+                        value = value.lower()
+                #
+                elif value in self.adts:
+                    if not self.treat_stdlib_adts_as_builtins:
+                        token = Name.Namespace
+                    else:
+                        token = Name.Builtin.Pseudo
+                        if self.algol_publication_mode:
+                            value = value.lower()
+                #
+                elif value in self.modules:
+                    token = Name.Namespace
+                #
+                elif value in self.types:
+                    token = Name.Class
+                #
+                elif value in self.procedures:
+                    token = Name.Function
+                #
+                elif value in self.variables:
+                    token = Name.Variable
+                #
+                elif value in self.constants:
+                    token = Name.Constant
+            #
+            elif token in Number:
+                #
+                # mark prefix number literals as error for PIM and ISO dialects
+                if self.dialect not in ('unknown', 'm2r10', 'objm2'):
+                    if "'" in value or value[0:2] in ('0b', '0x', '0u'):
+                        token = Error
+                #
+                elif self.dialect in ('m2r10', 'objm2'):
+                    # mark base-8 number literals as errors for M2 R10 and ObjM2
+                    if token is Number.Oct:
+                        token = Error
+                    # mark suffix base-16 literals as errors for M2 R10 and ObjM2
+                    elif token is Number.Hex and 'H' in value:
+                        token = Error
+                    # mark real numbers with E as errors for M2 R10 and ObjM2
+                    elif token is Number.Float and 'E' in value:
+                        token = Error
+            #
+            elif token in Comment:
+                #
+                # mark single line comment as error for PIM and ISO dialects
+                if token is Comment.Single:
+                    if self.dialect not in ('unknown', 'm2r10', 'objm2'):
+                        token = Error
+                #
+                if token is Comment.Preproc:
+                    # mark ISO pragma as error for PIM dialects
+                    if value.startswith('<*') and \
+                       self.dialect.startswith('m2pim'):
+                        token = Error
+                    # mark PIM pragma as comment for other dialects
+                    elif value.startswith('(*$') and \
+                            self.dialect != 'unknown' and \
+                            not self.dialect.startswith('m2pim'):
+                        token = Comment.Multiline
+            #
+            else:  # token is neither Name nor Comment
+                #
+                # mark lexemes matching the dialect's error token set as errors
+                if value in self.lexemes_to_reject:
+                    token = Error
+                #
+                # substitute lexemes when in Algol mode
+                if self.algol_publication_mode:
+                    if value == '#':
+                        value = '≠'
+                    elif value == '<=':
+                        value = '≤'
+                    elif value == '>=':
+                        value = '≥'
+                    elif value == '==':
+                        value = '≡'
+                    elif value == '*.':
+                        value = '•'
+
+            # return result
+            yield index, token, value
+
+    def analyse_text(text):
+        """It's Pascal-like, but does not use FUNCTION -- uses PROCEDURE
+        instead."""
+
+        # Check if this looks like Pascal, if not, bail out early
+        if not ('(*' in text and '*)' in text and ':=' in text):
+            return
+
+        result = 0
+        # Procedure is in Modula2
+        if re.search(r'\bPROCEDURE\b', text):
+            result += 0.6
+
+        # FUNCTION is only valid in Pascal, but not in Modula2
+        if re.search(r'\bFUNCTION\b', text):
+            result = 0.0
+
+        return result
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/mojo.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/mojo.py
new file mode 100644
index 0000000000000000000000000000000000000000..84aac46694607b0f385fc1aec747c3ffcde7ca74
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/mojo.py
@@ -0,0 +1,707 @@
+"""
+    pygments.lexers.mojo
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Mojo and related languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import keyword
+
+from pygments import unistring as uni
+from pygments.lexer import (
+    RegexLexer,
+    bygroups,
+    combined,
+    default,
+    include,
+    this,
+    using,
+    words,
+)
+from pygments.token import (
+    Comment,
+    # Error,
+    Keyword,
+    Name,
+    Number,
+    Operator,
+    Punctuation,
+    String,
+    Text,
+    Whitespace,
+)
+from pygments.util import shebang_matches
+
+__all__ = ["MojoLexer"]
+
+
+class MojoLexer(RegexLexer):
+    """
+    For Mojo source code (version 24.2.1).
+    """
+
+    name = "Mojo"
+    url = "https://docs.modular.com/mojo/"
+    aliases = ["mojo", "🔥"]
+    filenames = [
+        "*.mojo",
+        "*.🔥",
+    ]
+    mimetypes = [
+        "text/x-mojo",
+        "application/x-mojo",
+    ]
+    version_added = "2.18"
+
+    uni_name = f"[{uni.xid_start}][{uni.xid_continue}]*"
+
+    def innerstring_rules(ttype):
+        return [
+            # the old style '%s' % (...) string formatting (still valid in Py3)
+            (
+                r"%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?"
+                "[hlL]?[E-GXc-giorsaux%]",
+                String.Interpol,
+            ),
+            # the new style '{}'.format(...) string formatting
+            (
+                r"\{"
+                r"((\w+)((\.\w+)|(\[[^\]]+\]))*)?"  # field name
+                r"(\![sra])?"  # conversion
+                r"(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?"
+                r"\}",
+                String.Interpol,
+            ),
+            # backslashes, quotes and formatting signs must be parsed one at a time
+            (r'[^\\\'"%{\n]+', ttype),
+            (r'[\'"\\]', ttype),
+            # unhandled string formatting sign
+            (r"%|(\{{1,2})", ttype),
+            # newlines are an error (use "nl" state)
+        ]
+
+    def fstring_rules(ttype):
+        return [
+            # Assuming that a '}' is the closing brace after format specifier.
+            # Sadly, this means that we won't detect syntax error. But it's
+            # more important to parse correct syntax correctly, than to
+            # highlight invalid syntax.
+            (r"\}", String.Interpol),
+            (r"\{", String.Interpol, "expr-inside-fstring"),
+            # backslashes, quotes and formatting signs must be parsed one at a time
+            (r'[^\\\'"{}\n]+', ttype),
+            (r'[\'"\\]', ttype),
+            # newlines are an error (use "nl" state)
+        ]
+
+    tokens = {
+        "root": [
+            (r"\s+", Whitespace),
+            (
+                r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
+                bygroups(Whitespace, String.Affix, String.Doc),
+            ),
+            (
+                r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
+                bygroups(Whitespace, String.Affix, String.Doc),
+            ),
+            (r"\A#!.+$", Comment.Hashbang),
+            (r"#.*$", Comment.Single),
+            (r"\\\n", Whitespace),
+            (r"\\", Whitespace),
+            include("keywords"),
+            include("soft-keywords"),
+            # In the original PR, all the below here used ((?:\s|\\\s)+) to
+            # designate whitespace, but I can't find any example of this being
+            # needed in the example file, so we're replacing it with `\s+`.
+            (
+                r"(alias)(\s+)",
+                bygroups(Keyword, Whitespace),
+                "varname",  # TODO varname the right fit?
+            ),
+            (r"(var)(\s+)", bygroups(Keyword, Whitespace), "varname"),
+            (r"(def)(\s+)", bygroups(Keyword, Whitespace), "funcname"),
+            (r"(fn)(\s+)", bygroups(Keyword, Whitespace), "funcname"),
+            (
+                r"(class)(\s+)",
+                bygroups(Keyword, Whitespace),
+                "classname",
+            ),  # not implemented yet
+            (r"(struct)(\s+)", bygroups(Keyword, Whitespace), "structname"),
+            (r"(trait)(\s+)", bygroups(Keyword, Whitespace), "structname"),
+            (r"(from)(\s+)", bygroups(Keyword.Namespace, Whitespace), "fromimport"),
+            (r"(import)(\s+)", bygroups(Keyword.Namespace, Whitespace), "import"),
+            include("expr"),
+        ],
+        "expr": [
+            # raw f-strings
+            (
+                '(?i)(rf|fr)(""")',
+                bygroups(String.Affix, String.Double),
+                combined("rfstringescape", "tdqf"),
+            ),
+            (
+                "(?i)(rf|fr)(''')",
+                bygroups(String.Affix, String.Single),
+                combined("rfstringescape", "tsqf"),
+            ),
+            (
+                '(?i)(rf|fr)(")',
+                bygroups(String.Affix, String.Double),
+                combined("rfstringescape", "dqf"),
+            ),
+            (
+                "(?i)(rf|fr)(')",
+                bygroups(String.Affix, String.Single),
+                combined("rfstringescape", "sqf"),
+            ),
+            # non-raw f-strings
+            (
+                '([fF])(""")',
+                bygroups(String.Affix, String.Double),
+                combined("fstringescape", "tdqf"),
+            ),
+            (
+                "([fF])(''')",
+                bygroups(String.Affix, String.Single),
+                combined("fstringescape", "tsqf"),
+            ),
+            (
+                '([fF])(")',
+                bygroups(String.Affix, String.Double),
+                combined("fstringescape", "dqf"),
+            ),
+            (
+                "([fF])(')",
+                bygroups(String.Affix, String.Single),
+                combined("fstringescape", "sqf"),
+            ),
+            # raw bytes and strings
+            ('(?i)(rb|br|r)(""")', bygroups(String.Affix, String.Double), "tdqs"),
+            ("(?i)(rb|br|r)(''')", bygroups(String.Affix, String.Single), "tsqs"),
+            ('(?i)(rb|br|r)(")', bygroups(String.Affix, String.Double), "dqs"),
+            ("(?i)(rb|br|r)(')", bygroups(String.Affix, String.Single), "sqs"),
+            # non-raw strings
+            (
+                '([uU]?)(""")',
+                bygroups(String.Affix, String.Double),
+                combined("stringescape", "tdqs"),
+            ),
+            (
+                "([uU]?)(''')",
+                bygroups(String.Affix, String.Single),
+                combined("stringescape", "tsqs"),
+            ),
+            (
+                '([uU]?)(")',
+                bygroups(String.Affix, String.Double),
+                combined("stringescape", "dqs"),
+            ),
+            (
+                "([uU]?)(')",
+                bygroups(String.Affix, String.Single),
+                combined("stringescape", "sqs"),
+            ),
+            # non-raw bytes
+            (
+                '([bB])(""")',
+                bygroups(String.Affix, String.Double),
+                combined("bytesescape", "tdqs"),
+            ),
+            (
+                "([bB])(''')",
+                bygroups(String.Affix, String.Single),
+                combined("bytesescape", "tsqs"),
+            ),
+            (
+                '([bB])(")',
+                bygroups(String.Affix, String.Double),
+                combined("bytesescape", "dqs"),
+            ),
+            (
+                "([bB])(')",
+                bygroups(String.Affix, String.Single),
+                combined("bytesescape", "sqs"),
+            ),
+            (r"[^\S\n]+", Text),
+            include("numbers"),
+            (r"!=|==|<<|>>|:=|[-~+/*%=<>&^|.]", Operator),
+            (r"([]{}:\(\),;[])+", Punctuation),
+            (r"(in|is|and|or|not)\b", Operator.Word),
+            include("expr-keywords"),
+            include("builtins"),
+            include("magicfuncs"),
+            include("magicvars"),
+            include("name"),
+        ],
+        "expr-inside-fstring": [
+            (r"[{([]", Punctuation, "expr-inside-fstring-inner"),
+            # without format specifier
+            (
+                r"(=\s*)?"  # debug (https://bugs.python.org/issue36817)
+                r"(\![sraf])?"  # conversion
+                r"\}",
+                String.Interpol,
+                "#pop",
+            ),
+            # with format specifier
+            # we'll catch the remaining '}' in the outer scope
+            (
+                r"(=\s*)?"  # debug (https://bugs.python.org/issue36817)
+                r"(\![sraf])?"  # conversion
+                r":",
+                String.Interpol,
+                "#pop",
+            ),
+            (r"\s+", Whitespace),  # allow new lines
+            include("expr"),
+        ],
+        "expr-inside-fstring-inner": [
+            (r"[{([]", Punctuation, "expr-inside-fstring-inner"),
+            (r"[])}]", Punctuation, "#pop"),
+            (r"\s+", Whitespace),  # allow new lines
+            include("expr"),
+        ],
+        "expr-keywords": [
+            # Based on https://docs.python.org/3/reference/expressions.html
+            (
+                words(
+                    (
+                        "async for",  # TODO https://docs.modular.com/mojo/roadmap#no-async-for-or-async-with
+                        "async with",  # TODO https://docs.modular.com/mojo/roadmap#no-async-for-or-async-with
+                        "await",
+                        "else",
+                        "for",
+                        "if",
+                        "lambda",
+                        "yield",
+                        "yield from",
+                    ),
+                    suffix=r"\b",
+                ),
+                Keyword,
+            ),
+            (words(("True", "False", "None"), suffix=r"\b"), Keyword.Constant),
+        ],
+        "keywords": [
+            (
+                words(
+                    (
+                        "assert",
+                        "async",
+                        "await",
+                        "borrowed",
+                        "break",
+                        "continue",
+                        "del",
+                        "elif",
+                        "else",
+                        "except",
+                        "finally",
+                        "for",
+                        "global",
+                        "if",
+                        "lambda",
+                        "pass",
+                        "raise",
+                        "nonlocal",
+                        "return",
+                        "try",
+                        "while",
+                        "yield",
+                        "yield from",
+                        "as",
+                        "with",
+                    ),
+                    suffix=r"\b",
+                ),
+                Keyword,
+            ),
+            (words(("True", "False", "None"), suffix=r"\b"), Keyword.Constant),
+        ],
+        "soft-keywords": [
+            # `match`, `case` and `_` soft keywords
+            (
+                r"(^[ \t]*)"  # at beginning of line + possible indentation
+                r"(match|case)\b"  # a possible keyword
+                r"(?![ \t]*(?:"  # not followed by...
+                r"[:,;=^&|@~)\]}]|(?:" +  # characters and keywords that mean this isn't
+                # pattern matching (but None/True/False is ok)
+                r"|".join(k for k in keyword.kwlist if k[0].islower())
+                + r")\b))",
+                bygroups(Whitespace, Keyword),
+                "soft-keywords-inner",
+            ),
+        ],
+        "soft-keywords-inner": [
+            # optional `_` keyword
+            (r"(\s+)([^\n_]*)(_\b)", bygroups(Whitespace, using(this), Keyword)),
+            default("#pop"),
+        ],
+        "builtins": [
+            (
+                words(
+                    (
+                        "__import__",
+                        "abs",
+                        "aiter",
+                        "all",
+                        "any",
+                        "bin",
+                        "bool",
+                        "bytearray",
+                        "breakpoint",
+                        "bytes",
+                        "callable",
+                        "chr",
+                        "classmethod",
+                        "compile",
+                        "complex",
+                        "delattr",
+                        "dict",
+                        "dir",
+                        "divmod",
+                        "enumerate",
+                        "eval",
+                        "filter",
+                        "float",
+                        "format",
+                        "frozenset",
+                        "getattr",
+                        "globals",
+                        "hasattr",
+                        "hash",
+                        "hex",
+                        "id",
+                        "input",
+                        "int",
+                        "isinstance",
+                        "issubclass",
+                        "iter",
+                        "len",
+                        "list",
+                        "locals",
+                        "map",
+                        "max",
+                        "memoryview",
+                        "min",
+                        "next",
+                        "object",
+                        "oct",
+                        "open",
+                        "ord",
+                        "pow",
+                        "print",
+                        "property",
+                        "range",
+                        "repr",
+                        "reversed",
+                        "round",
+                        "set",
+                        "setattr",
+                        "slice",
+                        "sorted",
+                        "staticmethod",
+                        "str",
+                        "sum",
+                        "super",
+                        "tuple",
+                        "type",
+                        "vars",
+                        "zip",
+                        # Mojo builtin types: https://docs.modular.com/mojo/stdlib/builtin/
+                        "AnyType",
+                        "Coroutine",
+                        "DType",
+                        "Error",
+                        "Int",
+                        "List",
+                        "ListLiteral",
+                        "Scalar",
+                        "Int8",
+                        "UInt8",
+                        "Int16",
+                        "UInt16",
+                        "Int32",
+                        "UInt32",
+                        "Int64",
+                        "UInt64",
+                        "BFloat16",
+                        "Float16",
+                        "Float32",
+                        "Float64",
+                        "SIMD",
+                        "String",
+                        "Tensor",
+                        "Tuple",
+                        "Movable",
+                        "Copyable",
+                        "CollectionElement",
+                    ),
+                    prefix=r"(?>',
+    # Binary augmented
+    '+=', '-=', '*=', '/=', '%=', '**=', '&=', '|=', '^=', '<<=', '>>=',
+    # Comparison
+    '==', '!=', '<', '<=', '>', '>=', '<=>',
+    # Patterns and assignment
+    ':=', '?', '=~', '!~', '=>',
+    # Calls and sends
+    '.', '<-', '->',
+]
+_escape_pattern = (
+    r'(?:\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|'
+    r'\\["\'\\bftnr])')
+# _char = _escape_chars + [('.', String.Char)]
+_identifier = r'[_a-zA-Z]\w*'
+
+_constants = [
+    # Void constants
+    'null',
+    # Bool constants
+    'false', 'true',
+    # Double constants
+    'Infinity', 'NaN',
+    # Special objects
+    'M', 'Ref', 'throw', 'traceln',
+]
+
+_guards = [
+    'Any', 'Binding', 'Bool', 'Bytes', 'Char', 'DeepFrozen', 'Double',
+    'Empty', 'Int', 'List', 'Map', 'Near', 'NullOk', 'Same', 'Selfless',
+    'Set', 'Str', 'SubrangeGuard', 'Transparent', 'Void',
+]
+
+_safeScope = [
+    '_accumulateList', '_accumulateMap', '_auditedBy', '_bind',
+    '_booleanFlow', '_comparer', '_equalizer', '_iterForever', '_loop',
+    '_makeBytes', '_makeDouble', '_makeFinalSlot', '_makeInt', '_makeList',
+    '_makeMap', '_makeMessageDesc', '_makeOrderedSpace', '_makeParamDesc',
+    '_makeProtocolDesc', '_makeSourceSpan', '_makeString', '_makeVarSlot',
+    '_makeVerbFacet', '_mapExtract', '_matchSame', '_quasiMatcher',
+    '_slotToBinding', '_splitList', '_suchThat', '_switchFailed',
+    '_validateFor', 'b__quasiParser', 'eval', 'import', 'm__quasiParser',
+    'makeBrandPair', 'makeLazySlot', 'safeScope', 'simple__quasiParser',
+]
+
+
+class MonteLexer(RegexLexer):
+    """
+    Lexer for the Monte programming language.
+    """
+    name = 'Monte'
+    url = 'https://monte.readthedocs.io/'
+    aliases = ['monte']
+    filenames = ['*.mt']
+    version_added = '2.2'
+
+    tokens = {
+        'root': [
+            # Comments
+            (r'#[^\n]*\n', Comment),
+
+            # Docstrings
+            # Apologies for the non-greedy matcher here.
+            (r'/\*\*.*?\*/', String.Doc),
+
+            # `var` declarations
+            (r'\bvar\b', Keyword.Declaration, 'var'),
+
+            # `interface` declarations
+            (r'\binterface\b', Keyword.Declaration, 'interface'),
+
+            # method declarations
+            (words(_methods, prefix='\\b', suffix='\\b'),
+             Keyword, 'method'),
+
+            # All other declarations
+            (words(_declarations, prefix='\\b', suffix='\\b'),
+             Keyword.Declaration),
+
+            # Keywords
+            (words(_keywords, prefix='\\b', suffix='\\b'), Keyword),
+
+            # Literals
+            ('[+-]?0x[_0-9a-fA-F]+', Number.Hex),
+            (r'[+-]?[_0-9]+\.[_0-9]*([eE][+-]?[_0-9]+)?', Number.Float),
+            ('[+-]?[_0-9]+', Number.Integer),
+            ("'", String.Double, 'char'),
+            ('"', String.Double, 'string'),
+
+            # Quasiliterals
+            ('`', String.Backtick, 'ql'),
+
+            # Operators
+            (words(_operators), Operator),
+
+            # Verb operators
+            (_identifier + '=', Operator.Word),
+
+            # Safe scope constants
+            (words(_constants, prefix='\\b', suffix='\\b'),
+             Keyword.Pseudo),
+
+            # Safe scope guards
+            (words(_guards, prefix='\\b', suffix='\\b'), Keyword.Type),
+
+            # All other safe scope names
+            (words(_safeScope, prefix='\\b', suffix='\\b'),
+             Name.Builtin),
+
+            # Identifiers
+            (_identifier, Name),
+
+            # Punctuation
+            (r'\(|\)|\{|\}|\[|\]|:|,', Punctuation),
+
+            # Whitespace
+            (' +', Whitespace),
+
+            # Definite lexer errors
+            ('=', Error),
+        ],
+        'char': [
+            # It is definitely an error to have a char of width == 0.
+            ("'", Error, 'root'),
+            (_escape_pattern, String.Escape, 'charEnd'),
+            ('.', String.Char, 'charEnd'),
+        ],
+        'charEnd': [
+            ("'", String.Char, '#pop:2'),
+            # It is definitely an error to have a char of width > 1.
+            ('.', Error),
+        ],
+        # The state of things coming into an interface.
+        'interface': [
+            (' +', Whitespace),
+            (_identifier, Name.Class, '#pop'),
+            include('root'),
+        ],
+        # The state of things coming into a method.
+        'method': [
+            (' +', Whitespace),
+            (_identifier, Name.Function, '#pop'),
+            include('root'),
+        ],
+        'string': [
+            ('"', String.Double, 'root'),
+            (_escape_pattern, String.Escape),
+            (r'\n', String.Double),
+            ('.', String.Double),
+        ],
+        'ql': [
+            ('`', String.Backtick, 'root'),
+            (r'\$' + _escape_pattern, String.Escape),
+            (r'\$\$', String.Escape),
+            (r'@@', String.Escape),
+            (r'\$\{', String.Interpol, 'qlNest'),
+            (r'@\{', String.Interpol, 'qlNest'),
+            (r'\$' + _identifier, Name),
+            ('@' + _identifier, Name),
+            ('.', String.Backtick),
+        ],
+        'qlNest': [
+            (r'\}', String.Interpol, '#pop'),
+            include('root'),
+        ],
+        # The state of things immediately following `var`.
+        'var': [
+            (' +', Whitespace),
+            (_identifier, Name.Variable, '#pop'),
+            include('root'),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/mosel.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/mosel.py
new file mode 100644
index 0000000000000000000000000000000000000000..912f478773cb575412a937448e709bef101069e3
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/mosel.py
@@ -0,0 +1,447 @@
+"""
+    pygments.lexers.mosel
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the mosel language.
+    http://www.fico.com/en/products/fico-xpress-optimization
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['MoselLexer']
+
+FUNCTIONS = (
+    # core functions
+    '_',
+    'abs',
+    'arctan',
+    'asproc',
+    'assert',
+    'bitflip',
+    'bitneg',
+    'bitset',
+    'bitshift',
+    'bittest',
+    'bitval',
+    'ceil',
+    'cos',
+    'create',
+    'currentdate',
+    'currenttime',
+    'cutelt',
+    'cutfirst',
+    'cuthead',
+    'cutlast',
+    'cuttail',
+    'datablock',
+    'delcell',
+    'exists',
+    'exit',
+    'exp',
+    'exportprob',
+    'fclose',
+    'fflush',
+    'finalize',
+    'findfirst',
+    'findlast',
+    'floor',
+    'fopen',
+    'fselect',
+    'fskipline',
+    'fwrite',
+    'fwrite_',
+    'fwriteln',
+    'fwriteln_',
+    'getact',
+    'getcoeff',
+    'getcoeffs',
+    'getdual',
+    'getelt',
+    'getfid',
+    'getfirst',
+    'getfname',
+    'gethead',
+    'getlast',
+    'getobjval',
+    'getparam',
+    'getrcost',
+    'getreadcnt',
+    'getreverse',
+    'getsize',
+    'getslack',
+    'getsol',
+    'gettail',
+    'gettype',
+    'getvars',
+    'isdynamic',
+    'iseof',
+    'isfinite',
+    'ishidden',
+    'isinf',
+    'isnan',
+    'isodd',
+    'ln',
+    'localsetparam',
+    'log',
+    'makesos1',
+    'makesos2',
+    'maxlist',
+    'memoryuse',
+    'minlist',
+    'newmuid',
+    'publish',
+    'random',
+    'read',
+    'readln',
+    'reset',
+    'restoreparam',
+    'reverse',
+    'round',
+    'setcoeff',
+    'sethidden',
+    'setioerr',
+    'setmatherr',
+    'setname',
+    'setparam',
+    'setrandseed',
+    'setrange',
+    'settype',
+    'sin',
+    'splithead',
+    'splittail',
+    'sqrt',
+    'strfmt',
+    'substr',
+    'timestamp',
+    'unpublish',
+    'versionnum',
+    'versionstr',
+    'write',
+    'write_',
+    'writeln',
+    'writeln_',
+
+    # mosel exam mmxprs | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u
+    'addcut',
+    'addcuts',
+    'addmipsol',
+    'basisstability',
+    'calcsolinfo',
+    'clearmipdir',
+    'clearmodcut',
+    'command',
+    'copysoltoinit',
+    'crossoverlpsol',
+    'defdelayedrows',
+    'defsecurevecs',
+    'delcuts',
+    'dropcuts',
+    'estimatemarginals',
+    'fixglobal',
+    'flushmsgq',
+    'getbstat',
+    'getcnlist',
+    'getcplist',
+    'getdualray',
+    'getiis',
+    'getiissense',
+    'getiistype',
+    'getinfcause',
+    'getinfeas',
+    'getlb',
+    'getlct',
+    'getleft',
+    'getloadedlinctrs',
+    'getloadedmpvars',
+    'getname',
+    'getprimalray',
+    'getprobstat',
+    'getrange',
+    'getright',
+    'getsensrng',
+    'getsize',
+    'getsol',
+    'gettype',
+    'getub',
+    'getvars',
+    'gety',
+    'hasfeature',
+    'implies',
+    'indicator',
+    'initglobal',
+    'ishidden',
+    'isiisvalid',
+    'isintegral',
+    'loadbasis',
+    'loadcuts',
+    'loadlpsol',
+    'loadmipsol',
+    'loadprob',
+    'maximise',
+    'maximize',
+    'minimise',
+    'minimize',
+    'postsolve',
+    'readbasis',
+    'readdirs',
+    'readsol',
+    'refinemipsol',
+    'rejectintsol',
+    'repairinfeas',
+    'repairinfeas_deprec',
+    'resetbasis',
+    'resetiis',
+    'resetsol',
+    'savebasis',
+    'savemipsol',
+    'savesol',
+    'savestate',
+    'selectsol',
+    'setarchconsistency',
+    'setbstat',
+    'setcallback',
+    'setcbcutoff',
+    'setgndata',
+    'sethidden',
+    'setlb',
+    'setmipdir',
+    'setmodcut',
+    'setsol',
+    'setub',
+    'setucbdata',
+    'stopoptimise',
+    'stopoptimize',
+    'storecut',
+    'storecuts',
+    'unloadprob',
+    'uselastbarsol',
+    'writebasis',
+    'writedirs',
+    'writeprob',
+    'writesol',
+    'xor',
+    'xprs_addctr',
+    'xprs_addindic',
+
+    # mosel exam mmsystem | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u
+    'addmonths',
+    'copytext',
+    'cuttext',
+    'deltext',
+    'endswith',
+    'erase',
+    'expandpath',
+    'fcopy',
+    'fdelete',
+    'findfiles',
+    'findtext',
+    'fmove',
+    'formattext',
+    'getasnumber',
+    'getchar',
+    'getcwd',
+    'getdate',
+    'getday',
+    'getdaynum',
+    'getdays',
+    'getdirsep',
+    'getdsoparam',
+    'getendparse',
+    'getenv',
+    'getfsize',
+    'getfstat',
+    'getftime',
+    'gethour',
+    'getminute',
+    'getmonth',
+    'getmsec',
+    'getoserrmsg',
+    'getoserror',
+    'getpathsep',
+    'getqtype',
+    'getsecond',
+    'getsepchar',
+    'getsize',
+    'getstart',
+    'getsucc',
+    'getsysinfo',
+    'getsysstat',
+    'gettime',
+    'gettmpdir',
+    'gettrim',
+    'getweekday',
+    'getyear',
+    'inserttext',
+    'isvalid',
+    'jointext',
+    'makedir',
+    'makepath',
+    'newtar',
+    'newzip',
+    'nextfield',
+    'openpipe',
+    'parseextn',
+    'parseint',
+    'parsereal',
+    'parsetext',
+    'pastetext',
+    'pathmatch',
+    'pathsplit',
+    'qsort',
+    'quote',
+    'readtextline',
+    'regmatch',
+    'regreplace',
+    'removedir',
+    'removefiles',
+    'setchar',
+    'setdate',
+    'setday',
+    'setdsoparam',
+    'setendparse',
+    'setenv',
+    'sethour',
+    'setminute',
+    'setmonth',
+    'setmsec',
+    'setoserror',
+    'setqtype',
+    'setsecond',
+    'setsepchar',
+    'setstart',
+    'setsucc',
+    'settime',
+    'settrim',
+    'setyear',
+    'sleep',
+    'splittext',
+    'startswith',
+    'system',
+    'tarlist',
+    'textfmt',
+    'tolower',
+    'toupper',
+    'trim',
+    'untar',
+    'unzip',
+    'ziplist',
+
+    # mosel exam mmjobs | sed -n -e "s/ [pf][a-z]* \([a-zA-Z0-9_]*\).*/'\1',/p" | sort -u
+    'canceltimer',
+    'clearaliases',
+    'compile',
+    'connect',
+    'detach',
+    'disconnect',
+    'dropnextevent',
+    'findxsrvs',
+    'getaliases',
+    'getannidents',
+    'getannotations',
+    'getbanner',
+    'getclass',
+    'getdsoprop',
+    'getdsopropnum',
+    'getexitcode',
+    'getfromgid',
+    'getfromid',
+    'getfromuid',
+    'getgid',
+    'gethostalias',
+    'getid',
+    'getmodprop',
+    'getmodpropnum',
+    'getnextevent',
+    'getnode',
+    'getrmtid',
+    'getstatus',
+    'getsysinfo',
+    'gettimer',
+    'getuid',
+    'getvalue',
+    'isqueueempty',
+    'load',
+    'nullevent',
+    'peeknextevent',
+    'resetmodpar',
+    'run',
+    'send',
+    'setcontrol',
+    'setdefstream',
+    'setgid',
+    'sethostalias',
+    'setmodpar',
+    'settimer',
+    'setuid',
+    'setworkdir',
+    'stop',
+    'unload',
+    'wait',
+    'waitexpired',
+    'waitfor',
+    'waitforend',
+)
+
+
+class MoselLexer(RegexLexer):
+    """
+    For the Mosel optimization language.
+    """
+    name = 'Mosel'
+    aliases = ['mosel']
+    filenames = ['*.mos']
+    url = 'https://www.fico.com/fico-xpress-optimization/docs/latest/mosel/mosel_lang/dhtml/moselreflang.html'
+    version_added = '2.6'
+
+    tokens = {
+        'root': [
+            (r'\n', Text),
+            (r'\s+', Text.Whitespace),
+            (r'!.*?\n', Comment.Single),
+            (r'\(!(.|\n)*?!\)', Comment.Multiline),
+            (words((
+                'and', 'as', 'break', 'case', 'count', 'declarations', 'do',
+                'dynamic', 'elif', 'else', 'end-', 'end', 'evaluation', 'false',
+                'forall', 'forward', 'from', 'function', 'hashmap', 'if',
+                'imports', 'include', 'initialisations', 'initializations', 'inter',
+                'max', 'min', 'model', 'namespace', 'next', 'not', 'nsgroup',
+                'nssearch', 'of', 'options', 'or', 'package', 'parameters',
+                'procedure', 'public', 'prod', 'record', 'repeat', 'requirements',
+                'return', 'sum', 'then', 'to', 'true', 'union', 'until', 'uses',
+                'version', 'while', 'with'), prefix=r'\b', suffix=r'\b'),
+             Keyword.Builtin),
+            (words((
+                'range', 'array', 'set', 'list', 'mpvar', 'mpproblem', 'linctr',
+                'nlctr', 'integer', 'string', 'real', 'boolean', 'text', 'time',
+                'date', 'datetime', 'returned', 'Model', 'Mosel', 'counter',
+                'xmldoc', 'is_sos1', 'is_sos2', 'is_integer', 'is_binary',
+                'is_continuous', 'is_free', 'is_semcont', 'is_semint',
+                'is_partint'), prefix=r'\b', suffix=r'\b'),
+             Keyword.Type),
+            (r'(\+|\-|\*|/|=|<=|>=|\||\^|<|>|<>|\.\.|\.|:=|::|:|in|mod|div)',
+             Operator),
+            (r'[()\[\]{},;]+', Punctuation),
+            (words(FUNCTIONS,  prefix=r'\b', suffix=r'\b'), Name.Function),
+            (r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float),
+            (r'\d+([eE][+-]?\d+)?', Number.Integer),
+            (r'[+-]?Infinity', Number.Integer),
+            (r'0[xX][0-9a-fA-F]+', Number),
+            (r'"', String.Double, 'double_quote'),
+            (r'\'', String.Single, 'single_quote'),
+            (r'(\w+|(\.(?!\.)))', Text),
+        ],
+        'single_quote': [
+            (r'\'', String.Single, '#pop'),
+            (r'[^\']+', String.Single),
+        ],
+        'double_quote': [
+            (r'(\\"|\\[0-7]{1,3}\D|\\[abfnrtv]|\\\\)', String.Escape),
+            (r'\"', String.Double, '#pop'),
+            (r'[^"\\]+', String.Double),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ncl.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ncl.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2729efba6e23c1d9d3ca86c8971649f2e964c17
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ncl.py
@@ -0,0 +1,894 @@
+"""
+    pygments.lexers.ncl
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for NCAR Command Language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['NCLLexer']
+
+
+class NCLLexer(RegexLexer):
+    """
+    Lexer for NCL code.
+    """
+    name = 'NCL'
+    aliases = ['ncl']
+    filenames = ['*.ncl']
+    mimetypes = ['text/ncl']
+    url = 'https://www.ncl.ucar.edu'
+    version_added = '2.2'
+
+    flags = re.MULTILINE
+
+    tokens = {
+        'root': [
+            (r';.*\n', Comment),
+            include('strings'),
+            include('core'),
+            (r'[a-zA-Z_]\w*', Name),
+            include('nums'),
+            (r'[\s]+', Text),
+        ],
+        'core': [
+            # Statements
+            (words((
+                'begin', 'break', 'continue', 'create', 'defaultapp', 'do',
+                'else', 'end', 'external', 'exit', 'True', 'False', 'file', 'function',
+                'getvalues', 'graphic', 'group', 'if', 'list', 'load', 'local',
+                'new', '_Missing', 'Missing', 'noparent', 'procedure',
+                'quit', 'QUIT', 'Quit', 'record', 'return', 'setvalues', 'stop',
+                'then', 'while'), prefix=r'\b', suffix=r'\s*\b'),
+             Keyword),
+
+            # Data Types
+            (words((
+                'ubyte', 'uint', 'uint64', 'ulong', 'string', 'byte',
+                'character', 'double', 'float', 'integer', 'int64', 'logical',
+                'long', 'short', 'ushort', 'enumeric', 'numeric', 'snumeric'),
+                prefix=r'\b', suffix=r'\s*\b'),
+             Keyword.Type),
+
+            # Operators
+            (r'[\%^*+\-/<>]', Operator),
+
+            # punctuation:
+            (r'[\[\]():@$!&|.,\\{}]', Punctuation),
+            (r'[=:]', Punctuation),
+
+            # Intrinsics
+            (words((
+                'abs', 'acos', 'addfile', 'addfiles', 'all', 'angmom_atm', 'any',
+                'area_conserve_remap', 'area_hi2lores', 'area_poly_sphere',
+                'asciiread', 'asciiwrite', 'asin', 'atan', 'atan2', 'attsetvalues',
+                'avg', 'betainc', 'bin_avg', 'bin_sum', 'bw_bandpass_filter',
+                'cancor', 'cbinread', 'cbinwrite', 'cd_calendar', 'cd_inv_calendar',
+                'cdfbin_p', 'cdfbin_pr', 'cdfbin_s', 'cdfbin_xn', 'cdfchi_p',
+                'cdfchi_x', 'cdfgam_p', 'cdfgam_x', 'cdfnor_p', 'cdfnor_x',
+                'cdft_p', 'cdft_t', 'ceil', 'center_finite_diff',
+                'center_finite_diff_n', 'cfftb', 'cfftf', 'cfftf_frq_reorder',
+                'charactertodouble', 'charactertofloat', 'charactertointeger',
+                'charactertolong', 'charactertoshort', 'charactertostring',
+                'chartodouble', 'chartofloat', 'chartoint', 'chartointeger',
+                'chartolong', 'chartoshort', 'chartostring', 'chiinv', 'clear',
+                'color_index_to_rgba', 'conform', 'conform_dims', 'cos', 'cosh',
+                'count_unique_values', 'covcorm', 'covcorm_xy', 'craybinnumrec',
+                'craybinrecread', 'create_graphic', 'csa1', 'csa1d', 'csa1s',
+                'csa1x', 'csa1xd', 'csa1xs', 'csa2', 'csa2d', 'csa2l', 'csa2ld',
+                'csa2ls', 'csa2lx', 'csa2lxd', 'csa2lxs', 'csa2s', 'csa2x',
+                'csa2xd', 'csa2xs', 'csa3', 'csa3d', 'csa3l', 'csa3ld', 'csa3ls',
+                'csa3lx', 'csa3lxd', 'csa3lxs', 'csa3s', 'csa3x', 'csa3xd',
+                'csa3xs', 'csc2s', 'csgetp', 'css2c', 'cssetp', 'cssgrid', 'csstri',
+                'csvoro', 'cumsum', 'cz2ccm', 'datatondc', 'day_of_week',
+                'day_of_year', 'days_in_month', 'default_fillvalue', 'delete',
+                'depth_to_pres', 'destroy', 'determinant', 'dewtemp_trh',
+                'dgeevx_lapack', 'dim_acumrun_n', 'dim_avg', 'dim_avg_n',
+                'dim_avg_wgt', 'dim_avg_wgt_n', 'dim_cumsum', 'dim_cumsum_n',
+                'dim_gamfit_n', 'dim_gbits', 'dim_max', 'dim_max_n', 'dim_median',
+                'dim_median_n', 'dim_min', 'dim_min_n', 'dim_num', 'dim_num_n',
+                'dim_numrun_n', 'dim_pqsort', 'dim_pqsort_n', 'dim_product',
+                'dim_product_n', 'dim_rmsd', 'dim_rmsd_n', 'dim_rmvmean',
+                'dim_rmvmean_n', 'dim_rmvmed', 'dim_rmvmed_n', 'dim_spi_n',
+                'dim_standardize', 'dim_standardize_n', 'dim_stat4', 'dim_stat4_n',
+                'dim_stddev', 'dim_stddev_n', 'dim_sum', 'dim_sum_n', 'dim_sum_wgt',
+                'dim_sum_wgt_n', 'dim_variance', 'dim_variance_n', 'dimsizes',
+                'doubletobyte', 'doubletochar', 'doubletocharacter',
+                'doubletofloat', 'doubletoint', 'doubletointeger', 'doubletolong',
+                'doubletoshort', 'dpres_hybrid_ccm', 'dpres_plevel', 'draw',
+                'draw_color_palette', 'dsgetp', 'dsgrid2', 'dsgrid2d', 'dsgrid2s',
+                'dsgrid3', 'dsgrid3d', 'dsgrid3s', 'dspnt2', 'dspnt2d', 'dspnt2s',
+                'dspnt3', 'dspnt3d', 'dspnt3s', 'dssetp', 'dtrend', 'dtrend_msg',
+                'dtrend_msg_n', 'dtrend_n', 'dtrend_quadratic',
+                'dtrend_quadratic_msg_n', 'dv2uvf', 'dv2uvg', 'dz_height',
+                'echo_off', 'echo_on', 'eof2data', 'eof_varimax', 'eofcor',
+                'eofcor_pcmsg', 'eofcor_ts', 'eofcov', 'eofcov_pcmsg', 'eofcov_ts',
+                'eofunc', 'eofunc_ts', 'eofunc_varimax', 'equiv_sample_size', 'erf',
+                'erfc', 'esacr', 'esacv', 'esccr', 'esccv', 'escorc', 'escorc_n',
+                'escovc', 'exit', 'exp', 'exp_tapersh', 'exp_tapersh_wgts',
+                'exp_tapershC', 'ezfftb', 'ezfftb_n', 'ezfftf', 'ezfftf_n',
+                'f2fosh', 'f2foshv', 'f2fsh', 'f2fshv', 'f2gsh', 'f2gshv', 'fabs',
+                'fbindirread', 'fbindirwrite', 'fbinnumrec', 'fbinread',
+                'fbinrecread', 'fbinrecwrite', 'fbinwrite', 'fft2db', 'fft2df',
+                'fftshift', 'fileattdef', 'filechunkdimdef', 'filedimdef',
+                'fileexists', 'filegrpdef', 'filevarattdef', 'filevarchunkdef',
+                'filevarcompressleveldef', 'filevardef', 'filevardimsizes',
+                'filwgts_lancos', 'filwgts_lanczos', 'filwgts_normal',
+                'floattobyte', 'floattochar', 'floattocharacter', 'floattoint',
+                'floattointeger', 'floattolong', 'floattoshort', 'floor',
+                'fluxEddy', 'fo2fsh', 'fo2fshv', 'fourier_info', 'frame', 'fspan',
+                'ftcurv', 'ftcurvd', 'ftcurvi', 'ftcurvp', 'ftcurvpi', 'ftcurvps',
+                'ftcurvs', 'ftest', 'ftgetp', 'ftkurv', 'ftkurvd', 'ftkurvp',
+                'ftkurvpd', 'ftsetp', 'ftsurf', 'g2fsh', 'g2fshv', 'g2gsh',
+                'g2gshv', 'gamma', 'gammainc', 'gaus', 'gaus_lobat',
+                'gaus_lobat_wgt', 'gc_aangle', 'gc_clkwise', 'gc_dangle',
+                'gc_inout', 'gc_latlon', 'gc_onarc', 'gc_pnt2gc', 'gc_qarea',
+                'gc_tarea', 'generate_2d_array', 'get_color_index',
+                'get_color_rgba', 'get_cpu_time', 'get_isolines', 'get_ncl_version',
+                'get_script_name', 'get_script_prefix_name', 'get_sphere_radius',
+                'get_unique_values', 'getbitsone', 'getenv', 'getfiledimsizes',
+                'getfilegrpnames', 'getfilepath', 'getfilevaratts',
+                'getfilevarchunkdimsizes', 'getfilevardims', 'getfilevardimsizes',
+                'getfilevarnames', 'getfilevartypes', 'getvaratts', 'getvardims',
+                'gradsf', 'gradsg', 'greg2jul', 'grid2triple', 'hlsrgb', 'hsvrgb',
+                'hydro', 'hyi2hyo', 'idsfft', 'igradsf', 'igradsg', 'ilapsf',
+                'ilapsg', 'ilapvf', 'ilapvg', 'ind', 'ind_resolve', 'int2p',
+                'int2p_n', 'integertobyte', 'integertochar', 'integertocharacter',
+                'integertoshort', 'inttobyte', 'inttochar', 'inttoshort',
+                'inverse_matrix', 'isatt', 'isbigendian', 'isbyte', 'ischar',
+                'iscoord', 'isdefined', 'isdim', 'isdimnamed', 'isdouble',
+                'isenumeric', 'isfile', 'isfilepresent', 'isfilevar',
+                'isfilevaratt', 'isfilevarcoord', 'isfilevardim', 'isfloat',
+                'isfunc', 'isgraphic', 'isint', 'isint64', 'isinteger',
+                'isleapyear', 'islogical', 'islong', 'ismissing', 'isnan_ieee',
+                'isnumeric', 'ispan', 'isproc', 'isshort', 'issnumeric', 'isstring',
+                'isubyte', 'isuint', 'isuint64', 'isulong', 'isunlimited',
+                'isunsigned', 'isushort', 'isvar', 'jul2greg', 'kmeans_as136',
+                'kolsm2_n', 'kron_product', 'lapsf', 'lapsg', 'lapvf', 'lapvg',
+                'latlon2utm', 'lclvl', 'lderuvf', 'lderuvg', 'linint1', 'linint1_n',
+                'linint2', 'linint2_points', 'linmsg', 'linmsg_n', 'linrood_latwgt',
+                'linrood_wgt', 'list_files', 'list_filevars', 'list_hlus',
+                'list_procfuncs', 'list_vars', 'ListAppend', 'ListCount',
+                'ListGetType', 'ListIndex', 'ListIndexFromName', 'ListPop',
+                'ListPush', 'ListSetType', 'loadscript', 'local_max', 'local_min',
+                'log', 'log10', 'longtobyte', 'longtochar', 'longtocharacter',
+                'longtoint', 'longtointeger', 'longtoshort', 'lspoly', 'lspoly_n',
+                'mask', 'max', 'maxind', 'min', 'minind', 'mixed_layer_depth',
+                'mixhum_ptd', 'mixhum_ptrh', 'mjo_cross_coh2pha',
+                'mjo_cross_segment', 'moc_globe_atl', 'monthday', 'natgrid',
+                'natgridd', 'natgrids', 'ncargpath', 'ncargversion', 'ndctodata',
+                'ndtooned', 'new', 'NewList', 'ngezlogo', 'nggcog', 'nggetp',
+                'nglogo', 'ngsetp', 'NhlAddAnnotation', 'NhlAddData',
+                'NhlAddOverlay', 'NhlAddPrimitive', 'NhlAppGetDefaultParentId',
+                'NhlChangeWorkstation', 'NhlClassName', 'NhlClearWorkstation',
+                'NhlDataPolygon', 'NhlDataPolyline', 'NhlDataPolymarker',
+                'NhlDataToNDC', 'NhlDestroy', 'NhlDraw', 'NhlFrame', 'NhlFreeColor',
+                'NhlGetBB', 'NhlGetClassResources', 'NhlGetErrorObjectId',
+                'NhlGetNamedColorIndex', 'NhlGetParentId',
+                'NhlGetParentWorkstation', 'NhlGetWorkspaceObjectId',
+                'NhlIsAllocatedColor', 'NhlIsApp', 'NhlIsDataComm', 'NhlIsDataItem',
+                'NhlIsDataSpec', 'NhlIsTransform', 'NhlIsView', 'NhlIsWorkstation',
+                'NhlName', 'NhlNDCPolygon', 'NhlNDCPolyline', 'NhlNDCPolymarker',
+                'NhlNDCToData', 'NhlNewColor', 'NhlNewDashPattern', 'NhlNewMarker',
+                'NhlPalGetDefined', 'NhlRemoveAnnotation', 'NhlRemoveData',
+                'NhlRemoveOverlay', 'NhlRemovePrimitive', 'NhlSetColor',
+                'NhlSetDashPattern', 'NhlSetMarker', 'NhlUpdateData',
+                'NhlUpdateWorkstation', 'nice_mnmxintvl', 'nngetaspectd',
+                'nngetaspects', 'nngetp', 'nngetsloped', 'nngetslopes', 'nngetwts',
+                'nngetwtsd', 'nnpnt', 'nnpntd', 'nnpntend', 'nnpntendd',
+                'nnpntinit', 'nnpntinitd', 'nnpntinits', 'nnpnts', 'nnsetp', 'num',
+                'obj_anal_ic', 'omega_ccm', 'onedtond', 'overlay', 'paleo_outline',
+                'pdfxy_bin', 'poisson_grid_fill', 'pop_remap', 'potmp_insitu_ocn',
+                'prcwater_dp', 'pres2hybrid', 'pres_hybrid_ccm', 'pres_sigma',
+                'print', 'print_table', 'printFileVarSummary', 'printVarSummary',
+                'product', 'pslec', 'pslhor', 'pslhyp', 'qsort', 'rand',
+                'random_chi', 'random_gamma', 'random_normal', 'random_setallseed',
+                'random_uniform', 'rcm2points', 'rcm2rgrid', 'rdsstoi',
+                'read_colormap_file', 'reg_multlin', 'regcoef', 'regCoef_n',
+                'regline', 'relhum', 'replace_ieeenan', 'reshape', 'reshape_ind',
+                'rgba_to_color_index', 'rgbhls', 'rgbhsv', 'rgbyiq', 'rgrid2rcm',
+                'rhomb_trunc', 'rip_cape_2d', 'rip_cape_3d', 'round', 'rtest',
+                'runave', 'runave_n', 'set_default_fillvalue', 'set_sphere_radius',
+                'setfileoption', 'sfvp2uvf', 'sfvp2uvg', 'shaec', 'shagc',
+                'shgetnp', 'shgetp', 'shgrid', 'shorttobyte', 'shorttochar',
+                'shorttocharacter', 'show_ascii', 'shsec', 'shsetp', 'shsgc',
+                'shsgc_R42', 'sigma2hybrid', 'simpeq', 'simpne', 'sin',
+                'sindex_yrmo', 'sinh', 'sizeof', 'sleep', 'smth9', 'snindex_yrmo',
+                'solve_linsys', 'span_color_indexes', 'span_color_rgba',
+                'sparse_matrix_mult', 'spcorr', 'spcorr_n', 'specx_anal',
+                'specxy_anal', 'spei', 'sprintf', 'sprinti', 'sqrt', 'sqsort',
+                'srand', 'stat2', 'stat4', 'stat_medrng', 'stat_trim',
+                'status_exit', 'stdatmus_p2tdz', 'stdatmus_z2tdp', 'stddev',
+                'str_capital', 'str_concat', 'str_fields_count', 'str_get_cols',
+                'str_get_dq', 'str_get_field', 'str_get_nl', 'str_get_sq',
+                'str_get_tab', 'str_index_of_substr', 'str_insert', 'str_is_blank',
+                'str_join', 'str_left_strip', 'str_lower', 'str_match',
+                'str_match_ic', 'str_match_ic_regex', 'str_match_ind',
+                'str_match_ind_ic', 'str_match_ind_ic_regex', 'str_match_ind_regex',
+                'str_match_regex', 'str_right_strip', 'str_split',
+                'str_split_by_length', 'str_split_csv', 'str_squeeze', 'str_strip',
+                'str_sub_str', 'str_switch', 'str_upper', 'stringtochar',
+                'stringtocharacter', 'stringtodouble', 'stringtofloat',
+                'stringtoint', 'stringtointeger', 'stringtolong', 'stringtoshort',
+                'strlen', 'student_t', 'sum', 'svd_lapack', 'svdcov', 'svdcov_sv',
+                'svdstd', 'svdstd_sv', 'system', 'systemfunc', 'tan', 'tanh',
+                'taper', 'taper_n', 'tdclrs', 'tdctri', 'tdcudp', 'tdcurv',
+                'tddtri', 'tdez2d', 'tdez3d', 'tdgetp', 'tdgrds', 'tdgrid',
+                'tdgtrs', 'tdinit', 'tditri', 'tdlbla', 'tdlblp', 'tdlbls',
+                'tdline', 'tdlndp', 'tdlnpa', 'tdlpdp', 'tdmtri', 'tdotri',
+                'tdpara', 'tdplch', 'tdprpa', 'tdprpi', 'tdprpt', 'tdsetp',
+                'tdsort', 'tdstri', 'tdstrs', 'tdttri', 'thornthwaite', 'tobyte',
+                'tochar', 'todouble', 'tofloat', 'toint', 'toint64', 'tointeger',
+                'tolong', 'toshort', 'tosigned', 'tostring', 'tostring_with_format',
+                'totype', 'toubyte', 'touint', 'touint64', 'toulong', 'tounsigned',
+                'toushort', 'trend_manken', 'tri_trunc', 'triple2grid',
+                'triple2grid2d', 'trop_wmo', 'ttest', 'typeof', 'undef',
+                'unique_string', 'update', 'ushorttoint', 'ut_calendar',
+                'ut_inv_calendar', 'utm2latlon', 'uv2dv_cfd', 'uv2dvf', 'uv2dvg',
+                'uv2sfvpf', 'uv2sfvpg', 'uv2vr_cfd', 'uv2vrdvf', 'uv2vrdvg',
+                'uv2vrf', 'uv2vrg', 'v5d_close', 'v5d_create', 'v5d_setLowLev',
+                'v5d_setUnits', 'v5d_write', 'v5d_write_var', 'variance', 'vhaec',
+                'vhagc', 'vhsec', 'vhsgc', 'vibeta', 'vinth2p', 'vinth2p_ecmwf',
+                'vinth2p_ecmwf_nodes', 'vinth2p_nodes', 'vintp2p_ecmwf', 'vr2uvf',
+                'vr2uvg', 'vrdv2uvf', 'vrdv2uvg', 'wavelet', 'wavelet_default',
+                'weibull', 'wgt_area_smooth', 'wgt_areaave', 'wgt_areaave2',
+                'wgt_arearmse', 'wgt_arearmse2', 'wgt_areasum2', 'wgt_runave',
+                'wgt_runave_n', 'wgt_vert_avg_beta', 'wgt_volave', 'wgt_volave_ccm',
+                'wgt_volrmse', 'wgt_volrmse_ccm', 'where', 'wk_smooth121', 'wmbarb',
+                'wmbarbmap', 'wmdrft', 'wmgetp', 'wmlabs', 'wmsetp', 'wmstnm',
+                'wmvect', 'wmvectmap', 'wmvlbl', 'wrf_avo', 'wrf_cape_2d',
+                'wrf_cape_3d', 'wrf_dbz', 'wrf_eth', 'wrf_helicity', 'wrf_ij_to_ll',
+                'wrf_interp_1d', 'wrf_interp_2d_xy', 'wrf_interp_3d_z',
+                'wrf_latlon_to_ij', 'wrf_ll_to_ij', 'wrf_omega', 'wrf_pvo',
+                'wrf_rh', 'wrf_slp', 'wrf_smooth_2d', 'wrf_td', 'wrf_tk',
+                'wrf_updraft_helicity', 'wrf_uvmet', 'wrf_virtual_temp',
+                'wrf_wetbulb', 'wrf_wps_close_int', 'wrf_wps_open_int',
+                'wrf_wps_rddata_int', 'wrf_wps_rdhead_int', 'wrf_wps_read_int',
+                'wrf_wps_write_int', 'write_matrix', 'write_table', 'yiqrgb',
+                'z2geouv', 'zonal_mpsi', 'addfiles_GetVar', 'advect_variable',
+                'area_conserve_remap_Wrap', 'area_hi2lores_Wrap',
+                'array_append_record', 'assignFillValue', 'byte2flt',
+                'byte2flt_hdf', 'calcDayAnomTLL', 'calcMonAnomLLLT',
+                'calcMonAnomLLT', 'calcMonAnomTLL', 'calcMonAnomTLLL',
+                'calculate_monthly_values', 'cd_convert', 'changeCase',
+                'changeCaseChar', 'clmDayTLL', 'clmDayTLLL', 'clmMon2clmDay',
+                'clmMonLLLT', 'clmMonLLT', 'clmMonTLL', 'clmMonTLLL', 'closest_val',
+                'copy_VarAtts', 'copy_VarCoords', 'copy_VarCoords_1',
+                'copy_VarCoords_2', 'copy_VarMeta', 'copyatt', 'crossp3',
+                'cshstringtolist', 'cssgrid_Wrap', 'dble2flt', 'decimalPlaces',
+                'delete_VarAtts', 'dim_avg_n_Wrap', 'dim_avg_wgt_n_Wrap',
+                'dim_avg_wgt_Wrap', 'dim_avg_Wrap', 'dim_cumsum_n_Wrap',
+                'dim_cumsum_Wrap', 'dim_max_n_Wrap', 'dim_min_n_Wrap',
+                'dim_rmsd_n_Wrap', 'dim_rmsd_Wrap', 'dim_rmvmean_n_Wrap',
+                'dim_rmvmean_Wrap', 'dim_rmvmed_n_Wrap', 'dim_rmvmed_Wrap',
+                'dim_standardize_n_Wrap', 'dim_standardize_Wrap',
+                'dim_stddev_n_Wrap', 'dim_stddev_Wrap', 'dim_sum_n_Wrap',
+                'dim_sum_wgt_n_Wrap', 'dim_sum_wgt_Wrap', 'dim_sum_Wrap',
+                'dim_variance_n_Wrap', 'dim_variance_Wrap', 'dpres_plevel_Wrap',
+                'dtrend_leftdim', 'dv2uvF_Wrap', 'dv2uvG_Wrap', 'eof_north',
+                'eofcor_Wrap', 'eofcov_Wrap', 'eofunc_north', 'eofunc_ts_Wrap',
+                'eofunc_varimax_reorder', 'eofunc_varimax_Wrap', 'eofunc_Wrap',
+                'epsZero', 'f2fosh_Wrap', 'f2foshv_Wrap', 'f2fsh_Wrap',
+                'f2fshv_Wrap', 'f2gsh_Wrap', 'f2gshv_Wrap', 'fbindirSwap',
+                'fbinseqSwap1', 'fbinseqSwap2', 'flt2dble', 'flt2string',
+                'fo2fsh_Wrap', 'fo2fshv_Wrap', 'g2fsh_Wrap', 'g2fshv_Wrap',
+                'g2gsh_Wrap', 'g2gshv_Wrap', 'generate_resample_indices',
+                'generate_sample_indices', 'generate_unique_indices',
+                'genNormalDist', 'get1Dindex', 'get1Dindex_Collapse',
+                'get1Dindex_Exclude', 'get_file_suffix', 'GetFillColor',
+                'GetFillColorIndex', 'getFillValue', 'getind_latlon2d',
+                'getVarDimNames', 'getVarFillValue', 'grib_stime2itime',
+                'hyi2hyo_Wrap', 'ilapsF_Wrap', 'ilapsG_Wrap', 'ind_nearest_coord',
+                'indStrSubset', 'int2dble', 'int2flt', 'int2p_n_Wrap', 'int2p_Wrap',
+                'isMonotonic', 'isStrSubset', 'latGau', 'latGauWgt', 'latGlobeF',
+                'latGlobeFo', 'latRegWgt', 'linint1_n_Wrap', 'linint1_Wrap',
+                'linint2_points_Wrap', 'linint2_Wrap', 'local_max_1d',
+                'local_min_1d', 'lonFlip', 'lonGlobeF', 'lonGlobeFo', 'lonPivot',
+                'merge_levels_sfc', 'mod', 'month_to_annual',
+                'month_to_annual_weighted', 'month_to_season', 'month_to_season12',
+                'month_to_seasonN', 'monthly_total_to_daily_mean', 'nameDim',
+                'natgrid_Wrap', 'NewCosWeight', 'niceLatLon2D', 'NormCosWgtGlobe',
+                'numAsciiCol', 'numAsciiRow', 'numeric2int',
+                'obj_anal_ic_deprecated', 'obj_anal_ic_Wrap', 'omega_ccm_driver',
+                'omega_to_w', 'oneDtostring', 'pack_values', 'pattern_cor', 'pdfx',
+                'pdfxy', 'pdfxy_conform', 'pot_temp', 'pot_vort_hybrid',
+                'pot_vort_isobaric', 'pres2hybrid_Wrap', 'print_clock',
+                'printMinMax', 'quadroots', 'rcm2points_Wrap', 'rcm2rgrid_Wrap',
+                'readAsciiHead', 'readAsciiTable', 'reg_multlin_stats',
+                'region_ind', 'regline_stats', 'relhum_ttd', 'replaceSingleChar',
+                'RGBtoCmap', 'rgrid2rcm_Wrap', 'rho_mwjf', 'rm_single_dims',
+                'rmAnnCycle1D', 'rmInsufData', 'rmMonAnnCycLLLT', 'rmMonAnnCycLLT',
+                'rmMonAnnCycTLL', 'runave_n_Wrap', 'runave_Wrap', 'short2flt',
+                'short2flt_hdf', 'shsgc_R42_Wrap', 'sign_f90', 'sign_matlab',
+                'smth9_Wrap', 'smthClmDayTLL', 'smthClmDayTLLL', 'SqrtCosWeight',
+                'stat_dispersion', 'static_stability', 'stdMonLLLT', 'stdMonLLT',
+                'stdMonTLL', 'stdMonTLLL', 'symMinMaxPlt', 'table_attach_columns',
+                'table_attach_rows', 'time_to_newtime', 'transpose',
+                'triple2grid_Wrap', 'ut_convert', 'uv2dvF_Wrap', 'uv2dvG_Wrap',
+                'uv2vrF_Wrap', 'uv2vrG_Wrap', 'vr2uvF_Wrap', 'vr2uvG_Wrap',
+                'w_to_omega', 'wallClockElapseTime', 'wave_number_spc',
+                'wgt_areaave_Wrap', 'wgt_runave_leftdim', 'wgt_runave_n_Wrap',
+                'wgt_runave_Wrap', 'wgt_vertical_n', 'wind_component',
+                'wind_direction', 'yyyyddd_to_yyyymmdd', 'yyyymm_time',
+                'yyyymm_to_yyyyfrac', 'yyyymmdd_time', 'yyyymmdd_to_yyyyddd',
+                'yyyymmdd_to_yyyyfrac', 'yyyymmddhh_time', 'yyyymmddhh_to_yyyyfrac',
+                'zonal_mpsi_Wrap', 'zonalAve', 'calendar_decode2', 'cd_string',
+                'kf_filter', 'run_cor', 'time_axis_labels', 'ut_string',
+                'wrf_contour', 'wrf_map', 'wrf_map_overlay', 'wrf_map_overlays',
+                'wrf_map_resources', 'wrf_map_zoom', 'wrf_overlay', 'wrf_overlays',
+                'wrf_user_getvar', 'wrf_user_ij_to_ll', 'wrf_user_intrp2d',
+                'wrf_user_intrp3d', 'wrf_user_latlon_to_ij', 'wrf_user_list_times',
+                'wrf_user_ll_to_ij', 'wrf_user_unstagger', 'wrf_user_vert_interp',
+                'wrf_vector', 'gsn_add_annotation', 'gsn_add_polygon',
+                'gsn_add_polyline', 'gsn_add_polymarker',
+                'gsn_add_shapefile_polygons', 'gsn_add_shapefile_polylines',
+                'gsn_add_shapefile_polymarkers', 'gsn_add_text', 'gsn_attach_plots',
+                'gsn_blank_plot', 'gsn_contour', 'gsn_contour_map',
+                'gsn_contour_shade', 'gsn_coordinates', 'gsn_create_labelbar',
+                'gsn_create_legend', 'gsn_create_text',
+                'gsn_csm_attach_zonal_means', 'gsn_csm_blank_plot',
+                'gsn_csm_contour', 'gsn_csm_contour_map', 'gsn_csm_contour_map_ce',
+                'gsn_csm_contour_map_overlay', 'gsn_csm_contour_map_polar',
+                'gsn_csm_hov', 'gsn_csm_lat_time', 'gsn_csm_map', 'gsn_csm_map_ce',
+                'gsn_csm_map_polar', 'gsn_csm_pres_hgt',
+                'gsn_csm_pres_hgt_streamline', 'gsn_csm_pres_hgt_vector',
+                'gsn_csm_streamline', 'gsn_csm_streamline_contour_map',
+                'gsn_csm_streamline_contour_map_ce',
+                'gsn_csm_streamline_contour_map_polar', 'gsn_csm_streamline_map',
+                'gsn_csm_streamline_map_ce', 'gsn_csm_streamline_map_polar',
+                'gsn_csm_streamline_scalar', 'gsn_csm_streamline_scalar_map',
+                'gsn_csm_streamline_scalar_map_ce',
+                'gsn_csm_streamline_scalar_map_polar', 'gsn_csm_time_lat',
+                'gsn_csm_vector', 'gsn_csm_vector_map', 'gsn_csm_vector_map_ce',
+                'gsn_csm_vector_map_polar', 'gsn_csm_vector_scalar',
+                'gsn_csm_vector_scalar_map', 'gsn_csm_vector_scalar_map_ce',
+                'gsn_csm_vector_scalar_map_polar', 'gsn_csm_x2y', 'gsn_csm_x2y2',
+                'gsn_csm_xy', 'gsn_csm_xy2', 'gsn_csm_xy3', 'gsn_csm_y',
+                'gsn_define_colormap', 'gsn_draw_colormap', 'gsn_draw_named_colors',
+                'gsn_histogram', 'gsn_labelbar_ndc', 'gsn_legend_ndc', 'gsn_map',
+                'gsn_merge_colormaps', 'gsn_open_wks', 'gsn_panel', 'gsn_polygon',
+                'gsn_polygon_ndc', 'gsn_polyline', 'gsn_polyline_ndc',
+                'gsn_polymarker', 'gsn_polymarker_ndc', 'gsn_retrieve_colormap',
+                'gsn_reverse_colormap', 'gsn_streamline', 'gsn_streamline_map',
+                'gsn_streamline_scalar', 'gsn_streamline_scalar_map', 'gsn_table',
+                'gsn_text', 'gsn_text_ndc', 'gsn_vector', 'gsn_vector_map',
+                'gsn_vector_scalar', 'gsn_vector_scalar_map', 'gsn_xy', 'gsn_y',
+                'hsv2rgb', 'maximize_output', 'namedcolor2rgb', 'namedcolor2rgba',
+                'reset_device_coordinates', 'span_named_colors'), prefix=r'\b'),
+             Name.Builtin),
+
+            # Resources
+            (words((
+                'amDataXF', 'amDataYF', 'amJust', 'amOn', 'amOrthogonalPosF',
+                'amParallelPosF', 'amResizeNotify', 'amSide', 'amTrackData',
+                'amViewId', 'amZone', 'appDefaultParent', 'appFileSuffix',
+                'appResources', 'appSysDir', 'appUsrDir', 'caCopyArrays',
+                'caXArray', 'caXCast', 'caXMaxV', 'caXMinV', 'caXMissingV',
+                'caYArray', 'caYCast', 'caYMaxV', 'caYMinV', 'caYMissingV',
+                'cnCellFillEdgeColor', 'cnCellFillMissingValEdgeColor',
+                'cnConpackParams', 'cnConstFEnableFill', 'cnConstFLabelAngleF',
+                'cnConstFLabelBackgroundColor', 'cnConstFLabelConstantSpacingF',
+                'cnConstFLabelFont', 'cnConstFLabelFontAspectF',
+                'cnConstFLabelFontColor', 'cnConstFLabelFontHeightF',
+                'cnConstFLabelFontQuality', 'cnConstFLabelFontThicknessF',
+                'cnConstFLabelFormat', 'cnConstFLabelFuncCode', 'cnConstFLabelJust',
+                'cnConstFLabelOn', 'cnConstFLabelOrthogonalPosF',
+                'cnConstFLabelParallelPosF', 'cnConstFLabelPerimColor',
+                'cnConstFLabelPerimOn', 'cnConstFLabelPerimSpaceF',
+                'cnConstFLabelPerimThicknessF', 'cnConstFLabelSide',
+                'cnConstFLabelString', 'cnConstFLabelTextDirection',
+                'cnConstFLabelZone', 'cnConstFUseInfoLabelRes',
+                'cnExplicitLabelBarLabelsOn', 'cnExplicitLegendLabelsOn',
+                'cnExplicitLineLabelsOn', 'cnFillBackgroundColor', 'cnFillColor',
+                'cnFillColors', 'cnFillDotSizeF', 'cnFillDrawOrder', 'cnFillMode',
+                'cnFillOn', 'cnFillOpacityF', 'cnFillPalette', 'cnFillPattern',
+                'cnFillPatterns', 'cnFillScaleF', 'cnFillScales', 'cnFixFillBleed',
+                'cnGridBoundFillColor', 'cnGridBoundFillPattern',
+                'cnGridBoundFillScaleF', 'cnGridBoundPerimColor',
+                'cnGridBoundPerimDashPattern', 'cnGridBoundPerimOn',
+                'cnGridBoundPerimThicknessF', 'cnHighLabelAngleF',
+                'cnHighLabelBackgroundColor', 'cnHighLabelConstantSpacingF',
+                'cnHighLabelCount', 'cnHighLabelFont', 'cnHighLabelFontAspectF',
+                'cnHighLabelFontColor', 'cnHighLabelFontHeightF',
+                'cnHighLabelFontQuality', 'cnHighLabelFontThicknessF',
+                'cnHighLabelFormat', 'cnHighLabelFuncCode', 'cnHighLabelPerimColor',
+                'cnHighLabelPerimOn', 'cnHighLabelPerimSpaceF',
+                'cnHighLabelPerimThicknessF', 'cnHighLabelString', 'cnHighLabelsOn',
+                'cnHighLowLabelOverlapMode', 'cnHighUseLineLabelRes',
+                'cnInfoLabelAngleF', 'cnInfoLabelBackgroundColor',
+                'cnInfoLabelConstantSpacingF', 'cnInfoLabelFont',
+                'cnInfoLabelFontAspectF', 'cnInfoLabelFontColor',
+                'cnInfoLabelFontHeightF', 'cnInfoLabelFontQuality',
+                'cnInfoLabelFontThicknessF', 'cnInfoLabelFormat',
+                'cnInfoLabelFuncCode', 'cnInfoLabelJust', 'cnInfoLabelOn',
+                'cnInfoLabelOrthogonalPosF', 'cnInfoLabelParallelPosF',
+                'cnInfoLabelPerimColor', 'cnInfoLabelPerimOn',
+                'cnInfoLabelPerimSpaceF', 'cnInfoLabelPerimThicknessF',
+                'cnInfoLabelSide', 'cnInfoLabelString', 'cnInfoLabelTextDirection',
+                'cnInfoLabelZone', 'cnLabelBarEndLabelsOn', 'cnLabelBarEndStyle',
+                'cnLabelDrawOrder', 'cnLabelMasking', 'cnLabelScaleFactorF',
+                'cnLabelScaleValueF', 'cnLabelScalingMode', 'cnLegendLevelFlags',
+                'cnLevelCount', 'cnLevelFlag', 'cnLevelFlags', 'cnLevelSelectionMode',
+                'cnLevelSpacingF', 'cnLevels', 'cnLineColor', 'cnLineColors',
+                'cnLineDashPattern', 'cnLineDashPatterns', 'cnLineDashSegLenF',
+                'cnLineDrawOrder', 'cnLineLabelAngleF', 'cnLineLabelBackgroundColor',
+                'cnLineLabelConstantSpacingF', 'cnLineLabelCount',
+                'cnLineLabelDensityF', 'cnLineLabelFont', 'cnLineLabelFontAspectF',
+                'cnLineLabelFontColor', 'cnLineLabelFontColors',
+                'cnLineLabelFontHeightF', 'cnLineLabelFontQuality',
+                'cnLineLabelFontThicknessF', 'cnLineLabelFormat',
+                'cnLineLabelFuncCode', 'cnLineLabelInterval', 'cnLineLabelPerimColor',
+                'cnLineLabelPerimOn', 'cnLineLabelPerimSpaceF',
+                'cnLineLabelPerimThicknessF', 'cnLineLabelPlacementMode',
+                'cnLineLabelStrings', 'cnLineLabelsOn', 'cnLinePalette',
+                'cnLineThicknessF', 'cnLineThicknesses', 'cnLinesOn',
+                'cnLowLabelAngleF', 'cnLowLabelBackgroundColor',
+                'cnLowLabelConstantSpacingF', 'cnLowLabelCount', 'cnLowLabelFont',
+                'cnLowLabelFontAspectF', 'cnLowLabelFontColor',
+                'cnLowLabelFontHeightF', 'cnLowLabelFontQuality',
+                'cnLowLabelFontThicknessF', 'cnLowLabelFormat', 'cnLowLabelFuncCode',
+                'cnLowLabelPerimColor', 'cnLowLabelPerimOn', 'cnLowLabelPerimSpaceF',
+                'cnLowLabelPerimThicknessF', 'cnLowLabelString', 'cnLowLabelsOn',
+                'cnLowUseHighLabelRes', 'cnMaxDataValueFormat', 'cnMaxLevelCount',
+                'cnMaxLevelValF', 'cnMaxPointDistanceF', 'cnMinLevelValF',
+                'cnMissingValFillColor', 'cnMissingValFillPattern',
+                'cnMissingValFillScaleF', 'cnMissingValPerimColor',
+                'cnMissingValPerimDashPattern', 'cnMissingValPerimGridBoundOn',
+                'cnMissingValPerimOn', 'cnMissingValPerimThicknessF',
+                'cnMonoFillColor', 'cnMonoFillPattern', 'cnMonoFillScale',
+                'cnMonoLevelFlag', 'cnMonoLineColor', 'cnMonoLineDashPattern',
+                'cnMonoLineLabelFontColor', 'cnMonoLineThickness', 'cnNoDataLabelOn',
+                'cnNoDataLabelString', 'cnOutOfRangeFillColor',
+                'cnOutOfRangeFillPattern', 'cnOutOfRangeFillScaleF',
+                'cnOutOfRangePerimColor', 'cnOutOfRangePerimDashPattern',
+                'cnOutOfRangePerimOn', 'cnOutOfRangePerimThicknessF',
+                'cnRasterCellSizeF', 'cnRasterMinCellSizeF', 'cnRasterModeOn',
+                'cnRasterSampleFactorF', 'cnRasterSmoothingOn', 'cnScalarFieldData',
+                'cnSmoothingDistanceF', 'cnSmoothingOn', 'cnSmoothingTensionF',
+                'cnSpanFillPalette', 'cnSpanLinePalette', 'ctCopyTables',
+                'ctXElementSize', 'ctXMaxV', 'ctXMinV', 'ctXMissingV', 'ctXTable',
+                'ctXTableLengths', 'ctXTableType', 'ctYElementSize', 'ctYMaxV',
+                'ctYMinV', 'ctYMissingV', 'ctYTable', 'ctYTableLengths',
+                'ctYTableType', 'dcDelayCompute', 'errBuffer',
+                'errFileName', 'errFilePtr', 'errLevel', 'errPrint', 'errUnitNumber',
+                'gsClipOn', 'gsColors', 'gsEdgeColor', 'gsEdgeDashPattern',
+                'gsEdgeDashSegLenF', 'gsEdgeThicknessF', 'gsEdgesOn',
+                'gsFillBackgroundColor', 'gsFillColor', 'gsFillDotSizeF',
+                'gsFillIndex', 'gsFillLineThicknessF', 'gsFillOpacityF',
+                'gsFillScaleF', 'gsFont', 'gsFontAspectF', 'gsFontColor',
+                'gsFontHeightF', 'gsFontOpacityF', 'gsFontQuality',
+                'gsFontThicknessF', 'gsLineColor', 'gsLineDashPattern',
+                'gsLineDashSegLenF', 'gsLineLabelConstantSpacingF', 'gsLineLabelFont',
+                'gsLineLabelFontAspectF', 'gsLineLabelFontColor',
+                'gsLineLabelFontHeightF', 'gsLineLabelFontQuality',
+                'gsLineLabelFontThicknessF', 'gsLineLabelFuncCode',
+                'gsLineLabelString', 'gsLineOpacityF', 'gsLineThicknessF',
+                'gsMarkerColor', 'gsMarkerIndex', 'gsMarkerOpacityF', 'gsMarkerSizeF',
+                'gsMarkerThicknessF', 'gsSegments', 'gsTextAngleF',
+                'gsTextConstantSpacingF', 'gsTextDirection', 'gsTextFuncCode',
+                'gsTextJustification', 'gsnAboveYRefLineBarColors',
+                'gsnAboveYRefLineBarFillScales', 'gsnAboveYRefLineBarPatterns',
+                'gsnAboveYRefLineColor', 'gsnAddCyclic', 'gsnAttachBorderOn',
+                'gsnAttachPlotsXAxis', 'gsnBelowYRefLineBarColors',
+                'gsnBelowYRefLineBarFillScales', 'gsnBelowYRefLineBarPatterns',
+                'gsnBelowYRefLineColor', 'gsnBoxMargin', 'gsnCenterString',
+                'gsnCenterStringFontColor', 'gsnCenterStringFontHeightF',
+                'gsnCenterStringFuncCode', 'gsnCenterStringOrthogonalPosF',
+                'gsnCenterStringParallelPosF', 'gsnContourLineThicknessesScale',
+                'gsnContourNegLineDashPattern', 'gsnContourPosLineDashPattern',
+                'gsnContourZeroLineThicknessF', 'gsnDebugWriteFileName', 'gsnDraw',
+                'gsnFrame', 'gsnHistogramBarWidthPercent', 'gsnHistogramBinIntervals',
+                'gsnHistogramBinMissing', 'gsnHistogramBinWidth',
+                'gsnHistogramClassIntervals', 'gsnHistogramCompare',
+                'gsnHistogramComputePercentages',
+                'gsnHistogramComputePercentagesNoMissing',
+                'gsnHistogramDiscreteBinValues', 'gsnHistogramDiscreteClassValues',
+                'gsnHistogramHorizontal', 'gsnHistogramMinMaxBinsOn',
+                'gsnHistogramNumberOfBins', 'gsnHistogramPercentSign',
+                'gsnHistogramSelectNiceIntervals', 'gsnLeftString',
+                'gsnLeftStringFontColor', 'gsnLeftStringFontHeightF',
+                'gsnLeftStringFuncCode', 'gsnLeftStringOrthogonalPosF',
+                'gsnLeftStringParallelPosF', 'gsnMajorLatSpacing',
+                'gsnMajorLonSpacing', 'gsnMaskLambertConformal',
+                'gsnMaskLambertConformalOutlineOn', 'gsnMaximize',
+                'gsnMinorLatSpacing', 'gsnMinorLonSpacing', 'gsnPanelBottom',
+                'gsnPanelCenter', 'gsnPanelDebug', 'gsnPanelFigureStrings',
+                'gsnPanelFigureStringsBackgroundFillColor',
+                'gsnPanelFigureStringsFontHeightF', 'gsnPanelFigureStringsJust',
+                'gsnPanelFigureStringsPerimOn', 'gsnPanelLabelBar', 'gsnPanelLeft',
+                'gsnPanelMainFont', 'gsnPanelMainFontColor',
+                'gsnPanelMainFontHeightF', 'gsnPanelMainString', 'gsnPanelRight',
+                'gsnPanelRowSpec', 'gsnPanelScalePlotIndex', 'gsnPanelTop',
+                'gsnPanelXF', 'gsnPanelXWhiteSpacePercent', 'gsnPanelYF',
+                'gsnPanelYWhiteSpacePercent', 'gsnPaperHeight', 'gsnPaperMargin',
+                'gsnPaperOrientation', 'gsnPaperWidth', 'gsnPolar',
+                'gsnPolarLabelDistance', 'gsnPolarLabelFont',
+                'gsnPolarLabelFontHeightF', 'gsnPolarLabelSpacing', 'gsnPolarTime',
+                'gsnPolarUT', 'gsnRightString', 'gsnRightStringFontColor',
+                'gsnRightStringFontHeightF', 'gsnRightStringFuncCode',
+                'gsnRightStringOrthogonalPosF', 'gsnRightStringParallelPosF',
+                'gsnScalarContour', 'gsnScale', 'gsnShape', 'gsnSpreadColorEnd',
+                'gsnSpreadColorStart', 'gsnSpreadColors', 'gsnStringFont',
+                'gsnStringFontColor', 'gsnStringFontHeightF', 'gsnStringFuncCode',
+                'gsnTickMarksOn', 'gsnXAxisIrregular2Linear', 'gsnXAxisIrregular2Log',
+                'gsnXRefLine', 'gsnXRefLineColor', 'gsnXRefLineDashPattern',
+                'gsnXRefLineThicknessF', 'gsnXYAboveFillColors', 'gsnXYBarChart',
+                'gsnXYBarChartBarWidth', 'gsnXYBarChartColors',
+                'gsnXYBarChartColors2', 'gsnXYBarChartFillDotSizeF',
+                'gsnXYBarChartFillLineThicknessF', 'gsnXYBarChartFillOpacityF',
+                'gsnXYBarChartFillScaleF', 'gsnXYBarChartOutlineOnly',
+                'gsnXYBarChartOutlineThicknessF', 'gsnXYBarChartPatterns',
+                'gsnXYBarChartPatterns2', 'gsnXYBelowFillColors', 'gsnXYFillColors',
+                'gsnXYFillOpacities', 'gsnXYLeftFillColors', 'gsnXYRightFillColors',
+                'gsnYAxisIrregular2Linear', 'gsnYAxisIrregular2Log', 'gsnYRefLine',
+                'gsnYRefLineColor', 'gsnYRefLineColors', 'gsnYRefLineDashPattern',
+                'gsnYRefLineDashPatterns', 'gsnYRefLineThicknessF',
+                'gsnYRefLineThicknesses', 'gsnZonalMean', 'gsnZonalMeanXMaxF',
+                'gsnZonalMeanXMinF', 'gsnZonalMeanYRefLine', 'lbAutoManage',
+                'lbBottomMarginF', 'lbBoxCount', 'lbBoxEndCapStyle', 'lbBoxFractions',
+                'lbBoxLineColor', 'lbBoxLineDashPattern', 'lbBoxLineDashSegLenF',
+                'lbBoxLineThicknessF', 'lbBoxLinesOn', 'lbBoxMajorExtentF',
+                'lbBoxMinorExtentF', 'lbBoxSeparatorLinesOn', 'lbBoxSizing',
+                'lbFillBackground', 'lbFillColor', 'lbFillColors', 'lbFillDotSizeF',
+                'lbFillLineThicknessF', 'lbFillPattern', 'lbFillPatterns',
+                'lbFillScaleF', 'lbFillScales', 'lbJustification', 'lbLabelAlignment',
+                'lbLabelAngleF', 'lbLabelAutoStride', 'lbLabelBarOn',
+                'lbLabelConstantSpacingF', 'lbLabelDirection', 'lbLabelFont',
+                'lbLabelFontAspectF', 'lbLabelFontColor', 'lbLabelFontHeightF',
+                'lbLabelFontQuality', 'lbLabelFontThicknessF', 'lbLabelFuncCode',
+                'lbLabelJust', 'lbLabelOffsetF', 'lbLabelPosition', 'lbLabelStride',
+                'lbLabelStrings', 'lbLabelsOn', 'lbLeftMarginF', 'lbMaxLabelLenF',
+                'lbMinLabelSpacingF', 'lbMonoFillColor', 'lbMonoFillPattern',
+                'lbMonoFillScale', 'lbOrientation', 'lbPerimColor',
+                'lbPerimDashPattern', 'lbPerimDashSegLenF', 'lbPerimFill',
+                'lbPerimFillColor', 'lbPerimOn', 'lbPerimThicknessF',
+                'lbRasterFillOn', 'lbRightMarginF', 'lbTitleAngleF',
+                'lbTitleConstantSpacingF', 'lbTitleDirection', 'lbTitleExtentF',
+                'lbTitleFont', 'lbTitleFontAspectF', 'lbTitleFontColor',
+                'lbTitleFontHeightF', 'lbTitleFontQuality', 'lbTitleFontThicknessF',
+                'lbTitleFuncCode', 'lbTitleJust', 'lbTitleOffsetF', 'lbTitleOn',
+                'lbTitlePosition', 'lbTitleString', 'lbTopMarginF', 'lgAutoManage',
+                'lgBottomMarginF', 'lgBoxBackground', 'lgBoxLineColor',
+                'lgBoxLineDashPattern', 'lgBoxLineDashSegLenF', 'lgBoxLineThicknessF',
+                'lgBoxLinesOn', 'lgBoxMajorExtentF', 'lgBoxMinorExtentF',
+                'lgDashIndex', 'lgDashIndexes', 'lgItemCount', 'lgItemOrder',
+                'lgItemPlacement', 'lgItemPositions', 'lgItemType', 'lgItemTypes',
+                'lgJustification', 'lgLabelAlignment', 'lgLabelAngleF',
+                'lgLabelAutoStride', 'lgLabelConstantSpacingF', 'lgLabelDirection',
+                'lgLabelFont', 'lgLabelFontAspectF', 'lgLabelFontColor',
+                'lgLabelFontHeightF', 'lgLabelFontQuality', 'lgLabelFontThicknessF',
+                'lgLabelFuncCode', 'lgLabelJust', 'lgLabelOffsetF', 'lgLabelPosition',
+                'lgLabelStride', 'lgLabelStrings', 'lgLabelsOn', 'lgLeftMarginF',
+                'lgLegendOn', 'lgLineColor', 'lgLineColors', 'lgLineDashSegLenF',
+                'lgLineDashSegLens', 'lgLineLabelConstantSpacingF', 'lgLineLabelFont',
+                'lgLineLabelFontAspectF', 'lgLineLabelFontColor',
+                'lgLineLabelFontColors', 'lgLineLabelFontHeightF',
+                'lgLineLabelFontHeights', 'lgLineLabelFontQuality',
+                'lgLineLabelFontThicknessF', 'lgLineLabelFuncCode',
+                'lgLineLabelStrings', 'lgLineLabelsOn', 'lgLineThicknessF',
+                'lgLineThicknesses', 'lgMarkerColor', 'lgMarkerColors',
+                'lgMarkerIndex', 'lgMarkerIndexes', 'lgMarkerSizeF', 'lgMarkerSizes',
+                'lgMarkerThicknessF', 'lgMarkerThicknesses', 'lgMonoDashIndex',
+                'lgMonoItemType', 'lgMonoLineColor', 'lgMonoLineDashSegLen',
+                'lgMonoLineLabelFontColor', 'lgMonoLineLabelFontHeight',
+                'lgMonoLineThickness', 'lgMonoMarkerColor', 'lgMonoMarkerIndex',
+                'lgMonoMarkerSize', 'lgMonoMarkerThickness', 'lgOrientation',
+                'lgPerimColor', 'lgPerimDashPattern', 'lgPerimDashSegLenF',
+                'lgPerimFill', 'lgPerimFillColor', 'lgPerimOn', 'lgPerimThicknessF',
+                'lgRightMarginF', 'lgTitleAngleF', 'lgTitleConstantSpacingF',
+                'lgTitleDirection', 'lgTitleExtentF', 'lgTitleFont',
+                'lgTitleFontAspectF', 'lgTitleFontColor', 'lgTitleFontHeightF',
+                'lgTitleFontQuality', 'lgTitleFontThicknessF', 'lgTitleFuncCode',
+                'lgTitleJust', 'lgTitleOffsetF', 'lgTitleOn', 'lgTitlePosition',
+                'lgTitleString', 'lgTopMarginF', 'mpAreaGroupCount',
+                'mpAreaMaskingOn', 'mpAreaNames', 'mpAreaTypes', 'mpBottomAngleF',
+                'mpBottomMapPosF', 'mpBottomNDCF', 'mpBottomNPCF',
+                'mpBottomPointLatF', 'mpBottomPointLonF', 'mpBottomWindowF',
+                'mpCenterLatF', 'mpCenterLonF', 'mpCenterRotF', 'mpCountyLineColor',
+                'mpCountyLineDashPattern', 'mpCountyLineDashSegLenF',
+                'mpCountyLineThicknessF', 'mpDataBaseVersion', 'mpDataResolution',
+                'mpDataSetName', 'mpDefaultFillColor', 'mpDefaultFillPattern',
+                'mpDefaultFillScaleF', 'mpDynamicAreaGroups', 'mpEllipticalBoundary',
+                'mpFillAreaSpecifiers', 'mpFillBoundarySets', 'mpFillColor',
+                'mpFillColors', 'mpFillColors-default', 'mpFillDotSizeF',
+                'mpFillDrawOrder', 'mpFillOn', 'mpFillPatternBackground',
+                'mpFillPattern', 'mpFillPatterns', 'mpFillPatterns-default',
+                'mpFillScaleF', 'mpFillScales', 'mpFillScales-default',
+                'mpFixedAreaGroups', 'mpGeophysicalLineColor',
+                'mpGeophysicalLineDashPattern', 'mpGeophysicalLineDashSegLenF',
+                'mpGeophysicalLineThicknessF', 'mpGreatCircleLinesOn',
+                'mpGridAndLimbDrawOrder', 'mpGridAndLimbOn', 'mpGridLatSpacingF',
+                'mpGridLineColor', 'mpGridLineDashPattern', 'mpGridLineDashSegLenF',
+                'mpGridLineThicknessF', 'mpGridLonSpacingF', 'mpGridMaskMode',
+                'mpGridMaxLatF', 'mpGridPolarLonSpacingF', 'mpGridSpacingF',
+                'mpInlandWaterFillColor', 'mpInlandWaterFillPattern',
+                'mpInlandWaterFillScaleF', 'mpLabelDrawOrder', 'mpLabelFontColor',
+                'mpLabelFontHeightF', 'mpLabelsOn', 'mpLambertMeridianF',
+                'mpLambertParallel1F', 'mpLambertParallel2F', 'mpLandFillColor',
+                'mpLandFillPattern', 'mpLandFillScaleF', 'mpLeftAngleF',
+                'mpLeftCornerLatF', 'mpLeftCornerLonF', 'mpLeftMapPosF',
+                'mpLeftNDCF', 'mpLeftNPCF', 'mpLeftPointLatF',
+                'mpLeftPointLonF', 'mpLeftWindowF', 'mpLimbLineColor',
+                'mpLimbLineDashPattern', 'mpLimbLineDashSegLenF',
+                'mpLimbLineThicknessF', 'mpLimitMode', 'mpMaskAreaSpecifiers',
+                'mpMaskOutlineSpecifiers', 'mpMaxLatF', 'mpMaxLonF',
+                'mpMinLatF', 'mpMinLonF', 'mpMonoFillColor', 'mpMonoFillPattern',
+                'mpMonoFillScale', 'mpNationalLineColor', 'mpNationalLineDashPattern',
+                'mpNationalLineThicknessF', 'mpOceanFillColor', 'mpOceanFillPattern',
+                'mpOceanFillScaleF', 'mpOutlineBoundarySets', 'mpOutlineDrawOrder',
+                'mpOutlineMaskingOn', 'mpOutlineOn', 'mpOutlineSpecifiers',
+                'mpPerimDrawOrder', 'mpPerimLineColor', 'mpPerimLineDashPattern',
+                'mpPerimLineDashSegLenF', 'mpPerimLineThicknessF', 'mpPerimOn',
+                'mpPolyMode', 'mpProjection', 'mpProvincialLineColor',
+                'mpProvincialLineDashPattern', 'mpProvincialLineDashSegLenF',
+                'mpProvincialLineThicknessF', 'mpRelativeCenterLat',
+                'mpRelativeCenterLon', 'mpRightAngleF', 'mpRightCornerLatF',
+                'mpRightCornerLonF', 'mpRightMapPosF', 'mpRightNDCF',
+                'mpRightNPCF', 'mpRightPointLatF', 'mpRightPointLonF',
+                'mpRightWindowF', 'mpSatelliteAngle1F', 'mpSatelliteAngle2F',
+                'mpSatelliteDistF', 'mpShapeMode', 'mpSpecifiedFillColors',
+                'mpSpecifiedFillDirectIndexing', 'mpSpecifiedFillPatterns',
+                'mpSpecifiedFillPriority', 'mpSpecifiedFillScales',
+                'mpTopAngleF', 'mpTopMapPosF', 'mpTopNDCF', 'mpTopNPCF',
+                'mpTopPointLatF', 'mpTopPointLonF', 'mpTopWindowF',
+                'mpUSStateLineColor', 'mpUSStateLineDashPattern',
+                'mpUSStateLineDashSegLenF', 'mpUSStateLineThicknessF',
+                'pmAnnoManagers', 'pmAnnoViews', 'pmLabelBarDisplayMode',
+                'pmLabelBarHeightF', 'pmLabelBarKeepAspect', 'pmLabelBarOrthogonalPosF',
+                'pmLabelBarParallelPosF', 'pmLabelBarSide', 'pmLabelBarWidthF',
+                'pmLabelBarZone', 'pmLegendDisplayMode', 'pmLegendHeightF',
+                'pmLegendKeepAspect', 'pmLegendOrthogonalPosF',
+                'pmLegendParallelPosF', 'pmLegendSide', 'pmLegendWidthF',
+                'pmLegendZone', 'pmOverlaySequenceIds', 'pmTickMarkDisplayMode',
+                'pmTickMarkZone', 'pmTitleDisplayMode', 'pmTitleZone',
+                'prGraphicStyle', 'prPolyType', 'prXArray', 'prYArray',
+                'sfCopyData', 'sfDataArray', 'sfDataMaxV', 'sfDataMinV',
+                'sfElementNodes', 'sfExchangeDimensions', 'sfFirstNodeIndex',
+                'sfMissingValueV', 'sfXArray', 'sfXCActualEndF', 'sfXCActualStartF',
+                'sfXCEndIndex', 'sfXCEndSubsetV', 'sfXCEndV', 'sfXCStartIndex',
+                'sfXCStartSubsetV', 'sfXCStartV', 'sfXCStride', 'sfXCellBounds',
+                'sfYArray', 'sfYCActualEndF', 'sfYCActualStartF', 'sfYCEndIndex',
+                'sfYCEndSubsetV', 'sfYCEndV', 'sfYCStartIndex', 'sfYCStartSubsetV',
+                'sfYCStartV', 'sfYCStride', 'sfYCellBounds', 'stArrowLengthF',
+                'stArrowStride', 'stCrossoverCheckCount',
+                'stExplicitLabelBarLabelsOn', 'stLabelBarEndLabelsOn',
+                'stLabelFormat', 'stLengthCheckCount', 'stLevelColors',
+                'stLevelCount', 'stLevelPalette', 'stLevelSelectionMode',
+                'stLevelSpacingF', 'stLevels', 'stLineColor', 'stLineOpacityF',
+                'stLineStartStride', 'stLineThicknessF', 'stMapDirection',
+                'stMaxLevelCount', 'stMaxLevelValF', 'stMinArrowSpacingF',
+                'stMinDistanceF', 'stMinLevelValF', 'stMinLineSpacingF',
+                'stMinStepFactorF', 'stMonoLineColor', 'stNoDataLabelOn',
+                'stNoDataLabelString', 'stScalarFieldData', 'stScalarMissingValColor',
+                'stSpanLevelPalette', 'stStepSizeF', 'stStreamlineDrawOrder',
+                'stUseScalarArray', 'stVectorFieldData', 'stZeroFLabelAngleF',
+                'stZeroFLabelBackgroundColor', 'stZeroFLabelConstantSpacingF',
+                'stZeroFLabelFont', 'stZeroFLabelFontAspectF',
+                'stZeroFLabelFontColor', 'stZeroFLabelFontHeightF',
+                'stZeroFLabelFontQuality', 'stZeroFLabelFontThicknessF',
+                'stZeroFLabelFuncCode', 'stZeroFLabelJust', 'stZeroFLabelOn',
+                'stZeroFLabelOrthogonalPosF', 'stZeroFLabelParallelPosF',
+                'stZeroFLabelPerimColor', 'stZeroFLabelPerimOn',
+                'stZeroFLabelPerimSpaceF', 'stZeroFLabelPerimThicknessF',
+                'stZeroFLabelSide', 'stZeroFLabelString', 'stZeroFLabelTextDirection',
+                'stZeroFLabelZone', 'tfDoNDCOverlay', 'tfPlotManagerOn',
+                'tfPolyDrawList', 'tfPolyDrawOrder', 'tiDeltaF', 'tiMainAngleF',
+                'tiMainConstantSpacingF', 'tiMainDirection', 'tiMainFont',
+                'tiMainFontAspectF', 'tiMainFontColor', 'tiMainFontHeightF',
+                'tiMainFontQuality', 'tiMainFontThicknessF', 'tiMainFuncCode',
+                'tiMainJust', 'tiMainOffsetXF', 'tiMainOffsetYF', 'tiMainOn',
+                'tiMainPosition', 'tiMainSide', 'tiMainString', 'tiUseMainAttributes',
+                'tiXAxisAngleF', 'tiXAxisConstantSpacingF', 'tiXAxisDirection',
+                'tiXAxisFont', 'tiXAxisFontAspectF', 'tiXAxisFontColor',
+                'tiXAxisFontHeightF', 'tiXAxisFontQuality', 'tiXAxisFontThicknessF',
+                'tiXAxisFuncCode', 'tiXAxisJust', 'tiXAxisOffsetXF',
+                'tiXAxisOffsetYF', 'tiXAxisOn', 'tiXAxisPosition', 'tiXAxisSide',
+                'tiXAxisString', 'tiYAxisAngleF', 'tiYAxisConstantSpacingF',
+                'tiYAxisDirection', 'tiYAxisFont', 'tiYAxisFontAspectF',
+                'tiYAxisFontColor', 'tiYAxisFontHeightF', 'tiYAxisFontQuality',
+                'tiYAxisFontThicknessF', 'tiYAxisFuncCode', 'tiYAxisJust',
+                'tiYAxisOffsetXF', 'tiYAxisOffsetYF', 'tiYAxisOn', 'tiYAxisPosition',
+                'tiYAxisSide', 'tiYAxisString', 'tmBorderLineColor',
+                'tmBorderThicknessF', 'tmEqualizeXYSizes', 'tmLabelAutoStride',
+                'tmSciNoteCutoff', 'tmXBAutoPrecision', 'tmXBBorderOn',
+                'tmXBDataLeftF', 'tmXBDataRightF', 'tmXBFormat', 'tmXBIrrTensionF',
+                'tmXBIrregularPoints', 'tmXBLabelAngleF', 'tmXBLabelConstantSpacingF',
+                'tmXBLabelDeltaF', 'tmXBLabelDirection', 'tmXBLabelFont',
+                'tmXBLabelFontAspectF', 'tmXBLabelFontColor', 'tmXBLabelFontHeightF',
+                'tmXBLabelFontQuality', 'tmXBLabelFontThicknessF',
+                'tmXBLabelFuncCode', 'tmXBLabelJust', 'tmXBLabelStride', 'tmXBLabels',
+                'tmXBLabelsOn', 'tmXBMajorLengthF', 'tmXBMajorLineColor',
+                'tmXBMajorOutwardLengthF', 'tmXBMajorThicknessF', 'tmXBMaxLabelLenF',
+                'tmXBMaxTicks', 'tmXBMinLabelSpacingF', 'tmXBMinorLengthF',
+                'tmXBMinorLineColor', 'tmXBMinorOn', 'tmXBMinorOutwardLengthF',
+                'tmXBMinorPerMajor', 'tmXBMinorThicknessF', 'tmXBMinorValues',
+                'tmXBMode', 'tmXBOn', 'tmXBPrecision', 'tmXBStyle', 'tmXBTickEndF',
+                'tmXBTickSpacingF', 'tmXBTickStartF', 'tmXBValues', 'tmXMajorGrid',
+                'tmXMajorGridLineColor', 'tmXMajorGridLineDashPattern',
+                'tmXMajorGridThicknessF', 'tmXMinorGrid', 'tmXMinorGridLineColor',
+                'tmXMinorGridLineDashPattern', 'tmXMinorGridThicknessF',
+                'tmXTAutoPrecision', 'tmXTBorderOn', 'tmXTDataLeftF',
+                'tmXTDataRightF', 'tmXTFormat', 'tmXTIrrTensionF',
+                'tmXTIrregularPoints', 'tmXTLabelAngleF', 'tmXTLabelConstantSpacingF',
+                'tmXTLabelDeltaF', 'tmXTLabelDirection', 'tmXTLabelFont',
+                'tmXTLabelFontAspectF', 'tmXTLabelFontColor', 'tmXTLabelFontHeightF',
+                'tmXTLabelFontQuality', 'tmXTLabelFontThicknessF',
+                'tmXTLabelFuncCode', 'tmXTLabelJust', 'tmXTLabelStride', 'tmXTLabels',
+                'tmXTLabelsOn', 'tmXTMajorLengthF', 'tmXTMajorLineColor',
+                'tmXTMajorOutwardLengthF', 'tmXTMajorThicknessF', 'tmXTMaxLabelLenF',
+                'tmXTMaxTicks', 'tmXTMinLabelSpacingF', 'tmXTMinorLengthF',
+                'tmXTMinorLineColor', 'tmXTMinorOn', 'tmXTMinorOutwardLengthF',
+                'tmXTMinorPerMajor', 'tmXTMinorThicknessF', 'tmXTMinorValues',
+                'tmXTMode', 'tmXTOn', 'tmXTPrecision', 'tmXTStyle', 'tmXTTickEndF',
+                'tmXTTickSpacingF', 'tmXTTickStartF', 'tmXTValues', 'tmXUseBottom',
+                'tmYLAutoPrecision', 'tmYLBorderOn', 'tmYLDataBottomF',
+                'tmYLDataTopF', 'tmYLFormat', 'tmYLIrrTensionF',
+                'tmYLIrregularPoints', 'tmYLLabelAngleF', 'tmYLLabelConstantSpacingF',
+                'tmYLLabelDeltaF', 'tmYLLabelDirection', 'tmYLLabelFont',
+                'tmYLLabelFontAspectF', 'tmYLLabelFontColor', 'tmYLLabelFontHeightF',
+                'tmYLLabelFontQuality', 'tmYLLabelFontThicknessF',
+                'tmYLLabelFuncCode', 'tmYLLabelJust', 'tmYLLabelStride', 'tmYLLabels',
+                'tmYLLabelsOn', 'tmYLMajorLengthF', 'tmYLMajorLineColor',
+                'tmYLMajorOutwardLengthF', 'tmYLMajorThicknessF', 'tmYLMaxLabelLenF',
+                'tmYLMaxTicks', 'tmYLMinLabelSpacingF', 'tmYLMinorLengthF',
+                'tmYLMinorLineColor', 'tmYLMinorOn', 'tmYLMinorOutwardLengthF',
+                'tmYLMinorPerMajor', 'tmYLMinorThicknessF', 'tmYLMinorValues',
+                'tmYLMode', 'tmYLOn', 'tmYLPrecision', 'tmYLStyle', 'tmYLTickEndF',
+                'tmYLTickSpacingF', 'tmYLTickStartF', 'tmYLValues', 'tmYMajorGrid',
+                'tmYMajorGridLineColor', 'tmYMajorGridLineDashPattern',
+                'tmYMajorGridThicknessF', 'tmYMinorGrid', 'tmYMinorGridLineColor',
+                'tmYMinorGridLineDashPattern', 'tmYMinorGridThicknessF',
+                'tmYRAutoPrecision', 'tmYRBorderOn', 'tmYRDataBottomF',
+                'tmYRDataTopF', 'tmYRFormat', 'tmYRIrrTensionF',
+                'tmYRIrregularPoints', 'tmYRLabelAngleF', 'tmYRLabelConstantSpacingF',
+                'tmYRLabelDeltaF', 'tmYRLabelDirection', 'tmYRLabelFont',
+                'tmYRLabelFontAspectF', 'tmYRLabelFontColor', 'tmYRLabelFontHeightF',
+                'tmYRLabelFontQuality', 'tmYRLabelFontThicknessF',
+                'tmYRLabelFuncCode', 'tmYRLabelJust', 'tmYRLabelStride', 'tmYRLabels',
+                'tmYRLabelsOn', 'tmYRMajorLengthF', 'tmYRMajorLineColor',
+                'tmYRMajorOutwardLengthF', 'tmYRMajorThicknessF', 'tmYRMaxLabelLenF',
+                'tmYRMaxTicks', 'tmYRMinLabelSpacingF', 'tmYRMinorLengthF',
+                'tmYRMinorLineColor', 'tmYRMinorOn', 'tmYRMinorOutwardLengthF',
+                'tmYRMinorPerMajor', 'tmYRMinorThicknessF', 'tmYRMinorValues',
+                'tmYRMode', 'tmYROn', 'tmYRPrecision', 'tmYRStyle', 'tmYRTickEndF',
+                'tmYRTickSpacingF', 'tmYRTickStartF', 'tmYRValues', 'tmYUseLeft',
+                'trGridType', 'trLineInterpolationOn',
+                'trXAxisType', 'trXCoordPoints', 'trXInterPoints', 'trXLog',
+                'trXMaxF', 'trXMinF', 'trXReverse', 'trXSamples', 'trXTensionF',
+                'trYAxisType', 'trYCoordPoints', 'trYInterPoints', 'trYLog',
+                'trYMaxF', 'trYMinF', 'trYReverse', 'trYSamples', 'trYTensionF',
+                'txAngleF', 'txBackgroundFillColor', 'txConstantSpacingF', 'txDirection',
+                'txFont', 'HLU-Fonts', 'txFontAspectF', 'txFontColor',
+                'txFontHeightF', 'txFontOpacityF', 'txFontQuality',
+                'txFontThicknessF', 'txFuncCode', 'txJust', 'txPerimColor',
+                'txPerimDashLengthF', 'txPerimDashPattern', 'txPerimOn',
+                'txPerimSpaceF', 'txPerimThicknessF', 'txPosXF', 'txPosYF',
+                'txString', 'vcExplicitLabelBarLabelsOn', 'vcFillArrowEdgeColor',
+                'vcFillArrowEdgeThicknessF', 'vcFillArrowFillColor',
+                'vcFillArrowHeadInteriorXF', 'vcFillArrowHeadMinFracXF',
+                'vcFillArrowHeadMinFracYF', 'vcFillArrowHeadXF', 'vcFillArrowHeadYF',
+                'vcFillArrowMinFracWidthF', 'vcFillArrowWidthF', 'vcFillArrowsOn',
+                'vcFillOverEdge', 'vcGlyphOpacityF', 'vcGlyphStyle',
+                'vcLabelBarEndLabelsOn', 'vcLabelFontColor', 'vcLabelFontHeightF',
+                'vcLabelsOn', 'vcLabelsUseVectorColor', 'vcLevelColors',
+                'vcLevelCount', 'vcLevelPalette', 'vcLevelSelectionMode',
+                'vcLevelSpacingF', 'vcLevels', 'vcLineArrowColor',
+                'vcLineArrowHeadMaxSizeF', 'vcLineArrowHeadMinSizeF',
+                'vcLineArrowThicknessF', 'vcMagnitudeFormat',
+                'vcMagnitudeScaleFactorF', 'vcMagnitudeScaleValueF',
+                'vcMagnitudeScalingMode', 'vcMapDirection', 'vcMaxLevelCount',
+                'vcMaxLevelValF', 'vcMaxMagnitudeF', 'vcMinAnnoAngleF',
+                'vcMinAnnoArrowAngleF', 'vcMinAnnoArrowEdgeColor',
+                'vcMinAnnoArrowFillColor', 'vcMinAnnoArrowLineColor',
+                'vcMinAnnoArrowMinOffsetF', 'vcMinAnnoArrowSpaceF',
+                'vcMinAnnoArrowUseVecColor', 'vcMinAnnoBackgroundColor',
+                'vcMinAnnoConstantSpacingF', 'vcMinAnnoExplicitMagnitudeF',
+                'vcMinAnnoFont', 'vcMinAnnoFontAspectF', 'vcMinAnnoFontColor',
+                'vcMinAnnoFontHeightF', 'vcMinAnnoFontQuality',
+                'vcMinAnnoFontThicknessF', 'vcMinAnnoFuncCode', 'vcMinAnnoJust',
+                'vcMinAnnoOn', 'vcMinAnnoOrientation', 'vcMinAnnoOrthogonalPosF',
+                'vcMinAnnoParallelPosF', 'vcMinAnnoPerimColor', 'vcMinAnnoPerimOn',
+                'vcMinAnnoPerimSpaceF', 'vcMinAnnoPerimThicknessF', 'vcMinAnnoSide',
+                'vcMinAnnoString1', 'vcMinAnnoString1On', 'vcMinAnnoString2',
+                'vcMinAnnoString2On', 'vcMinAnnoTextDirection', 'vcMinAnnoZone',
+                'vcMinDistanceF', 'vcMinFracLengthF', 'vcMinLevelValF',
+                'vcMinMagnitudeF', 'vcMonoFillArrowEdgeColor',
+                'vcMonoFillArrowFillColor', 'vcMonoLineArrowColor',
+                'vcMonoWindBarbColor', 'vcNoDataLabelOn', 'vcNoDataLabelString',
+                'vcPositionMode', 'vcRefAnnoAngleF', 'vcRefAnnoArrowAngleF',
+                'vcRefAnnoArrowEdgeColor', 'vcRefAnnoArrowFillColor',
+                'vcRefAnnoArrowLineColor', 'vcRefAnnoArrowMinOffsetF',
+                'vcRefAnnoArrowSpaceF', 'vcRefAnnoArrowUseVecColor',
+                'vcRefAnnoBackgroundColor', 'vcRefAnnoConstantSpacingF',
+                'vcRefAnnoExplicitMagnitudeF', 'vcRefAnnoFont',
+                'vcRefAnnoFontAspectF', 'vcRefAnnoFontColor', 'vcRefAnnoFontHeightF',
+                'vcRefAnnoFontQuality', 'vcRefAnnoFontThicknessF',
+                'vcRefAnnoFuncCode', 'vcRefAnnoJust', 'vcRefAnnoOn',
+                'vcRefAnnoOrientation', 'vcRefAnnoOrthogonalPosF',
+                'vcRefAnnoParallelPosF', 'vcRefAnnoPerimColor', 'vcRefAnnoPerimOn',
+                'vcRefAnnoPerimSpaceF', 'vcRefAnnoPerimThicknessF', 'vcRefAnnoSide',
+                'vcRefAnnoString1', 'vcRefAnnoString1On', 'vcRefAnnoString2',
+                'vcRefAnnoString2On', 'vcRefAnnoTextDirection', 'vcRefAnnoZone',
+                'vcRefLengthF', 'vcRefMagnitudeF', 'vcScalarFieldData',
+                'vcScalarMissingValColor', 'vcScalarValueFormat',
+                'vcScalarValueScaleFactorF', 'vcScalarValueScaleValueF',
+                'vcScalarValueScalingMode', 'vcSpanLevelPalette', 'vcUseRefAnnoRes',
+                'vcUseScalarArray', 'vcVectorDrawOrder', 'vcVectorFieldData',
+                'vcWindBarbCalmCircleSizeF', 'vcWindBarbColor',
+                'vcWindBarbLineThicknessF', 'vcWindBarbScaleFactorF',
+                'vcWindBarbTickAngleF', 'vcWindBarbTickLengthF',
+                'vcWindBarbTickSpacingF', 'vcZeroFLabelAngleF',
+                'vcZeroFLabelBackgroundColor', 'vcZeroFLabelConstantSpacingF',
+                'vcZeroFLabelFont', 'vcZeroFLabelFontAspectF',
+                'vcZeroFLabelFontColor', 'vcZeroFLabelFontHeightF',
+                'vcZeroFLabelFontQuality', 'vcZeroFLabelFontThicknessF',
+                'vcZeroFLabelFuncCode', 'vcZeroFLabelJust', 'vcZeroFLabelOn',
+                'vcZeroFLabelOrthogonalPosF', 'vcZeroFLabelParallelPosF',
+                'vcZeroFLabelPerimColor', 'vcZeroFLabelPerimOn',
+                'vcZeroFLabelPerimSpaceF', 'vcZeroFLabelPerimThicknessF',
+                'vcZeroFLabelSide', 'vcZeroFLabelString', 'vcZeroFLabelTextDirection',
+                'vcZeroFLabelZone', 'vfCopyData', 'vfDataArray',
+                'vfExchangeDimensions', 'vfExchangeUVData', 'vfMagMaxV', 'vfMagMinV',
+                'vfMissingUValueV', 'vfMissingVValueV', 'vfPolarData',
+                'vfSingleMissingValue', 'vfUDataArray', 'vfUMaxV', 'vfUMinV',
+                'vfVDataArray', 'vfVMaxV', 'vfVMinV', 'vfXArray', 'vfXCActualEndF',
+                'vfXCActualStartF', 'vfXCEndIndex', 'vfXCEndSubsetV', 'vfXCEndV',
+                'vfXCStartIndex', 'vfXCStartSubsetV', 'vfXCStartV', 'vfXCStride',
+                'vfYArray', 'vfYCActualEndF', 'vfYCActualStartF', 'vfYCEndIndex',
+                'vfYCEndSubsetV', 'vfYCEndV', 'vfYCStartIndex', 'vfYCStartSubsetV',
+                'vfYCStartV', 'vfYCStride', 'vpAnnoManagerId', 'vpClipOn',
+                'vpHeightF', 'vpKeepAspect', 'vpOn', 'vpUseSegments', 'vpWidthF',
+                'vpXF', 'vpYF', 'wkAntiAlias', 'wkBackgroundColor', 'wkBackgroundOpacityF',
+                'wkColorMapLen', 'wkColorMap', 'wkColorModel', 'wkDashTableLength',
+                'wkDefGraphicStyleId', 'wkDeviceLowerX', 'wkDeviceLowerY',
+                'wkDeviceUpperX', 'wkDeviceUpperY', 'wkFileName', 'wkFillTableLength',
+                'wkForegroundColor', 'wkFormat', 'wkFullBackground', 'wkGksWorkId',
+                'wkHeight', 'wkMarkerTableLength', 'wkMetaName', 'wkOrientation',
+                'wkPDFFileName', 'wkPDFFormat', 'wkPDFResolution', 'wkPSFileName',
+                'wkPSFormat', 'wkPSResolution', 'wkPaperHeightF', 'wkPaperSize',
+                'wkPaperWidthF', 'wkPause', 'wkTopLevelViews', 'wkViews',
+                'wkVisualType', 'wkWidth', 'wkWindowId', 'wkXColorMode', 'wsCurrentSize',
+                'wsMaximumSize', 'wsThresholdSize', 'xyComputeXMax',
+                'xyComputeXMin', 'xyComputeYMax', 'xyComputeYMin', 'xyCoordData',
+                'xyCoordDataSpec', 'xyCurveDrawOrder', 'xyDashPattern',
+                'xyDashPatterns', 'xyExplicitLabels', 'xyExplicitLegendLabels',
+                'xyLabelMode', 'xyLineColor', 'xyLineColors', 'xyLineDashSegLenF',
+                'xyLineLabelConstantSpacingF', 'xyLineLabelFont',
+                'xyLineLabelFontAspectF', 'xyLineLabelFontColor',
+                'xyLineLabelFontColors', 'xyLineLabelFontHeightF',
+                'xyLineLabelFontQuality', 'xyLineLabelFontThicknessF',
+                'xyLineLabelFuncCode', 'xyLineThicknessF', 'xyLineThicknesses',
+                'xyMarkLineMode', 'xyMarkLineModes', 'xyMarker', 'xyMarkerColor',
+                'xyMarkerColors', 'xyMarkerSizeF', 'xyMarkerSizes',
+                'xyMarkerThicknessF', 'xyMarkerThicknesses', 'xyMarkers',
+                'xyMonoDashPattern', 'xyMonoLineColor', 'xyMonoLineLabelFontColor',
+                'xyMonoLineThickness', 'xyMonoMarkLineMode', 'xyMonoMarker',
+                'xyMonoMarkerColor', 'xyMonoMarkerSize', 'xyMonoMarkerThickness',
+                'xyXIrrTensionF', 'xyXIrregularPoints', 'xyXStyle', 'xyYIrrTensionF',
+                'xyYIrregularPoints', 'xyYStyle'), prefix=r'\b'),
+             Name.Builtin),
+
+            # Booleans
+            (r'\.(True|False)\.', Name.Builtin),
+            # Comparing Operators
+            (r'\.(eq|ne|lt|le|gt|ge|not|and|or|xor)\.', Operator.Word),
+        ],
+
+        'strings': [
+            (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double),
+        ],
+
+        'nums': [
+            (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer),
+            (r'[+-]?\d*\.\d+(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float),
+            (r'[+-]?\d+\.\d*(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/nimrod.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/nimrod.py
new file mode 100644
index 0000000000000000000000000000000000000000..b99e246d707f30d4c0c3fab50d7293fcd34e3510
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/nimrod.py
@@ -0,0 +1,199 @@
+"""
+    pygments.lexers.nimrod
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Nim language (formerly known as Nimrod).
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, default, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error
+
+__all__ = ['NimrodLexer']
+
+
+class NimrodLexer(RegexLexer):
+    """
+    For Nim source code.
+    """
+
+    name = 'Nimrod'
+    url = 'http://nim-lang.org/'
+    aliases = ['nimrod', 'nim']
+    filenames = ['*.nim', '*.nimrod']
+    mimetypes = ['text/x-nim']
+    version_added = '1.5'
+
+    flags = re.MULTILINE | re.IGNORECASE
+
+    def underscorize(words):
+        newWords = []
+        new = []
+        for word in words:
+            for ch in word:
+                new.append(ch)
+                new.append("_?")
+            newWords.append(''.join(new))
+            new = []
+        return "|".join(newWords)
+
+    keywords = [
+        'addr', 'and', 'as', 'asm', 'bind', 'block', 'break', 'case',
+        'cast', 'concept', 'const', 'continue', 'converter', 'defer', 'discard',
+        'distinct', 'div', 'do', 'elif', 'else', 'end', 'enum', 'except',
+        'export', 'finally', 'for', 'if', 'in', 'yield', 'interface',
+        'is', 'isnot', 'iterator', 'let', 'mixin', 'mod',
+        'not', 'notin', 'object', 'of', 'or', 'out', 'ptr', 'raise',
+        'ref', 'return', 'shl', 'shr', 'static', 'try',
+        'tuple', 'type', 'using', 'when', 'while', 'xor'
+    ]
+
+    keywordsPseudo = [
+        'nil', 'true', 'false'
+    ]
+
+    opWords = [
+        'and', 'or', 'not', 'xor', 'shl', 'shr', 'div', 'mod', 'in',
+        'notin', 'is', 'isnot'
+    ]
+
+    types = [
+        'int', 'int8', 'int16', 'int32', 'int64', 'float', 'float32', 'float64',
+        'bool', 'char', 'range', 'array', 'seq', 'set', 'string'
+    ]
+
+    tokens = {
+        'root': [
+            # Comments
+            (r'##\[', String.Doc, 'doccomment'),
+            (r'##.*$', String.Doc),
+            (r'#\[', Comment.Multiline, 'comment'),
+            (r'#.*$', Comment),
+
+            # Pragmas
+            (r'\{\.', String.Other, 'pragma'),
+
+            # Operators
+            (r'[*=><+\-/@$~&%!?|\\\[\]]', Operator),
+            (r'\.\.|\.|,|\[\.|\.\]|\{\.|\.\}|\(\.|\.\)|\{|\}|\(|\)|:|\^|`|;',
+             Punctuation),
+
+            # Case statement branch
+            (r'(\n\s*)(of)(\s)', bygroups(Text.Whitespace, Keyword,
+                                          Text.Whitespace), 'casebranch'),
+
+            # Strings
+            (r'(?:[\w]+)"', String, 'rdqs'),
+            (r'"""', String.Double, 'tdqs'),
+            ('"', String, 'dqs'),
+
+            # Char
+            ("'", String.Char, 'chars'),
+
+            # Keywords
+            (rf'({underscorize(opWords)})\b', Operator.Word),
+            (r'(proc|func|method|macro|template)(\s)(?![(\[\]])',
+             bygroups(Keyword, Text.Whitespace), 'funcname'),
+            (rf'({underscorize(keywords)})\b', Keyword),
+            (r'({})\b'.format(underscorize(['from', 'import', 'include', 'export'])),
+             Keyword.Namespace),
+            (r'(v_?a_?r)\b', Keyword.Declaration),
+            (rf'({underscorize(types)})\b', Name.Builtin),
+            (rf'({underscorize(keywordsPseudo)})\b', Keyword.Pseudo),
+
+            # Identifiers
+            (r'\b((?![_\d])\w)(((?!_)\w)|(_(?!_)\w))*', Name),
+
+            # Numbers
+            (r'[0-9][0-9_]*(?=([e.]|\'f(32|64)))',
+             Number.Float, ('float-suffix', 'float-number')),
+            (r'0x[a-f0-9][a-f0-9_]*', Number.Hex, 'int-suffix'),
+            (r'0b[01][01_]*', Number.Bin, 'int-suffix'),
+            (r'0o[0-7][0-7_]*', Number.Oct, 'int-suffix'),
+            (r'[0-9][0-9_]*', Number.Integer, 'int-suffix'),
+
+            # Whitespace
+            (r'\s+', Text.Whitespace),
+            (r'.+$', Error),
+        ],
+        'chars': [
+            (r'\\([\\abcefnrtvl"\']|x[a-f0-9]{2}|[0-9]{1,3})', String.Escape),
+            (r"'", String.Char, '#pop'),
+            (r".", String.Char)
+        ],
+        'strings': [
+            (r'(?|>=|>>|>|<=|<<|<|\+|-|=|/|\*|%|\+=|-=|!|@', Operator),
+            (r'\(|\)|\[|\]|,|\.\.\.|\.\.|\.|::|:', Punctuation),
+            (r'`\{[^`]*`\}', Text),  # Extern blocks won't be Lexed by Nit
+            (r'[\r\n\t ]+', Text),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/nix.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/nix.py
new file mode 100644
index 0000000000000000000000000000000000000000..16572afe48a097271dcf017a1d1f53e89b000916
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/nix.py
@@ -0,0 +1,144 @@
+"""
+    pygments.lexers.nix
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the NixOS Nix language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Literal
+
+__all__ = ['NixLexer']
+
+
+class NixLexer(RegexLexer):
+    """
+    For the Nix language.
+    """
+
+    name = 'Nix'
+    url = 'http://nixos.org/nix/'
+    aliases = ['nixos', 'nix']
+    filenames = ['*.nix']
+    mimetypes = ['text/x-nix']
+    version_added = '2.0'
+
+    keywords = ['rec', 'with', 'let', 'in', 'inherit', 'assert', 'if',
+                'else', 'then', '...']
+    builtins = ['import', 'abort', 'baseNameOf', 'dirOf', 'isNull', 'builtins',
+                'map', 'removeAttrs', 'throw', 'toString', 'derivation']
+    operators = ['++', '+', '?', '.', '!', '//', '==', '/',
+                 '!=', '&&', '||', '->', '=', '<', '>', '*', '-']
+
+    punctuations = ["(", ")", "[", "]", ";", "{", "}", ":", ",", "@"]
+
+    tokens = {
+        'root': [
+            # comments starting with #
+            (r'#.*$', Comment.Single),
+
+            # multiline comments
+            (r'/\*', Comment.Multiline, 'comment'),
+
+            # whitespace
+            (r'\s+', Text),
+
+            # keywords
+            ('({})'.format('|'.join(re.escape(entry) + '\\b' for entry in keywords)), Keyword),
+
+            # highlight the builtins
+            ('({})'.format('|'.join(re.escape(entry) + '\\b' for entry in builtins)),
+             Name.Builtin),
+
+            (r'\b(true|false|null)\b', Name.Constant),
+
+            # floats
+            (r'-?(\d+\.\d*|\.\d+)([eE][-+]?\d+)?', Number.Float),
+
+            # integers
+            (r'-?[0-9]+', Number.Integer),
+
+            # paths
+            (r'[\w.+-]*(\/[\w.+-]+)+', Literal),
+            (r'~(\/[\w.+-]+)+', Literal),
+            (r'\<[\w.+-]+(\/[\w.+-]+)*\>', Literal),
+
+            # operators
+            ('({})'.format('|'.join(re.escape(entry) for entry in operators)),
+             Operator),
+
+            # word operators
+            (r'\b(or|and)\b', Operator.Word),
+
+            (r'\{', Punctuation, 'block'),
+
+            # punctuations
+            ('({})'.format('|'.join(re.escape(entry) for entry in punctuations)), Punctuation),
+
+            # strings
+            (r'"', String.Double, 'doublequote'),
+            (r"''", String.Multiline, 'multiline'),
+
+            # urls
+            (r'[a-zA-Z][a-zA-Z0-9\+\-\.]*\:[\w%/?:@&=+$,\\.!~*\'-]+', Literal),
+
+            # names of variables
+            (r'[\w-]+(?=\s*=)', String.Symbol),
+            (r'[a-zA-Z_][\w\'-]*', Text),
+
+            (r"\$\{", String.Interpol, 'antiquote'),
+        ],
+        'comment': [
+            (r'[^/*]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline),
+        ],
+        'multiline': [
+            (r"''(\$|'|\\n|\\r|\\t|\\)", String.Escape),
+            (r"''", String.Multiline, '#pop'),
+            (r'\$\{', String.Interpol, 'antiquote'),
+            (r"[^'\$]+", String.Multiline),
+            (r"\$[^\{']", String.Multiline),
+            (r"'[^']", String.Multiline),
+            (r"\$(?=')", String.Multiline),
+        ],
+        'doublequote': [
+            (r'\\(\\|"|\$|n)', String.Escape),
+            (r'"', String.Double, '#pop'),
+            (r'\$\{', String.Interpol, 'antiquote'),
+            (r'[^"\\\$]+', String.Double),
+            (r'\$[^\{"]', String.Double),
+            (r'\$(?=")', String.Double),
+            (r'\\', String.Double),
+        ],
+        'antiquote': [
+            (r"\}", String.Interpol, '#pop'),
+            # TODO: we should probably escape also here ''${ \${
+            (r"\$\{", String.Interpol, '#push'),
+            include('root'),
+        ],
+        'block': [
+            (r"\}", Punctuation, '#pop'),
+            include('root'),
+        ],
+    }
+
+    def analyse_text(text):
+        rv = 0.0
+        # TODO: let/in
+        if re.search(r'import.+?<[^>]+>', text):
+            rv += 0.4
+        if re.search(r'mkDerivation\s+(\(|\{|rec)', text):
+            rv += 0.4
+        if re.search(r'=\s+mkIf\s+', text):
+            rv += 0.4
+        if re.search(r'\{[a-zA-Z,\s]+\}:', text):
+            rv += 0.1
+        return rv
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/numbair.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/numbair.py
new file mode 100644
index 0000000000000000000000000000000000000000..28395b41a4d302aadba80c4413293dd288779407
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/numbair.py
@@ -0,0 +1,63 @@
+"""
+    pygments.lexers.numbair
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for other Numba Intermediate Representation.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, words
+from pygments.token import Whitespace, Name, String,  Punctuation, Keyword, \
+    Operator, Number
+
+__all__ = ["NumbaIRLexer"]
+
+class NumbaIRLexer(RegexLexer):
+    """
+    Lexer for Numba IR
+    """
+    name = 'Numba_IR'
+    url = "https://numba.readthedocs.io/en/stable/developer/architecture.html#stage-2-generate-the-numba-ir"
+    aliases = ['numba_ir', 'numbair']
+    filenames = ['*.numba_ir']
+    mimetypes = ['text/x-numba_ir', 'text/x-numbair']
+    version_added = '2.19'
+
+    identifier = r'\$[a-zA-Z0-9._]+'
+    fun_or_var = r'([a-zA-Z_]+[a-zA-Z0-9]*)'
+
+    tokens = {
+        'root' : [
+            (r'(label)(\ [0-9]+)(:)$',
+                bygroups(Keyword, Name.Label, Punctuation)),
+
+            (r'=', Operator),
+            include('whitespace'),
+            include('keyword'),
+
+            (identifier, Name.Variable),
+            (fun_or_var + r'(\()',
+                bygroups(Name.Function, Punctuation)),
+            (fun_or_var + r'(\=)',
+                bygroups(Name.Attribute, Punctuation)),
+            (fun_or_var, Name.Constant),
+            (r'[0-9]+', Number),
+
+            # 
+            (r'<[^>\n]*>', String),
+
+            (r'[=<>{}\[\]()*.,!\':]|x\b', Punctuation)
+        ],
+
+        'keyword':[
+            (words((
+                'del', 'jump', 'call', 'branch',
+            ), suffix=' '), Keyword),
+        ],
+
+        'whitespace': [
+            (r'(\n|\s)+', Whitespace),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/oberon.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/oberon.py
new file mode 100644
index 0000000000000000000000000000000000000000..11cf571dfd6097354117f2842246eeb06ff696ea
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/oberon.py
@@ -0,0 +1,120 @@
+"""
+    pygments.lexers.oberon
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Oberon family languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['ComponentPascalLexer']
+
+
+class ComponentPascalLexer(RegexLexer):
+    """
+    For Component Pascal source code.
+    """
+    name = 'Component Pascal'
+    aliases = ['componentpascal', 'cp']
+    filenames = ['*.cp', '*.cps']
+    mimetypes = ['text/x-component-pascal']
+    url = 'https://blackboxframework.org'
+    version_added = '2.1'
+
+    flags = re.MULTILINE | re.DOTALL
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            include('comments'),
+            include('punctuation'),
+            include('numliterals'),
+            include('strings'),
+            include('operators'),
+            include('builtins'),
+            include('identifiers'),
+        ],
+        'whitespace': [
+            (r'\n+', Text),  # blank lines
+            (r'\s+', Text),  # whitespace
+        ],
+        'comments': [
+            (r'\(\*([^$].*?)\*\)', Comment.Multiline),
+            # TODO: nested comments (* (* ... *) ... (* ... *) *) not supported!
+        ],
+        'punctuation': [
+            (r'[()\[\]{},.:;|]', Punctuation),
+        ],
+        'numliterals': [
+            (r'[0-9A-F]+X\b', Number.Hex),                 # char code
+            (r'[0-9A-F]+[HL]\b', Number.Hex),              # hexadecimal number
+            (r'[0-9]+\.[0-9]+E[+-][0-9]+', Number.Float),  # real number
+            (r'[0-9]+\.[0-9]+', Number.Float),             # real number
+            (r'[0-9]+', Number.Integer),                   # decimal whole number
+        ],
+        'strings': [
+            (r"'[^\n']*'", String),  # single quoted string
+            (r'"[^\n"]*"', String),  # double quoted string
+        ],
+        'operators': [
+            # Arithmetic Operators
+            (r'[+-]', Operator),
+            (r'[*/]', Operator),
+            # Relational Operators
+            (r'[=#<>]', Operator),
+            # Dereferencing Operator
+            (r'\^', Operator),
+            # Logical AND Operator
+            (r'&', Operator),
+            # Logical NOT Operator
+            (r'~', Operator),
+            # Assignment Symbol
+            (r':=', Operator),
+            # Range Constructor
+            (r'\.\.', Operator),
+            (r'\$', Operator),
+        ],
+        'identifiers': [
+            (r'([a-zA-Z_$][\w$]*)', Name),
+        ],
+        'builtins': [
+            (words((
+                'ANYPTR', 'ANYREC', 'BOOLEAN', 'BYTE', 'CHAR', 'INTEGER', 'LONGINT',
+                'REAL', 'SET', 'SHORTCHAR', 'SHORTINT', 'SHORTREAL'
+                ), suffix=r'\b'), Keyword.Type),
+            (words((
+                'ABS', 'ABSTRACT', 'ARRAY', 'ASH', 'ASSERT', 'BEGIN', 'BITS', 'BY',
+                'CAP', 'CASE', 'CHR', 'CLOSE', 'CONST', 'DEC', 'DIV', 'DO', 'ELSE',
+                'ELSIF', 'EMPTY', 'END', 'ENTIER', 'EXCL', 'EXIT', 'EXTENSIBLE', 'FOR',
+                'HALT', 'IF', 'IMPORT', 'IN', 'INC', 'INCL', 'IS', 'LEN', 'LIMITED',
+                'LONG', 'LOOP', 'MAX', 'MIN', 'MOD', 'MODULE', 'NEW', 'ODD', 'OF',
+                'OR', 'ORD', 'OUT', 'POINTER', 'PROCEDURE', 'RECORD', 'REPEAT', 'RETURN',
+                'SHORT', 'SHORTCHAR', 'SHORTINT', 'SIZE', 'THEN', 'TYPE', 'TO', 'UNTIL',
+                'VAR', 'WHILE', 'WITH'
+                ), suffix=r'\b'), Keyword.Reserved),
+            (r'(TRUE|FALSE|NIL|INF)\b', Keyword.Constant),
+        ]
+    }
+
+    def analyse_text(text):
+        """The only other lexer using .cp is the C++ one, so we check if for
+        a few common Pascal keywords here. Those are unfortunately quite
+        common across various business languages as well."""
+        result = 0
+        if 'BEGIN' in text:
+            result += 0.01
+        if 'END' in text:
+            result += 0.01
+        if 'PROCEDURE' in text:
+            result += 0.01
+        if 'MODULE' in text:
+            result += 0.01
+
+        return result
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/objective.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/objective.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce281fbd198802ba63742086f850b339d82e3b0d
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/objective.py
@@ -0,0 +1,513 @@
+"""
+    pygments.lexers.objective
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Objective-C family languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, this, words, \
+    inherit, default
+from pygments.token import Text, Keyword, Name, String, Operator, \
+    Number, Punctuation, Literal, Comment, Whitespace
+
+from pygments.lexers.c_cpp import CLexer, CppLexer
+
+__all__ = ['ObjectiveCLexer', 'ObjectiveCppLexer', 'LogosLexer', 'SwiftLexer']
+
+
+def objective(baselexer):
+    """
+    Generate a subclass of baselexer that accepts the Objective-C syntax
+    extensions.
+    """
+
+    # Have to be careful not to accidentally match JavaDoc/Doxygen syntax here,
+    # since that's quite common in ordinary C/C++ files.  It's OK to match
+    # JavaDoc/Doxygen keywords that only apply to Objective-C, mind.
+    #
+    # The upshot of this is that we CANNOT match @class or @interface
+    _oc_keywords = re.compile(r'@(?:end|implementation|protocol)')
+
+    # Matches [ ? identifier  ( identifier ? ] |  identifier? : )
+    # (note the identifier is *optional* when there is a ':'!)
+    _oc_message = re.compile(r'\[\s*[a-zA-Z_]\w*\s+'
+                             r'(?:[a-zA-Z_]\w*\s*\]|'
+                             r'(?:[a-zA-Z_]\w*)?:)')
+
+    class GeneratedObjectiveCVariant(baselexer):
+        """
+        Implements Objective-C syntax on top of an existing C family lexer.
+        """
+
+        tokens = {
+            'statements': [
+                (r'@"', String, 'string'),
+                (r'@(YES|NO)', Number),
+                (r"@'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+                (r'@(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
+                (r'@(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+                (r'@0x[0-9a-fA-F]+[Ll]?', Number.Hex),
+                (r'@0[0-7]+[Ll]?', Number.Oct),
+                (r'@\d+[Ll]?', Number.Integer),
+                (r'@\(', Literal, 'literal_number'),
+                (r'@\[', Literal, 'literal_array'),
+                (r'@\{', Literal, 'literal_dictionary'),
+                (words((
+                    '@selector', '@private', '@protected', '@public', '@encode',
+                    '@synchronized', '@try', '@throw', '@catch', '@finally',
+                    '@end', '@property', '@synthesize', '__bridge', '__bridge_transfer',
+                    '__autoreleasing', '__block', '__weak', '__strong', 'weak', 'strong',
+                    'copy', 'retain', 'assign', 'unsafe_unretained', 'atomic', 'nonatomic',
+                    'readonly', 'readwrite', 'setter', 'getter', 'typeof', 'in',
+                    'out', 'inout', 'release', 'class', '@dynamic', '@optional',
+                    '@required', '@autoreleasepool', '@import'), suffix=r'\b'),
+                 Keyword),
+                (words(('id', 'instancetype', 'Class', 'IMP', 'SEL', 'BOOL',
+                        'IBOutlet', 'IBAction', 'unichar'), suffix=r'\b'),
+                 Keyword.Type),
+                (r'@(true|false|YES|NO)\n', Name.Builtin),
+                (r'(YES|NO|nil|self|super)\b', Name.Builtin),
+                # Carbon types
+                (r'(Boolean|UInt8|SInt8|UInt16|SInt16|UInt32|SInt32)\b', Keyword.Type),
+                # Carbon built-ins
+                (r'(TRUE|FALSE)\b', Name.Builtin),
+                (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Text),
+                 ('#pop', 'oc_classname')),
+                (r'(@class|@protocol)(\s+)', bygroups(Keyword, Text),
+                 ('#pop', 'oc_forward_classname')),
+                # @ can also prefix other expressions like @{...} or @(...)
+                (r'@', Punctuation),
+                inherit,
+            ],
+            'oc_classname': [
+                # interface definition that inherits
+                (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)(\{)',
+                 bygroups(Name.Class, Text, Name.Class, Text, Punctuation),
+                 ('#pop', 'oc_ivars')),
+                (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?',
+                 bygroups(Name.Class, Text, Name.Class), '#pop'),
+                # interface definition for a category
+                (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)(\{)',
+                 bygroups(Name.Class, Text, Name.Label, Text, Punctuation),
+                 ('#pop', 'oc_ivars')),
+                (r'([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))',
+                 bygroups(Name.Class, Text, Name.Label), '#pop'),
+                # simple interface / implementation
+                (r'([a-zA-Z$_][\w$]*)(\s*)(\{)',
+                 bygroups(Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')),
+                (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop')
+            ],
+            'oc_forward_classname': [
+                (r'([a-zA-Z$_][\w$]*)(\s*,\s*)',
+                 bygroups(Name.Class, Text), 'oc_forward_classname'),
+                (r'([a-zA-Z$_][\w$]*)(\s*;?)',
+                 bygroups(Name.Class, Text), '#pop')
+            ],
+            'oc_ivars': [
+                include('whitespace'),
+                include('statements'),
+                (';', Punctuation),
+                (r'\{', Punctuation, '#push'),
+                (r'\}', Punctuation, '#pop'),
+            ],
+            'root': [
+                # methods
+                (r'^([-+])(\s*)'                         # method marker
+                 r'(\(.*?\))?(\s*)'                      # return type
+                 r'([a-zA-Z$_][\w$]*:?)',        # begin of method name
+                 bygroups(Punctuation, Text, using(this),
+                          Text, Name.Function),
+                 'method'),
+                inherit,
+            ],
+            'method': [
+                include('whitespace'),
+                # TODO unsure if ellipses are allowed elsewhere, see
+                # discussion in Issue 789
+                (r',', Punctuation),
+                (r'\.\.\.', Punctuation),
+                (r'(\(.*?\))(\s*)([a-zA-Z$_][\w$]*)',
+                 bygroups(using(this), Text, Name.Variable)),
+                (r'[a-zA-Z$_][\w$]*:', Name.Function),
+                (';', Punctuation, '#pop'),
+                (r'\{', Punctuation, 'function'),
+                default('#pop'),
+            ],
+            'literal_number': [
+                (r'\(', Punctuation, 'literal_number_inner'),
+                (r'\)', Literal, '#pop'),
+                include('statement'),
+            ],
+            'literal_number_inner': [
+                (r'\(', Punctuation, '#push'),
+                (r'\)', Punctuation, '#pop'),
+                include('statement'),
+            ],
+            'literal_array': [
+                (r'\[', Punctuation, 'literal_array_inner'),
+                (r'\]', Literal, '#pop'),
+                include('statement'),
+            ],
+            'literal_array_inner': [
+                (r'\[', Punctuation, '#push'),
+                (r'\]', Punctuation, '#pop'),
+                include('statement'),
+            ],
+            'literal_dictionary': [
+                (r'\}', Literal, '#pop'),
+                include('statement'),
+            ],
+        }
+
+        def analyse_text(text):
+            if _oc_keywords.search(text):
+                return 1.0
+            elif '@"' in text:  # strings
+                return 0.8
+            elif re.search('@[0-9]+', text):
+                return 0.7
+            elif _oc_message.search(text):
+                return 0.8
+            return 0
+
+        def get_tokens_unprocessed(self, text, stack=('root',)):
+            from pygments.lexers._cocoa_builtins import COCOA_INTERFACES, \
+                COCOA_PROTOCOLS, COCOA_PRIMITIVES
+
+            for index, token, value in \
+                    baselexer.get_tokens_unprocessed(self, text, stack):
+                if token is Name or token is Name.Class:
+                    if value in COCOA_INTERFACES or value in COCOA_PROTOCOLS \
+                       or value in COCOA_PRIMITIVES:
+                        token = Name.Builtin.Pseudo
+
+                yield index, token, value
+
+    return GeneratedObjectiveCVariant
+
+
+class ObjectiveCLexer(objective(CLexer)):
+    """
+    For Objective-C source code with preprocessor directives.
+    """
+
+    name = 'Objective-C'
+    url = 'https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/ProgrammingWithObjectiveC/Introduction/Introduction.html'
+    aliases = ['objective-c', 'objectivec', 'obj-c', 'objc']
+    filenames = ['*.m', '*.h']
+    mimetypes = ['text/x-objective-c']
+    version_added = ''
+    priority = 0.05    # Lower than C
+
+
+class ObjectiveCppLexer(objective(CppLexer)):
+    """
+    For Objective-C++ source code with preprocessor directives.
+    """
+
+    name = 'Objective-C++'
+    aliases = ['objective-c++', 'objectivec++', 'obj-c++', 'objc++']
+    filenames = ['*.mm', '*.hh']
+    mimetypes = ['text/x-objective-c++']
+    version_added = ''
+    priority = 0.05    # Lower than C++
+
+
+class LogosLexer(ObjectiveCppLexer):
+    """
+    For Logos + Objective-C source code with preprocessor directives.
+    """
+
+    name = 'Logos'
+    aliases = ['logos']
+    filenames = ['*.x', '*.xi', '*.xm', '*.xmi']
+    mimetypes = ['text/x-logos']
+    version_added = '1.6'
+    priority = 0.25
+
+    tokens = {
+        'statements': [
+            (r'(%orig|%log)\b', Keyword),
+            (r'(%c)\b(\()(\s*)([a-zA-Z$_][\w$]*)(\s*)(\))',
+             bygroups(Keyword, Punctuation, Text, Name.Class, Text, Punctuation)),
+            (r'(%init)\b(\()',
+             bygroups(Keyword, Punctuation), 'logos_init_directive'),
+            (r'(%init)(?=\s*;)', bygroups(Keyword)),
+            (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)',
+             bygroups(Keyword, Text, Name.Class), '#pop'),
+            (r'(%subclass)(\s+)', bygroups(Keyword, Text),
+             ('#pop', 'logos_classname')),
+            inherit,
+        ],
+        'logos_init_directive': [
+            (r'\s+', Text),
+            (',', Punctuation, ('logos_init_directive', '#pop')),
+            (r'([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)',
+             bygroups(Name.Class, Text, Punctuation, Text, Text)),
+            (r'([a-zA-Z$_][\w$]*)', Name.Class),
+            (r'\)', Punctuation, '#pop'),
+        ],
+        'logos_classname': [
+            (r'([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?',
+             bygroups(Name.Class, Text, Name.Class), '#pop'),
+            (r'([a-zA-Z$_][\w$]*)', Name.Class, '#pop')
+        ],
+        'root': [
+            (r'(%subclass)(\s+)', bygroups(Keyword, Text),
+             'logos_classname'),
+            (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)',
+             bygroups(Keyword, Text, Name.Class)),
+            (r'(%config)(\s*\(\s*)(\w+)(\s*=)(.*?)(\)\s*)',
+             bygroups(Keyword, Text, Name.Variable, Text, String, Text)),
+            (r'(%ctor)(\s*)(\{)', bygroups(Keyword, Text, Punctuation),
+             'function'),
+            (r'(%new)(\s*)(\()(.*?)(\))',
+             bygroups(Keyword, Text, Keyword, String, Keyword)),
+            (r'(\s*)(%end)(\s*)', bygroups(Text, Keyword, Text)),
+            inherit,
+        ],
+    }
+
+    _logos_keywords = re.compile(r'%(?:hook|ctor|init|c\()')
+
+    def analyse_text(text):
+        if LogosLexer._logos_keywords.search(text):
+            return 1.0
+        return 0
+
+
+class SwiftLexer(RegexLexer):
+    """
+    For Swift source.
+    """
+    name = 'Swift'
+    url = 'https://www.swift.org/'
+    filenames = ['*.swift']
+    aliases = ['swift']
+    mimetypes = ['text/x-swift']
+    version_added = '2.0'
+
+    tokens = {
+        'root': [
+            # Whitespace and Comments
+            (r'\n', Text),
+            (r'\s+', Whitespace),
+            (r'//', Comment.Single, 'comment-single'),
+            (r'/\*', Comment.Multiline, 'comment-multi'),
+            (r'#(if|elseif|else|endif|available)\b', Comment.Preproc, 'preproc'),
+
+            # Keywords
+            include('keywords'),
+
+            # Global Types
+            (words((
+                'Array', 'AutoreleasingUnsafeMutablePointer', 'BidirectionalReverseView',
+                'Bit', 'Bool', 'CFunctionPointer', 'COpaquePointer', 'CVaListPointer',
+                'Character', 'ClosedInterval', 'CollectionOfOne', 'ContiguousArray',
+                'Dictionary', 'DictionaryGenerator', 'DictionaryIndex', 'Double',
+                'EmptyCollection', 'EmptyGenerator', 'EnumerateGenerator',
+                'EnumerateSequence', 'FilterCollectionView',
+                'FilterCollectionViewIndex', 'FilterGenerator', 'FilterSequenceView',
+                'Float', 'Float80', 'FloatingPointClassification', 'GeneratorOf',
+                'GeneratorOfOne', 'GeneratorSequence', 'HalfOpenInterval', 'HeapBuffer',
+                'HeapBufferStorage', 'ImplicitlyUnwrappedOptional', 'IndexingGenerator',
+                'Int', 'Int16', 'Int32', 'Int64', 'Int8', 'LazyBidirectionalCollection',
+                'LazyForwardCollection', 'LazyRandomAccessCollection',
+                'LazySequence', 'MapCollectionView', 'MapSequenceGenerator',
+                'MapSequenceView', 'MirrorDisposition', 'ObjectIdentifier', 'OnHeap',
+                'Optional', 'PermutationGenerator', 'QuickLookObject',
+                'RandomAccessReverseView', 'Range', 'RangeGenerator', 'RawByte', 'Repeat',
+                'ReverseBidirectionalIndex', 'ReverseRandomAccessIndex', 'SequenceOf',
+                'SinkOf', 'Slice', 'StaticString', 'StrideThrough', 'StrideThroughGenerator',
+                'StrideTo', 'StrideToGenerator', 'String', 'UInt', 'UInt16', 'UInt32',
+                'UInt64', 'UInt8', 'UTF16', 'UTF32', 'UTF8', 'UnicodeDecodingResult',
+                'UnicodeScalar', 'Unmanaged', 'UnsafeBufferPointer',
+                'UnsafeBufferPointerGenerator', 'UnsafeMutableBufferPointer',
+                'UnsafeMutablePointer', 'UnsafePointer', 'Zip2', 'ZipGenerator2',
+                # Protocols
+                'AbsoluteValuable', 'AnyObject', 'ArrayLiteralConvertible',
+                'BidirectionalIndexType', 'BitwiseOperationsType',
+                'BooleanLiteralConvertible', 'BooleanType', 'CVarArgType',
+                'CollectionType', 'Comparable', 'DebugPrintable',
+                'DictionaryLiteralConvertible', 'Equatable',
+                'ExtendedGraphemeClusterLiteralConvertible',
+                'ExtensibleCollectionType', 'FloatLiteralConvertible',
+                'FloatingPointType', 'ForwardIndexType', 'GeneratorType', 'Hashable',
+                'IntegerArithmeticType', 'IntegerLiteralConvertible', 'IntegerType',
+                'IntervalType', 'MirrorType', 'MutableCollectionType', 'MutableSliceable',
+                'NilLiteralConvertible', 'OutputStreamType', 'Printable',
+                'RandomAccessIndexType', 'RangeReplaceableCollectionType',
+                'RawOptionSetType', 'RawRepresentable', 'Reflectable', 'SequenceType',
+                'SignedIntegerType', 'SignedNumberType', 'SinkType', 'Sliceable',
+                'Streamable', 'Strideable', 'StringInterpolationConvertible',
+                'StringLiteralConvertible', 'UnicodeCodecType',
+                'UnicodeScalarLiteralConvertible', 'UnsignedIntegerType',
+                '_ArrayBufferType', '_BidirectionalIndexType', '_CocoaStringType',
+                '_CollectionType', '_Comparable', '_ExtensibleCollectionType',
+                '_ForwardIndexType', '_Incrementable', '_IntegerArithmeticType',
+                '_IntegerType', '_ObjectiveCBridgeable', '_RandomAccessIndexType',
+                '_RawOptionSetType', '_SequenceType', '_Sequence_Type',
+                '_SignedIntegerType', '_SignedNumberType', '_Sliceable', '_Strideable',
+                '_SwiftNSArrayRequiredOverridesType', '_SwiftNSArrayType',
+                '_SwiftNSCopyingType', '_SwiftNSDictionaryRequiredOverridesType',
+                '_SwiftNSDictionaryType', '_SwiftNSEnumeratorType',
+                '_SwiftNSFastEnumerationType', '_SwiftNSStringRequiredOverridesType',
+                '_SwiftNSStringType', '_UnsignedIntegerType',
+                # Variables
+                'C_ARGC', 'C_ARGV', 'Process',
+                # Typealiases
+                'Any', 'AnyClass', 'BooleanLiteralType', 'CBool', 'CChar', 'CChar16',
+                'CChar32', 'CDouble', 'CFloat', 'CInt', 'CLong', 'CLongLong', 'CShort',
+                'CSignedChar', 'CUnsignedInt', 'CUnsignedLong', 'CUnsignedShort',
+                'CWideChar', 'ExtendedGraphemeClusterType', 'Float32', 'Float64',
+                'FloatLiteralType', 'IntMax', 'IntegerLiteralType', 'StringLiteralType',
+                'UIntMax', 'UWord', 'UnicodeScalarType', 'Void', 'Word',
+                # Foundation/Cocoa
+                'NSErrorPointer', 'NSObjectProtocol', 'Selector'), suffix=r'\b'),
+             Name.Builtin),
+            # Functions
+            (words((
+                'abs', 'advance', 'alignof', 'alignofValue', 'assert', 'assertionFailure',
+                'contains', 'count', 'countElements', 'debugPrint', 'debugPrintln',
+                'distance', 'dropFirst', 'dropLast', 'dump', 'enumerate', 'equal',
+                'extend', 'fatalError', 'filter', 'find', 'first', 'getVaList', 'indices',
+                'insert', 'isEmpty', 'join', 'last', 'lazy', 'lexicographicalCompare',
+                'map', 'max', 'maxElement', 'min', 'minElement', 'numericCast', 'overlaps',
+                'partition', 'precondition', 'preconditionFailure', 'prefix', 'print',
+                'println', 'reduce', 'reflect', 'removeAll', 'removeAtIndex', 'removeLast',
+                'removeRange', 'reverse', 'sizeof', 'sizeofValue', 'sort', 'sorted',
+                'splice', 'split', 'startsWith', 'stride', 'strideof', 'strideofValue',
+                'suffix', 'swap', 'toDebugString', 'toString', 'transcode',
+                'underestimateCount', 'unsafeAddressOf', 'unsafeBitCast', 'unsafeDowncast',
+                'withExtendedLifetime', 'withUnsafeMutablePointer',
+                'withUnsafeMutablePointers', 'withUnsafePointer', 'withUnsafePointers',
+                'withVaList'), suffix=r'\b'),
+             Name.Builtin.Pseudo),
+
+            # Implicit Block Variables
+            (r'\$\d+', Name.Variable),
+
+            # Binary Literal
+            (r'0b[01_]+', Number.Bin),
+            # Octal Literal
+            (r'0o[0-7_]+', Number.Oct),
+            # Hexadecimal Literal
+            (r'0x[0-9a-fA-F_]+', Number.Hex),
+            # Decimal Literal
+            (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|'
+             r'\.[0-9_]*|[eE][+\-]?[0-9_]+)', Number.Float),
+            (r'[0-9][0-9_]*', Number.Integer),
+            # String Literal
+            (r'"""', String, 'string-multi'),
+            (r'"', String, 'string'),
+
+            # Operators and Punctuation
+            (r'[(){}\[\].,:;=@#`?]|->|[<&?](?=\w)|(?<=\w)[>!?]', Punctuation),
+            (r'[/=\-+!*%<>&|^?~]+', Operator),
+
+            # Identifier
+            (r'[a-zA-Z_]\w*', Name)
+        ],
+        'keywords': [
+            (words((
+                'as', 'async', 'await', 'break', 'case', 'catch', 'continue', 'default', 'defer',
+                'do', 'else', 'fallthrough', 'for', 'guard', 'if', 'in', 'is',
+                'repeat', 'return', '#selector', 'switch', 'throw', 'try',
+                'where', 'while'), suffix=r'\b'),
+             Keyword),
+            (r'@availability\([^)]+\)', Keyword.Reserved),
+            (words((
+                'associativity', 'convenience', 'dynamic', 'didSet', 'final',
+                'get', 'indirect', 'infix', 'inout', 'lazy', 'left', 'mutating',
+                'none', 'nonmutating', 'optional', 'override', 'postfix',
+                'precedence', 'prefix', 'Protocol', 'required', 'rethrows',
+                'right', 'set', 'throws', 'Type', 'unowned', 'weak', 'willSet',
+                '@availability', '@autoclosure', '@noreturn',
+                '@NSApplicationMain', '@NSCopying', '@NSManaged', '@objc',
+                '@UIApplicationMain', '@IBAction', '@IBDesignable',
+                '@IBInspectable', '@IBOutlet'), suffix=r'\b'),
+             Keyword.Reserved),
+            (r'(as|dynamicType|false|is|nil|self|Self|super|true|__COLUMN__'
+             r'|__FILE__|__FUNCTION__|__LINE__|_'
+             r'|#(?:file|line|column|function))\b', Keyword.Constant),
+            (r'import\b', Keyword.Declaration, 'module'),
+            (r'(class|enum|extension|struct|protocol)(\s+)([a-zA-Z_]\w*)',
+             bygroups(Keyword.Declaration, Whitespace, Name.Class)),
+            (r'(func)(\s+)([a-zA-Z_]\w*)',
+             bygroups(Keyword.Declaration, Whitespace, Name.Function)),
+            (r'(var|let)(\s+)([a-zA-Z_]\w*)', bygroups(Keyword.Declaration,
+             Whitespace, Name.Variable)),
+            (words((
+                'actor', 'associatedtype', 'class', 'deinit', 'enum', 'extension', 'func', 'import',
+                'init', 'internal', 'let', 'operator', 'private', 'protocol', 'public',
+                'static', 'struct', 'subscript', 'typealias', 'var'), suffix=r'\b'),
+             Keyword.Declaration)
+        ],
+        'comment': [
+            (r':param: [a-zA-Z_]\w*|:returns?:|(FIXME|MARK|TODO):',
+             Comment.Special)
+        ],
+
+        # Nested
+        'comment-single': [
+            (r'\n', Whitespace, '#pop'),
+            include('comment'),
+            (r'[^\n]+', Comment.Single)
+        ],
+        'comment-multi': [
+            include('comment'),
+            (r'[^*/]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]+', Comment.Multiline)
+        ],
+        'module': [
+            (r'\n', Whitespace, '#pop'),
+            (r'[a-zA-Z_]\w*', Name.Class),
+            include('root')
+        ],
+        'preproc': [
+            (r'\n', Whitespace, '#pop'),
+            include('keywords'),
+            (r'[A-Za-z]\w*', Comment.Preproc),
+            include('root')
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            include("string-common"),
+        ],
+        'string-multi': [
+            (r'"""', String, '#pop'),
+            include("string-common"),
+        ],
+        'string-common': [
+            (r'\\\(', String.Interpol, 'string-intp'),
+            (r"""\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}"""
+             r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}""", String.Escape),
+            (r'[^\\"]+', String),
+            (r'\\', String)
+        ],
+        'string-intp': [
+            (r'\(', String.Interpol, '#push'),
+            (r'\)', String.Interpol, '#pop'),
+            include('root')
+        ]
+    }
+
+    def get_tokens_unprocessed(self, text):
+        from pygments.lexers._cocoa_builtins import COCOA_INTERFACES, \
+            COCOA_PROTOCOLS, COCOA_PRIMITIVES
+
+        for index, token, value in \
+                RegexLexer.get_tokens_unprocessed(self, text):
+            if token is Name or token is Name.Class:
+                if value in COCOA_INTERFACES or value in COCOA_PROTOCOLS \
+                   or value in COCOA_PRIMITIVES:
+                    token = Name.Builtin.Pseudo
+
+            yield index, token, value
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ooc.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ooc.py
new file mode 100644
index 0000000000000000000000000000000000000000..f6aea8c830689523903c913699927004208652bb
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ooc.py
@@ -0,0 +1,84 @@
+"""
+    pygments.lexers.ooc
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Ooc language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['OocLexer']
+
+
+class OocLexer(RegexLexer):
+    """
+    For Ooc source code
+    """
+    name = 'Ooc'
+    url = 'https://ooc-lang.github.io/'
+    aliases = ['ooc']
+    filenames = ['*.ooc']
+    mimetypes = ['text/x-ooc']
+    version_added = '1.2'
+
+    tokens = {
+        'root': [
+            (words((
+                'class', 'interface', 'implement', 'abstract', 'extends', 'from',
+                'this', 'super', 'new', 'const', 'final', 'static', 'import',
+                'use', 'extern', 'inline', 'proto', 'break', 'continue',
+                'fallthrough', 'operator', 'if', 'else', 'for', 'while', 'do',
+                'switch', 'case', 'as', 'in', 'version', 'return', 'true',
+                'false', 'null'), prefix=r'\b', suffix=r'\b'),
+             Keyword),
+            (r'include\b', Keyword, 'include'),
+            (r'(cover)([ \t]+)(from)([ \t]+)(\w+[*@]?)',
+             bygroups(Keyword, Text, Keyword, Text, Name.Class)),
+            (r'(func)((?:[ \t]|\\\n)+)(~[a-z_]\w*)',
+             bygroups(Keyword, Text, Name.Function)),
+            (r'\bfunc\b', Keyword),
+            # Note: %= not listed on https://ooc-lang.github.io/docs/lang/operators/
+            (r'//.*', Comment),
+            (r'(?s)/\*.*?\*/', Comment.Multiline),
+            (r'(==?|\+=?|-[=>]?|\*=?|/=?|:=|!=?|%=?|\?|>{1,3}=?|<{1,3}=?|\.\.|'
+             r'&&?|\|\|?|\^=?)', Operator),
+            (r'(\.)([ \t]*)([a-z]\w*)', bygroups(Operator, Text,
+                                                 Name.Function)),
+            (r'[A-Z][A-Z0-9_]+', Name.Constant),
+            (r'[A-Z]\w*([@*]|\[[ \t]*\])?', Name.Class),
+
+            (r'([a-z]\w*(?:~[a-z]\w*)?)((?:[ \t]|\\\n)*)(?=\()',
+             bygroups(Name.Function, Text)),
+            (r'[a-z]\w*', Name.Variable),
+
+            # : introduces types
+            (r'[:(){}\[\];,]', Punctuation),
+
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'0c[0-9]+', Number.Oct),
+            (r'0b[01]+', Number.Bin),
+            (r'[0-9_]\.[0-9_]*(?!\.)', Number.Float),
+            (r'[0-9_]+', Number.Decimal),
+
+            (r'"(?:\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\"])*"',
+             String.Double),
+            (r"'(?:\\.|\\[0-9]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'",
+             String.Char),
+            (r'@', Punctuation),  # pointer dereference
+            (r'\.', Punctuation),  # imports or chain operator
+
+            (r'\\[ \t\n]', Text),
+            (r'[ \t]+', Text),
+        ],
+        'include': [
+            (r'[\w/]+', Name),
+            (r',', Punctuation),
+            (r'[ \t]', Text),
+            (r'[;\n]', Text, '#pop'),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/openscad.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/openscad.py
new file mode 100644
index 0000000000000000000000000000000000000000..1ec8fea051ca0d0c1e13a281158bfe85c63ec365
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/openscad.py
@@ -0,0 +1,96 @@
+"""
+    pygments.lexers.openscad
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the OpenSCAD languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words, include
+from pygments.token import Text, Comment, Punctuation, Operator, Keyword, Name, Number, Whitespace, Literal, String
+
+__all__ = ['OpenScadLexer']
+
+
+class OpenScadLexer(RegexLexer):
+    """For openSCAD code.
+    """
+    name = "OpenSCAD"
+    url = "https://openscad.org/"
+    aliases = ["openscad"]
+    filenames = ["*.scad"]
+    mimetypes = ["application/x-openscad"]
+    version_added = '2.16'
+
+    tokens = {
+        "root": [
+            (r"[^\S\n]+", Whitespace),
+            (r'//', Comment.Single, 'comment-single'),
+            (r'/\*', Comment.Multiline, 'comment-multi'),
+            (r"[{}\[\]\(\),;:]", Punctuation),
+            (r"[*!#%\-+=?/]", Operator),
+            (r"<=|<|==|!=|>=|>|&&|\|\|", Operator),
+            (r"\$(f[asn]|t|vp[rtd]|children)", Operator),
+            (r"(undef|PI)\b", Keyword.Constant),
+            (
+                r"(use|include)((?:\s|\\\\s)+)",
+                bygroups(Keyword.Namespace, Text),
+                "includes",
+            ),
+            (r"(module)(\s*)([^\s\(]+)",
+             bygroups(Keyword.Namespace, Whitespace, Name.Namespace)),
+            (r"(function)(\s*)([^\s\(]+)",
+             bygroups(Keyword.Declaration, Whitespace, Name.Function)),
+            (words(("true", "false"), prefix=r"\b", suffix=r"\b"), Literal),
+            (words((
+                "function", "module", "include", "use", "for",
+                "intersection_for", "if", "else", "return"
+                ), prefix=r"\b", suffix=r"\b"), Keyword
+            ),
+            (words((
+                "circle", "square", "polygon", "text", "sphere", "cube",
+                "cylinder", "polyhedron", "translate", "rotate", "scale",
+                "resize", "mirror", "multmatrix", "color", "offset", "hull",
+                "minkowski", "union", "difference", "intersection", "abs",
+                "sign", "sin", "cos", "tan", "acos", "asin", "atan", "atan2",
+                "floor", "round", "ceil", "ln", "log", "pow", "sqrt", "exp",
+                "rands", "min", "max", "concat", "lookup", "str", "chr",
+                "search", "version", "version_num", "norm", "cross",
+                "parent_module", "echo", "import", "import_dxf",
+                "dxf_linear_extrude", "linear_extrude", "rotate_extrude",
+                "surface", "projection", "render", "dxf_cross",
+                "dxf_dim", "let", "assign", "len"
+                ), prefix=r"\b", suffix=r"\b"),
+                Name.Builtin
+            ),
+            (r"\bchildren\b", Name.Builtin.Pseudo),
+            (r'""".*?"""', String.Double),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"-?\d+(\.\d+)?(e[+-]?\d+)?", Number),
+            (r"\w+", Name),
+        ],
+        "includes": [
+            (
+                r"(<)([^>]*)(>)",
+                bygroups(Punctuation, Comment.PreprocFile, Punctuation),
+            ),
+        ],
+        'comment': [
+            (r':param: [a-zA-Z_]\w*|:returns?:|(FIXME|MARK|TODO):',
+             Comment.Special)
+        ],
+        'comment-single': [
+            (r'\n', Text, '#pop'),
+            include('comment'),
+            (r'[^\n]+', Comment.Single)
+        ],
+        'comment-multi': [
+            include('comment'),
+            (r'[^*/]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/other.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/other.py
new file mode 100644
index 0000000000000000000000000000000000000000..8767d73037b75095486aa46ad3b89a002156fc8b
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/other.py
@@ -0,0 +1,41 @@
+"""
+    pygments.lexers.other
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Just export lexer classes previously contained in this module.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+# ruff: noqa: F401
+from pygments.lexers.sql import SqlLexer, MySqlLexer, SqliteConsoleLexer
+from pygments.lexers.shell import BashLexer, BashSessionLexer, BatchLexer, \
+    TcshLexer
+from pygments.lexers.robotframework import RobotFrameworkLexer
+from pygments.lexers.testing import GherkinLexer
+from pygments.lexers.esoteric import BrainfuckLexer, BefungeLexer, RedcodeLexer
+from pygments.lexers.prolog import LogtalkLexer
+from pygments.lexers.snobol import SnobolLexer
+from pygments.lexers.rebol import RebolLexer
+from pygments.lexers.configs import KconfigLexer, Cfengine3Lexer
+from pygments.lexers.modeling import ModelicaLexer
+from pygments.lexers.scripting import AppleScriptLexer, MOOCodeLexer, \
+    HybrisLexer
+from pygments.lexers.graphics import PostScriptLexer, GnuplotLexer, \
+    AsymptoteLexer, PovrayLexer
+from pygments.lexers.business import ABAPLexer, OpenEdgeLexer, \
+    GoodDataCLLexer, MaqlLexer
+from pygments.lexers.automation import AutoItLexer, AutohotkeyLexer
+from pygments.lexers.dsls import ProtoBufLexer, BroLexer, PuppetLexer, \
+    MscgenLexer, VGLLexer
+from pygments.lexers.basic import CbmBasicV2Lexer
+from pygments.lexers.pawn import SourcePawnLexer, PawnLexer
+from pygments.lexers.ecl import ECLLexer
+from pygments.lexers.urbi import UrbiscriptLexer
+from pygments.lexers.smalltalk import SmalltalkLexer, NewspeakLexer
+from pygments.lexers.installers import NSISLexer, RPMSpecLexer
+from pygments.lexers.textedit import AwkLexer
+from pygments.lexers.smv import NuSMVLexer
+
+__all__ = []
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/parasail.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/parasail.py
new file mode 100644
index 0000000000000000000000000000000000000000..9b409609f8b1ea3211f0364273dd8280b65e0d0c
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/parasail.py
@@ -0,0 +1,78 @@
+"""
+    pygments.lexers.parasail
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for ParaSail.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Literal
+
+__all__ = ['ParaSailLexer']
+
+
+class ParaSailLexer(RegexLexer):
+    """
+    For ParaSail source code.
+    """
+
+    name = 'ParaSail'
+    url = 'http://www.parasail-lang.org'
+    aliases = ['parasail']
+    filenames = ['*.psi', '*.psl']
+    mimetypes = ['text/x-parasail']
+    version_added = '2.1'
+
+    flags = re.MULTILINE
+
+    tokens = {
+        'root': [
+            (r'[^\S\n]+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'\b(and|or|xor)=', Operator.Word),
+            (r'\b(and(\s+then)?|or(\s+else)?|xor|rem|mod|'
+             r'(is|not)\s+null)\b',
+             Operator.Word),
+            # Keywords
+            (r'\b(abs|abstract|all|block|class|concurrent|const|continue|'
+             r'each|end|exit|extends|exports|forward|func|global|implements|'
+             r'import|in|interface|is|lambda|locked|new|not|null|of|op|'
+             r'optional|private|queued|ref|return|reverse|separate|some|'
+             r'type|until|var|with|'
+             # Control flow
+             r'if|then|else|elsif|case|for|while|loop)\b',
+             Keyword.Reserved),
+            (r'(abstract\s+)?(interface|class|op|func|type)',
+             Keyword.Declaration),
+            # Literals
+            (r'"[^"]*"', String),
+            (r'\\[\'ntrf"0]', String.Escape),
+            (r'#[a-zA-Z]\w*', Literal),       # Enumeration
+            include('numbers'),
+            (r"'[^']'", String.Char),
+            (r'[a-zA-Z]\w*', Name),
+            # Operators and Punctuation
+            (r'(<==|==>|<=>|\*\*=|<\|=|<<=|>>=|==|!=|=\?|<=|>=|'
+             r'\*\*|<<|>>|=>|:=|\+=|-=|\*=|\|=|\||/=|\+|-|\*|/|'
+             r'\.\.|<\.\.|\.\.<|<\.\.<)',
+             Operator),
+            (r'(<|>|\[|\]|\(|\)|\||:|;|,|.|\{|\}|->)',
+             Punctuation),
+            (r'\n+', Text),
+        ],
+        'numbers': [
+            (r'\d[0-9_]*#[0-9a-fA-F][0-9a-fA-F_]*#', Number.Hex),  # any base
+            (r'0[xX][0-9a-fA-F][0-9a-fA-F_]*', Number.Hex),        # C-like hex
+            (r'0[bB][01][01_]*', Number.Bin),                      # C-like bin
+            (r'\d[0-9_]*\.\d[0-9_]*[eE][+-]\d[0-9_]*',             # float exp
+             Number.Float),
+            (r'\d[0-9_]*\.\d[0-9_]*', Number.Float),               # float
+            (r'\d[0-9_]*', Number.Integer),                        # integer
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/parsers.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/parsers.py
new file mode 100644
index 0000000000000000000000000000000000000000..baa49b05407cbb85e9f0b72d653fd862f0323d34
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/parsers.py
@@ -0,0 +1,798 @@
+"""
+    pygments.lexers.parsers
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for parser generators.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, DelegatingLexer, \
+    include, bygroups, using
+from pygments.token import Punctuation, Other, Text, Comment, Operator, \
+    Keyword, Name, String, Number, Whitespace
+from pygments.lexers.jvm import JavaLexer
+from pygments.lexers.c_cpp import CLexer, CppLexer
+from pygments.lexers.objective import ObjectiveCLexer
+from pygments.lexers.d import DLexer
+from pygments.lexers.dotnet import CSharpLexer
+from pygments.lexers.ruby import RubyLexer
+from pygments.lexers.python import PythonLexer
+from pygments.lexers.perl import PerlLexer
+
+__all__ = ['RagelLexer', 'RagelEmbeddedLexer', 'RagelCLexer', 'RagelDLexer',
+           'RagelCppLexer', 'RagelObjectiveCLexer', 'RagelRubyLexer',
+           'RagelJavaLexer', 'AntlrLexer', 'AntlrPythonLexer',
+           'AntlrPerlLexer', 'AntlrRubyLexer', 'AntlrCppLexer',
+           'AntlrCSharpLexer', 'AntlrObjectiveCLexer',
+           'AntlrJavaLexer', 'AntlrActionScriptLexer',
+           'TreetopLexer', 'EbnfLexer']
+
+
+class RagelLexer(RegexLexer):
+    """A pure `Ragel `_ lexer.
+
+    Use this for fragments of Ragel.  For ``.rl`` files, use
+    :class:`RagelEmbeddedLexer` instead (or one of the language-specific
+    subclasses).
+    """
+
+    name = 'Ragel'
+    url = 'http://www.colm.net/open-source/ragel/'
+    aliases = ['ragel']
+    filenames = []
+    version_added = '1.1'
+
+    tokens = {
+        'whitespace': [
+            (r'\s+', Whitespace)
+        ],
+        'comments': [
+            (r'\#.*$', Comment),
+        ],
+        'keywords': [
+            (r'(access|action|alphtype)\b', Keyword),
+            (r'(getkey|write|machine|include)\b', Keyword),
+            (r'(any|ascii|extend|alpha|digit|alnum|lower|upper)\b', Keyword),
+            (r'(xdigit|cntrl|graph|print|punct|space|zlen|empty)\b', Keyword)
+        ],
+        'numbers': [
+            (r'0x[0-9A-Fa-f]+', Number.Hex),
+            (r'[+-]?[0-9]+', Number.Integer),
+        ],
+        'literals': [
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+            (r'\[(\\\\|\\[^\\]|[^\\\]])*\]', String),          # square bracket literals
+            (r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/', String.Regex),  # regular expressions
+        ],
+        'identifiers': [
+            (r'[a-zA-Z_]\w*', Name.Variable),
+        ],
+        'operators': [
+            (r',', Operator),                           # Join
+            (r'\||&|--?', Operator),                    # Union, Intersection and Subtraction
+            (r'\.|<:|:>>?', Operator),                  # Concatention
+            (r':', Operator),                           # Label
+            (r'->', Operator),                          # Epsilon Transition
+            (r'(>|\$|%|<|@|<>)(/|eof\b)', Operator),    # EOF Actions
+            (r'(>|\$|%|<|@|<>)(!|err\b)', Operator),    # Global Error Actions
+            (r'(>|\$|%|<|@|<>)(\^|lerr\b)', Operator),  # Local Error Actions
+            (r'(>|\$|%|<|@|<>)(~|to\b)', Operator),     # To-State Actions
+            (r'(>|\$|%|<|@|<>)(\*|from\b)', Operator),  # From-State Actions
+            (r'>|@|\$|%', Operator),                    # Transition Actions and Priorities
+            (r'\*|\?|\+|\{[0-9]*,[0-9]*\}', Operator),  # Repetition
+            (r'!|\^', Operator),                        # Negation
+            (r'\(|\)', Operator),                       # Grouping
+        ],
+        'root': [
+            include('literals'),
+            include('whitespace'),
+            include('comments'),
+            include('keywords'),
+            include('numbers'),
+            include('identifiers'),
+            include('operators'),
+            (r'\{', Punctuation, 'host'),
+            (r'=', Operator),
+            (r';', Punctuation),
+        ],
+        'host': [
+            (r'(' + r'|'.join((  # keep host code in largest possible chunks
+                r'[^{}\'"/#]+',  # exclude unsafe characters
+                r'[^\\]\\[{}]',  # allow escaped { or }
+
+                # strings and comments may safely contain unsafe characters
+                r'"(\\\\|\\[^\\]|[^"\\])*"',
+                r"'(\\\\|\\[^\\]|[^'\\])*'",
+                r'//.*$\n?',            # single line comment
+                r'/\*(.|\n)*?\*/',      # multi-line javadoc-style comment
+                r'\#.*$\n?',            # ruby comment
+
+                # regular expression: There's no reason for it to start
+                # with a * and this stops confusion with comments.
+                r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/',
+
+                # / is safe now that we've handled regex and javadoc comments
+                r'/',
+            )) + r')+', Other),
+
+            (r'\{', Punctuation, '#push'),
+            (r'\}', Punctuation, '#pop'),
+        ],
+    }
+
+
+class RagelEmbeddedLexer(RegexLexer):
+    """
+    A lexer for Ragel embedded in a host language file.
+
+    This will only highlight Ragel statements. If you want host language
+    highlighting then call the language-specific Ragel lexer.
+    """
+
+    name = 'Embedded Ragel'
+    aliases = ['ragel-em']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    tokens = {
+        'root': [
+            (r'(' + r'|'.join((   # keep host code in largest possible chunks
+                r'[^%\'"/#]+',    # exclude unsafe characters
+                r'%(?=[^%]|$)',   # a single % sign is okay, just not 2 of them
+
+                # strings and comments may safely contain unsafe characters
+                r'"(\\\\|\\[^\\]|[^"\\])*"',
+                r"'(\\\\|\\[^\\]|[^'\\])*'",
+                r'/\*(.|\n)*?\*/',      # multi-line javadoc-style comment
+                r'//.*$\n?',  # single line comment
+                r'\#.*$\n?',  # ruby/ragel comment
+                r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/',  # regular expression
+
+                # / is safe now that we've handled regex and javadoc comments
+                r'/',
+            )) + r')+', Other),
+
+            # Single Line FSM.
+            # Please don't put a quoted newline in a single line FSM.
+            # That's just mean. It will break this.
+            (r'(%%)(?![{%])(.*)($|;)(\n?)', bygroups(Punctuation,
+                                                     using(RagelLexer),
+                                                     Punctuation, Text)),
+
+            # Multi Line FSM.
+            (r'(%%%%|%%)\{', Punctuation, 'multi-line-fsm'),
+        ],
+        'multi-line-fsm': [
+            (r'(' + r'|'.join((  # keep ragel code in largest possible chunks.
+                r'(' + r'|'.join((
+                    r'[^}\'"\[/#]',   # exclude unsafe characters
+                    r'\}(?=[^%]|$)',   # } is okay as long as it's not followed by %
+                    r'\}%(?=[^%]|$)',  # ...well, one %'s okay, just not two...
+                    r'[^\\]\\[{}]',   # ...and } is okay if it's escaped
+
+                    # allow / if it's preceded with one of these symbols
+                    # (ragel EOF actions)
+                    r'(>|\$|%|<|@|<>)/',
+
+                    # specifically allow regex followed immediately by *
+                    # so it doesn't get mistaken for a comment
+                    r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/\*',
+
+                    # allow / as long as it's not followed by another / or by a *
+                    r'/(?=[^/*]|$)',
+
+                    # We want to match as many of these as we can in one block.
+                    # Not sure if we need the + sign here,
+                    # does it help performance?
+                )) + r')+',
+
+                # strings and comments may safely contain unsafe characters
+                r'"(\\\\|\\[^\\]|[^"\\])*"',
+                r"'(\\\\|\\[^\\]|[^'\\])*'",
+                r"\[(\\\\|\\[^\\]|[^\]\\])*\]",  # square bracket literal
+                r'/\*(.|\n)*?\*/',          # multi-line javadoc-style comment
+                r'//.*$\n?',                # single line comment
+                r'\#.*$\n?',                # ruby/ragel comment
+            )) + r')+', using(RagelLexer)),
+
+            (r'\}%%', Punctuation, '#pop'),
+        ]
+    }
+
+    def analyse_text(text):
+        return '@LANG: indep' in text
+
+
+class RagelRubyLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in a Ruby host file.
+    """
+
+    name = 'Ragel in Ruby Host'
+    aliases = ['ragel-ruby', 'ragel-rb']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(RubyLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: ruby' in text
+
+
+class RagelCLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in a C host file.
+    """
+
+    name = 'Ragel in C Host'
+    aliases = ['ragel-c']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(CLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: c' in text
+
+
+class RagelDLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in a D host file.
+    """
+
+    name = 'Ragel in D Host'
+    aliases = ['ragel-d']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(DLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: d' in text
+
+
+class RagelCppLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in a C++ host file.
+    """
+
+    name = 'Ragel in CPP Host'
+    aliases = ['ragel-cpp']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(CppLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: c++' in text
+
+
+class RagelObjectiveCLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in an Objective C host file.
+    """
+
+    name = 'Ragel in Objective C Host'
+    aliases = ['ragel-objc']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(ObjectiveCLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: objc' in text
+
+
+class RagelJavaLexer(DelegatingLexer):
+    """
+    A lexer for Ragel in a Java host file.
+    """
+
+    name = 'Ragel in Java Host'
+    aliases = ['ragel-java']
+    filenames = ['*.rl']
+    url = 'http://www.colm.net/open-source/ragel/'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(JavaLexer, RagelEmbeddedLexer, **options)
+
+    def analyse_text(text):
+        return '@LANG: java' in text
+
+
+class AntlrLexer(RegexLexer):
+    """
+    Generic ANTLR Lexer.
+    Should not be called directly, instead
+    use DelegatingLexer for your target language.
+    """
+
+    name = 'ANTLR'
+    aliases = ['antlr']
+    filenames = []
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    _id = r'[A-Za-z]\w*'
+    _TOKEN_REF = r'[A-Z]\w*'
+    _RULE_REF = r'[a-z]\w*'
+    _STRING_LITERAL = r'\'(?:\\\\|\\\'|[^\']*)\''
+    _INT = r'[0-9]+'
+
+    tokens = {
+        'whitespace': [
+            (r'\s+', Whitespace),
+        ],
+        'comments': [
+            (r'//.*$', Comment),
+            (r'/\*(.|\n)*?\*/', Comment),
+        ],
+        'root': [
+            include('whitespace'),
+            include('comments'),
+
+            (r'(lexer|parser|tree)?(\s*)(grammar\b)(\s*)(' + _id + ')(;)',
+             bygroups(Keyword, Whitespace, Keyword, Whitespace, Name.Class,
+                      Punctuation)),
+            # optionsSpec
+            (r'options\b', Keyword, 'options'),
+            # tokensSpec
+            (r'tokens\b', Keyword, 'tokens'),
+            # attrScope
+            (r'(scope)(\s*)(' + _id + r')(\s*)(\{)',
+             bygroups(Keyword, Whitespace, Name.Variable, Whitespace,
+                      Punctuation), 'action'),
+            # exception
+            (r'(catch|finally)\b', Keyword, 'exception'),
+            # action
+            (r'(@' + _id + r')(\s*)(::)?(\s*)(' + _id + r')(\s*)(\{)',
+             bygroups(Name.Label, Whitespace, Punctuation, Whitespace,
+                      Name.Label, Whitespace, Punctuation), 'action'),
+            # rule
+            (r'((?:protected|private|public|fragment)\b)?(\s*)(' + _id + ')(!)?',
+             bygroups(Keyword, Whitespace, Name.Label, Punctuation),
+             ('rule-alts', 'rule-prelims')),
+        ],
+        'exception': [
+            (r'\n', Whitespace, '#pop'),
+            (r'\s', Whitespace),
+            include('comments'),
+
+            (r'\[', Punctuation, 'nested-arg-action'),
+            (r'\{', Punctuation, 'action'),
+        ],
+        'rule-prelims': [
+            include('whitespace'),
+            include('comments'),
+
+            (r'returns\b', Keyword),
+            (r'\[', Punctuation, 'nested-arg-action'),
+            (r'\{', Punctuation, 'action'),
+            # throwsSpec
+            (r'(throws)(\s+)(' + _id + ')',
+             bygroups(Keyword, Whitespace, Name.Label)),
+            (r'(,)(\s*)(' + _id + ')',
+             bygroups(Punctuation, Whitespace, Name.Label)),  # Additional throws
+            # optionsSpec
+            (r'options\b', Keyword, 'options'),
+            # ruleScopeSpec - scope followed by target language code or name of action
+            # TODO finish implementing other possibilities for scope
+            # L173 ANTLRv3.g from ANTLR book
+            (r'(scope)(\s+)(\{)', bygroups(Keyword, Whitespace, Punctuation),
+             'action'),
+            (r'(scope)(\s+)(' + _id + r')(\s*)(;)',
+             bygroups(Keyword, Whitespace, Name.Label, Whitespace, Punctuation)),
+            # ruleAction
+            (r'(@' + _id + r')(\s*)(\{)',
+             bygroups(Name.Label, Whitespace, Punctuation), 'action'),
+            # finished prelims, go to rule alts!
+            (r':', Punctuation, '#pop')
+        ],
+        'rule-alts': [
+            include('whitespace'),
+            include('comments'),
+
+            # These might need to go in a separate 'block' state triggered by (
+            (r'options\b', Keyword, 'options'),
+            (r':', Punctuation),
+
+            # literals
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+            (r'<<([^>]|>[^>])>>', String),
+            # identifiers
+            # Tokens start with capital letter.
+            (r'\$?[A-Z_]\w*', Name.Constant),
+            # Rules start with small letter.
+            (r'\$?[a-z_]\w*', Name.Variable),
+            # operators
+            (r'(\+|\||->|=>|=|\(|\)|\.\.|\.|\?|\*|\^|!|\#|~)', Operator),
+            (r',', Punctuation),
+            (r'\[', Punctuation, 'nested-arg-action'),
+            (r'\{', Punctuation, 'action'),
+            (r';', Punctuation, '#pop')
+        ],
+        'tokens': [
+            include('whitespace'),
+            include('comments'),
+            (r'\{', Punctuation),
+            (r'(' + _TOKEN_REF + r')(\s*)(=)?(\s*)(' + _STRING_LITERAL
+             + r')?(\s*)(;)',
+             bygroups(Name.Label, Whitespace, Punctuation, Whitespace,
+                      String, Whitespace, Punctuation)),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'options': [
+            include('whitespace'),
+            include('comments'),
+            (r'\{', Punctuation),
+            (r'(' + _id + r')(\s*)(=)(\s*)(' +
+             '|'.join((_id, _STRING_LITERAL, _INT, r'\*')) + r')(\s*)(;)',
+             bygroups(Name.Variable, Whitespace, Punctuation, Whitespace,
+                      Text, Whitespace, Punctuation)),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'action': [
+            (r'(' + r'|'.join((    # keep host code in largest possible chunks
+                r'[^${}\'"/\\]+',  # exclude unsafe characters
+
+                # strings and comments may safely contain unsafe characters
+                r'"(\\\\|\\[^\\]|[^"\\])*"',
+                r"'(\\\\|\\[^\\]|[^'\\])*'",
+                r'//.*$\n?',            # single line comment
+                r'/\*(.|\n)*?\*/',      # multi-line javadoc-style comment
+
+                # regular expression: There's no reason for it to start
+                # with a * and this stops confusion with comments.
+                r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/',
+
+                # backslashes are okay, as long as we are not backslashing a %
+                r'\\(?!%)',
+
+                # Now that we've handled regex and javadoc comments
+                # it's safe to let / through.
+                r'/',
+            )) + r')+', Other),
+            (r'(\\)(%)', bygroups(Punctuation, Other)),
+            (r'(\$[a-zA-Z]+)(\.?)(text|value)?',
+             bygroups(Name.Variable, Punctuation, Name.Property)),
+            (r'\{', Punctuation, '#push'),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'nested-arg-action': [
+            (r'(' + r'|'.join((    # keep host code in largest possible chunks.
+                r'[^$\[\]\'"/]+',  # exclude unsafe characters
+
+                # strings and comments may safely contain unsafe characters
+                r'"(\\\\|\\[^\\]|[^"\\])*"',
+                r"'(\\\\|\\[^\\]|[^'\\])*'",
+                r'//.*$\n?',            # single line comment
+                r'/\*(.|\n)*?\*/',      # multi-line javadoc-style comment
+
+                # regular expression: There's no reason for it to start
+                # with a * and this stops confusion with comments.
+                r'/(?!\*)(\\\\|\\[^\\]|[^/\\])*/',
+
+                # Now that we've handled regex and javadoc comments
+                # it's safe to let / through.
+                r'/',
+            )) + r')+', Other),
+
+
+            (r'\[', Punctuation, '#push'),
+            (r'\]', Punctuation, '#pop'),
+            (r'(\$[a-zA-Z]+)(\.?)(text|value)?',
+             bygroups(Name.Variable, Punctuation, Name.Property)),
+            (r'(\\\\|\\\]|\\\[|[^\[\]])+', Other),
+        ]
+    }
+
+    def analyse_text(text):
+        return re.search(r'^\s*grammar\s+[a-zA-Z0-9]+\s*;', text, re.M)
+
+
+# http://www.antlr.org/wiki/display/ANTLR3/Code+Generation+Targets
+
+class AntlrCppLexer(DelegatingLexer):
+    """
+    ANTLR with C++ Target
+    """
+
+    name = 'ANTLR With CPP Target'
+    aliases = ['antlr-cpp']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(CppLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*C\s*;', text, re.M)
+
+
+class AntlrObjectiveCLexer(DelegatingLexer):
+    """
+    ANTLR with Objective-C Target
+    """
+
+    name = 'ANTLR With ObjectiveC Target'
+    aliases = ['antlr-objc']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(ObjectiveCLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*ObjC\s*;', text)
+
+
+class AntlrCSharpLexer(DelegatingLexer):
+    """
+    ANTLR with C# Target
+    """
+
+    name = 'ANTLR With C# Target'
+    aliases = ['antlr-csharp', 'antlr-c#']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(CSharpLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*CSharp2\s*;', text, re.M)
+
+
+class AntlrPythonLexer(DelegatingLexer):
+    """
+    ANTLR with Python Target
+    """
+
+    name = 'ANTLR With Python Target'
+    aliases = ['antlr-python']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(PythonLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*Python\s*;', text, re.M)
+
+
+class AntlrJavaLexer(DelegatingLexer):
+    """
+    ANTLR with Java Target
+    """
+
+    name = 'ANTLR With Java Target'
+    aliases = ['antlr-java']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(JavaLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        # Antlr language is Java by default
+        return AntlrLexer.analyse_text(text) and 0.9
+
+
+class AntlrRubyLexer(DelegatingLexer):
+    """
+    ANTLR with Ruby Target
+    """
+
+    name = 'ANTLR With Ruby Target'
+    aliases = ['antlr-ruby', 'antlr-rb']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(RubyLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*Ruby\s*;', text, re.M)
+
+
+class AntlrPerlLexer(DelegatingLexer):
+    """
+    ANTLR with Perl Target
+    """
+
+    name = 'ANTLR With Perl Target'
+    aliases = ['antlr-perl']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        super().__init__(PerlLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*Perl5\s*;', text, re.M)
+
+
+class AntlrActionScriptLexer(DelegatingLexer):
+    """
+    ANTLR with ActionScript Target
+    """
+
+    name = 'ANTLR With ActionScript Target'
+    aliases = ['antlr-actionscript', 'antlr-as']
+    filenames = ['*.G', '*.g']
+    url = 'https://www.antlr.org'
+    version_added = '1.1'
+
+    def __init__(self, **options):
+        from pygments.lexers.actionscript import ActionScriptLexer
+        super().__init__(ActionScriptLexer, AntlrLexer, **options)
+
+    def analyse_text(text):
+        return AntlrLexer.analyse_text(text) and \
+            re.search(r'^\s*language\s*=\s*ActionScript\s*;', text, re.M)
+
+
+class TreetopBaseLexer(RegexLexer):
+    """
+    A base lexer for `Treetop `_ grammars.
+    Not for direct use; use :class:`TreetopLexer` instead.
+
+    .. versionadded:: 1.6
+    """
+
+    tokens = {
+        'root': [
+            include('space'),
+            (r'require[ \t]+[^\n\r]+[\n\r]', Other),
+            (r'module\b', Keyword.Namespace, 'module'),
+            (r'grammar\b', Keyword, 'grammar'),
+        ],
+        'module': [
+            include('space'),
+            include('end'),
+            (r'module\b', Keyword, '#push'),
+            (r'grammar\b', Keyword, 'grammar'),
+            (r'[A-Z]\w*(?:::[A-Z]\w*)*', Name.Namespace),
+        ],
+        'grammar': [
+            include('space'),
+            include('end'),
+            (r'rule\b', Keyword, 'rule'),
+            (r'include\b', Keyword, 'include'),
+            (r'[A-Z]\w*', Name),
+        ],
+        'include': [
+            include('space'),
+            (r'[A-Z]\w*(?:::[A-Z]\w*)*', Name.Class, '#pop'),
+        ],
+        'rule': [
+            include('space'),
+            include('end'),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+            (r'([A-Za-z_]\w*)(:)', bygroups(Name.Label, Punctuation)),
+            (r'[A-Za-z_]\w*', Name),
+            (r'[()]', Punctuation),
+            (r'[?+*/&!~]', Operator),
+            (r'\[(?:\\.|\[:\^?[a-z]+:\]|[^\\\]])+\]', String.Regex),
+            (r'([0-9]*)(\.\.)([0-9]*)',
+             bygroups(Number.Integer, Operator, Number.Integer)),
+            (r'(<)([^>]+)(>)', bygroups(Punctuation, Name.Class, Punctuation)),
+            (r'\{', Punctuation, 'inline_module'),
+            (r'\.', String.Regex),
+        ],
+        'inline_module': [
+            (r'\{', Other, 'ruby'),
+            (r'\}', Punctuation, '#pop'),
+            (r'[^{}]+', Other),
+        ],
+        'ruby': [
+            (r'\{', Other, '#push'),
+            (r'\}', Other, '#pop'),
+            (r'[^{}]+', Other),
+        ],
+        'space': [
+            (r'[ \t\n\r]+', Whitespace),
+            (r'#[^\n]*', Comment.Single),
+        ],
+        'end': [
+            (r'end\b', Keyword, '#pop'),
+        ],
+    }
+
+
+class TreetopLexer(DelegatingLexer):
+    """
+    A lexer for Treetop grammars.
+    """
+
+    name = 'Treetop'
+    aliases = ['treetop']
+    filenames = ['*.treetop', '*.tt']
+    url = 'https://cjheath.github.io/treetop'
+    version_added = '1.6'
+
+    def __init__(self, **options):
+        super().__init__(RubyLexer, TreetopBaseLexer, **options)
+
+
+class EbnfLexer(RegexLexer):
+    """
+    Lexer for `ISO/IEC 14977 EBNF
+    `_
+    grammars.
+    """
+
+    name = 'EBNF'
+    aliases = ['ebnf']
+    filenames = ['*.ebnf']
+    mimetypes = ['text/x-ebnf']
+    url = 'https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_Form'
+    version_added = '2.0'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            include('comment_start'),
+            include('identifier'),
+            (r'=', Operator, 'production'),
+        ],
+        'production': [
+            include('whitespace'),
+            include('comment_start'),
+            include('identifier'),
+            (r'"[^"]*"', String.Double),
+            (r"'[^']*'", String.Single),
+            (r'(\?[^?]*\?)', Name.Entity),
+            (r'[\[\]{}(),|]', Punctuation),
+            (r'-', Operator),
+            (r';', Punctuation, '#pop'),
+            (r'\.', Punctuation, '#pop'),
+        ],
+        'whitespace': [
+            (r'\s+', Text),
+        ],
+        'comment_start': [
+            (r'\(\*', Comment.Multiline, 'comment'),
+        ],
+        'comment': [
+            (r'[^*)]', Comment.Multiline),
+            include('comment_start'),
+            (r'\*\)', Comment.Multiline, '#pop'),
+            (r'[*)]', Comment.Multiline),
+        ],
+        'identifier': [
+            (r'([a-zA-Z][\w \-]*)', Keyword),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pascal.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pascal.py
new file mode 100644
index 0000000000000000000000000000000000000000..ad8a0aec62a515a601b0b866ae4d64cf62c28afb
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pascal.py
@@ -0,0 +1,644 @@
+"""
+    pygments.lexers.pascal
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Pascal family languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer
+from pygments.util import get_bool_opt, get_list_opt
+from pygments.token import Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error, Whitespace
+from pygments.scanner import Scanner
+
+# compatibility import
+from pygments.lexers.modula2 import Modula2Lexer # noqa: F401
+
+__all__ = ['DelphiLexer', 'PortugolLexer']
+
+
+class PortugolLexer(Lexer):
+    """For Portugol, a Pascal dialect with keywords in Portuguese."""
+    name = 'Portugol'
+    aliases = ['portugol']
+    filenames = ['*.alg', '*.portugol']
+    mimetypes = []
+    url = "https://www.apoioinformatica.inf.br/produtos/visualg/linguagem"
+    version_added = ''
+
+    def __init__(self, **options):
+        Lexer.__init__(self, **options)
+        self.lexer = DelphiLexer(**options, portugol=True)
+
+    def get_tokens_unprocessed(self, text):
+        return self.lexer.get_tokens_unprocessed(text)
+
+
+class DelphiLexer(Lexer):
+    """
+    For Delphi (Borland Object Pascal),
+    Turbo Pascal and Free Pascal source code.
+
+    Additional options accepted:
+
+    `turbopascal`
+        Highlight Turbo Pascal specific keywords (default: ``True``).
+    `delphi`
+        Highlight Borland Delphi specific keywords (default: ``True``).
+    `freepascal`
+        Highlight Free Pascal specific keywords (default: ``True``).
+    `units`
+        A list of units that should be considered builtin, supported are
+        ``System``, ``SysUtils``, ``Classes`` and ``Math``.
+        Default is to consider all of them builtin.
+    """
+    name = 'Delphi'
+    aliases = ['delphi', 'pas', 'pascal', 'objectpascal']
+    filenames = ['*.pas', '*.dpr']
+    mimetypes = ['text/x-pascal']
+    url = 'https://www.embarcadero.com/products/delphi'
+    version_added = ''
+
+    TURBO_PASCAL_KEYWORDS = (
+        'absolute', 'and', 'array', 'asm', 'begin', 'break', 'case',
+        'const', 'constructor', 'continue', 'destructor', 'div', 'do',
+        'downto', 'else', 'end', 'file', 'for', 'function', 'goto',
+        'if', 'implementation', 'in', 'inherited', 'inline', 'interface',
+        'label', 'mod', 'nil', 'not', 'object', 'of', 'on', 'operator',
+        'or', 'packed', 'procedure', 'program', 'record', 'reintroduce',
+        'repeat', 'self', 'set', 'shl', 'shr', 'string', 'then', 'to',
+        'type', 'unit', 'until', 'uses', 'var', 'while', 'with', 'xor'
+    )
+
+    DELPHI_KEYWORDS = (
+        'as', 'class', 'except', 'exports', 'finalization', 'finally',
+        'initialization', 'is', 'library', 'on', 'property', 'raise',
+        'threadvar', 'try'
+    )
+
+    FREE_PASCAL_KEYWORDS = (
+        'dispose', 'exit', 'false', 'new', 'true'
+    )
+
+    BLOCK_KEYWORDS = {
+        'begin', 'class', 'const', 'constructor', 'destructor', 'end',
+        'finalization', 'function', 'implementation', 'initialization',
+        'label', 'library', 'operator', 'procedure', 'program', 'property',
+        'record', 'threadvar', 'type', 'unit', 'uses', 'var'
+    }
+
+    FUNCTION_MODIFIERS = {
+        'alias', 'cdecl', 'export', 'inline', 'interrupt', 'nostackframe',
+        'pascal', 'register', 'safecall', 'softfloat', 'stdcall',
+        'varargs', 'name', 'dynamic', 'near', 'virtual', 'external',
+        'override', 'assembler'
+    }
+
+    # XXX: those aren't global. but currently we know no way for defining
+    #      them just for the type context.
+    DIRECTIVES = {
+        'absolute', 'abstract', 'assembler', 'cppdecl', 'default', 'far',
+        'far16', 'forward', 'index', 'oldfpccall', 'private', 'protected',
+        'published', 'public'
+    }
+
+    BUILTIN_TYPES = {
+        'ansichar', 'ansistring', 'bool', 'boolean', 'byte', 'bytebool',
+        'cardinal', 'char', 'comp', 'currency', 'double', 'dword',
+        'extended', 'int64', 'integer', 'iunknown', 'longbool', 'longint',
+        'longword', 'pansichar', 'pansistring', 'pbool', 'pboolean',
+        'pbyte', 'pbytearray', 'pcardinal', 'pchar', 'pcomp', 'pcurrency',
+        'pdate', 'pdatetime', 'pdouble', 'pdword', 'pextended', 'phandle',
+        'pint64', 'pinteger', 'plongint', 'plongword', 'pointer',
+        'ppointer', 'pshortint', 'pshortstring', 'psingle', 'psmallint',
+        'pstring', 'pvariant', 'pwidechar', 'pwidestring', 'pword',
+        'pwordarray', 'pwordbool', 'real', 'real48', 'shortint',
+        'shortstring', 'single', 'smallint', 'string', 'tclass', 'tdate',
+        'tdatetime', 'textfile', 'thandle', 'tobject', 'ttime', 'variant',
+        'widechar', 'widestring', 'word', 'wordbool'
+    }
+
+    BUILTIN_UNITS = {
+        'System': (
+            'abs', 'acquireexceptionobject', 'addr', 'ansitoutf8',
+            'append', 'arctan', 'assert', 'assigned', 'assignfile',
+            'beginthread', 'blockread', 'blockwrite', 'break', 'chdir',
+            'chr', 'close', 'closefile', 'comptocurrency', 'comptodouble',
+            'concat', 'continue', 'copy', 'cos', 'dec', 'delete',
+            'dispose', 'doubletocomp', 'endthread', 'enummodules',
+            'enumresourcemodules', 'eof', 'eoln', 'erase', 'exceptaddr',
+            'exceptobject', 'exclude', 'exit', 'exp', 'filepos', 'filesize',
+            'fillchar', 'finalize', 'findclasshinstance', 'findhinstance',
+            'findresourcehinstance', 'flush', 'frac', 'freemem',
+            'get8087cw', 'getdir', 'getlasterror', 'getmem',
+            'getmemorymanager', 'getmodulefilename', 'getvariantmanager',
+            'halt', 'hi', 'high', 'inc', 'include', 'initialize', 'insert',
+            'int', 'ioresult', 'ismemorymanagerset', 'isvariantmanagerset',
+            'length', 'ln', 'lo', 'low', 'mkdir', 'move', 'new', 'odd',
+            'olestrtostring', 'olestrtostrvar', 'ord', 'paramcount',
+            'paramstr', 'pi', 'pos', 'pred', 'ptr', 'pucs4chars', 'random',
+            'randomize', 'read', 'readln', 'reallocmem',
+            'releaseexceptionobject', 'rename', 'reset', 'rewrite', 'rmdir',
+            'round', 'runerror', 'seek', 'seekeof', 'seekeoln',
+            'set8087cw', 'setlength', 'setlinebreakstyle',
+            'setmemorymanager', 'setstring', 'settextbuf',
+            'setvariantmanager', 'sin', 'sizeof', 'slice', 'sqr', 'sqrt',
+            'str', 'stringofchar', 'stringtoolestr', 'stringtowidechar',
+            'succ', 'swap', 'trunc', 'truncate', 'typeinfo',
+            'ucs4stringtowidestring', 'unicodetoutf8', 'uniquestring',
+            'upcase', 'utf8decode', 'utf8encode', 'utf8toansi',
+            'utf8tounicode', 'val', 'vararrayredim', 'varclear',
+            'widecharlentostring', 'widecharlentostrvar',
+            'widechartostring', 'widechartostrvar',
+            'widestringtoucs4string', 'write', 'writeln'
+        ),
+        'SysUtils': (
+            'abort', 'addexitproc', 'addterminateproc', 'adjustlinebreaks',
+            'allocmem', 'ansicomparefilename', 'ansicomparestr',
+            'ansicomparetext', 'ansidequotedstr', 'ansiextractquotedstr',
+            'ansilastchar', 'ansilowercase', 'ansilowercasefilename',
+            'ansipos', 'ansiquotedstr', 'ansisamestr', 'ansisametext',
+            'ansistrcomp', 'ansistricomp', 'ansistrlastchar', 'ansistrlcomp',
+            'ansistrlicomp', 'ansistrlower', 'ansistrpos', 'ansistrrscan',
+            'ansistrscan', 'ansistrupper', 'ansiuppercase',
+            'ansiuppercasefilename', 'appendstr', 'assignstr', 'beep',
+            'booltostr', 'bytetocharindex', 'bytetocharlen', 'bytetype',
+            'callterminateprocs', 'changefileext', 'charlength',
+            'chartobyteindex', 'chartobytelen', 'comparemem', 'comparestr',
+            'comparetext', 'createdir', 'createguid', 'currentyear',
+            'currtostr', 'currtostrf', 'date', 'datetimetofiledate',
+            'datetimetostr', 'datetimetostring', 'datetimetosystemtime',
+            'datetimetotimestamp', 'datetostr', 'dayofweek', 'decodedate',
+            'decodedatefully', 'decodetime', 'deletefile', 'directoryexists',
+            'diskfree', 'disksize', 'disposestr', 'encodedate', 'encodetime',
+            'exceptionerrormessage', 'excludetrailingbackslash',
+            'excludetrailingpathdelimiter', 'expandfilename',
+            'expandfilenamecase', 'expanduncfilename', 'extractfiledir',
+            'extractfiledrive', 'extractfileext', 'extractfilename',
+            'extractfilepath', 'extractrelativepath', 'extractshortpathname',
+            'fileage', 'fileclose', 'filecreate', 'filedatetodatetime',
+            'fileexists', 'filegetattr', 'filegetdate', 'fileisreadonly',
+            'fileopen', 'fileread', 'filesearch', 'fileseek', 'filesetattr',
+            'filesetdate', 'filesetreadonly', 'filewrite', 'finalizepackage',
+            'findclose', 'findcmdlineswitch', 'findfirst', 'findnext',
+            'floattocurr', 'floattodatetime', 'floattodecimal', 'floattostr',
+            'floattostrf', 'floattotext', 'floattotextfmt', 'fmtloadstr',
+            'fmtstr', 'forcedirectories', 'format', 'formatbuf', 'formatcurr',
+            'formatdatetime', 'formatfloat', 'freeandnil', 'getcurrentdir',
+            'getenvironmentvariable', 'getfileversion', 'getformatsettings',
+            'getlocaleformatsettings', 'getmodulename', 'getpackagedescription',
+            'getpackageinfo', 'gettime', 'guidtostring', 'incamonth',
+            'includetrailingbackslash', 'includetrailingpathdelimiter',
+            'incmonth', 'initializepackage', 'interlockeddecrement',
+            'interlockedexchange', 'interlockedexchangeadd',
+            'interlockedincrement', 'inttohex', 'inttostr', 'isdelimiter',
+            'isequalguid', 'isleapyear', 'ispathdelimiter', 'isvalidident',
+            'languages', 'lastdelimiter', 'loadpackage', 'loadstr',
+            'lowercase', 'msecstotimestamp', 'newstr', 'nextcharindex', 'now',
+            'outofmemoryerror', 'quotedstr', 'raiselastoserror',
+            'raiselastwin32error', 'removedir', 'renamefile', 'replacedate',
+            'replacetime', 'safeloadlibrary', 'samefilename', 'sametext',
+            'setcurrentdir', 'showexception', 'sleep', 'stralloc', 'strbufsize',
+            'strbytetype', 'strcat', 'strcharlength', 'strcomp', 'strcopy',
+            'strdispose', 'strecopy', 'strend', 'strfmt', 'stricomp',
+            'stringreplace', 'stringtoguid', 'strlcat', 'strlcomp', 'strlcopy',
+            'strlen', 'strlfmt', 'strlicomp', 'strlower', 'strmove', 'strnew',
+            'strnextchar', 'strpas', 'strpcopy', 'strplcopy', 'strpos',
+            'strrscan', 'strscan', 'strtobool', 'strtobooldef', 'strtocurr',
+            'strtocurrdef', 'strtodate', 'strtodatedef', 'strtodatetime',
+            'strtodatetimedef', 'strtofloat', 'strtofloatdef', 'strtoint',
+            'strtoint64', 'strtoint64def', 'strtointdef', 'strtotime',
+            'strtotimedef', 'strupper', 'supports', 'syserrormessage',
+            'systemtimetodatetime', 'texttofloat', 'time', 'timestamptodatetime',
+            'timestamptomsecs', 'timetostr', 'trim', 'trimleft', 'trimright',
+            'tryencodedate', 'tryencodetime', 'tryfloattocurr', 'tryfloattodatetime',
+            'trystrtobool', 'trystrtocurr', 'trystrtodate', 'trystrtodatetime',
+            'trystrtofloat', 'trystrtoint', 'trystrtoint64', 'trystrtotime',
+            'unloadpackage', 'uppercase', 'widecomparestr', 'widecomparetext',
+            'widefmtstr', 'wideformat', 'wideformatbuf', 'widelowercase',
+            'widesamestr', 'widesametext', 'wideuppercase', 'win32check',
+            'wraptext'
+        ),
+        'Classes': (
+            'activateclassgroup', 'allocatehwnd', 'bintohex', 'checksynchronize',
+            'collectionsequal', 'countgenerations', 'deallocatehwnd', 'equalrect',
+            'extractstrings', 'findclass', 'findglobalcomponent', 'getclass',
+            'groupdescendantswith', 'hextobin', 'identtoint',
+            'initinheritedcomponent', 'inttoident', 'invalidpoint',
+            'isuniqueglobalcomponentname', 'linestart', 'objectbinarytotext',
+            'objectresourcetotext', 'objecttexttobinary', 'objecttexttoresource',
+            'pointsequal', 'readcomponentres', 'readcomponentresex',
+            'readcomponentresfile', 'rect', 'registerclass', 'registerclassalias',
+            'registerclasses', 'registercomponents', 'registerintegerconsts',
+            'registernoicon', 'registernonactivex', 'smallpoint', 'startclassgroup',
+            'teststreamformat', 'unregisterclass', 'unregisterclasses',
+            'unregisterintegerconsts', 'unregistermoduleclasses',
+            'writecomponentresfile'
+        ),
+        'Math': (
+            'arccos', 'arccosh', 'arccot', 'arccoth', 'arccsc', 'arccsch', 'arcsec',
+            'arcsech', 'arcsin', 'arcsinh', 'arctan2', 'arctanh', 'ceil',
+            'comparevalue', 'cosecant', 'cosh', 'cot', 'cotan', 'coth', 'csc',
+            'csch', 'cycletodeg', 'cycletograd', 'cycletorad', 'degtocycle',
+            'degtograd', 'degtorad', 'divmod', 'doubledecliningbalance',
+            'ensurerange', 'floor', 'frexp', 'futurevalue', 'getexceptionmask',
+            'getprecisionmode', 'getroundmode', 'gradtocycle', 'gradtodeg',
+            'gradtorad', 'hypot', 'inrange', 'interestpayment', 'interestrate',
+            'internalrateofreturn', 'intpower', 'isinfinite', 'isnan', 'iszero',
+            'ldexp', 'lnxp1', 'log10', 'log2', 'logn', 'max', 'maxintvalue',
+            'maxvalue', 'mean', 'meanandstddev', 'min', 'minintvalue', 'minvalue',
+            'momentskewkurtosis', 'netpresentvalue', 'norm', 'numberofperiods',
+            'payment', 'periodpayment', 'poly', 'popnstddev', 'popnvariance',
+            'power', 'presentvalue', 'radtocycle', 'radtodeg', 'radtograd',
+            'randg', 'randomrange', 'roundto', 'samevalue', 'sec', 'secant',
+            'sech', 'setexceptionmask', 'setprecisionmode', 'setroundmode',
+            'sign', 'simpleroundto', 'sincos', 'sinh', 'slndepreciation', 'stddev',
+            'sum', 'sumint', 'sumofsquares', 'sumsandsquares', 'syddepreciation',
+            'tan', 'tanh', 'totalvariance', 'variance'
+        )
+    }
+
+    ASM_REGISTERS = {
+        'ah', 'al', 'ax', 'bh', 'bl', 'bp', 'bx', 'ch', 'cl', 'cr0',
+        'cr1', 'cr2', 'cr3', 'cr4', 'cs', 'cx', 'dh', 'di', 'dl', 'dr0',
+        'dr1', 'dr2', 'dr3', 'dr4', 'dr5', 'dr6', 'dr7', 'ds', 'dx',
+        'eax', 'ebp', 'ebx', 'ecx', 'edi', 'edx', 'es', 'esi', 'esp',
+        'fs', 'gs', 'mm0', 'mm1', 'mm2', 'mm3', 'mm4', 'mm5', 'mm6',
+        'mm7', 'si', 'sp', 'ss', 'st0', 'st1', 'st2', 'st3', 'st4', 'st5',
+        'st6', 'st7', 'xmm0', 'xmm1', 'xmm2', 'xmm3', 'xmm4', 'xmm5',
+        'xmm6', 'xmm7'
+    }
+
+    ASM_INSTRUCTIONS = {
+        'aaa', 'aad', 'aam', 'aas', 'adc', 'add', 'and', 'arpl', 'bound',
+        'bsf', 'bsr', 'bswap', 'bt', 'btc', 'btr', 'bts', 'call', 'cbw',
+        'cdq', 'clc', 'cld', 'cli', 'clts', 'cmc', 'cmova', 'cmovae',
+        'cmovb', 'cmovbe', 'cmovc', 'cmovcxz', 'cmove', 'cmovg',
+        'cmovge', 'cmovl', 'cmovle', 'cmovna', 'cmovnae', 'cmovnb',
+        'cmovnbe', 'cmovnc', 'cmovne', 'cmovng', 'cmovnge', 'cmovnl',
+        'cmovnle', 'cmovno', 'cmovnp', 'cmovns', 'cmovnz', 'cmovo',
+        'cmovp', 'cmovpe', 'cmovpo', 'cmovs', 'cmovz', 'cmp', 'cmpsb',
+        'cmpsd', 'cmpsw', 'cmpxchg', 'cmpxchg486', 'cmpxchg8b', 'cpuid',
+        'cwd', 'cwde', 'daa', 'das', 'dec', 'div', 'emms', 'enter', 'hlt',
+        'ibts', 'icebp', 'idiv', 'imul', 'in', 'inc', 'insb', 'insd',
+        'insw', 'int', 'int01', 'int03', 'int1', 'int3', 'into', 'invd',
+        'invlpg', 'iret', 'iretd', 'iretw', 'ja', 'jae', 'jb', 'jbe',
+        'jc', 'jcxz', 'jcxz', 'je', 'jecxz', 'jg', 'jge', 'jl', 'jle',
+        'jmp', 'jna', 'jnae', 'jnb', 'jnbe', 'jnc', 'jne', 'jng', 'jnge',
+        'jnl', 'jnle', 'jno', 'jnp', 'jns', 'jnz', 'jo', 'jp', 'jpe',
+        'jpo', 'js', 'jz', 'lahf', 'lar', 'lcall', 'lds', 'lea', 'leave',
+        'les', 'lfs', 'lgdt', 'lgs', 'lidt', 'ljmp', 'lldt', 'lmsw',
+        'loadall', 'loadall286', 'lock', 'lodsb', 'lodsd', 'lodsw',
+        'loop', 'loope', 'loopne', 'loopnz', 'loopz', 'lsl', 'lss', 'ltr',
+        'mov', 'movd', 'movq', 'movsb', 'movsd', 'movsw', 'movsx',
+        'movzx', 'mul', 'neg', 'nop', 'not', 'or', 'out', 'outsb', 'outsd',
+        'outsw', 'pop', 'popa', 'popad', 'popaw', 'popf', 'popfd', 'popfw',
+        'push', 'pusha', 'pushad', 'pushaw', 'pushf', 'pushfd', 'pushfw',
+        'rcl', 'rcr', 'rdmsr', 'rdpmc', 'rdshr', 'rdtsc', 'rep', 'repe',
+        'repne', 'repnz', 'repz', 'ret', 'retf', 'retn', 'rol', 'ror',
+        'rsdc', 'rsldt', 'rsm', 'sahf', 'sal', 'salc', 'sar', 'sbb',
+        'scasb', 'scasd', 'scasw', 'seta', 'setae', 'setb', 'setbe',
+        'setc', 'setcxz', 'sete', 'setg', 'setge', 'setl', 'setle',
+        'setna', 'setnae', 'setnb', 'setnbe', 'setnc', 'setne', 'setng',
+        'setnge', 'setnl', 'setnle', 'setno', 'setnp', 'setns', 'setnz',
+        'seto', 'setp', 'setpe', 'setpo', 'sets', 'setz', 'sgdt', 'shl',
+        'shld', 'shr', 'shrd', 'sidt', 'sldt', 'smi', 'smint', 'smintold',
+        'smsw', 'stc', 'std', 'sti', 'stosb', 'stosd', 'stosw', 'str',
+        'sub', 'svdc', 'svldt', 'svts', 'syscall', 'sysenter', 'sysexit',
+        'sysret', 'test', 'ud1', 'ud2', 'umov', 'verr', 'verw', 'wait',
+        'wbinvd', 'wrmsr', 'wrshr', 'xadd', 'xbts', 'xchg', 'xlat',
+        'xlatb', 'xor'
+    }
+
+    PORTUGOL_KEYWORDS = (
+        'aleatorio',
+        'algoritmo',
+        'arquivo',
+        'ate',
+        'caso',
+        'cronometro',
+        'debug',
+        'e',
+        'eco',
+        'enquanto',
+        'entao',
+        'escolha',
+        'escreva',
+        'escreval',
+        'faca',
+        'falso',
+        'fimalgoritmo',
+        'fimenquanto',
+        'fimescolha',
+        'fimfuncao',
+        'fimpara',
+        'fimprocedimento',
+        'fimrepita',
+        'fimse',
+        'funcao',
+        'inicio',
+        'int',
+        'interrompa',
+        'leia',
+        'limpatela',
+        'mod',
+        'nao',
+        'ou',
+        'outrocaso',
+        'para',
+        'passo',
+        'pausa',
+        'procedimento',
+        'repita',
+        'retorne',
+        'se',
+        'senao',
+        'timer',
+        'var',
+        'vetor',
+        'verdadeiro',
+        'xou',
+        'div',
+        'mod',
+        'abs',
+        'arccos',
+        'arcsen',
+        'arctan',
+        'cos',
+        'cotan',
+        'Exp',
+        'grauprad',
+        'int',
+        'log',
+        'logn',
+        'pi',
+        'quad',
+        'radpgrau',
+        'raizq',
+        'rand',
+        'randi',
+        'sen',
+        'Tan',
+        'asc',
+        'carac',
+        'caracpnum',
+        'compr',
+        'copia',
+        'maiusc',
+        'minusc',
+        'numpcarac',
+        'pos',
+    )
+
+    PORTUGOL_BUILTIN_TYPES = {
+        'inteiro', 'real', 'caractere', 'logico'
+    }
+
+    def __init__(self, **options):
+        Lexer.__init__(self, **options)
+        self.keywords = set()
+        self.builtins = set()
+        if get_bool_opt(options, 'portugol', False):
+            self.keywords.update(self.PORTUGOL_KEYWORDS)
+            self.builtins.update(self.PORTUGOL_BUILTIN_TYPES)
+            self.is_portugol = True
+        else:
+            self.is_portugol = False
+
+            if get_bool_opt(options, 'turbopascal', True):
+                self.keywords.update(self.TURBO_PASCAL_KEYWORDS)
+            if get_bool_opt(options, 'delphi', True):
+                self.keywords.update(self.DELPHI_KEYWORDS)
+            if get_bool_opt(options, 'freepascal', True):
+                self.keywords.update(self.FREE_PASCAL_KEYWORDS)
+            for unit in get_list_opt(options, 'units', list(self.BUILTIN_UNITS)):
+                self.builtins.update(self.BUILTIN_UNITS[unit])
+
+    def get_tokens_unprocessed(self, text):
+        scanner = Scanner(text, re.DOTALL | re.MULTILINE | re.IGNORECASE)
+        stack = ['initial']
+        in_function_block = False
+        in_property_block = False
+        was_dot = False
+        next_token_is_function = False
+        next_token_is_property = False
+        collect_labels = False
+        block_labels = set()
+        brace_balance = [0, 0]
+
+        while not scanner.eos:
+            token = Error
+
+            if stack[-1] == 'initial':
+                if scanner.scan(r'\s+'):
+                    token = Whitespace
+                elif not self.is_portugol and scanner.scan(r'\{.*?\}|\(\*.*?\*\)'):
+                    if scanner.match.startswith('$'):
+                        token = Comment.Preproc
+                    else:
+                        token = Comment.Multiline
+                elif scanner.scan(r'//.*?$'):
+                    token = Comment.Single
+                elif self.is_portugol and scanner.scan(r'(<\-)|(>=)|(<=)|%|<|>|-|\+|\*|\=|(<>)|\/|\.|:|,'):
+                    token = Operator
+                elif not self.is_portugol and scanner.scan(r'[-+*\/=<>:;,.@\^]'):
+                    token = Operator
+                    # stop label highlighting on next ";"
+                    if collect_labels and scanner.match == ';':
+                        collect_labels = False
+                elif scanner.scan(r'[\(\)\[\]]+'):
+                    token = Punctuation
+                    # abort function naming ``foo = Function(...)``
+                    next_token_is_function = False
+                    # if we are in a function block we count the open
+                    # braces because ootherwise it's impossible to
+                    # determine the end of the modifier context
+                    if in_function_block or in_property_block:
+                        if scanner.match == '(':
+                            brace_balance[0] += 1
+                        elif scanner.match == ')':
+                            brace_balance[0] -= 1
+                        elif scanner.match == '[':
+                            brace_balance[1] += 1
+                        elif scanner.match == ']':
+                            brace_balance[1] -= 1
+                elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'):
+                    lowercase_name = scanner.match.lower()
+                    if lowercase_name == 'result':
+                        token = Name.Builtin.Pseudo
+                    elif lowercase_name in self.keywords:
+                        token = Keyword
+                        # if we are in a special block and a
+                        # block ending keyword occurs (and the parenthesis
+                        # is balanced) we end the current block context
+                        if self.is_portugol:
+                            if lowercase_name in ('funcao', 'procedimento'):
+                                in_function_block = True
+                                next_token_is_function = True
+                        else:
+                            if (in_function_block or in_property_block) and \
+                                    lowercase_name in self.BLOCK_KEYWORDS and \
+                                    brace_balance[0] <= 0 and \
+                                    brace_balance[1] <= 0:
+                                in_function_block = False
+                                in_property_block = False
+                                brace_balance = [0, 0]
+                                block_labels = set()
+                            if lowercase_name in ('label', 'goto'):
+                                collect_labels = True
+                            elif lowercase_name == 'asm':
+                                stack.append('asm')
+                            elif lowercase_name == 'property':
+                                in_property_block = True
+                                next_token_is_property = True
+                            elif lowercase_name in ('procedure', 'operator',
+                                                    'function', 'constructor',
+                                                    'destructor'):
+                                in_function_block = True
+                                next_token_is_function = True
+                    # we are in a function block and the current name
+                    # is in the set of registered modifiers. highlight
+                    # it as pseudo keyword
+                    elif not self.is_portugol and in_function_block and \
+                            lowercase_name in self.FUNCTION_MODIFIERS:
+                        token = Keyword.Pseudo
+                    # if we are in a property highlight some more
+                    # modifiers
+                    elif not self.is_portugol and in_property_block and \
+                            lowercase_name in ('read', 'write'):
+                        token = Keyword.Pseudo
+                        next_token_is_function = True
+                    # if the last iteration set next_token_is_function
+                    # to true we now want this name highlighted as
+                    # function. so do that and reset the state
+                    elif next_token_is_function:
+                        # Look if the next token is a dot. If yes it's
+                        # not a function, but a class name and the
+                        # part after the dot a function name
+                        if not self.is_portugol and scanner.test(r'\s*\.\s*'):
+                            token = Name.Class
+                        # it's not a dot, our job is done
+                        else:
+                            token = Name.Function
+                            next_token_is_function = False
+
+                            if self.is_portugol:
+                                block_labels.add(scanner.match.lower())
+
+                    # same for properties
+                    elif not self.is_portugol and next_token_is_property:
+                        token = Name.Property
+                        next_token_is_property = False
+                    # Highlight this token as label and add it
+                    # to the list of known labels
+                    elif not self.is_portugol and collect_labels:
+                        token = Name.Label
+                        block_labels.add(scanner.match.lower())
+                    # name is in list of known labels
+                    elif lowercase_name in block_labels:
+                        token = Name.Label
+                    elif self.is_portugol and lowercase_name in self.PORTUGOL_BUILTIN_TYPES:
+                        token = Keyword.Type
+                    elif not self.is_portugol and lowercase_name in self.BUILTIN_TYPES:
+                        token = Keyword.Type
+                    elif not self.is_portugol and lowercase_name in self.DIRECTIVES:
+                        token = Keyword.Pseudo
+                    # builtins are just builtins if the token
+                    # before isn't a dot
+                    elif not self.is_portugol and not was_dot and lowercase_name in self.builtins:
+                        token = Name.Builtin
+                    else:
+                        token = Name
+                elif self.is_portugol and scanner.scan(r"\""):
+                    token = String
+                    stack.append('string')
+                elif not self.is_portugol and scanner.scan(r"'"):
+                    token = String
+                    stack.append('string')
+                elif not self.is_portugol and scanner.scan(r'\#(\d+|\$[0-9A-Fa-f]+)'):
+                    token = String.Char
+                elif not self.is_portugol and scanner.scan(r'\$[0-9A-Fa-f]+'):
+                    token = Number.Hex
+                elif scanner.scan(r'\d+(?![eE]|\.[^.])'):
+                    token = Number.Integer
+                elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'):
+                    token = Number.Float
+                else:
+                    # if the stack depth is deeper than once, pop
+                    if len(stack) > 1:
+                        stack.pop()
+                    scanner.get_char()
+
+            elif stack[-1] == 'string':
+                if self.is_portugol:
+                    if scanner.scan(r"''"):
+                        token = String.Escape
+                    elif scanner.scan(r"\""):
+                        token = String
+                        stack.pop()
+                    elif scanner.scan(r"[^\"]*"):
+                        token = String
+                    else:
+                        scanner.get_char()
+                        stack.pop()
+                else:
+                    if scanner.scan(r"''"):
+                        token = String.Escape
+                    elif scanner.scan(r"'"):
+                        token = String
+                        stack.pop()
+                    elif scanner.scan(r"[^']*"):
+                        token = String
+                    else:
+                        scanner.get_char()
+                        stack.pop()
+            elif not self.is_portugol and stack[-1] == 'asm':
+                if scanner.scan(r'\s+'):
+                    token = Whitespace
+                elif scanner.scan(r'end'):
+                    token = Keyword
+                    stack.pop()
+                elif scanner.scan(r'\{.*?\}|\(\*.*?\*\)'):
+                    if scanner.match.startswith('$'):
+                        token = Comment.Preproc
+                    else:
+                        token = Comment.Multiline
+                elif scanner.scan(r'//.*?$'):
+                    token = Comment.Single
+                elif scanner.scan(r"'"):
+                    token = String
+                    stack.append('string')
+                elif scanner.scan(r'@@[A-Za-z_][A-Za-z_0-9]*'):
+                    token = Name.Label
+                elif scanner.scan(r'[A-Za-z_][A-Za-z_0-9]*'):
+                    lowercase_name = scanner.match.lower()
+                    if lowercase_name in self.ASM_INSTRUCTIONS:
+                        token = Keyword
+                    elif lowercase_name in self.ASM_REGISTERS:
+                        token = Name.Builtin
+                    else:
+                        token = Name
+                elif scanner.scan(r'[-+*\/=<>:;,.@\^]+'):
+                    token = Operator
+                elif scanner.scan(r'[\(\)\[\]]+'):
+                    token = Punctuation
+                elif scanner.scan(r'\$[0-9A-Fa-f]+'):
+                    token = Number.Hex
+                elif scanner.scan(r'\d+(?![eE]|\.[^.])'):
+                    token = Number.Integer
+                elif scanner.scan(r'\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)'):
+                    token = Number.Float
+                else:
+                    scanner.get_char()
+                    stack.pop()
+
+            # save the dot!!!11
+            if not self.is_portugol and scanner.match.strip():
+                was_dot = scanner.match == '.'
+
+            yield scanner.start_pos, token, scanner.match or ''
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pawn.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pawn.py
new file mode 100644
index 0000000000000000000000000000000000000000..9b4234c3a274ed46b01f0aee17df86e7868e1502
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pawn.py
@@ -0,0 +1,202 @@
+"""
+    pygments.lexers.pawn
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Pawn languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+from pygments.util import get_bool_opt
+
+__all__ = ['SourcePawnLexer', 'PawnLexer']
+
+
+class SourcePawnLexer(RegexLexer):
+    """
+    For SourcePawn source code with preprocessor directives.
+    """
+    name = 'SourcePawn'
+    aliases = ['sp']
+    filenames = ['*.sp']
+    mimetypes = ['text/x-sourcepawn']
+    url = 'https://github.com/alliedmodders/sourcepawn'
+    version_added = '1.6'
+
+    #: optional Comment or Whitespace
+    _ws = r'(?:\s|//.*?\n|/\*.*?\*/)+'
+    #: only one /* */ style comment
+    _ws1 = r'\s*(?:/[*].*?[*]/\s*)*'
+
+    tokens = {
+        'root': [
+            # preprocessor directives: without whitespace
+            (r'^#if\s+0', Comment.Preproc, 'if0'),
+            ('^#', Comment.Preproc, 'macro'),
+            # or with whitespace
+            ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'),
+            ('^' + _ws1 + '#', Comment.Preproc, 'macro'),
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'\\\n', Text),  # line continuation
+            (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
+            (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline),
+            (r'[{}]', Punctuation),
+            (r'L?"', String, 'string'),
+            (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+            (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
+            (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+            (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
+            (r'0[0-7]+[LlUu]*', Number.Oct),
+            (r'\d+[LlUu]*', Number.Integer),
+            (r'[~!%^&*+=|?:<>/-]', Operator),
+            (r'[()\[\],.;]', Punctuation),
+            (r'(case|const|continue|native|'
+             r'default|else|enum|for|if|new|operator|'
+             r'public|return|sizeof|static|decl|struct|switch)\b', Keyword),
+            (r'(bool|Float)\b', Keyword.Type),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'[a-zA-Z_]\w*', Name),
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+            (r'[^\\"\n]+', String),  # all other characters
+            (r'\\\n', String),       # line continuation
+            (r'\\', String),         # stray backslash
+        ],
+        'macro': [
+            (r'[^/\n]+', Comment.Preproc),
+            (r'/\*(.|\n)*?\*/', Comment.Multiline),
+            (r'//.*?\n', Comment.Single, '#pop'),
+            (r'/', Comment.Preproc),
+            (r'(?<=\\)\n', Comment.Preproc),
+            (r'\n', Comment.Preproc, '#pop'),
+        ],
+        'if0': [
+            (r'^\s*#if.*?(?/-]', Operator),
+            (r'[()\[\],.;]', Punctuation),
+            (r'(switch|case|default|const|new|static|char|continue|break|'
+             r'if|else|for|while|do|operator|enum|'
+             r'public|return|sizeof|tagof|state|goto)\b', Keyword),
+            (r'(bool|Float)\b', Keyword.Type),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'[a-zA-Z_]\w*', Name),
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
+            (r'[^\\"\n]+', String),  # all other characters
+            (r'\\\n', String),       # line continuation
+            (r'\\', String),         # stray backslash
+        ],
+        'macro': [
+            (r'[^/\n]+', Comment.Preproc),
+            (r'/\*(.|\n)*?\*/', Comment.Multiline),
+            (r'//.*?\n', Comment.Single, '#pop'),
+            (r'/', Comment.Preproc),
+            (r'(?<=\\)\n', Comment.Preproc),
+            (r'\n', Comment.Preproc, '#pop'),
+        ],
+        'if0': [
+            (r'^\s*#if.*?(?<-]', Operator),
+            (r'[a-zA-Z][a-zA-Z0-9_-]*', Name),
+            (r'\?[a-zA-Z][a-zA-Z0-9_-]*', Name.Variable),
+            (r'[0-9]+\.[0-9]+', Number.Float),
+            (r'[0-9]+', Number.Integer),
+        ],
+        'keywords': [
+            (words((
+                ':requirements', ':types', ':constants',
+                ':predicates', ':functions', ':action', ':agent',
+                ':parameters', ':precondition', ':effect',
+                ':durative-action', ':duration', ':condition',
+                ':derived', ':domain', ':objects', ':init',
+                ':goal', ':metric', ':length', ':serial', ':parallel',
+                # the following are requirements
+                ':strips', ':typing', ':negative-preconditions',
+                ':disjunctive-preconditions', ':equality',
+                ':existential-preconditions', ':universal-preconditions',
+                ':conditional-effects', ':fluents', ':numeric-fluents',
+                ':object-fluents', ':adl', ':durative-actions',
+                ':continuous-effects', ':derived-predicates',
+                ':time-intial-literals', ':preferences',
+                ':constraints', ':action-costs', ':multi-agent',
+                ':unfactored-privacy', ':factored-privacy',
+                ':non-deterministic'
+                ), suffix=r'\b'), Keyword)
+        ],
+        'builtins': [
+            (words((
+                'define', 'domain', 'object', 'either', 'and',
+                'forall', 'preference', 'imply', 'or', 'exists',
+                'not', 'when', 'assign', 'scale-up', 'scale-down',
+                'increase', 'decrease', 'at', 'over', 'start',
+                'end', 'all', 'problem', 'always', 'sometime',
+                'within', 'at-most-once', 'sometime-after',
+                'sometime-before', 'always-within', 'hold-during',
+                'hold-after', 'minimize', 'maximize',
+                'total-time', 'is-violated'), suffix=r'\b'),
+                Name.Builtin)
+        ]
+    }
+
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/perl.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/perl.py
new file mode 100644
index 0000000000000000000000000000000000000000..7448d2a1e2bb822837079cfec4690b56825bf0b7
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/perl.py
@@ -0,0 +1,733 @@
+"""
+    pygments.lexers.perl
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Perl, Raku and related languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, ExtendedRegexLexer, include, bygroups, \
+    using, this, default, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+from pygments.util import shebang_matches
+
+__all__ = ['PerlLexer', 'Perl6Lexer']
+
+
+class PerlLexer(RegexLexer):
+    """
+    For Perl source code.
+    """
+
+    name = 'Perl'
+    url = 'https://www.perl.org'
+    aliases = ['perl', 'pl']
+    filenames = ['*.pl', '*.pm', '*.t', '*.perl']
+    mimetypes = ['text/x-perl', 'application/x-perl']
+    version_added = ''
+
+    flags = re.DOTALL | re.MULTILINE
+    # TODO: give this to a perl guy who knows how to parse perl...
+    tokens = {
+        'balanced-regex': [
+            (r'/(\\\\|\\[^\\]|[^\\/])*/[egimosx]*', String.Regex, '#pop'),
+            (r'!(\\\\|\\[^\\]|[^\\!])*![egimosx]*', String.Regex, '#pop'),
+            (r'\\(\\\\|[^\\])*\\[egimosx]*', String.Regex, '#pop'),
+            (r'\{(\\\\|\\[^\\]|[^\\}])*\}[egimosx]*', String.Regex, '#pop'),
+            (r'<(\\\\|\\[^\\]|[^\\>])*>[egimosx]*', String.Regex, '#pop'),
+            (r'\[(\\\\|\\[^\\]|[^\\\]])*\][egimosx]*', String.Regex, '#pop'),
+            (r'\((\\\\|\\[^\\]|[^\\)])*\)[egimosx]*', String.Regex, '#pop'),
+            (r'@(\\\\|\\[^\\]|[^\\@])*@[egimosx]*', String.Regex, '#pop'),
+            (r'%(\\\\|\\[^\\]|[^\\%])*%[egimosx]*', String.Regex, '#pop'),
+            (r'\$(\\\\|\\[^\\]|[^\\$])*\$[egimosx]*', String.Regex, '#pop'),
+        ],
+        'root': [
+            (r'\A\#!.+?$', Comment.Hashbang),
+            (r'\#.*?$', Comment.Single),
+            (r'^=[a-zA-Z0-9]+\s+.*?\n=cut', Comment.Multiline),
+            (words((
+                'case', 'continue', 'do', 'else', 'elsif', 'for', 'foreach',
+                'if', 'last', 'my', 'next', 'our', 'redo', 'reset', 'then',
+                'unless', 'until', 'while', 'print', 'new', 'BEGIN',
+                'CHECK', 'INIT', 'END', 'return'), suffix=r'\b'),
+             Keyword),
+            (r'(format)(\s+)(\w+)(\s*)(=)(\s*\n)',
+             bygroups(Keyword, Whitespace, Name, Whitespace, Punctuation, Whitespace), 'format'),
+            (r'(eq|lt|gt|le|ge|ne|not|and|or|cmp)\b', Operator.Word),
+            # common delimiters
+            (r's/(\\\\|\\[^\\]|[^\\/])*/(\\\\|\\[^\\]|[^\\/])*/[egimosx]*',
+                String.Regex),
+            (r's!(\\\\|\\!|[^!])*!(\\\\|\\!|[^!])*![egimosx]*', String.Regex),
+            (r's\\(\\\\|[^\\])*\\(\\\\|[^\\])*\\[egimosx]*', String.Regex),
+            (r's@(\\\\|\\[^\\]|[^\\@])*@(\\\\|\\[^\\]|[^\\@])*@[egimosx]*',
+                String.Regex),
+            (r's%(\\\\|\\[^\\]|[^\\%])*%(\\\\|\\[^\\]|[^\\%])*%[egimosx]*',
+                String.Regex),
+            # balanced delimiters
+            (r's\{(\\\\|\\[^\\]|[^\\}])*\}\s*', String.Regex, 'balanced-regex'),
+            (r's<(\\\\|\\[^\\]|[^\\>])*>\s*', String.Regex, 'balanced-regex'),
+            (r's\[(\\\\|\\[^\\]|[^\\\]])*\]\s*', String.Regex,
+                'balanced-regex'),
+            (r's\((\\\\|\\[^\\]|[^\\)])*\)\s*', String.Regex,
+                'balanced-regex'),
+
+            (r'm?/(\\\\|\\[^\\]|[^\\/\n])*/[gcimosx]*', String.Regex),
+            (r'm(?=[/!\\{<\[(@%$])', String.Regex, 'balanced-regex'),
+            (r'((?<==~)|(?<=\())\s*/(\\\\|\\[^\\]|[^\\/])*/[gcimosx]*',
+                String.Regex),
+            (r'\s+', Whitespace),
+            (words((
+                'abs', 'accept', 'alarm', 'atan2', 'bind', 'binmode', 'bless', 'caller', 'chdir',
+                'chmod', 'chomp', 'chop', 'chown', 'chr', 'chroot', 'close', 'closedir', 'connect',
+                'continue', 'cos', 'crypt', 'dbmclose', 'dbmopen', 'defined', 'delete', 'die',
+                'dump', 'each', 'endgrent', 'endhostent', 'endnetent', 'endprotoent',
+                'endpwent', 'endservent', 'eof', 'eval', 'exec', 'exists', 'exit', 'exp', 'fcntl',
+                'fileno', 'flock', 'fork', 'format', 'formline', 'getc', 'getgrent', 'getgrgid',
+                'getgrnam', 'gethostbyaddr', 'gethostbyname', 'gethostent', 'getlogin',
+                'getnetbyaddr', 'getnetbyname', 'getnetent', 'getpeername', 'getpgrp',
+                'getppid', 'getpriority', 'getprotobyname', 'getprotobynumber',
+                'getprotoent', 'getpwent', 'getpwnam', 'getpwuid', 'getservbyname',
+                'getservbyport', 'getservent', 'getsockname', 'getsockopt', 'glob', 'gmtime',
+                'goto', 'grep', 'hex', 'import', 'index', 'int', 'ioctl', 'join', 'keys', 'kill', 'last',
+                'lc', 'lcfirst', 'length', 'link', 'listen', 'local', 'localtime', 'log', 'lstat',
+                'map', 'mkdir', 'msgctl', 'msgget', 'msgrcv', 'msgsnd', 'my', 'next', 'oct', 'open',
+                'opendir', 'ord', 'our', 'pack', 'pipe', 'pop', 'pos', 'printf',
+                'prototype', 'push', 'quotemeta', 'rand', 'read', 'readdir',
+                'readline', 'readlink', 'readpipe', 'recv', 'redo', 'ref', 'rename',
+                'reverse', 'rewinddir', 'rindex', 'rmdir', 'scalar', 'seek', 'seekdir',
+                'select', 'semctl', 'semget', 'semop', 'send', 'setgrent', 'sethostent', 'setnetent',
+                'setpgrp', 'setpriority', 'setprotoent', 'setpwent', 'setservent',
+                'setsockopt', 'shift', 'shmctl', 'shmget', 'shmread', 'shmwrite', 'shutdown',
+                'sin', 'sleep', 'socket', 'socketpair', 'sort', 'splice', 'split', 'sprintf', 'sqrt',
+                'srand', 'stat', 'study', 'substr', 'symlink', 'syscall', 'sysopen', 'sysread',
+                'sysseek', 'system', 'syswrite', 'tell', 'telldir', 'tie', 'tied', 'time', 'times', 'tr',
+                'truncate', 'uc', 'ucfirst', 'umask', 'undef', 'unlink', 'unpack', 'unshift', 'untie',
+                'utime', 'values', 'vec', 'wait', 'waitpid', 'wantarray', 'warn', 'write'), suffix=r'\b'),
+             Name.Builtin),
+            (r'((__(DATA|DIE|WARN)__)|(STD(IN|OUT|ERR)))\b', Name.Builtin.Pseudo),
+            (r'(<<)([\'"]?)([a-zA-Z_]\w*)(\2;?\n.*?\n)(\3)(\n)',
+             bygroups(String, String, String.Delimiter, String, String.Delimiter, Whitespace)),
+            (r'__END__', Comment.Preproc, 'end-part'),
+            (r'\$\^[ADEFHILMOPSTWX]', Name.Variable.Global),
+            (r"\$[\\\"\[\]'&`+*.,;=%~?@$!<>(^|/-](?!\w)", Name.Variable.Global),
+            (r'[$@%#]+', Name.Variable, 'varname'),
+            (r'0_?[0-7]+(_[0-7]+)*', Number.Oct),
+            (r'0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*', Number.Hex),
+            (r'0b[01]+(_[01]+)*', Number.Bin),
+            (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?',
+             Number.Float),
+            (r'(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*', Number.Float),
+            (r'\d+(_\d+)*', Number.Integer),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+            (r'`(\\\\|\\[^\\]|[^`\\])*`', String.Backtick),
+            (r'<([^\s>]+)>', String.Regex),
+            (r'(q|qq|qw|qr|qx)\{', String.Other, 'cb-string'),
+            (r'(q|qq|qw|qr|qx)\(', String.Other, 'rb-string'),
+            (r'(q|qq|qw|qr|qx)\[', String.Other, 'sb-string'),
+            (r'(q|qq|qw|qr|qx)\<', String.Other, 'lt-string'),
+            (r'(q|qq|qw|qr|qx)([\W_])(.|\n)*?\2', String.Other),
+            (r'(package)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)',
+             bygroups(Keyword, Whitespace, Name.Namespace)),
+            (r'(use|require|no)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)',
+             bygroups(Keyword, Whitespace, Name.Namespace)),
+            (r'(sub)(\s+)', bygroups(Keyword, Whitespace), 'funcname'),
+            (words((
+                'no', 'package', 'require', 'use'), suffix=r'\b'),
+             Keyword),
+            (r'(\[\]|\*\*|::|<<|>>|>=|<=>|<=|={3}|!=|=~|'
+             r'!~|&&?|\|\||\.{1,3})', Operator),
+            (r'[-+/*%=<>&^|!\\~]=?', Operator),
+            (r'[()\[\]:;,<>/?{}]', Punctuation),  # yes, there's no shortage
+                                                  # of punctuation in Perl!
+            (r'(?=\w)', Name, 'name'),
+        ],
+        'format': [
+            (r'\.\n', String.Interpol, '#pop'),
+            (r'[^\n]*\n', String.Interpol),
+        ],
+        'varname': [
+            (r'\s+', Whitespace),
+            (r'\{', Punctuation, '#pop'),    # hash syntax?
+            (r'\)|,', Punctuation, '#pop'),  # argument specifier
+            (r'\w+::', Name.Namespace),
+            (r'[\w:]+', Name.Variable, '#pop'),
+        ],
+        'name': [
+            (r'[a-zA-Z_]\w*(::[a-zA-Z_]\w*)*(::)?(?=\s*->)', Name.Namespace, '#pop'),
+            (r'[a-zA-Z_]\w*(::[a-zA-Z_]\w*)*::', Name.Namespace, '#pop'),
+            (r'[\w:]+', Name, '#pop'),
+            (r'[A-Z_]+(?=\W)', Name.Constant, '#pop'),
+            (r'(?=\W)', Text, '#pop'),
+        ],
+        'funcname': [
+            (r'[a-zA-Z_]\w*[!?]?', Name.Function),
+            (r'\s+', Whitespace),
+            # argument declaration
+            (r'(\([$@%]*\))(\s*)', bygroups(Punctuation, Whitespace)),
+            (r';', Punctuation, '#pop'),
+            (r'.*?\{', Punctuation, '#pop'),
+        ],
+        'cb-string': [
+            (r'\\[{}\\]', String.Other),
+            (r'\\', String.Other),
+            (r'\{', String.Other, 'cb-string'),
+            (r'\}', String.Other, '#pop'),
+            (r'[^{}\\]+', String.Other)
+        ],
+        'rb-string': [
+            (r'\\[()\\]', String.Other),
+            (r'\\', String.Other),
+            (r'\(', String.Other, 'rb-string'),
+            (r'\)', String.Other, '#pop'),
+            (r'[^()]+', String.Other)
+        ],
+        'sb-string': [
+            (r'\\[\[\]\\]', String.Other),
+            (r'\\', String.Other),
+            (r'\[', String.Other, 'sb-string'),
+            (r'\]', String.Other, '#pop'),
+            (r'[^\[\]]+', String.Other)
+        ],
+        'lt-string': [
+            (r'\\[<>\\]', String.Other),
+            (r'\\', String.Other),
+            (r'\<', String.Other, 'lt-string'),
+            (r'\>', String.Other, '#pop'),
+            (r'[^<>]+', String.Other)
+        ],
+        'end-part': [
+            (r'.+', Comment.Preproc, '#pop')
+        ]
+    }
+
+    def analyse_text(text):
+        if shebang_matches(text, r'perl'):
+            return True
+
+        result = 0
+
+        if re.search(r'(?:my|our)\s+[$@%(]', text):
+            result += 0.9
+
+        if ':=' in text:
+            # := is not valid Perl, but it appears in unicon, so we should
+            # become less confident if we think we found Perl with :=
+            result /= 2
+
+        return result
+
+
+class Perl6Lexer(ExtendedRegexLexer):
+    """
+    For Raku (a.k.a. Perl 6) source code.
+    """
+
+    name = 'Perl6'
+    url = 'https://www.raku.org'
+    aliases = ['perl6', 'pl6', 'raku']
+    filenames = ['*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6',
+                 '*.6pm', '*.p6m', '*.pm6', '*.t', '*.raku', '*.rakumod',
+                 '*.rakutest', '*.rakudoc']
+    mimetypes = ['text/x-perl6', 'application/x-perl6']
+    version_added = '2.0'
+    flags = re.MULTILINE | re.DOTALL
+
+    PERL6_IDENTIFIER_RANGE = r"['\w:-]"
+
+    PERL6_KEYWORDS = (
+        #Phasers
+        'BEGIN','CATCH','CHECK','CLOSE','CONTROL','DOC','END','ENTER','FIRST',
+        'INIT','KEEP','LAST','LEAVE','NEXT','POST','PRE','QUIT','UNDO',
+        #Keywords
+        'anon','augment','but','class','constant','default','does','else',
+        'elsif','enum','for','gather','given','grammar','has','if','import',
+        'is','let','loop','made','make','method','module','multi','my','need',
+        'orwith','our','proceed','proto','repeat','require','return',
+        'return-rw','returns','role','rule','state','sub','submethod','subset',
+        'succeed','supersede','token','try','unit','unless','until','use',
+        'when','while','with','without',
+        #Traits
+        'export','native','repr','required','rw','symbol',
+    )
+
+    PERL6_BUILTINS = (
+        'ACCEPTS','abs','abs2rel','absolute','accept','accessed','acos',
+        'acosec','acosech','acosh','acotan','acotanh','acquire','act','action',
+        'actions','add','add_attribute','add_enum_value','add_fallback',
+        'add_method','add_parent','add_private_method','add_role','add_trustee',
+        'adverb','after','all','allocate','allof','allowed','alternative-names',
+        'annotations','antipair','antipairs','any','anyof','app_lifetime',
+        'append','arch','archname','args','arity','Array','asec','asech','asin',
+        'asinh','ASSIGN-KEY','ASSIGN-POS','assuming','ast','at','atan','atan2',
+        'atanh','AT-KEY','atomic-assign','atomic-dec-fetch','atomic-fetch',
+        'atomic-fetch-add','atomic-fetch-dec','atomic-fetch-inc',
+        'atomic-fetch-sub','atomic-inc-fetch','AT-POS','attributes','auth',
+        'await','backtrace','Bag','BagHash','bail-out','base','basename',
+        'base-repeating','batch','BIND-KEY','BIND-POS','bind-stderr',
+        'bind-stdin','bind-stdout','bind-udp','bits','bless','block','Bool',
+        'bool-only','bounds','break','Bridge','broken','BUILD','build-date',
+        'bytes','cache','callframe','calling-package','CALL-ME','callsame',
+        'callwith','can','cancel','candidates','cando','can-ok','canonpath',
+        'caps','caption','Capture','cas','catdir','categorize','categorize-list',
+        'catfile','catpath','cause','ceiling','cglobal','changed','Channel',
+        'chars','chdir','child','child-name','child-typename','chmod','chomp',
+        'chop','chr','chrs','chunks','cis','classify','classify-list','cleanup',
+        'clone','close','closed','close-stdin','cmp-ok','code','codes','collate',
+        'column','comb','combinations','command','comment','compiler','Complex',
+        'compose','compose_type','composer','condition','config',
+        'configure_destroy','configure_type_checking','conj','connect',
+        'constraints','construct','contains','contents','copy','cos','cosec',
+        'cosech','cosh','cotan','cotanh','count','count-only','cpu-cores',
+        'cpu-usage','CREATE','create_type','cross','cue','curdir','curupdir','d',
+        'Date','DateTime','day','daycount','day-of-month','day-of-week',
+        'day-of-year','days-in-month','declaration','decode','decoder','deepmap',
+        'default','defined','DEFINITE','delayed','DELETE-KEY','DELETE-POS',
+        'denominator','desc','DESTROY','destroyers','devnull','diag',
+        'did-you-mean','die','dies-ok','dir','dirname','dir-sep','DISTROnames',
+        'do','does','does-ok','done','done-testing','duckmap','dynamic','e',
+        'eager','earlier','elems','emit','enclosing','encode','encoder',
+        'encoding','end','ends-with','enum_from_value','enum_value_list',
+        'enum_values','enums','eof','EVAL','eval-dies-ok','EVALFILE',
+        'eval-lives-ok','exception','excludes-max','excludes-min','EXISTS-KEY',
+        'EXISTS-POS','exit','exitcode','exp','expected','explicitly-manage',
+        'expmod','extension','f','fail','fails-like','fc','feature','file',
+        'filename','find_method','find_method_qualified','finish','first','flat',
+        'flatmap','flip','floor','flunk','flush','fmt','format','formatter',
+        'freeze','from','from-list','from-loop','from-posix','full',
+        'full-barrier','get','get_value','getc','gist','got','grab','grabpairs',
+        'grep','handle','handled','handles','hardware','has_accessor','Hash',
+        'head','headers','hh-mm-ss','hidden','hides','hour','how','hyper','id',
+        'illegal','im','in','indent','index','indices','indir','infinite',
+        'infix','infix:<+>','infix:<->','install_method_cache','Instant',
+        'instead','Int','int-bounds','interval','in-timezone','invalid-str',
+        'invert','invocant','IO','IO::Notification.watch-path','is_trusted',
+        'is_type','isa','is-absolute','isa-ok','is-approx','is-deeply',
+        'is-hidden','is-initial-thread','is-int','is-lazy','is-leap-year',
+        'isNaN','isnt','is-prime','is-relative','is-routine','is-setting',
+        'is-win','item','iterator','join','keep','kept','KERNELnames','key',
+        'keyof','keys','kill','kv','kxxv','l','lang','last','lastcall','later',
+        'lazy','lc','leading','level','like','line','lines','link','List',
+        'listen','live','lives-ok','local','lock','log','log10','lookup','lsb',
+        'made','MAIN','make','Map','match','max','maxpairs','merge','message',
+        'method','method_table','methods','migrate','min','minmax','minpairs',
+        'minute','misplaced','Mix','MixHash','mkdir','mode','modified','month',
+        'move','mro','msb','multi','multiness','my','name','named','named_names',
+        'narrow','nativecast','native-descriptor','nativesizeof','new','new_type',
+        'new-from-daycount','new-from-pairs','next','nextcallee','next-handle',
+        'nextsame','nextwith','NFC','NFD','NFKC','NFKD','nl-in','nl-out',
+        'nodemap','nok','none','norm','not','note','now','nude','Num',
+        'numerator','Numeric','of','offset','offset-in-hours','offset-in-minutes',
+        'ok','old','on-close','one','on-switch','open','opened','operation',
+        'optional','ord','ords','orig','os-error','osname','out-buffer','pack',
+        'package','package-kind','package-name','packages','pair','pairs',
+        'pairup','parameter','params','parent','parent-name','parents','parse',
+        'parse-base','parsefile','parse-names','parts','pass','path','path-sep',
+        'payload','peer-host','peer-port','periods','perl','permutations','phaser',
+        'pick','pickpairs','pid','placeholder','plan','plus','polar','poll',
+        'polymod','pop','pos','positional','posix','postfix','postmatch',
+        'precomp-ext','precomp-target','pred','prefix','prematch','prepend',
+        'print','printf','print-nl','print-to','private','private_method_table',
+        'proc','produce','Promise','prompt','protect','pull-one','push',
+        'push-all','push-at-least','push-exactly','push-until-lazy','put',
+        'qualifier-type','quit','r','race','radix','rand','range','Rat','raw',
+        're','read','readchars','readonly','ready','Real','reallocate','reals',
+        'reason','rebless','receive','recv','redispatcher','redo','reduce',
+        'rel2abs','relative','release','rename','repeated','replacement',
+        'report','reserved','resolve','restore','result','resume','rethrow',
+        'reverse','right','rindex','rmdir','role','roles_to_compose','rolish',
+        'roll','rootdir','roots','rotate','rotor','round','roundrobin',
+        'routine-type','run','rwx','s','samecase','samemark','samewith','say',
+        'schedule-on','scheduler','scope','sec','sech','second','seek','self',
+        'send','Set','set_hidden','set_name','set_package','set_rw','set_value',
+        'SetHash','set-instruments','setup_finalization','shape','share','shell',
+        'shift','sibling','sigil','sign','signal','signals','signature','sin',
+        'sinh','sink','sink-all','skip','skip-at-least','skip-at-least-pull-one',
+        'skip-one','skip-rest','sleep','sleep-timer','sleep-until','Slip','slurp',
+        'slurp-rest','slurpy','snap','snapper','so','socket-host','socket-port',
+        'sort','source','source-package','spawn','SPEC','splice','split',
+        'splitdir','splitpath','sprintf','spurt','sqrt','squish','srand','stable',
+        'start','started','starts-with','status','stderr','stdout','Str',
+        'sub_signature','subbuf','subbuf-rw','subname','subparse','subst',
+        'subst-mutate','substr','substr-eq','substr-rw','subtest','succ','sum',
+        'Supply','symlink','t','tail','take','take-rw','tan','tanh','tap',
+        'target','target-name','tc','tclc','tell','then','throttle','throw',
+        'throws-like','timezone','tmpdir','to','today','todo','toggle','to-posix',
+        'total','trailing','trans','tree','trim','trim-leading','trim-trailing',
+        'truncate','truncated-to','trusts','try_acquire','trying','twigil','type',
+        'type_captures','typename','uc','udp','uncaught_handler','unimatch',
+        'uniname','uninames','uniparse','uniprop','uniprops','unique','unival',
+        'univals','unlike','unlink','unlock','unpack','unpolar','unshift',
+        'unwrap','updir','USAGE','use-ok','utc','val','value','values','VAR',
+        'variable','verbose-config','version','VMnames','volume','vow','w','wait',
+        'warn','watch','watch-path','week','weekday-of-month','week-number',
+        'week-year','WHAT','when','WHERE','WHEREFORE','WHICH','WHO',
+        'whole-second','WHY','wordcase','words','workaround','wrap','write',
+        'write-to','x','yada','year','yield','yyyy-mm-dd','z','zip','zip-latest',
+
+    )
+
+    PERL6_BUILTIN_CLASSES = (
+        #Booleans
+        'False','True',
+        #Classes
+        'Any','Array','Associative','AST','atomicint','Attribute','Backtrace',
+        'Backtrace::Frame','Bag','Baggy','BagHash','Blob','Block','Bool','Buf',
+        'Callable','CallFrame','Cancellation','Capture','CArray','Channel','Code',
+        'compiler','Complex','ComplexStr','Cool','CurrentThreadScheduler',
+        'Cursor','Date','Dateish','DateTime','Distro','Duration','Encoding',
+        'Exception','Failure','FatRat','Grammar','Hash','HyperWhatever','Instant',
+        'Int','int16','int32','int64','int8','IntStr','IO','IO::ArgFiles',
+        'IO::CatHandle','IO::Handle','IO::Notification','IO::Path',
+        'IO::Path::Cygwin','IO::Path::QNX','IO::Path::Unix','IO::Path::Win32',
+        'IO::Pipe','IO::Socket','IO::Socket::Async','IO::Socket::INET','IO::Spec',
+        'IO::Spec::Cygwin','IO::Spec::QNX','IO::Spec::Unix','IO::Spec::Win32',
+        'IO::Special','Iterable','Iterator','Junction','Kernel','Label','List',
+        'Lock','Lock::Async','long','longlong','Macro','Map','Match',
+        'Metamodel::AttributeContainer','Metamodel::C3MRO','Metamodel::ClassHOW',
+        'Metamodel::EnumHOW','Metamodel::Finalization','Metamodel::MethodContainer',
+        'Metamodel::MROBasedMethodDispatch','Metamodel::MultipleInheritance',
+        'Metamodel::Naming','Metamodel::Primitives','Metamodel::PrivateMethodContainer',
+        'Metamodel::RoleContainer','Metamodel::Trusting','Method','Mix','MixHash',
+        'Mixy','Mu','NFC','NFD','NFKC','NFKD','Nil','Num','num32','num64',
+        'Numeric','NumStr','ObjAt','Order','Pair','Parameter','Perl','Pod::Block',
+        'Pod::Block::Code','Pod::Block::Comment','Pod::Block::Declarator',
+        'Pod::Block::Named','Pod::Block::Para','Pod::Block::Table','Pod::Heading',
+        'Pod::Item','Pointer','Positional','PositionalBindFailover','Proc',
+        'Proc::Async','Promise','Proxy','PseudoStash','QuantHash','Range','Rat',
+        'Rational','RatStr','Real','Regex','Routine','Scalar','Scheduler',
+        'Semaphore','Seq','Set','SetHash','Setty','Signature','size_t','Slip',
+        'Stash','Str','StrDistance','Stringy','Sub','Submethod','Supplier',
+        'Supplier::Preserving','Supply','Systemic','Tap','Telemetry',
+        'Telemetry::Instrument::Thread','Telemetry::Instrument::Usage',
+        'Telemetry::Period','Telemetry::Sampler','Thread','ThreadPoolScheduler',
+        'UInt','uint16','uint32','uint64','uint8','Uni','utf8','Variable',
+        'Version','VM','Whatever','WhateverCode','WrapHandle'
+    )
+
+    PERL6_OPERATORS = (
+        'X', 'Z', 'after', 'also', 'and', 'andthen', 'before', 'cmp', 'div',
+        'eq', 'eqv', 'extra', 'ff', 'fff', 'ge', 'gt', 'le', 'leg', 'lt', 'm',
+        'mm', 'mod', 'ne', 'or', 'orelse', 'rx', 's', 'tr', 'x', 'xor', 'xx',
+        '++', '--', '**', '!', '+', '-', '~', '?', '|', '||', '+^', '~^', '?^',
+        '^', '*', '/', '%', '%%', '+&', '+<', '+>', '~&', '~<', '~>', '?&',
+        'gcd', 'lcm', '+', '-', '+|', '+^', '~|', '~^', '?|', '?^',
+        '~', '&', '^', 'but', 'does', '<=>', '..', '..^', '^..', '^..^',
+        '!=', '==', '<', '<=', '>', '>=', '~~', '===', '!eqv',
+        '&&', '||', '^^', '//', 'min', 'max', '??', '!!', 'ff', 'fff', 'so',
+        'not', '<==', '==>', '<<==', '==>>','unicmp',
+    )
+
+    # Perl 6 has a *lot* of possible bracketing characters
+    # this list was lifted from STD.pm6 (https://github.com/perl6/std)
+    PERL6_BRACKETS = {
+        '\u0028': '\u0029', '\u003c': '\u003e', '\u005b': '\u005d',
+        '\u007b': '\u007d', '\u00ab': '\u00bb', '\u0f3a': '\u0f3b',
+        '\u0f3c': '\u0f3d', '\u169b': '\u169c', '\u2018': '\u2019',
+        '\u201a': '\u2019', '\u201b': '\u2019', '\u201c': '\u201d',
+        '\u201e': '\u201d', '\u201f': '\u201d', '\u2039': '\u203a',
+        '\u2045': '\u2046', '\u207d': '\u207e', '\u208d': '\u208e',
+        '\u2208': '\u220b', '\u2209': '\u220c', '\u220a': '\u220d',
+        '\u2215': '\u29f5', '\u223c': '\u223d', '\u2243': '\u22cd',
+        '\u2252': '\u2253', '\u2254': '\u2255', '\u2264': '\u2265',
+        '\u2266': '\u2267', '\u2268': '\u2269', '\u226a': '\u226b',
+        '\u226e': '\u226f', '\u2270': '\u2271', '\u2272': '\u2273',
+        '\u2274': '\u2275', '\u2276': '\u2277', '\u2278': '\u2279',
+        '\u227a': '\u227b', '\u227c': '\u227d', '\u227e': '\u227f',
+        '\u2280': '\u2281', '\u2282': '\u2283', '\u2284': '\u2285',
+        '\u2286': '\u2287', '\u2288': '\u2289', '\u228a': '\u228b',
+        '\u228f': '\u2290', '\u2291': '\u2292', '\u2298': '\u29b8',
+        '\u22a2': '\u22a3', '\u22a6': '\u2ade', '\u22a8': '\u2ae4',
+        '\u22a9': '\u2ae3', '\u22ab': '\u2ae5', '\u22b0': '\u22b1',
+        '\u22b2': '\u22b3', '\u22b4': '\u22b5', '\u22b6': '\u22b7',
+        '\u22c9': '\u22ca', '\u22cb': '\u22cc', '\u22d0': '\u22d1',
+        '\u22d6': '\u22d7', '\u22d8': '\u22d9', '\u22da': '\u22db',
+        '\u22dc': '\u22dd', '\u22de': '\u22df', '\u22e0': '\u22e1',
+        '\u22e2': '\u22e3', '\u22e4': '\u22e5', '\u22e6': '\u22e7',
+        '\u22e8': '\u22e9', '\u22ea': '\u22eb', '\u22ec': '\u22ed',
+        '\u22f0': '\u22f1', '\u22f2': '\u22fa', '\u22f3': '\u22fb',
+        '\u22f4': '\u22fc', '\u22f6': '\u22fd', '\u22f7': '\u22fe',
+        '\u2308': '\u2309', '\u230a': '\u230b', '\u2329': '\u232a',
+        '\u23b4': '\u23b5', '\u2768': '\u2769', '\u276a': '\u276b',
+        '\u276c': '\u276d', '\u276e': '\u276f', '\u2770': '\u2771',
+        '\u2772': '\u2773', '\u2774': '\u2775', '\u27c3': '\u27c4',
+        '\u27c5': '\u27c6', '\u27d5': '\u27d6', '\u27dd': '\u27de',
+        '\u27e2': '\u27e3', '\u27e4': '\u27e5', '\u27e6': '\u27e7',
+        '\u27e8': '\u27e9', '\u27ea': '\u27eb', '\u2983': '\u2984',
+        '\u2985': '\u2986', '\u2987': '\u2988', '\u2989': '\u298a',
+        '\u298b': '\u298c', '\u298d': '\u298e', '\u298f': '\u2990',
+        '\u2991': '\u2992', '\u2993': '\u2994', '\u2995': '\u2996',
+        '\u2997': '\u2998', '\u29c0': '\u29c1', '\u29c4': '\u29c5',
+        '\u29cf': '\u29d0', '\u29d1': '\u29d2', '\u29d4': '\u29d5',
+        '\u29d8': '\u29d9', '\u29da': '\u29db', '\u29f8': '\u29f9',
+        '\u29fc': '\u29fd', '\u2a2b': '\u2a2c', '\u2a2d': '\u2a2e',
+        '\u2a34': '\u2a35', '\u2a3c': '\u2a3d', '\u2a64': '\u2a65',
+        '\u2a79': '\u2a7a', '\u2a7d': '\u2a7e', '\u2a7f': '\u2a80',
+        '\u2a81': '\u2a82', '\u2a83': '\u2a84', '\u2a8b': '\u2a8c',
+        '\u2a91': '\u2a92', '\u2a93': '\u2a94', '\u2a95': '\u2a96',
+        '\u2a97': '\u2a98', '\u2a99': '\u2a9a', '\u2a9b': '\u2a9c',
+        '\u2aa1': '\u2aa2', '\u2aa6': '\u2aa7', '\u2aa8': '\u2aa9',
+        '\u2aaa': '\u2aab', '\u2aac': '\u2aad', '\u2aaf': '\u2ab0',
+        '\u2ab3': '\u2ab4', '\u2abb': '\u2abc', '\u2abd': '\u2abe',
+        '\u2abf': '\u2ac0', '\u2ac1': '\u2ac2', '\u2ac3': '\u2ac4',
+        '\u2ac5': '\u2ac6', '\u2acd': '\u2ace', '\u2acf': '\u2ad0',
+        '\u2ad1': '\u2ad2', '\u2ad3': '\u2ad4', '\u2ad5': '\u2ad6',
+        '\u2aec': '\u2aed', '\u2af7': '\u2af8', '\u2af9': '\u2afa',
+        '\u2e02': '\u2e03', '\u2e04': '\u2e05', '\u2e09': '\u2e0a',
+        '\u2e0c': '\u2e0d', '\u2e1c': '\u2e1d', '\u2e20': '\u2e21',
+        '\u3008': '\u3009', '\u300a': '\u300b', '\u300c': '\u300d',
+        '\u300e': '\u300f', '\u3010': '\u3011', '\u3014': '\u3015',
+        '\u3016': '\u3017', '\u3018': '\u3019', '\u301a': '\u301b',
+        '\u301d': '\u301e', '\ufd3e': '\ufd3f', '\ufe17': '\ufe18',
+        '\ufe35': '\ufe36', '\ufe37': '\ufe38', '\ufe39': '\ufe3a',
+        '\ufe3b': '\ufe3c', '\ufe3d': '\ufe3e', '\ufe3f': '\ufe40',
+        '\ufe41': '\ufe42', '\ufe43': '\ufe44', '\ufe47': '\ufe48',
+        '\ufe59': '\ufe5a', '\ufe5b': '\ufe5c', '\ufe5d': '\ufe5e',
+        '\uff08': '\uff09', '\uff1c': '\uff1e', '\uff3b': '\uff3d',
+        '\uff5b': '\uff5d', '\uff5f': '\uff60', '\uff62': '\uff63',
+    }
+
+    def _build_word_match(words, boundary_regex_fragment=None, prefix='', suffix=''):
+        if boundary_regex_fragment is None:
+            return r'\b(' + prefix + r'|'.join(re.escape(x) for x in words) + \
+                suffix + r')\b'
+        else:
+            return r'(? 0:
+                    next_open_pos = text.find(opening_chars, search_pos + n_chars)
+                    next_close_pos = text.find(closing_chars, search_pos + n_chars)
+
+                    if next_close_pos == -1:
+                        next_close_pos = len(text)
+                        nesting_level = 0
+                    elif next_open_pos != -1 and next_open_pos < next_close_pos:
+                        nesting_level += 1
+                        search_pos = next_open_pos
+                    else:  # next_close_pos < next_open_pos
+                        nesting_level -= 1
+                        search_pos = next_close_pos
+
+                end_pos = next_close_pos
+
+            if end_pos < 0:     # if we didn't find a closer, just highlight the
+                                # rest of the text in this class
+                end_pos = len(text)
+
+            if adverbs is not None and re.search(r':to\b', adverbs):
+                heredoc_terminator = text[match.start('delimiter') + n_chars:end_pos]
+                end_heredoc = re.search(r'^\s*' + re.escape(heredoc_terminator) +
+                                        r'\s*$', text[end_pos:], re.MULTILINE)
+
+                if end_heredoc:
+                    end_pos += end_heredoc.end()
+                else:
+                    end_pos = len(text)
+
+            yield match.start(), token_class, text[match.start():end_pos + n_chars]
+            context.pos = end_pos + n_chars
+
+        return callback
+
+    def opening_brace_callback(lexer, match, context):
+        stack = context.stack
+
+        yield match.start(), Text, context.text[match.start():match.end()]
+        context.pos = match.end()
+
+        # if we encounter an opening brace and we're one level
+        # below a token state, it means we need to increment
+        # the nesting level for braces so we know later when
+        # we should return to the token rules.
+        if len(stack) > 2 and stack[-2] == 'token':
+            context.perl6_token_nesting_level += 1
+
+    def closing_brace_callback(lexer, match, context):
+        stack = context.stack
+
+        yield match.start(), Text, context.text[match.start():match.end()]
+        context.pos = match.end()
+
+        # if we encounter a free closing brace and we're one level
+        # below a token state, it means we need to check the nesting
+        # level to see if we need to return to the token state.
+        if len(stack) > 2 and stack[-2] == 'token':
+            context.perl6_token_nesting_level -= 1
+            if context.perl6_token_nesting_level == 0:
+                stack.pop()
+
+    def embedded_perl6_callback(lexer, match, context):
+        context.perl6_token_nesting_level = 1
+        yield match.start(), Text, context.text[match.start():match.end()]
+        context.pos = match.end()
+        context.stack.append('root')
+
+    # If you're modifying these rules, be careful if you need to process '{' or '}'
+    # characters. We have special logic for processing these characters (due to the fact
+    # that you can nest Perl 6 code in regex blocks), so if you need to process one of
+    # them, make sure you also process the corresponding one!
+    tokens = {
+        'common': [
+            (r'#[`|=](?P(?P[' + ''.join(PERL6_BRACKETS) + r'])(?P=first_char)*)',
+             brackets_callback(Comment.Multiline)),
+            (r'#[^\n]*$', Comment.Single),
+            (r'^(\s*)=begin\s+(\w+)\b.*?^\1=end\s+\2', Comment.Multiline),
+            (r'^(\s*)=for.*?\n\s*?\n', Comment.Multiline),
+            (r'^=.*?\n\s*?\n', Comment.Multiline),
+            (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)',
+             bygroups(Keyword, Name), 'token-sym-brackets'),
+            (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + r')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?',
+             bygroups(Keyword, Name), 'pre-token'),
+            # deal with a special case in the Perl 6 grammar (role q { ... })
+            (r'(role)(\s+)(q)(\s*)', bygroups(Keyword, Whitespace, Name, Whitespace)),
+            (_build_word_match(PERL6_KEYWORDS, PERL6_IDENTIFIER_RANGE), Keyword),
+            (_build_word_match(PERL6_BUILTIN_CLASSES, PERL6_IDENTIFIER_RANGE, suffix='(?::[UD])?'),
+             Name.Builtin),
+            (_build_word_match(PERL6_BUILTINS, PERL6_IDENTIFIER_RANGE), Name.Builtin),
+            # copied from PerlLexer
+            (r'[$@%&][.^:?=!~]?' + PERL6_IDENTIFIER_RANGE + '+(?:<<.*?>>|<.*?>|«.*?»)*',
+             Name.Variable),
+            (r'\$[!/](?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable.Global),
+            (r'::\?\w+', Name.Variable.Global),
+            (r'[$@%&]\*' + PERL6_IDENTIFIER_RANGE + '+(?:<<.*?>>|<.*?>|«.*?»)*',
+             Name.Variable.Global),
+            (r'\$(?:<.*?>)+', Name.Variable),
+            (r'(?:q|qq|Q)[a-zA-Z]?\s*(?P:[\w\s:]+)?\s*(?P(?P[^0-9a-zA-Z:\s])'
+             r'(?P=first_char)*)', brackets_callback(String)),
+            # copied from PerlLexer
+            (r'0_?[0-7]+(_[0-7]+)*', Number.Oct),
+            (r'0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*', Number.Hex),
+            (r'0b[01]+(_[01]+)*', Number.Bin),
+            (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?',
+             Number.Float),
+            (r'(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*', Number.Float),
+            (r'\d+(_\d+)*', Number.Integer),
+            (r'(?<=~~)\s*/(?:\\\\|\\/|.)*?/', String.Regex),
+            (r'(?<=[=(,])\s*/(?:\\\\|\\/|.)*?/', String.Regex),
+            (r'm\w+(?=\()', Name),
+            (r'(?:m|ms|rx)\s*(?P:[\w\s:]+)?\s*(?P(?P[^\w:\s])'
+             r'(?P=first_char)*)', brackets_callback(String.Regex)),
+            (r'(?:s|ss|tr)\s*(?::[\w\s:]+)?\s*/(?:\\\\|\\/|.)*?/(?:\\\\|\\/|.)*?/',
+             String.Regex),
+            (r'<[^\s=].*?\S>', String),
+            (_build_word_match(PERL6_OPERATORS), Operator),
+            (r'\w' + PERL6_IDENTIFIER_RANGE + '*', Name),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+        ],
+        'root': [
+            include('common'),
+            (r'\{', opening_brace_callback),
+            (r'\}', closing_brace_callback),
+            (r'.+?', Text),
+        ],
+        'pre-token': [
+            include('common'),
+            (r'\{', Text, ('#pop', 'token')),
+            (r'.+?', Text),
+        ],
+        'token-sym-brackets': [
+            (r'(?P(?P[' + ''.join(PERL6_BRACKETS) + '])(?P=first_char)*)',
+             brackets_callback(Name), ('#pop', 'pre-token')),
+            default(('#pop', 'pre-token')),
+        ],
+        'token': [
+            (r'\}', Text, '#pop'),
+            (r'(?<=:)(?:my|our|state|constant|temp|let).*?;', using(this)),
+            # make sure that quotes in character classes aren't treated as strings
+            (r'<(?:[-!?+.]\s*)?\[.*?\]>', String.Regex),
+            # make sure that '#' characters in quotes aren't treated as comments
+            (r"(?my|our)\s+)?(?:module|class|role|enum|grammar)', line)
+            if class_decl:
+                if saw_perl_decl or class_decl.group('scope') is not None:
+                    return True
+                rating = 0.05
+                continue
+            break
+
+        if ':=' in text:
+            # Same logic as above for PerlLexer
+            rating /= 2
+
+        return rating
+
+    def __init__(self, **options):
+        super().__init__(**options)
+        self.encoding = options.get('encoding', 'utf-8')
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/phix.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/phix.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4cb35d5b13c96d4619a2801715117b788b47226
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/phix.py
@@ -0,0 +1,363 @@
+"""
+    pygments.lexers.phix
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Phix.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Whitespace
+
+__all__ = ['PhixLexer']
+
+
+class PhixLexer(RegexLexer):
+    """
+    Pygments Lexer for Phix files (.exw).
+    See http://phix.x10.mx
+    """
+
+    name = 'Phix'
+    url = 'http://phix.x10.mx'
+    aliases = ['phix']
+    filenames = ['*.exw']
+    mimetypes = ['text/x-phix']
+    version_added = '2.14'
+
+    flags = re.MULTILINE    # nb: **NOT** re.DOTALL! (totally spanners comment handling)
+
+    preproc = (
+        'ifdef', 'elsifdef', 'elsedef'
+    )
+    # Note these lists are auto-generated by pwa/p2js.exw, when pwa\src\p2js_keywords.e (etc)
+    #     change, though of course subsequent copy/commit/pull requests are all manual steps.
+    types = (
+        'string', 'nullable_string', 'atom_string', 'atom', 'bool', 'boolean',
+        'cdCanvan', 'cdCanvas', 'complex', 'CURLcode', 'dictionary', 'int',
+        'integer', 'Ihandle', 'Ihandles', 'Ihandln', 'mpfr', 'mpq', 'mpz',
+        'mpz_or_string', 'number', 'rid_string', 'seq', 'sequence', 'timedate',
+        'object'
+    )
+    keywords = (
+        'abstract', 'class', 'continue', 'export', 'extends', 'nullable',
+        'private', 'public', 'static', 'struct', 'trace',
+        'and', 'break', 'by', 'case', 'catch', 'const', 'constant', 'debug',
+        'default', 'do', 'else', 'elsif', 'end', 'enum', 'exit', 'fallthru',
+        'fallthrough', 'for', 'forward', 'function', 'global', 'if', 'in',
+        'include', 'js', 'javascript', 'javascript_semantics', 'let', 'not',
+        'or', 'procedure', 'profile', 'profile_time', 'return', 'safe_mode',
+        'switch', 'then', 'to', 'try', 'type', 'type_check', 'until', 'warning',
+        'while', 'with', 'without', 'xor'
+    )
+    routines = (
+        'abort', 'abs', 'adjust_timedate', 'and_bits', 'and_bitsu', 'apply',
+        'append', 'arccos', 'arcsin', 'arctan', 'assert', 'atan2',
+        'atom_to_float32', 'atom_to_float64', 'bankers_rounding', 'beep',
+        'begins', 'binary_search', 'bits_to_int', 'bk_color', 'bytes_to_int',
+        'call_func', 'call_proc', 'cdCanvasActivate', 'cdCanvasArc',
+        'cdCanvasBegin', 'cdCanvasBox', 'cdCanvasChord', 'cdCanvasCircle',
+        'cdCanvasClear', 'cdCanvasEnd', 'cdCanvasFlush', 'cdCanvasFont',
+        'cdCanvasGetImageRGB', 'cdCanvasGetSize', 'cdCanvasGetTextAlignment',
+        'cdCanvasGetTextSize', 'cdCanvasLine', 'cdCanvasMark',
+        'cdCanvasMarkSize', 'cdCanvasMultiLineVectorText', 'cdCanvasPixel',
+        'cdCanvasRect', 'cdCanvasRoundedBox', 'cdCanvasRoundedRect',
+        'cdCanvasSector', 'cdCanvasSetAttribute', 'cdCanvasSetBackground',
+        'cdCanvasSetFillMode', 'cdCanvasSetForeground',
+        'cdCanvasSetInteriorStyle', 'cdCanvasSetLineStyle',
+        'cdCanvasSetLineWidth', 'cdCanvasSetTextAlignment', 'cdCanvasText',
+        'cdCanvasSetTextOrientation', 'cdCanvasGetTextOrientation',
+        'cdCanvasVectorText', 'cdCanvasVectorTextDirection',
+        'cdCanvasVectorTextSize', 'cdCanvasVertex', 'cdCreateCanvas',
+        'cdDecodeAlpha', 'cdDecodeColor', 'cdDecodeColorAlpha', 'cdEncodeAlpha',
+        'cdEncodeColor', 'cdEncodeColorAlpha', 'cdKillCanvas', 'cdVersion',
+        'cdVersionDate', 'ceil', 'change_timezone', 'choose', 'clear_screen',
+        'columnize', 'command_line', 'compare', 'complex_abs', 'complex_add',
+        'complex_arg', 'complex_conjugate', 'complex_cos', 'complex_cosh',
+        'complex_div', 'complex_exp', 'complex_imag', 'complex_inv',
+        'complex_log', 'complex_mul', 'complex_neg', 'complex_new',
+        'complex_norm', 'complex_power', 'complex_rho', 'complex_real',
+        'complex_round', 'complex_sin', 'complex_sinh', 'complex_sprint',
+        'complex_sqrt', 'complex_sub', 'complex_theta', 'concat', 'cos',
+        'crash', 'custom_sort', 'date', 'day_of_week', 'day_of_year',
+        'days_in_month', 'decode_base64', 'decode_flags', 'deep_copy', 'deld',
+        'deserialize', 'destroy_dict', 'destroy_queue', 'destroy_stack',
+        'dict_name', 'dict_size', 'elapsed', 'elapsed_short', 'encode_base64',
+        'equal', 'even', 'exp', 'extract', 'factorial', 'factors',
+        'file_size_k', 'find', 'find_all', 'find_any', 'find_replace', 'filter',
+        'flatten', 'float32_to_atom', 'float64_to_atom', 'floor',
+        'format_timedate', 'free_console', 'from_polar', 'gcd', 'get_file_base',
+        'get_file_extension', 'get_file_name', 'get_file_name_and_path',
+        'get_file_path', 'get_file_path_and_name', 'get_maxprime', 'get_prime',
+        'get_primes', 'get_primes_le', 'get_proper_dir', 'get_proper_path',
+        'get_rand', 'get_routine_info', 'get_test_abort', 'get_test_logfile',
+        'get_test_pause', 'get_test_verbosity', 'get_tzid', 'getd', 'getdd',
+        'getd_all_keys', 'getd_by_index', 'getd_index', 'getd_partial_key',
+        'glAttachShader', 'glBindBuffer', 'glBindTexture', 'glBufferData',
+        'glCanvasSpecialText', 'glClear', 'glClearColor', 'glColor',
+        'glCompileShader', 'glCreateBuffer', 'glCreateProgram',
+        'glCreateShader', 'glCreateTexture', 'glDeleteProgram',
+        'glDeleteShader', 'glDrawArrays', 'glEnable',
+        'glEnableVertexAttribArray', 'glFloat32Array', 'glInt32Array',
+        'glFlush', 'glGetAttribLocation', 'glGetError', 'glGetProgramInfoLog',
+        'glGetProgramParameter', 'glGetShaderInfoLog', 'glGetShaderParameter',
+        'glGetUniformLocation', 'glLinkProgram', 'glLoadIdentity',
+        'glMatrixMode', 'glOrtho', 'glRotatef', 'glShadeModel',
+        'glShaderSource', 'glSimpleA7texcoords', 'glTexImage2Dc',
+        'glTexParameteri', 'glTranslate', 'glUniform1f', 'glUniform1i',
+        'glUniformMatrix4fv', 'glUseProgram', 'glVertex',
+        'glVertexAttribPointer', 'glViewport', 'head', 'hsv_to_rgb', 'iff',
+        'iif', 'include_file', 'incl0de_file', 'insert', 'instance',
+        'int_to_bits', 'int_to_bytes', 'is_dict', 'is_integer', 's_leap_year',
+        'is_prime', 'is_prime2', 'islower', 'isupper', 'Icallback',
+        'iup_isdouble', 'iup_isprint', 'iup_XkeyBase', 'IupAppend', 'IupAlarm',
+        'IupBackgroundBox', 'IupButton', 'IupCalendar', 'IupCanvas',
+        'IupClipboard', 'IupClose', 'IupCloseOnEscape', 'IupControlsOpen',
+        'IupDatePick', 'IupDestroy', 'IupDialog', 'IupDrawArc', 'IupDrawBegin',
+        'IupDrawEnd', 'IupDrawGetSize', 'IupDrawGetTextSize', 'IupDrawLine',
+        'IupDrawRectangle', 'IupDrawText', 'IupExpander', 'IupFill',
+        'IupFlatLabel', 'IupFlatList', 'IupFlatTree', 'IupFlush', 'IupFrame',
+        'IupGetAttribute', 'IupGetAttributeId', 'IupGetAttributePtr',
+        'IupGetBrother', 'IupGetChild', 'IupGetChildCount', 'IupGetClassName',
+        'IupGetDialog', 'IupGetDialogChild', 'IupGetDouble', 'IupGetFocus',
+        'IupGetGlobal', 'IupGetGlobalInt', 'IupGetGlobalIntInt', 'IupGetInt',
+        'IupGetInt2', 'IupGetIntId', 'IupGetIntInt', 'IupGetParent',
+        'IupGLCanvas', 'IupGLCanvasOpen', 'IupGLMakeCurrent', 'IupGraph',
+        'IupHbox', 'IupHide', 'IupImage', 'IupImageRGBA', 'IupItem',
+        'iupKeyCodeToName', 'IupLabel', 'IupLink', 'IupList', 'IupMap',
+        'IupMenu', 'IupMenuItem', 'IupMessage', 'IupMessageDlg', 'IupMultiBox',
+        'IupMultiLine', 'IupNextField', 'IupNormaliser', 'IupOpen',
+        'IupPlayInput', 'IupPopup', 'IupPreviousField', 'IupProgressBar',
+        'IupRadio', 'IupRecordInput', 'IupRedraw', 'IupRefresh',
+        'IupRefreshChildren', 'IupSeparator', 'IupSetAttribute',
+        'IupSetAttributes', 'IupSetAttributeHandle', 'IupSetAttributeId',
+        'IupSetAttributePtr', 'IupSetCallback', 'IupSetCallbacks',
+        'IupSetDouble', 'IupSetFocus', 'IupSetGlobal', 'IupSetGlobalInt',
+        'IupSetGlobalFunction', 'IupSetHandle', 'IupSetInt',
+        'IupSetStrAttribute', 'IupSetStrGlobal', 'IupShow', 'IupShowXY',
+        'IupSplit', 'IupStoreAttribute', 'IupSubmenu', 'IupTable',
+        'IupTableClearSelected', 'IupTableClick_cb', 'IupTableGetSelected',
+        'IupTableResize_cb', 'IupTableSetData', 'IupTabs', 'IupText',
+        'IupTimer', 'IupToggle', 'IupTreeAddNodes', 'IupTreeView', 'IupUpdate',
+        'IupValuator', 'IupVbox', 'join', 'join_by', 'join_path', 'k_perm',
+        'largest', 'lcm', 'length', 'log', 'log10', 'log2', 'lower',
+        'm4_crossProduct', 'm4_inverse', 'm4_lookAt', 'm4_multiply',
+        'm4_normalize', 'm4_perspective', 'm4_subtractVectors', 'm4_xRotate',
+        'm4_yRotate', 'machine_bits', 'machine_word', 'match', 'match_all',
+        'match_replace', 'max', 'maxsq', 'min', 'minsq', 'mod', 'mpfr_add',
+        'mpfr_ceil', 'mpfr_cmp', 'mpfr_cmp_si', 'mpfr_const_pi', 'mpfr_div',
+        'mpfr_div_si', 'mpfr_div_z', 'mpfr_floor', 'mpfr_free', 'mpfr_get_d',
+        'mpfr_get_default_precision', 'mpfr_get_default_rounding_mode',
+        'mpfr_get_fixed', 'mpfr_get_precision', 'mpfr_get_si', 'mpfr_init',
+        'mpfr_inits', 'mpfr_init_set', 'mpfr_init_set_q', 'mpfr_init_set_z',
+        'mpfr_mul', 'mpfr_mul_si', 'mpfr_pow_si', 'mpfr_set', 'mpfr_set_d',
+        'mpfr_set_default_precision', 'mpfr_set_default_rounding_mode',
+        'mpfr_set_precision', 'mpfr_set_q', 'mpfr_set_si', 'mpfr_set_str',
+        'mpfr_set_z', 'mpfr_si_div', 'mpfr_si_sub', 'mpfr_sqrt', 'mpfr_sub',
+        'mpfr_sub_si', 'mpq_abs', 'mpq_add', 'mpq_add_si', 'mpq_canonicalize',
+        'mpq_cmp', 'mpq_cmp_si', 'mpq_div', 'mpq_div_2exp', 'mpq_free',
+        'mpq_get_den', 'mpq_get_num', 'mpq_get_str', 'mpq_init', 'mpq_init_set',
+        'mpq_init_set_si', 'mpq_init_set_str', 'mpq_init_set_z', 'mpq_inits',
+        'mpq_inv', 'mpq_mul', 'mpq_neg', 'mpq_set', 'mpq_set_si', 'mpq_set_str',
+        'mpq_set_z', 'mpq_sub', 'mpz_abs', 'mpz_add', 'mpz_addmul',
+        'mpz_addmul_ui', 'mpz_addmul_si', 'mpz_add_si', 'mpz_add_ui', 'mpz_and',
+        'mpz_bin_uiui', 'mpz_cdiv_q', 'mpz_cmp', 'mpz_cmp_si', 'mpz_divexact',
+        'mpz_divexact_ui', 'mpz_divisible_p', 'mpz_divisible_ui_p', 'mpz_even',
+        'mpz_fac_ui', 'mpz_factorstring', 'mpz_fdiv_q', 'mpz_fdiv_q_2exp',
+        'mpz_fdiv_q_ui', 'mpz_fdiv_qr', 'mpz_fdiv_r', 'mpz_fdiv_ui',
+        'mpz_fib_ui', 'mpz_fib2_ui', 'mpz_fits_atom', 'mpz_fits_integer',
+        'mpz_free', 'mpz_gcd', 'mpz_gcd_ui', 'mpz_get_atom', 'mpz_get_integer',
+        'mpz_get_short_str', 'mpz_get_str', 'mpz_init', 'mpz_init_set',
+        'mpz_inits', 'mpz_invert', 'mpz_lcm', 'mpz_lcm_ui', 'mpz_max',
+        'mpz_min', 'mpz_mod', 'mpz_mod_ui', 'mpz_mul', 'mpz_mul_2exp',
+        'mpz_mul_d', 'mpz_mul_si', 'mpz_neg', 'mpz_nthroot', 'mpz_odd',
+        'mpz_pollard_rho', 'mpz_pow_ui', 'mpz_powm', 'mpz_powm_ui', 'mpz_prime',
+        'mpz_prime_factors', 'mpz_prime_mr', 'mpz_rand', 'mpz_rand_ui',
+        'mpz_re_compose', 'mpz_remove', 'mpz_scan0', 'mpz_scan1', 'mpz_set',
+        'mpz_set_d', 'mpz_set_si', 'mpz_set_str', 'mpz_set_v', 'mpz_sign',
+        'mpz_sizeinbase', 'mpz_sqrt', 'mpz_sub', 'mpz_sub_si', 'mpz_sub_ui',
+        'mpz_si_sub', 'mpz_tdiv_q_2exp', 'mpz_tdiv_r_2exp', 'mpz_tstbit',
+        'mpz_ui_pow_ui', 'mpz_xor', 'named_dict', 'new_dict', 'new_queue',
+        'new_stack', 'not_bits', 'not_bitsu', 'odd', 'or_all', 'or_allu',
+        'or_bits', 'or_bitsu', 'ord', 'ordinal', 'ordinant',
+        'override_timezone', 'pad', 'pad_head', 'pad_tail', 'parse_date_string',
+        'papply', 'peep', 'peepn', 'peep_dict', 'permute', 'permutes',
+        'platform', 'pop', 'popn', 'pop_dict', 'power', 'pp', 'ppEx', 'ppExf',
+        'ppf', 'ppOpt', 'pq_add', 'pq_destroy', 'pq_empty', 'pq_new', 'pq_peek',
+        'pq_pop', 'pq_pop_data', 'pq_size', 'prepend', 'prime_factors',
+        'printf', 'product', 'proper', 'push', 'pushn', 'putd', 'puts',
+        'queue_empty', 'queue_size', 'rand', 'rand_range', 'reinstate',
+        'remainder', 'remove', 'remove_all', 'repeat', 'repeatch', 'replace',
+        'requires', 'reverse', 'rfind', 'rgb', 'rmatch', 'rmdr', 'rnd', 'round',
+        'routine_id', 'scanf', 'serialize', 'series', 'set_rand',
+        'set_test_abort', 'set_test_logfile', 'set_test_module',
+        'set_test_pause', 'set_test_verbosity', 'set_timedate_formats',
+        'set_timezone', 'setd', 'setd_default', 'shorten', 'sha256',
+        'shift_bits', 'shuffle', 'sign', 'sin', 'smallest', 'sort',
+        'sort_columns', 'speak', 'splice', 'split', 'split_any', 'split_by',
+        'sprint', 'sprintf', 'sq_abs', 'sq_add', 'sq_and', 'sq_and_bits',
+        'sq_arccos', 'sq_arcsin', 'sq_arctan', 'sq_atom', 'sq_ceil', 'sq_cmp',
+        'sq_cos', 'sq_div', 'sq_even', 'sq_eq', 'sq_floor', 'sq_floor_div',
+        'sq_ge', 'sq_gt', 'sq_int', 'sq_le', 'sq_log', 'sq_log10', 'sq_log2',
+        'sq_lt', 'sq_max', 'sq_min', 'sq_mod', 'sq_mul', 'sq_ne', 'sq_not',
+        'sq_not_bits', 'sq_odd', 'sq_or', 'sq_or_bits', 'sq_power', 'sq_rand',
+        'sq_remainder', 'sq_rmdr', 'sq_rnd', 'sq_round', 'sq_seq', 'sq_sign',
+        'sq_sin', 'sq_sqrt', 'sq_str', 'sq_sub', 'sq_tan', 'sq_trunc',
+        'sq_uminus', 'sq_xor', 'sq_xor_bits', 'sqrt', 'square_free',
+        'stack_empty', 'stack_size', 'substitute', 'substitute_all', 'sum',
+        'tail', 'tan', 'test_equal', 'test_fail', 'test_false',
+        'test_not_equal', 'test_pass', 'test_summary', 'test_true',
+        'text_color', 'throw', 'time', 'timedate_diff', 'timedelta',
+        'to_integer', 'to_number', 'to_rgb', 'to_string', 'traverse_dict',
+        'traverse_dict_partial_key', 'trim', 'trim_head', 'trim_tail', 'trunc',
+        'tagset', 'tagstart', 'typeof', 'unique', 'unix_dict', 'upper',
+        'utf8_to_utf32', 'utf32_to_utf8', 'version', 'vlookup', 'vslice',
+        'wglGetProcAddress', 'wildcard_file', 'wildcard_match', 'with_rho',
+        'with_theta', 'xml_new_doc', 'xml_new_element', 'xml_set_attribute',
+        'xml_sprint', 'xor_bits', 'xor_bitsu',
+        'accept', 'allocate', 'allocate_string', 'allow_break', 'ARM',
+        'atom_to_float80', 'c_func', 'c_proc', 'call_back', 'chdir',
+        'check_break', 'clearDib', 'close', 'closesocket', 'console',
+        'copy_file', 'create', 'create_directory', 'create_thread',
+        'curl_easy_cleanup', 'curl_easy_get_file', 'curl_easy_init',
+        'curl_easy_perform', 'curl_easy_perform_ex', 'curl_easy_setopt',
+        'curl_easy_strerror', 'curl_global_cleanup', 'curl_global_init',
+        'curl_slist_append', 'curl_slist_free_all', 'current_dir', 'cursor',
+        'define_c_func', 'define_c_proc', 'delete', 'delete_cs', 'delete_file',
+        'dir', 'DLL', 'drawDib', 'drawShadedPolygonToDib', 'ELF32', 'ELF64',
+        'enter_cs', 'eval', 'exit_thread', 'free', 'file_exists', 'final',
+        'float80_to_atom', 'format', 'get_bytes', 'get_file_date',
+        'get_file_size', 'get_file_type', 'get_interpreter', 'get_key',
+        'get_socket_error', 'get_text', 'get_thread_exitcode', 'get_thread_id',
+        'getc', 'getenv', 'gets', 'getsockaddr', 'glBegin', 'glCallList',
+        'glFrustum', 'glGenLists', 'glGetString', 'glLight', 'glMaterial',
+        'glNewList', 'glNormal', 'glPopMatrix', 'glPushMatrix', 'glRotate',
+        'glEnd', 'glEndList', 'glTexImage2D', 'goto', 'GUI', 'icons', 'ilASM',
+        'include_files', 'include_paths', 'init_cs', 'ip_to_string',
+        'IupConfig', 'IupConfigDialogClosed', 'IupConfigDialogShow',
+        'IupConfigGetVariableInt', 'IupConfigLoad', 'IupConfigSave',
+        'IupConfigSetVariableInt', 'IupExitLoop', 'IupFileDlg', 'IupFileList',
+        'IupGLSwapBuffers', 'IupHelp', 'IupLoopStep', 'IupMainLoop',
+        'IupNormalizer', 'IupPlot', 'IupPlotAdd', 'IupPlotBegin', 'IupPlotEnd',
+        'IupPlotInsert', 'IupSaveImage', 'IupTreeGetUserId', 'IupUser',
+        'IupVersion', 'IupVersionDate', 'IupVersionNumber', 'IupVersionShow',
+        'killDib', 'leave_cs', 'listen', 'manifest', 'mem_copy', 'mem_set',
+        'mpfr_gamma', 'mpfr_printf', 'mpfr_sprintf', 'mpz_export', 'mpz_import',
+        'namespace', 'new', 'newDib', 'open', 'open_dll', 'PE32', 'PE64',
+        'peek', 'peek_string', 'peek1s', 'peek1u', 'peek2s', 'peek2u', 'peek4s',
+        'peek4u', 'peek8s', 'peek8u', 'peekNS', 'peekns', 'peeknu', 'poke',
+        'poke2', 'poke4', 'poke8', 'pokeN', 'poke_string', 'poke_wstring',
+        'position', 'progress', 'prompt_number', 'prompt_string', 'read_file',
+        'read_lines', 'recv', 'resume_thread', 'seek', 'select', 'send',
+        'setHandler', 'shutdown', 'sleep', 'SO', 'sockaddr_in', 'socket',
+        'split_path', 'suspend_thread', 'system', 'system_exec', 'system_open',
+        'system_wait', 'task_clock_start', 'task_clock_stop', 'task_create',
+        'task_delay', 'task_list', 'task_schedule', 'task_self', 'task_status',
+        'task_suspend', 'task_yield', 'thread_safe_string', 'try_cs',
+        'utf8_to_utf16', 'utf16_to_utf8', 'utf16_to_utf32', 'utf32_to_utf16',
+        'video_config', 'WSACleanup', 'wait_thread', 'walk_dir', 'where',
+        'write_lines', 'wait_key'
+    )
+    constants = (
+        'ANY_QUEUE', 'ASCENDING', 'BLACK', 'BLOCK_CURSOR', 'BLUE',
+        'BRIGHT_CYAN', 'BRIGHT_BLUE', 'BRIGHT_GREEN', 'BRIGHT_MAGENTA',
+        'BRIGHT_RED', 'BRIGHT_WHITE', 'BROWN', 'C_DWORD', 'C_INT', 'C_POINTER',
+        'C_USHORT', 'C_WORD', 'CD_AMBER', 'CD_BLACK', 'CD_BLUE', 'CD_BOLD',
+        'CD_BOLD_ITALIC', 'CD_BOX', 'CD_CENTER', 'CD_CIRCLE', 'CD_CLOSED_LINES',
+        'CD_CONTINUOUS', 'CD_CUSTOM', 'CD_CYAN', 'CD_DARK_BLUE', 'CD_DARK_CYAN',
+        'CD_DARK_GRAY', 'CD_DARK_GREY', 'CD_DARK_GREEN', 'CD_DARK_MAGENTA',
+        'CD_DARK_RED', 'CD_DARK_YELLOW', 'CD_DASH_DOT', 'CD_DASH_DOT_DOT',
+        'CD_DASHED', 'CD_DBUFFER', 'CD_DEG2RAD', 'CD_DIAMOND', 'CD_DOTTED',
+        'CD_EAST', 'CD_EVENODD', 'CD_FILL', 'CD_GL', 'CD_GRAY', 'CD_GREY',
+        'CD_GREEN', 'CD_HATCH', 'CD_HOLLOW', 'CD_HOLLOW_BOX',
+        'CD_HOLLOW_CIRCLE', 'CD_HOLLOW_DIAMOND', 'CD_INDIGO', 'CD_ITALIC',
+        'CD_IUP', 'CD_IUPDBUFFER', 'CD_LIGHT_BLUE', 'CD_LIGHT_GRAY',
+        'CD_LIGHT_GREY', 'CD_LIGHT_GREEN', 'CD_LIGHT_PARCHMENT', 'CD_MAGENTA',
+        'CD_NAVY', 'CD_NORTH', 'CD_NORTH_EAST', 'CD_NORTH_WEST', 'CD_OLIVE',
+        'CD_OPEN_LINES', 'CD_ORANGE', 'CD_PARCHMENT', 'CD_PATTERN',
+        'CD_PRINTER', 'CD_PURPLE', 'CD_PLAIN', 'CD_PLUS', 'CD_QUERY',
+        'CD_RAD2DEG', 'CD_RED', 'CD_SILVER', 'CD_SOLID', 'CD_SOUTH_EAST',
+        'CD_SOUTH_WEST', 'CD_STAR', 'CD_STIPPLE', 'CD_STRIKEOUT',
+        'CD_UNDERLINE', 'CD_WEST', 'CD_WHITE', 'CD_WINDING', 'CD_VIOLET',
+        'CD_X', 'CD_YELLOW', 'CURLE_OK', 'CURLOPT_MAIL_FROM',
+        'CURLOPT_MAIL_RCPT', 'CURLOPT_PASSWORD', 'CURLOPT_READDATA',
+        'CURLOPT_READFUNCTION', 'CURLOPT_SSL_VERIFYPEER',
+        'CURLOPT_SSL_VERIFYHOST', 'CURLOPT_UPLOAD', 'CURLOPT_URL',
+        'CURLOPT_USE_SSL', 'CURLOPT_USERNAME', 'CURLOPT_VERBOSE',
+        'CURLOPT_WRITEFUNCTION', 'CURLUSESSL_ALL', 'CYAN', 'D_NAME',
+        'D_ATTRIBUTES', 'D_SIZE', 'D_YEAR', 'D_MONTH', 'D_DAY', 'D_HOUR',
+        'D_MINUTE', 'D_SECOND', 'D_CREATION', 'D_LASTACCESS', 'D_MODIFICATION',
+        'DT_YEAR', 'DT_MONTH', 'DT_DAY', 'DT_HOUR', 'DT_MINUTE', 'DT_SECOND',
+        'DT_DOW', 'DT_MSEC', 'DT_DOY', 'DT_GMT', 'EULER', 'E_CODE', 'E_ADDR',
+        'E_LINE', 'E_RTN', 'E_NAME', 'E_FILE', 'E_PATH', 'E_USER', 'false',
+        'False', 'FALSE', 'FIFO_QUEUE', 'FILETYPE_DIRECTORY', 'FILETYPE_FILE',
+        'GET_EOF', 'GET_FAIL', 'GET_IGNORE', 'GET_SUCCESS',
+        'GL_AMBIENT_AND_DIFFUSE', 'GL_ARRAY_BUFFER', 'GL_CLAMP',
+        'GL_CLAMP_TO_BORDER', 'GL_CLAMP_TO_EDGE', 'GL_COLOR_BUFFER_BIT',
+        'GL_COMPILE', 'GL_COMPILE_STATUS', 'GL_CULL_FACE',
+        'GL_DEPTH_BUFFER_BIT', 'GL_DEPTH_TEST', 'GL_EXTENSIONS', 'GL_FLAT',
+        'GL_FLOAT', 'GL_FRAGMENT_SHADER', 'GL_FRONT', 'GL_LIGHT0',
+        'GL_LIGHTING', 'GL_LINEAR', 'GL_LINK_STATUS', 'GL_MODELVIEW',
+        'GL_NEAREST', 'GL_NO_ERROR', 'GL_NORMALIZE', 'GL_POSITION',
+        'GL_PROJECTION', 'GL_QUAD_STRIP', 'GL_QUADS', 'GL_RENDERER',
+        'GL_REPEAT', 'GL_RGB', 'GL_RGBA', 'GL_SMOOTH', 'GL_STATIC_DRAW',
+        'GL_TEXTURE_2D', 'GL_TEXTURE_MAG_FILTER', 'GL_TEXTURE_MIN_FILTER',
+        'GL_TEXTURE_WRAP_S', 'GL_TEXTURE_WRAP_T', 'GL_TRIANGLES',
+        'GL_UNSIGNED_BYTE', 'GL_VENDOR', 'GL_VERSION', 'GL_VERTEX_SHADER',
+        'GRAY', 'GREEN', 'GT_LF_STRIPPED', 'GT_WHOLE_FILE', 'INVLN10',
+        'IUP_CLOSE', 'IUP_CONTINUE', 'IUP_DEFAULT', 'IUP_BLACK', 'IUP_BLUE',
+        'IUP_BUTTON1', 'IUP_BUTTON3', 'IUP_CENTER', 'IUP_CYAN', 'IUP_DARK_BLUE',
+        'IUP_DARK_CYAN', 'IUP_DARK_GRAY', 'IUP_DARK_GREY', 'IUP_DARK_GREEN',
+        'IUP_DARK_MAGENTA', 'IUP_DARK_RED', 'IUP_GRAY', 'IUP_GREY', 'IUP_GREEN',
+        'IUP_IGNORE', 'IUP_INDIGO', 'IUP_MAGENTA', 'IUP_MASK_INT',
+        'IUP_MASK_UINT', 'IUP_MOUSEPOS', 'IUP_NAVY', 'IUP_OLIVE', 'IUP_RECTEXT',
+        'IUP_RED', 'IUP_LIGHT_BLUE', 'IUP_LIGHT_GRAY', 'IUP_LIGHT_GREY',
+        'IUP_LIGHT_GREEN', 'IUP_ORANGE', 'IUP_PARCHMENT', 'IUP_PURPLE',
+        'IUP_SILVER', 'IUP_TEAL', 'IUP_VIOLET', 'IUP_WHITE', 'IUP_YELLOW',
+        'K_BS', 'K_cA', 'K_cC', 'K_cD', 'K_cF5', 'K_cK', 'K_cM', 'K_cN', 'K_cO',
+        'K_cP', 'K_cR', 'K_cS', 'K_cT', 'K_cW', 'K_CR', 'K_DEL', 'K_DOWN',
+        'K_END', 'K_ESC', 'K_F1', 'K_F2', 'K_F3', 'K_F4', 'K_F5', 'K_F6',
+        'K_F7', 'K_F8', 'K_F9', 'K_F10', 'K_F11', 'K_F12', 'K_HOME', 'K_INS',
+        'K_LEFT', 'K_MIDDLE', 'K_PGDN', 'K_PGUP', 'K_RIGHT', 'K_SP', 'K_TAB',
+        'K_UP', 'K_h', 'K_i', 'K_j', 'K_p', 'K_r', 'K_s', 'JS', 'LIFO_QUEUE',
+        'LINUX', 'MAX_HEAP', 'MAGENTA', 'MIN_HEAP', 'Nan', 'NO_CURSOR', 'null',
+        'NULL', 'PI', 'pp_Ascii', 'pp_Brkt', 'pp_Date', 'pp_File', 'pp_FltFmt',
+        'pp_Indent', 'pp_IntCh', 'pp_IntFmt', 'pp_Maxlen', 'pp_Nest',
+        'pp_Pause', 'pp_Q22', 'pp_StrFmt', 'RED', 'SEEK_OK', 'SLASH',
+        'TEST_ABORT', 'TEST_CRASH', 'TEST_PAUSE', 'TEST_PAUSE_FAIL',
+        'TEST_QUIET', 'TEST_SHOW_ALL', 'TEST_SHOW_FAILED', 'TEST_SUMMARY',
+        'true', 'True', 'TRUE', 'VC_SCRNLINES', 'WHITE', 'WINDOWS', 'YELLOW'
+    )
+
+    tokens = {
+        'root': [
+            (r"\s+", Whitespace),
+            (r'/\*|--/\*|#\[', Comment.Multiline, 'comment'),
+            (r'(?://|--|#!).*$', Comment.Single),
+#Alt:
+#           (r'//.*$|--.*$|#!.*$', Comment.Single),
+            (r'"([^"\\]|\\.)*"', String.Other),
+            (r'\'[^\']*\'', String.Other),
+            (r'`[^`]*`', String.Other),
+
+            (words(types, prefix=r'\b', suffix=r'\b'), Name.Function),
+            (words(routines, prefix=r'\b', suffix=r'\b'), Name.Function),
+            (words(preproc, prefix=r'\b', suffix=r'\b'), Keyword.Declaration),
+            (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword.Declaration),
+            (words(constants, prefix=r'\b', suffix=r'\b'), Name.Constant),
+            # Aside: Phix only supports/uses the ascii/non-unicode tilde
+            (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|\.(){},?:\[\]$\\;#]', Operator),
+            (r'[\w-]+', Text)
+        ],
+        'comment': [
+            (r'[^*/#]+', Comment.Multiline),
+            (r'/\*|#\[', Comment.Multiline, '#push'),
+            (r'\*/|#\]', Comment.Multiline, '#pop'),
+            (r'[*/#]', Comment.Multiline)
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/php.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/php.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac935c4717498a7c2c3acb40156f80c2fc69d901
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/php.py
@@ -0,0 +1,335 @@
+"""
+    pygments.lexers.php
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for PHP and related languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, include, bygroups, default, \
+    using, this, words, do_insertions, line_re
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Other, Generic
+from pygments.util import get_bool_opt, get_list_opt, shebang_matches
+
+__all__ = ['ZephirLexer', 'PsyshConsoleLexer', 'PhpLexer']
+
+
+class ZephirLexer(RegexLexer):
+    """
+    For Zephir language source code.
+
+    Zephir is a compiled high level language aimed
+    to the creation of C-extensions for PHP.
+    """
+
+    name = 'Zephir'
+    url = 'http://zephir-lang.com/'
+    aliases = ['zephir']
+    filenames = ['*.zep']
+    version_added = '2.0'
+
+    zephir_keywords = ['fetch', 'echo', 'isset', 'empty']
+    zephir_type = ['bit', 'bits', 'string']
+
+    flags = re.DOTALL | re.MULTILINE
+
+    tokens = {
+        'commentsandwhitespace': [
+            (r'\s+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline)
+        ],
+        'slashstartsregex': [
+            include('commentsandwhitespace'),
+            (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+             r'([gim]+\b|\B)', String.Regex, '#pop'),
+            (r'/', Operator, '#pop'),
+            default('#pop')
+        ],
+        'badregex': [
+            (r'\n', Text, '#pop')
+        ],
+        'root': [
+            (r'^(?=\s|/)', Text, 'slashstartsregex'),
+            include('commentsandwhitespace'),
+            (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
+             r'(<<|>>>?|==?|!=?|->|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
+            (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+            (r'[})\].]', Punctuation),
+            (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|loop|'
+             r'require|inline|throw|try|catch|finally|new|delete|typeof|instanceof|void|'
+             r'namespace|use|extends|this|fetch|isset|unset|echo|fetch|likely|unlikely|'
+             r'empty)\b', Keyword, 'slashstartsregex'),
+            (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'),
+            (r'(abstract|boolean|bool|char|class|const|double|enum|export|extends|final|'
+             r'native|goto|implements|import|int|string|interface|long|ulong|char|uchar|'
+             r'float|unsigned|private|protected|public|short|static|self|throws|reverse|'
+             r'transient|volatile|readonly)\b', Keyword.Reserved),
+            (r'(true|false|null|undefined)\b', Keyword.Constant),
+            (r'(Array|Boolean|Date|_REQUEST|_COOKIE|_SESSION|'
+             r'_GET|_POST|_SERVER|this|stdClass|range|count|iterator|'
+             r'window)\b', Name.Builtin),
+            (r'[$a-zA-Z_][\w\\]*', Name.Other),
+            (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'[0-9]+', Number.Integer),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+        ]
+    }
+
+
+class PsyshConsoleLexer(Lexer):
+    """
+    For PsySH console output, such as:
+
+    .. sourcecode:: psysh
+
+        >>> $greeting = function($name): string {
+        ...     return "Hello, {$name}";
+        ... };
+        => Closure($name): string {#2371 …3}
+        >>> $greeting('World')
+        => "Hello, World"
+    """
+    name = 'PsySH console session for PHP'
+    url = 'https://psysh.org/'
+    aliases = ['psysh']
+    version_added = '2.7'
+
+    def __init__(self, **options):
+        options['startinline'] = True
+        Lexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        phplexer = PhpLexer(**self.options)
+        curcode = ''
+        insertions = []
+        for match in line_re.finditer(text):
+            line = match.group()
+            if line.startswith('>>> ') or line.startswith('... '):
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, line[:4])]))
+                curcode += line[4:]
+            elif line.rstrip() == '...':
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, '...')]))
+                curcode += line[3:]
+            else:
+                if curcode:
+                    yield from do_insertions(
+                        insertions, phplexer.get_tokens_unprocessed(curcode))
+                    curcode = ''
+                    insertions = []
+                yield match.start(), Generic.Output, line
+        if curcode:
+            yield from do_insertions(insertions,
+                                     phplexer.get_tokens_unprocessed(curcode))
+
+
+class PhpLexer(RegexLexer):
+    """
+    For PHP source code.
+    For PHP embedded in HTML, use the `HtmlPhpLexer`.
+
+    Additional options accepted:
+
+    `startinline`
+        If given and ``True`` the lexer starts highlighting with
+        php code (i.e.: no starting ``>> from pygments.lexers._php_builtins import MODULES
+            >>> MODULES.keys()
+            ['PHP Options/Info', 'Zip', 'dba', ...]
+
+        In fact the names of those modules match the module names from
+        the php documentation.
+    """
+
+    name = 'PHP'
+    url = 'https://www.php.net/'
+    aliases = ['php', 'php3', 'php4', 'php5']
+    filenames = ['*.php', '*.php[345]', '*.inc']
+    mimetypes = ['text/x-php']
+    version_added = ''
+
+    # Note that a backslash is included, PHP uses a backslash as a namespace
+    # separator.
+    _ident_inner = r'(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*'
+    # But not inside strings.
+    _ident_nons = r'(?:[_a-z]|[^\x00-\x7f])(?:\w|[^\x00-\x7f])*'
+
+    flags = re.IGNORECASE | re.DOTALL | re.MULTILINE
+    tokens = {
+        'root': [
+            (r'<\?(php)?', Comment.Preproc, 'php'),
+            (r'[^<]+', Other),
+            (r'<', Other)
+        ],
+        'php': [
+            (r'\?>', Comment.Preproc, '#pop'),
+            (r'(<<<)([\'"]?)(' + _ident_nons + r')(\2\n.*?\n\s*)(\3)(;?)(\n)',
+             bygroups(String, String, String.Delimiter, String, String.Delimiter,
+                      Punctuation, Text)),
+            (r'\s+', Text),
+            (r'#\[', Punctuation, 'attribute'),
+            (r'#.*?\n', Comment.Single),
+            (r'//.*?\n', Comment.Single),
+            # put the empty comment here, it is otherwise seen as
+            # the start of a docstring
+            (r'/\*\*/', Comment.Multiline),
+            (r'/\*\*.*?\*/', String.Doc),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'(->|::)(\s*)(' + _ident_nons + ')',
+             bygroups(Operator, Text, Name.Attribute)),
+            (r'[~!%^&*+=|:.<>/@-]+', Operator),
+            (r'\?', Operator),  # don't add to the charclass above!
+            (r'[\[\]{}();,]+', Punctuation),
+            (r'(new)(\s+)(class)\b', bygroups(Keyword, Text, Keyword)),
+            (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+            (r'(function)(\s*)(?=\()', bygroups(Keyword, Text)),
+            (r'(function)(\s+)(&?)(\s*)',
+             bygroups(Keyword, Text, Operator, Text), 'functionname'),
+            (r'(const)(\s+)(' + _ident_inner + ')',
+             bygroups(Keyword, Text, Name.Constant)),
+            # source: https://www.php.net/manual/en/reserved.keywords.php
+            (r'(and|E_PARSE|old_function|E_ERROR|or|as|E_WARNING|parent|'
+             r'eval|PHP_OS|break|exit|case|extends|PHP_VERSION|cfunction|'
+             r'FALSE|print|for|require|continue|foreach|require_once|'
+             r'declare|return|default|static|do|switch|die|stdClass|'
+             r'echo|else|TRUE|elseif|var|empty|if|xor|enddeclare|include|'
+             r'virtual|endfor|include_once|while|endforeach|global|'
+             r'endif|list|endswitch|new|endwhile|not|'
+             r'array|E_ALL|NULL|final|php_user_filter|interface|'
+             r'implements|public|private|protected|abstract|clone|try|'
+             r'catch|throw|this|use|namespace|trait|yield( from)?|'
+             r'finally|match|readonly)\b', Keyword),
+            (r'(true|false|null)\b', Keyword.Constant),
+            include('magicconstants'),
+            (r'\$\{', Name.Variable, 'variablevariable'),
+            (r'\$+' + _ident_inner, Name.Variable),
+            (_ident_inner, Name.Other),
+            (r'(\d+\.\d*|\d*\.\d+)(e[+-]?[0-9]+)?', Number.Float),
+            (r'\d+e[+-]?[0-9]+', Number.Float),
+            (r'0[0-7]+', Number.Oct),
+            (r'0x[a-f0-9]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'0b[01]+', Number.Bin),
+            (r"'([^'\\]*(?:\\.[^'\\]*)*)'", String.Single),
+            (r'`([^`\\]*(?:\\.[^`\\]*)*)`', String.Backtick),
+            (r'"', String.Double, 'string'),
+        ],
+        'variablevariable': [
+            (r'\}', Name.Variable, '#pop'),
+            include('php')
+        ],
+        'magicfuncs': [
+            # source: http://php.net/manual/en/language.oop5.magic.php
+            (words((
+                '__construct', '__destruct', '__call', '__callStatic', '__get', '__set',
+                '__isset', '__unset', '__sleep', '__wakeup', '__toString', '__invoke',
+                '__set_state', '__clone', '__debugInfo',), suffix=r'\b'),
+             Name.Function.Magic),
+        ],
+        'magicconstants': [
+            # source: https://www.php.net/manual/en/language.constants.magic.php
+            (words((
+                '__LINE__', '__FILE__', '__DIR__', '__FUNCTION__', '__CLASS__',
+                '__TRAIT__', '__METHOD__', '__NAMESPACE__', '__PROPERTY__',),
+                suffix=r'\b'),
+             Name.Constant),
+        ],
+        'classname': [
+            (_ident_inner, Name.Class, '#pop')
+        ],
+        'functionname': [
+            include('magicfuncs'),
+            (_ident_inner, Name.Function, '#pop'),
+            default('#pop')
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (r'[^{$"\\]+', String.Double),
+            (r'\\([nrt"$\\]|[0-7]{1,3}|x[0-9a-f]{1,2})', String.Escape),
+            (r'\$' + _ident_nons + r'(\[\S+?\]|->' + _ident_nons + ')?',
+             String.Interpol),
+            (r'(\{\$\{)(.*?)(\}\})',
+             bygroups(String.Interpol, using(this, _startinline=True),
+                      String.Interpol)),
+            (r'(\{)(\$.*?)(\})',
+             bygroups(String.Interpol, using(this, _startinline=True),
+                      String.Interpol)),
+            (r'(\$\{)(\S+)(\})',
+             bygroups(String.Interpol, Name.Variable, String.Interpol)),
+            (r'[${\\]', String.Double)
+        ],
+        'attribute': [
+            (r'\]', Punctuation, '#pop'),
+            (r'\(', Punctuation, 'attributeparams'),
+            (_ident_inner, Name.Decorator),
+            include('php')
+        ],
+        'attributeparams': [
+            (r'\)', Punctuation, '#pop'),
+            include('php')
+        ],
+    }
+
+    def __init__(self, **options):
+        self.funcnamehighlighting = get_bool_opt(
+            options, 'funcnamehighlighting', True)
+        self.disabledmodules = get_list_opt(
+            options, 'disabledmodules', ['unknown'])
+        self.startinline = get_bool_opt(options, 'startinline', False)
+
+        # private option argument for the lexer itself
+        if '_startinline' in options:
+            self.startinline = options.pop('_startinline')
+
+        # collect activated functions in a set
+        self._functions = set()
+        if self.funcnamehighlighting:
+            from pygments.lexers._php_builtins import MODULES
+            for key, value in MODULES.items():
+                if key not in self.disabledmodules:
+                    self._functions.update(value)
+        RegexLexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        stack = ['root']
+        if self.startinline:
+            stack.append('php')
+        for index, token, value in \
+                RegexLexer.get_tokens_unprocessed(self, text, stack):
+            if token is Name.Other:
+                if value in self._functions:
+                    yield index, Name.Builtin, value
+                    continue
+            yield index, token, value
+
+    def analyse_text(text):
+        if shebang_matches(text, r'php'):
+            return True
+        rv = 0.0
+        if re.search(r'<\?(?!xml)', text):
+            rv += 0.3
+        return rv
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pointless.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pointless.py
new file mode 100644
index 0000000000000000000000000000000000000000..aab0a5684fe7b6c7c37372fefbf48630d2c96d4f
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pointless.py
@@ -0,0 +1,70 @@
+"""
+    pygments.lexers.pointless
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Pointless.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Comment, Error, Keyword, Name, Number, Operator, \
+    Punctuation, String, Text
+
+__all__ = ['PointlessLexer']
+
+
+class PointlessLexer(RegexLexer):
+    """
+    For Pointless source code.
+    """
+
+    name = 'Pointless'
+    url = 'https://ptls.dev'
+    aliases = ['pointless']
+    filenames = ['*.ptls']
+    version_added = '2.7'
+
+    ops = words([
+        "+", "-", "*", "/", "**", "%", "+=", "-=", "*=",
+        "/=", "**=", "%=", "|>", "=", "==", "!=", "<", ">",
+        "<=", ">=", "=>", "$", "++",
+    ])
+
+    keywords = words([
+        "if", "then", "else", "where", "with", "cond",
+        "case", "and", "or", "not", "in", "as", "for",
+        "requires", "throw", "try", "catch", "when",
+        "yield", "upval",
+    ], suffix=r'\b')
+
+    tokens = {
+        'root': [
+            (r'[ \n\r]+', Text),
+            (r'--.*$', Comment.Single),
+            (r'"""', String, 'multiString'),
+            (r'"', String, 'string'),
+            (r'[\[\](){}:;,.]', Punctuation),
+            (ops, Operator),
+            (keywords, Keyword),
+            (r'\d+|\d*\.\d+', Number),
+            (r'(true|false)\b', Name.Builtin),
+            (r'[A-Z][a-zA-Z0-9]*\b', String.Symbol),
+            (r'output\b', Name.Variable.Magic),
+            (r'(export|import)\b', Keyword.Namespace),
+            (r'[a-z][a-zA-Z0-9]*\b', Name.Variable)
+        ],
+        'multiString': [
+            (r'\\.', String.Escape),
+            (r'"""', String, '#pop'),
+            (r'"', String),
+            (r'[^\\"]+', String),
+        ],
+        'string': [
+            (r'\\.', String.Escape),
+            (r'"', String, '#pop'),
+            (r'\n', Error),
+            (r'[^\\"]+', String),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pony.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pony.py
new file mode 100644
index 0000000000000000000000000000000000000000..62d16d7d651b3395c761ecef98fe3f43d59ca33d
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/pony.py
@@ -0,0 +1,93 @@
+"""
+    pygments.lexers.pony
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Pony and related languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['PonyLexer']
+
+
+class PonyLexer(RegexLexer):
+    """
+    For Pony source code.
+    """
+
+    name = 'Pony'
+    aliases = ['pony']
+    filenames = ['*.pony']
+    url = 'https://www.ponylang.io'
+    version_added = '2.4'
+
+    _caps = r'(iso|trn|ref|val|box|tag)'
+
+    tokens = {
+        'root': [
+            (r'\n', Text),
+            (r'[^\S\n]+', Text),
+            (r'//.*\n', Comment.Single),
+            (r'/\*', Comment.Multiline, 'nested_comment'),
+            (r'"""(?:.|\n)*?"""', String.Doc),
+            (r'"', String, 'string'),
+            (r'\'.*\'', String.Char),
+            (r'=>|[]{}:().~;,|&!^?[]', Punctuation),
+            (words((
+                'addressof', 'and', 'as', 'consume', 'digestof', 'is', 'isnt',
+                'not', 'or'),
+                suffix=r'\b'),
+             Operator.Word),
+            (r'!=|==|<<|>>|[-+/*%=<>]', Operator),
+            (words((
+                'box', 'break', 'compile_error', 'compile_intrinsic',
+                'continue', 'do', 'else', 'elseif', 'embed', 'end', 'error',
+                'for', 'if', 'ifdef', 'in', 'iso', 'lambda', 'let', 'match',
+                'object', 'recover', 'ref', 'repeat', 'return', 'tag', 'then',
+                'this', 'trn', 'try', 'until', 'use', 'var', 'val', 'where',
+                'while', 'with', '#any', '#read', '#send', '#share'),
+                suffix=r'\b'),
+             Keyword),
+            (r'(actor|class|struct|primitive|interface|trait|type)((?:\s)+)',
+             bygroups(Keyword, Text), 'typename'),
+            (r'(new|fun|be)((?:\s)+)', bygroups(Keyword, Text), 'methodname'),
+            (words((
+                'I8', 'U8', 'I16', 'U16', 'I32', 'U32', 'I64', 'U64', 'I128',
+                'U128', 'ILong', 'ULong', 'ISize', 'USize', 'F32', 'F64',
+                'Bool', 'Pointer', 'None', 'Any', 'Array', 'String',
+                'Iterator'),
+                suffix=r'\b'),
+             Name.Builtin.Type),
+            (r'_?[A-Z]\w*', Name.Type),
+            (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'(true|false)\b', Name.Builtin),
+            (r'_\d*', Name),
+            (r'_?[a-z][\w\']*', Name)
+        ],
+        'typename': [
+            (_caps + r'?((?:\s)*)(_?[A-Z]\w*)',
+             bygroups(Keyword, Text, Name.Class), '#pop')
+        ],
+        'methodname': [
+            (_caps + r'?((?:\s)*)(_?[a-z]\w*)',
+             bygroups(Keyword, Text, Name.Function), '#pop')
+        ],
+        'nested_comment': [
+            (r'[^*/]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r'\\"', String),
+            (r'[^\\"]+', String)
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/praat.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/praat.py
new file mode 100644
index 0000000000000000000000000000000000000000..c50e9279f88c4f235262f748f62563b099f58d0b
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/praat.py
@@ -0,0 +1,303 @@
+"""
+    pygments.lexers.praat
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Praat
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, bygroups, include
+from pygments.token import Name, Text, Comment, Keyword, String, Punctuation, \
+    Number, Operator, Whitespace
+
+__all__ = ['PraatLexer']
+
+
+class PraatLexer(RegexLexer):
+    """
+    For Praat scripts.
+    """
+
+    name = 'Praat'
+    url = 'http://www.praat.org'
+    aliases = ['praat']
+    filenames = ['*.praat', '*.proc', '*.psc']
+    version_added = '2.1'
+
+    keywords = (
+        'if', 'then', 'else', 'elsif', 'elif', 'endif', 'fi', 'for', 'from', 'to',
+        'endfor', 'endproc', 'while', 'endwhile', 'repeat', 'until', 'select', 'plus',
+        'minus', 'demo', 'assert', 'stopwatch', 'nocheck', 'nowarn', 'noprogress',
+        'editor', 'endeditor', 'clearinfo',
+    )
+
+    functions_string = (
+        'backslashTrigraphsToUnicode', 'chooseDirectory', 'chooseReadFile',
+        'chooseWriteFile', 'date', 'demoKey', 'do', 'environment', 'extractLine',
+        'extractWord', 'fixed', 'info', 'left', 'mid', 'percent', 'readFile', 'replace',
+        'replace_regex', 'right', 'selected', 'string', 'unicodeToBackslashTrigraphs',
+    )
+
+    functions_numeric = (
+        'abs', 'appendFile', 'appendFileLine', 'appendInfo', 'appendInfoLine', 'arccos',
+        'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'barkToHertz',
+        'beginPause', 'beginSendPraat', 'besselI', 'besselK', 'beta', 'beta2',
+        'binomialP', 'binomialQ', 'boolean', 'ceiling', 'chiSquareP', 'chiSquareQ',
+        'choice', 'comment', 'cos', 'cosh', 'createDirectory', 'deleteFile',
+        'demoClicked', 'demoClickedIn', 'demoCommandKeyPressed',
+        'demoExtraControlKeyPressed', 'demoInput', 'demoKeyPressed',
+        'demoOptionKeyPressed', 'demoShiftKeyPressed', 'demoShow', 'demoWaitForInput',
+        'demoWindowTitle', 'demoX', 'demoY', 'differenceLimensToPhon', 'do', 'editor',
+        'endPause', 'endSendPraat', 'endsWith', 'erb', 'erbToHertz', 'erf', 'erfc',
+        'exitScript', 'exp', 'extractNumber', 'fileReadable', 'fisherP', 'fisherQ',
+        'floor', 'gaussP', 'gaussQ', 'hertzToBark', 'hertzToErb', 'hertzToMel',
+        'hertzToSemitones', 'imax', 'imin', 'incompleteBeta', 'incompleteGammaP', 'index',
+        'index_regex', 'integer', 'invBinomialP', 'invBinomialQ', 'invChiSquareQ', 'invFisherQ',
+        'invGaussQ', 'invSigmoid', 'invStudentQ', 'length', 'ln', 'lnBeta', 'lnGamma',
+        'log10', 'log2', 'max', 'melToHertz', 'min', 'minusObject', 'natural', 'number',
+        'numberOfColumns', 'numberOfRows', 'numberOfSelected', 'objectsAreIdentical',
+        'option', 'optionMenu', 'pauseScript', 'phonToDifferenceLimens', 'plusObject',
+        'positive', 'randomBinomial', 'randomGauss', 'randomInteger', 'randomPoisson',
+        'randomUniform', 'real', 'readFile', 'removeObject', 'rindex', 'rindex_regex',
+        'round', 'runScript', 'runSystem', 'runSystem_nocheck', 'selectObject',
+        'selected', 'semitonesToHertz', 'sentence', 'sentencetext', 'sigmoid', 'sin', 'sinc',
+        'sincpi', 'sinh', 'soundPressureToPhon', 'sqrt', 'startsWith', 'studentP',
+        'studentQ', 'tan', 'tanh', 'text', 'variableExists', 'word', 'writeFile', 'writeFileLine',
+        'writeInfo', 'writeInfoLine',
+    )
+
+    functions_array = (
+        'linear', 'randomGauss', 'randomInteger', 'randomUniform', 'zero',
+    )
+
+    objects = (
+        'Activation', 'AffineTransform', 'AmplitudeTier', 'Art', 'Artword',
+        'Autosegment', 'BarkFilter', 'BarkSpectrogram', 'CCA', 'Categories',
+        'Cepstrogram', 'Cepstrum', 'Cepstrumc', 'ChebyshevSeries', 'ClassificationTable',
+        'Cochleagram', 'Collection', 'ComplexSpectrogram', 'Configuration', 'Confusion',
+        'ContingencyTable', 'Corpus', 'Correlation', 'Covariance',
+        'CrossCorrelationTable', 'CrossCorrelationTables', 'DTW', 'DataModeler',
+        'Diagonalizer', 'Discriminant', 'Dissimilarity', 'Distance', 'Distributions',
+        'DurationTier', 'EEG', 'ERP', 'ERPTier', 'EditCostsTable', 'EditDistanceTable',
+        'Eigen', 'Excitation', 'Excitations', 'ExperimentMFC', 'FFNet', 'FeatureWeights',
+        'FileInMemory', 'FilesInMemory', 'Formant', 'FormantFilter', 'FormantGrid',
+        'FormantModeler', 'FormantPoint', 'FormantTier', 'GaussianMixture', 'HMM',
+        'HMM_Observation', 'HMM_ObservationSequence', 'HMM_State', 'HMM_StateSequence',
+        'Harmonicity', 'ISpline', 'Index', 'Intensity', 'IntensityTier', 'IntervalTier',
+        'KNN', 'KlattGrid', 'KlattTable', 'LFCC', 'LPC', 'Label', 'LegendreSeries',
+        'LinearRegression', 'LogisticRegression', 'LongSound', 'Ltas', 'MFCC', 'MSpline',
+        'ManPages', 'Manipulation', 'Matrix', 'MelFilter', 'MelSpectrogram',
+        'MixingMatrix', 'Movie', 'Network', 'Object', 'OTGrammar', 'OTHistory', 'OTMulti',
+        'PCA', 'PairDistribution', 'ParamCurve', 'Pattern', 'Permutation', 'Photo',
+        'Pitch', 'PitchModeler', 'PitchTier', 'PointProcess', 'Polygon', 'Polynomial',
+        'PowerCepstrogram', 'PowerCepstrum', 'Procrustes', 'RealPoint', 'RealTier',
+        'ResultsMFC', 'Roots', 'SPINET', 'SSCP', 'SVD', 'Salience', 'ScalarProduct',
+        'Similarity', 'SimpleString', 'SortedSetOfString', 'Sound', 'Speaker',
+        'Spectrogram', 'Spectrum', 'SpectrumTier', 'SpeechSynthesizer', 'SpellingChecker',
+        'Strings', 'StringsIndex', 'Table', 'TableOfReal', 'TextGrid', 'TextInterval',
+        'TextPoint', 'TextTier', 'Tier', 'Transition', 'VocalTract', 'VocalTractTier',
+        'Weight', 'WordList',
+    )
+
+    variables_numeric = (
+        'macintosh', 'windows', 'unix', 'praatVersion', 'pi', 'e', 'undefined',
+    )
+
+    variables_string = (
+        'praatVersion', 'tab', 'shellDirectory', 'homeDirectory',
+        'preferencesDirectory', 'newline', 'temporaryDirectory',
+        'defaultDirectory',
+    )
+
+    object_attributes = (
+        'ncol', 'nrow', 'xmin', 'ymin', 'xmax', 'ymax', 'nx', 'ny', 'dx', 'dy',
+    )
+
+    tokens = {
+        'root': [
+            (r'(\s+)(#.*?$)',  bygroups(Whitespace, Comment.Single)),
+            (r'^#.*?$',        Comment.Single),
+            (r';[^\n]*',       Comment.Single),
+            (r'\s+',           Whitespace),
+
+            (r'\bprocedure\b', Keyword,       'procedure_definition'),
+            (r'\bcall\b',      Keyword,       'procedure_call'),
+            (r'@',             Name.Function, 'procedure_call'),
+
+            include('function_call'),
+
+            (words(keywords, suffix=r'\b'), Keyword),
+
+            (r'(\bform\b)(\s+)([^\n]+)',
+             bygroups(Keyword, Whitespace, String), 'old_form'),
+
+            (r'(print(?:line|tab)?|echo|exit|asserterror|pause|send(?:praat|socket)|'
+             r'include|execute|system(?:_nocheck)?)(\s+)',
+             bygroups(Keyword, Whitespace), 'string_unquoted'),
+
+            (r'(goto|label)(\s+)(\w+)', bygroups(Keyword, Whitespace, Name.Label)),
+
+            include('variable_name'),
+            include('number'),
+
+            (r'"', String, 'string'),
+
+            (words((objects), suffix=r'(?=\s+\S+\n)'), Name.Class, 'string_unquoted'),
+
+            (r'\b[A-Z]', Keyword, 'command'),
+            (r'(\.{3}|[)(,])', Punctuation),
+        ],
+        'command': [
+            (r'( ?[\w()-]+ ?)', Keyword),
+
+            include('string_interpolated'),
+
+            (r'\.{3}', Keyword, ('#pop', 'old_arguments')),
+            (r':', Keyword, ('#pop', 'comma_list')),
+            (r'\s', Whitespace, '#pop'),
+        ],
+        'procedure_call': [
+            (r'\s+', Whitespace),
+            (r'([\w.]+)(?:(:)|(?:(\s*)(\()))',
+             bygroups(Name.Function, Punctuation,
+                      Text.Whitespace, Punctuation), '#pop'),
+            (r'([\w.]+)', Name.Function, ('#pop', 'old_arguments')),
+        ],
+        'procedure_definition': [
+            (r'\s', Whitespace),
+            (r'([\w.]+)(\s*?[(:])',
+             bygroups(Name.Function, Whitespace), '#pop'),
+            (r'([\w.]+)([^\n]*)',
+             bygroups(Name.Function, Text), '#pop'),
+        ],
+        'function_call': [
+            (words(functions_string, suffix=r'\$(?=\s*[:(])'), Name.Function, 'function'),
+            (words(functions_array, suffix=r'#(?=\s*[:(])'),   Name.Function, 'function'),
+            (words(functions_numeric, suffix=r'(?=\s*[:(])'),  Name.Function, 'function'),
+        ],
+        'function': [
+            (r'\s+',   Whitespace),
+            (r':',     Punctuation, ('#pop', 'comma_list')),
+            (r'\s*\(', Punctuation, ('#pop', 'comma_list')),
+        ],
+        'comma_list': [
+            (r'(\s*\n\s*)(\.{3})', bygroups(Whitespace, Punctuation)),
+
+            (r'(\s*)(?:([)\]])|(\n))', bygroups(
+                Whitespace, Punctuation, Whitespace), '#pop'),
+
+            (r'\s+', Whitespace),
+            (r'"',   String, 'string'),
+            (r'\b(if|then|else|fi|endif)\b', Keyword),
+
+            include('function_call'),
+            include('variable_name'),
+            include('operator'),
+            include('number'),
+
+            (r'[()]', Text),
+            (r',', Punctuation),
+        ],
+        'old_arguments': [
+            (r'\n', Whitespace, '#pop'),
+
+            include('variable_name'),
+            include('operator'),
+            include('number'),
+
+            (r'"', String, 'string'),
+            (r'[^\n]', Text),
+        ],
+        'number': [
+            (r'\n', Whitespace, '#pop'),
+            (r'\b\d+(\.\d*)?([eE][-+]?\d+)?%?', Number),
+        ],
+        'object_reference': [
+            include('string_interpolated'),
+            (r'([a-z][a-zA-Z0-9_]*|\d+)', Name.Builtin),
+
+            (words(object_attributes, prefix=r'\.'), Name.Builtin, '#pop'),
+
+            (r'\$', Name.Builtin),
+            (r'\[', Text, '#pop'),
+        ],
+        'variable_name': [
+            include('operator'),
+            include('number'),
+
+            (words(variables_string,  suffix=r'\$'), Name.Variable.Global),
+            (words(variables_numeric,
+             suffix=r'(?=[^a-zA-Z0-9_."\'$#\[:(]|\s|^|$)'),
+             Name.Variable.Global),
+
+            (words(objects, prefix=r'\b', suffix=r"(_)"),
+             bygroups(Name.Builtin, Name.Builtin),
+             'object_reference'),
+
+            (r'\.?_?[a-z][\w.]*(\$|#)?', Text),
+            (r'[\[\]]', Punctuation, 'comma_list'),
+
+            include('string_interpolated'),
+        ],
+        'operator': [
+            (r'([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)',       Operator),
+            (r'(?', Punctuation),
+            (r'"(?:\\x[0-9a-fA-F]+\\|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|'
+             r'\\[0-7]+\\|\\["\\abcefnrstv]|[^\\"])*"', String.Double),
+            (r"'(?:''|[^'])*'", String.Atom),  # quoted atom
+            # Needs to not be followed by an atom.
+            # (r'=(?=\s|[a-zA-Z\[])', Operator),
+            (r'is\b', Operator),
+            (r'(<|>|=<|>=|==|=:=|=|/|//|\*|\+|-)(?=\s|[a-zA-Z0-9\[])',
+             Operator),
+            (r'(mod|div|not)\b', Operator),
+            (r'_', Keyword),  # The don't-care variable
+            (r'([a-z]+)(:)', bygroups(Name.Namespace, Punctuation)),
+            (r'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
+             r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)'
+             r'(\s*)(:-|-->)',
+             bygroups(Name.Function, Text, Operator)),  # function defn
+            (r'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
+             r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)'
+             r'(\s*)(\()',
+             bygroups(Name.Function, Text, Punctuation)),
+            (r'[a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]'
+             r'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*',
+             String.Atom),  # atom, characters
+            # This one includes !
+            (r'[#&*+\-./:<=>?@\\^~\u00a1-\u00bf\u2010-\u303f]+',
+             String.Atom),  # atom, graphics
+            (r'[A-Z_]\w*', Name.Variable),
+            (r'\s+|[\u2000-\u200f\ufff0-\ufffe\uffef]', Text),
+        ],
+        'nested-comment': [
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'[^*/]+', Comment.Multiline),
+            (r'[*/]', Comment.Multiline),
+        ],
+    }
+
+    def analyse_text(text):
+        """Competes with IDL and Visual Prolog on *.pro"""
+        if ':-' in text:
+            # Visual Prolog also uses :-
+            return 0.5
+        else:
+            return 0
+
+
+class LogtalkLexer(RegexLexer):
+    """
+    For Logtalk source code.
+    """
+
+    name = 'Logtalk'
+    url = 'http://logtalk.org/'
+    aliases = ['logtalk']
+    filenames = ['*.lgt', '*.logtalk']
+    mimetypes = ['text/x-logtalk']
+    version_added = '0.10'
+
+    tokens = {
+        'root': [
+            # Directives
+            (r'^\s*:-\s', Punctuation, 'directive'),
+            # Comments
+            (r'%.*?\n', Comment),
+            (r'/\*(.|\n)*?\*/', Comment),
+            # Whitespace
+            (r'\n', Text),
+            (r'\s+', Text),
+            # Numbers
+            (r"0'[\\]?.", Number),
+            (r'0b[01]+', Number.Bin),
+            (r'0o[0-7]+', Number.Oct),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number),
+            # Variables
+            (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable),
+            # Event handlers
+            (r'(after|before)(?=[(])', Keyword),
+            # Message forwarding handler
+            (r'forward(?=[(])', Keyword),
+            # Execution-context methods
+            (r'(context|parameter|this|se(lf|nder))(?=[(])', Keyword),
+            # Reflection
+            (r'(current_predicate|predicate_property)(?=[(])', Keyword),
+            # DCGs and term expansion
+            (r'(expand_(goal|term)|(goal|term)_expansion|phrase)(?=[(])', Keyword),
+            # Entity
+            (r'(abolish|c(reate|urrent))_(object|protocol|category)(?=[(])', Keyword),
+            (r'(object|protocol|category)_property(?=[(])', Keyword),
+            # Entity relations
+            (r'co(mplements_object|nforms_to_protocol)(?=[(])', Keyword),
+            (r'extends_(object|protocol|category)(?=[(])', Keyword),
+            (r'imp(lements_protocol|orts_category)(?=[(])', Keyword),
+            (r'(instantiat|specializ)es_class(?=[(])', Keyword),
+            # Events
+            (r'(current_event|(abolish|define)_events)(?=[(])', Keyword),
+            # Flags
+            (r'(create|current|set)_logtalk_flag(?=[(])', Keyword),
+            # Compiling, loading, and library paths
+            (r'logtalk_(compile|l(ibrary_path|oad|oad_context)|make(_target_action)?)(?=[(])', Keyword),
+            (r'\blogtalk_make\b', Keyword),
+            # Database
+            (r'(clause|retract(all)?)(?=[(])', Keyword),
+            (r'a(bolish|ssert(a|z))(?=[(])', Keyword),
+            # Control constructs
+            (r'(ca(ll|tch)|throw)(?=[(])', Keyword),
+            (r'(fa(il|lse)|true|(instantiation|system)_error)\b', Keyword),
+            (r'(uninstantiation|type|domain|existence|permission|representation|evaluation|resource|syntax)_error(?=[(])', Keyword),
+            # All solutions
+            (r'((bag|set)of|f(ind|or)all)(?=[(])', Keyword),
+            # Multi-threading predicates
+            (r'threaded(_(ca(ll|ncel)|once|ignore|exit|peek|wait|notify))?(?=[(])', Keyword),
+            # Engine predicates
+            (r'threaded_engine(_(create|destroy|self|next|next_reified|yield|post|fetch))?(?=[(])', Keyword),
+            # Term unification
+            (r'(subsumes_term|unify_with_occurs_check)(?=[(])', Keyword),
+            # Term creation and decomposition
+            (r'(functor|arg|copy_term|numbervars|term_variables)(?=[(])', Keyword),
+            # Evaluable functors
+            (r'(div|rem|m(ax|in|od)|abs|sign)(?=[(])', Keyword),
+            (r'float(_(integer|fractional)_part)?(?=[(])', Keyword),
+            (r'(floor|t(an|runcate)|round|ceiling)(?=[(])', Keyword),
+            # Other arithmetic functors
+            (r'(cos|a(cos|sin|tan|tan2)|exp|log|s(in|qrt)|xor)(?=[(])', Keyword),
+            # Term testing
+            (r'(var|atom(ic)?|integer|float|c(allable|ompound)|n(onvar|umber)|ground|acyclic_term)(?=[(])', Keyword),
+            # Term comparison
+            (r'compare(?=[(])', Keyword),
+            # Stream selection and control
+            (r'(curren|se)t_(in|out)put(?=[(])', Keyword),
+            (r'(open|close)(?=[(])', Keyword),
+            (r'flush_output(?=[(])', Keyword),
+            (r'(at_end_of_stream|flush_output)\b', Keyword),
+            (r'(stream_property|at_end_of_stream|set_stream_position)(?=[(])', Keyword),
+            # Character and byte input/output
+            (r'(nl|(get|peek|put)_(byte|c(har|ode)))(?=[(])', Keyword),
+            (r'\bnl\b', Keyword),
+            # Term input/output
+            (r'read(_term)?(?=[(])', Keyword),
+            (r'write(q|_(canonical|term))?(?=[(])', Keyword),
+            (r'(current_)?op(?=[(])', Keyword),
+            (r'(current_)?char_conversion(?=[(])', Keyword),
+            # Atomic term processing
+            (r'atom_(length|c(hars|o(ncat|des)))(?=[(])', Keyword),
+            (r'(char_code|sub_atom)(?=[(])', Keyword),
+            (r'number_c(har|ode)s(?=[(])', Keyword),
+            # Implementation defined hooks functions
+            (r'(se|curren)t_prolog_flag(?=[(])', Keyword),
+            (r'\bhalt\b', Keyword),
+            (r'halt(?=[(])', Keyword),
+            # Message sending operators
+            (r'(::|:|\^\^)', Operator),
+            # External call
+            (r'[{}]', Keyword),
+            # Logic and control
+            (r'(ignore|once)(?=[(])', Keyword),
+            (r'\brepeat\b', Keyword),
+            # Sorting
+            (r'(key)?sort(?=[(])', Keyword),
+            # Bitwise functors
+            (r'(>>|<<|/\\|\\\\|\\)', Operator),
+            # Predicate aliases
+            (r'\bas\b', Operator),
+            # Arithmetic evaluation
+            (r'\bis\b', Keyword),
+            # Arithmetic comparison
+            (r'(=:=|=\\=|<|=<|>=|>)', Operator),
+            # Term creation and decomposition
+            (r'=\.\.', Operator),
+            # Term unification
+            (r'(=|\\=)', Operator),
+            # Term comparison
+            (r'(==|\\==|@=<|@<|@>=|@>)', Operator),
+            # Evaluable functors
+            (r'(//|[-+*/])', Operator),
+            (r'\b(e|pi|div|mod|rem)\b', Operator),
+            # Other arithmetic functors
+            (r'\b\*\*\b', Operator),
+            # DCG rules
+            (r'-->', Operator),
+            # Control constructs
+            (r'([!;]|->)', Operator),
+            # Logic and control
+            (r'\\+', Operator),
+            # Mode operators
+            (r'[?@]', Operator),
+            # Existential quantifier
+            (r'\^', Operator),
+            # Punctuation
+            (r'[()\[\],.|]', Text),
+            # Atoms
+            (r"[a-z][a-zA-Z0-9_]*", Text),
+            (r"'", String, 'quoted_atom'),
+            # Double-quoted terms
+            (r'"', String, 'double_quoted_term'),
+        ],
+
+        'quoted_atom': [
+            (r"''", String),
+            (r"'", String, '#pop'),
+            (r'\\([\\abfnrtv"\']|(x[a-fA-F0-9]+|[0-7]+)\\)', String.Escape),
+            (r"[^\\'\n]+", String),
+            (r'\\', String),
+        ],
+
+        'double_quoted_term': [
+            (r'""', String),
+            (r'"', String, '#pop'),
+            (r'\\([\\abfnrtv"\']|(x[a-fA-F0-9]+|[0-7]+)\\)', String.Escape),
+            (r'[^\\"\n]+', String),
+            (r'\\', String),
+        ],
+
+        'directive': [
+            # Conditional compilation directives
+            (r'(el)?if(?=[(])', Keyword, 'root'),
+            (r'(e(lse|ndif))(?=[.])', Keyword, 'root'),
+            # Entity directives
+            (r'(category|object|protocol)(?=[(])', Keyword, 'entityrelations'),
+            (r'(end_(category|object|protocol))(?=[.])', Keyword, 'root'),
+            # Predicate scope directives
+            (r'(public|protected|private)(?=[(])', Keyword, 'root'),
+            # Other directives
+            (r'e(n(coding|sure_loaded)|xport)(?=[(])', Keyword, 'root'),
+            (r'in(clude|itialization|fo)(?=[(])', Keyword, 'root'),
+            (r'(built_in|dynamic|synchronized|threaded)(?=[.])', Keyword, 'root'),
+            (r'(alias|d(ynamic|iscontiguous)|m(eta_(non_terminal|predicate)|ode|ultifile)|s(et_(logtalk|prolog)_flag|ynchronized))(?=[(])', Keyword, 'root'),
+            (r'op(?=[(])', Keyword, 'root'),
+            (r'(c(alls|oinductive)|module|reexport|use(s|_module))(?=[(])', Keyword, 'root'),
+            (r'[a-z][a-zA-Z0-9_]*(?=[(])', Text, 'root'),
+            (r'[a-z][a-zA-Z0-9_]*(?=[.])', Text, 'root'),
+        ],
+
+        'entityrelations': [
+            (r'(complements|extends|i(nstantiates|mp(lements|orts))|specializes)(?=[(])', Keyword),
+            # Numbers
+            (r"0'[\\]?.", Number),
+            (r'0b[01]+', Number.Bin),
+            (r'0o[0-7]+', Number.Oct),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number),
+            # Variables
+            (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable),
+            # Atoms
+            (r"[a-z][a-zA-Z0-9_]*", Text),
+            (r"'", String, 'quoted_atom'),
+            # Double-quoted terms
+            (r'"', String, 'double_quoted_term'),
+            # End of entity-opening directive
+            (r'([)]\.)', Text, 'root'),
+            # Scope operator
+            (r'(::)', Operator),
+            # Punctuation
+            (r'[()\[\],.|]', Text),
+            # Comments
+            (r'%.*?\n', Comment),
+            (r'/\*(.|\n)*?\*/', Comment),
+            # Whitespace
+            (r'\n', Text),
+            (r'\s+', Text),
+        ]
+    }
+
+    def analyse_text(text):
+        if ':- object(' in text:
+            return 1.0
+        elif ':- protocol(' in text:
+            return 1.0
+        elif ':- category(' in text:
+            return 1.0
+        elif re.search(r'^:-\s[a-z]', text, re.M):
+            return 0.9
+        else:
+            return 0.0
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/promql.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/promql.py
new file mode 100644
index 0000000000000000000000000000000000000000..5cfa73fe1e619918e80828ab1529f268938adcd0
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/promql.py
@@ -0,0 +1,176 @@
+"""
+    pygments.lexers.promql
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Prometheus Query Language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, default, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, \
+    Punctuation, String, Whitespace
+
+__all__ = ["PromQLLexer"]
+
+
+class PromQLLexer(RegexLexer):
+    """
+    For PromQL queries.
+
+    For details about the grammar see:
+    https://github.com/prometheus/prometheus/tree/master/promql/parser
+
+    .. versionadded: 2.7
+    """
+
+    name = "PromQL"
+    url = 'https://prometheus.io/docs/prometheus/latest/querying/basics/'
+    aliases = ["promql"]
+    filenames = ["*.promql"]
+    version_added = ''
+
+    base_keywords = (
+        words(
+            (
+                "bool",
+                "by",
+                "group_left",
+                "group_right",
+                "ignoring",
+                "offset",
+                "on",
+                "without",
+            ),
+            suffix=r"\b",
+        ),
+        Keyword,
+    )
+
+    aggregator_keywords = (
+        words(
+            (
+                "sum",
+                "min",
+                "max",
+                "avg",
+                "group",
+                "stddev",
+                "stdvar",
+                "count",
+                "count_values",
+                "bottomk",
+                "topk",
+                "quantile",
+            ),
+            suffix=r"\b",
+        ),
+        Keyword,
+    )
+
+    function_keywords = (
+        words(
+            (
+                "abs",
+                "absent",
+                "absent_over_time",
+                "avg_over_time",
+                "ceil",
+                "changes",
+                "clamp_max",
+                "clamp_min",
+                "count_over_time",
+                "day_of_month",
+                "day_of_week",
+                "days_in_month",
+                "delta",
+                "deriv",
+                "exp",
+                "floor",
+                "histogram_quantile",
+                "holt_winters",
+                "hour",
+                "idelta",
+                "increase",
+                "irate",
+                "label_join",
+                "label_replace",
+                "ln",
+                "log10",
+                "log2",
+                "max_over_time",
+                "min_over_time",
+                "minute",
+                "month",
+                "predict_linear",
+                "quantile_over_time",
+                "rate",
+                "resets",
+                "round",
+                "scalar",
+                "sort",
+                "sort_desc",
+                "sqrt",
+                "stddev_over_time",
+                "stdvar_over_time",
+                "sum_over_time",
+                "time",
+                "timestamp",
+                "vector",
+                "year",
+            ),
+            suffix=r"\b",
+        ),
+        Keyword.Reserved,
+    )
+
+    tokens = {
+        "root": [
+            (r"\n", Whitespace),
+            (r"\s+", Whitespace),
+            (r",", Punctuation),
+            # Keywords
+            base_keywords,
+            aggregator_keywords,
+            function_keywords,
+            # Offsets
+            (r"[1-9][0-9]*[smhdwy]", String),
+            # Numbers
+            (r"-?[0-9]+\.[0-9]+", Number.Float),
+            (r"-?[0-9]+", Number.Integer),
+            # Comments
+            (r"#.*?$", Comment.Single),
+            # Operators
+            (r"(\+|\-|\*|\/|\%|\^)", Operator),
+            (r"==|!=|>=|<=|<|>", Operator),
+            (r"and|or|unless", Operator.Word),
+            # Metrics
+            (r"[_a-zA-Z][a-zA-Z0-9_]+", Name.Variable),
+            # Params
+            (r'(["\'])(.*?)(["\'])', bygroups(Punctuation, String, Punctuation)),
+            # Other states
+            (r"\(", Operator, "function"),
+            (r"\)", Operator),
+            (r"\{", Punctuation, "labels"),
+            (r"\[", Punctuation, "range"),
+        ],
+        "labels": [
+            (r"\}", Punctuation, "#pop"),
+            (r"\n", Whitespace),
+            (r"\s+", Whitespace),
+            (r",", Punctuation),
+            (r'([_a-zA-Z][a-zA-Z0-9_]*?)(\s*?)(=~|!=|=|!~)(\s*?)("|\')(.*?)("|\')',
+             bygroups(Name.Label, Whitespace, Operator, Whitespace,
+                      Punctuation, String, Punctuation)),
+        ],
+        "range": [
+            (r"\]", Punctuation, "#pop"),
+            (r"[1-9][0-9]*[smhdwy]", String),
+        ],
+        "function": [
+            (r"\)", Operator, "#pop"),
+            (r"\(", Operator, "#push"),
+            default("#pop"),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/prql.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/prql.py
new file mode 100644
index 0000000000000000000000000000000000000000..a62cb6bcd1a9ff89660b996ebda8106f687d296a
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/prql.py
@@ -0,0 +1,251 @@
+"""
+    pygments.lexers.prql
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the PRQL query language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, combined, words, include, bygroups
+from pygments.token import Comment, Literal, Keyword, Name, Number, Operator, \
+    Punctuation, String, Text, Whitespace
+
+__all__ = ['PrqlLexer']
+
+
+class PrqlLexer(RegexLexer):
+    """
+    For PRQL source code.
+
+    grammar: https://github.com/PRQL/prql/tree/main/grammars
+    """
+
+    name = 'PRQL'
+    url = 'https://prql-lang.org/'
+    aliases = ['prql']
+    filenames = ['*.prql']
+    mimetypes = ['application/prql', 'application/x-prql']
+    version_added = '2.17'
+
+    builtinTypes = words((
+        "bool",
+        "int",
+        "int8", "int16", "int32", "int64", "int128",
+        "float",
+        "text",
+        "set"), suffix=r'\b')
+
+    def innerstring_rules(ttype):
+        return [
+            # the new style '{}'.format(...) string formatting
+            (r'\{'
+             r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?'  # field name
+             r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
+             r'\}', String.Interpol),
+
+            (r'[^\\\'"%{\n]+', ttype),
+            (r'[\'"\\]', ttype),
+            (r'%|(\{{1,2})', ttype)
+        ]
+
+    def fstring_rules(ttype):
+        return [
+            (r'\}', String.Interpol),
+            (r'\{', String.Interpol, 'expr-inside-fstring'),
+            (r'[^\\\'"{}\n]+', ttype),
+            (r'[\'"\\]', ttype),
+        ]
+
+    tokens = {
+        'root': [
+
+            # Comments
+            (r'#!.*', String.Doc),
+            (r'#.*', Comment.Single),
+
+            # Whitespace
+            (r'\s+', Whitespace),
+
+            # Modules
+            (r'^(\s*)(module)(\s*)',
+             bygroups(Whitespace, Keyword.Namespace, Whitespace),
+             'imports'),
+
+            (builtinTypes, Keyword.Type),
+
+            # Main
+            (r'^prql ', Keyword.Reserved),
+
+            ('let', Keyword.Declaration),
+
+            include('keywords'),
+            include('expr'),
+
+            # Transforms
+            (r'^[A-Za-z_][a-zA-Z0-9_]*', Keyword),
+        ],
+        'expr': [
+            # non-raw f-strings
+            ('(f)(""")', bygroups(String.Affix, String.Double),
+             combined('fstringescape', 'tdqf')),
+            ("(f)(''')", bygroups(String.Affix, String.Single),
+             combined('fstringescape', 'tsqf')),
+            ('(f)(")', bygroups(String.Affix, String.Double),
+             combined('fstringescape', 'dqf')),
+            ("(f)(')", bygroups(String.Affix, String.Single),
+             combined('fstringescape', 'sqf')),
+
+            # non-raw s-strings
+            ('(s)(""")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'tdqf')),
+            ("(s)(''')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'tsqf')),
+            ('(s)(")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'dqf')),
+            ("(s)(')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'sqf')),
+
+            # raw strings
+            ('(?i)(r)(""")',
+             bygroups(String.Affix, String.Double), 'tdqs'),
+            ("(?i)(r)(''')",
+             bygroups(String.Affix, String.Single), 'tsqs'),
+            ('(?i)(r)(")',
+             bygroups(String.Affix, String.Double), 'dqs'),
+            ("(?i)(r)(')",
+             bygroups(String.Affix, String.Single), 'sqs'),
+
+            # non-raw strings
+            ('"""', String.Double, combined('stringescape', 'tdqs')),
+            ("'''", String.Single, combined('stringescape', 'tsqs')),
+            ('"', String.Double, combined('stringescape', 'dqs')),
+            ("'", String.Single, combined('stringescape', 'sqs')),
+
+            # Time and dates
+            (r'@\d{4}-\d{2}-\d{2}T\d{2}(:\d{2})?(:\d{2})?(\.\d{1,6})?(Z|[+-]\d{1,2}(:\d{1,2})?)?', Literal.Date),
+            (r'@\d{4}-\d{2}-\d{2}', Literal.Date),
+            (r'@\d{2}(:\d{2})?(:\d{2})?(\.\d{1,6})?(Z|[+-]\d{1,2}(:\d{1,2})?)?', Literal.Date),
+
+            (r'[^\S\n]+', Text),
+            include('numbers'),
+            (r'->|=>|==|!=|>=|<=|~=|&&|\|\||\?\?|\/\/', Operator),
+            (r'[-~+/*%=<>&^|.@]', Operator),
+            (r'[]{}:(),;[]', Punctuation),
+            include('functions'),
+
+            # Variable Names
+            (r'[A-Za-z_][a-zA-Z0-9_]*', Name.Variable),
+        ],
+        'numbers': [
+            (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)'
+             r'([eE][+-]?\d(?:_?\d)*)?', Number.Float),
+            (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float),
+            (r'0[oO](?:_?[0-7])+', Number.Oct),
+            (r'0[bB](?:_?[01])+', Number.Bin),
+            (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex),
+            (r'\d(?:_?\d)*', Number.Integer),
+        ],
+        'fstringescape': [
+            include('stringescape'),
+        ],
+        'bytesescape': [
+            (r'\\([\\bfnrt"\']|\n|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
+        ],
+        'stringescape': [
+            (r'\\(N\{.*?\}|u\{[a-fA-F0-9]{1,6}\})', String.Escape),
+            include('bytesescape')
+        ],
+        'fstrings-single': fstring_rules(String.Single),
+        'fstrings-double': fstring_rules(String.Double),
+        'strings-single': innerstring_rules(String.Single),
+        'strings-double': innerstring_rules(String.Double),
+        'dqf': [
+            (r'"', String.Double, '#pop'),
+            (r'\\\\|\\"|\\\n', String.Escape),  # included here for raw strings
+            include('fstrings-double')
+        ],
+        'sqf': [
+            (r"'", String.Single, '#pop'),
+            (r"\\\\|\\'|\\\n", String.Escape),  # included here for raw strings
+            include('fstrings-single')
+        ],
+        'dqs': [
+            (r'"', String.Double, '#pop'),
+            (r'\\\\|\\"|\\\n', String.Escape),  # included here for raw strings
+            include('strings-double')
+        ],
+        'sqs': [
+            (r"'", String.Single, '#pop'),
+            (r"\\\\|\\'|\\\n", String.Escape),  # included here for raw strings
+            include('strings-single')
+        ],
+        'tdqf': [
+            (r'"""', String.Double, '#pop'),
+            include('fstrings-double'),
+            (r'\n', String.Double)
+        ],
+        'tsqf': [
+            (r"'''", String.Single, '#pop'),
+            include('fstrings-single'),
+            (r'\n', String.Single)
+        ],
+        'tdqs': [
+            (r'"""', String.Double, '#pop'),
+            include('strings-double'),
+            (r'\n', String.Double)
+        ],
+        'tsqs': [
+            (r"'''", String.Single, '#pop'),
+            include('strings-single'),
+            (r'\n', String.Single)
+        ],
+
+        'expr-inside-fstring': [
+            (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
+            # without format specifier
+            (r'(=\s*)?'         # debug (https://bugs.python.org/issue36817)
+             r'\}', String.Interpol, '#pop'),
+            # with format specifier
+            # we'll catch the remaining '}' in the outer scope
+            (r'(=\s*)?'         # debug (https://bugs.python.org/issue36817)
+             r':', String.Interpol, '#pop'),
+            (r'\s+', Whitespace),  # allow new lines
+            include('expr'),
+        ],
+        'expr-inside-fstring-inner': [
+            (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
+            (r'[])}]', Punctuation, '#pop'),
+            (r'\s+', Whitespace),  # allow new lines
+            include('expr'),
+        ],
+        'keywords': [
+            (words((
+                'into', 'case', 'type', 'module', 'internal',
+            ), suffix=r'\b'),
+                Keyword),
+            (words(('true', 'false', 'null'), suffix=r'\b'), Keyword.Constant),
+        ],
+        'functions': [
+            (words((
+                "min", "max", "sum", "average", "stddev", "every", "any",
+                "concat_array", "count", "lag", "lead", "first", "last",
+                "rank", "rank_dense", "row_number", "round", "as", "in",
+                "tuple_every", "tuple_map", "tuple_zip", "_eq", "_is_null",
+                "from_text", "lower", "upper", "read_parquet", "read_csv"),
+                suffix=r'\b'),
+             Name.Function),
+        ],
+
+        'comment': [
+            (r'-(?!\})', Comment.Multiline),
+            (r'\{-', Comment.Multiline, 'comment'),
+            (r'[^-}]', Comment.Multiline),
+            (r'-\}', Comment.Multiline, '#pop'),
+        ],
+
+        'imports': [
+            (r'\w+(\.\w+)*', Name.Class, '#pop'),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ptx.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ptx.py
new file mode 100644
index 0000000000000000000000000000000000000000..685c7357ecb2548069d3b5506fae5f8ba8ceb65b
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ptx.py
@@ -0,0 +1,119 @@
+"""
+    pygments.lexers.ptx
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for other PTX language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Comment, Keyword, Name, String, Number, \
+    Punctuation, Whitespace, Operator
+
+__all__ = ["PtxLexer"]
+
+
+class PtxLexer(RegexLexer):
+    """
+    For NVIDIA `PTX `_
+    source.
+    """
+    name = 'PTX'
+    url = "https://docs.nvidia.com/cuda/parallel-thread-execution/"
+    filenames = ['*.ptx']
+    aliases = ['ptx']
+    mimetypes = ['text/x-ptx']
+    version_added = '2.16'
+
+    #: optional Comment or Whitespace
+    string = r'"[^"]*?"'
+    followsym = r'[a-zA-Z0-9_$]'
+    identifier = r'([-a-zA-Z$._][\w\-$.]*|' + string + ')'
+    block_label = r'(' + identifier + r'|(\d+))'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+
+            (block_label + r'\s*:', Name.Label),
+
+            include('keyword'),
+
+            (r'%' + identifier, Name.Variable),
+            (r'%\d+', Name.Variable.Anonymous),
+            (r'c?' + string, String),
+            (identifier, Name.Variable),
+            (r';', Punctuation),
+            (r'[*+-/]', Operator),
+
+            (r'0[xX][a-fA-F0-9]+', Number),
+            (r'-?\d+(?:[.]\d+)?(?:[eE][-+]?\d+(?:[.]\d+)?)?', Number),
+
+            (r'[=<>{}\[\]()*.,!]|x\b', Punctuation)
+
+        ],
+        'whitespace': [
+            (r'(\n|\s+)+', Whitespace),
+            (r'//.*?\n', Comment)
+        ],
+
+        'keyword': [
+            # Instruction keywords
+            (words((
+                'abs', 'discard', 'min', 'shf', 'vadd',
+                'activemask', 'div', 'mma', 'shfl', 'vadd2',
+                'add', 'dp2a', 'mov', 'shl', 'vadd4',
+                'addc', 'dp4a', 'movmatrix', 'shr', 'vavrg2',
+                'alloca', 'elect', 'mul', 'sin', 'vavrg4',
+                'and', 'ex2', 'mul24', 'slct', 'vmad',
+                'applypriority', 'exit', 'multimem', 'sqrt', 'vmax',
+                'atom', 'fence', 'nanosleep', 'st', 'vmax2',
+                'bar', 'fma', 'neg', 'stackrestore', 'vmax4',
+                'barrier', 'fns', 'not', 'stacksave', 'vmin',
+                'bfe', 'getctarank', 'or', 'stmatrix', 'vmin2',
+                'bfi', 'griddepcontrol', 'pmevent', 'sub', 'vmin4',
+                'bfind', 'isspacep', 'popc', 'subc', 'vote',
+                'bmsk', 'istypep', 'prefetch', 'suld', 'vset',
+                'bra', 'ld', 'prefetchu', 'suq', 'vset2',
+                'brev', 'ldmatrix', 'prmt', 'sured', 'vset4',
+                'brkpt', 'ldu', 'rcp', 'sust', 'vshl',
+                'brx', 'lg2', 'red', 'szext', 'vshr',
+                'call', 'lop3', 'redux', 'tanh', 'vsub',
+                'clz', 'mad', 'rem', 'testp', 'vsub2',
+                'cnot', 'mad24', 'ret', 'tex', 'vsub4',
+                'copysign', 'madc', 'rsqrt', 'tld4', 'wgmma',
+                'cos', 'mapa', 'sad', 'trap', 'wmma',
+                'cp', 'match', 'selp', 'txq', 'xor',
+                'createpolicy', 'max', 'set', 'vabsdiff', 'cvt',
+                'mbarrier', 'setmaxnreg', 'vabsdiff2', 'cvta',
+                'membar', 'setp', 'vabsdiff4')), Keyword),
+            # State Spaces and Suffixes
+            (words((
+                'reg', '.sreg', '.const', '.global',
+                '.local', '.param', '.shared', '.tex',
+                '.wide', '.loc'
+            )), Keyword.Pseudo),
+            # PTX Directives
+            (words((
+                '.address_size', '.explicitcluster', '.maxnreg', '.section',
+                '.alias', '.extern', '.maxntid', '.shared',
+                '.align', '.file', '.minnctapersm', '.sreg',
+                '.branchtargets', '.func', '.noreturn', '.target',
+                '.callprototype', '.global', '.param', '.tex',
+                '.calltargets', '.loc', '.pragma', '.version',
+                '.common', '.local', '.reg', '.visible',
+                '.const', '.maxclusterrank', '.reqnctapercluster', '.weak',
+                '.entry', '.maxnctapersm', '.reqntid')), Keyword.Reserved),
+            # Fundamental Types
+            (words((
+                '.s8', '.s16', '.s32', '.s64',
+                '.u8', '.u16', '.u32', '.u64',
+                '.f16', '.f16x2', '.f32', '.f64',
+                '.b8', '.b16', '.b32', '.b64',
+                '.pred'
+            )), Keyword.Type)
+        ],
+
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/python.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/python.py
new file mode 100644
index 0000000000000000000000000000000000000000..b296c8dbe4ed35625167711800abd9642b61c84b
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/python.py
@@ -0,0 +1,1204 @@
+"""
+    pygments.lexers.python
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Python and related languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import keyword
+
+from pygments.lexer import DelegatingLexer, RegexLexer, include, \
+    bygroups, using, default, words, combined, this
+from pygments.util import get_bool_opt, shebang_matches
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Generic, Other, Error, Whitespace
+from pygments import unistring as uni
+
+__all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer',
+           'Python2Lexer', 'Python2TracebackLexer',
+           'CythonLexer', 'DgLexer', 'NumPyLexer']
+
+
+class PythonLexer(RegexLexer):
+    """
+    For Python source code (version 3.x).
+
+    .. versionchanged:: 2.5
+       This is now the default ``PythonLexer``.  It is still available as the
+       alias ``Python3Lexer``.
+    """
+
+    name = 'Python'
+    url = 'https://www.python.org'
+    aliases = ['python', 'py', 'sage', 'python3', 'py3', 'bazel', 'starlark', 'pyi']
+    filenames = [
+        '*.py',
+        '*.pyw',
+        # Type stubs
+        '*.pyi',
+        # Jython
+        '*.jy',
+        # Sage
+        '*.sage',
+        # SCons
+        '*.sc',
+        'SConstruct',
+        'SConscript',
+        # Skylark/Starlark (used by Bazel, Buck, and Pants)
+        '*.bzl',
+        'BUCK',
+        'BUILD',
+        'BUILD.bazel',
+        'WORKSPACE',
+        # Twisted Application infrastructure
+        '*.tac',
+        # Execubot level format
+        '*.pye',
+    ]
+    mimetypes = ['text/x-python', 'application/x-python',
+                 'text/x-python3', 'application/x-python3']
+    version_added = '0.10'
+
+    uni_name = f"[{uni.xid_start}][{uni.xid_continue}]*"
+
+    def innerstring_rules(ttype):
+        return [
+            # the old style '%s' % (...) string formatting (still valid in Py3)
+            (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
+             '[hlL]?[E-GXc-giorsaux%]', String.Interpol),
+            # the new style '{}'.format(...) string formatting
+            (r'\{'
+             r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?'  # field name
+             r'(\![sra])?'                       # conversion
+             r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
+             r'\}', String.Interpol),
+
+            # backslashes, quotes and formatting signs must be parsed one at a time
+            (r'[^\\\'"%{\n]+', ttype),
+            (r'[\'"\\]', ttype),
+            # unhandled string formatting sign
+            (r'%|(\{{1,2})', ttype)
+            # newlines are an error (use "nl" state)
+        ]
+
+    def fstring_rules(ttype):
+        return [
+            # Assuming that a '}' is the closing brace after format specifier.
+            # Sadly, this means that we won't detect syntax error. But it's
+            # more important to parse correct syntax correctly, than to
+            # highlight invalid syntax.
+            (r'\}', String.Interpol),
+            (r'\{', String.Interpol, 'expr-inside-fstring'),
+            # backslashes, quotes and formatting signs must be parsed one at a time
+            (r'[^\\\'"{}\n]+', ttype),
+            (r'[\'"\\]', ttype),
+            # newlines are an error (use "nl" state)
+        ]
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
+             bygroups(Whitespace, String.Affix, String.Doc)),
+            (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
+             bygroups(Whitespace, String.Affix, String.Doc)),
+            (r'\A#!.+$', Comment.Hashbang),
+            (r'#.*$', Comment.Single),
+            (r'\\\n', Text),
+            (r'\\', Text),
+            include('keywords'),
+            include('soft-keywords'),
+            (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'funcname'),
+            (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'classname'),
+            (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
+             'fromimport'),
+            (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
+             'import'),
+            include('expr'),
+        ],
+        'expr': [
+            # raw f-strings and t-strings
+            ('(?i)(r[ft]|[ft]r)(""")',
+             bygroups(String.Affix, String.Double),
+             combined('rfstringescape', 'tdqf')),
+            ("(?i)(r[ft]|[ft]r)(''')",
+             bygroups(String.Affix, String.Single),
+             combined('rfstringescape', 'tsqf')),
+            ('(?i)(r[ft]|[ft]r)(")',
+             bygroups(String.Affix, String.Double),
+             combined('rfstringescape', 'dqf')),
+            ("(?i)(r[ft]|[ft]r)(')",
+             bygroups(String.Affix, String.Single),
+             combined('rfstringescape', 'sqf')),
+            # non-raw f-strings and t-strings
+            ('([fFtT])(""")', bygroups(String.Affix, String.Double),
+             combined('fstringescape', 'tdqf')),
+            ("([fFtT])(''')", bygroups(String.Affix, String.Single),
+             combined('fstringescape', 'tsqf')),
+            ('([fFtT])(")', bygroups(String.Affix, String.Double),
+             combined('fstringescape', 'dqf')),
+            ("([fFtT])(')", bygroups(String.Affix, String.Single),
+             combined('fstringescape', 'sqf')),
+            # raw bytes and strings
+            ('(?i)(rb|br|r)(""")',
+             bygroups(String.Affix, String.Double), 'tdqs'),
+            ("(?i)(rb|br|r)(''')",
+             bygroups(String.Affix, String.Single), 'tsqs'),
+            ('(?i)(rb|br|r)(")',
+             bygroups(String.Affix, String.Double), 'dqs'),
+            ("(?i)(rb|br|r)(')",
+             bygroups(String.Affix, String.Single), 'sqs'),
+            # non-raw strings
+            ('([uU]?)(""")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'tdqs')),
+            ("([uU]?)(''')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'tsqs')),
+            ('([uU]?)(")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'dqs')),
+            ("([uU]?)(')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'sqs')),
+            # non-raw bytes
+            ('([bB])(""")', bygroups(String.Affix, String.Double),
+             combined('bytesescape', 'tdqs')),
+            ("([bB])(''')", bygroups(String.Affix, String.Single),
+             combined('bytesescape', 'tsqs')),
+            ('([bB])(")', bygroups(String.Affix, String.Double),
+             combined('bytesescape', 'dqs')),
+            ("([bB])(')", bygroups(String.Affix, String.Single),
+             combined('bytesescape', 'sqs')),
+
+            (r'[^\S\n]+', Text),
+            include('numbers'),
+            (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator),
+            (r'[]{}:(),;[]', Punctuation),
+            (r'(in|is|and|or|not)\b', Operator.Word),
+            include('expr-keywords'),
+            include('builtins'),
+            include('magicfuncs'),
+            include('magicvars'),
+            include('name'),
+        ],
+        'expr-inside-fstring': [
+            (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
+            # without format specifier
+            (r'(=\s*)?'         # debug (https://bugs.python.org/issue36817)
+             r'(\![sraf])?'     # conversion
+             r'\}', String.Interpol, '#pop'),
+            # with format specifier
+            # we'll catch the remaining '}' in the outer scope
+            (r'(=\s*)?'         # debug (https://bugs.python.org/issue36817)
+             r'(\![sraf])?'     # conversion
+             r':', String.Interpol, '#pop'),
+            (r'\s+', Whitespace),  # allow new lines
+            include('expr'),
+        ],
+        'expr-inside-fstring-inner': [
+            (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
+            (r'[])}]', Punctuation, '#pop'),
+            (r'\s+', Whitespace),  # allow new lines
+            include('expr'),
+        ],
+        'expr-keywords': [
+            # Based on https://docs.python.org/3/reference/expressions.html
+            (words((
+                'async for', 'await', 'else', 'for', 'if', 'lambda',
+                'yield', 'yield from'), suffix=r'\b'),
+             Keyword),
+            (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
+        ],
+        'keywords': [
+            (words((
+                'assert', 'async', 'await', 'break', 'continue', 'del', 'elif',
+                'else', 'except', 'finally', 'for', 'global', 'if', 'lambda',
+                'pass', 'raise', 'nonlocal', 'return', 'try', 'while', 'yield',
+                'yield from', 'as', 'with'), suffix=r'\b'),
+             Keyword),
+            (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
+        ],
+        'soft-keywords': [
+            # `match`, `case` and `_` soft keywords
+            (r'(^[ \t]*)'              # at beginning of line + possible indentation
+             r'(match|case)\b'         # a possible keyword
+             r'(?![ \t]*(?:'           # not followed by...
+             r'[:,;=^&|@~)\]}]|(?:' +  # characters and keywords that mean this isn't
+                                       # pattern matching (but None/True/False is ok)
+             r'|'.join(k for k in keyword.kwlist if k[0].islower()) + r')\b))',
+             bygroups(Text, Keyword), 'soft-keywords-inner'),
+        ],
+        'soft-keywords-inner': [
+            # optional `_` keyword
+            (r'(\s+)([^\n_]*)(_\b)', bygroups(Whitespace, using(this), Keyword)),
+            default('#pop')
+        ],
+        'builtins': [
+            (words((
+                '__import__', 'abs', 'aiter', 'all', 'any', 'bin', 'bool', 'bytearray',
+                'breakpoint', 'bytes', 'callable', 'chr', 'classmethod', 'compile',
+                'complex', 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval',
+                'filter', 'float', 'format', 'frozenset', 'getattr', 'globals',
+                'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'isinstance',
+                'issubclass', 'iter', 'len', 'list', 'locals', 'map', 'max',
+                'memoryview', 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow',
+                'print', 'property', 'range', 'repr', 'reversed', 'round', 'set',
+                'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super',
+                'tuple', 'type', 'vars', 'zip'), prefix=r'(?>|[-~+/*%=<>&^|.]', Operator),
+            include('keywords'),
+            (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'funcname'),
+            (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Whitespace), 'classname'),
+            (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
+             'fromimport'),
+            (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Whitespace),
+             'import'),
+            include('builtins'),
+            include('magicfuncs'),
+            include('magicvars'),
+            include('backtick'),
+            ('([rR]|[uUbB][rR]|[rR][uUbB])(""")',
+             bygroups(String.Affix, String.Double), 'tdqs'),
+            ("([rR]|[uUbB][rR]|[rR][uUbB])(''')",
+             bygroups(String.Affix, String.Single), 'tsqs'),
+            ('([rR]|[uUbB][rR]|[rR][uUbB])(")',
+             bygroups(String.Affix, String.Double), 'dqs'),
+            ("([rR]|[uUbB][rR]|[rR][uUbB])(')",
+             bygroups(String.Affix, String.Single), 'sqs'),
+            ('([uUbB]?)(""")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'tdqs')),
+            ("([uUbB]?)(''')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'tsqs')),
+            ('([uUbB]?)(")', bygroups(String.Affix, String.Double),
+             combined('stringescape', 'dqs')),
+            ("([uUbB]?)(')", bygroups(String.Affix, String.Single),
+             combined('stringescape', 'sqs')),
+            include('name'),
+            include('numbers'),
+        ],
+        'keywords': [
+            (words((
+                'assert', 'break', 'continue', 'del', 'elif', 'else', 'except',
+                'exec', 'finally', 'for', 'global', 'if', 'lambda', 'pass',
+                'print', 'raise', 'return', 'try', 'while', 'yield',
+                'yield from', 'as', 'with'), suffix=r'\b'),
+             Keyword),
+        ],
+        'builtins': [
+            (words((
+                '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin',
+                'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod',
+                'cmp', 'coerce', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod',
+                'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float',
+                'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id',
+                'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len',
+                'list', 'locals', 'long', 'map', 'max', 'min', 'next', 'object',
+                'oct', 'open', 'ord', 'pow', 'property', 'range', 'raw_input', 'reduce',
+                'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice',
+                'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type',
+                'unichr', 'unicode', 'vars', 'xrange', 'zip'),
+                prefix=r'(?>> )(.*\n)', bygroups(Generic.Prompt, Other.Code), 'continuations'),
+            # This happens, e.g., when tracebacks are embedded in documentation;
+            # trailing whitespaces are often stripped in such contexts.
+            (r'(>>>)(\n)', bygroups(Generic.Prompt, Whitespace)),
+            (r'(\^C)?Traceback \(most recent call last\):\n', Other.Traceback, 'traceback'),
+            # SyntaxError starts with this
+            (r'  File "[^"]+", line \d+', Other.Traceback, 'traceback'),
+            (r'.*\n', Generic.Output),
+        ],
+        'continuations': [
+            (r'(\.\.\. )(.*\n)', bygroups(Generic.Prompt, Other.Code)),
+            # See above.
+            (r'(\.\.\.)(\n)', bygroups(Generic.Prompt, Whitespace)),
+            default('#pop'),
+        ],
+        'traceback': [
+            # As soon as we see a traceback, consume everything until the next
+            # >>> prompt.
+            (r'(?=>>>( |$))', Text, '#pop'),
+            (r'(KeyboardInterrupt)(\n)', bygroups(Name.Class, Whitespace)),
+            (r'.*\n', Other.Traceback),
+        ],
+    }
+
+
+class PythonConsoleLexer(DelegatingLexer):
+    """
+    For Python console output or doctests, such as:
+
+    .. sourcecode:: pycon
+
+        >>> a = 'foo'
+        >>> print(a)
+        foo
+        >>> 1 / 0
+        Traceback (most recent call last):
+          File "", line 1, in 
+        ZeroDivisionError: integer division or modulo by zero
+
+    Additional options:
+
+    `python3`
+        Use Python 3 lexer for code.  Default is ``True``.
+
+        .. versionadded:: 1.0
+        .. versionchanged:: 2.5
+           Now defaults to ``True``.
+    """
+
+    name = 'Python console session'
+    aliases = ['pycon', 'python-console']
+    mimetypes = ['text/x-python-doctest']
+    url = 'https://python.org'
+    version_added = ''
+
+    def __init__(self, **options):
+        python3 = get_bool_opt(options, 'python3', True)
+        if python3:
+            pylexer = PythonLexer
+            tblexer = PythonTracebackLexer
+        else:
+            pylexer = Python2Lexer
+            tblexer = Python2TracebackLexer
+        # We have two auxiliary lexers. Use DelegatingLexer twice with
+        # different tokens.  TODO: DelegatingLexer should support this
+        # directly, by accepting a tuplet of auxiliary lexers and a tuple of
+        # distinguishing tokens. Then we wouldn't need this intermediary
+        # class.
+        class _ReplaceInnerCode(DelegatingLexer):
+            def __init__(self, **options):
+                super().__init__(pylexer, _PythonConsoleLexerBase, Other.Code, **options)
+        super().__init__(tblexer, _ReplaceInnerCode, Other.Traceback, **options)
+
+
+class PythonTracebackLexer(RegexLexer):
+    """
+    For Python 3.x tracebacks, with support for chained exceptions.
+
+    .. versionchanged:: 2.5
+       This is now the default ``PythonTracebackLexer``.  It is still available
+       as the alias ``Python3TracebackLexer``.
+    """
+
+    name = 'Python Traceback'
+    aliases = ['pytb', 'py3tb']
+    filenames = ['*.pytb', '*.py3tb']
+    mimetypes = ['text/x-python-traceback', 'text/x-python3-traceback']
+    url = 'https://python.org'
+    version_added = '1.0'
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'^(\^C)?Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'),
+            (r'^During handling of the above exception, another '
+             r'exception occurred:\n\n', Generic.Traceback),
+            (r'^The above exception was the direct cause of the '
+             r'following exception:\n\n', Generic.Traceback),
+            (r'^(?=  File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
+            (r'^.*\n', Other),
+        ],
+        'intb': [
+            (r'^(  File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
+             bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
+            (r'^(  File )("[^"]+")(, line )(\d+)(\n)',
+             bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
+            (r'^(    )(.+)(\n)',
+             bygroups(Whitespace, using(PythonLexer), Whitespace), 'markers'),
+            (r'^([ \t]*)(\.\.\.)(\n)',
+             bygroups(Whitespace, Comment, Whitespace)),  # for doctests...
+            (r'^([^:]+)(: )(.+)(\n)',
+             bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
+            (r'^([a-zA-Z_][\w.]*)(:?\n)',
+             bygroups(Generic.Error, Whitespace), '#pop'),
+            default('#pop'),
+        ],
+        'markers': [
+            # Either `PEP 657 `
+            # error locations in Python 3.11+, or single-caret markers
+            # for syntax errors before that.
+            (r'^( {4,})([~^]+)(\n)',
+             bygroups(Whitespace, Punctuation.Marker, Whitespace),
+             '#pop'),
+            default('#pop'),
+        ],
+    }
+
+
+Python3TracebackLexer = PythonTracebackLexer
+
+
+class Python2TracebackLexer(RegexLexer):
+    """
+    For Python tracebacks.
+
+    .. versionchanged:: 2.5
+       This class has been renamed from ``PythonTracebackLexer``.
+       ``PythonTracebackLexer`` now refers to the Python 3 variant.
+    """
+
+    name = 'Python 2.x Traceback'
+    aliases = ['py2tb']
+    filenames = ['*.py2tb']
+    mimetypes = ['text/x-python2-traceback']
+    url = 'https://python.org'
+    version_added = '0.7'
+
+    tokens = {
+        'root': [
+            # Cover both (most recent call last) and (innermost last)
+            # The optional ^C allows us to catch keyboard interrupt signals.
+            (r'^(\^C)?(Traceback.*\n)',
+             bygroups(Text, Generic.Traceback), 'intb'),
+            # SyntaxError starts with this.
+            (r'^(?=  File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
+            (r'^.*\n', Other),
+        ],
+        'intb': [
+            (r'^(  File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
+             bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
+            (r'^(  File )("[^"]+")(, line )(\d+)(\n)',
+             bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
+            (r'^(    )(.+)(\n)',
+             bygroups(Text, using(Python2Lexer), Whitespace), 'marker'),
+            (r'^([ \t]*)(\.\.\.)(\n)',
+             bygroups(Text, Comment, Whitespace)),  # for doctests...
+            (r'^([^:]+)(: )(.+)(\n)',
+             bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
+            (r'^([a-zA-Z_]\w*)(:?\n)',
+             bygroups(Generic.Error, Whitespace), '#pop')
+        ],
+        'marker': [
+            # For syntax errors.
+            (r'( {4,})(\^)', bygroups(Text, Punctuation.Marker), '#pop'),
+            default('#pop'),
+        ],
+    }
+
+
+class CythonLexer(RegexLexer):
+    """
+    For Pyrex and Cython source code.
+    """
+
+    name = 'Cython'
+    url = 'https://cython.org'
+    aliases = ['cython', 'pyx', 'pyrex']
+    filenames = ['*.pyx', '*.pxd', '*.pxi']
+    mimetypes = ['text/x-cython', 'application/x-cython']
+    version_added = '1.1'
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Whitespace, String.Doc)),
+            (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Whitespace, String.Doc)),
+            (r'[^\S\n]+', Text),
+            (r'#.*$', Comment),
+            (r'[]{}:(),;[]', Punctuation),
+            (r'\\\n', Whitespace),
+            (r'\\', Text),
+            (r'(in|is|and|or|not)\b', Operator.Word),
+            (r'(<)([a-zA-Z0-9.?]+)(>)',
+             bygroups(Punctuation, Keyword.Type, Punctuation)),
+            (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator),
+            (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)',
+             bygroups(Keyword, Number.Integer, Operator, Whitespace, Operator,
+                      Name, Punctuation)),
+            include('keywords'),
+            (r'(def|property)(\s+)', bygroups(Keyword, Whitespace), 'funcname'),
+            (r'(cp?def)(\s+)', bygroups(Keyword, Whitespace), 'cdef'),
+            # (should actually start a block with only cdefs)
+            (r'(cdef)(:)', bygroups(Keyword, Punctuation)),
+            (r'(class|cppclass|struct)(\s+)', bygroups(Keyword, Whitespace), 'classname'),
+            (r'(from)(\s+)', bygroups(Keyword, Whitespace), 'fromimport'),
+            (r'(c?import)(\s+)', bygroups(Keyword, Whitespace), 'import'),
+            include('builtins'),
+            include('backtick'),
+            ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'),
+            ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'),
+            ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'),
+            ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'),
+            ('[uU]?"""', String, combined('stringescape', 'tdqs')),
+            ("[uU]?'''", String, combined('stringescape', 'tsqs')),
+            ('[uU]?"', String, combined('stringescape', 'dqs')),
+            ("[uU]?'", String, combined('stringescape', 'sqs')),
+            include('name'),
+            include('numbers'),
+        ],
+        'keywords': [
+            (words((
+                'assert', 'async', 'await', 'break', 'by', 'continue', 'ctypedef', 'del',
+                'elif', 'else', 'except', 'except?', 'exec', 'finally', 'for', 'fused', 'gil',
+                'global', 'if', 'include', 'lambda', 'namespace', 'new', 'noexcept','nogil',
+                'pass', 'print', 'raise', 'return', 'try', 'while', 'yield', 'as', 'with'),
+             suffix=r'\b'),
+             Keyword),
+             (words(('True', 'False', 'None', 'NULL'), suffix=r'\b'), Keyword.Constant),
+            (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc),
+        ],
+        'builtins': [
+            (words((
+                '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', 'bint',
+                'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'char', 'chr',
+                'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr',
+                'dict', 'dir', 'divmod', 'double', 'enumerate', 'eval', 'execfile', 'exit',
+                'file', 'filter', 'float', 'frozenset', 'getattr', 'globals',
+                'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance',
+                'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max',
+                'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property',
+                'Py_ssize_t', 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed',
+                'round', 'set', 'setattr', 'size_t', 'slice', 'sorted', 'staticmethod',
+                'ssize_t', 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode',
+                'unsigned', 'vars', 'xrange', 'zip'), prefix=r'(??/\\:']?:)(\s*)(\{)",
+             bygroups(Name.Function, Whitespace, Operator, Whitespace, Punctuation),
+             "functions"),
+            # Variable Names
+            (r"([.]?[a-zA-Z][\w.]*)(\s*)([-.~=!@#$%^&*_+|,<>?/\\:']?:)",
+             bygroups(Name.Variable, Whitespace, Operator)),
+            # Functions
+            (r"\{", Punctuation, "functions"),
+            # Parentheses
+            (r"\(", Punctuation, "parentheses"),
+            # Brackets
+            (r"\[", Punctuation, "brackets"),
+            # Errors
+            (r"'`([a-zA-Z][\w.]*)?", Name.Exception),
+            # File Symbols
+            (r"`:([a-zA-Z/][\w./]*)?", String.Symbol),
+            # Symbols
+            (r"`([a-zA-Z][\w.]*)?", String.Symbol),
+            # Numbers
+            include("numbers"),
+            # Variable Names
+            (r"[a-zA-Z][\w.]*", Name),
+            # Operators
+            (r"[-=+*#$%@!~^&:.,<>'\\|/?_]", Operator),
+            # Punctuation
+            (r";", Punctuation),
+        ],
+        "functions": [
+            include("root"),
+            (r"\}", Punctuation, "#pop"),
+        ],
+        "parentheses": [
+            include("root"),
+            (r"\)", Punctuation, "#pop"),
+        ],
+        "brackets": [
+            include("root"),
+            (r"\]", Punctuation, "#pop"),
+        ],
+        "numbers": [
+            # Binary Values
+            (r"[01]+b", Number.Bin),
+            # Nulls/Infinities
+            (r"0[nNwW][cefghijmndzuvtp]?", Number),
+            # Timestamps
+            ((r"(?:[0-9]{4}[.][0-9]{2}[.][0-9]{2}|[0-9]+)"
+              "D(?:[0-9](?:[0-9](?::[0-9]{2}"
+              "(?::[0-9]{2}(?:[.][0-9]*)?)?)?)?)?"), Literal.Date),
+            # Datetimes
+            ((r"[0-9]{4}[.][0-9]{2}"
+              "(?:m|[.][0-9]{2}(?:T(?:[0-9]{2}:[0-9]{2}"
+              "(?::[0-9]{2}(?:[.][0-9]*)?)?)?)?)"), Literal.Date),
+            # Times
+            (r"[0-9]{2}:[0-9]{2}(?::[0-9]{2}(?:[.][0-9]{1,3})?)?",
+             Literal.Date),
+            # GUIDs
+            (r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
+             Number.Hex),
+            # Byte Vectors
+            (r"0x[0-9a-fA-F]+", Number.Hex),
+            # Floats
+            (r"([0-9]*[.]?[0-9]+|[0-9]+[.]?[0-9]*)[eE][+-]?[0-9]+[ef]?",
+             Number.Float),
+            (r"([0-9]*[.][0-9]+|[0-9]+[.][0-9]*)[ef]?", Number.Float),
+            (r"[0-9]+[ef]", Number.Float),
+            # Characters
+            (r"[0-9]+c", Number),
+            # Integers
+            (r"[0-9]+[ihtuv]", Number.Integer),
+            # Long Integers
+            (r"[0-9]+[jnp]?", Number.Integer.Long),
+        ],
+        "comments": [
+            (r"[^\\]+", Comment.Multiline),
+            (r"^\\", Comment.Multiline, "#pop"),
+            (r"\\", Comment.Multiline),
+        ],
+        "strings": [
+            (r'[^"\\]+', String.Double),
+            (r"\\.", String.Escape),
+            (r'"', String.Double, "#pop"),
+        ],
+    }
+
+
+class QLexer(KLexer):
+    """
+    For `Q `_ source code.
+    """
+
+    name = "Q"
+    aliases = ["q"]
+    filenames = ["*.q"]
+    version_added = '2.12'
+
+    tokens = {
+        "root": [
+            (words(("aj", "aj0", "ajf", "ajf0", "all", "and", "any", "asc",
+                    "asof", "attr", "avgs", "ceiling", "cols", "count", "cross",
+                    "csv", "cut", "deltas", "desc", "differ", "distinct", "dsave",
+                    "each", "ej", "ema", "eval", "except", "fby", "fills", "first",
+                    "fkeys", "flip", "floor", "get", "group", "gtime", "hclose",
+                    "hcount", "hdel", "hsym", "iasc", "idesc", "ij", "ijf",
+                    "inter", "inv", "key", "keys", "lj", "ljf", "load", "lower",
+                    "lsq", "ltime", "ltrim", "mavg", "maxs", "mcount", "md5",
+                    "mdev", "med", "meta", "mins", "mmax", "mmin", "mmu", "mod",
+                    "msum", "neg", "next", "not", "null", "or", "over", "parse",
+                    "peach", "pj", "prds", "prior", "prev", "rand", "rank", "ratios",
+                    "raze", "read0", "read1", "reciprocal", "reval", "reverse",
+                    "rload", "rotate", "rsave", "rtrim", "save", "scan", "scov",
+                    "sdev", "set", "show", "signum", "ssr", "string", "sublist",
+                    "sums", "sv", "svar", "system", "tables", "til", "trim", "txf",
+                    "type", "uj", "ujf", "ungroup", "union", "upper", "upsert",
+                    "value", "view", "views", "vs", "where", "wj", "wj1", "ww",
+                    "xasc", "xbar", "xcol", "xcols", "xdesc", "xgroup", "xkey",
+                    "xlog", "xprev", "xrank"),
+                    suffix=r"\b"), Name.Builtin,
+            ),
+            inherit,
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/qlik.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/qlik.py
new file mode 100644
index 0000000000000000000000000000000000000000..6972ed9d03bf447f6215fbba65e96eb4455a8248
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/qlik.py
@@ -0,0 +1,117 @@
+"""
+    pygments.lexers.qlik
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the qlik scripting language
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, \
+    Punctuation, String, Text
+from pygments.lexers._qlik_builtins import OPERATORS_LIST, STATEMENT_LIST, \
+    SCRIPT_FUNCTIONS, CONSTANT_LIST
+
+__all__ = ["QlikLexer"]
+
+
+class QlikLexer(RegexLexer):
+    """
+    Lexer for qlik code, including .qvs files
+    """
+
+    name = "Qlik"
+    aliases = ["qlik", "qlikview", "qliksense", "qlikscript"]
+    filenames = ["*.qvs", "*.qvw"]
+    url = "https://qlik.com"
+    version_added = '2.12'
+
+    flags = re.IGNORECASE
+
+    tokens = {
+        # Handle multi-line comments
+        "comment": [
+            (r"\*/", Comment.Multiline, "#pop"),
+            (r"[^*]+", Comment.Multiline),
+        ],
+        # Handle numbers
+        "numerics": [
+            (r"\b\d+\.\d+(e\d+)?[fd]?\b", Number.Float),
+            (r"\b\d+\b", Number.Integer),
+        ],
+        # Handle variable names in things
+        "interp": [
+            (
+                r"(\$\()(\w+)(\))",
+                bygroups(String.Interpol, Name.Variable, String.Interpol),
+            ),
+        ],
+        # Handle strings
+        "string": [
+            (r"'", String, "#pop"),
+            include("interp"),
+            (r"[^'$]+", String),
+            (r"\$", String),
+        ],
+        #
+        "assignment": [
+            (r";", Punctuation, "#pop"),
+            include("root"),
+        ],
+        "field_name_quote": [
+            (r'"', String.Symbol, "#pop"),
+            include("interp"),
+            (r"[^\"$]+", String.Symbol),
+            (r"\$", String.Symbol),
+        ],
+        "field_name_bracket": [
+            (r"\]", String.Symbol, "#pop"),
+            include("interp"),
+            (r"[^\]$]+", String.Symbol),
+            (r"\$", String.Symbol),
+        ],
+        "function": [(r"\)", Punctuation, "#pop"), include("root")],
+        "root": [
+            # Whitespace and comments
+            (r"\s+", Text.Whitespace),
+            (r"/\*", Comment.Multiline, "comment"),
+            (r"//.*\n", Comment.Single),
+            # variable assignment
+            (r"(let|set)(\s+)", bygroups(Keyword.Declaration, Text.Whitespace),
+             "assignment"),
+            # Word operators
+            (words(OPERATORS_LIST["words"], prefix=r"\b", suffix=r"\b"),
+             Operator.Word),
+            # Statements
+            (words(STATEMENT_LIST, suffix=r"\b"), Keyword),
+            # Table names
+            (r"[a-z]\w*:", Keyword.Declaration),
+            # Constants
+            (words(CONSTANT_LIST, suffix=r"\b"), Keyword.Constant),
+            # Functions
+            (words(SCRIPT_FUNCTIONS, suffix=r"(?=\s*\()"), Name.Builtin,
+             "function"),
+            # interpolation - e.g. $(variableName)
+            include("interp"),
+            # Quotes denote a field/file name
+            (r'"', String.Symbol, "field_name_quote"),
+            # Square brackets denote a field/file name
+            (r"\[", String.Symbol, "field_name_bracket"),
+            # Strings
+            (r"'", String, "string"),
+            # Numbers
+            include("numerics"),
+            # Operator symbols
+            (words(OPERATORS_LIST["symbols"]), Operator),
+            # Strings denoted by single quotes
+            (r"'.+?'", String),
+            # Words as text
+            (r"\b\w+\b", Text),
+            # Basic punctuation
+            (r"[,;.()\\/]", Punctuation),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/qvt.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/qvt.py
new file mode 100644
index 0000000000000000000000000000000000000000..02d6fcf7d21003379377858820df83389a2b21c5
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/qvt.py
@@ -0,0 +1,153 @@
+"""
+    pygments.lexers.qvt
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for QVT Operational language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, include, combined, default, \
+    words
+from pygments.token import Text, Comment, Operator, Keyword, Punctuation, \
+    Name, String, Number
+
+__all__ = ['QVToLexer']
+
+
+class QVToLexer(RegexLexer):
+    """
+    For the QVT Operational Mapping language.
+
+    Reference for implementing this: «Meta Object Facility (MOF) 2.0
+    Query/View/Transformation Specification», Version 1.1 - January 2011
+    (https://www.omg.org/spec/QVT/1.1/), see §8.4, «Concrete Syntax» in
+    particular.
+
+    Notable tokens assignments:
+
+    - Name.Class is assigned to the identifier following any of the following
+      keywords: metamodel, class, exception, primitive, enum, transformation
+      or library
+
+    - Name.Function is assigned to the names of mappings and queries
+
+    - Name.Builtin.Pseudo is assigned to the pre-defined variables 'this',
+      'self' and 'result'.
+    """
+    # With obvious borrowings & inspiration from the Java, Python and C lexers
+
+    name = 'QVTO'
+    aliases = ['qvto', 'qvt']
+    filenames = ['*.qvto']
+    url = 'https://www.omg.org/spec/QVT/1.1'
+    version_added = ''
+
+    tokens = {
+        'root': [
+            (r'\n', Text),
+            (r'[^\S\n]+', Text),
+            (r'(--|//)(\s*)(directive:)?(.*)$',
+             bygroups(Comment, Comment, Comment.Preproc, Comment)),
+            # Uncomment the following if you want to distinguish between
+            # '/*' and '/**', à la javadoc
+            # (r'/[*]{2}(.|\n)*?[*]/', Comment.Multiline),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+            (r'\\\n', Text),
+            (r'(and|not|or|xor|##?)\b', Operator.Word),
+            (r'(:{1,2}=|[-+]=)\b', Operator.Word),
+            (r'(@|<<|>>)\b', Keyword),  # stereotypes
+            (r'!=|<>|==|=|!->|->|>=|<=|[.]{3}|[+/*%=<>&|.~]', Operator),
+            (r'[]{}:(),;[]', Punctuation),
+            (r'(true|false|unlimited|null)\b', Keyword.Constant),
+            (r'(this|self|result)\b', Name.Builtin.Pseudo),
+            (r'(var)\b', Keyword.Declaration),
+            (r'(from|import)\b', Keyword.Namespace, 'fromimport'),
+            (r'(metamodel|class|exception|primitive|enum|transformation|'
+             r'library)(\s+)(\w+)',
+             bygroups(Keyword.Word, Text, Name.Class)),
+            (r'(exception)(\s+)(\w+)',
+             bygroups(Keyword.Word, Text, Name.Exception)),
+            (r'(main)\b', Name.Function),
+            (r'(mapping|helper|query)(\s+)',
+             bygroups(Keyword.Declaration, Text), 'operation'),
+            (r'(assert)(\s+)\b', bygroups(Keyword, Text), 'assert'),
+            (r'(Bag|Collection|Dict|OrderedSet|Sequence|Set|Tuple|List)\b',
+             Keyword.Type),
+            include('keywords'),
+            ('"', String, combined('stringescape', 'dqs')),
+            ("'", String, combined('stringescape', 'sqs')),
+            include('name'),
+            include('numbers'),
+            # (r'([a-zA-Z_]\w*)(::)([a-zA-Z_]\w*)',
+            # bygroups(Text, Text, Text)),
+        ],
+
+        'fromimport': [
+            (r'(?:[ \t]|\\\n)+', Text),
+            (r'[a-zA-Z_][\w.]*', Name.Namespace),
+            default('#pop'),
+        ],
+
+        'operation': [
+            (r'::', Text),
+            (r'(.*::)([a-zA-Z_]\w*)([ \t]*)(\()',
+             bygroups(Text, Name.Function, Text, Punctuation), '#pop')
+        ],
+
+        'assert': [
+            (r'(warning|error|fatal)\b', Keyword, '#pop'),
+            default('#pop'),  # all else: go back
+        ],
+
+        'keywords': [
+            (words((
+                'abstract', 'access', 'any', 'assert', 'blackbox', 'break',
+                'case', 'collect', 'collectNested', 'collectOne', 'collectselect',
+                'collectselectOne', 'composes', 'compute', 'configuration',
+                'constructor', 'continue', 'datatype', 'default', 'derived',
+                'disjuncts', 'do', 'elif', 'else', 'end', 'endif', 'except',
+                'exists', 'extends', 'forAll', 'forEach', 'forOne', 'from', 'if',
+                'implies', 'in', 'inherits', 'init', 'inout', 'intermediate',
+                'invresolve', 'invresolveIn', 'invresolveone', 'invresolveoneIn',
+                'isUnique', 'iterate', 'late', 'let', 'literal', 'log', 'map',
+                'merges', 'modeltype', 'new', 'object', 'one', 'ordered', 'out',
+                'package', 'population', 'property', 'raise', 'readonly',
+                'references', 'refines', 'reject', 'resolve', 'resolveIn',
+                'resolveone', 'resolveoneIn', 'return', 'select', 'selectOne',
+                'sortedBy', 'static', 'switch', 'tag', 'then', 'try', 'typedef',
+                'unlimited', 'uses', 'when', 'where', 'while', 'with', 'xcollect',
+                'xmap', 'xselect'), suffix=r'\b'), Keyword),
+        ],
+
+        # There is no need to distinguish between String.Single and
+        # String.Double: 'strings' is factorised for 'dqs' and 'sqs'
+        'strings': [
+            (r'[^\\\'"\n]+', String),
+            # quotes, percents and backslashes must be parsed one at a time
+            (r'[\'"\\]', String),
+        ],
+        'stringescape': [
+            (r'\\([\\btnfr"\']|u[0-3][0-7]{2}|u[0-7]{1,2})', String.Escape)
+        ],
+        'dqs': [  # double-quoted string
+            (r'"', String, '#pop'),
+            (r'\\\\|\\"', String.Escape),
+            include('strings')
+        ],
+        'sqs': [  # single-quoted string
+            (r"'", String, '#pop'),
+            (r"\\\\|\\'", String.Escape),
+            include('strings')
+        ],
+        'name': [
+            (r'[a-zA-Z_]\w*', Name),
+        ],
+        # numbers: excerpt taken from the python lexer
+        'numbers': [
+            (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+            (r'\d+[eE][+-]?[0-9]+', Number.Float),
+            (r'\d+', Number.Integer)
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/r.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/r.py
new file mode 100644
index 0000000000000000000000000000000000000000..7dcc148a9cb52683e928a307444dd5bfd311a45f
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/r.py
@@ -0,0 +1,196 @@
+"""
+    pygments.lexers.r
+    ~~~~~~~~~~~~~~~~~
+
+    Lexers for the R/S languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, include, do_insertions
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Generic, Whitespace
+
+__all__ = ['RConsoleLexer', 'SLexer', 'RdLexer']
+
+
+line_re  = re.compile('.*?\n')
+
+
+class RConsoleLexer(Lexer):
+    """
+    For R console transcripts or R CMD BATCH output files.
+    """
+
+    name = 'RConsole'
+    aliases = ['rconsole', 'rout']
+    filenames = ['*.Rout']
+    url = 'https://www.r-project.org'
+    version_added = ''
+    _example = "rconsole/r-console-transcript.Rout"
+
+    def get_tokens_unprocessed(self, text):
+        slexer = SLexer(**self.options)
+
+        current_code_block = ''
+        insertions = []
+
+        for match in line_re.finditer(text):
+            line = match.group()
+            if line.startswith('>') or line.startswith('+'):
+                # Colorize the prompt as such,
+                # then put rest of line into current_code_block
+                insertions.append((len(current_code_block),
+                                   [(0, Generic.Prompt, line[:2])]))
+                current_code_block += line[2:]
+            else:
+                # We have reached a non-prompt line!
+                # If we have stored prompt lines, need to process them first.
+                if current_code_block:
+                    # Weave together the prompts and highlight code.
+                    yield from do_insertions(
+                        insertions, slexer.get_tokens_unprocessed(current_code_block))
+                    # Reset vars for next code block.
+                    current_code_block = ''
+                    insertions = []
+                # Now process the actual line itself, this is output from R.
+                yield match.start(), Generic.Output, line
+
+        # If we happen to end on a code block with nothing after it, need to
+        # process the last code block. This is neither elegant nor DRY so
+        # should be changed.
+        if current_code_block:
+            yield from do_insertions(
+                insertions, slexer.get_tokens_unprocessed(current_code_block))
+
+
+class SLexer(RegexLexer):
+    """
+    For S, S-plus, and R source code.
+    """
+
+    name = 'S'
+    aliases = ['splus', 's', 'r']
+    filenames = ['*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron']
+    mimetypes = ['text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r',
+                 'text/x-R', 'text/x-r-history', 'text/x-r-profile']
+    url = 'https://www.r-project.org'
+    version_added = '0.10'
+
+    valid_name = r'`[^`\\]*(?:\\.[^`\\]*)*`|(?:[a-zA-Z]|\.[A-Za-z_.])[\w.]*|\.'
+    tokens = {
+        'comments': [
+            (r'#.*$', Comment.Single),
+        ],
+        'valid_name': [
+            (valid_name, Name),
+        ],
+        'function_name': [
+            (rf'({valid_name})\s*(?=\()', Name.Function),
+        ],
+        'punctuation': [
+            (r'\[{1,2}|\]{1,2}|\(|\)|;|,', Punctuation),
+        ],
+        'keywords': [
+            (r'(if|else|for|while|repeat|in|next|break|return|switch|function)'
+             r'(?![\w.])',
+             Keyword.Reserved),
+        ],
+        'operators': [
+            (r'<>?|-|==|<=|>=|\|>|<|>|&&?|!=|\|\|?|\?', Operator),
+            (r'\*|\+|\^|/|!|%[^%]*%|=|~|\$|@|:{1,3}', Operator),
+        ],
+        'builtin_symbols': [
+            (r'(NULL|NA(_(integer|real|complex|character)_)?|'
+             r'letters|LETTERS|Inf|TRUE|FALSE|NaN|pi|\.\.(\.|[0-9]+))'
+             r'(?![\w.])',
+             Keyword.Constant),
+            (r'(T|F)\b', Name.Builtin.Pseudo),
+        ],
+        'numbers': [
+            # hex number
+            (r'0[xX][a-fA-F0-9]+([pP][0-9]+)?[Li]?', Number.Hex),
+            # decimal number
+            (r'[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)([eE][+-]?[0-9]+)?[Li]?',
+             Number),
+        ],
+        'statements': [
+            include('comments'),
+            # whitespaces
+            (r'\s+', Whitespace),
+            (r'\'', String, 'string_squote'),
+            (r'\"', String, 'string_dquote'),
+            include('builtin_symbols'),
+            include('keywords'),
+            include('function_name'),
+            include('valid_name'),
+            include('numbers'),
+            include('punctuation'),
+            include('operators'),
+        ],
+        'root': [
+            # calls:
+            include('statements'),
+            # blocks:
+            (r'\{|\}', Punctuation),
+            # (r'\{', Punctuation, 'block'),
+            (r'.', Text),
+        ],
+        # 'block': [
+        #    include('statements'),
+        #    ('\{', Punctuation, '#push'),
+        #    ('\}', Punctuation, '#pop')
+        # ],
+        'string_squote': [
+            (r'([^\'\\]|\\.)*\'', String, '#pop'),
+        ],
+        'string_dquote': [
+            (r'([^"\\]|\\.)*"', String, '#pop'),
+        ],
+    }
+
+    def analyse_text(text):
+        if re.search(r'[a-z0-9_\])\s]<-(?!-)', text):
+            return 0.11
+
+
+class RdLexer(RegexLexer):
+    """
+    Pygments Lexer for R documentation (Rd) files
+
+    This is a very minimal implementation, highlighting little more
+    than the macros. A description of Rd syntax is found in `Writing R
+    Extensions `_
+    and `Parsing Rd files `_.
+    """
+    name = 'Rd'
+    aliases = ['rd']
+    filenames = ['*.Rd']
+    mimetypes = ['text/x-r-doc']
+    url = 'http://cran.r-project.org/doc/manuals/R-exts.html'
+    version_added = '1.6'
+
+    # To account for verbatim / LaTeX-like / and R-like areas
+    # would require parsing.
+    tokens = {
+        'root': [
+            # catch escaped brackets and percent sign
+            (r'\\[\\{}%]', String.Escape),
+            # comments
+            (r'%.*$', Comment),
+            # special macros with no arguments
+            (r'\\(?:cr|l?dots|R|tab)\b', Keyword.Constant),
+            # macros
+            (r'\\[a-zA-Z]+\b', Keyword),
+            # special preprocessor macros
+            (r'^\s*#(?:ifn?def|endif).*\b', Comment.Preproc),
+            # non-escaped brackets
+            (r'[{}]', Name.Builtin),
+            # everything else
+            (r'[^\\%\n{}]+', Text),
+            (r'.', Text),
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rdf.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rdf.py
new file mode 100644
index 0000000000000000000000000000000000000000..f2ecf7a11f3072e3f009c22ebd0808905e2fbbcf
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rdf.py
@@ -0,0 +1,468 @@
+"""
+    pygments.lexers.rdf
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for semantic web and RDF query languages and markup.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, default
+from pygments.token import Keyword, Punctuation, String, Number, Operator, \
+    Generic, Whitespace, Name, Literal, Comment, Text
+
+__all__ = ['SparqlLexer', 'TurtleLexer', 'ShExCLexer']
+
+
+class SparqlLexer(RegexLexer):
+    """
+    Lexer for SPARQL query language.
+    """
+    name = 'SPARQL'
+    aliases = ['sparql']
+    filenames = ['*.rq', '*.sparql']
+    mimetypes = ['application/sparql-query']
+    url = 'https://www.w3.org/TR/sparql11-query'
+    version_added = '2.0'
+
+    # character group definitions ::
+
+    PN_CHARS_BASE_GRP = ('a-zA-Z'
+                         '\u00c0-\u00d6'
+                         '\u00d8-\u00f6'
+                         '\u00f8-\u02ff'
+                         '\u0370-\u037d'
+                         '\u037f-\u1fff'
+                         '\u200c-\u200d'
+                         '\u2070-\u218f'
+                         '\u2c00-\u2fef'
+                         '\u3001-\ud7ff'
+                         '\uf900-\ufdcf'
+                         '\ufdf0-\ufffd')
+
+    PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_')
+
+    PN_CHARS_GRP = (PN_CHARS_U_GRP +
+                    r'\-' +
+                    r'0-9' +
+                    '\u00b7' +
+                    '\u0300-\u036f' +
+                    '\u203f-\u2040')
+
+    HEX_GRP = '0-9A-Fa-f'
+
+    PN_LOCAL_ESC_CHARS_GRP = r' _~.\-!$&"()*+,;=/?#@%'
+
+    # terminal productions ::
+
+    PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']'
+
+    PN_CHARS_U = '[' + PN_CHARS_U_GRP + ']'
+
+    PN_CHARS = '[' + PN_CHARS_GRP + ']'
+
+    HEX = '[' + HEX_GRP + ']'
+
+    PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']'
+
+    IRIREF = r'<(?:[^<>"{}|^`\\\x00-\x20])*>'
+
+    BLANK_NODE_LABEL = '_:[0-9' + PN_CHARS_U_GRP + '](?:[' + PN_CHARS_GRP + \
+                       '.]*' + PN_CHARS + ')?'
+
+    PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?'
+
+    VARNAME = '[0-9' + PN_CHARS_U_GRP + '][' + PN_CHARS_U_GRP + \
+              '0-9\u00b7\u0300-\u036f\u203f-\u2040]*'
+
+    PERCENT = '%' + HEX + HEX
+
+    PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS
+
+    PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')'
+
+    PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' +
+                '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' +
+                PN_CHARS_GRP + ':]|' + PLX + '))?')
+
+    EXPONENT = r'[eE][+-]?\d+'
+
+    # Lexer token definitions ::
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            # keywords ::
+            (r'(?i)(select|construct|describe|ask|where|filter|group\s+by|minus|'
+             r'distinct|reduced|from\s+named|from|order\s+by|desc|asc|limit|'
+             r'offset|values|bindings|load|into|clear|drop|create|add|move|copy|'
+             r'insert\s+data|delete\s+data|delete\s+where|with|delete|insert|'
+             r'using\s+named|using|graph|default|named|all|optional|service|'
+             r'silent|bind|undef|union|not\s+in|in|as|having|to|prefix|base)\b', Keyword),
+            (r'(a)\b', Keyword),
+            # IRIs ::
+            ('(' + IRIREF + ')', Name.Label),
+            # blank nodes ::
+            ('(' + BLANK_NODE_LABEL + ')', Name.Label),
+            #  # variables ::
+            ('[?$]' + VARNAME, Name.Variable),
+            # prefixed names ::
+            (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + r')?',
+             bygroups(Name.Namespace, Punctuation, Name.Tag)),
+            # function names ::
+            (r'(?i)(str|lang|langmatches|datatype|bound|iri|uri|bnode|rand|abs|'
+             r'ceil|floor|round|concat|strlen|ucase|lcase|encode_for_uri|'
+             r'contains|strstarts|strends|strbefore|strafter|year|month|day|'
+             r'hours|minutes|seconds|timezone|tz|now|uuid|struuid|md5|sha1|sha256|sha384|'
+             r'sha512|coalesce|if|strlang|strdt|sameterm|isiri|isuri|isblank|'
+             r'isliteral|isnumeric|regex|substr|replace|exists|not\s+exists|'
+             r'count|sum|min|max|avg|sample|group_concat|separator)\b',
+             Name.Function),
+            # boolean literals ::
+            (r'(true|false)', Keyword.Constant),
+            # double literals ::
+            (r'[+\-]?(\d+\.\d*' + EXPONENT + r'|\.?\d+' + EXPONENT + ')', Number.Float),
+            # decimal literals ::
+            (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float),
+            # integer literals ::
+            (r'[+\-]?\d+', Number.Integer),
+            # operators ::
+            (r'(\|\||&&|=|\*|\-|\+|/|!=|<=|>=|!|<|>)', Operator),
+            # punctuation characters ::
+            (r'[(){}.;,:^\[\]]', Punctuation),
+            # line comments ::
+            (r'#[^\n]*', Comment),
+            # strings ::
+            (r'"""', String, 'triple-double-quoted-string'),
+            (r'"', String, 'single-double-quoted-string'),
+            (r"'''", String, 'triple-single-quoted-string'),
+            (r"'", String, 'single-single-quoted-string'),
+        ],
+        'triple-double-quoted-string': [
+            (r'"""', String, 'end-of-string'),
+            (r'[^\\]+', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'single-double-quoted-string': [
+            (r'"', String, 'end-of-string'),
+            (r'[^"\\\n]+', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'triple-single-quoted-string': [
+            (r"'''", String, 'end-of-string'),
+            (r'[^\\]+', String),
+            (r'\\', String.Escape, 'string-escape'),
+        ],
+        'single-single-quoted-string': [
+            (r"'", String, 'end-of-string'),
+            (r"[^'\\\n]+", String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'string-escape': [
+            (r'u' + HEX + '{4}', String.Escape, '#pop'),
+            (r'U' + HEX + '{8}', String.Escape, '#pop'),
+            (r'.', String.Escape, '#pop'),
+        ],
+        'end-of-string': [
+            (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)',
+             bygroups(Operator, Name.Function), '#pop:2'),
+            (r'\^\^', Operator, '#pop:2'),
+            default('#pop:2'),
+        ],
+    }
+
+
+class TurtleLexer(RegexLexer):
+    """
+    Lexer for Turtle data language.
+    """
+    name = 'Turtle'
+    aliases = ['turtle']
+    filenames = ['*.ttl']
+    mimetypes = ['text/turtle', 'application/x-turtle']
+    url = 'https://www.w3.org/TR/turtle'
+    version_added = '2.1'
+
+    # character group definitions ::
+    PN_CHARS_BASE_GRP = ('a-zA-Z'
+                         '\u00c0-\u00d6'
+                         '\u00d8-\u00f6'
+                         '\u00f8-\u02ff'
+                         '\u0370-\u037d'
+                         '\u037f-\u1fff'
+                         '\u200c-\u200d'
+                         '\u2070-\u218f'
+                         '\u2c00-\u2fef'
+                         '\u3001-\ud7ff'
+                         '\uf900-\ufdcf'
+                         '\ufdf0-\ufffd')
+
+    PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_')
+
+    PN_CHARS_GRP = (PN_CHARS_U_GRP +
+                    r'\-' +
+                    r'0-9' +
+                    '\u00b7' +
+                    '\u0300-\u036f' +
+                    '\u203f-\u2040')
+
+    PN_CHARS = '[' + PN_CHARS_GRP + ']'
+
+    PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']'
+
+    PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?'
+
+    HEX_GRP = '0-9A-Fa-f'
+
+    HEX = '[' + HEX_GRP + ']'
+
+    PERCENT = '%' + HEX + HEX
+
+    PN_LOCAL_ESC_CHARS_GRP = r' _~.\-!$&"()*+,;=/?#@%'
+
+    PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']'
+
+    PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS
+
+    PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')'
+
+    PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' +
+                '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' +
+                PN_CHARS_GRP + ':]|' + PLX + '))?')
+
+    patterns = {
+        'PNAME_NS': r'((?:[a-zA-Z][\w-]*)?\:)',  # Simplified character range
+        'IRIREF': r'(<[^<>"{}|^`\\\x00-\x20]*>)'
+    }
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+
+            # Base / prefix
+            (r'(@base|BASE)(\s+){IRIREF}(\s*)(\.?)'.format(**patterns),
+             bygroups(Keyword, Whitespace, Name.Variable, Whitespace,
+                      Punctuation)),
+            (r'(@prefix|PREFIX)(\s+){PNAME_NS}(\s+){IRIREF}(\s*)(\.?)'.format(**patterns),
+             bygroups(Keyword, Whitespace, Name.Namespace, Whitespace,
+                      Name.Variable, Whitespace, Punctuation)),
+
+            # The shorthand predicate 'a'
+            (r'(?<=\s)a(?=\s)', Keyword.Type),
+
+            # IRIREF
+            (r'{IRIREF}'.format(**patterns), Name.Variable),
+
+            # PrefixedName
+            (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + r')?',
+             bygroups(Name.Namespace, Punctuation, Name.Tag)),
+
+            # BlankNodeLabel
+            (r'(_)(:)([' + PN_CHARS_U_GRP + r'0-9]([' + PN_CHARS_GRP + r'.]*' + PN_CHARS + ')?)',
+             bygroups(Name.Namespace, Punctuation, Name.Tag)),
+
+            # Comment
+            (r'#([^\n]+|$)', Comment),
+
+            (r'\b(true|false)\b', Literal),
+            (r'[+\-]?\d*\.\d+', Number.Float),
+            (r'[+\-]?\d*(:?\.\d+)?E[+\-]?\d+', Number.Float),
+            (r'[+\-]?\d+', Number.Integer),
+            (r'[\[\](){}.;,:^]', Punctuation),
+
+            (r'"""', String, 'triple-double-quoted-string'),
+            (r'"', String, 'single-double-quoted-string'),
+            (r"'''", String, 'triple-single-quoted-string'),
+            (r"'", String, 'single-single-quoted-string'),
+        ],
+        'triple-double-quoted-string': [
+            (r'"""', String, 'end-of-string'),
+            (r'[^\\]+(?=""")', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'single-double-quoted-string': [
+            (r'"', String, 'end-of-string'),
+            (r'[^"\\\n]+', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'triple-single-quoted-string': [
+            (r"'''", String, 'end-of-string'),
+            (r"[^\\]+(?=''')", String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'single-single-quoted-string': [
+            (r"'", String, 'end-of-string'),
+            (r"[^'\\\n]+", String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'string-escape': [
+            (r'.', String, '#pop'),
+        ],
+        'end-of-string': [
+            (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)',
+             bygroups(Operator, Generic.Emph), '#pop:2'),
+
+            (r'(\^\^){IRIREF}'.format(**patterns), bygroups(Operator, Generic.Emph), '#pop:2'),
+
+            default('#pop:2'),
+
+        ],
+    }
+
+    # Turtle and Tera Term macro files share the same file extension
+    # but each has a recognizable and distinct syntax.
+    def analyse_text(text):
+        for t in ('@base ', 'BASE ', '@prefix ', 'PREFIX '):
+            if re.search(rf'^\s*{t}', text):
+                return 0.80
+
+
+class ShExCLexer(RegexLexer):
+    """
+    Lexer for ShExC shape expressions language syntax.
+    """
+    name = 'ShExC'
+    aliases = ['shexc', 'shex']
+    filenames = ['*.shex']
+    mimetypes = ['text/shex']
+    url = 'https://shex.io/shex-semantics/#shexc'
+    version_added = ''
+
+    # character group definitions ::
+
+    PN_CHARS_BASE_GRP = ('a-zA-Z'
+                         '\u00c0-\u00d6'
+                         '\u00d8-\u00f6'
+                         '\u00f8-\u02ff'
+                         '\u0370-\u037d'
+                         '\u037f-\u1fff'
+                         '\u200c-\u200d'
+                         '\u2070-\u218f'
+                         '\u2c00-\u2fef'
+                         '\u3001-\ud7ff'
+                         '\uf900-\ufdcf'
+                         '\ufdf0-\ufffd')
+
+    PN_CHARS_U_GRP = (PN_CHARS_BASE_GRP + '_')
+
+    PN_CHARS_GRP = (PN_CHARS_U_GRP +
+                    r'\-' +
+                    r'0-9' +
+                    '\u00b7' +
+                    '\u0300-\u036f' +
+                    '\u203f-\u2040')
+
+    HEX_GRP = '0-9A-Fa-f'
+
+    PN_LOCAL_ESC_CHARS_GRP = r"_~.\-!$&'()*+,;=/?#@%"
+
+    # terminal productions ::
+
+    PN_CHARS_BASE = '[' + PN_CHARS_BASE_GRP + ']'
+
+    PN_CHARS_U = '[' + PN_CHARS_U_GRP + ']'
+
+    PN_CHARS = '[' + PN_CHARS_GRP + ']'
+
+    HEX = '[' + HEX_GRP + ']'
+
+    PN_LOCAL_ESC_CHARS = '[' + PN_LOCAL_ESC_CHARS_GRP + ']'
+
+    UCHAR_NO_BACKSLASH = '(?:u' + HEX + '{4}|U' + HEX + '{8})'
+
+    UCHAR = r'\\' + UCHAR_NO_BACKSLASH
+
+    IRIREF = r'<(?:[^\x00-\x20<>"{}|^`\\]|' + UCHAR + ')*>'
+
+    BLANK_NODE_LABEL = '_:[0-9' + PN_CHARS_U_GRP + '](?:[' + PN_CHARS_GRP + \
+                       '.]*' + PN_CHARS + ')?'
+
+    PN_PREFIX = PN_CHARS_BASE + '(?:[' + PN_CHARS_GRP + '.]*' + PN_CHARS + ')?'
+
+    PERCENT = '%' + HEX + HEX
+
+    PN_LOCAL_ESC = r'\\' + PN_LOCAL_ESC_CHARS
+
+    PLX = '(?:' + PERCENT + ')|(?:' + PN_LOCAL_ESC + ')'
+
+    PN_LOCAL = ('(?:[' + PN_CHARS_U_GRP + ':0-9' + ']|' + PLX + ')' +
+                '(?:(?:[' + PN_CHARS_GRP + '.:]|' + PLX + ')*(?:[' +
+                PN_CHARS_GRP + ':]|' + PLX + '))?')
+
+    EXPONENT = r'[eE][+-]?\d+'
+
+    # Lexer token definitions ::
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            # keywords ::
+            (r'(?i)(base|prefix|start|external|'
+             r'literal|iri|bnode|nonliteral|length|minlength|maxlength|'
+             r'mininclusive|minexclusive|maxinclusive|maxexclusive|'
+             r'totaldigits|fractiondigits|'
+             r'closed|extra)\b', Keyword),
+            (r'(a)\b', Keyword),
+            # IRIs ::
+            ('(' + IRIREF + ')', Name.Label),
+            # blank nodes ::
+            ('(' + BLANK_NODE_LABEL + ')', Name.Label),
+            # prefixed names ::
+            (r'(' + PN_PREFIX + r')?(\:)(' + PN_LOCAL + ')?',
+             bygroups(Name.Namespace, Punctuation, Name.Tag)),
+            # boolean literals ::
+            (r'(true|false)', Keyword.Constant),
+            # double literals ::
+            (r'[+\-]?(\d+\.\d*' + EXPONENT + r'|\.?\d+' + EXPONENT + ')', Number.Float),
+            # decimal literals ::
+            (r'[+\-]?(\d+\.\d*|\.\d+)', Number.Float),
+            # integer literals ::
+            (r'[+\-]?\d+', Number.Integer),
+            # operators ::
+            (r'[@|$&=*+?^\-~]', Operator),
+            # operator keywords ::
+            (r'(?i)(and|or|not)\b', Operator.Word),
+            # punctuation characters ::
+            (r'[(){}.;,:^\[\]]', Punctuation),
+            # line comments ::
+            (r'#[^\n]*', Comment),
+            # strings ::
+            (r'"""', String, 'triple-double-quoted-string'),
+            (r'"', String, 'single-double-quoted-string'),
+            (r"'''", String, 'triple-single-quoted-string'),
+            (r"'", String, 'single-single-quoted-string'),
+        ],
+        'triple-double-quoted-string': [
+            (r'"""', String, 'end-of-string'),
+            (r'[^\\]+', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'single-double-quoted-string': [
+            (r'"', String, 'end-of-string'),
+            (r'[^"\\\n]+', String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'triple-single-quoted-string': [
+            (r"'''", String, 'end-of-string'),
+            (r'[^\\]+', String),
+            (r'\\', String.Escape, 'string-escape'),
+        ],
+        'single-single-quoted-string': [
+            (r"'", String, 'end-of-string'),
+            (r"[^'\\\n]+", String),
+            (r'\\', String, 'string-escape'),
+        ],
+        'string-escape': [
+            (UCHAR_NO_BACKSLASH, String.Escape, '#pop'),
+            (r'.', String.Escape, '#pop'),
+        ],
+        'end-of-string': [
+            (r'(@)([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)',
+             bygroups(Operator, Name.Function), '#pop:2'),
+            (r'\^\^', Operator, '#pop:2'),
+            default('#pop:2'),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rebol.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rebol.py
new file mode 100644
index 0000000000000000000000000000000000000000..fcff62be14de1f3d89c43600daa10f33e7acbc8a
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rebol.py
@@ -0,0 +1,419 @@
+"""
+    pygments.lexers.rebol
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the REBOL and related languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Generic, Whitespace
+
+__all__ = ['RebolLexer', 'RedLexer']
+
+
+class RebolLexer(RegexLexer):
+    """
+    A REBOL lexer.
+    """
+    name = 'REBOL'
+    aliases = ['rebol']
+    filenames = ['*.r', '*.r3', '*.reb']
+    mimetypes = ['text/x-rebol']
+    url = 'http://www.rebol.com'
+    version_added = '1.1'
+
+    flags = re.IGNORECASE | re.MULTILINE
+
+    escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)'
+
+    def word_callback(lexer, match):
+        word = match.group()
+
+        if re.match(".*:$", word):
+            yield match.start(), Generic.Subheading, word
+        elif re.match(
+            r'(native|alias|all|any|as-string|as-binary|bind|bound\?|case|'
+            r'catch|checksum|comment|debase|dehex|exclude|difference|disarm|'
+            r'either|else|enbase|foreach|remove-each|form|free|get|get-env|if|'
+            r'in|intersect|loop|minimum-of|maximum-of|mold|new-line|'
+            r'new-line\?|not|now|prin|print|reduce|compose|construct|repeat|'
+            r'reverse|save|script\?|set|shift|switch|throw|to-hex|trace|try|'
+            r'type\?|union|unique|unless|unprotect|unset|until|use|value\?|'
+            r'while|compress|decompress|secure|open|close|read|read-io|'
+            r'write-io|write|update|query|wait|input\?|exp|log-10|log-2|'
+            r'log-e|square-root|cosine|sine|tangent|arccosine|arcsine|'
+            r'arctangent|protect|lowercase|uppercase|entab|detab|connected\?|'
+            r'browse|launch|stats|get-modes|set-modes|to-local-file|'
+            r'to-rebol-file|encloak|decloak|create-link|do-browser|bind\?|'
+            r'hide|draw|show|size-text|textinfo|offset-to-caret|'
+            r'caret-to-offset|local-request-file|rgb-to-hsv|hsv-to-rgb|'
+            r'crypt-strength\?|dh-make-key|dh-generate-key|dh-compute-key|'
+            r'dsa-make-key|dsa-generate-key|dsa-make-signature|'
+            r'dsa-verify-signature|rsa-make-key|rsa-generate-key|'
+            r'rsa-encrypt)$', word):
+            yield match.start(), Name.Builtin, word
+        elif re.match(
+            r'(add|subtract|multiply|divide|remainder|power|and~|or~|xor~|'
+            r'minimum|maximum|negate|complement|absolute|random|head|tail|'
+            r'next|back|skip|at|pick|first|second|third|fourth|fifth|sixth|'
+            r'seventh|eighth|ninth|tenth|last|path|find|select|make|to|copy\*|'
+            r'insert|remove|change|poke|clear|trim|sort|min|max|abs|cp|'
+            r'copy)$', word):
+            yield match.start(), Name.Function, word
+        elif re.match(
+            r'(error|source|input|license|help|install|echo|Usage|with|func|'
+            r'throw-on-error|function|does|has|context|probe|\?\?|as-pair|'
+            r'mod|modulo|round|repend|about|set-net|append|join|rejoin|reform|'
+            r'remold|charset|array|replace|move|extract|forskip|forall|alter|'
+            r'first+|also|take|for|forever|dispatch|attempt|what-dir|'
+            r'change-dir|clean-path|list-dir|dirize|rename|split-path|delete|'
+            r'make-dir|delete-dir|in-dir|confirm|dump-obj|upgrade|what|'
+            r'build-tag|process-source|build-markup|decode-cgi|read-cgi|'
+            r'write-user|save-user|set-user-name|protect-system|parse-xml|'
+            r'cvs-date|cvs-version|do-boot|get-net-info|desktop|layout|'
+            r'scroll-para|get-face|alert|set-face|uninstall|unfocus|'
+            r'request-dir|center-face|do-events|net-error|decode-url|'
+            r'parse-header|parse-header-date|parse-email-addrs|import-email|'
+            r'send|build-attach-body|resend|show-popup|hide-popup|open-events|'
+            r'find-key-face|do-face|viewtop|confine|find-window|'
+            r'insert-event-func|remove-event-func|inform|dump-pane|dump-face|'
+            r'flag-face|deflag-face|clear-fields|read-net|vbug|path-thru|'
+            r'read-thru|load-thru|do-thru|launch-thru|load-image|'
+            r'request-download|do-face-alt|set-font|set-para|get-style|'
+            r'set-style|make-face|stylize|choose|hilight-text|hilight-all|'
+            r'unlight-text|focus|scroll-drag|clear-face|reset-face|scroll-face|'
+            r'resize-face|load-stock|load-stock-block|notify|request|flash|'
+            r'request-color|request-pass|request-text|request-list|'
+            r'request-date|request-file|dbug|editor|link-relative-path|'
+            r'emailer|parse-error)$', word):
+            yield match.start(), Keyword.Namespace, word
+        elif re.match(
+            r'(halt|quit|do|load|q|recycle|call|run|ask|parse|view|unview|'
+            r'return|exit|break)$', word):
+            yield match.start(), Name.Exception, word
+        elif re.match('REBOL$', word):
+            yield match.start(), Generic.Heading, word
+        elif re.match("to-.*", word):
+            yield match.start(), Keyword, word
+        elif re.match(r'(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|<>|<|>|<=|>=)$',
+                      word):
+            yield match.start(), Operator, word
+        elif re.match(r".*\?$", word):
+            yield match.start(), Keyword, word
+        elif re.match(r".*\!$", word):
+            yield match.start(), Keyword.Type, word
+        elif re.match("'.*", word):
+            yield match.start(), Name.Variable.Instance, word  # lit-word
+        elif re.match("#.*", word):
+            yield match.start(), Name.Label, word  # issue
+        elif re.match("%.*", word):
+            yield match.start(), Name.Decorator, word  # file
+        else:
+            yield match.start(), Name.Variable, word
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'#"', String.Char, 'char'),
+            (r'#\{[0-9a-f]*\}', Number.Hex),
+            (r'2#\{', Number.Hex, 'bin2'),
+            (r'64#\{[0-9a-z+/=\s]*\}', Number.Hex),
+            (r'"', String, 'string'),
+            (r'\{', String, 'string2'),
+            (r';#+.*\n', Comment.Special),
+            (r';\*+.*\n', Comment.Preproc),
+            (r';.*\n', Comment),
+            (r'%"', Name.Decorator, 'stringFile'),
+            (r'%[^(^{")\s\[\]]+', Name.Decorator),
+            (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float),  # money
+            (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other),    # time
+            (r'\d+[\-/][0-9a-z]+[\-/]\d+(\/\d+\:\d+((\:\d+)?'
+             r'([.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other),   # date
+            (r'\d+(\.\d+)+\.\d+', Keyword.Constant),             # tuple
+            (r'\d+X\d+', Keyword.Constant),                   # pair
+            (r'[+-]?\d+(\'\d+)?([.,]\d*)?E[+-]?\d+', Number.Float),
+            (r'[+-]?\d+(\'\d+)?[.,]\d*', Number.Float),
+            (r'[+-]?\d+(\'\d+)?', Number),
+            (r'[\[\]()]', Generic.Strong),
+            (r'[a-z]+[^(^{"\s:)]*://[^(^{"\s)]*', Name.Decorator),  # url
+            (r'mailto:[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator),  # url
+            (r'[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator),         # email
+            (r'comment\s"', Comment, 'commentString1'),
+            (r'comment\s\{', Comment, 'commentString2'),
+            (r'comment\s\[', Comment, 'commentBlock'),
+            (r'comment\s[^(\s{"\[]+', Comment),
+            (r'/[^(^{")\s/[\]]*', Name.Attribute),
+            (r'([^(^{")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback),
+            (r'<[\w:.-]*>', Name.Tag),
+            (r'<[^(<>\s")]+', Name.Tag, 'tag'),
+            (r'([^(^{")\s]+)', Text),
+        ],
+        'string': [
+            (r'[^(^")]+', String),
+            (escape_re, String.Escape),
+            (r'[(|)]+', String),
+            (r'\^.', String.Escape),
+            (r'"', String, '#pop'),
+        ],
+        'string2': [
+            (r'[^(^{})]+', String),
+            (escape_re, String.Escape),
+            (r'[(|)]+', String),
+            (r'\^.', String.Escape),
+            (r'\{', String, '#push'),
+            (r'\}', String, '#pop'),
+        ],
+        'stringFile': [
+            (r'[^(^")]+', Name.Decorator),
+            (escape_re, Name.Decorator),
+            (r'\^.', Name.Decorator),
+            (r'"', Name.Decorator, '#pop'),
+        ],
+        'char': [
+            (escape_re + '"', String.Char, '#pop'),
+            (r'\^."', String.Char, '#pop'),
+            (r'."', String.Char, '#pop'),
+        ],
+        'tag': [
+            (escape_re, Name.Tag),
+            (r'"', Name.Tag, 'tagString'),
+            (r'[^(<>\r\n")]+', Name.Tag),
+            (r'>', Name.Tag, '#pop'),
+        ],
+        'tagString': [
+            (r'[^(^")]+', Name.Tag),
+            (escape_re, Name.Tag),
+            (r'[(|)]+', Name.Tag),
+            (r'\^.', Name.Tag),
+            (r'"', Name.Tag, '#pop'),
+        ],
+        'tuple': [
+            (r'(\d+\.)+', Keyword.Constant),
+            (r'\d+', Keyword.Constant, '#pop'),
+        ],
+        'bin2': [
+            (r'\s+', Number.Hex),
+            (r'([01]\s*){8}', Number.Hex),
+            (r'\}', Number.Hex, '#pop'),
+        ],
+        'commentString1': [
+            (r'[^(^")]+', Comment),
+            (escape_re, Comment),
+            (r'[(|)]+', Comment),
+            (r'\^.', Comment),
+            (r'"', Comment, '#pop'),
+        ],
+        'commentString2': [
+            (r'[^(^{})]+', Comment),
+            (escape_re, Comment),
+            (r'[(|)]+', Comment),
+            (r'\^.', Comment),
+            (r'\{', Comment, '#push'),
+            (r'\}', Comment, '#pop'),
+        ],
+        'commentBlock': [
+            (r'\[', Comment, '#push'),
+            (r'\]', Comment, '#pop'),
+            (r'"', Comment, "commentString1"),
+            (r'\{', Comment, "commentString2"),
+            (r'[^(\[\]"{)]+', Comment),
+        ],
+    }
+
+    def analyse_text(text):
+        """
+        Check if code contains REBOL header and so it probably not R code
+        """
+        if re.match(r'^\s*REBOL\s*\[', text, re.IGNORECASE):
+            # The code starts with REBOL header
+            return 1.0
+        elif re.search(r'\s*REBOL\s*\[', text, re.IGNORECASE):
+            # The code contains REBOL header but also some text before it
+            return 0.5
+
+
+class RedLexer(RegexLexer):
+    """
+    A Red-language lexer.
+    """
+    name = 'Red'
+    aliases = ['red', 'red/system']
+    filenames = ['*.red', '*.reds']
+    mimetypes = ['text/x-red', 'text/x-red-system']
+    url = 'https://www.red-lang.org'
+    version_added = '2.0'
+
+    flags = re.IGNORECASE | re.MULTILINE
+
+    escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)'
+
+    def word_callback(lexer, match):
+        word = match.group()
+
+        if re.match(".*:$", word):
+            yield match.start(), Generic.Subheading, word
+        elif re.match(r'(if|unless|either|any|all|while|until|loop|repeat|'
+                      r'foreach|forall|func|function|does|has|switch|'
+                      r'case|reduce|compose|get|set|print|prin|equal\?|'
+                      r'not-equal\?|strict-equal\?|lesser\?|greater\?|lesser-or-equal\?|'
+                      r'greater-or-equal\?|same\?|not|type\?|stats|'
+                      r'bind|union|replace|charset|routine)$', word):
+            yield match.start(), Name.Builtin, word
+        elif re.match(r'(make|random|reflect|to|form|mold|absolute|add|divide|multiply|negate|'
+                      r'power|remainder|round|subtract|even\?|odd\?|and~|complement|or~|xor~|'
+                      r'append|at|back|change|clear|copy|find|head|head\?|index\?|insert|'
+                      r'length\?|next|pick|poke|remove|reverse|select|sort|skip|swap|tail|tail\?|'
+                      r'take|trim|create|close|delete|modify|open|open\?|query|read|rename|'
+                      r'update|write)$', word):
+            yield match.start(), Name.Function, word
+        elif re.match(r'(yes|on|no|off|true|false|tab|cr|lf|newline|escape|slash|sp|space|null|'
+                      r'none|crlf|dot|null-byte)$', word):
+            yield match.start(), Name.Builtin.Pseudo, word
+        elif re.match(r'(#system-global|#include|#enum|#define|#either|#if|#import|#export|'
+                      r'#switch|#default|#get-definition)$', word):
+            yield match.start(), Keyword.Namespace, word
+        elif re.match(r'(system|halt|quit|quit-return|do|load|q|recycle|call|run|ask|parse|'
+                      r'raise-error|return|exit|break|alias|push|pop|probe|\?\?|spec-of|body-of|'
+                      r'quote|forever)$', word):
+            yield match.start(), Name.Exception, word
+        elif re.match(r'(action\?|block\?|char\?|datatype\?|file\?|function\?|get-path\?|zero\?|'
+                      r'get-word\?|integer\?|issue\?|lit-path\?|lit-word\?|logic\?|native\?|'
+                      r'op\?|paren\?|path\?|refinement\?|set-path\?|set-word\?|string\?|unset\?|'
+                      r'any-struct\?|none\?|word\?|any-series\?)$', word):
+            yield match.start(), Keyword, word
+        elif re.match(r'(JNICALL|stdcall|cdecl|infix)$', word):
+            yield match.start(), Keyword.Namespace, word
+        elif re.match("to-.*", word):
+            yield match.start(), Keyword, word
+        elif re.match(r'(\+|-\*\*|-|\*\*|//|/|\*|and|or|xor|=\?|===|==|=|<>|<=|>=|'
+                      r'<<<|>>>|<<|>>|<|>%)$', word):
+            yield match.start(), Operator, word
+        elif re.match(r".*\!$", word):
+            yield match.start(), Keyword.Type, word
+        elif re.match("'.*", word):
+            yield match.start(), Name.Variable.Instance, word  # lit-word
+        elif re.match("#.*", word):
+            yield match.start(), Name.Label, word  # issue
+        elif re.match("%.*", word):
+            yield match.start(), Name.Decorator, word  # file
+        elif re.match(":.*", word):
+            yield match.start(), Generic.Subheading, word  # get-word
+        else:
+            yield match.start(), Name.Variable, word
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'#"', String.Char, 'char'),
+            (r'#\{[0-9a-f\s]*\}', Number.Hex),
+            (r'2#\{', Number.Hex, 'bin2'),
+            (r'64#\{[0-9a-z+/=\s]*\}', Number.Hex),
+            (r'([0-9a-f]+)(h)((\s)|(?=[\[\]{}"()]))',
+             bygroups(Number.Hex, Name.Variable, Whitespace)),
+            (r'"', String, 'string'),
+            (r'\{', String, 'string2'),
+            (r';#+.*\n', Comment.Special),
+            (r';\*+.*\n', Comment.Preproc),
+            (r';.*\n', Comment),
+            (r'%"', Name.Decorator, 'stringFile'),
+            (r'%[^(^{")\s\[\]]+', Name.Decorator),
+            (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float),  # money
+            (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other),    # time
+            (r'\d+[\-/][0-9a-z]+[\-/]\d+(/\d+:\d+((:\d+)?'
+             r'([\.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other),   # date
+            (r'\d+(\.\d+)+\.\d+', Keyword.Constant),             # tuple
+            (r'\d+X\d+', Keyword.Constant),                   # pair
+            (r'[+-]?\d+(\'\d+)?([.,]\d*)?E[+-]?\d+', Number.Float),
+            (r'[+-]?\d+(\'\d+)?[.,]\d*', Number.Float),
+            (r'[+-]?\d+(\'\d+)?', Number),
+            (r'[\[\]()]', Generic.Strong),
+            (r'[a-z]+[^(^{"\s:)]*://[^(^{"\s)]*', Name.Decorator),  # url
+            (r'mailto:[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator),  # url
+            (r'[^(^{"@\s)]+@[^(^{"@\s)]+', Name.Decorator),         # email
+            (r'comment\s"', Comment, 'commentString1'),
+            (r'comment\s\{', Comment, 'commentString2'),
+            (r'comment\s\[', Comment, 'commentBlock'),
+            (r'comment\s[^(\s{"\[]+', Comment),
+            (r'/[^(^{^")\s/[\]]*', Name.Attribute),
+            (r'([^(^{^")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback),
+            (r'<[\w:.-]*>', Name.Tag),
+            (r'<[^(<>\s")]+', Name.Tag, 'tag'),
+            (r'([^(^{")\s]+)', Text),
+        ],
+        'string': [
+            (r'[^(^")]+', String),
+            (escape_re, String.Escape),
+            (r'[(|)]+', String),
+            (r'\^.', String.Escape),
+            (r'"', String, '#pop'),
+        ],
+        'string2': [
+            (r'[^(^{})]+', String),
+            (escape_re, String.Escape),
+            (r'[(|)]+', String),
+            (r'\^.', String.Escape),
+            (r'\{', String, '#push'),
+            (r'\}', String, '#pop'),
+        ],
+        'stringFile': [
+            (r'[^(^")]+', Name.Decorator),
+            (escape_re, Name.Decorator),
+            (r'\^.', Name.Decorator),
+            (r'"', Name.Decorator, '#pop'),
+        ],
+        'char': [
+            (escape_re + '"', String.Char, '#pop'),
+            (r'\^."', String.Char, '#pop'),
+            (r'."', String.Char, '#pop'),
+        ],
+        'tag': [
+            (escape_re, Name.Tag),
+            (r'"', Name.Tag, 'tagString'),
+            (r'[^(<>\r\n")]+', Name.Tag),
+            (r'>', Name.Tag, '#pop'),
+        ],
+        'tagString': [
+            (r'[^(^")]+', Name.Tag),
+            (escape_re, Name.Tag),
+            (r'[(|)]+', Name.Tag),
+            (r'\^.', Name.Tag),
+            (r'"', Name.Tag, '#pop'),
+        ],
+        'tuple': [
+            (r'(\d+\.)+', Keyword.Constant),
+            (r'\d+', Keyword.Constant, '#pop'),
+        ],
+        'bin2': [
+            (r'\s+', Number.Hex),
+            (r'([01]\s*){8}', Number.Hex),
+            (r'\}', Number.Hex, '#pop'),
+        ],
+        'commentString1': [
+            (r'[^(^")]+', Comment),
+            (escape_re, Comment),
+            (r'[(|)]+', Comment),
+            (r'\^.', Comment),
+            (r'"', Comment, '#pop'),
+        ],
+        'commentString2': [
+            (r'[^(^{})]+', Comment),
+            (escape_re, Comment),
+            (r'[(|)]+', Comment),
+            (r'\^.', Comment),
+            (r'\{', Comment, '#push'),
+            (r'\}', Comment, '#pop'),
+        ],
+        'commentBlock': [
+            (r'\[', Comment, '#push'),
+            (r'\]', Comment, '#pop'),
+            (r'"', Comment, "commentString1"),
+            (r'\{', Comment, "commentString2"),
+            (r'[^(\[\]"{)]+', Comment),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rego.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rego.py
new file mode 100644
index 0000000000000000000000000000000000000000..b2e8b75473ce1f8b72a94f7de904d9f3b23c3a9f
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rego.py
@@ -0,0 +1,57 @@
+"""
+    pygments.lexers.rego
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Rego policy languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Comment, Operator, Keyword, Name, String, Number, Punctuation, Whitespace
+
+class RegoLexer(RegexLexer):
+    """
+    For Rego source.
+    """
+    name = 'Rego'
+    url = 'https://www.openpolicyagent.org/docs/latest/policy-language/'
+    filenames = ['*.rego']
+    aliases = ['rego']
+    mimetypes = ['text/x-rego']
+    version_added = '2.19'
+
+    reserved_words = (
+        'as', 'contains', 'data', 'default', 'else', 'every', 'false',
+        'if', 'in', 'import', 'package', 'not', 'null',
+        'some', 'true', 'with'
+    )
+
+    builtins = (
+        # https://www.openpolicyagent.org/docs/latest/philosophy/#the-opa-document-model
+        'data',  # Global variable for accessing base and virtual documents
+        'input', # Represents synchronously pushed base documents
+    )
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'\s+', Whitespace),
+            (r'#.*?$', Comment.Single),
+            (words(reserved_words, suffix=r'\b'), Keyword),
+            (words(builtins, suffix=r'\b'), Name.Builtin),
+            (r'[a-zA-Z_][a-zA-Z0-9_]*', Name),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r'`[^`]*`', String.Backtick),
+            (r'-?\d+(\.\d+)?', Number),
+            (r'(==|!=|<=|>=|:=)', Operator),  # Compound operators
+            (r'[=<>+\-*/%&|]', Operator),     # Single-character operators
+            (r'[\[\]{}(),.:;]', Punctuation),
+        ]
+    }
+
+__all__ = ['RegoLexer']
+
+
+
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rell.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rell.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d54ed2a5162448386aa30d1d2f2f99f9e836c20
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rell.py
@@ -0,0 +1,68 @@
+"""
+    pygments.lexers.rell
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Rell language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, default, words
+from pygments.token import Comment, Keyword, Name, String, Number, \
+        Punctuation, Whitespace
+
+__all__ = ['RellLexer']
+
+
+class RellLexer(RegexLexer):
+    """
+    A Lexer for Rell.
+    """
+    name = 'Rell'
+    url = 'https://docs.chromia.com/rell/rell-intro'
+    aliases = ['rell']
+    filenames = ['*.rell']
+    mimetypes = ['text/x-rell']
+    version_added = '2.20'
+
+    ident = r'[a-zA-Z_][a-zA-Z0-9_]*'
+
+    tokens = {
+        'root': [
+            (words((
+                'big_integer', 'boolean', 'byte_array', 'decimal', 'gtv',
+                'integer', 'json', 'list', 'map', 'mutable', 'set', 'text',
+                'virtual'), suffix=r'\b'),
+             Keyword.Type),
+            (r'(false|true|null)\b', Keyword.Constant),
+            (r'(entity|enum|namespace|object|struct)\b', Keyword.Declaration),
+            (r'(function|operation|query)\b', Keyword.Declaration, 'function'),
+            (words((
+                'abstract', 'and', 'break', 'continue', 'create', 'delete',
+                'else', 'for', 'if', 'import', 'in', 'index', 'key', 'limit',
+                'module', 'not', 'offset', 'or', 'override', 'return', 'update',
+                'val', 'var', 'when', 'while'), suffix=r'\b'),
+             Keyword.Reserved),
+            (r'//.*?$', Comment.Single),
+            (r'/\*(.|\n|\r)*?\*/', Comment.Multiline),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r'\'(\\\\|\\\'|[^\\\'])*\'', String.Single),
+            (r'-?[0-9]*\.[0-9]+([eE][+-][0-9]+)?', Number.Float),
+            (r'-?[0-9]+([eE][+-][0-9]+|[lL])?', Number.Integer),
+            (r'x(\'[a-fA-F0-9]*\'|"[a-fA-F0-9]*")', String.Binary),
+            (r'(\.)([ \n\t\r]*)(' + ident + ')',
+                bygroups(Punctuation, Whitespace, Name.Attribute)),
+            (r'[{}():;,]+', Punctuation),
+            (r'[ \n\t\r]+', Whitespace),
+            (r'@[a-zA-Z_][a-zA-Z0-9_]*', Name.Decorator),
+            (r'[~^*!%&\[\]<>|+=/?\-@\$]', Punctuation.Marker),
+            (ident, Name),
+            (r'(\.)+', Punctuation),
+        ],
+        'function': [
+            (r'[ \n\t\r]+', Whitespace),
+            (ident, Name.Function, '#pop'),
+            default('#pop'),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/resource.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/resource.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb6cb4a2ee6b9a661613d68fcd8f1fc2d77e96a2
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/resource.py
@@ -0,0 +1,83 @@
+"""
+    pygments.lexers.resource
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for resource definition files.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Comment, String, Number, Operator, Text, \
+    Keyword, Name
+
+__all__ = ['ResourceLexer']
+
+
+class ResourceLexer(RegexLexer):
+    """Lexer for ICU Resource bundles.
+    """
+    name = 'ResourceBundle'
+    aliases = ['resourcebundle', 'resource']
+    filenames = []
+    url = 'https://unicode-org.github.io/icu/userguide/locale/resources.html'
+    version_added = '2.0'
+
+    _types = (':table', ':array', ':string', ':bin', ':import', ':intvector',
+              ':int', ':alias')
+
+    flags = re.MULTILINE | re.IGNORECASE
+    tokens = {
+        'root': [
+            (r'//.*?$', Comment),
+            (r'"', String, 'string'),
+            (r'-?\d+', Number.Integer),
+            (r'[,{}]', Operator),
+            (r'([^\s{{:]+)(\s*)({}?)'.format('|'.join(_types)),
+             bygroups(Name, Text, Keyword)),
+            (r'\s+', Text),
+            (words(_types), Keyword),
+        ],
+        'string': [
+            (r'(\\x[0-9a-f]{2}|\\u[0-9a-f]{4}|\\U00[0-9a-f]{6}|'
+             r'\\[0-7]{1,3}|\\c.|\\[abtnvfre\'"?\\]|\\\{|[^"{\\])+', String),
+            (r'\{', String.Escape, 'msgname'),
+            (r'"', String, '#pop')
+        ],
+        'msgname': [
+            (r'([^{},]+)(\s*)', bygroups(Name, String.Escape), ('#pop', 'message'))
+        ],
+        'message': [
+            (r'\{', String.Escape, 'msgname'),
+            (r'\}', String.Escape, '#pop'),
+            (r'(,)(\s*)([a-z]+)(\s*\})',
+             bygroups(Operator, String.Escape, Keyword, String.Escape), '#pop'),
+            (r'(,)(\s*)([a-z]+)(\s*)(,)(\s*)(offset)(\s*)(:)(\s*)(-?\d+)(\s*)',
+             bygroups(Operator, String.Escape, Keyword, String.Escape, Operator,
+                      String.Escape, Operator.Word, String.Escape, Operator,
+                      String.Escape, Number.Integer, String.Escape), 'choice'),
+            (r'(,)(\s*)([a-z]+)(\s*)(,)(\s*)',
+             bygroups(Operator, String.Escape, Keyword, String.Escape, Operator,
+                      String.Escape), 'choice'),
+            (r'\s+', String.Escape)
+        ],
+        'choice': [
+            (r'(=|<|>|<=|>=|!=)(-?\d+)(\s*\{)',
+             bygroups(Operator, Number.Integer, String.Escape), 'message'),
+            (r'([a-z]+)(\s*\{)', bygroups(Keyword.Type, String.Escape), 'str'),
+            (r'\}', String.Escape, ('#pop', '#pop')),
+            (r'\s+', String.Escape)
+        ],
+        'str': [
+            (r'\}', String.Escape, '#pop'),
+            (r'\{', String.Escape, 'msgname'),
+            (r'[^{}]+', String)
+        ]
+    }
+
+    def analyse_text(text):
+        if text.startswith('root:table'):
+            return 1.0
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ride.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ride.py
new file mode 100644
index 0000000000000000000000000000000000000000..1ed8f38412baa74fc0ed6b53709cdcb41afeb893
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ride.py
@@ -0,0 +1,138 @@
+"""
+    pygments.lexers.ride
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Ride programming language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, include
+from pygments.token import Comment, Keyword, Name, Number, Punctuation, \
+    String, Text
+
+__all__ = ['RideLexer']
+
+
+class RideLexer(RegexLexer):
+    """
+    For Ride source code.
+    """
+
+    name = 'Ride'
+    aliases = ['ride']
+    filenames = ['*.ride']
+    mimetypes = ['text/x-ride']
+    url = 'https://docs.waves.tech/en/ride'
+    version_added = '2.6'
+
+    validName = r'[a-zA-Z_][a-zA-Z0-9_\']*'
+
+    builtinOps = (
+        '||', '|', '>=', '>', '==', '!',
+        '=', '<=', '<', '::', ':+', ':', '!=', '/',
+        '.', '=>', '-', '+', '*', '&&', '%', '++',
+    )
+
+    globalVariablesName = (
+        'NOALG', 'MD5', 'SHA1', 'SHA224', 'SHA256', 'SHA384', 'SHA512',
+        'SHA3224', 'SHA3256', 'SHA3384', 'SHA3512', 'nil', 'this', 'unit',
+        'height', 'lastBlock', 'Buy', 'Sell', 'CEILING', 'FLOOR', 'DOWN',
+        'HALFDOWN', 'HALFEVEN', 'HALFUP', 'UP',
+    )
+
+    typesName = (
+        'Unit', 'Int', 'Boolean', 'ByteVector', 'String', 'Address', 'Alias',
+        'Transfer', 'AssetPair', 'DataEntry', 'Order', 'Transaction',
+        'GenesisTransaction', 'PaymentTransaction', 'ReissueTransaction',
+        'BurnTransaction', 'MassTransferTransaction', 'ExchangeTransaction',
+        'TransferTransaction', 'SetAssetScriptTransaction',
+        'InvokeScriptTransaction', 'IssueTransaction', 'LeaseTransaction',
+        'LeaseCancelTransaction', 'CreateAliasTransaction',
+        'SetScriptTransaction', 'SponsorFeeTransaction', 'DataTransaction',
+        'WriteSet', 'AttachedPayment', 'ScriptTransfer', 'TransferSet',
+        'ScriptResult', 'Invocation', 'Asset', 'BlockInfo', 'Issue', 'Reissue',
+        'Burn', 'NoAlg', 'Md5', 'Sha1', 'Sha224', 'Sha256', 'Sha384', 'Sha512',
+        'Sha3224', 'Sha3256', 'Sha3384', 'Sha3512', 'BinaryEntry',
+        'BooleanEntry', 'IntegerEntry', 'StringEntry', 'List', 'Ceiling',
+        'Down', 'Floor', 'HalfDown', 'HalfEven', 'HalfUp', 'Up',
+    )
+
+    functionsName = (
+        'fraction', 'size', 'toBytes', 'take', 'drop', 'takeRight', 'dropRight',
+        'toString', 'isDefined', 'extract', 'throw', 'getElement', 'value',
+        'cons', 'toUtf8String', 'toInt', 'indexOf', 'lastIndexOf', 'split',
+        'parseInt', 'parseIntValue', 'keccak256', 'blake2b256', 'sha256',
+        'sigVerify', 'toBase58String', 'fromBase58String', 'toBase64String',
+        'fromBase64String', 'transactionById', 'transactionHeightById',
+        'getInteger', 'getBoolean', 'getBinary', 'getString',
+        'addressFromPublicKey', 'addressFromString', 'addressFromRecipient',
+        'assetBalance', 'wavesBalance', 'getIntegerValue', 'getBooleanValue',
+        'getBinaryValue', 'getStringValue', 'addressFromStringValue',
+        'assetInfo', 'rsaVerify', 'checkMerkleProof', 'median',
+        'valueOrElse', 'valueOrErrorMessage', 'contains', 'log', 'pow',
+        'toBase16String', 'fromBase16String', 'blockInfoByHeight',
+        'transferTransactionById',
+    )
+
+    reservedWords = words((
+        'match', 'case', 'else', 'func', 'if',
+        'let', 'then', '@Callable', '@Verifier',
+    ), suffix=r'\b')
+
+    tokens = {
+        'root': [
+            # Comments
+            (r'#.*', Comment.Single),
+            # Whitespace
+            (r'\s+', Text),
+            # Strings
+            (r'"', String, 'doublequote'),
+            (r'utf8\'', String, 'utf8quote'),
+            (r'base(58|64|16)\'', String, 'singlequote'),
+            # Keywords
+            (reservedWords, Keyword.Reserved),
+            (r'\{-#.*?#-\}', Keyword.Reserved),
+            (r'FOLD<\d+>', Keyword.Reserved),
+            # Types
+            (words(typesName), Keyword.Type),
+            # Main
+            # (specialName, Keyword.Reserved),
+            # Prefix Operators
+            (words(builtinOps, prefix=r'\(', suffix=r'\)'), Name.Function),
+            # Infix Operators
+            (words(builtinOps), Name.Function),
+            (words(globalVariablesName), Name.Function),
+            (words(functionsName), Name.Function),
+            # Numbers
+            include('numbers'),
+            # Variable Names
+            (validName, Name.Variable),
+            # Parens
+            (r'[,()\[\]{}]', Punctuation),
+        ],
+
+        'doublequote': [
+            (r'\\u[0-9a-fA-F]{4}', String.Escape),
+            (r'\\[nrfvb\\"]', String.Escape),
+            (r'[^"]', String),
+            (r'"', String, '#pop'),
+        ],
+
+        'utf8quote': [
+            (r'\\u[0-9a-fA-F]{4}', String.Escape),
+            (r'\\[nrfvb\\\']', String.Escape),
+            (r'[^\']', String),
+            (r'\'', String, '#pop'),
+        ],
+
+        'singlequote': [
+            (r'[^\']', String),
+            (r'\'', String, '#pop'),
+        ],
+
+        'numbers': [
+            (r'_?\d+', Number.Integer),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rita.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rita.py
new file mode 100644
index 0000000000000000000000000000000000000000..99a574c2a1a4d83050389f07dbb5052a3c39b180
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rita.py
@@ -0,0 +1,42 @@
+"""
+    pygments.lexers.rita
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for RITA language
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Comment, Operator, Keyword, Name, Literal, \
+    Punctuation, Whitespace
+
+__all__ = ['RitaLexer']
+
+
+class RitaLexer(RegexLexer):
+    """
+    Lexer for RITA.
+    """
+    name = 'Rita'
+    url = 'https://github.com/zaibacu/rita-dsl'
+    filenames = ['*.rita']
+    aliases = ['rita']
+    mimetypes = ['text/rita']
+    version_added = '2.11'
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'\s+', Whitespace),
+            (r'#(.*?)\n', Comment.Single),
+            (r'@(.*?)\n', Operator),  # Yes, whole line as an operator
+            (r'"(\w|\d|\s|(\\")|[\'_\-./,\?\!])+?"', Literal),
+            (r'\'(\w|\d|\s|(\\\')|["_\-./,\?\!])+?\'', Literal),
+            (r'([A-Z_]+)', Keyword),
+            (r'([a-z0-9_]+)', Name),
+            (r'((->)|[!?+*|=])', Operator),
+            (r'[\(\),\{\}]', Punctuation)
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rnc.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rnc.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ddd33e44474451c6f1cff5a9ac35eb40df3783a
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rnc.py
@@ -0,0 +1,66 @@
+"""
+    pygments.lexers.rnc
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Relax-NG Compact syntax
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Punctuation
+
+__all__ = ['RNCCompactLexer']
+
+
+class RNCCompactLexer(RegexLexer):
+    """
+    For RelaxNG-compact syntax.
+    """
+
+    name = 'Relax-NG Compact'
+    url = 'http://relaxng.org'
+    aliases = ['rng-compact', 'rnc']
+    filenames = ['*.rnc']
+    version_added = '2.2'
+
+    tokens = {
+        'root': [
+            (r'namespace\b', Keyword.Namespace),
+            (r'(?:default|datatypes)\b', Keyword.Declaration),
+            (r'##.*$', Comment.Preproc),
+            (r'#.*$', Comment.Single),
+            (r'"[^"]*"', String.Double),
+            # TODO single quoted strings and escape sequences outside of
+            # double-quoted strings
+            (r'(?:element|attribute|mixed)\b', Keyword.Declaration, 'variable'),
+            (r'(text\b|xsd:[^ ]+)', Keyword.Type, 'maybe_xsdattributes'),
+            (r'[,?&*=|~]|>>', Operator),
+            (r'[(){}]', Punctuation),
+            (r'.', Text),
+        ],
+
+        # a variable has been declared using `element` or `attribute`
+        'variable': [
+            (r'[^{]+', Name.Variable),
+            (r'\{', Punctuation, '#pop'),
+        ],
+
+        # after an xsd: declaration there may be attributes
+        'maybe_xsdattributes': [
+            (r'\{', Punctuation, 'xsdattributes'),
+            (r'\}', Punctuation, '#pop'),
+            (r'.', Text),
+        ],
+
+        # attributes take the form { key1 = value1 key2 = value2 ... }
+        'xsdattributes': [
+            (r'[^ =}]', Name.Attribute),
+            (r'=', Operator),
+            (r'"[^"]*"', String.Double),
+            (r'\}', Punctuation, '#pop'),
+            (r'.', Text),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/roboconf.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/roboconf.py
new file mode 100644
index 0000000000000000000000000000000000000000..bf0570cc37161e82e34a31113731d723b662128f
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/roboconf.py
@@ -0,0 +1,81 @@
+"""
+    pygments.lexers.roboconf
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Roboconf DSL.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, re
+from pygments.token import Text, Operator, Keyword, Name, Comment
+
+__all__ = ['RoboconfGraphLexer', 'RoboconfInstancesLexer']
+
+
+class RoboconfGraphLexer(RegexLexer):
+    """
+    Lexer for Roboconf graph files.
+    """
+    name = 'Roboconf Graph'
+    aliases = ['roboconf-graph']
+    filenames = ['*.graph']
+    url = 'https://roboconf.github.io/en/user-guide/graph-definition.html'
+    version_added = '2.1'
+
+    flags = re.IGNORECASE | re.MULTILINE
+    tokens = {
+        'root': [
+            # Skip white spaces
+            (r'\s+', Text),
+
+            # There is one operator
+            (r'=', Operator),
+
+            # Keywords
+            (words(('facet', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword),
+            (words((
+                'installer', 'extends', 'exports', 'imports', 'facets',
+                'children'), suffix=r'\s*:?', prefix=r'\b'), Name),
+
+            # Comments
+            (r'#.*\n', Comment),
+
+            # Default
+            (r'[^#]', Text),
+            (r'.*\n', Text)
+        ]
+    }
+
+
+class RoboconfInstancesLexer(RegexLexer):
+    """
+    Lexer for Roboconf instances files.
+    """
+    name = 'Roboconf Instances'
+    aliases = ['roboconf-instances']
+    filenames = ['*.instances']
+    url = 'https://roboconf.github.io'
+    version_added = '2.1'
+
+    flags = re.IGNORECASE | re.MULTILINE
+    tokens = {
+        'root': [
+
+            # Skip white spaces
+            (r'\s+', Text),
+
+            # Keywords
+            (words(('instance of', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword),
+            (words(('name', 'count'), suffix=r's*:?', prefix=r'\b'), Name),
+            (r'\s*[\w.-]+\s*:', Name),
+
+            # Comments
+            (r'#.*\n', Comment),
+
+            # Default
+            (r'[^#]', Text),
+            (r'.*\n', Text)
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/robotframework.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/robotframework.py
new file mode 100644
index 0000000000000000000000000000000000000000..a481136cc7b8c08182e10fc5c57d9aa41ceedaf1
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/robotframework.py
@@ -0,0 +1,551 @@
+"""
+    pygments.lexers.robotframework
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Robot Framework.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+#  Copyright 2012 Nokia Siemens Networks Oyj
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import re
+
+from pygments.lexer import Lexer
+from pygments.token import Token
+
+__all__ = ['RobotFrameworkLexer']
+
+
+HEADING = Token.Generic.Heading
+SETTING = Token.Keyword.Namespace
+IMPORT = Token.Name.Namespace
+TC_KW_NAME = Token.Generic.Subheading
+KEYWORD = Token.Name.Function
+ARGUMENT = Token.String
+VARIABLE = Token.Name.Variable
+COMMENT = Token.Comment
+SEPARATOR = Token.Punctuation
+SYNTAX = Token.Punctuation
+GHERKIN = Token.Generic.Emph
+ERROR = Token.Error
+
+
+def normalize(string, remove=''):
+    string = string.lower()
+    for char in remove + ' ':
+        if char in string:
+            string = string.replace(char, '')
+    return string
+
+
+class RobotFrameworkLexer(Lexer):
+    """
+    For Robot Framework test data.
+
+    Supports both space and pipe separated plain text formats.
+    """
+    name = 'RobotFramework'
+    url = 'http://robotframework.org'
+    aliases = ['robotframework']
+    filenames = ['*.robot', '*.resource']
+    mimetypes = ['text/x-robotframework']
+    version_added = '1.6'
+
+    def __init__(self, **options):
+        options['tabsize'] = 2
+        options['encoding'] = 'UTF-8'
+        Lexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        row_tokenizer = RowTokenizer()
+        var_tokenizer = VariableTokenizer()
+        index = 0
+        for row in text.splitlines():
+            for value, token in row_tokenizer.tokenize(row):
+                for value, token in var_tokenizer.tokenize(value, token):
+                    if value:
+                        yield index, token, str(value)
+                        index += len(value)
+
+
+class VariableTokenizer:
+
+    def tokenize(self, string, token):
+        var = VariableSplitter(string, identifiers='$@%&')
+        if var.start < 0 or token in (COMMENT, ERROR):
+            yield string, token
+            return
+        for value, token in self._tokenize(var, string, token):
+            if value:
+                yield value, token
+
+    def _tokenize(self, var, string, orig_token):
+        before = string[:var.start]
+        yield before, orig_token
+        yield var.identifier + '{', SYNTAX
+        yield from self.tokenize(var.base, VARIABLE)
+        yield '}', SYNTAX
+        if var.index is not None:
+            yield '[', SYNTAX
+            yield from self.tokenize(var.index, VARIABLE)
+            yield ']', SYNTAX
+        yield from self.tokenize(string[var.end:], orig_token)
+
+
+class RowTokenizer:
+
+    def __init__(self):
+        self._table = UnknownTable()
+        self._splitter = RowSplitter()
+        testcases = TestCaseTable()
+        settings = SettingTable(testcases.set_default_template)
+        variables = VariableTable()
+        keywords = KeywordTable()
+        self._tables = {'settings': settings, 'setting': settings,
+                        'metadata': settings,
+                        'variables': variables, 'variable': variables,
+                        'testcases': testcases, 'testcase': testcases,
+                        'tasks': testcases, 'task': testcases,
+                        'keywords': keywords, 'keyword': keywords,
+                        'userkeywords': keywords, 'userkeyword': keywords}
+
+    def tokenize(self, row):
+        commented = False
+        heading = False
+        for index, value in enumerate(self._splitter.split(row)):
+            # First value, and every second after that, is a separator.
+            index, separator = divmod(index-1, 2)
+            if value.startswith('#'):
+                commented = True
+            elif index == 0 and value.startswith('*'):
+                self._table = self._start_table(value)
+                heading = True
+            yield from self._tokenize(value, index, commented,
+                                      separator, heading)
+        self._table.end_row()
+
+    def _start_table(self, header):
+        name = normalize(header, remove='*')
+        return self._tables.get(name, UnknownTable())
+
+    def _tokenize(self, value, index, commented, separator, heading):
+        if commented:
+            yield value, COMMENT
+        elif separator:
+            yield value, SEPARATOR
+        elif heading:
+            yield value, HEADING
+        else:
+            yield from self._table.tokenize(value, index)
+
+
+class RowSplitter:
+    _space_splitter = re.compile('( {2,})')
+    _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))')
+
+    def split(self, row):
+        splitter = (row.startswith('| ') and self._split_from_pipes
+                    or self._split_from_spaces)
+        yield from splitter(row)
+        yield '\n'
+
+    def _split_from_spaces(self, row):
+        yield ''  # Start with (pseudo)separator similarly as with pipes
+        yield from self._space_splitter.split(row)
+
+    def _split_from_pipes(self, row):
+        _, separator, rest = self._pipe_splitter.split(row, 1)
+        yield separator
+        while self._pipe_splitter.search(rest):
+            cell, separator, rest = self._pipe_splitter.split(rest, 1)
+            yield cell
+            yield separator
+        yield rest
+
+
+class Tokenizer:
+    _tokens = None
+
+    def __init__(self):
+        self._index = 0
+
+    def tokenize(self, value):
+        values_and_tokens = self._tokenize(value, self._index)
+        self._index += 1
+        if isinstance(values_and_tokens, type(Token)):
+            values_and_tokens = [(value, values_and_tokens)]
+        return values_and_tokens
+
+    def _tokenize(self, value, index):
+        index = min(index, len(self._tokens) - 1)
+        return self._tokens[index]
+
+    def _is_assign(self, value):
+        if value.endswith('='):
+            value = value[:-1].strip()
+        var = VariableSplitter(value, identifiers='$@&')
+        return var.start == 0 and var.end == len(value)
+
+
+class Comment(Tokenizer):
+    _tokens = (COMMENT,)
+
+
+class Setting(Tokenizer):
+    _tokens = (SETTING, ARGUMENT)
+    _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown',
+                         'suitepostcondition', 'testsetup', 'tasksetup', 'testprecondition',
+                         'testteardown','taskteardown', 'testpostcondition', 'testtemplate', 'tasktemplate')
+    _import_settings = ('library', 'resource', 'variables')
+    _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags',
+                       'testtimeout','tasktimeout')
+    _custom_tokenizer = None
+
+    def __init__(self, template_setter=None):
+        Tokenizer.__init__(self)
+        self._template_setter = template_setter
+
+    def _tokenize(self, value, index):
+        if index == 1 and self._template_setter:
+            self._template_setter(value)
+        if index == 0:
+            normalized = normalize(value)
+            if normalized in self._keyword_settings:
+                self._custom_tokenizer = KeywordCall(support_assign=False)
+            elif normalized in self._import_settings:
+                self._custom_tokenizer = ImportSetting()
+            elif normalized not in self._other_settings:
+                return ERROR
+        elif self._custom_tokenizer:
+            return self._custom_tokenizer.tokenize(value)
+        return Tokenizer._tokenize(self, value, index)
+
+
+class ImportSetting(Tokenizer):
+    _tokens = (IMPORT, ARGUMENT)
+
+
+class TestCaseSetting(Setting):
+    _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition',
+                         'template')
+    _import_settings = ()
+    _other_settings = ('documentation', 'tags', 'timeout')
+
+    def _tokenize(self, value, index):
+        if index == 0:
+            type = Setting._tokenize(self, value[1:-1], index)
+            return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)]
+        return Setting._tokenize(self, value, index)
+
+
+class KeywordSetting(TestCaseSetting):
+    _keyword_settings = ('teardown',)
+    _other_settings = ('documentation', 'arguments', 'return', 'timeout', 'tags')
+
+
+class Variable(Tokenizer):
+    _tokens = (SYNTAX, ARGUMENT)
+
+    def _tokenize(self, value, index):
+        if index == 0 and not self._is_assign(value):
+            return ERROR
+        return Tokenizer._tokenize(self, value, index)
+
+
+class KeywordCall(Tokenizer):
+    _tokens = (KEYWORD, ARGUMENT)
+
+    def __init__(self, support_assign=True):
+        Tokenizer.__init__(self)
+        self._keyword_found = not support_assign
+        self._assigns = 0
+
+    def _tokenize(self, value, index):
+        if not self._keyword_found and self._is_assign(value):
+            self._assigns += 1
+            return SYNTAX  # VariableTokenizer tokenizes this later.
+        if self._keyword_found:
+            return Tokenizer._tokenize(self, value, index - self._assigns)
+        self._keyword_found = True
+        return GherkinTokenizer().tokenize(value, KEYWORD)
+
+
+class GherkinTokenizer:
+    _gherkin_prefix = re.compile('^(Given|When|Then|And|But) ', re.IGNORECASE)
+
+    def tokenize(self, value, token):
+        match = self._gherkin_prefix.match(value)
+        if not match:
+            return [(value, token)]
+        end = match.end()
+        return [(value[:end], GHERKIN), (value[end:], token)]
+
+
+class TemplatedKeywordCall(Tokenizer):
+    _tokens = (ARGUMENT,)
+
+
+class ForLoop(Tokenizer):
+
+    def __init__(self):
+        Tokenizer.__init__(self)
+        self._in_arguments = False
+
+    def _tokenize(self, value, index):
+        token = self._in_arguments and ARGUMENT or SYNTAX
+        if value.upper() in ('IN', 'IN RANGE'):
+            self._in_arguments = True
+        return token
+
+
+class _Table:
+    _tokenizer_class = None
+
+    def __init__(self, prev_tokenizer=None):
+        self._tokenizer = self._tokenizer_class()
+        self._prev_tokenizer = prev_tokenizer
+        self._prev_values_on_row = []
+
+    def tokenize(self, value, index):
+        if self._continues(value, index):
+            self._tokenizer = self._prev_tokenizer
+            yield value, SYNTAX
+        else:
+            yield from self._tokenize(value, index)
+        self._prev_values_on_row.append(value)
+
+    def _continues(self, value, index):
+        return value == '...' and all(self._is_empty(t)
+                                      for t in self._prev_values_on_row)
+
+    def _is_empty(self, value):
+        return value in ('', '\\')
+
+    def _tokenize(self, value, index):
+        return self._tokenizer.tokenize(value)
+
+    def end_row(self):
+        self.__init__(prev_tokenizer=self._tokenizer)
+
+
+class UnknownTable(_Table):
+    _tokenizer_class = Comment
+
+    def _continues(self, value, index):
+        return False
+
+
+class VariableTable(_Table):
+    _tokenizer_class = Variable
+
+
+class SettingTable(_Table):
+    _tokenizer_class = Setting
+
+    def __init__(self, template_setter, prev_tokenizer=None):
+        _Table.__init__(self, prev_tokenizer)
+        self._template_setter = template_setter
+
+    def _tokenize(self, value, index):
+        if index == 0 and normalize(value) == 'testtemplate':
+            self._tokenizer = Setting(self._template_setter)
+        return _Table._tokenize(self, value, index)
+
+    def end_row(self):
+        self.__init__(self._template_setter, prev_tokenizer=self._tokenizer)
+
+
+class TestCaseTable(_Table):
+    _setting_class = TestCaseSetting
+    _test_template = None
+    _default_template = None
+
+    @property
+    def _tokenizer_class(self):
+        if self._test_template or (self._default_template and
+                                   self._test_template is not False):
+            return TemplatedKeywordCall
+        return KeywordCall
+
+    def _continues(self, value, index):
+        return index > 0 and _Table._continues(self, value, index)
+
+    def _tokenize(self, value, index):
+        if index == 0:
+            if value:
+                self._test_template = None
+            return GherkinTokenizer().tokenize(value, TC_KW_NAME)
+        if index == 1 and self._is_setting(value):
+            if self._is_template(value):
+                self._test_template = False
+                self._tokenizer = self._setting_class(self.set_test_template)
+            else:
+                self._tokenizer = self._setting_class()
+        if index == 1 and self._is_for_loop(value):
+            self._tokenizer = ForLoop()
+        if index == 1 and self._is_empty(value):
+            return [(value, SYNTAX)]
+        return _Table._tokenize(self, value, index)
+
+    def _is_setting(self, value):
+        return value.startswith('[') and value.endswith(']')
+
+    def _is_template(self, value):
+        return normalize(value) == '[template]'
+
+    def _is_for_loop(self, value):
+        return value.startswith(':') and normalize(value, remove=':') == 'for'
+
+    def set_test_template(self, template):
+        self._test_template = self._is_template_set(template)
+
+    def set_default_template(self, template):
+        self._default_template = self._is_template_set(template)
+
+    def _is_template_set(self, template):
+        return normalize(template) not in ('', '\\', 'none', '${empty}')
+
+
+class KeywordTable(TestCaseTable):
+    _tokenizer_class = KeywordCall
+    _setting_class = KeywordSetting
+
+    def _is_template(self, value):
+        return False
+
+
+# Following code copied directly from Robot Framework 2.7.5.
+
+class VariableSplitter:
+
+    def __init__(self, string, identifiers):
+        self.identifier = None
+        self.base = None
+        self.index = None
+        self.start = -1
+        self.end = -1
+        self._identifiers = identifiers
+        self._may_have_internal_variables = False
+        try:
+            self._split(string)
+        except ValueError:
+            pass
+        else:
+            self._finalize()
+
+    def get_replaced_base(self, variables):
+        if self._may_have_internal_variables:
+            return variables.replace_string(self.base)
+        return self.base
+
+    def _finalize(self):
+        self.identifier = self._variable_chars[0]
+        self.base = ''.join(self._variable_chars[2:-1])
+        self.end = self.start + len(self._variable_chars)
+        if self._has_list_or_dict_variable_index():
+            self.index = ''.join(self._list_and_dict_variable_index_chars[1:-1])
+            self.end += len(self._list_and_dict_variable_index_chars)
+
+    def _has_list_or_dict_variable_index(self):
+        return self._list_and_dict_variable_index_chars\
+        and self._list_and_dict_variable_index_chars[-1] == ']'
+
+    def _split(self, string):
+        start_index, max_index = self._find_variable(string)
+        self.start = start_index
+        self._open_curly = 1
+        self._state = self._variable_state
+        self._variable_chars = [string[start_index], '{']
+        self._list_and_dict_variable_index_chars = []
+        self._string = string
+        start_index += 2
+        for index, char in enumerate(string[start_index:]):
+            index += start_index  # Giving start to enumerate only in Py 2.6+
+            try:
+                self._state(char, index)
+            except StopIteration:
+                return
+            if index  == max_index and not self._scanning_list_variable_index():
+                return
+
+    def _scanning_list_variable_index(self):
+        return self._state in [self._waiting_list_variable_index_state,
+                               self._list_variable_index_state]
+
+    def _find_variable(self, string):
+        max_end_index = string.rfind('}')
+        if max_end_index == -1:
+            raise ValueError('No variable end found')
+        if self._is_escaped(string, max_end_index):
+            return self._find_variable(string[:max_end_index])
+        start_index = self._find_start_index(string, 1, max_end_index)
+        if start_index == -1:
+            raise ValueError('No variable start found')
+        return start_index, max_end_index
+
+    def _find_start_index(self, string, start, end):
+        index = string.find('{', start, end) - 1
+        if index < 0:
+            return -1
+        if self._start_index_is_ok(string, index):
+            return index
+        return self._find_start_index(string, index+2, end)
+
+    def _start_index_is_ok(self, string, index):
+        return string[index] in self._identifiers\
+        and not self._is_escaped(string, index)
+
+    def _is_escaped(self, string, index):
+        escaped = False
+        while index > 0 and string[index-1] == '\\':
+            index -= 1
+            escaped = not escaped
+        return escaped
+
+    def _variable_state(self, char, index):
+        self._variable_chars.append(char)
+        if char == '}' and not self._is_escaped(self._string, index):
+            self._open_curly -= 1
+            if self._open_curly == 0:
+                if not self._is_list_or_dict_variable():
+                    raise StopIteration
+                self._state = self._waiting_list_variable_index_state
+        elif char in self._identifiers:
+            self._state = self._internal_variable_start_state
+
+    def _is_list_or_dict_variable(self):
+        return self._variable_chars[0] in ('@','&')
+
+    def _internal_variable_start_state(self, char, index):
+        self._state = self._variable_state
+        if char == '{':
+            self._variable_chars.append(char)
+            self._open_curly += 1
+            self._may_have_internal_variables = True
+        else:
+            self._variable_state(char, index)
+
+    def _waiting_list_variable_index_state(self, char, index):
+        if char != '[':
+            raise StopIteration
+        self._list_and_dict_variable_index_chars.append(char)
+        self._state = self._list_variable_index_state
+
+    def _list_variable_index_state(self, char, index):
+        self._list_and_dict_variable_index_chars.append(char)
+        if char == ']':
+            raise StopIteration
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ruby.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ruby.py
new file mode 100644
index 0000000000000000000000000000000000000000..6832652a6cf52d22520a14a9adb78c8366b1f724
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/ruby.py
@@ -0,0 +1,518 @@
+"""
+    pygments.lexers.ruby
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Ruby and related languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, ExtendedRegexLexer, include, \
+    bygroups, default, LexerContext, do_insertions, words, line_re
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error, Generic, Whitespace
+from pygments.util import shebang_matches
+
+__all__ = ['RubyLexer', 'RubyConsoleLexer', 'FancyLexer']
+
+
+RUBY_OPERATORS = (
+    '*', '**', '-', '+', '-@', '+@', '/', '%', '&', '|', '^', '`', '~',
+    '[]', '[]=', '<<', '>>', '<', '<>', '<=>', '>', '>=', '==', '==='
+)
+
+
+class RubyLexer(ExtendedRegexLexer):
+    """
+    For Ruby source code.
+    """
+
+    name = 'Ruby'
+    url = 'http://www.ruby-lang.org'
+    aliases = ['ruby', 'rb', 'duby']
+    filenames = ['*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec',
+                 '*.rbx', '*.duby', 'Gemfile', 'Vagrantfile']
+    mimetypes = ['text/x-ruby', 'application/x-ruby']
+    version_added = ''
+
+    flags = re.DOTALL | re.MULTILINE
+
+    def heredoc_callback(self, match, ctx):
+        # okay, this is the hardest part of parsing Ruby...
+        # match: 1 = <<[-~]?, 2 = quote? 3 = name 4 = quote? 5 = rest of line
+
+        start = match.start(1)
+        yield start, Operator, match.group(1)        # <<[-~]?
+        yield match.start(2), String.Heredoc, match.group(2)   # quote ", ', `
+        yield match.start(3), String.Delimiter, match.group(3) # heredoc name
+        yield match.start(4), String.Heredoc, match.group(4)   # quote again
+
+        heredocstack = ctx.__dict__.setdefault('heredocstack', [])
+        outermost = not bool(heredocstack)
+        heredocstack.append((match.group(1) in ('<<-', '<<~'), match.group(3)))
+
+        ctx.pos = match.start(5)
+        ctx.end = match.end(5)
+        # this may find other heredocs, so limit the recursion depth
+        if len(heredocstack) < 100:
+            yield from self.get_tokens_unprocessed(context=ctx)
+        else:
+            yield ctx.pos, String.Heredoc, match.group(5)
+        ctx.pos = match.end()
+
+        if outermost:
+            # this is the outer heredoc again, now we can process them all
+            for tolerant, hdname in heredocstack:
+                lines = []
+                for match in line_re.finditer(ctx.text, ctx.pos):
+                    if tolerant:
+                        check = match.group().strip()
+                    else:
+                        check = match.group().rstrip()
+                    if check == hdname:
+                        for amatch in lines:
+                            yield amatch.start(), String.Heredoc, amatch.group()
+                        yield match.start(), String.Delimiter, match.group()
+                        ctx.pos = match.end()
+                        break
+                    else:
+                        lines.append(match)
+                else:
+                    # end of heredoc not found -- error!
+                    for amatch in lines:
+                        yield amatch.start(), Error, amatch.group()
+            ctx.end = len(ctx.text)
+            del heredocstack[:]
+
+    def gen_rubystrings_rules():
+        def intp_regex_callback(self, match, ctx):
+            yield match.start(1), String.Regex, match.group(1)  # begin
+            nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
+            for i, t, v in self.get_tokens_unprocessed(context=nctx):
+                yield match.start(3)+i, t, v
+            yield match.start(4), String.Regex, match.group(4)  # end[mixounse]*
+            ctx.pos = match.end()
+
+        def intp_string_callback(self, match, ctx):
+            yield match.start(1), String.Other, match.group(1)
+            nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
+            for i, t, v in self.get_tokens_unprocessed(context=nctx):
+                yield match.start(3)+i, t, v
+            yield match.start(4), String.Other, match.group(4)  # end
+            ctx.pos = match.end()
+
+        states = {}
+        states['strings'] = [
+            # easy ones
+            (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol),
+            (words(RUBY_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol),
+            (r":'(\\\\|\\[^\\]|[^'\\])*'", String.Symbol),
+            (r':"', String.Symbol, 'simple-sym'),
+            (r'([a-zA-Z_]\w*)(:)(?!:)',
+             bygroups(String.Symbol, Punctuation)),  # Since Ruby 1.9
+            (r'"', String.Double, 'simple-string-double'),
+            (r"'", String.Single, 'simple-string-single'),
+            (r'(?', '<>', 'ab'):
+            states[name+'-intp-string'] = [
+                (r'\\[\\' + bracecc + ']', String.Other),
+                (lbrace, String.Other, '#push'),
+                (rbrace, String.Other, '#pop'),
+                include('string-intp-escaped'),
+                (r'[\\#' + bracecc + ']', String.Other),
+                (r'[^\\#' + bracecc + ']+', String.Other),
+            ]
+            states['strings'].append((r'%[QWx]?' + lbrace, String.Other,
+                                      name+'-intp-string'))
+            states[name+'-string'] = [
+                (r'\\[\\' + bracecc + ']', String.Other),
+                (lbrace, String.Other, '#push'),
+                (rbrace, String.Other, '#pop'),
+                (r'[\\#' + bracecc + ']', String.Other),
+                (r'[^\\#' + bracecc + ']+', String.Other),
+            ]
+            states['strings'].append((r'%[qsw]' + lbrace, String.Other,
+                                      name+'-string'))
+            states[name+'-regex'] = [
+                (r'\\[\\' + bracecc + ']', String.Regex),
+                (lbrace, String.Regex, '#push'),
+                (rbrace + '[mixounse]*', String.Regex, '#pop'),
+                include('string-intp'),
+                (r'[\\#' + bracecc + ']', String.Regex),
+                (r'[^\\#' + bracecc + ']+', String.Regex),
+            ]
+            states['strings'].append((r'%r' + lbrace, String.Regex,
+                                      name+'-regex'))
+
+        # these must come after %!
+        states['strings'] += [
+            # %r regex
+            (r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[mixounse]*)',
+             intp_regex_callback),
+            # regular fancy strings with qsw
+            (r'%[qsw]([\W_])((?:\\\1|(?!\1).)*)\1', String.Other),
+            (r'(%[QWx]([\W_]))((?:\\\2|(?!\2).)*)(\2)',
+             intp_string_callback),
+            # special forms of fancy strings after operators or
+            # in method calls with braces
+            (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+             bygroups(Whitespace, String.Other, None)),
+            # and because of fixed width lookbehinds the whole thing a
+            # second time for line startings...
+            (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+             bygroups(Whitespace, String.Other, None)),
+            # all regular fancy strings without qsw
+            (r'(%([^a-zA-Z0-9\s]))((?:\\\2|(?!\2).)*)(\2)',
+             intp_string_callback),
+        ]
+
+        return states
+
+    tokens = {
+        'root': [
+            (r'\A#!.+?$', Comment.Hashbang),
+            (r'#.*?$', Comment.Single),
+            (r'=begin\s.*?\n=end.*?$', Comment.Multiline),
+            # keywords
+            (words((
+                'BEGIN', 'END', 'alias', 'begin', 'break', 'case', 'defined?',
+                'do', 'else', 'elsif', 'end', 'ensure', 'for', 'if', 'in', 'next', 'redo',
+                'rescue', 'raise', 'retry', 'return', 'super', 'then', 'undef',
+                'unless', 'until', 'when', 'while', 'yield'), suffix=r'\b'),
+             Keyword),
+            # start of function, class and module names
+            (r'(module)(\s+)([a-zA-Z_]\w*'
+             r'(?:::[a-zA-Z_]\w*)*)',
+             bygroups(Keyword, Whitespace, Name.Namespace)),
+            (r'(def)(\s+)', bygroups(Keyword, Whitespace), 'funcname'),
+            (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'),
+            (r'(class)(\s+)', bygroups(Keyword, Whitespace), 'classname'),
+            # special methods
+            (words((
+                'initialize', 'new', 'loop', 'include', 'extend', 'raise', 'attr_reader',
+                'attr_writer', 'attr_accessor', 'attr', 'catch', 'throw', 'private',
+                'module_function', 'public', 'protected', 'true', 'false', 'nil'),
+                suffix=r'\b'),
+             Keyword.Pseudo),
+            (r'(not|and|or)\b', Operator.Word),
+            (words((
+                'autoload', 'block_given', 'const_defined', 'eql', 'equal', 'frozen', 'include',
+                'instance_of', 'is_a', 'iterator', 'kind_of', 'method_defined', 'nil',
+                'private_method_defined', 'protected_method_defined',
+                'public_method_defined', 'respond_to', 'tainted'), suffix=r'\?'),
+             Name.Builtin),
+            (r'(chomp|chop|exit|gsub|sub)!', Name.Builtin),
+            (words((
+                'Array', 'Float', 'Integer', 'String', '__id__', '__send__', 'abort',
+                'ancestors', 'at_exit', 'autoload', 'binding', 'callcc', 'caller',
+                'catch', 'chomp', 'chop', 'class_eval', 'class_variables',
+                'clone', 'const_defined?', 'const_get', 'const_missing', 'const_set',
+                'constants', 'display', 'dup', 'eval', 'exec', 'exit', 'extend', 'fail', 'fork',
+                'format', 'freeze', 'getc', 'gets', 'global_variables', 'gsub',
+                'hash', 'id', 'included_modules', 'inspect', 'instance_eval',
+                'instance_method', 'instance_methods',
+                'instance_variable_get', 'instance_variable_set', 'instance_variables',
+                'lambda', 'load', 'local_variables', 'loop',
+                'method', 'method_missing', 'methods', 'module_eval', 'name',
+                'object_id', 'open', 'p', 'print', 'printf', 'private_class_method',
+                'private_instance_methods',
+                'private_methods', 'proc', 'protected_instance_methods',
+                'protected_methods', 'public_class_method',
+                'public_instance_methods', 'public_methods',
+                'putc', 'puts', 'raise', 'rand', 'readline', 'readlines', 'require',
+                'scan', 'select', 'self', 'send', 'set_trace_func', 'singleton_methods', 'sleep',
+                'split', 'sprintf', 'srand', 'sub', 'syscall', 'system', 'taint',
+                'test', 'throw', 'to_a', 'to_s', 'trace_var', 'trap', 'untaint',
+                'untrace_var', 'warn'), prefix=r'(?~!:])|'
+             r'(?<=(?:\s|;)when\s)|'
+             r'(?<=(?:\s|;)or\s)|'
+             r'(?<=(?:\s|;)and\s)|'
+             r'(?<=\.index\s)|'
+             r'(?<=\.scan\s)|'
+             r'(?<=\.sub\s)|'
+             r'(?<=\.sub!\s)|'
+             r'(?<=\.gsub\s)|'
+             r'(?<=\.gsub!\s)|'
+             r'(?<=\.match\s)|'
+             r'(?<=(?:\s|;)if\s)|'
+             r'(?<=(?:\s|;)elsif\s)|'
+             r'(?<=^when\s)|'
+             r'(?<=^index\s)|'
+             r'(?<=^scan\s)|'
+             r'(?<=^sub\s)|'
+             r'(?<=^gsub\s)|'
+             r'(?<=^sub!\s)|'
+             r'(?<=^gsub!\s)|'
+             r'(?<=^match\s)|'
+             r'(?<=^if\s)|'
+             r'(?<=^elsif\s)'
+             r')(\s*)(/)', bygroups(Text, String.Regex), 'multiline-regex'),
+            # multiline regex (in method calls or subscripts)
+            (r'(?<=\(|,|\[)/', String.Regex, 'multiline-regex'),
+            # multiline regex (this time the funny no whitespace rule)
+            (r'(\s+)(/)(?![\s=])', bygroups(Whitespace, String.Regex),
+             'multiline-regex'),
+            # lex numbers and ignore following regular expressions which
+            # are division operators in fact (grrrr. i hate that. any
+            # better ideas?)
+            # since pygments 0.7 we also eat a "?" operator after numbers
+            # so that the char operator does not work. Chars are not allowed
+            # there so that you can use the ternary operator.
+            # stupid example:
+            #   x>=0?n[x]:""
+            (r'(0_?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?',
+             bygroups(Number.Oct, Whitespace, Operator)),
+            (r'(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?',
+             bygroups(Number.Hex, Whitespace, Operator)),
+            (r'(0b[01]+(?:_[01]+)*)(\s*)([/?])?',
+             bygroups(Number.Bin, Whitespace, Operator)),
+            (r'([\d]+(?:_\d+)*)(\s*)([/?])?',
+             bygroups(Number.Integer, Whitespace, Operator)),
+            # Names
+            (r'@@[a-zA-Z_]\w*', Name.Variable.Class),
+            (r'@[a-zA-Z_]\w*', Name.Variable.Instance),
+            (r'\$\w+', Name.Variable.Global),
+            (r'\$[!@&`\'+~=/\\,;.<>_*$?:"^-]', Name.Variable.Global),
+            (r'\$-[0adFiIlpvw]', Name.Variable.Global),
+            (r'::', Operator),
+            include('strings'),
+            # chars
+            (r'\?(\\[MC]-)*'  # modifiers
+             r'(\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)'
+             r'(?!\w)',
+             String.Char),
+            (r'[A-Z]\w+', Name.Constant),
+            # this is needed because ruby attributes can look
+            # like keywords (class) or like this: ` ?!?
+            (words(RUBY_OPERATORS, prefix=r'(\.|::)'),
+             bygroups(Operator, Name.Operator)),
+            (r'(\.|::)([a-zA-Z_]\w*[!?]?|[*%&^`~+\-/\[<>=])',
+             bygroups(Operator, Name)),
+            (r'[a-zA-Z_]\w*[!?]?', Name),
+            (r'(\[|\]|\*\*|<>?|>=|<=|<=>|=~|={3}|'
+             r'!~|&&?|\|\||\.{1,3})', Operator),
+            (r'[-+/*%=<>&!^|~]=?', Operator),
+            (r'[(){};,/?:\\]', Punctuation),
+            (r'\s+', Whitespace)
+        ],
+        'funcname': [
+            (r'\(', Punctuation, 'defexpr'),
+            (r'(?:([a-zA-Z_]\w*)(\.))?'  # optional scope name, like "self."
+             r'('
+                r'[a-zA-Z\u0080-\uffff][a-zA-Z0-9_\u0080-\uffff]*[!?=]?'  # method name
+                r'|!=|!~|=~|\*\*?|[-+!~]@?|[/%&|^]|<=>|<[<=]?|>[>=]?|===?'  # or operator override
+                r'|\[\]=?'  # or element reference/assignment override
+                r'|`'  # or the undocumented backtick override
+             r')',
+             bygroups(Name.Class, Operator, Name.Function), '#pop'),
+            default('#pop')
+        ],
+        'classname': [
+            (r'\(', Punctuation, 'defexpr'),
+            (r'<<', Operator, '#pop'),
+            (r'[A-Z_]\w*', Name.Class, '#pop'),
+            default('#pop')
+        ],
+        'defexpr': [
+            (r'(\))(\.|::)?', bygroups(Punctuation, Operator), '#pop'),
+            (r'\(', Operator, '#push'),
+            include('root')
+        ],
+        'in-intp': [
+            (r'\{', String.Interpol, '#push'),
+            (r'\}', String.Interpol, '#pop'),
+            include('root'),
+        ],
+        'string-intp': [
+            (r'#\{', String.Interpol, 'in-intp'),
+            (r'#@@?[a-zA-Z_]\w*', String.Interpol),
+            (r'#\$[a-zA-Z_]\w*', String.Interpol)
+        ],
+        'string-intp-escaped': [
+            include('string-intp'),
+            (r'\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})',
+             String.Escape)
+        ],
+        'interpolated-regex': [
+            include('string-intp'),
+            (r'[\\#]', String.Regex),
+            (r'[^\\#]+', String.Regex),
+        ],
+        'interpolated-string': [
+            include('string-intp'),
+            (r'[\\#]', String.Other),
+            (r'[^\\#]+', String.Other),
+        ],
+        'multiline-regex': [
+            include('string-intp'),
+            (r'\\\\', String.Regex),
+            (r'\\/', String.Regex),
+            (r'[\\#]', String.Regex),
+            (r'[^\\/#]+', String.Regex),
+            (r'/[mixounse]*', String.Regex, '#pop'),
+        ],
+        'end-part': [
+            (r'.+', Comment.Preproc, '#pop')
+        ]
+    }
+    tokens.update(gen_rubystrings_rules())
+
+    def analyse_text(text):
+        return shebang_matches(text, r'ruby(1\.\d)?')
+
+
+class RubyConsoleLexer(Lexer):
+    """
+    For Ruby interactive console (**irb**) output.
+    """
+    name = 'Ruby irb session'
+    aliases = ['rbcon', 'irb']
+    mimetypes = ['text/x-ruby-shellsession']
+    url = 'https://www.ruby-lang.org'
+    version_added = ''
+    _example = 'rbcon/console'
+
+    _prompt_re = re.compile(r'irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] '
+                            r'|>> |\?> ')
+
+    def get_tokens_unprocessed(self, text):
+        rblexer = RubyLexer(**self.options)
+
+        curcode = ''
+        insertions = []
+        for match in line_re.finditer(text):
+            line = match.group()
+            m = self._prompt_re.match(line)
+            if m is not None:
+                end = m.end()
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, line[:end])]))
+                curcode += line[end:]
+            else:
+                if curcode:
+                    yield from do_insertions(
+                        insertions, rblexer.get_tokens_unprocessed(curcode))
+                    curcode = ''
+                    insertions = []
+                yield match.start(), Generic.Output, line
+        if curcode:
+            yield from do_insertions(
+                insertions, rblexer.get_tokens_unprocessed(curcode))
+
+
+class FancyLexer(RegexLexer):
+    """
+    Pygments Lexer For Fancy.
+
+    Fancy is a self-hosted, pure object-oriented, dynamic,
+    class-based, concurrent general-purpose programming language
+    running on Rubinius, the Ruby VM.
+    """
+    name = 'Fancy'
+    url = 'https://github.com/bakkdoor/fancy'
+    filenames = ['*.fy', '*.fancypack']
+    aliases = ['fancy', 'fy']
+    mimetypes = ['text/x-fancysrc']
+    version_added = '1.5'
+
+    tokens = {
+        # copied from PerlLexer:
+        'balanced-regex': [
+            (r'/(\\\\|\\[^\\]|[^/\\])*/[egimosx]*', String.Regex, '#pop'),
+            (r'!(\\\\|\\[^\\]|[^!\\])*![egimosx]*', String.Regex, '#pop'),
+            (r'\\(\\\\|[^\\])*\\[egimosx]*', String.Regex, '#pop'),
+            (r'\{(\\\\|\\[^\\]|[^}\\])*\}[egimosx]*', String.Regex, '#pop'),
+            (r'<(\\\\|\\[^\\]|[^>\\])*>[egimosx]*', String.Regex, '#pop'),
+            (r'\[(\\\\|\\[^\\]|[^\]\\])*\][egimosx]*', String.Regex, '#pop'),
+            (r'\((\\\\|\\[^\\]|[^)\\])*\)[egimosx]*', String.Regex, '#pop'),
+            (r'@(\\\\|\\[^\\]|[^@\\])*@[egimosx]*', String.Regex, '#pop'),
+            (r'%(\\\\|\\[^\\]|[^%\\])*%[egimosx]*', String.Regex, '#pop'),
+            (r'\$(\\\\|\\[^\\]|[^$\\])*\$[egimosx]*', String.Regex, '#pop'),
+        ],
+        'root': [
+            (r'\s+', Whitespace),
+
+            # balanced delimiters (copied from PerlLexer):
+            (r's\{(\\\\|\\[^\\]|[^}\\])*\}\s*', String.Regex, 'balanced-regex'),
+            (r's<(\\\\|\\[^\\]|[^>\\])*>\s*', String.Regex, 'balanced-regex'),
+            (r's\[(\\\\|\\[^\\]|[^\]\\])*\]\s*', String.Regex, 'balanced-regex'),
+            (r's\((\\\\|\\[^\\]|[^)\\])*\)\s*', String.Regex, 'balanced-regex'),
+            (r'm?/(\\\\|\\[^\\]|[^///\n])*/[gcimosx]*', String.Regex),
+            (r'm(?=[/!\\{<\[(@%$])', String.Regex, 'balanced-regex'),
+
+            # Comments
+            (r'#(.*?)\n', Comment.Single),
+            # Symbols
+            (r'\'([^\'\s\[\](){}]+|\[\])', String.Symbol),
+            # Multi-line DoubleQuotedString
+            (r'"""(\\\\|\\[^\\]|[^\\])*?"""', String),
+            # DoubleQuotedString
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+            # keywords
+            (r'(def|class|try|catch|finally|retry|return|return_local|match|'
+             r'case|->|=>)\b', Keyword),
+            # constants
+            (r'(self|super|nil|false|true)\b', Name.Constant),
+            (r'[(){};,/?|:\\]', Punctuation),
+            # names
+            (words((
+                'Object', 'Array', 'Hash', 'Directory', 'File', 'Class', 'String',
+                'Number', 'Enumerable', 'FancyEnumerable', 'Block', 'TrueClass',
+                'NilClass', 'FalseClass', 'Tuple', 'Symbol', 'Stack', 'Set',
+                'FancySpec', 'Method', 'Package', 'Range'), suffix=r'\b'),
+             Name.Builtin),
+            # functions
+            (r'[a-zA-Z](\w|[-+?!=*/^><%])*:', Name.Function),
+            # operators, must be below functions
+            (r'[-+*/~,<>=&!?%^\[\].$]+', Operator),
+            (r'[A-Z]\w*', Name.Constant),
+            (r'@[a-zA-Z_]\w*', Name.Variable.Instance),
+            (r'@@[a-zA-Z_]\w*', Name.Variable.Class),
+            ('@@?', Operator),
+            (r'[a-zA-Z_]\w*', Name),
+            # numbers - / checks are necessary to avoid mismarking regexes,
+            # see comment in RubyLexer
+            (r'(0[oO]?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?',
+             bygroups(Number.Oct, Whitespace, Operator)),
+            (r'(0[xX][0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*)(\s*)([/?])?',
+             bygroups(Number.Hex, Whitespace, Operator)),
+            (r'(0[bB][01]+(?:_[01]+)*)(\s*)([/?])?',
+             bygroups(Number.Bin, Whitespace, Operator)),
+            (r'([\d]+(?:_\d+)*)(\s*)([/?])?',
+             bygroups(Number.Integer, Whitespace, Operator)),
+            (r'\d+([eE][+-]?[0-9]+)|\d+\.\d+([eE][+-]?[0-9]+)?', Number.Float),
+            (r'\d+', Number.Integer)
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rust.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rust.py
new file mode 100644
index 0000000000000000000000000000000000000000..6c9e6217a4e88ca15012c5ea57181a8eaa37e0cf
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/rust.py
@@ -0,0 +1,222 @@
+"""
+    pygments.lexers.rust
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Rust language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, words, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+
+__all__ = ['RustLexer']
+
+
+class RustLexer(RegexLexer):
+    """
+    Lexer for the Rust programming language (version 1.47).
+    """
+    name = 'Rust'
+    url = 'https://www.rust-lang.org/'
+    filenames = ['*.rs', '*.rs.in']
+    aliases = ['rust', 'rs']
+    mimetypes = ['text/rust', 'text/x-rust']
+    version_added = '1.6'
+
+    keyword_types = (words((
+        'u8', 'u16', 'u32', 'u64', 'u128', 'i8', 'i16', 'i32', 'i64', 'i128',
+        'usize', 'isize', 'f32', 'f64', 'char', 'str', 'bool',
+    ), suffix=r'\b'), Keyword.Type)
+
+    builtin_funcs_types = (words((
+        'Copy', 'Send', 'Sized', 'Sync', 'Unpin',
+        'Drop', 'Fn', 'FnMut', 'FnOnce', 'drop',
+        'Box', 'ToOwned', 'Clone',
+        'PartialEq', 'PartialOrd', 'Eq', 'Ord',
+        'AsRef', 'AsMut', 'Into', 'From', 'Default',
+        'Iterator', 'Extend', 'IntoIterator', 'DoubleEndedIterator',
+        'ExactSizeIterator',
+        'Option', 'Some', 'None',
+        'Result', 'Ok', 'Err',
+        'String', 'ToString', 'Vec',
+    ), suffix=r'\b'), Name.Builtin)
+
+    builtin_macros = (words((
+        'asm', 'assert', 'assert_eq', 'assert_ne', 'cfg', 'column',
+        'compile_error', 'concat', 'concat_idents', 'dbg', 'debug_assert',
+        'debug_assert_eq', 'debug_assert_ne', 'env', 'eprint', 'eprintln',
+        'file', 'format', 'format_args', 'format_args_nl', 'global_asm',
+        'include', 'include_bytes', 'include_str',
+        'is_aarch64_feature_detected',
+        'is_arm_feature_detected',
+        'is_mips64_feature_detected',
+        'is_mips_feature_detected',
+        'is_powerpc64_feature_detected',
+        'is_powerpc_feature_detected',
+        'is_x86_feature_detected',
+        'line', 'llvm_asm', 'log_syntax', 'macro_rules', 'matches',
+        'module_path', 'option_env', 'panic', 'print', 'println', 'stringify',
+        'thread_local', 'todo', 'trace_macros', 'unimplemented', 'unreachable',
+        'vec', 'write', 'writeln',
+    ), suffix=r'!'), Name.Function.Magic)
+
+    tokens = {
+        'root': [
+            # rust allows a file to start with a shebang, but if the first line
+            # starts with #![ then it's not a shebang but a crate attribute.
+            (r'#![^[\r\n].*$', Comment.Preproc),
+            default('base'),
+        ],
+        'base': [
+            # Whitespace and Comments
+            (r'\n', Whitespace),
+            (r'\s+', Whitespace),
+            (r'//!.*?\n', String.Doc),
+            (r'///(\n|[^/].*?\n)', String.Doc),
+            (r'//(.*?)\n', Comment.Single),
+            (r'/\*\*(\n|[^/*])', String.Doc, 'doccomment'),
+            (r'/\*!', String.Doc, 'doccomment'),
+            (r'/\*', Comment.Multiline, 'comment'),
+
+            # Macro parameters
+            (r"""\$([a-zA-Z_]\w*|\(,?|\),?|,?)""", Comment.Preproc),
+            # Keywords
+            (words(('as', 'async', 'await', 'box', 'const', 'crate', 'dyn',
+                    'else', 'extern', 'for', 'if', 'impl', 'in', 'loop',
+                    'match', 'move', 'mut', 'pub', 'ref', 'return', 'static',
+                    'super', 'trait', 'unsafe', 'use', 'where', 'while'),
+                   suffix=r'\b'), Keyword),
+            (words(('abstract', 'become', 'do', 'final', 'macro', 'override',
+                    'priv', 'typeof', 'try', 'unsized', 'virtual', 'yield'),
+                   suffix=r'\b'), Keyword.Reserved),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'self\b', Name.Builtin.Pseudo),
+            (r'mod\b', Keyword, 'modname'),
+            (r'let\b', Keyword.Declaration),
+            (r'fn\b', Keyword, 'funcname'),
+            (r'(struct|enum|type|union)\b', Keyword, 'typename'),
+            (r'(default)(\s+)(type|fn)\b', bygroups(Keyword, Whitespace, Keyword)),
+            keyword_types,
+            (r'[sS]elf\b', Name.Builtin.Pseudo),
+            # Prelude (taken from Rust's src/libstd/prelude.rs)
+            builtin_funcs_types,
+            builtin_macros,
+            # Path separators, so types don't catch them.
+            (r'::\b', Punctuation),
+            # Types in positions.
+            (r'(?::|->)', Punctuation, 'typename'),
+            # Labels
+            (r'(break|continue)(\b\s*)(\'[A-Za-z_]\w*)?',
+             bygroups(Keyword, Text.Whitespace, Name.Label)),
+
+            # Character literals
+            (r"""'(\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""",
+             String.Char),
+            (r"""b'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""",
+             String.Char),
+
+            # Binary literals
+            (r'0b[01_]+', Number.Bin, 'number_lit'),
+            # Octal literals
+            (r'0o[0-7_]+', Number.Oct, 'number_lit'),
+            # Hexadecimal literals
+            (r'0[xX][0-9a-fA-F_]+', Number.Hex, 'number_lit'),
+            # Decimal literals
+            (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|'
+             r'\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)', Number.Float,
+             'number_lit'),
+            (r'[0-9][0-9_]*', Number.Integer, 'number_lit'),
+
+            # String literals
+            (r'b"', String, 'bytestring'),
+            (r'"', String, 'string'),
+            (r'(?s)b?r(#*)".*?"\1', String),
+
+            # Lifetime names
+            (r"'", Operator, 'lifetime'),
+
+            # Operators and Punctuation
+            (r'\.\.=?', Operator),
+            (r'[{}()\[\],.;]', Punctuation),
+            (r'[+\-*/%&|<>^!~@=:?]', Operator),
+
+            # Identifiers
+            (r'[a-zA-Z_]\w*', Name),
+            # Raw identifiers
+            (r'r#[a-zA-Z_]\w*', Name),
+
+            # Attributes
+            (r'#!?\[', Comment.Preproc, 'attribute['),
+
+            # Misc
+            # Lone hashes: not used in Rust syntax, but allowed in macro
+            # arguments, most famously for quote::quote!()
+            (r'#', Punctuation),
+        ],
+        'comment': [
+            (r'[^*/]+', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline),
+        ],
+        'doccomment': [
+            (r'[^*/]+', String.Doc),
+            (r'/\*', String.Doc, '#push'),
+            (r'\*/', String.Doc, '#pop'),
+            (r'[*/]', String.Doc),
+        ],
+        'modname': [
+            (r'\s+', Whitespace),
+            (r'[a-zA-Z_]\w*', Name.Namespace, '#pop'),
+            default('#pop'),
+        ],
+        'funcname': [
+            (r'\s+', Whitespace),
+            (r'[a-zA-Z_]\w*', Name.Function, '#pop'),
+            default('#pop'),
+        ],
+        'typename': [
+            (r'\s+', Whitespace),
+            (r'&', Keyword.Pseudo),
+            (r"'", Operator, 'lifetime'),
+            builtin_funcs_types,
+            keyword_types,
+            (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+            default('#pop'),
+        ],
+        'lifetime': [
+            (r"(static|_)", Name.Builtin),
+            (r"[a-zA-Z_]+\w*", Name.Attribute),
+            default('#pop'),
+        ],
+        'number_lit': [
+            (r'[ui](8|16|32|64|size)', Keyword, '#pop'),
+            (r'f(32|64)', Keyword, '#pop'),
+            default('#pop'),
+        ],
+        'string': [
+            (r'"', String, '#pop'),
+            (r"""\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}""", String.Escape),
+            (r'[^\\"]+', String),
+            (r'\\', String),
+        ],
+        'bytestring': [
+            (r"""\\x[89a-fA-F][0-9a-fA-F]""", String.Escape),
+            include('string'),
+        ],
+        'attribute_common': [
+            (r'"', String, 'string'),
+            (r'\[', Comment.Preproc, 'attribute['),
+        ],
+        'attribute[': [
+            include('attribute_common'),
+            (r'\]', Comment.Preproc, '#pop'),
+            (r'[^"\]\[]+', Comment.Preproc),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sas.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sas.py
new file mode 100644
index 0000000000000000000000000000000000000000..35ee8541a9c8817f4050957bc62238a94f938e38
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sas.py
@@ -0,0 +1,227 @@
+"""
+    pygments.lexers.sas
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for SAS.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Comment, Keyword, Name, Number, String, Text, \
+    Other, Generic
+
+__all__ = ['SASLexer']
+
+
+class SASLexer(RegexLexer):
+    """
+    For SAS files.
+    """
+    # Syntax from syntax/sas.vim by James Kidd 
+
+    name      = 'SAS'
+    aliases   = ['sas']
+    filenames = ['*.SAS', '*.sas']
+    mimetypes = ['text/x-sas', 'text/sas', 'application/x-sas']
+    url = 'https://en.wikipedia.org/wiki/SAS_(software)'
+    version_added = '2.2'
+    flags     = re.IGNORECASE | re.MULTILINE
+
+    builtins_macros = (
+        "bquote", "nrbquote", "cmpres", "qcmpres", "compstor", "datatyp",
+        "display", "do", "else", "end", "eval", "global", "goto", "if",
+        "index", "input", "keydef", "label", "left", "length", "let",
+        "local", "lowcase", "macro", "mend", "nrquote",
+        "nrstr", "put", "qleft", "qlowcase", "qscan",
+        "qsubstr", "qsysfunc", "qtrim", "quote", "qupcase", "scan",
+        "str", "substr", "superq", "syscall", "sysevalf", "sysexec",
+        "sysfunc", "sysget", "syslput", "sysprod", "sysrc", "sysrput",
+        "then", "to", "trim", "unquote", "until", "upcase", "verify",
+        "while", "window"
+    )
+
+    builtins_conditionals = (
+        "do", "if", "then", "else", "end", "until", "while"
+    )
+
+    builtins_statements = (
+        "abort", "array", "attrib", "by", "call", "cards", "cards4",
+        "catname", "continue", "datalines", "datalines4", "delete", "delim",
+        "delimiter", "display", "dm", "drop", "endsas", "error", "file",
+        "filename", "footnote", "format", "goto", "in", "infile", "informat",
+        "input", "keep", "label", "leave", "length", "libname", "link",
+        "list", "lostcard", "merge", "missing", "modify", "options", "output",
+        "out", "page", "put", "redirect", "remove", "rename", "replace",
+        "retain", "return", "select", "set", "skip", "startsas", "stop",
+        "title", "update", "waitsas", "where", "window", "x", "systask"
+    )
+
+    builtins_sql = (
+        "add", "and", "alter", "as", "cascade", "check", "create",
+        "delete", "describe", "distinct", "drop", "foreign", "from",
+        "group", "having", "index", "insert", "into", "in", "key", "like",
+        "message", "modify", "msgtype", "not", "null", "on", "or",
+        "order", "primary", "references", "reset", "restrict", "select",
+        "set", "table", "unique", "update", "validate", "view", "where"
+    )
+
+    builtins_functions = (
+        "abs", "addr", "airy", "arcos", "arsin", "atan", "attrc",
+        "attrn", "band", "betainv", "blshift", "bnot", "bor",
+        "brshift", "bxor", "byte", "cdf", "ceil", "cexist", "cinv",
+        "close", "cnonct", "collate", "compbl", "compound",
+        "compress", "cos", "cosh", "css", "curobs", "cv", "daccdb",
+        "daccdbsl", "daccsl", "daccsyd", "dacctab", "dairy", "date",
+        "datejul", "datepart", "datetime", "day", "dclose", "depdb",
+        "depdbsl", "depsl", "depsyd",
+        "deptab", "dequote", "dhms", "dif", "digamma",
+        "dim", "dinfo", "dnum", "dopen", "doptname", "doptnum",
+        "dread", "dropnote", "dsname", "erf", "erfc", "exist", "exp",
+        "fappend", "fclose", "fcol", "fdelete", "fetch", "fetchobs",
+        "fexist", "fget", "fileexist", "filename", "fileref",
+        "finfo", "finv", "fipname", "fipnamel", "fipstate", "floor",
+        "fnonct", "fnote", "fopen", "foptname", "foptnum", "fpoint",
+        "fpos", "fput", "fread", "frewind", "frlen", "fsep", "fuzz",
+        "fwrite", "gaminv", "gamma", "getoption", "getvarc", "getvarn",
+        "hbound", "hms", "hosthelp", "hour", "ibessel", "index",
+        "indexc", "indexw", "input", "inputc", "inputn", "int",
+        "intck", "intnx", "intrr", "irr", "jbessel", "juldate",
+        "kurtosis", "lag", "lbound", "left", "length", "lgamma",
+        "libname", "libref", "log", "log10", "log2", "logpdf", "logpmf",
+        "logsdf", "lowcase", "max", "mdy", "mean", "min", "minute",
+        "mod", "month", "mopen", "mort", "n", "netpv", "nmiss",
+        "normal", "note", "npv", "open", "ordinal", "pathname",
+        "pdf", "peek", "peekc", "pmf", "point", "poisson", "poke",
+        "probbeta", "probbnml", "probchi", "probf", "probgam",
+        "probhypr", "probit", "probnegb", "probnorm", "probt",
+        "put", "putc", "putn", "qtr", "quote", "ranbin", "rancau",
+        "ranexp", "rangam", "range", "rank", "rannor", "ranpoi",
+        "rantbl", "rantri", "ranuni", "repeat", "resolve", "reverse",
+        "rewind", "right", "round", "saving", "scan", "sdf", "second",
+        "sign", "sin", "sinh", "skewness", "soundex", "spedis",
+        "sqrt", "std", "stderr", "stfips", "stname", "stnamel",
+        "substr", "sum", "symget", "sysget", "sysmsg", "sysprod",
+        "sysrc", "system", "tan", "tanh", "time", "timepart", "tinv",
+        "tnonct", "today", "translate", "tranwrd", "trigamma",
+        "trim", "trimn", "trunc", "uniform", "upcase", "uss", "var",
+        "varfmt", "varinfmt", "varlabel", "varlen", "varname",
+        "varnum", "varray", "varrayx", "vartype", "verify", "vformat",
+        "vformatd", "vformatdx", "vformatn", "vformatnx", "vformatw",
+        "vformatwx", "vformatx", "vinarray", "vinarrayx", "vinformat",
+        "vinformatd", "vinformatdx", "vinformatn", "vinformatnx",
+        "vinformatw", "vinformatwx", "vinformatx", "vlabel",
+        "vlabelx", "vlength", "vlengthx", "vname", "vnamex", "vtype",
+        "vtypex", "weekday", "year", "yyq", "zipfips", "zipname",
+        "zipnamel", "zipstate"
+    )
+
+    tokens = {
+        'root': [
+            include('comments'),
+            include('proc-data'),
+            include('cards-datalines'),
+            include('logs'),
+            include('general'),
+            (r'.', Text),
+        ],
+        # SAS is multi-line regardless, but * is ended by ;
+        'comments': [
+            (r'^\s*\*.*?;', Comment),
+            (r'/\*.*?\*/', Comment),
+            (r'^\s*\*(.|\n)*?;', Comment.Multiline),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+        ],
+        # Special highlight for proc, data, quit, run
+        'proc-data': [
+            (r'(^|;)\s*(proc \w+|data|run|quit)[\s;]',
+             Keyword.Reserved),
+        ],
+        # Special highlight cards and datalines
+        'cards-datalines': [
+            (r'^\s*(datalines|cards)\s*;\s*$', Keyword, 'data'),
+        ],
+        'data': [
+            (r'(.|\n)*^\s*;\s*$', Other, '#pop'),
+        ],
+        # Special highlight for put NOTE|ERROR|WARNING (order matters)
+        'logs': [
+            (r'\n?^\s*%?put ', Keyword, 'log-messages'),
+        ],
+        'log-messages': [
+            (r'NOTE(:|-).*', Generic, '#pop'),
+            (r'WARNING(:|-).*', Generic.Emph, '#pop'),
+            (r'ERROR(:|-).*', Generic.Error, '#pop'),
+            include('general'),
+        ],
+        'general': [
+            include('keywords'),
+            include('vars-strings'),
+            include('special'),
+            include('numbers'),
+        ],
+        # Keywords, statements, functions, macros
+        'keywords': [
+            (words(builtins_statements,
+                   prefix = r'\b',
+                   suffix = r'\b'),
+             Keyword),
+            (words(builtins_sql,
+                   prefix = r'\b',
+                   suffix = r'\b'),
+             Keyword),
+            (words(builtins_conditionals,
+                   prefix = r'\b',
+                   suffix = r'\b'),
+             Keyword),
+            (words(builtins_macros,
+                   prefix = r'%',
+                   suffix = r'\b'),
+             Name.Builtin),
+            (words(builtins_functions,
+                   prefix = r'\b',
+                   suffix = r'\('),
+             Name.Builtin),
+        ],
+        # Strings and user-defined variables and macros (order matters)
+        'vars-strings': [
+            (r'&[a-z_]\w{0,31}\.?', Name.Variable),
+            (r'%[a-z_]\w{0,31}', Name.Function),
+            (r'\'', String, 'string_squote'),
+            (r'"', String, 'string_dquote'),
+        ],
+        'string_squote': [
+            ('\'', String, '#pop'),
+            (r'\\\\|\\"|\\\n', String.Escape),
+            # AFAIK, macro variables are not evaluated in single quotes
+            # (r'&', Name.Variable, 'validvar'),
+            (r'[^$\'\\]+', String),
+            (r'[$\'\\]', String),
+        ],
+        'string_dquote': [
+            (r'"', String, '#pop'),
+            (r'\\\\|\\"|\\\n', String.Escape),
+            (r'&', Name.Variable, 'validvar'),
+            (r'[^$&"\\]+', String),
+            (r'[$"\\]', String),
+        ],
+        'validvar': [
+            (r'[a-z_]\w{0,31}\.?', Name.Variable, '#pop'),
+        ],
+        # SAS numbers and special variables
+        'numbers': [
+            (r'\b[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)(E[+-]?[0-9]+)?i?\b',
+             Number),
+        ],
+        'special': [
+            (r'(null|missing|_all_|_automatic_|_character_|_n_|'
+             r'_infile_|_name_|_null_|_numeric_|_user_|_webout_)',
+             Keyword.Constant),
+        ],
+        # 'operators': [
+        #     (r'(-|=|<=|>=|<|>|<>|&|!=|'
+        #      r'\||\*|\+|\^|/|!|~|~=)', Operator)
+        # ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/savi.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/savi.py
new file mode 100644
index 0000000000000000000000000000000000000000..548a05665fb166f2e3bbd400a3a09630b709eb4e
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/savi.py
@@ -0,0 +1,171 @@
+"""
+    pygments.lexers.savi
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Savi.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, include
+from pygments.token import Whitespace, Keyword, Name, String, Number, \
+  Operator, Punctuation, Comment, Generic, Error
+
+__all__ = ['SaviLexer']
+
+
+# The canonical version of this file can be found in the following repository,
+# where it is kept in sync with any language changes, as well as the other
+# pygments-like lexers that are maintained for use with other tools:
+# - https://github.com/savi-lang/savi/blob/main/tooling/pygments/lexers/savi.py
+#
+# If you're changing this file in the pygments repository, please ensure that
+# any changes you make are also propagated to the official Savi repository,
+# in order to avoid accidental clobbering of your changes later when an update
+# from the Savi repository flows forward into the pygments repository.
+#
+# If you're changing this file in the Savi repository, please ensure that
+# any changes you make are also reflected in the other pygments-like lexers
+# (rouge, vscode, etc) so that all of the lexers can be kept cleanly in sync.
+
+class SaviLexer(RegexLexer):
+    """
+    For Savi source code.
+
+    .. versionadded: 2.10
+    """
+
+    name = 'Savi'
+    url = 'https://github.com/savi-lang/savi'
+    aliases = ['savi']
+    filenames = ['*.savi']
+    version_added = ''
+
+    tokens = {
+      "root": [
+        # Line Comment
+        (r'//.*?$', Comment.Single),
+
+        # Doc Comment
+        (r'::.*?$', Comment.Single),
+
+        # Capability Operator
+        (r'(\')(\w+)(?=[^\'])', bygroups(Operator, Name)),
+
+        # Double-Quote String
+        (r'\w?"', String.Double, "string.double"),
+
+        # Single-Char String
+        (r"'", String.Char, "string.char"),
+
+        # Type Name
+        (r'(_?[A-Z]\w*)', Name.Class),
+
+        # Nested Type Name
+        (r'(\.)(\s*)(_?[A-Z]\w*)', bygroups(Punctuation, Whitespace, Name.Class)),
+
+        # Declare
+        (r'^([ \t]*)(:\w+)',
+          bygroups(Whitespace, Name.Tag),
+          "decl"),
+
+        # Error-Raising Calls/Names
+        (r'((\w+|\+|\-|\*)\!)', Generic.Deleted),
+
+        # Numeric Values
+        (r'\b\d([\d_]*(\.[\d_]+)?)\b', Number),
+
+        # Hex Numeric Values
+        (r'\b0x([0-9a-fA-F_]+)\b', Number.Hex),
+
+        # Binary Numeric Values
+        (r'\b0b([01_]+)\b', Number.Bin),
+
+        # Function Call (with braces)
+        (r'\w+(?=\()', Name.Function),
+
+        # Function Call (with receiver)
+        (r'(\.)(\s*)(\w+)', bygroups(Punctuation, Whitespace, Name.Function)),
+
+        # Function Call (with self receiver)
+        (r'(@)(\w+)', bygroups(Punctuation, Name.Function)),
+
+        # Parenthesis
+        (r'\(', Punctuation, "root"),
+        (r'\)', Punctuation, "#pop"),
+
+        # Brace
+        (r'\{', Punctuation, "root"),
+        (r'\}', Punctuation, "#pop"),
+
+        # Bracket
+        (r'\[', Punctuation, "root"),
+        (r'(\])(\!)', bygroups(Punctuation, Generic.Deleted), "#pop"),
+        (r'\]', Punctuation, "#pop"),
+
+        # Punctuation
+        (r'[,;:\.@]', Punctuation),
+
+        # Piping Operators
+        (r'(\|\>)', Operator),
+
+        # Branching Operators
+        (r'(\&\&|\|\||\?\?|\&\?|\|\?|\.\?)', Operator),
+
+        # Comparison Operators
+        (r'(\<\=\>|\=\~|\=\=|\<\=|\>\=|\<|\>)', Operator),
+
+        # Arithmetic Operators
+        (r'(\+|\-|\/|\*|\%)', Operator),
+
+        # Assignment Operators
+        (r'(\=)', Operator),
+
+        # Other Operators
+        (r'(\!|\<\<|\<|\&|\|)', Operator),
+
+        # Identifiers
+        (r'\b\w+\b', Name),
+
+        # Whitespace
+        (r'[ \t\r]+\n*|\n+', Whitespace),
+      ],
+
+      # Declare (nested rules)
+      "decl": [
+        (r'\b[a-z_]\w*\b(?!\!)', Keyword.Declaration),
+        (r':', Punctuation, "#pop"),
+        (r'\n', Whitespace, "#pop"),
+        include("root"),
+      ],
+
+      # Double-Quote String (nested rules)
+      "string.double": [
+        (r'\\\(', String.Interpol, "string.interpolation"),
+        (r'\\u[0-9a-fA-F]{4}', String.Escape),
+        (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        (r'\\[bfnrt\\\']', String.Escape),
+        (r'\\"', String.Escape),
+        (r'"', String.Double, "#pop"),
+        (r'[^\\"]+', String.Double),
+        (r'.', Error),
+      ],
+
+      # Single-Char String (nested rules)
+      "string.char": [
+        (r'\\u[0-9a-fA-F]{4}', String.Escape),
+        (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        (r'\\[bfnrt\\\']', String.Escape),
+        (r"\\'", String.Escape),
+        (r"'", String.Char, "#pop"),
+        (r"[^\\']+", String.Char),
+        (r'.', Error),
+      ],
+
+      # Interpolation inside String (nested rules)
+      "string.interpolation": [
+        (r"\)", String.Interpol, "#pop"),
+        include("root"),
+      ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/scdoc.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/scdoc.py
new file mode 100644
index 0000000000000000000000000000000000000000..9b2e01da1f8443a0f7ddf8952531fb94f1e35e2e
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/scdoc.py
@@ -0,0 +1,85 @@
+"""
+    pygments.lexers.scdoc
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for scdoc, a simple man page generator.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, this
+from pygments.token import Text, Comment, Keyword, String, Generic
+
+__all__ = ['ScdocLexer']
+
+
+class ScdocLexer(RegexLexer):
+    """
+    `scdoc` is a simple man page generator for POSIX systems written in C99.
+    """
+    name = 'scdoc'
+    url = 'https://git.sr.ht/~sircmpwn/scdoc'
+    aliases = ['scdoc', 'scd']
+    filenames = ['*.scd', '*.scdoc']
+    version_added = '2.5'
+    flags = re.MULTILINE
+
+    tokens = {
+        'root': [
+            # comment
+            (r'^(;.+\n)', bygroups(Comment)),
+
+            # heading with pound prefix
+            (r'^(#)([^#].+\n)', bygroups(Generic.Heading, Text)),
+            (r'^(#{2})(.+\n)', bygroups(Generic.Subheading, Text)),
+            # bulleted lists
+            (r'^(\s*)([*-])(\s)(.+\n)',
+            bygroups(Text, Keyword, Text, using(this, state='inline'))),
+            # numbered lists
+            (r'^(\s*)(\.+\.)( .+\n)',
+            bygroups(Text, Keyword, using(this, state='inline'))),
+            # quote
+            (r'^(\s*>\s)(.+\n)', bygroups(Keyword, Generic.Emph)),
+            # text block
+            (r'^(```\n)([\w\W]*?)(^```$)', bygroups(String, Text, String)),
+
+            include('inline'),
+        ],
+        'inline': [
+            # escape
+            (r'\\.', Text),
+            # underlines
+            (r'(\s)(_[^_]+_)(\W|\n)', bygroups(Text, Generic.Emph, Text)),
+            # bold
+            (r'(\s)(\*[^*]+\*)(\W|\n)', bygroups(Text, Generic.Strong, Text)),
+            # inline code
+            (r'`[^`]+`', String.Backtick),
+
+            # general text, must come last!
+            (r'[^\\\s]+', Text),
+            (r'.', Text),
+        ],
+    }
+
+    def analyse_text(text):
+        """We checks for bold and underline text with * and _. Also
+        every scdoc file must start with a strictly defined first line."""
+        result = 0
+
+        if '*' in text:
+            result += 0.01
+
+        if '_' in text:
+            result += 0.01
+
+        # name(section) ["left_footer" ["center_header"]]
+        first_line = text.partition('\n')[0]
+        scdoc_preamble_pattern = r'^.*\([1-7]\)( "[^"]+"){0,2}$'
+
+        if re.search(scdoc_preamble_pattern, first_line):
+            result += 0.5
+
+        return result
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/scripting.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/scripting.py
new file mode 100644
index 0000000000000000000000000000000000000000..2adb7bbd0194cd2482a6c2d99d573a8486084bf8
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/scripting.py
@@ -0,0 +1,1638 @@
+"""
+    pygments.lexers.scripting
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for scripting and embedded languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, default, combined, \
+    words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error, Whitespace, Other
+from pygments.util import get_bool_opt, get_list_opt
+
+__all__ = ['LuaLexer', 'LuauLexer', 'MoonScriptLexer', 'ChaiscriptLexer', 'LSLLexer',
+           'AppleScriptLexer', 'RexxLexer', 'MOOCodeLexer', 'HybrisLexer',
+           'EasytrieveLexer', 'JclLexer', 'MiniScriptLexer']
+
+
+def all_lua_builtins():
+    from pygments.lexers._lua_builtins import MODULES
+    return [w for values in MODULES.values() for w in values]
+
+class LuaLexer(RegexLexer):
+    """
+    For Lua source code.
+
+    Additional options accepted:
+
+    `func_name_highlighting`
+        If given and ``True``, highlight builtin function names
+        (default: ``True``).
+    `disabled_modules`
+        If given, must be a list of module names whose function names
+        should not be highlighted. By default all modules are highlighted.
+
+        To get a list of allowed modules have a look into the
+        `_lua_builtins` module:
+
+        .. sourcecode:: pycon
+
+            >>> from pygments.lexers._lua_builtins import MODULES
+            >>> MODULES.keys()
+            ['string', 'coroutine', 'modules', 'io', 'basic', ...]
+    """
+
+    name = 'Lua'
+    url = 'https://www.lua.org/'
+    aliases = ['lua']
+    filenames = ['*.lua', '*.wlua']
+    mimetypes = ['text/x-lua', 'application/x-lua']
+    version_added = ''
+
+    _comment_multiline = r'(?:--\[(?P=*)\[[\w\W]*?\](?P=level)\])'
+    _comment_single = r'(?:--.*$)'
+    _space = r'(?:\s+(?!\s))'
+    _s = rf'(?:{_comment_multiline}|{_comment_single}|{_space})'
+    # A lookahead-safe version of _s that avoids catastrophic backtracking.
+    # The _comment_multiline pattern contains [\w\W]*? which, when used
+    # inside a lookahead with a * quantifier, causes exponential blowup.
+    # This version skips only whitespace; comments between an identifier
+    # and a following [.:] or ( are rare enough to sacrifice.
+    _s_la = r'\s'
+    _name = r'(?:[^\W\d]\w*)'
+
+    tokens = {
+        'root': [
+            # Lua allows a file to start with a shebang.
+            (r'#!.*', Comment.Preproc),
+            default('base'),
+        ],
+        'ws': [
+            (_comment_multiline, Comment.Multiline),
+            (_comment_single, Comment.Single),
+            (_space, Whitespace),
+        ],
+        'base': [
+            include('ws'),
+
+            (r'(?i)0x[\da-f]*(\.[\da-f]*)?(p[+-]?\d+)?', Number.Hex),
+            (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float),
+            (r'(?i)\d+e[+-]?\d+', Number.Float),
+            (r'\d+', Number.Integer),
+
+            # multiline strings
+            (r'(?s)\[(=*)\[.*?\]\1\]', String),
+
+            (r'::', Punctuation, 'label'),
+            (r'\.{3}', Punctuation),
+            (r'[=<>|~&+\-*/%#^]+|\.\.', Operator),
+            (r'[\[\]{}().,:;]+', Punctuation),
+            (r'(and|or|not)\b', Operator.Word),
+
+            (words([
+                'break', 'do', 'else', 'elseif', 'end', 'for', 'if', 'in',
+                'repeat', 'return', 'then', 'until', 'while'
+            ], suffix=r'\b'), Keyword.Reserved),
+            (r'goto\b', Keyword.Reserved, 'goto'),
+            (r'(local)\b', Keyword.Declaration),
+            (r'(true|false|nil)\b', Keyword.Constant),
+
+            (r'(function)\b', Keyword.Reserved, 'funcname'),
+
+            (words(all_lua_builtins(), suffix=r"\b"), Name.Builtin),
+            (fr'[A-Za-z_]\w*(?={_s_la}*[.:])', Name.Variable, 'varname'),
+            (fr'[A-Za-z_]\w*(?={_s_la}*\()', Name.Function),
+            (r'[A-Za-z_]\w*', Name.Variable),
+
+            ("'", String.Single, combined('stringescape', 'sqs')),
+            ('"', String.Double, combined('stringescape', 'dqs'))
+        ],
+
+        'varname': [
+            include('ws'),
+            (r'\.\.', Operator, '#pop'),
+            (r'[.:]', Punctuation),
+            (rf'{_name}(?={_s_la}*[.:])', Name.Property),
+            (rf'{_name}(?={_s_la}*\()', Name.Function, '#pop'),
+            (_name, Name.Property, '#pop'),
+        ],
+
+        'funcname': [
+            include('ws'),
+            (r'[.:]', Punctuation),
+            (rf'{_name}(?={_s_la}*[.:])', Name.Class),
+            (_name, Name.Function, '#pop'),
+            # inline function
+            (r'\(', Punctuation, '#pop'),
+        ],
+
+        'goto': [
+            include('ws'),
+            (_name, Name.Label, '#pop'),
+        ],
+
+        'label': [
+            include('ws'),
+            (r'::', Punctuation, '#pop'),
+            (_name, Name.Label),
+        ],
+
+        'stringescape': [
+            (r'\\([abfnrtv\\"\']|[\r\n]{1,2}|z\s*|x[0-9a-fA-F]{2}|\d{1,3}|'
+             r'u\{[0-9a-fA-F]+\})', String.Escape),
+        ],
+
+        'sqs': [
+            (r"'", String.Single, '#pop'),
+            (r"[^\\']+", String.Single),
+        ],
+
+        'dqs': [
+            (r'"', String.Double, '#pop'),
+            (r'[^\\"]+', String.Double),
+        ]
+    }
+
+    def __init__(self, **options):
+        self.func_name_highlighting = get_bool_opt(
+            options, 'func_name_highlighting', True)
+        self.disabled_modules = get_list_opt(options, 'disabled_modules', [])
+
+        self._functions = set()
+        if self.func_name_highlighting:
+            from pygments.lexers._lua_builtins import MODULES
+            for mod, func in MODULES.items():
+                if mod not in self.disabled_modules:
+                    self._functions.update(func)
+        RegexLexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        for index, token, value in \
+                RegexLexer.get_tokens_unprocessed(self, text):
+            if token is Name.Builtin and value not in self._functions:
+                if '.' in value:
+                    a, b = value.split('.')
+                    yield index, Name, a
+                    yield index + len(a), Punctuation, '.'
+                    yield index + len(a) + 1, Name, b
+                else:
+                    yield index, Name, value
+                continue
+            yield index, token, value
+
+def _luau_make_expression(should_pop, _s, _s_la):
+    temp_list = [
+        (r'0[xX][\da-fA-F_]*', Number.Hex, '#pop'),
+        (r'0[bB][\d_]*', Number.Bin, '#pop'),
+        (r'\.?\d[\d_]*(?:\.[\d_]*)?(?:[eE][+-]?[\d_]+)?', Number.Float, '#pop'),
+
+        (words((
+            'true', 'false', 'nil'
+        ), suffix=r'\b'), Keyword.Constant, '#pop'),
+
+        (r'\[(=*)\[[.\n]*?\]\1\]', String, '#pop'),
+
+        (r'(\.)([a-zA-Z_]\w*)(?=%s*[({"\'])', bygroups(Punctuation, Name.Function), '#pop'),
+        (r'(\.)([a-zA-Z_]\w*)', bygroups(Punctuation, Name.Variable), '#pop'),
+
+        (rf'[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*(?={_s_la}*[({{"\'])', Name.Other, '#pop'),
+        (r'[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*', Name, '#pop'),
+    ]
+    if should_pop:
+        return temp_list
+    return [entry[:2] for entry in temp_list]
+
+def _luau_make_expression_special(should_pop):
+    temp_list = [
+        (r'\{', Punctuation, ('#pop', 'closing_brace_base', 'expression')),
+        (r'\(', Punctuation, ('#pop', 'closing_parenthesis_base', 'expression')),
+
+        (r'::?', Punctuation, ('#pop', 'type_end', 'type_start')),
+
+        (r"'", String.Single, ('#pop', 'string_single')),
+        (r'"', String.Double, ('#pop', 'string_double')),
+        (r'`', String.Backtick, ('#pop', 'string_interpolated')),
+    ]
+    if should_pop:
+        return temp_list
+    return [(entry[0], entry[1], entry[2][1:]) for entry in temp_list]
+
+class LuauLexer(RegexLexer):
+    """
+    For Luau source code.
+
+    Additional options accepted:
+
+    `include_luau_builtins`
+        If given and ``True``, automatically highlight Luau builtins
+        (default: ``True``).
+    `include_roblox_builtins`
+        If given and ``True``, automatically highlight Roblox-specific builtins
+        (default: ``False``).
+    `additional_builtins`
+        If given, must be a list of additional builtins to highlight.
+    `disabled_builtins`
+        If given, must be a list of builtins that will not be highlighted.
+    """
+
+    name = 'Luau'
+    url = 'https://luau-lang.org/'
+    aliases = ['luau']
+    filenames = ['*.luau']
+    version_added = '2.18'
+
+    _comment_multiline = r'(?:--\[(?P=*)\[[\w\W]*?\](?P=level)\])'
+    _comment_single = r'(?:--.*$)'
+    _s = r'(?:{}|{}|{})'.format(_comment_multiline, _comment_single, r'\s+')
+    # Lookahead-safe version — avoids catastrophic backtracking from
+    # [\w\W]*? inside _comment_multiline when combined with * quantifier.
+    _s_la = r'\s'
+
+    tokens = {
+        'root': [
+            (r'#!.*', Comment.Hashbang, 'base'),
+            default('base'),
+        ],
+
+        'ws': [
+            (_comment_multiline, Comment.Multiline),
+            (_comment_single, Comment.Single),
+            (r'\s+', Whitespace),
+        ],
+
+        'base': [
+            include('ws'),
+
+            *_luau_make_expression_special(False),
+            (r'\.\.\.', Punctuation),
+
+            (rf'type\b(?={_s}+[a-zA-Z_])', Keyword.Reserved, 'type_declaration'),
+            (rf'export\b(?={_s}+[a-zA-Z_])', Keyword.Reserved),
+
+            (r'(?:\.\.|//|[+\-*\/%^<>=])=?', Operator, 'expression'),
+            (r'~=', Operator, 'expression'),
+
+            (words((
+                'and', 'or', 'not'
+            ), suffix=r'\b'), Operator.Word, 'expression'),
+
+            (words((
+                'elseif', 'for', 'if', 'in', 'repeat', 'return', 'until',
+                'while'), suffix=r'\b'), Keyword.Reserved, 'expression'),
+            (r'local\b', Keyword.Declaration, 'expression'),
+
+            (r'function\b', Keyword.Reserved, ('expression', 'func_name')),
+
+            (r'[\])};]+', Punctuation),
+
+            include('expression_static'),
+            *_luau_make_expression(False, _s, _s_la),
+
+            (r'[\[.,]', Punctuation, 'expression'),
+        ],
+        'expression_static': [
+            (words((
+                'break', 'continue', 'do', 'else', 'elseif', 'end', 'for',
+                'if', 'in', 'repeat', 'return', 'then', 'until', 'while'),
+                suffix=r'\b'), Keyword.Reserved),
+        ],
+        'expression': [
+            include('ws'),
+
+            (r'if\b', Keyword.Reserved, ('ternary', 'expression')),
+
+            (r'local\b', Keyword.Declaration),
+            *_luau_make_expression_special(True),
+            (r'\.\.\.', Punctuation, '#pop'),
+
+            (r'function\b', Keyword.Reserved, 'func_name'),
+
+            include('expression_static'),
+            *_luau_make_expression(True, _s, _s_la),
+
+            default('#pop'),
+        ],
+        'ternary': [
+            include('ws'),
+
+            (r'else\b', Keyword.Reserved, '#pop'),
+            (words((
+                'then', 'elseif',
+            ), suffix=r'\b'), Operator.Reserved, 'expression'),
+
+            default('#pop'),
+        ],
+
+        'closing_brace_pop': [
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'closing_parenthesis_pop': [
+            (r'\)', Punctuation, '#pop'),
+        ],
+        'closing_gt_pop': [
+            (r'>', Punctuation, '#pop'),
+        ],
+
+        'closing_parenthesis_base': [
+            include('closing_parenthesis_pop'),
+            include('base'),
+        ],
+        'closing_parenthesis_type': [
+            include('closing_parenthesis_pop'),
+            include('type'),
+        ],
+        'closing_brace_base': [
+            include('closing_brace_pop'),
+            include('base'),
+        ],
+        'closing_brace_type': [
+            include('closing_brace_pop'),
+            include('type'),
+        ],
+        'closing_gt_type': [
+            include('closing_gt_pop'),
+            include('type'),
+        ],
+
+        'string_escape': [
+            (r'\\z\s*', String.Escape),
+            (r'\\(?:[abfnrtvz\\"\'`\{\n])|[\r\n]{1,2}|x[\da-fA-F]{2}|\d{1,3}|'
+             r'u\{\}[\da-fA-F]*\}', String.Escape),
+        ],
+        'string_single': [
+            include('string_escape'),
+
+            (r"'", String.Single, "#pop"),
+            (r"[^\\']+", String.Single),
+        ],
+        'string_double': [
+            include('string_escape'),
+
+            (r'"', String.Double, "#pop"),
+            (r'[^\\"]+', String.Double),
+        ],
+        'string_interpolated': [
+            include('string_escape'),
+
+            (r'\{', Punctuation, ('closing_brace_base', 'expression')),
+
+            (r'`', String.Backtick, "#pop"),
+            (r'[^\\`\{]+', String.Backtick),
+        ],
+
+        'func_name': [
+            include('ws'),
+
+            (r'[.:]', Punctuation),
+            (rf'[a-zA-Z_]\w*(?={_s_la}*[.:])', Name.Class),
+            (r'[a-zA-Z_]\w*', Name.Function),
+
+            (r'<', Punctuation, 'closing_gt_type'),
+
+            (r'\(', Punctuation, '#pop'),
+        ],
+
+        'type': [
+            include('ws'),
+
+            (r'\(', Punctuation, 'closing_parenthesis_type'),
+            (r'\{', Punctuation, 'closing_brace_type'),
+            (r'<', Punctuation, 'closing_gt_type'),
+
+            (r"'", String.Single, 'string_single'),
+            (r'"', String.Double, 'string_double'),
+
+            (r'[|&\.,\[\]:=]+', Punctuation),
+            (r'->', Punctuation),
+
+            (r'typeof\(', Name.Builtin, ('closing_parenthesis_base',
+                                         'expression')),
+            (r'[a-zA-Z_]\w*', Name.Class),
+        ],
+        'type_start': [
+            include('ws'),
+
+            (r'\(', Punctuation, ('#pop', 'closing_parenthesis_type')),
+            (r'\{', Punctuation, ('#pop', 'closing_brace_type')),
+            (r'<', Punctuation, ('#pop', 'closing_gt_type')),
+
+            (r"'", String.Single, ('#pop', 'string_single')),
+            (r'"', String.Double, ('#pop', 'string_double')),
+
+            (r'typeof\(', Name.Builtin, ('#pop', 'closing_parenthesis_base',
+                                         'expression')),
+            (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+        ],
+        'type_end': [
+            include('ws'),
+
+            (r'[|&\.]', Punctuation, 'type_start'),
+            (r'->', Punctuation, 'type_start'),
+
+            (r'<', Punctuation, 'closing_gt_type'),
+
+            default('#pop'),
+        ],
+        'type_declaration': [
+            include('ws'),
+
+            (r'[a-zA-Z_]\w*', Name.Class),
+            (r'<', Punctuation, 'closing_gt_type'),
+
+            (r'=', Punctuation, ('#pop', 'type_end', 'type_start')),
+        ],
+    }
+
+    def __init__(self, **options):
+        self.include_luau_builtins = get_bool_opt(
+            options, 'include_luau_builtins', True)
+        self.include_roblox_builtins = get_bool_opt(
+            options, 'include_roblox_builtins', False)
+        self.additional_builtins = get_list_opt(options, 'additional_builtins', [])
+        self.disabled_builtins = get_list_opt(options, 'disabled_builtins', [])
+
+        self._builtins = set(self.additional_builtins)
+        if self.include_luau_builtins:
+            from pygments.lexers._luau_builtins import LUAU_BUILTINS
+            self._builtins.update(LUAU_BUILTINS)
+        if self.include_roblox_builtins:
+            from pygments.lexers._luau_builtins import ROBLOX_BUILTINS
+            self._builtins.update(ROBLOX_BUILTINS)
+        if self.additional_builtins:
+            self._builtins.update(self.additional_builtins)
+        self._builtins.difference_update(self.disabled_builtins)
+
+        RegexLexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        for index, token, value in \
+                RegexLexer.get_tokens_unprocessed(self, text):
+            if token is Name or token is Name.Other:
+                split_value = value.split('.')
+                complete_value = []
+                new_index = index
+                for position in range(len(split_value), 0, -1):
+                    potential_string = '.'.join(split_value[:position])
+                    if potential_string in self._builtins:
+                        yield index, Name.Builtin, potential_string
+                        new_index += len(potential_string)
+
+                        if complete_value:
+                            yield new_index, Punctuation, '.'
+                            new_index += 1
+                        break
+                    complete_value.insert(0, split_value[position - 1])
+
+                for position, substring in enumerate(complete_value):
+                    if position + 1 == len(complete_value):
+                        if token is Name:
+                            yield new_index, Name.Variable, substring
+                            continue
+                        yield new_index, Name.Function, substring
+                        continue
+                    yield new_index, Name.Variable, substring
+                    new_index += len(substring)
+                    yield new_index, Punctuation, '.'
+                    new_index += 1
+
+                continue
+            yield index, token, value
+
+class MoonScriptLexer(LuaLexer):
+    """
+    For MoonScript source code.
+    """
+
+    name = 'MoonScript'
+    url = 'http://moonscript.org'
+    aliases = ['moonscript', 'moon']
+    filenames = ['*.moon']
+    mimetypes = ['text/x-moonscript', 'application/x-moonscript']
+    version_added = '1.5'
+
+    tokens = {
+        'root': [
+            (r'#!(.*?)$', Comment.Preproc),
+            default('base'),
+        ],
+        'base': [
+            ('--.*$', Comment.Single),
+            (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float),
+            (r'(?i)\d+e[+-]?\d+', Number.Float),
+            (r'(?i)0x[0-9a-f]*', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'\n', Whitespace),
+            (r'[^\S\n]+', Text),
+            (r'(?s)\[(=*)\[.*?\]\1\]', String),
+            (r'(->|=>)', Name.Function),
+            (r':[a-zA-Z_]\w*', Name.Variable),
+            (r'(==|!=|~=|<=|>=|\.\.\.|\.\.|[=+\-*/%^<>#!.\\:])', Operator),
+            (r'[;,]', Punctuation),
+            (r'[\[\]{}()]', Keyword.Type),
+            (r'[a-zA-Z_]\w*:', Name.Variable),
+            (words((
+                'class', 'extends', 'if', 'then', 'super', 'do', 'with',
+                'import', 'export', 'while', 'elseif', 'return', 'for', 'in',
+                'from', 'when', 'using', 'else', 'and', 'or', 'not', 'switch',
+                'break'), suffix=r'\b'),
+             Keyword),
+            (r'(true|false|nil)\b', Keyword.Constant),
+            (r'(and|or|not)\b', Operator.Word),
+            (r'(self)\b', Name.Builtin.Pseudo),
+            (r'@@?([a-zA-Z_]\w*)?', Name.Variable.Class),
+            (r'[A-Z]\w*', Name.Class),  # proper name
+            (words(all_lua_builtins(), suffix=r"\b"), Name.Builtin),
+            (r'[A-Za-z_]\w*', Name),
+            ("'", String.Single, combined('stringescape', 'sqs')),
+            ('"', String.Double, combined('stringescape', 'dqs'))
+        ],
+        'stringescape': [
+            (r'''\\([abfnrtv\\"']|\d{1,3})''', String.Escape)
+        ],
+        'strings': [
+            (r'[^#\\\'"]+', String),
+            # note that strings are multi-line.
+            # hashmarks, quotes and backslashes must be parsed one at a time
+        ],
+        'interpoling_string': [
+            (r'\}', String.Interpol, "#pop"),
+            include('base')
+        ],
+        'dqs': [
+            (r'"', String.Double, '#pop'),
+            (r'\\.|\'', String),  # double-quoted string don't need ' escapes
+            (r'#\{', String.Interpol, "interpoling_string"),
+            (r'#', String),
+            include('strings')
+        ],
+        'sqs': [
+            (r"'", String.Single, '#pop'),
+            (r'#|\\.|"', String),  # single quoted strings don't need " escapses
+            include('strings')
+        ]
+    }
+
+    def get_tokens_unprocessed(self, text):
+        # set . as Operator instead of Punctuation
+        for index, token, value in LuaLexer.get_tokens_unprocessed(self, text):
+            if token == Punctuation and value == ".":
+                token = Operator
+            yield index, token, value
+
+
+class ChaiscriptLexer(RegexLexer):
+    """
+    For ChaiScript source code.
+    """
+
+    name = 'ChaiScript'
+    url = 'http://chaiscript.com/'
+    aliases = ['chaiscript', 'chai']
+    filenames = ['*.chai']
+    mimetypes = ['text/x-chaiscript', 'application/x-chaiscript']
+    version_added = '2.0'
+
+    flags = re.DOTALL | re.MULTILINE
+
+    tokens = {
+        'commentsandwhitespace': [
+            (r'\s+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'^\#.*?\n', Comment.Single)
+        ],
+        'slashstartsregex': [
+            include('commentsandwhitespace'),
+            (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
+             r'([gim]+\b|\B)', String.Regex, '#pop'),
+            (r'(?=/)', Text, ('#pop', 'badregex')),
+            default('#pop')
+        ],
+        'badregex': [
+            (r'\n', Text, '#pop')
+        ],
+        'root': [
+            include('commentsandwhitespace'),
+            (r'\n', Text),
+            (r'[^\S\n]+', Text),
+            (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|\.\.'
+             r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
+            (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
+            (r'[})\].]', Punctuation),
+            (r'[=+\-*/]', Operator),
+            (r'(for|in|while|do|break|return|continue|if|else|'
+             r'throw|try|catch'
+             r')\b', Keyword, 'slashstartsregex'),
+            (r'(var)\b', Keyword.Declaration, 'slashstartsregex'),
+            (r'(attr|def|fun)\b', Keyword.Reserved),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'(eval|throw)\b', Name.Builtin),
+            (r'`\S+`', Name.Builtin),
+            (r'[$a-zA-Z_]\w*', Name.Other),
+            (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'[0-9]+', Number.Integer),
+            (r'"', String.Double, 'dqstring'),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
+        ],
+        'dqstring': [
+            (r'\$\{[^"}]+?\}', String.Interpol),
+            (r'\$', String.Double),
+            (r'\\\\', String.Double),
+            (r'\\"', String.Double),
+            (r'[^\\"$]+', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+    }
+
+
+class LSLLexer(RegexLexer):
+    """
+    For Second Life's Linden Scripting Language source code.
+    """
+
+    name = 'LSL'
+    aliases = ['lsl']
+    filenames = ['*.lsl']
+    mimetypes = ['text/x-lsl']
+    url = 'https://wiki.secondlife.com/wiki/Linden_Scripting_Language'
+    version_added = '2.0'
+
+    flags = re.MULTILINE
+
+    lsl_keywords = r'\b(?:do|else|for|if|jump|return|while)\b'
+    lsl_types = r'\b(?:float|integer|key|list|quaternion|rotation|string|vector)\b'
+    lsl_states = r'\b(?:(?:state)\s+\w+|default)\b'
+    lsl_events = r'\b(?:state_(?:entry|exit)|touch(?:_(?:start|end))?|(?:land_)?collision(?:_(?:start|end))?|timer|listen|(?:no_)?sensor|control|(?:not_)?at_(?:rot_)?target|money|email|run_time_permissions|changed|attach|dataserver|moving_(?:start|end)|link_message|(?:on|object)_rez|remote_data|http_re(?:sponse|quest)|path_update|transaction_result)\b'
+    lsl_functions_builtin = r'\b(?:ll(?:ReturnObjectsBy(?:ID|Owner)|Json(?:2List|[GS]etValue|ValueType)|Sin|Cos|Tan|Atan2|Sqrt|Pow|Abs|Fabs|Frand|Floor|Ceil|Round|Vec(?:Mag|Norm|Dist)|Rot(?:Between|2(?:Euler|Fwd|Left|Up))|(?:Euler|Axes)2Rot|Whisper|(?:Region|Owner)?Say|Shout|Listen(?:Control|Remove)?|Sensor(?:Repeat|Remove)?|Detected(?:Name|Key|Owner|Type|Pos|Vel|Grab|Rot|Group|LinkNumber)|Die|Ground|Wind|(?:[GS]et)(?:AnimationOverride|MemoryLimit|PrimMediaParams|ParcelMusicURL|Object(?:Desc|Name)|PhysicsMaterial|Status|Scale|Color|Alpha|Texture|Pos|Rot|Force|Torque)|ResetAnimationOverride|(?:Scale|Offset|Rotate)Texture|(?:Rot)?Target(?:Remove)?|(?:Stop)?MoveToTarget|Apply(?:Rotational)?Impulse|Set(?:KeyframedMotion|ContentType|RegionPos|(?:Angular)?Velocity|Buoyancy|HoverHeight|ForceAndTorque|TimerEvent|ScriptState|Damage|TextureAnim|Sound(?:Queueing|Radius)|Vehicle(?:Type|(?:Float|Vector|Rotation)Param)|(?:Touch|Sit)?Text|Camera(?:Eye|At)Offset|PrimitiveParams|ClickAction|Link(?:Alpha|Color|PrimitiveParams(?:Fast)?|Texture(?:Anim)?|Camera|Media)|RemoteScriptAccessPin|PayPrice|LocalRot)|ScaleByFactor|Get(?:(?:Max|Min)ScaleFactor|ClosestNavPoint|StaticPath|SimStats|Env|PrimitiveParams|Link(?:PrimitiveParams|Number(?:OfSides)?|Key|Name|Media)|HTTPHeader|FreeURLs|Object(?:Details|PermMask|PrimCount)|Parcel(?:MaxPrims|Details|Prim(?:Count|Owners))|Attached|(?:SPMax|Free|Used)Memory|Region(?:Name|TimeDilation|FPS|Corner|AgentCount)|Root(?:Position|Rotation)|UnixTime|(?:Parcel|Region)Flags|(?:Wall|GMT)clock|SimulatorHostname|BoundingBox|GeometricCenter|Creator|NumberOf(?:Prims|NotecardLines|Sides)|Animation(?:List)?|(?:Camera|Local)(?:Pos|Rot)|Vel|Accel|Omega|Time(?:stamp|OfDay)|(?:Object|CenterOf)?Mass|MassMKS|Energy|Owner|(?:Owner)?Key|SunDirection|Texture(?:Offset|Scale|Rot)|Inventory(?:Number|Name|Key|Type|Creator|PermMask)|Permissions(?:Key)?|StartParameter|List(?:Length|EntryType)|Date|Agent(?:Size|Info|Language|List)|LandOwnerAt|NotecardLine|Script(?:Name|State))|(?:Get|Reset|GetAndReset)Time|PlaySound(?:Slave)?|LoopSound(?:Master|Slave)?|(?:Trigger|Stop|Preload)Sound|(?:(?:Get|Delete)Sub|Insert)String|To(?:Upper|Lower)|Give(?:InventoryList|Money)|RezObject|(?:Stop)?LookAt|Sleep|CollisionFilter|(?:Take|Release)Controls|DetachFromAvatar|AttachToAvatar(?:Temp)?|InstantMessage|(?:GetNext)?Email|StopHover|MinEventDelay|RotLookAt|String(?:Length|Trim)|(?:Start|Stop)Animation|TargetOmega|RequestPermissions|(?:Create|Break)Link|BreakAllLinks|(?:Give|Remove)Inventory|Water|PassTouches|Request(?:Agent|Inventory)Data|TeleportAgent(?:Home|GlobalCoords)?|ModifyLand|CollisionSound|ResetScript|MessageLinked|PushObject|PassCollisions|AxisAngle2Rot|Rot2(?:Axis|Angle)|A(?:cos|sin)|AngleBetween|AllowInventoryDrop|SubStringIndex|List2(?:CSV|Integer|Json|Float|String|Key|Vector|Rot|List(?:Strided)?)|DeleteSubList|List(?:Statistics|Sort|Randomize|(?:Insert|Find|Replace)List)|EdgeOfWorld|AdjustSoundVolume|Key2Name|TriggerSoundLimited|EjectFromLand|(?:CSV|ParseString)2List|OverMyLand|SameGroup|UnSit|Ground(?:Slope|Normal|Contour)|GroundRepel|(?:Set|Remove)VehicleFlags|(?:AvatarOn)?(?:Link)?SitTarget|Script(?:Danger|Profiler)|Dialog|VolumeDetect|ResetOtherScript|RemoteLoadScriptPin|(?:Open|Close)RemoteDataChannel|SendRemoteData|RemoteDataReply|(?:Integer|String)ToBase64|XorBase64|Log(?:10)?|Base64To(?:String|Integer)|ParseStringKeepNulls|RezAtRoot|RequestSimulatorData|ForceMouselook|(?:Load|Release|(?:E|Une)scape)URL|ParcelMedia(?:CommandList|Query)|ModPow|MapDestination|(?:RemoveFrom|AddTo|Reset)Land(?:Pass|Ban)List|(?:Set|Clear)CameraParams|HTTP(?:Request|Response)|TextBox|DetectedTouch(?:UV|Face|Pos|(?:N|Bin)ormal|ST)|(?:MD5|SHA1|DumpList2)String|Request(?:Secure)?URL|Clear(?:Prim|Link)Media|(?:Link)?ParticleSystem|(?:Get|Request)(?:Username|DisplayName)|RegionSayTo|CastRay|GenerateKey|TransferLindenDollars|ManageEstateAccess|(?:Create|Delete)Character|ExecCharacterCmd|Evade|FleeFrom|NavigateTo|PatrolPoints|Pursue|UpdateCharacter|WanderWithin))\b'
+    lsl_constants_float = r'\b(?:DEG_TO_RAD|PI(?:_BY_TWO)?|RAD_TO_DEG|SQRT2|TWO_PI)\b'
+    lsl_constants_integer = r'\b(?:JSON_APPEND|STATUS_(?:PHYSICS|ROTATE_[XYZ]|PHANTOM|SANDBOX|BLOCK_GRAB(?:_OBJECT)?|(?:DIE|RETURN)_AT_EDGE|CAST_SHADOWS|OK|MALFORMED_PARAMS|TYPE_MISMATCH|BOUNDS_ERROR|NOT_(?:FOUND|SUPPORTED)|INTERNAL_ERROR|WHITELIST_FAILED)|AGENT(?:_(?:BY_(?:LEGACY_|USER)NAME|FLYING|ATTACHMENTS|SCRIPTED|MOUSELOOK|SITTING|ON_OBJECT|AWAY|WALKING|IN_AIR|TYPING|CROUCHING|BUSY|ALWAYS_RUN|AUTOPILOT|LIST_(?:PARCEL(?:_OWNER)?|REGION)))?|CAMERA_(?:PITCH|DISTANCE|BEHINDNESS_(?:ANGLE|LAG)|(?:FOCUS|POSITION)(?:_(?:THRESHOLD|LOCKED|LAG))?|FOCUS_OFFSET|ACTIVE)|ANIM_ON|LOOP|REVERSE|PING_PONG|SMOOTH|ROTATE|SCALE|ALL_SIDES|LINK_(?:ROOT|SET|ALL_(?:OTHERS|CHILDREN)|THIS)|ACTIVE|PASSIVE|SCRIPTED|CONTROL_(?:FWD|BACK|(?:ROT_)?(?:LEFT|RIGHT)|UP|DOWN|(?:ML_)?LBUTTON)|PERMISSION_(?:RETURN_OBJECTS|DEBIT|OVERRIDE_ANIMATIONS|SILENT_ESTATE_MANAGEMENT|TAKE_CONTROLS|TRIGGER_ANIMATION|ATTACH|CHANGE_LINKS|(?:CONTROL|TRACK)_CAMERA|TELEPORT)|INVENTORY_(?:TEXTURE|SOUND|OBJECT|SCRIPT|LANDMARK|CLOTHING|NOTECARD|BODYPART|ANIMATION|GESTURE|ALL|NONE)|CHANGED_(?:INVENTORY|COLOR|SHAPE|SCALE|TEXTURE|LINK|ALLOWED_DROP|OWNER|REGION(?:_START)?|TELEPORT|MEDIA)|OBJECT_(?:(?:PHYSICS|SERVER|STREAMING)_COST|UNKNOWN_DETAIL|CHARACTER_TIME|PHANTOM|PHYSICS|TEMP_ON_REZ|NAME|DESC|POS|PRIM_EQUIVALENCE|RETURN_(?:PARCEL(?:_OWNER)?|REGION)|ROO?T|VELOCITY|OWNER|GROUP|CREATOR|ATTACHED_POINT|RENDER_WEIGHT|PATHFINDING_TYPE|(?:RUNNING|TOTAL)_SCRIPT_COUNT|SCRIPT_(?:MEMORY|TIME))|TYPE_(?:INTEGER|FLOAT|STRING|KEY|VECTOR|ROTATION|INVALID)|(?:DEBUG|PUBLIC)_CHANNEL|ATTACH_(?:AVATAR_CENTER|CHEST|HEAD|BACK|PELVIS|MOUTH|CHIN|NECK|NOSE|BELLY|[LR](?:SHOULDER|HAND|FOOT|EAR|EYE|[UL](?:ARM|LEG)|HIP)|(?:LEFT|RIGHT)_PEC|HUD_(?:CENTER_[12]|TOP_(?:RIGHT|CENTER|LEFT)|BOTTOM(?:_(?:RIGHT|LEFT))?))|LAND_(?:LEVEL|RAISE|LOWER|SMOOTH|NOISE|REVERT)|DATA_(?:ONLINE|NAME|BORN|SIM_(?:POS|STATUS|RATING)|PAYINFO)|PAYMENT_INFO_(?:ON_FILE|USED)|REMOTE_DATA_(?:CHANNEL|REQUEST|REPLY)|PSYS_(?:PART_(?:BF_(?:ZERO|ONE(?:_MINUS_(?:DEST_COLOR|SOURCE_(ALPHA|COLOR)))?|DEST_COLOR|SOURCE_(ALPHA|COLOR))|BLEND_FUNC_(DEST|SOURCE)|FLAGS|(?:START|END)_(?:COLOR|ALPHA|SCALE|GLOW)|MAX_AGE|(?:RIBBON|WIND|INTERP_(?:COLOR|SCALE)|BOUNCE|FOLLOW_(?:SRC|VELOCITY)|TARGET_(?:POS|LINEAR)|EMISSIVE)_MASK)|SRC_(?:MAX_AGE|PATTERN|ANGLE_(?:BEGIN|END)|BURST_(?:RATE|PART_COUNT|RADIUS|SPEED_(?:MIN|MAX))|ACCEL|TEXTURE|TARGET_KEY|OMEGA|PATTERN_(?:DROP|EXPLODE|ANGLE(?:_CONE(?:_EMPTY)?)?)))|VEHICLE_(?:REFERENCE_FRAME|TYPE_(?:NONE|SLED|CAR|BOAT|AIRPLANE|BALLOON)|(?:LINEAR|ANGULAR)_(?:FRICTION_TIMESCALE|MOTOR_DIRECTION)|LINEAR_MOTOR_OFFSET|HOVER_(?:HEIGHT|EFFICIENCY|TIMESCALE)|BUOYANCY|(?:LINEAR|ANGULAR)_(?:DEFLECTION_(?:EFFICIENCY|TIMESCALE)|MOTOR_(?:DECAY_)?TIMESCALE)|VERTICAL_ATTRACTION_(?:EFFICIENCY|TIMESCALE)|BANKING_(?:EFFICIENCY|MIX|TIMESCALE)|FLAG_(?:NO_DEFLECTION_UP|LIMIT_(?:ROLL_ONLY|MOTOR_UP)|HOVER_(?:(?:WATER|TERRAIN|UP)_ONLY|GLOBAL_HEIGHT)|MOUSELOOK_(?:STEER|BANK)|CAMERA_DECOUPLED))|PRIM_(?:TYPE(?:_(?:BOX|CYLINDER|PRISM|SPHERE|TORUS|TUBE|RING|SCULPT))?|HOLE_(?:DEFAULT|CIRCLE|SQUARE|TRIANGLE)|MATERIAL(?:_(?:STONE|METAL|GLASS|WOOD|FLESH|PLASTIC|RUBBER))?|SHINY_(?:NONE|LOW|MEDIUM|HIGH)|BUMP_(?:NONE|BRIGHT|DARK|WOOD|BARK|BRICKS|CHECKER|CONCRETE|TILE|STONE|DISKS|GRAVEL|BLOBS|SIDING|LARGETILE|STUCCO|SUCTION|WEAVE)|TEXGEN_(?:DEFAULT|PLANAR)|SCULPT_(?:TYPE_(?:SPHERE|TORUS|PLANE|CYLINDER|MASK)|FLAG_(?:MIRROR|INVERT))|PHYSICS(?:_(?:SHAPE_(?:CONVEX|NONE|PRIM|TYPE)))?|(?:POS|ROT)_LOCAL|SLICE|TEXT|FLEXIBLE|POINT_LIGHT|TEMP_ON_REZ|PHANTOM|POSITION|SIZE|ROTATION|TEXTURE|NAME|OMEGA|DESC|LINK_TARGET|COLOR|BUMP_SHINY|FULLBRIGHT|TEXGEN|GLOW|MEDIA_(?:ALT_IMAGE_ENABLE|CONTROLS|(?:CURRENT|HOME)_URL|AUTO_(?:LOOP|PLAY|SCALE|ZOOM)|FIRST_CLICK_INTERACT|(?:WIDTH|HEIGHT)_PIXELS|WHITELIST(?:_ENABLE)?|PERMS_(?:INTERACT|CONTROL)|PARAM_MAX|CONTROLS_(?:STANDARD|MINI)|PERM_(?:NONE|OWNER|GROUP|ANYONE)|MAX_(?:URL_LENGTH|WHITELIST_(?:SIZE|COUNT)|(?:WIDTH|HEIGHT)_PIXELS)))|MASK_(?:BASE|OWNER|GROUP|EVERYONE|NEXT)|PERM_(?:TRANSFER|MODIFY|COPY|MOVE|ALL)|PARCEL_(?:MEDIA_COMMAND_(?:STOP|PAUSE|PLAY|LOOP|TEXTURE|URL|TIME|AGENT|UNLOAD|AUTO_ALIGN|TYPE|SIZE|DESC|LOOP_SET)|FLAG_(?:ALLOW_(?:FLY|(?:GROUP_)?SCRIPTS|LANDMARK|TERRAFORM|DAMAGE|CREATE_(?:GROUP_)?OBJECTS)|USE_(?:ACCESS_(?:GROUP|LIST)|BAN_LIST|LAND_PASS_LIST)|LOCAL_SOUND_ONLY|RESTRICT_PUSHOBJECT|ALLOW_(?:GROUP|ALL)_OBJECT_ENTRY)|COUNT_(?:TOTAL|OWNER|GROUP|OTHER|SELECTED|TEMP)|DETAILS_(?:NAME|DESC|OWNER|GROUP|AREA|ID|SEE_AVATARS))|LIST_STAT_(?:MAX|MIN|MEAN|MEDIAN|STD_DEV|SUM(?:_SQUARES)?|NUM_COUNT|GEOMETRIC_MEAN|RANGE)|PAY_(?:HIDE|DEFAULT)|REGION_FLAG_(?:ALLOW_DAMAGE|FIXED_SUN|BLOCK_TERRAFORM|SANDBOX|DISABLE_(?:COLLISIONS|PHYSICS)|BLOCK_FLY|ALLOW_DIRECT_TELEPORT|RESTRICT_PUSHOBJECT)|HTTP_(?:METHOD|MIMETYPE|BODY_(?:MAXLENGTH|TRUNCATED)|CUSTOM_HEADER|PRAGMA_NO_CACHE|VERBOSE_THROTTLE|VERIFY_CERT)|STRING_(?:TRIM(?:_(?:HEAD|TAIL))?)|CLICK_ACTION_(?:NONE|TOUCH|SIT|BUY|PAY|OPEN(?:_MEDIA)?|PLAY|ZOOM)|TOUCH_INVALID_FACE|PROFILE_(?:NONE|SCRIPT_MEMORY)|RC_(?:DATA_FLAGS|DETECT_PHANTOM|GET_(?:LINK_NUM|NORMAL|ROOT_KEY)|MAX_HITS|REJECT_(?:TYPES|AGENTS|(?:NON)?PHYSICAL|LAND))|RCERR_(?:CAST_TIME_EXCEEDED|SIM_PERF_LOW|UNKNOWN)|ESTATE_ACCESS_(?:ALLOWED_(?:AGENT|GROUP)_(?:ADD|REMOVE)|BANNED_AGENT_(?:ADD|REMOVE))|DENSITY|FRICTION|RESTITUTION|GRAVITY_MULTIPLIER|KFM_(?:COMMAND|CMD_(?:PLAY|STOP|PAUSE|SET_MODE)|MODE|FORWARD|LOOP|PING_PONG|REVERSE|DATA|ROTATION|TRANSLATION)|ERR_(?:GENERIC|PARCEL_PERMISSIONS|MALFORMED_PARAMS|RUNTIME_PERMISSIONS|THROTTLED)|CHARACTER_(?:CMD_(?:(?:SMOOTH_)?STOP|JUMP)|DESIRED_(?:TURN_)?SPEED|RADIUS|STAY_WITHIN_PARCEL|LENGTH|ORIENTATION|ACCOUNT_FOR_SKIPPED_FRAMES|AVOIDANCE_MODE|TYPE(?:_(?:[A-D]|NONE))?|MAX_(?:DECEL|TURN_RADIUS|(?:ACCEL|SPEED)))|PURSUIT_(?:OFFSET|FUZZ_FACTOR|GOAL_TOLERANCE|INTERCEPT)|REQUIRE_LINE_OF_SIGHT|FORCE_DIRECT_PATH|VERTICAL|HORIZONTAL|AVOID_(?:CHARACTERS|DYNAMIC_OBSTACLES|NONE)|PU_(?:EVADE_(?:HIDDEN|SPOTTED)|FAILURE_(?:DYNAMIC_PATHFINDING_DISABLED|INVALID_(?:GOAL|START)|NO_(?:NAVMESH|VALID_DESTINATION)|OTHER|TARGET_GONE|(?:PARCEL_)?UNREACHABLE)|(?:GOAL|SLOWDOWN_DISTANCE)_REACHED)|TRAVERSAL_TYPE(?:_(?:FAST|NONE|SLOW))?|CONTENT_TYPE_(?:ATOM|FORM|HTML|JSON|LLSD|RSS|TEXT|XHTML|XML)|GCNP_(?:RADIUS|STATIC)|(?:PATROL|WANDER)_PAUSE_AT_WAYPOINTS|OPT_(?:AVATAR|CHARACTER|EXCLUSION_VOLUME|LEGACY_LINKSET|MATERIAL_VOLUME|OTHER|STATIC_OBSTACLE|WALKABLE)|SIM_STAT_PCT_CHARS_STEPPED)\b'
+    lsl_constants_integer_boolean = r'\b(?:FALSE|TRUE)\b'
+    lsl_constants_rotation = r'\b(?:ZERO_ROTATION)\b'
+    lsl_constants_string = r'\b(?:EOF|JSON_(?:ARRAY|DELETE|FALSE|INVALID|NULL|NUMBER|OBJECT|STRING|TRUE)|NULL_KEY|TEXTURE_(?:BLANK|DEFAULT|MEDIA|PLYWOOD|TRANSPARENT)|URL_REQUEST_(?:GRANTED|DENIED))\b'
+    lsl_constants_vector = r'\b(?:TOUCH_INVALID_(?:TEXCOORD|VECTOR)|ZERO_VECTOR)\b'
+    lsl_invalid_broken = r'\b(?:LAND_(?:LARGE|MEDIUM|SMALL)_BRUSH)\b'
+    lsl_invalid_deprecated = r'\b(?:ATTACH_[LR]PEC|DATA_RATING|OBJECT_ATTACHMENT_(?:GEOMETRY_BYTES|SURFACE_AREA)|PRIM_(?:CAST_SHADOWS|MATERIAL_LIGHT|TYPE_LEGACY)|PSYS_SRC_(?:INNER|OUTER)ANGLE|VEHICLE_FLAG_NO_FLY_UP|ll(?:Cloud|Make(?:Explosion|Fountain|Smoke|Fire)|RemoteDataSetRegion|Sound(?:Preload)?|XorBase64Strings(?:Correct)?))\b'
+    lsl_invalid_illegal = r'\b(?:event)\b'
+    lsl_invalid_unimplemented = r'\b(?:CHARACTER_(?:MAX_ANGULAR_(?:ACCEL|SPEED)|TURN_SPEED_MULTIPLIER)|PERMISSION_(?:CHANGE_(?:JOINTS|PERMISSIONS)|RELEASE_OWNERSHIP|REMAP_CONTROLS)|PRIM_PHYSICS_MATERIAL|PSYS_SRC_OBJ_REL_MASK|ll(?:CollisionSprite|(?:Stop)?PointAt|(?:(?:Refresh|Set)Prim)URL|(?:Take|Release)Camera|RemoteLoadScript))\b'
+    lsl_reserved_godmode = r'\b(?:ll(?:GodLikeRezObject|Set(?:Inventory|Object)PermMask))\b'
+    lsl_reserved_log = r'\b(?:print)\b'
+    lsl_operators = r'\+\+|\-\-|<<|>>|&&?|\|\|?|\^|~|[!%<>=*+\-/]=?'
+
+    tokens = {
+        'root':
+        [
+            (r'//.*?\n',                          Comment.Single),
+            (r'/\*',                              Comment.Multiline, 'comment'),
+            (r'"',                                String.Double, 'string'),
+            (lsl_keywords,                        Keyword),
+            (lsl_types,                           Keyword.Type),
+            (lsl_states,                          Name.Class),
+            (lsl_events,                          Name.Builtin),
+            (lsl_functions_builtin,               Name.Function),
+            (lsl_constants_float,                 Keyword.Constant),
+            (lsl_constants_integer,               Keyword.Constant),
+            (lsl_constants_integer_boolean,       Keyword.Constant),
+            (lsl_constants_rotation,              Keyword.Constant),
+            (lsl_constants_string,                Keyword.Constant),
+            (lsl_constants_vector,                Keyword.Constant),
+            (lsl_invalid_broken,                  Error),
+            (lsl_invalid_deprecated,              Error),
+            (lsl_invalid_illegal,                 Error),
+            (lsl_invalid_unimplemented,           Error),
+            (lsl_reserved_godmode,                Keyword.Reserved),
+            (lsl_reserved_log,                    Keyword.Reserved),
+            (r'\b([a-zA-Z_]\w*)\b',     Name.Variable),
+            (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d*', Number.Float),
+            (r'(\d+\.\d*|\.\d+)',                 Number.Float),
+            (r'0[xX][0-9a-fA-F]+',                Number.Hex),
+            (r'\d+',                              Number.Integer),
+            (lsl_operators,                       Operator),
+            (r':=?',                              Error),
+            (r'[,;{}()\[\]]',                     Punctuation),
+            (r'\n+',                              Whitespace),
+            (r'\s+',                              Whitespace)
+        ],
+        'comment':
+        [
+            (r'[^*/]+',                           Comment.Multiline),
+            (r'/\*',                              Comment.Multiline, '#push'),
+            (r'\*/',                              Comment.Multiline, '#pop'),
+            (r'[*/]',                             Comment.Multiline)
+        ],
+        'string':
+        [
+            (r'\\([nt"\\])',                      String.Escape),
+            (r'"',                                String.Double, '#pop'),
+            (r'\\.',                              Error),
+            (r'[^"\\]+',                          String.Double),
+        ]
+    }
+
+
+class AppleScriptLexer(RegexLexer):
+    """
+    For AppleScript source code,
+    including `AppleScript Studio
+    `_.
+    Contributed by Andreas Amann .
+    """
+
+    name = 'AppleScript'
+    url = 'https://developer.apple.com/library/archive/documentation/AppleScript/Conceptual/AppleScriptLangGuide/introduction/ASLR_intro.html'
+    aliases = ['applescript']
+    filenames = ['*.applescript']
+    version_added = '1.0'
+
+    flags = re.MULTILINE | re.DOTALL
+
+    Identifiers = r'[a-zA-Z]\w*'
+
+    # XXX: use words() for all of these
+    Literals = ('AppleScript', 'current application', 'false', 'linefeed',
+                'missing value', 'pi', 'quote', 'result', 'return', 'space',
+                'tab', 'text item delimiters', 'true', 'version')
+    Classes = ('alias ', 'application ', 'boolean ', 'class ', 'constant ',
+               'date ', 'file ', 'integer ', 'list ', 'number ', 'POSIX file ',
+               'real ', 'record ', 'reference ', 'RGB color ', 'script ',
+               'text ', 'unit types', '(?:Unicode )?text', 'string')
+    BuiltIn = ('attachment', 'attribute run', 'character', 'day', 'month',
+               'paragraph', 'word', 'year')
+    HandlerParams = ('about', 'above', 'against', 'apart from', 'around',
+                     'aside from', 'at', 'below', 'beneath', 'beside',
+                     'between', 'for', 'given', 'instead of', 'on', 'onto',
+                     'out of', 'over', 'since')
+    Commands = ('ASCII (character|number)', 'activate', 'beep', 'choose URL',
+                'choose application', 'choose color', 'choose file( name)?',
+                'choose folder', 'choose from list',
+                'choose remote application', 'clipboard info',
+                'close( access)?', 'copy', 'count', 'current date', 'delay',
+                'delete', 'display (alert|dialog)', 'do shell script',
+                'duplicate', 'exists', 'get eof', 'get volume settings',
+                'info for', 'launch', 'list (disks|folder)', 'load script',
+                'log', 'make', 'mount volume', 'new', 'offset',
+                'open( (for access|location))?', 'path to', 'print', 'quit',
+                'random number', 'read', 'round', 'run( script)?',
+                'say', 'scripting components',
+                'set (eof|the clipboard to|volume)', 'store script',
+                'summarize', 'system attribute', 'system info',
+                'the clipboard', 'time to GMT', 'write', 'quoted form')
+    References = ('(in )?back of', '(in )?front of', '[0-9]+(st|nd|rd|th)',
+                  'first', 'second', 'third', 'fourth', 'fifth', 'sixth',
+                  'seventh', 'eighth', 'ninth', 'tenth', 'after', 'back',
+                  'before', 'behind', 'every', 'front', 'index', 'last',
+                  'middle', 'some', 'that', 'through', 'thru', 'where', 'whose')
+    Operators = ("and", "or", "is equal", "equals", "(is )?equal to", "is not",
+                 "isn't", "isn't equal( to)?", "is not equal( to)?",
+                 "doesn't equal", "does not equal", "(is )?greater than",
+                 "comes after", "is not less than or equal( to)?",
+                 "isn't less than or equal( to)?", "(is )?less than",
+                 "comes before", "is not greater than or equal( to)?",
+                 "isn't greater than or equal( to)?",
+                 "(is  )?greater than or equal( to)?", "is not less than",
+                 "isn't less than", "does not come before",
+                 "doesn't come before", "(is )?less than or equal( to)?",
+                 "is not greater than", "isn't greater than",
+                 "does not come after", "doesn't come after", "starts? with",
+                 "begins? with", "ends? with", "contains?", "does not contain",
+                 "doesn't contain", "is in", "is contained by", "is not in",
+                 "is not contained by", "isn't contained by", "div", "mod",
+                 "not", "(a  )?(ref( to)?|reference to)", "is", "does")
+    Control = ('considering', 'else', 'error', 'exit', 'from', 'if',
+               'ignoring', 'in', 'repeat', 'tell', 'then', 'times', 'to',
+               'try', 'until', 'using terms from', 'while', 'whith',
+               'with timeout( of)?', 'with transaction', 'by', 'continue',
+               'end', 'its?', 'me', 'my', 'return', 'of', 'as')
+    Declarations = ('global', 'local', 'prop(erty)?', 'set', 'get')
+    Reserved = ('but', 'put', 'returning', 'the')
+    StudioClasses = ('action cell', 'alert reply', 'application', 'box',
+                     'browser( cell)?', 'bundle', 'button( cell)?', 'cell',
+                     'clip view', 'color well', 'color-panel',
+                     'combo box( item)?', 'control',
+                     'data( (cell|column|item|row|source))?', 'default entry',
+                     'dialog reply', 'document', 'drag info', 'drawer',
+                     'event', 'font(-panel)?', 'formatter',
+                     'image( (cell|view))?', 'matrix', 'menu( item)?', 'item',
+                     'movie( view)?', 'open-panel', 'outline view', 'panel',
+                     'pasteboard', 'plugin', 'popup button',
+                     'progress indicator', 'responder', 'save-panel',
+                     'scroll view', 'secure text field( cell)?', 'slider',
+                     'sound', 'split view', 'stepper', 'tab view( item)?',
+                     'table( (column|header cell|header view|view))',
+                     'text( (field( cell)?|view))?', 'toolbar( item)?',
+                     'user-defaults', 'view', 'window')
+    StudioEvents = ('accept outline drop', 'accept table drop', 'action',
+                    'activated', 'alert ended', 'awake from nib', 'became key',
+                    'became main', 'begin editing', 'bounds changed',
+                    'cell value', 'cell value changed', 'change cell value',
+                    'change item value', 'changed', 'child of item',
+                    'choose menu item', 'clicked', 'clicked toolbar item',
+                    'closed', 'column clicked', 'column moved',
+                    'column resized', 'conclude drop', 'data representation',
+                    'deminiaturized', 'dialog ended', 'document nib name',
+                    'double clicked', 'drag( (entered|exited|updated))?',
+                    'drop', 'end editing', 'exposed', 'idle', 'item expandable',
+                    'item value', 'item value changed', 'items changed',
+                    'keyboard down', 'keyboard up', 'launched',
+                    'load data representation', 'miniaturized', 'mouse down',
+                    'mouse dragged', 'mouse entered', 'mouse exited',
+                    'mouse moved', 'mouse up', 'moved',
+                    'number of browser rows', 'number of items',
+                    'number of rows', 'open untitled', 'opened', 'panel ended',
+                    'parameters updated', 'plugin loaded', 'prepare drop',
+                    'prepare outline drag', 'prepare outline drop',
+                    'prepare table drag', 'prepare table drop',
+                    'read from file', 'resigned active', 'resigned key',
+                    'resigned main', 'resized( sub views)?',
+                    'right mouse down', 'right mouse dragged',
+                    'right mouse up', 'rows changed', 'scroll wheel',
+                    'selected tab view item', 'selection changed',
+                    'selection changing', 'should begin editing',
+                    'should close', 'should collapse item',
+                    'should end editing', 'should expand item',
+                    'should open( untitled)?',
+                    'should quit( after last window closed)?',
+                    'should select column', 'should select item',
+                    'should select row', 'should select tab view item',
+                    'should selection change', 'should zoom', 'shown',
+                    'update menu item', 'update parameters',
+                    'update toolbar item', 'was hidden', 'was miniaturized',
+                    'will become active', 'will close', 'will dismiss',
+                    'will display browser cell', 'will display cell',
+                    'will display item cell', 'will display outline cell',
+                    'will finish launching', 'will hide', 'will miniaturize',
+                    'will move', 'will open', 'will pop up', 'will quit',
+                    'will resign active', 'will resize( sub views)?',
+                    'will select tab view item', 'will show', 'will zoom',
+                    'write to file', 'zoomed')
+    StudioCommands = ('animate', 'append', 'call method', 'center',
+                      'close drawer', 'close panel', 'display',
+                      'display alert', 'display dialog', 'display panel', 'go',
+                      'hide', 'highlight', 'increment', 'item for',
+                      'load image', 'load movie', 'load nib', 'load panel',
+                      'load sound', 'localized string', 'lock focus', 'log',
+                      'open drawer', 'path for', 'pause', 'perform action',
+                      'play', 'register', 'resume', 'scroll', 'select( all)?',
+                      'show', 'size to fit', 'start', 'step back',
+                      'step forward', 'stop', 'synchronize', 'unlock focus',
+                      'update')
+    StudioProperties = ('accepts arrow key', 'action method', 'active',
+                        'alignment', 'allowed identifiers',
+                        'allows branch selection', 'allows column reordering',
+                        'allows column resizing', 'allows column selection',
+                        'allows customization',
+                        'allows editing text attributes',
+                        'allows empty selection', 'allows mixed state',
+                        'allows multiple selection', 'allows reordering',
+                        'allows undo', 'alpha( value)?', 'alternate image',
+                        'alternate increment value', 'alternate title',
+                        'animation delay', 'associated file name',
+                        'associated object', 'auto completes', 'auto display',
+                        'auto enables items', 'auto repeat',
+                        'auto resizes( outline column)?',
+                        'auto save expanded items', 'auto save name',
+                        'auto save table columns', 'auto saves configuration',
+                        'auto scroll', 'auto sizes all columns to fit',
+                        'auto sizes cells', 'background color', 'bezel state',
+                        'bezel style', 'bezeled', 'border rect', 'border type',
+                        'bordered', 'bounds( rotation)?', 'box type',
+                        'button returned', 'button type',
+                        'can choose directories', 'can choose files',
+                        'can draw', 'can hide',
+                        'cell( (background color|size|type))?', 'characters',
+                        'class', 'click count', 'clicked( data)? column',
+                        'clicked data item', 'clicked( data)? row',
+                        'closeable', 'collating', 'color( (mode|panel))',
+                        'command key down', 'configuration',
+                        'content(s| (size|view( margins)?))?', 'context',
+                        'continuous', 'control key down', 'control size',
+                        'control tint', 'control view',
+                        'controller visible', 'coordinate system',
+                        'copies( on scroll)?', 'corner view', 'current cell',
+                        'current column', 'current( field)?  editor',
+                        'current( menu)? item', 'current row',
+                        'current tab view item', 'data source',
+                        'default identifiers', 'delta (x|y|z)',
+                        'destination window', 'directory', 'display mode',
+                        'displayed cell', 'document( (edited|rect|view))?',
+                        'double value', 'dragged column', 'dragged distance',
+                        'dragged items', 'draws( cell)? background',
+                        'draws grid', 'dynamically scrolls', 'echos bullets',
+                        'edge', 'editable', 'edited( data)? column',
+                        'edited data item', 'edited( data)? row', 'enabled',
+                        'enclosing scroll view', 'ending page',
+                        'error handling', 'event number', 'event type',
+                        'excluded from windows menu', 'executable path',
+                        'expanded', 'fax number', 'field editor', 'file kind',
+                        'file name', 'file type', 'first responder',
+                        'first visible column', 'flipped', 'floating',
+                        'font( panel)?', 'formatter', 'frameworks path',
+                        'frontmost', 'gave up', 'grid color', 'has data items',
+                        'has horizontal ruler', 'has horizontal scroller',
+                        'has parent data item', 'has resize indicator',
+                        'has shadow', 'has sub menu', 'has vertical ruler',
+                        'has vertical scroller', 'header cell', 'header view',
+                        'hidden', 'hides when deactivated', 'highlights by',
+                        'horizontal line scroll', 'horizontal page scroll',
+                        'horizontal ruler view', 'horizontally resizable',
+                        'icon image', 'id', 'identifier',
+                        'ignores multiple clicks',
+                        'image( (alignment|dims when disabled|frame style|scaling))?',
+                        'imports graphics', 'increment value',
+                        'indentation per level', 'indeterminate', 'index',
+                        'integer value', 'intercell spacing', 'item height',
+                        'key( (code|equivalent( modifier)?|window))?',
+                        'knob thickness', 'label', 'last( visible)? column',
+                        'leading offset', 'leaf', 'level', 'line scroll',
+                        'loaded', 'localized sort', 'location', 'loop mode',
+                        'main( (bunde|menu|window))?', 'marker follows cell',
+                        'matrix mode', 'maximum( content)? size',
+                        'maximum visible columns',
+                        'menu( form representation)?', 'miniaturizable',
+                        'miniaturized', 'minimized image', 'minimized title',
+                        'minimum column width', 'minimum( content)? size',
+                        'modal', 'modified', 'mouse down state',
+                        'movie( (controller|file|rect))?', 'muted', 'name',
+                        'needs display', 'next state', 'next text',
+                        'number of tick marks', 'only tick mark values',
+                        'opaque', 'open panel', 'option key down',
+                        'outline table column', 'page scroll', 'pages across',
+                        'pages down', 'palette label', 'pane splitter',
+                        'parent data item', 'parent window', 'pasteboard',
+                        'path( (names|separator))?', 'playing',
+                        'plays every frame', 'plays selection only', 'position',
+                        'preferred edge', 'preferred type', 'pressure',
+                        'previous text', 'prompt', 'properties',
+                        'prototype cell', 'pulls down', 'rate',
+                        'released when closed', 'repeated',
+                        'requested print time', 'required file type',
+                        'resizable', 'resized column', 'resource path',
+                        'returns records', 'reuses columns', 'rich text',
+                        'roll over', 'row height', 'rulers visible',
+                        'save panel', 'scripts path', 'scrollable',
+                        'selectable( identifiers)?', 'selected cell',
+                        'selected( data)? columns?', 'selected data items?',
+                        'selected( data)? rows?', 'selected item identifier',
+                        'selection by rect', 'send action on arrow key',
+                        'sends action when done editing', 'separates columns',
+                        'separator item', 'sequence number', 'services menu',
+                        'shared frameworks path', 'shared support path',
+                        'sheet', 'shift key down', 'shows alpha',
+                        'shows state by', 'size( mode)?',
+                        'smart insert delete enabled', 'sort case sensitivity',
+                        'sort column', 'sort order', 'sort type',
+                        'sorted( data rows)?', 'sound', 'source( mask)?',
+                        'spell checking enabled', 'starting page', 'state',
+                        'string value', 'sub menu', 'super menu', 'super view',
+                        'tab key traverses cells', 'tab state', 'tab type',
+                        'tab view', 'table view', 'tag', 'target( printer)?',
+                        'text color', 'text container insert',
+                        'text container origin', 'text returned',
+                        'tick mark position', 'time stamp',
+                        'title(d| (cell|font|height|position|rect))?',
+                        'tool tip', 'toolbar', 'trailing offset', 'transparent',
+                        'treat packages as directories', 'truncated labels',
+                        'types', 'unmodified characters', 'update views',
+                        'use sort indicator', 'user defaults',
+                        'uses data source', 'uses ruler',
+                        'uses threaded animation',
+                        'uses title from previous column', 'value wraps',
+                        'version',
+                        'vertical( (line scroll|page scroll|ruler view))?',
+                        'vertically resizable', 'view',
+                        'visible( document rect)?', 'volume', 'width', 'window',
+                        'windows menu', 'wraps', 'zoomable', 'zoomed')
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'¬\n', String.Escape),
+            (r"'s\s+", Text),  # This is a possessive, consider moving
+            (r'(--|#).*?$', Comment),
+            (r'\(\*', Comment.Multiline, 'comment'),
+            (r'[(){}!,.:]', Punctuation),
+            (r'(«)([^»]+)(»)',
+             bygroups(Text, Name.Builtin, Text)),
+            (r'\b((?:considering|ignoring)\s*)'
+             r'(application responses|case|diacriticals|hyphens|'
+             r'numeric strings|punctuation|white space)',
+             bygroups(Keyword, Name.Builtin)),
+            (r'(-|\*|\+|&|≠|>=?|<=?|=|≥|≤|/|÷|\^)', Operator),
+            (r"\b({})\b".format('|'.join(Operators)), Operator.Word),
+            (r'^(\s*(?:on|end)\s+)'
+             r'({})'.format('|'.join(StudioEvents[::-1])),
+             bygroups(Keyword, Name.Function)),
+            (r'^(\s*)(in|on|script|to)(\s+)', bygroups(Text, Keyword, Text)),
+            (r'\b(as )({})\b'.format('|'.join(Classes)),
+             bygroups(Keyword, Name.Class)),
+            (r'\b({})\b'.format('|'.join(Literals)), Name.Constant),
+            (r'\b({})\b'.format('|'.join(Commands)), Name.Builtin),
+            (r'\b({})\b'.format('|'.join(Control)), Keyword),
+            (r'\b({})\b'.format('|'.join(Declarations)), Keyword),
+            (r'\b({})\b'.format('|'.join(Reserved)), Name.Builtin),
+            (r'\b({})s?\b'.format('|'.join(BuiltIn)), Name.Builtin),
+            (r'\b({})\b'.format('|'.join(HandlerParams)), Name.Builtin),
+            (r'\b({})\b'.format('|'.join(StudioProperties)), Name.Attribute),
+            (r'\b({})s?\b'.format('|'.join(StudioClasses)), Name.Builtin),
+            (r'\b({})\b'.format('|'.join(StudioCommands)), Name.Builtin),
+            (r'\b({})\b'.format('|'.join(References)), Name.Builtin),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
+            (rf'\b({Identifiers})\b', Name.Variable),
+            (r'[-+]?(\d+\.\d*|\d*\.\d+)(E[-+][0-9]+)?', Number.Float),
+            (r'[-+]?\d+', Number.Integer),
+        ],
+        'comment': [
+            (r'\(\*', Comment.Multiline, '#push'),
+            (r'\*\)', Comment.Multiline, '#pop'),
+            ('[^*(]+', Comment.Multiline),
+            ('[*(]', Comment.Multiline),
+        ],
+    }
+
+
+class RexxLexer(RegexLexer):
+    """
+    Rexx is a scripting language available for
+    a wide range of different platforms with its roots found on mainframe
+    systems. It is popular for I/O- and data based tasks and can act as glue
+    language to bind different applications together.
+    """
+    name = 'Rexx'
+    url = 'http://www.rexxinfo.org/'
+    aliases = ['rexx', 'arexx']
+    filenames = ['*.rexx', '*.rex', '*.rx', '*.arexx']
+    mimetypes = ['text/x-rexx']
+    version_added = '2.0'
+    flags = re.IGNORECASE
+
+    tokens = {
+        'root': [
+            (r'\s+', Whitespace),
+            (r'/\*', Comment.Multiline, 'comment'),
+            (r'"', String, 'string_double'),
+            (r"'", String, 'string_single'),
+            (r'[0-9]+(\.[0-9]+)?(e[+-]?[0-9])?', Number),
+            (r'([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b',
+             bygroups(Name.Function, Whitespace, Operator, Whitespace,
+                      Keyword.Declaration)),
+            (r'([a-z_]\w*)(\s*)(:)',
+             bygroups(Name.Label, Whitespace, Operator)),
+            include('function'),
+            include('keyword'),
+            include('operator'),
+            (r'[a-z_]\w*', Text),
+        ],
+        'function': [
+            (words((
+                'abbrev', 'abs', 'address', 'arg', 'b2x', 'bitand', 'bitor', 'bitxor',
+                'c2d', 'c2x', 'center', 'charin', 'charout', 'chars', 'compare',
+                'condition', 'copies', 'd2c', 'd2x', 'datatype', 'date', 'delstr',
+                'delword', 'digits', 'errortext', 'form', 'format', 'fuzz', 'insert',
+                'lastpos', 'left', 'length', 'linein', 'lineout', 'lines', 'max',
+                'min', 'overlay', 'pos', 'queued', 'random', 'reverse', 'right', 'sign',
+                'sourceline', 'space', 'stream', 'strip', 'substr', 'subword', 'symbol',
+                'time', 'trace', 'translate', 'trunc', 'value', 'verify', 'word',
+                'wordindex', 'wordlength', 'wordpos', 'words', 'x2b', 'x2c', 'x2d',
+                'xrange'), suffix=r'(\s*)(\()'),
+             bygroups(Name.Builtin, Whitespace, Operator)),
+        ],
+        'keyword': [
+            (r'(address|arg|by|call|do|drop|else|end|exit|for|forever|if|'
+             r'interpret|iterate|leave|nop|numeric|off|on|options|parse|'
+             r'pull|push|queue|return|say|select|signal|to|then|trace|until|'
+             r'while)\b', Keyword.Reserved),
+        ],
+        'operator': [
+            (r'(-|//|/|\(|\)|\*\*|\*|\\<<|\\<|\\==|\\=|\\>>|\\>|\\|\|\||\||'
+             r'&&|&|%|\+|<<=|<<|<=|<>|<|==|=|><|>=|>>=|>>|>|¬<<|¬<|¬==|¬=|'
+             r'¬>>|¬>|¬|\.|,)', Operator),
+        ],
+        'string_double': [
+            (r'[^"\n]+', String),
+            (r'""', String),
+            (r'"', String, '#pop'),
+            (r'\n', Text, '#pop'),  # Stray linefeed also terminates strings.
+        ],
+        'string_single': [
+            (r'[^\'\n]+', String),
+            (r'\'\'', String),
+            (r'\'', String, '#pop'),
+            (r'\n', Text, '#pop'),  # Stray linefeed also terminates strings.
+        ],
+        'comment': [
+            (r'[^*]+', Comment.Multiline),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'\*', Comment.Multiline),
+        ]
+    }
+
+    def _c(s):
+        return re.compile(s, re.MULTILINE)
+    _ADDRESS_COMMAND_PATTERN = _c(r'^\s*address\s+command\b')
+    _ADDRESS_PATTERN = _c(r'^\s*address\s+')
+    _DO_WHILE_PATTERN = _c(r'^\s*do\s+while\b')
+    _IF_THEN_DO_PATTERN = _c(r'^\s*if\b.+\bthen\s+do\s*$')
+    _PROCEDURE_PATTERN = _c(r'^\s*([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b')
+    _ELSE_DO_PATTERN = _c(r'\belse\s+do\s*$')
+    _PARSE_ARG_PATTERN = _c(r'^\s*parse\s+(upper\s+)?(arg|value)\b')
+    PATTERNS_AND_WEIGHTS = (
+        (_ADDRESS_COMMAND_PATTERN, 0.2),
+        (_ADDRESS_PATTERN, 0.05),
+        (_DO_WHILE_PATTERN, 0.1),
+        (_ELSE_DO_PATTERN, 0.1),
+        (_IF_THEN_DO_PATTERN, 0.1),
+        (_PROCEDURE_PATTERN, 0.5),
+        (_PARSE_ARG_PATTERN, 0.2),
+    )
+
+    def analyse_text(text):
+        """
+        Check for initial comment and patterns that distinguish Rexx from other
+        C-like languages.
+        """
+        if re.search(r'/\*\**\s*rexx', text, re.IGNORECASE):
+            # Header matches MVS Rexx requirements, this is certainly a Rexx
+            # script.
+            return 1.0
+        elif text.startswith('/*'):
+            # Header matches general Rexx requirements; the source code might
+            # still be any language using C comments such as C++, C# or Java.
+            lowerText = text.lower()
+            result = sum(weight
+                         for (pattern, weight) in RexxLexer.PATTERNS_AND_WEIGHTS
+                         if pattern.search(lowerText)) + 0.01
+            return min(result, 1.0)
+
+
+class MOOCodeLexer(RegexLexer):
+    """
+    For MOOCode (the MOO scripting language).
+    """
+    name = 'MOOCode'
+    url = 'http://www.moo.mud.org/'
+    filenames = ['*.moo']
+    aliases = ['moocode', 'moo']
+    mimetypes = ['text/x-moocode']
+    version_added = '0.9'
+
+    tokens = {
+        'root': [
+            # Numbers
+            (r'(0|[1-9][0-9_]*)', Number.Integer),
+            # Strings
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+            # exceptions
+            (r'(E_PERM|E_DIV)', Name.Exception),
+            # db-refs
+            (r'((#[-0-9]+)|(\$\w+))', Name.Entity),
+            # Keywords
+            (r'\b(if|else|elseif|endif|for|endfor|fork|endfork|while'
+             r'|endwhile|break|continue|return|try'
+             r'|except|endtry|finally|in)\b', Keyword),
+            # builtins
+            (r'(random|length)', Name.Builtin),
+            # special variables
+            (r'(player|caller|this|args)', Name.Variable.Instance),
+            # skip whitespace
+            (r'\s+', Text),
+            (r'\n', Text),
+            # other operators
+            (r'([!;=,{}&|:.\[\]@()<>?]+)', Operator),
+            # function call
+            (r'(\w+)(\()', bygroups(Name.Function, Operator)),
+            # variables
+            (r'(\w+)', Text),
+        ]
+    }
+
+
+class HybrisLexer(RegexLexer):
+    """
+    For Hybris source code.
+    """
+
+    name = 'Hybris'
+    aliases = ['hybris']
+    filenames = ['*.hyb']
+    mimetypes = ['text/x-hybris', 'application/x-hybris']
+    url = 'https://github.com/evilsocket/hybris'
+    version_added = '1.4'
+
+    flags = re.MULTILINE | re.DOTALL
+
+    tokens = {
+        'root': [
+            # method names
+            (r'^(\s*(?:function|method|operator\s+)+?)'
+             r'([a-zA-Z_]\w*)'
+             r'(\s*)(\()', bygroups(Keyword, Name.Function, Text, Operator)),
+            (r'[^\S\n]+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'@[a-zA-Z_][\w.]*', Name.Decorator),
+            (r'(break|case|catch|next|default|do|else|finally|for|foreach|of|'
+             r'unless|if|new|return|switch|me|throw|try|while)\b', Keyword),
+            (r'(extends|private|protected|public|static|throws|function|method|'
+             r'operator)\b', Keyword.Declaration),
+            (r'(true|false|null|__FILE__|__LINE__|__VERSION__|__LIB_PATH__|'
+             r'__INC_PATH__)\b', Keyword.Constant),
+            (r'(class|struct)(\s+)',
+             bygroups(Keyword.Declaration, Text), 'class'),
+            (r'(import|include)(\s+)',
+             bygroups(Keyword.Namespace, Text), 'import'),
+            (words((
+                'gc_collect', 'gc_mm_items', 'gc_mm_usage', 'gc_collect_threshold',
+                'urlencode', 'urldecode', 'base64encode', 'base64decode', 'sha1', 'crc32',
+                'sha2', 'md5', 'md5_file', 'acos', 'asin', 'atan', 'atan2', 'ceil', 'cos',
+                'cosh', 'exp', 'fabs', 'floor', 'fmod', 'log', 'log10', 'pow', 'sin',
+                'sinh', 'sqrt', 'tan', 'tanh', 'isint', 'isfloat', 'ischar', 'isstring',
+                'isarray', 'ismap', 'isalias', 'typeof', 'sizeof', 'toint', 'tostring',
+                'fromxml', 'toxml', 'binary', 'pack', 'load', 'eval', 'var_names',
+                'var_values', 'user_functions', 'dyn_functions', 'methods', 'call',
+                'call_method', 'mknod', 'mkfifo', 'mount', 'umount2', 'umount', 'ticks',
+                'usleep', 'sleep', 'time', 'strtime', 'strdate', 'dllopen', 'dlllink',
+                'dllcall', 'dllcall_argv', 'dllclose', 'env', 'exec', 'fork', 'getpid',
+                'wait', 'popen', 'pclose', 'exit', 'kill', 'pthread_create',
+                'pthread_create_argv', 'pthread_exit', 'pthread_join', 'pthread_kill',
+                'smtp_send', 'http_get', 'http_post', 'http_download', 'socket', 'bind',
+                'listen', 'accept', 'getsockname', 'getpeername', 'settimeout', 'connect',
+                'server', 'recv', 'send', 'close', 'print', 'println', 'printf', 'input',
+                'readline', 'serial_open', 'serial_fcntl', 'serial_get_attr',
+                'serial_get_ispeed', 'serial_get_ospeed', 'serial_set_attr',
+                'serial_set_ispeed', 'serial_set_ospeed', 'serial_write', 'serial_read',
+                'serial_close', 'xml_load', 'xml_parse', 'fopen', 'fseek', 'ftell',
+                'fsize', 'fread', 'fwrite', 'fgets', 'fclose', 'file', 'readdir',
+                'pcre_replace', 'size', 'pop', 'unmap', 'has', 'keys', 'values',
+                'length', 'find', 'substr', 'replace', 'split', 'trim', 'remove',
+                'contains', 'join'), suffix=r'\b'),
+             Name.Builtin),
+            (words((
+                'MethodReference', 'Runner', 'Dll', 'Thread', 'Pipe', 'Process',
+                'Runnable', 'CGI', 'ClientSocket', 'Socket', 'ServerSocket',
+                'File', 'Console', 'Directory', 'Exception'), suffix=r'\b'),
+             Keyword.Type),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+            (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char),
+            (r'(\.)([a-zA-Z_]\w*)',
+             bygroups(Operator, Name.Attribute)),
+            (r'[a-zA-Z_]\w*:', Name.Label),
+            (r'[a-zA-Z_$]\w*', Name),
+            (r'[~^*!%&\[\](){}<>|+=:;,./?\-@]+', Operator),
+            (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+            (r'0x[0-9a-f]+', Number.Hex),
+            (r'[0-9]+L?', Number.Integer),
+            (r'\n', Text),
+        ],
+        'class': [
+            (r'[a-zA-Z_]\w*', Name.Class, '#pop')
+        ],
+        'import': [
+            (r'[\w.]+\*?', Name.Namespace, '#pop')
+        ],
+    }
+
+    def analyse_text(text):
+        """public method and private method don't seem to be quite common
+        elsewhere."""
+        result = 0
+        if re.search(r'\b(?:public|private)\s+method\b', text):
+            result += 0.01
+        return result
+
+
+
+class EasytrieveLexer(RegexLexer):
+    """
+    Easytrieve Plus is a programming language for extracting, filtering and
+    converting sequential data. Furthermore it can layout data for reports.
+    It is mainly used on mainframe platforms and can access several of the
+    mainframe's native file formats. It is somewhat comparable to awk.
+    """
+    name = 'Easytrieve'
+    aliases = ['easytrieve']
+    filenames = ['*.ezt', '*.mac']
+    mimetypes = ['text/x-easytrieve']
+    url = 'https://www.broadcom.com/products/mainframe/application-development/easytrieve-report-generator'
+    version_added = '2.1'
+    flags = 0
+
+    # Note: We cannot use r'\b' at the start and end of keywords because
+    # Easytrieve Plus delimiter characters are:
+    #
+    #   * space ( )
+    #   * apostrophe (')
+    #   * period (.)
+    #   * comma (,)
+    #   * parenthesis ( and )
+    #   * colon (:)
+    #
+    # Additionally words end once a '*' appears, indicatins a comment.
+    _DELIMITERS = r' \'.,():\n'
+    _DELIMITERS_OR_COMENT = _DELIMITERS + '*'
+    _DELIMITER_PATTERN = '[' + _DELIMITERS + ']'
+    _DELIMITER_PATTERN_CAPTURE = '(' + _DELIMITER_PATTERN + ')'
+    _NON_DELIMITER_OR_COMMENT_PATTERN = '[^' + _DELIMITERS_OR_COMENT + ']'
+    _OPERATORS_PATTERN = '[.+\\-/=\\[\\](){}<>;,&%¬]'
+    _KEYWORDS = [
+        'AFTER-BREAK', 'AFTER-LINE', 'AFTER-SCREEN', 'AIM', 'AND', 'ATTR',
+        'BEFORE', 'BEFORE-BREAK', 'BEFORE-LINE', 'BEFORE-SCREEN', 'BUSHU',
+        'BY', 'CALL', 'CASE', 'CHECKPOINT', 'CHKP', 'CHKP-STATUS', 'CLEAR',
+        'CLOSE', 'COL', 'COLOR', 'COMMIT', 'CONTROL', 'COPY', 'CURSOR', 'D',
+        'DECLARE', 'DEFAULT', 'DEFINE', 'DELETE', 'DENWA', 'DISPLAY', 'DLI',
+        'DO', 'DUPLICATE', 'E', 'ELSE', 'ELSE-IF', 'END', 'END-CASE',
+        'END-DO', 'END-IF', 'END-PROC', 'ENDPAGE', 'ENDTABLE', 'ENTER', 'EOF',
+        'EQ', 'ERROR', 'EXIT', 'EXTERNAL', 'EZLIB', 'F1', 'F10', 'F11', 'F12',
+        'F13', 'F14', 'F15', 'F16', 'F17', 'F18', 'F19', 'F2', 'F20', 'F21',
+        'F22', 'F23', 'F24', 'F25', 'F26', 'F27', 'F28', 'F29', 'F3', 'F30',
+        'F31', 'F32', 'F33', 'F34', 'F35', 'F36', 'F4', 'F5', 'F6', 'F7',
+        'F8', 'F9', 'FETCH', 'FILE-STATUS', 'FILL', 'FINAL', 'FIRST',
+        'FIRST-DUP', 'FOR', 'GE', 'GET', 'GO', 'GOTO', 'GQ', 'GR', 'GT',
+        'HEADING', 'HEX', 'HIGH-VALUES', 'IDD', 'IDMS', 'IF', 'IN', 'INSERT',
+        'JUSTIFY', 'KANJI-DATE', 'KANJI-DATE-LONG', 'KANJI-TIME', 'KEY',
+        'KEY-PRESSED', 'KOKUGO', 'KUN', 'LAST-DUP', 'LE', 'LEVEL', 'LIKE',
+        'LINE', 'LINE-COUNT', 'LINE-NUMBER', 'LINK', 'LIST', 'LOW-VALUES',
+        'LQ', 'LS', 'LT', 'MACRO', 'MASK', 'MATCHED', 'MEND', 'MESSAGE',
+        'MOVE', 'MSTART', 'NE', 'NEWPAGE', 'NOMASK', 'NOPRINT', 'NOT',
+        'NOTE', 'NOVERIFY', 'NQ', 'NULL', 'OF', 'OR', 'OTHERWISE', 'PA1',
+        'PA2', 'PA3', 'PAGE-COUNT', 'PAGE-NUMBER', 'PARM-REGISTER',
+        'PATH-ID', 'PATTERN', 'PERFORM', 'POINT', 'POS', 'PRIMARY', 'PRINT',
+        'PROCEDURE', 'PROGRAM', 'PUT', 'READ', 'RECORD', 'RECORD-COUNT',
+        'RECORD-LENGTH', 'REFRESH', 'RELEASE', 'RENUM', 'REPEAT', 'REPORT',
+        'REPORT-INPUT', 'RESHOW', 'RESTART', 'RETRIEVE', 'RETURN-CODE',
+        'ROLLBACK', 'ROW', 'S', 'SCREEN', 'SEARCH', 'SECONDARY', 'SELECT',
+        'SEQUENCE', 'SIZE', 'SKIP', 'SOKAKU', 'SORT', 'SQL', 'STOP', 'SUM',
+        'SYSDATE', 'SYSDATE-LONG', 'SYSIN', 'SYSIPT', 'SYSLST', 'SYSPRINT',
+        'SYSSNAP', 'SYSTIME', 'TALLY', 'TERM-COLUMNS', 'TERM-NAME',
+        'TERM-ROWS', 'TERMINATION', 'TITLE', 'TO', 'TRANSFER', 'TRC',
+        'UNIQUE', 'UNTIL', 'UPDATE', 'UPPERCASE', 'USER', 'USERID', 'VALUE',
+        'VERIFY', 'W', 'WHEN', 'WHILE', 'WORK', 'WRITE', 'X', 'XDM', 'XRST'
+    ]
+
+    tokens = {
+        'root': [
+            (r'\*.*\n', Comment.Single),
+            (r'\n+', Whitespace),
+            # Macro argument
+            (r'&' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+\.', Name.Variable,
+             'after_macro_argument'),
+            # Macro call
+            (r'%' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name.Variable),
+            (r'(FILE|MACRO|REPORT)(\s+)',
+             bygroups(Keyword.Declaration, Whitespace), 'after_declaration'),
+            (r'(JOB|PARM)' + r'(' + _DELIMITER_PATTERN + r')',
+             bygroups(Keyword.Declaration, Operator)),
+            (words(_KEYWORDS, suffix=_DELIMITER_PATTERN_CAPTURE),
+             bygroups(Keyword.Reserved, Operator)),
+            (_OPERATORS_PATTERN, Operator),
+            # Procedure declaration
+            (r'(' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+)(\s*)(\.?)(\s*)(PROC)(\s*\n)',
+             bygroups(Name.Function, Whitespace, Operator, Whitespace,
+                      Keyword.Declaration, Whitespace)),
+            (r'[0-9]+\.[0-9]*', Number.Float),
+            (r'[0-9]+', Number.Integer),
+            (r"'(''|[^'])*'", String),
+            (r'\s+', Whitespace),
+            # Everything else just belongs to a name
+            (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name),
+         ],
+        'after_declaration': [
+            (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name.Function),
+            default('#pop'),
+        ],
+        'after_macro_argument': [
+            (r'\*.*\n', Comment.Single, '#pop'),
+            (r'\s+', Whitespace, '#pop'),
+            (_OPERATORS_PATTERN, Operator, '#pop'),
+            (r"'(''|[^'])*'", String, '#pop'),
+            # Everything else just belongs to a name
+            (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name),
+        ],
+    }
+    _COMMENT_LINE_REGEX = re.compile(r'^\s*\*')
+    _MACRO_HEADER_REGEX = re.compile(r'^\s*MACRO')
+
+    def analyse_text(text):
+        """
+        Perform a structural analysis for basic Easytrieve constructs.
+        """
+        result = 0.0
+        lines = text.split('\n')
+        hasEndProc = False
+        hasHeaderComment = False
+        hasFile = False
+        hasJob = False
+        hasProc = False
+        hasParm = False
+        hasReport = False
+
+        def isCommentLine(line):
+            return EasytrieveLexer._COMMENT_LINE_REGEX.match(lines[0]) is not None
+
+        def isEmptyLine(line):
+            return not bool(line.strip())
+
+        # Remove possible empty lines and header comments.
+        while lines and (isEmptyLine(lines[0]) or isCommentLine(lines[0])):
+            if not isEmptyLine(lines[0]):
+                hasHeaderComment = True
+            del lines[0]
+
+        if EasytrieveLexer._MACRO_HEADER_REGEX.match(lines[0]):
+            # Looks like an Easytrieve macro.
+            result = 0.4
+            if hasHeaderComment:
+                result += 0.4
+        else:
+            # Scan the source for lines starting with indicators.
+            for line in lines:
+                words = line.split()
+                if (len(words) >= 2):
+                    firstWord = words[0]
+                    if not hasReport:
+                        if not hasJob:
+                            if not hasFile:
+                                if not hasParm:
+                                    if firstWord == 'PARM':
+                                        hasParm = True
+                                if firstWord == 'FILE':
+                                    hasFile = True
+                            if firstWord == 'JOB':
+                                hasJob = True
+                        elif firstWord == 'PROC':
+                            hasProc = True
+                        elif firstWord == 'END-PROC':
+                            hasEndProc = True
+                        elif firstWord == 'REPORT':
+                            hasReport = True
+
+            # Weight the findings.
+            if hasJob and (hasProc == hasEndProc):
+                if hasHeaderComment:
+                    result += 0.1
+                if hasParm:
+                    if hasProc:
+                        # Found PARM, JOB and PROC/END-PROC:
+                        # pretty sure this is Easytrieve.
+                        result += 0.8
+                    else:
+                        # Found PARAM and  JOB: probably this is Easytrieve
+                        result += 0.5
+                else:
+                    # Found JOB and possibly other keywords: might be Easytrieve
+                    result += 0.11
+                    if hasParm:
+                        # Note: PARAM is not a proper English word, so this is
+                        # regarded a much better indicator for Easytrieve than
+                        # the other words.
+                        result += 0.2
+                    if hasFile:
+                        result += 0.01
+                    if hasReport:
+                        result += 0.01
+        assert 0.0 <= result <= 1.0
+        return result
+
+
+class JclLexer(RegexLexer):
+    """
+    Job Control Language (JCL)
+    is a scripting language used on mainframe platforms to instruct the system
+    on how to run a batch job or start a subsystem. It is somewhat
+    comparable to MS DOS batch and Unix shell scripts.
+    """
+    name = 'JCL'
+    aliases = ['jcl']
+    filenames = ['*.jcl']
+    mimetypes = ['text/x-jcl']
+    url = 'https://en.wikipedia.org/wiki/Job_Control_Language'
+    version_added = '2.1'
+
+    flags = re.IGNORECASE
+
+    tokens = {
+        'root': [
+            (r'//\*.*\n', Comment.Single),
+            (r'//', Keyword.Pseudo, 'statement'),
+            (r'/\*', Keyword.Pseudo, 'jes2_statement'),
+            # TODO: JES3 statement
+            (r'.*\n', Other)  # Input text or inline code in any language.
+        ],
+        'statement': [
+            (r'\s*\n', Whitespace, '#pop'),
+            (r'([a-z]\w*)(\s+)(exec|job)(\s*)',
+             bygroups(Name.Label, Whitespace, Keyword.Reserved, Whitespace),
+             'option'),
+            (r'[a-z]\w*', Name.Variable, 'statement_command'),
+            (r'\s+', Whitespace, 'statement_command'),
+        ],
+        'statement_command': [
+            (r'\s+(command|cntl|dd|endctl|endif|else|include|jcllib|'
+             r'output|pend|proc|set|then|xmit)\s+', Keyword.Reserved, 'option'),
+            include('option')
+        ],
+        'jes2_statement': [
+            (r'\s*\n', Whitespace, '#pop'),
+            (r'\$', Keyword, 'option'),
+            (r'\b(jobparam|message|netacct|notify|output|priority|route|'
+             r'setup|signoff|xeq|xmit)\b', Keyword, 'option'),
+        ],
+        'option': [
+            # (r'\n', Text, 'root'),
+            (r'\*', Name.Builtin),
+            (r'[\[\](){}<>;,]', Punctuation),
+            (r'[-+*/=&%]', Operator),
+            (r'[a-z_]\w*', Name),
+            (r'\d+\.\d*', Number.Float),
+            (r'\.\d+', Number.Float),
+            (r'\d+', Number.Integer),
+            (r"'", String, 'option_string'),
+            (r'[ \t]+', Whitespace, 'option_comment'),
+            (r'\.', Punctuation),
+        ],
+        'option_string': [
+            (r"(\n)(//)", bygroups(Text, Keyword.Pseudo)),
+            (r"''", String),
+            (r"[^']", String),
+            (r"'", String, '#pop'),
+        ],
+        'option_comment': [
+            # (r'\n', Text, 'root'),
+            (r'.+', Comment.Single),
+        ]
+    }
+
+    _JOB_HEADER_PATTERN = re.compile(r'^//[a-z#$@][a-z0-9#$@]{0,7}\s+job(\s+.*)?$',
+                                     re.IGNORECASE)
+
+    def analyse_text(text):
+        """
+        Recognize JCL job by header.
+        """
+        result = 0.0
+        lines = text.split('\n')
+        if len(lines) > 0:
+            if JclLexer._JOB_HEADER_PATTERN.match(lines[0]):
+                result = 1.0
+        assert 0.0 <= result <= 1.0
+        return result
+
+
+class MiniScriptLexer(RegexLexer):
+    """
+    For MiniScript source code.
+    """
+
+    name = 'MiniScript'
+    url = 'https://miniscript.org'
+    aliases = ['miniscript', 'ms']
+    filenames = ['*.ms']
+    mimetypes = ['text/x-minicript', 'application/x-miniscript']
+    version_added = '2.6'
+
+    tokens = {
+        'root': [
+            (r'#!(.*?)$', Comment.Preproc),
+            default('base'),
+        ],
+        'base': [
+            ('//.*$', Comment.Single),
+            (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number),
+            (r'(?i)\d+e[+-]?\d+', Number),
+            (r'\d+', Number),
+            (r'\n', Text),
+            (r'[^\S\n]+', Text),
+            (r'"', String, 'string_double'),
+            (r'(==|!=|<=|>=|[=+\-*/%^<>.:])', Operator),
+            (r'[;,\[\]{}()]', Punctuation),
+            (words((
+                'break', 'continue', 'else', 'end', 'for', 'function', 'if',
+                'in', 'isa', 'then', 'repeat', 'return', 'while'), suffix=r'\b'),
+             Keyword),
+            (words((
+                'abs', 'acos', 'asin', 'atan', 'ceil', 'char', 'cos', 'floor',
+                'log', 'round', 'rnd', 'pi', 'sign', 'sin', 'sqrt', 'str', 'tan',
+                'hasIndex', 'indexOf', 'len', 'val', 'code', 'remove', 'lower',
+                'upper', 'replace', 'split', 'indexes', 'values', 'join', 'sum',
+                'sort', 'shuffle', 'push', 'pop', 'pull', 'range',
+                'print', 'input', 'time', 'wait', 'locals', 'globals', 'outer',
+                'yield'), suffix=r'\b'),
+             Name.Builtin),
+            (r'(true|false|null)\b', Keyword.Constant),
+            (r'(and|or|not|new)\b', Operator.Word),
+            (r'(self|super|__isa)\b', Name.Builtin.Pseudo),
+            (r'[a-zA-Z_]\w*', Name.Variable)
+        ],
+        'string_double': [
+            (r'[^"\n]+', String),
+            (r'""', String),
+            (r'"', String, '#pop'),
+            (r'\n', Text, '#pop'),  # Stray linefeed also terminates strings.
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sgf.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sgf.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d7c30432cc39e99f3cacebde19db1550c814ac8
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sgf.py
@@ -0,0 +1,59 @@
+"""
+    pygments.lexers.sgf
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Smart Game Format (sgf) file format.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Name, Literal, String, Punctuation, Whitespace
+
+__all__ = ["SmartGameFormatLexer"]
+
+
+class SmartGameFormatLexer(RegexLexer):
+    """
+    Lexer for Smart Game Format (sgf) file format.
+
+    The format is used to store game records of board games for two players
+    (mainly Go game).
+    """
+    name = 'SmartGameFormat'
+    url = 'https://www.red-bean.com/sgf/'
+    aliases = ['sgf']
+    filenames = ['*.sgf']
+    version_added = '2.4'
+
+    tokens = {
+        'root': [
+            (r'[():;]+', Punctuation),
+            # tokens:
+            (r'(A[BW]|AE|AN|AP|AR|AS|[BW]L|BM|[BW]R|[BW]S|[BW]T|CA|CH|CP|CR|'
+             r'DD|DM|DO|DT|EL|EV|EX|FF|FG|G[BW]|GC|GM|GN|HA|HO|ID|IP|IT|IY|KM|'
+             r'KO|LB|LN|LT|L|MA|MN|M|N|OB|OM|ON|OP|OT|OV|P[BW]|PC|PL|PM|RE|RG|'
+             r'RO|RU|SO|SC|SE|SI|SL|SO|SQ|ST|SU|SZ|T[BW]|TC|TE|TM|TR|UC|US|VW|'
+             r'V|[BW]|C)',
+             Name.Builtin),
+            # number:
+            (r'(\[)([0-9.]+)(\])',
+             bygroups(Punctuation, Literal.Number, Punctuation)),
+            # date:
+            (r'(\[)([0-9]{4}-[0-9]{2}-[0-9]{2})(\])',
+             bygroups(Punctuation, Literal.Date, Punctuation)),
+            # point:
+            (r'(\[)([a-z]{2})(\])',
+             bygroups(Punctuation, String, Punctuation)),
+            # double points:
+            (r'(\[)([a-z]{2})(:)([a-z]{2})(\])',
+             bygroups(Punctuation, String, Punctuation, String, Punctuation)),
+
+            (r'(\[)([\w\s#()+,\-.:?]+)(\])',
+             bygroups(Punctuation, String, Punctuation)),
+            (r'(\[)(\s.*)(\])',
+             bygroups(Punctuation, Whitespace, Punctuation)),
+            (r'\s+', Whitespace)
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/shell.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/shell.py
new file mode 100644
index 0000000000000000000000000000000000000000..c1ce07fcc8aa5ac7aab453df30af21f94d859f3e
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/shell.py
@@ -0,0 +1,902 @@
+"""
+    pygments.lexers.shell
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for various shells.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, \
+    include, default, this, using, words, line_re
+from pygments.token import Punctuation, Whitespace, \
+    Text, Comment, Operator, Keyword, Name, String, Number, Generic
+from pygments.util import shebang_matches
+
+__all__ = ['BashLexer', 'BashSessionLexer', 'TcshLexer', 'BatchLexer',
+           'SlurmBashLexer', 'MSDOSSessionLexer', 'PowerShellLexer',
+           'PowerShellSessionLexer', 'TcshSessionLexer', 'FishShellLexer',
+           'ExeclineLexer']
+
+
+class BashLexer(RegexLexer):
+    """
+    Lexer for (ba|k|z|)sh shell scripts.
+    """
+
+    name = 'Bash'
+    aliases = ['bash', 'sh', 'ksh', 'zsh', 'shell', 'openrc']
+    filenames = ['*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass',
+                 '*.exheres-0', '*.exlib', '*.zsh',
+                 '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc',
+                 '.kshrc', 'kshrc',
+                 'PKGBUILD']
+    mimetypes = ['application/x-sh', 'application/x-shellscript', 'text/x-shellscript']
+    url = 'https://en.wikipedia.org/wiki/Unix_shell'
+    version_added = '0.6'
+
+    tokens = {
+        'root': [
+            include('basic'),
+            (r'`', String.Backtick, 'backticks'),
+            include('data'),
+            include('interp'),
+        ],
+        'interp': [
+            (r'\$\(\(', Keyword, 'math'),
+            (r'\$\(', Keyword, 'paren'),
+            (r'\$\{#?', String.Interpol, 'curly'),
+            (r'\$[a-zA-Z_]\w*', Name.Variable),  # user variable
+            (r'\$(?:\d+|[#$?!_*@-])', Name.Variable),      # builtin
+            (r'\$', Text),
+        ],
+        'basic': [
+            (r'\b(if|fi|else|while|in|do|done|for|then|return|function|case|'
+             r'select|break|continue|until|esac|elif)(\s*)\b',
+             bygroups(Keyword, Whitespace)),
+            (r'\b(alias|bg|bind|builtin|caller|cd|command|compgen|'
+             r'complete|declare|dirs|disown|echo|enable|eval|exec|exit|'
+             r'export|false|fc|fg|getopts|hash|help|history|jobs|kill|let|'
+             r'local|logout|popd|printf|pushd|pwd|read|readonly|set|shift|'
+             r'shopt|source|suspend|test|time|times|trap|true|type|typeset|'
+             r'ulimit|umask|unalias|unset|wait)(?=[\s)`])',
+             Name.Builtin),
+            (r'\A#!.+\n', Comment.Hashbang),
+            (r'#.*\n', Comment.Single),
+            (r'\\[\w\W]', String.Escape),
+            (r'(\b\w+)(\s*)(\+?=)', bygroups(Name.Variable, Whitespace, Operator)),
+            (r'[\[\]{}()=]', Operator),
+            (r'<<<', Operator),  # here-string
+            (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
+            (r'&&|\|\|', Operator),
+        ],
+        'data': [
+            (r'(?s)\$?"(\\.|[^"\\$])*"', String.Double),
+            (r'"', String.Double, 'string'),
+            (r"(?s)\$'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+            (r"(?s)'.*?'", String.Single),
+            (r';', Punctuation),
+            (r'&', Punctuation),
+            (r'\|', Punctuation),
+            (r'\s+', Whitespace),
+            (r'\d+\b', Number),
+            (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text),
+            (r'<', Text),
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (r'(?s)(\\\\|\\[0-7]+|\\.|[^"\\$])+', String.Double),
+            include('interp'),
+        ],
+        'curly': [
+            (r'\}', String.Interpol, '#pop'),
+            (r':-', Keyword),
+            (r'\w+', Name.Variable),
+            (r'[^}:"\'`$\\]+', Punctuation),
+            (r':', Punctuation),
+            include('root'),
+        ],
+        'paren': [
+            (r'\)', Keyword, '#pop'),
+            include('root'),
+        ],
+        'math': [
+            (r'\)\)', Keyword, '#pop'),
+            (r'\*\*|\|\||<<|>>|[-+*/%^|&<>]', Operator),
+            (r'\d+#[\da-zA-Z]+', Number),
+            (r'\d+#(?! )', Number),
+            (r'0[xX][\da-fA-F]+', Number),
+            (r'\d+', Number),
+            (r'[a-zA-Z_]\w*', Name.Variable),  # user variable
+            include('root'),
+        ],
+        'backticks': [
+            (r'`', String.Backtick, '#pop'),
+            include('root'),
+        ],
+    }
+
+    def analyse_text(text):
+        if shebang_matches(text, r'(ba|z|)sh'):
+            return 1
+        if text.startswith('$ '):
+            return 0.2
+
+
+class SlurmBashLexer(BashLexer):
+    """
+    Lexer for (ba|k|z|)sh Slurm scripts.
+    """
+
+    name = 'Slurm'
+    aliases = ['slurm', 'sbatch']
+    filenames = ['*.sl']
+    mimetypes = []
+    version_added = '2.4'
+    EXTRA_KEYWORDS = {'srun'}
+
+    def get_tokens_unprocessed(self, text):
+        for index, token, value in BashLexer.get_tokens_unprocessed(self, text):
+            if token is Text and value in self.EXTRA_KEYWORDS:
+                yield index, Name.Builtin, value
+            elif token is Comment.Single and 'SBATCH' in value:
+                yield index, Keyword.Pseudo, value
+            else:
+                yield index, token, value
+
+
+class ShellSessionBaseLexer(Lexer):
+    """
+    Base lexer for shell sessions.
+
+    .. versionadded:: 2.1
+    """
+
+    _bare_continuation = False
+    _venv = re.compile(r'^(\([^)]*\))(\s*)')
+
+    def get_tokens_unprocessed(self, text):
+        innerlexer = self._innerLexerCls(**self.options)
+
+        pos = 0
+        curcode = ''
+        insertions = []
+        backslash_continuation = False
+
+        for match in line_re.finditer(text):
+            line = match.group()
+
+            venv_match = self._venv.match(line)
+            if venv_match:
+                venv = venv_match.group(1)
+                venv_whitespace = venv_match.group(2)
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt.VirtualEnv, venv)]))
+                if venv_whitespace:
+                    insertions.append((len(curcode),
+                                       [(0, Text, venv_whitespace)]))
+                line = line[venv_match.end():]
+
+            m = self._ps1rgx.match(line)
+            if m:
+                # To support output lexers (say diff output), the output
+                # needs to be broken by prompts whenever the output lexer
+                # changes.
+                if not insertions:
+                    pos = match.start()
+
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt, m.group(1))]))
+                curcode += m.group(2)
+                backslash_continuation = curcode.endswith('\\\n')
+            elif backslash_continuation:
+                if line.startswith(self._ps2):
+                    insertions.append((len(curcode),
+                                       [(0, Generic.Prompt,
+                                         line[:len(self._ps2)])]))
+                    curcode += line[len(self._ps2):]
+                else:
+                    curcode += line
+                backslash_continuation = curcode.endswith('\\\n')
+            elif self._bare_continuation and line.startswith(self._ps2):
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt,
+                                     line[:len(self._ps2)])]))
+                curcode += line[len(self._ps2):]
+            else:
+                if insertions:
+                    toks = innerlexer.get_tokens_unprocessed(curcode)
+                    for i, t, v in do_insertions(insertions, toks):
+                        yield pos+i, t, v
+                yield match.start(), Generic.Output, line
+                insertions = []
+                curcode = ''
+        if insertions:
+            for i, t, v in do_insertions(insertions,
+                                         innerlexer.get_tokens_unprocessed(curcode)):
+                yield pos+i, t, v
+
+
+class BashSessionLexer(ShellSessionBaseLexer):
+    """
+    Lexer for Bash shell sessions, i.e. command lines, including a
+    prompt, interspersed with output.
+    """
+
+    name = 'Bash Session'
+    aliases = ['console', 'shell-session']
+    filenames = ['*.sh-session', '*.shell-session']
+    mimetypes = ['application/x-shell-session', 'application/x-sh-session']
+    url = 'https://en.wikipedia.org/wiki/Unix_shell'
+    version_added = '1.1'
+    _example = "console/example.sh-session"
+
+    _innerLexerCls = BashLexer
+    _ps1rgx = re.compile(
+        r'^((?:(?:\[.*?\])|(?:\(\S+\))?(?:| |sh\S*?|\w+\S+[@:]\S+(?:\s+\S+)' \
+        r'?|\[\S+[@:][^\n]+\].+))\s*[$#%]\s*)(.*\n?)')
+    _ps2 = '> '
+
+
+class BatchLexer(RegexLexer):
+    """
+    Lexer for the DOS/Windows Batch file format.
+    """
+    name = 'Batchfile'
+    aliases = ['batch', 'bat', 'dosbatch', 'winbatch']
+    filenames = ['*.bat', '*.cmd']
+    mimetypes = ['application/x-dos-batch']
+    url = 'https://en.wikipedia.org/wiki/Batch_file'
+    version_added = '0.7'
+
+    flags = re.MULTILINE | re.IGNORECASE
+
+    _nl = r'\n\x1a'
+    _punct = r'&<>|'
+    _ws = r'\t\v\f\r ,;=\xa0'
+    _nlws = r'\s\x1a\xa0,;='
+    _space = rf'(?:(?:(?:\^[{_nl}])?[{_ws}])+)'
+    _keyword_terminator = (rf'(?=(?:\^[{_nl}]?)?[{_ws}+./:[\\\]]|[{_nl}{_punct}(])')
+    _token_terminator = rf'(?=\^?[{_ws}]|[{_punct}{_nl}])'
+    _start_label = rf'((?:(?<=^[^:])|^[^:]?)[{_ws}]*)(:)'
+    _label = rf'(?:(?:[^{_nlws}{_punct}+:^]|\^[{_nl}]?[\w\W])*)'
+    _label_compound = rf'(?:(?:[^{_nlws}{_punct}+:^)]|\^[{_nl}]?[^)])*)'
+    _number = rf'(?:-?(?:0[0-7]+|0x[\da-f]+|\d+){_token_terminator})'
+    _opword = r'(?:equ|geq|gtr|leq|lss|neq)'
+    _string = rf'(?:"[^{_nl}"]*(?:"|(?=[{_nl}])))'
+    _variable = (r'(?:(?:%(?:\*|(?:~[a-z]*(?:\$[^:]+:)?)?\d|'
+                 rf'[^%:{_nl}]+(?::(?:~(?:-?\d+)?(?:,(?:-?\d+)?)?|(?:[^%{_nl}^]|'
+                 rf'\^[^%{_nl}])[^={_nl}]*=(?:[^%{_nl}^]|\^[^%{_nl}])*)?)?%))|'
+                 rf'(?:\^?![^!:{_nl}]+(?::(?:~(?:-?\d+)?(?:,(?:-?\d+)?)?|(?:'
+                 rf'[^!{_nl}^]|\^[^!{_nl}])[^={_nl}]*=(?:[^!{_nl}^]|\^[^!{_nl}])*)?)?\^?!))')
+    _core_token = rf'(?:(?:(?:\^[{_nl}]?)?[^"{_nlws}{_punct}])+)'
+    _core_token_compound = rf'(?:(?:(?:\^[{_nl}]?)?[^"{_nlws}{_punct})])+)'
+    _token = rf'(?:[{_punct}]+|{_core_token})'
+    _token_compound = rf'(?:[{_punct}]+|{_core_token_compound})'
+    _stoken = (rf'(?:[{_punct}]+|(?:{_string}|{_variable}|{_core_token})+)')
+
+    def _make_begin_state(compound, _core_token=_core_token,
+                          _core_token_compound=_core_token_compound,
+                          _keyword_terminator=_keyword_terminator,
+                          _nl=_nl, _punct=_punct, _string=_string,
+                          _space=_space, _start_label=_start_label,
+                          _stoken=_stoken, _token_terminator=_token_terminator,
+                          _variable=_variable, _ws=_ws):
+        rest = '(?:{}|{}|[^"%{}{}{}])*'.format(_string, _variable, _nl, _punct,
+                                            ')' if compound else '')
+        rest_of_line = rf'(?:(?:[^{_nl}^]|\^[{_nl}]?[\w\W])*)'
+        rest_of_line_compound = rf'(?:(?:[^{_nl}^)]|\^[{_nl}]?[^)])*)'
+        set_space = rf'((?:(?:\^[{_nl}]?)?[^\S\n])*)'
+        suffix = ''
+        if compound:
+            _keyword_terminator = rf'(?:(?=\))|{_keyword_terminator})'
+            _token_terminator = rf'(?:(?=\))|{_token_terminator})'
+            suffix = '/compound'
+        return [
+            ((r'\)', Punctuation, '#pop') if compound else
+             (rf'\)((?=\()|{_token_terminator}){rest_of_line}',
+              Comment.Single)),
+            (rf'(?={_start_label})', Text, f'follow{suffix}'),
+            (_space, using(this, state='text')),
+            include(f'redirect{suffix}'),
+            (rf'[{_nl}]+', Text),
+            (r'\(', Punctuation, 'root/compound'),
+            (r'@+', Punctuation),
+            (rf'((?:for|if|rem)(?:(?=(?:\^[{_nl}]?)?/)|(?:(?!\^)|'
+             rf'(?<=m))(?:(?=\()|{_token_terminator})))({_space}?{_core_token_compound if compound else _core_token}?(?:\^[{_nl}]?)?/(?:\^[{_nl}]?)?\?)',
+             bygroups(Keyword, using(this, state='text')),
+             f'follow{suffix}'),
+            (rf'(goto{_keyword_terminator})({rest}(?:\^[{_nl}]?)?/(?:\^[{_nl}]?)?\?{rest})',
+             bygroups(Keyword, using(this, state='text')),
+             f'follow{suffix}'),
+            (words(('assoc', 'break', 'cd', 'chdir', 'cls', 'color', 'copy',
+                    'date', 'del', 'dir', 'dpath', 'echo', 'endlocal', 'erase',
+                    'exit', 'ftype', 'keys', 'md', 'mkdir', 'mklink', 'move',
+                    'path', 'pause', 'popd', 'prompt', 'pushd', 'rd', 'ren',
+                    'rename', 'rmdir', 'setlocal', 'shift', 'start', 'time',
+                    'title', 'type', 'ver', 'verify', 'vol'),
+                   suffix=_keyword_terminator), Keyword, f'follow{suffix}'),
+            (rf'(call)({_space}?)(:)',
+             bygroups(Keyword, using(this, state='text'), Punctuation),
+             f'call{suffix}'),
+            (rf'call{_keyword_terminator}', Keyword),
+            (rf'(for{_token_terminator}(?!\^))({_space})(/f{_token_terminator})',
+             bygroups(Keyword, using(this, state='text'), Keyword),
+             ('for/f', 'for')),
+            (rf'(for{_token_terminator}(?!\^))({_space})(/l{_token_terminator})',
+             bygroups(Keyword, using(this, state='text'), Keyword),
+             ('for/l', 'for')),
+            (rf'for{_token_terminator}(?!\^)', Keyword, ('for2', 'for')),
+            (rf'(goto{_keyword_terminator})({_space}?)(:?)',
+             bygroups(Keyword, using(this, state='text'), Punctuation),
+             f'label{suffix}'),
+            (rf'(if(?:(?=\()|{_token_terminator})(?!\^))({_space}?)((?:/i{_token_terminator})?)({_space}?)((?:not{_token_terminator})?)({_space}?)',
+             bygroups(Keyword, using(this, state='text'), Keyword,
+                      using(this, state='text'), Keyword,
+                      using(this, state='text')), ('(?', 'if')),
+            (rf'rem(((?=\()|{_token_terminator}){_space}?{_stoken}?.*|{_keyword_terminator}{rest_of_line_compound if compound else rest_of_line})',
+             Comment.Single, f'follow{suffix}'),
+            (rf'(set{_keyword_terminator}){set_space}(/a)',
+             bygroups(Keyword, using(this, state='text'), Keyword),
+             f'arithmetic{suffix}'),
+            (r'(set{}){}((?:/p)?){}((?:(?:(?:\^[{}]?)?[^"{}{}^={}]|'
+             r'\^[{}]?[^"=])+)?)((?:(?:\^[{}]?)?=)?)'.format(_keyword_terminator, set_space, set_space, _nl, _nl, _punct,
+              ')' if compound else '', _nl, _nl),
+             bygroups(Keyword, using(this, state='text'), Keyword,
+                      using(this, state='text'), using(this, state='variable'),
+                      Punctuation),
+             f'follow{suffix}'),
+            default(f'follow{suffix}')
+        ]
+
+    def _make_follow_state(compound, _label=_label,
+                           _label_compound=_label_compound, _nl=_nl,
+                           _space=_space, _start_label=_start_label,
+                           _token=_token, _token_compound=_token_compound,
+                           _ws=_ws):
+        suffix = '/compound' if compound else ''
+        state = []
+        if compound:
+            state.append((r'(?=\))', Text, '#pop'))
+        state += [
+            (rf'{_start_label}([{_ws}]*)({_label_compound if compound else _label})(.*)',
+             bygroups(Text, Punctuation, Text, Name.Label, Comment.Single)),
+            include(f'redirect{suffix}'),
+            (rf'(?=[{_nl}])', Text, '#pop'),
+            (r'\|\|?|&&?', Punctuation, '#pop'),
+            include('text')
+        ]
+        return state
+
+    def _make_arithmetic_state(compound, _nl=_nl, _punct=_punct,
+                               _string=_string, _variable=_variable,
+                               _ws=_ws, _nlws=_nlws):
+        op = r'=+\-*/!~'
+        state = []
+        if compound:
+            state.append((r'(?=\))', Text, '#pop'))
+        state += [
+            (r'0[0-7]+', Number.Oct),
+            (r'0x[\da-f]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'[(),]+', Punctuation),
+            (rf'([{op}]|%|\^\^)+', Operator),
+            (r'({}|{}|(\^[{}]?)?[^(){}%\^"{}{}]|\^[{}]?{})+'.format(_string, _variable, _nl, op, _nlws, _punct, _nlws,
+              r'[^)]' if compound else r'[\w\W]'),
+             using(this, state='variable')),
+            (r'(?=[\x00|&])', Text, '#pop'),
+            include('follow')
+        ]
+        return state
+
+    def _make_call_state(compound, _label=_label,
+                         _label_compound=_label_compound):
+        state = []
+        if compound:
+            state.append((r'(?=\))', Text, '#pop'))
+        state.append((r'(:?)(%s)' % (_label_compound if compound else _label),
+                      bygroups(Punctuation, Name.Label), '#pop'))
+        return state
+
+    def _make_label_state(compound, _label=_label,
+                          _label_compound=_label_compound, _nl=_nl,
+                          _punct=_punct, _string=_string, _variable=_variable):
+        state = []
+        if compound:
+            state.append((r'(?=\))', Text, '#pop'))
+        state.append((r'({}?)((?:{}|{}|\^[{}]?{}|[^"%^{}{}{}])*)'.format(_label_compound if compound else _label, _string,
+                       _variable, _nl, r'[^)]' if compound else r'[\w\W]', _nl,
+                       _punct, r')' if compound else ''),
+                      bygroups(Name.Label, Comment.Single), '#pop'))
+        return state
+
+    def _make_redirect_state(compound,
+                             _core_token_compound=_core_token_compound,
+                             _nl=_nl, _punct=_punct, _stoken=_stoken,
+                             _string=_string, _space=_space,
+                             _variable=_variable, _nlws=_nlws):
+        stoken_compound = (rf'(?:[{_punct}]+|(?:{_string}|{_variable}|{_core_token_compound})+)')
+        return [
+            (rf'((?:(?<=[{_nlws}])\d)?)(>>?&|<&)([{_nlws}]*)(\d)',
+             bygroups(Number.Integer, Punctuation, Text, Number.Integer)),
+            (rf'((?:(?<=[{_nlws}])(?>?|<)({_space}?{stoken_compound if compound else _stoken})',
+             bygroups(Number.Integer, Punctuation, using(this, state='text')))
+        ]
+
+    tokens = {
+        'root': _make_begin_state(False),
+        'follow': _make_follow_state(False),
+        'arithmetic': _make_arithmetic_state(False),
+        'call': _make_call_state(False),
+        'label': _make_label_state(False),
+        'redirect': _make_redirect_state(False),
+        'root/compound': _make_begin_state(True),
+        'follow/compound': _make_follow_state(True),
+        'arithmetic/compound': _make_arithmetic_state(True),
+        'call/compound': _make_call_state(True),
+        'label/compound': _make_label_state(True),
+        'redirect/compound': _make_redirect_state(True),
+        'variable-or-escape': [
+            (_variable, Name.Variable),
+            (rf'%%|\^[{_nl}]?(\^!|[\w\W])', String.Escape)
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (_variable, Name.Variable),
+            (r'\^!|%%', String.Escape),
+            (rf'[^"%^{_nl}]+|[%^]', String.Double),
+            default('#pop')
+        ],
+        'sqstring': [
+            include('variable-or-escape'),
+            (r'[^%]+|%', String.Single)
+        ],
+        'bqstring': [
+            include('variable-or-escape'),
+            (r'[^%]+|%', String.Backtick)
+        ],
+        'text': [
+            (r'"', String.Double, 'string'),
+            include('variable-or-escape'),
+            (rf'[^"%^{_nlws}{_punct}\d)]+|.', Text)
+        ],
+        'variable': [
+            (r'"', String.Double, 'string'),
+            include('variable-or-escape'),
+            (rf'[^"%^{_nl}]+|.', Name.Variable)
+        ],
+        'for': [
+            (rf'({_space})(in)({_space})(\()',
+             bygroups(using(this, state='text'), Keyword,
+                      using(this, state='text'), Punctuation), '#pop'),
+            include('follow')
+        ],
+        'for2': [
+            (r'\)', Punctuation),
+            (rf'({_space})(do{_token_terminator})',
+             bygroups(using(this, state='text'), Keyword), '#pop'),
+            (rf'[{_nl}]+', Text),
+            include('follow')
+        ],
+        'for/f': [
+            (rf'(")((?:{_variable}|[^"])*?")([{_nlws}]*)(\))',
+             bygroups(String.Double, using(this, state='string'), Text,
+                      Punctuation)),
+            (r'"', String.Double, ('#pop', 'for2', 'string')),
+            (rf"('(?:%%|{_variable}|[\w\W])*?')([{_nlws}]*)(\))",
+             bygroups(using(this, state='sqstring'), Text, Punctuation)),
+            (rf'(`(?:%%|{_variable}|[\w\W])*?`)([{_nlws}]*)(\))',
+             bygroups(using(this, state='bqstring'), Text, Punctuation)),
+            include('for2')
+        ],
+        'for/l': [
+            (r'-?\d+', Number.Integer),
+            include('for2')
+        ],
+        'if': [
+            (rf'((?:cmdextversion|errorlevel){_token_terminator})({_space})(\d+)',
+             bygroups(Keyword, using(this, state='text'),
+                      Number.Integer), '#pop'),
+            (rf'(defined{_token_terminator})({_space})({_stoken})',
+             bygroups(Keyword, using(this, state='text'),
+                      using(this, state='variable')), '#pop'),
+            (rf'(exist{_token_terminator})({_space}{_stoken})',
+             bygroups(Keyword, using(this, state='text')), '#pop'),
+            (rf'({_number}{_space})({_opword})({_space}{_number})',
+             bygroups(using(this, state='arithmetic'), Operator.Word,
+                      using(this, state='arithmetic')), '#pop'),
+            (_stoken, using(this, state='text'), ('#pop', 'if2')),
+        ],
+        'if2': [
+            (rf'({_space}?)(==)({_space}?{_stoken})',
+             bygroups(using(this, state='text'), Operator,
+                      using(this, state='text')), '#pop'),
+            (rf'({_space})({_opword})({_space}{_stoken})',
+             bygroups(using(this, state='text'), Operator.Word,
+                      using(this, state='text')), '#pop')
+        ],
+        '(?': [
+            (_space, using(this, state='text')),
+            (r'\(', Punctuation, ('#pop', 'else?', 'root/compound')),
+            default('#pop')
+        ],
+        'else?': [
+            (_space, using(this, state='text')),
+            (rf'else{_token_terminator}', Keyword, '#pop'),
+            default('#pop')
+        ]
+    }
+
+
+class MSDOSSessionLexer(ShellSessionBaseLexer):
+    """
+    Lexer for MS DOS shell sessions, i.e. command lines, including a
+    prompt, interspersed with output.
+    """
+
+    name = 'MSDOS Session'
+    aliases = ['doscon']
+    filenames = []
+    mimetypes = []
+    url = 'https://en.wikipedia.org/wiki/MS-DOS'
+    version_added = '2.1'
+    _example = "doscon/session"
+
+    _innerLexerCls = BatchLexer
+    _ps1rgx = re.compile(r'^([^>]*>)(.*\n?)')
+    _ps2 = 'More? '
+
+
+class TcshLexer(RegexLexer):
+    """
+    Lexer for tcsh scripts.
+    """
+
+    name = 'Tcsh'
+    aliases = ['tcsh', 'csh']
+    filenames = ['*.tcsh', '*.csh']
+    mimetypes = ['application/x-csh']
+    url = 'https://www.tcsh.org'
+    version_added = '0.10'
+
+    tokens = {
+        'root': [
+            include('basic'),
+            (r'\$\(', Keyword, 'paren'),
+            (r'\$\{#?', Keyword, 'curly'),
+            (r'`', String.Backtick, 'backticks'),
+            include('data'),
+        ],
+        'basic': [
+            (r'\b(if|endif|else|while|then|foreach|case|default|'
+             r'break|continue|goto|breaksw|end|switch|endsw)\s*\b',
+             Keyword),
+            (r'\b(alias|alloc|bg|bindkey|builtins|bye|caller|cd|chdir|'
+             r'complete|dirs|echo|echotc|eval|exec|exit|fg|filetest|getxvers|'
+             r'glob|getspath|hashstat|history|hup|inlib|jobs|kill|'
+             r'limit|log|login|logout|ls-F|migrate|newgrp|nice|nohup|notify|'
+             r'onintr|popd|printenv|pushd|rehash|repeat|rootnode|popd|pushd|'
+             r'set|shift|sched|setenv|setpath|settc|setty|setxvers|shift|'
+             r'source|stop|suspend|source|suspend|telltc|time|'
+             r'umask|unalias|uncomplete|unhash|universe|unlimit|unset|unsetenv|'
+             r'ver|wait|warp|watchlog|where|which)\s*\b',
+             Name.Builtin),
+            (r'#.*', Comment),
+            (r'\\[\w\W]', String.Escape),
+            (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Text, Operator)),
+            (r'[\[\]{}()=]+', Operator),
+            (r'<<\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
+            (r';', Punctuation),
+        ],
+        'data': [
+            (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double),
+            (r"(?s)'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+            (r'\s+', Text),
+            (r'[^=\s\[\]{}()$"\'`\\;#]+', Text),
+            (r'\d+(?= |\Z)', Number),
+            (r'\$#?(\w+|.)', Name.Variable),
+        ],
+        'curly': [
+            (r'\}', Keyword, '#pop'),
+            (r':-', Keyword),
+            (r'\w+', Name.Variable),
+            (r'[^}:"\'`$]+', Punctuation),
+            (r':', Punctuation),
+            include('root'),
+        ],
+        'paren': [
+            (r'\)', Keyword, '#pop'),
+            include('root'),
+        ],
+        'backticks': [
+            (r'`', String.Backtick, '#pop'),
+            include('root'),
+        ],
+    }
+
+
+class TcshSessionLexer(ShellSessionBaseLexer):
+    """
+    Lexer for Tcsh sessions, i.e. command lines, including a
+    prompt, interspersed with output.
+    """
+
+    name = 'Tcsh Session'
+    aliases = ['tcshcon']
+    filenames = []
+    mimetypes = []
+    url = 'https://www.tcsh.org'
+    version_added = '2.1'
+    _example = "tcshcon/session"
+
+    _innerLexerCls = TcshLexer
+    _ps1rgx = re.compile(r'^([^>]+>)(.*\n?)')
+    _ps2 = '? '
+
+
+class PowerShellLexer(RegexLexer):
+    """
+    For Windows PowerShell code.
+    """
+    name = 'PowerShell'
+    aliases = ['powershell', 'pwsh', 'posh', 'ps1', 'psm1']
+    filenames = ['*.ps1', '*.psm1']
+    mimetypes = ['text/x-powershell']
+    url = 'https://learn.microsoft.com/en-us/powershell'
+    version_added = '1.5'
+
+    flags = re.DOTALL | re.IGNORECASE | re.MULTILINE
+
+    keywords = (
+        'while validateset validaterange validatepattern validatelength '
+        'validatecount until trap switch return ref process param parameter in '
+        'if global: local: function foreach for finally filter end elseif else '
+        'dynamicparam do default continue cmdletbinding break begin alias \\? '
+        '% #script #private #local #global mandatory parametersetname position '
+        'valuefrompipeline valuefrompipelinebypropertyname '
+        'valuefromremainingarguments helpmessage try catch throw').split()
+
+    operators = (
+        'and as band bnot bor bxor casesensitive ccontains ceq cge cgt cle '
+        'clike clt cmatch cne cnotcontains cnotlike cnotmatch contains '
+        'creplace eq exact f file ge gt icontains ieq ige igt ile ilike ilt '
+        'imatch ine inotcontains inotlike inotmatch ireplace is isnot le like '
+        'lt match ne not notcontains notlike notmatch or regex replace '
+        'wildcard').split()
+
+    verbs = (
+        'write where watch wait use update unregister unpublish unprotect '
+        'unlock uninstall undo unblock trace test tee take sync switch '
+        'suspend submit stop step start split sort skip show set send select '
+        'search scroll save revoke resume restore restart resolve resize '
+        'reset request repair rename remove register redo receive read push '
+        'publish protect pop ping out optimize open new move mount merge '
+        'measure lock limit join invoke install initialize import hide group '
+        'grant get format foreach find export expand exit enter enable edit '
+        'dismount disconnect disable deny debug cxnew copy convertto '
+        'convertfrom convert connect confirm compress complete compare close '
+        'clear checkpoint block backup assert approve aggregate add').split()
+
+    aliases_ = (
+        'ac asnp cat cd cfs chdir clc clear clhy cli clp cls clv cnsn '
+        'compare copy cp cpi cpp curl cvpa dbp del diff dir dnsn ebp echo epal '
+        'epcsv epsn erase etsn exsn fc fhx fl foreach ft fw gal gbp gc gci gcm '
+        'gcs gdr ghy gi gjb gl gm gmo gp gps gpv group gsn gsnp gsv gu gv gwmi '
+        'h history icm iex ihy ii ipal ipcsv ipmo ipsn irm ise iwmi iwr kill lp '
+        'ls man md measure mi mount move mp mv nal ndr ni nmo npssc nsn nv ogv '
+        'oh popd ps pushd pwd r rbp rcjb rcsn rd rdr ren ri rjb rm rmdir rmo '
+        'rni rnp rp rsn rsnp rujb rv rvpa rwmi sajb sal saps sasv sbp sc select '
+        'set shcm si sl sleep sls sort sp spjb spps spsv start sujb sv swmi tee '
+        'trcm type wget where wjb write').split()
+
+    commenthelp = (
+        'component description example externalhelp forwardhelpcategory '
+        'forwardhelptargetname functionality inputs link '
+        'notes outputs parameter remotehelprunspace role synopsis').split()
+
+    tokens = {
+        'root': [
+            # we need to count pairs of parentheses for correct highlight
+            # of '$(...)' blocks in strings
+            (r'\(', Punctuation, 'child'),
+            (r'\s+', Text),
+            (r'^(\s*#[#\s]*)(\.(?:{}))([^\n]*$)'.format('|'.join(commenthelp)),
+             bygroups(Comment, String.Doc, Comment)),
+            (r'#[^\n]*?$', Comment),
+            (r'(<|<)#', Comment.Multiline, 'multline'),
+            (r'@"\n', String.Heredoc, 'heredoc-double'),
+            (r"@'\n.*?\n'@", String.Heredoc),
+            # escaped syntax
+            (r'`[\'"$@-]', Punctuation),
+            (r'"', String.Double, 'string'),
+            (r"'([^']|'')*'", String.Single),
+            (r'(\$|@@|@)((global|script|private|env):)?\w+',
+             Name.Variable),
+            (r'({})\b'.format('|'.join(keywords)), Keyword),
+            (r'-({})\b'.format('|'.join(operators)), Operator),
+            (r'({})-[a-z_]\w*\b'.format('|'.join(verbs)), Name.Builtin),
+            (r'({})\s'.format('|'.join(aliases_)), Name.Builtin),
+            (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant),  # .net [type]s
+            (r'-[a-z_]\w*', Name),
+            (r'\w+', Name),
+            (r'[.,;:@{}\[\]$()=+*/\\&%!~?^`|<>-]', Punctuation),
+        ],
+        'child': [
+            (r'\)', Punctuation, '#pop'),
+            include('root'),
+        ],
+        'multline': [
+            (r'[^#&.]+', Comment.Multiline),
+            (r'#(>|>)', Comment.Multiline, '#pop'),
+            (r'\.({})'.format('|'.join(commenthelp)), String.Doc),
+            (r'[#&.]', Comment.Multiline),
+        ],
+        'string': [
+            (r"`[0abfnrtv'\"$`]", String.Escape),
+            (r'[^$`"]+', String.Double),
+            (r'\$\(', Punctuation, 'child'),
+            (r'""', String.Double),
+            (r'[`$]', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'heredoc-double': [
+            (r'\n"@', String.Heredoc, '#pop'),
+            (r'\$\(', Punctuation, 'child'),
+            (r'[^@\n]+"]', String.Heredoc),
+            (r".", String.Heredoc),
+        ]
+    }
+
+
+class PowerShellSessionLexer(ShellSessionBaseLexer):
+    """
+    Lexer for PowerShell sessions, i.e. command lines, including a
+    prompt, interspersed with output.
+    """
+
+    name = 'PowerShell Session'
+    aliases = ['pwsh-session', 'ps1con']
+    filenames = []
+    mimetypes = []
+    url = 'https://learn.microsoft.com/en-us/powershell'
+    version_added = '2.1'
+    _example = "pwsh-session/session"
+
+    _innerLexerCls = PowerShellLexer
+    _bare_continuation = True
+    _ps1rgx = re.compile(r'^((?:\[[^]]+\]: )?PS[^>]*> ?)(.*\n?)')
+    _ps2 = '> '
+
+
+class FishShellLexer(RegexLexer):
+    """
+    Lexer for Fish shell scripts.
+    """
+
+    name = 'Fish'
+    aliases = ['fish', 'fishshell']
+    filenames = ['*.fish', '*.load']
+    mimetypes = ['application/x-fish']
+    url = 'https://fishshell.com'
+    version_added = '2.1'
+
+    tokens = {
+        'root': [
+            include('basic'),
+            include('data'),
+            include('interp'),
+        ],
+        'interp': [
+            (r'\$\(\(', Keyword, 'math'),
+            (r'\(', Keyword, 'paren'),
+            (r'\$#?(\w+|.)', Name.Variable),
+        ],
+        'basic': [
+            (r'\b(begin|end|if|else|while|break|for|in|return|function|block|'
+             r'case|continue|switch|not|and|or|set|echo|exit|pwd|true|false|'
+             r'cd|count|test)(\s*)\b',
+             bygroups(Keyword, Text)),
+            (r'\b(alias|bg|bind|breakpoint|builtin|command|commandline|'
+             r'complete|contains|dirh|dirs|emit|eval|exec|fg|fish|fish_config|'
+             r'fish_indent|fish_pager|fish_prompt|fish_right_prompt|'
+             r'fish_update_completions|fishd|funced|funcsave|functions|help|'
+             r'history|isatty|jobs|math|mimedb|nextd|open|popd|prevd|psub|'
+             r'pushd|random|read|set_color|source|status|trap|type|ulimit|'
+             r'umask|vared|fc|getopts|hash|kill|printf|time|wait)\s*\b(?!\.)',
+             Name.Builtin),
+            (r'#.*\n', Comment),
+            (r'\\[\w\W]', String.Escape),
+            (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Whitespace, Operator)),
+            (r'[\[\]()=]', Operator),
+            (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
+        ],
+        'data': [
+            (r'(?s)\$?"(\\\\|\\[0-7]+|\\.|[^"\\$])*"', String.Double),
+            (r'"', String.Double, 'string'),
+            (r"(?s)\$'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
+            (r"(?s)'.*?'", String.Single),
+            (r';', Punctuation),
+            (r'&|\||\^|<|>', Operator),
+            (r'\s+', Text),
+            (r'\d+(?= |\Z)', Number),
+            (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text),
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (r'(?s)(\\\\|\\[0-7]+|\\.|[^"\\$])+', String.Double),
+            include('interp'),
+        ],
+        'paren': [
+            (r'\)', Keyword, '#pop'),
+            include('root'),
+        ],
+        'math': [
+            (r'\)\)', Keyword, '#pop'),
+            (r'[-+*/%^|&]|\*\*|\|\|', Operator),
+            (r'\d+#\d+', Number),
+            (r'\d+#(?! )', Number),
+            (r'\d+', Number),
+            include('root'),
+        ],
+    }
+
+class ExeclineLexer(RegexLexer):
+    """
+    Lexer for Laurent Bercot's execline language.
+    """
+
+    name = 'execline'
+    aliases = ['execline']
+    filenames = ['*.exec']
+    url = 'https://skarnet.org/software/execline'
+    version_added = '2.7'
+
+    tokens = {
+        'root': [
+            include('basic'),
+            include('data'),
+            include('interp')
+        ],
+        'interp': [
+            (r'\$\{', String.Interpol, 'curly'),
+            (r'\$[\w@#]+', Name.Variable),  # user variable
+            (r'\$', Text),
+        ],
+        'basic': [
+            (r'\b(background|backtick|cd|define|dollarat|elgetopt|'
+             r'elgetpositionals|elglob|emptyenv|envfile|exec|execlineb|'
+             r'exit|export|fdblock|fdclose|fdmove|fdreserve|fdswap|'
+             r'forbacktickx|foreground|forstdin|forx|getcwd|getpid|heredoc|'
+             r'homeof|if|ifelse|ifte|ifthenelse|importas|loopwhilex|'
+             r'multidefine|multisubstitute|pipeline|piperw|posix-cd|'
+             r'redirfd|runblock|shift|trap|tryexec|umask|unexport|wait|'
+             r'withstdinas)\b', Name.Builtin),
+            (r'\A#!.+\n', Comment.Hashbang),
+            (r'#.*\n', Comment.Single),
+            (r'[{}]', Operator)
+        ],
+        'data': [
+            (r'(?s)"(\\.|[^"\\$])*"', String.Double),
+            (r'"', String.Double, 'string'),
+            (r'\s+', Text),
+            (r'[^\s{}$"\\]+', Text)
+        ],
+        'string': [
+            (r'"', String.Double, '#pop'),
+            (r'(?s)(\\\\|\\.|[^"\\$])+', String.Double),
+            include('interp'),
+        ],
+        'curly': [
+            (r'\}', String.Interpol, '#pop'),
+            (r'[\w#@]+', Name.Variable),
+            include('root')
+        ]
+
+    }
+
+    def analyse_text(text):
+        if shebang_matches(text, r'execlineb'):
+            return 1
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sieve.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sieve.py
new file mode 100644
index 0000000000000000000000000000000000000000..4bda3517e83e2ba20c57b165b237f845796d6727
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sieve.py
@@ -0,0 +1,78 @@
+"""
+    pygments.lexers.sieve
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Sieve file format.
+
+    https://tools.ietf.org/html/rfc5228
+    https://tools.ietf.org/html/rfc5173
+    https://tools.ietf.org/html/rfc5229
+    https://tools.ietf.org/html/rfc5230
+    https://tools.ietf.org/html/rfc5232
+    https://tools.ietf.org/html/rfc5235
+    https://tools.ietf.org/html/rfc5429
+    https://tools.ietf.org/html/rfc8580
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Comment, Name, Literal, String, Text, Punctuation, \
+    Keyword
+
+__all__ = ["SieveLexer"]
+
+
+class SieveLexer(RegexLexer):
+    """
+    Lexer for sieve format.
+    """
+    name = 'Sieve'
+    filenames = ['*.siv', '*.sieve']
+    aliases = ['sieve']
+    url = 'https://en.wikipedia.org/wiki/Sieve_(mail_filtering_language)'
+    version_added = '2.6'
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'[();,{}\[\]]', Punctuation),
+            # import:
+            (r'(?i)require',
+             Keyword.Namespace),
+            # tags:
+            (r'(?i)(:)(addresses|all|contains|content|create|copy|comparator|'
+             r'count|days|detail|domain|fcc|flags|from|handle|importance|is|'
+             r'localpart|length|lowerfirst|lower|matches|message|mime|options|'
+             r'over|percent|quotewildcard|raw|regex|specialuse|subject|text|'
+             r'under|upperfirst|upper|value)',
+             bygroups(Name.Tag, Name.Tag)),
+            # tokens:
+            (r'(?i)(address|addflag|allof|anyof|body|discard|elsif|else|envelope|'
+             r'ereject|exists|false|fileinto|if|hasflag|header|keep|'
+             r'notify_method_capability|notify|not|redirect|reject|removeflag|'
+             r'setflag|size|spamtest|stop|string|true|vacation|virustest)',
+             Name.Builtin),
+            (r'(?i)set',
+             Keyword.Declaration),
+            # number:
+            (r'([0-9.]+)([kmgKMG])?',
+             bygroups(Literal.Number, Literal.Number)),
+            # comment:
+            (r'#.*$',
+             Comment.Single),
+            (r'/\*.*\*/',
+             Comment.Multiline),
+            # string:
+            (r'"[^"]*?"',
+             String),
+            # text block:
+            (r'text:',
+             Name.Tag, 'text'),
+        ],
+        'text': [
+            (r'[^.].*?\n', String),
+            (r'^\.', Punctuation, "#pop"),
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/slash.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/slash.py
new file mode 100644
index 0000000000000000000000000000000000000000..2bcbdf877548eb9d670ee2384aed01d2d631dc16
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/slash.py
@@ -0,0 +1,183 @@
+"""
+    pygments.lexers.slash
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Slash programming language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import ExtendedRegexLexer, bygroups, DelegatingLexer
+from pygments.token import Name, Number, String, Comment, Punctuation, \
+    Other, Keyword, Operator, Whitespace
+
+__all__ = ['SlashLexer']
+
+
+class SlashLanguageLexer(ExtendedRegexLexer):
+    _nkw = r'(?=[^a-zA-Z_0-9])'
+
+    def move_state(new_state):
+        return ("#pop", new_state)
+
+    def right_angle_bracket(lexer, match, ctx):
+        if len(ctx.stack) > 1 and ctx.stack[-2] == "string":
+            ctx.stack.pop()
+        yield match.start(), String.Interpol, '}'
+        ctx.pos = match.end()
+        pass
+
+    tokens = {
+        "root": [
+            (r"<%=",        Comment.Preproc,    move_state("slash")),
+            (r"<%!!",       Comment.Preproc,    move_state("slash")),
+            (r"<%#.*?%>",   Comment.Multiline),
+            (r"<%",         Comment.Preproc,    move_state("slash")),
+            (r".|\n",       Other),
+        ],
+        "string": [
+            (r"\\",         String.Escape,      move_state("string_e")),
+            (r"\"",         String,             move_state("slash")),
+            (r"#\{",        String.Interpol,    "slash"),
+            (r'.|\n',       String),
+        ],
+        "string_e": [
+            (r'n',                  String.Escape,      move_state("string")),
+            (r't',                  String.Escape,      move_state("string")),
+            (r'r',                  String.Escape,      move_state("string")),
+            (r'e',                  String.Escape,      move_state("string")),
+            (r'x[a-fA-F0-9]{2}',    String.Escape,      move_state("string")),
+            (r'.',                  String.Escape,      move_state("string")),
+        ],
+        "regexp": [
+            (r'}[a-z]*',            String.Regex,       move_state("slash")),
+            (r'\\(.|\n)',           String.Regex),
+            (r'{',                  String.Regex,       "regexp_r"),
+            (r'.|\n',               String.Regex),
+        ],
+        "regexp_r": [
+            (r'}[a-z]*',            String.Regex,       "#pop"),
+            (r'\\(.|\n)',           String.Regex),
+            (r'{',                  String.Regex,       "regexp_r"),
+        ],
+        "slash": [
+            (r"%>",                     Comment.Preproc,    move_state("root")),
+            (r"\"",                     String,             move_state("string")),
+            (r"'[a-zA-Z0-9_]+",         String),
+            (r'%r{',                    String.Regex,       move_state("regexp")),
+            (r'/\*.*?\*/',              Comment.Multiline),
+            (r"(#|//).*?\n",            Comment.Single),
+            (r'-?[0-9]+e[+-]?[0-9]+',   Number.Float),
+            (r'-?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?', Number.Float),
+            (r'-?[0-9]+',               Number.Integer),
+            (r'nil'+_nkw,               Name.Builtin),
+            (r'true'+_nkw,              Name.Builtin),
+            (r'false'+_nkw,             Name.Builtin),
+            (r'self'+_nkw,              Name.Builtin),
+            (r'(class)(\s+)([A-Z][a-zA-Z0-9_\']*)',
+                bygroups(Keyword, Whitespace, Name.Class)),
+            (r'class'+_nkw,             Keyword),
+            (r'extends'+_nkw,           Keyword),
+            (r'(def)(\s+)(self)(\s*)(\.)(\s*)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)',
+                bygroups(Keyword, Whitespace, Name.Builtin, Whitespace, Punctuation, Whitespace, Name.Function)),
+            (r'(def)(\s+)([a-z_][a-zA-Z0-9_\']*=?|<<|>>|==|<=>|<=|<|>=|>|\+|-(self)?|~(self)?|\*|/|%|^|&&|&|\||\[\]=?)',
+                bygroups(Keyword, Whitespace, Name.Function)),
+            (r'def'+_nkw,               Keyword),
+            (r'if'+_nkw,                Keyword),
+            (r'elsif'+_nkw,             Keyword),
+            (r'else'+_nkw,              Keyword),
+            (r'unless'+_nkw,            Keyword),
+            (r'for'+_nkw,               Keyword),
+            (r'in'+_nkw,                Keyword),
+            (r'while'+_nkw,             Keyword),
+            (r'until'+_nkw,             Keyword),
+            (r'and'+_nkw,               Keyword),
+            (r'or'+_nkw,                Keyword),
+            (r'not'+_nkw,               Keyword),
+            (r'lambda'+_nkw,            Keyword),
+            (r'try'+_nkw,               Keyword),
+            (r'catch'+_nkw,             Keyword),
+            (r'return'+_nkw,            Keyword),
+            (r'next'+_nkw,              Keyword),
+            (r'last'+_nkw,              Keyword),
+            (r'throw'+_nkw,             Keyword),
+            (r'use'+_nkw,               Keyword),
+            (r'switch'+_nkw,            Keyword),
+            (r'\\',                     Keyword),
+            (r'λ',                      Keyword),
+            (r'__FILE__'+_nkw,          Name.Builtin.Pseudo),
+            (r'__LINE__'+_nkw,          Name.Builtin.Pseudo),
+            (r'[A-Z][a-zA-Z0-9_\']*'+_nkw, Name.Constant),
+            (r'[a-z_][a-zA-Z0-9_\']*'+_nkw, Name),
+            (r'@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Instance),
+            (r'@@[a-z_][a-zA-Z0-9_\']*'+_nkw, Name.Variable.Class),
+            (r'\(',                     Punctuation),
+            (r'\)',                     Punctuation),
+            (r'\[',                     Punctuation),
+            (r'\]',                     Punctuation),
+            (r'\{',                     Punctuation),
+            (r'\}',                     right_angle_bracket),
+            (r';',                      Punctuation),
+            (r',',                      Punctuation),
+            (r'<<=',                    Operator),
+            (r'>>=',                    Operator),
+            (r'<<',                     Operator),
+            (r'>>',                     Operator),
+            (r'==',                     Operator),
+            (r'!=',                     Operator),
+            (r'=>',                     Operator),
+            (r'=',                      Operator),
+            (r'<=>',                    Operator),
+            (r'<=',                     Operator),
+            (r'>=',                     Operator),
+            (r'<',                      Operator),
+            (r'>',                      Operator),
+            (r'\+\+',                   Operator),
+            (r'\+=',                    Operator),
+            (r'-=',                     Operator),
+            (r'\*\*=',                  Operator),
+            (r'\*=',                    Operator),
+            (r'\*\*',                   Operator),
+            (r'\*',                     Operator),
+            (r'/=',                     Operator),
+            (r'\+',                     Operator),
+            (r'-',                      Operator),
+            (r'/',                      Operator),
+            (r'%=',                     Operator),
+            (r'%',                      Operator),
+            (r'^=',                     Operator),
+            (r'&&=',                    Operator),
+            (r'&=',                     Operator),
+            (r'&&',                     Operator),
+            (r'&',                      Operator),
+            (r'\|\|=',                  Operator),
+            (r'\|=',                    Operator),
+            (r'\|\|',                   Operator),
+            (r'\|',                     Operator),
+            (r'!',                      Operator),
+            (r'\.\.\.',                 Operator),
+            (r'\.\.',                   Operator),
+            (r'\.',                     Operator),
+            (r'::',                     Operator),
+            (r':',                      Operator),
+            (r'(\s|\n)+',               Whitespace),
+            (r'[a-z_][a-zA-Z0-9_\']*',  Name.Variable),
+        ],
+    }
+
+
+class SlashLexer(DelegatingLexer):
+    """
+    Lexer for the Slash programming language.
+    """
+
+    name = 'Slash'
+    aliases = ['slash']
+    filenames = ['*.sla']
+    url = 'https://github.com/arturadib/Slash-A'
+    version_added = '2.4'
+
+    def __init__(self, **options):
+        from pygments.lexers.web import HtmlLexer
+        super().__init__(HtmlLexer, SlashLanguageLexer, **options)
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/smalltalk.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/smalltalk.py
new file mode 100644
index 0000000000000000000000000000000000000000..69e0519b77f612145f5ac56f3260bfe4ad863d97
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/smalltalk.py
@@ -0,0 +1,194 @@
+"""
+    pygments.lexers.smalltalk
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Smalltalk and related languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['SmalltalkLexer', 'NewspeakLexer']
+
+
+class SmalltalkLexer(RegexLexer):
+    """
+    For Smalltalk syntax.
+    Contributed by Stefan Matthias Aust.
+    Rewritten by Nils Winter.
+    """
+    name = 'Smalltalk'
+    url = 'http://www.smalltalk.org/'
+    filenames = ['*.st']
+    aliases = ['smalltalk', 'squeak', 'st']
+    mimetypes = ['text/x-smalltalk']
+    version_added = '0.10'
+
+    tokens = {
+        'root': [
+            (r'(<)(\w+:)(.*?)(>)', bygroups(Text, Keyword, Text, Text)),
+            include('squeak fileout'),
+            include('whitespaces'),
+            include('method definition'),
+            (r'(\|)([\w\s]*)(\|)', bygroups(Operator, Name.Variable, Operator)),
+            include('objects'),
+            (r'\^|\:=|\_', Operator),
+            # temporaries
+            (r'[\]({}.;!]', Text),
+        ],
+        'method definition': [
+            # Not perfect can't allow whitespaces at the beginning and the
+            # without breaking everything
+            (r'([a-zA-Z]+\w*:)(\s*)(\w+)',
+             bygroups(Name.Function, Text, Name.Variable)),
+            (r'^(\b[a-zA-Z]+\w*\b)(\s*)$', bygroups(Name.Function, Text)),
+            (r'^([-+*/\\~<>=|&!?,@%]+)(\s*)(\w+)(\s*)$',
+             bygroups(Name.Function, Text, Name.Variable, Text)),
+        ],
+        'blockvariables': [
+            include('whitespaces'),
+            (r'(:)(\s*)(\w+)',
+             bygroups(Operator, Text, Name.Variable)),
+            (r'\|', Operator, '#pop'),
+            default('#pop'),  # else pop
+        ],
+        'literals': [
+            (r"'(''|[^'])*'", String, 'afterobject'),
+            (r'\$.', String.Char, 'afterobject'),
+            (r'#\(', String.Symbol, 'parenth'),
+            (r'\)', Text, 'afterobject'),
+            (r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number, 'afterobject'),
+        ],
+        '_parenth_helper': [
+            include('whitespaces'),
+            (r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number),
+            (r'[-+*/\\~<>=|&#!?,@%\w:]+', String.Symbol),
+            # literals
+            (r"'(''|[^'])*'", String),
+            (r'\$.', String.Char),
+            (r'#*\(', String.Symbol, 'inner_parenth'),
+        ],
+        'parenth': [
+            # This state is a bit tricky since
+            # we can't just pop this state
+            (r'\)', String.Symbol, ('root', 'afterobject')),
+            include('_parenth_helper'),
+        ],
+        'inner_parenth': [
+            (r'\)', String.Symbol, '#pop'),
+            include('_parenth_helper'),
+        ],
+        'whitespaces': [
+            # skip whitespace and comments
+            (r'\s+', Text),
+            (r'"(""|[^"])*"', Comment),
+        ],
+        'objects': [
+            (r'\[', Text, 'blockvariables'),
+            (r'\]', Text, 'afterobject'),
+            (r'\b(self|super|true|false|nil|thisContext)\b',
+             Name.Builtin.Pseudo, 'afterobject'),
+            (r'\b[A-Z]\w*(?!:)\b', Name.Class, 'afterobject'),
+            (r'\b[a-z]\w*(?!:)\b', Name.Variable, 'afterobject'),
+            (r'#("(""|[^"])*"|[-+*/\\~<>=|&!?,@%]+|[\w:]+)',
+             String.Symbol, 'afterobject'),
+            include('literals'),
+        ],
+        'afterobject': [
+            (r'! !$', Keyword, '#pop'),  # squeak chunk delimiter
+            include('whitespaces'),
+            (r'\b(ifTrue:|ifFalse:|whileTrue:|whileFalse:|timesRepeat:)',
+             Name.Builtin, '#pop'),
+            (r'\b(new\b(?!:))', Name.Builtin),
+            (r'\:=|\_', Operator, '#pop'),
+            (r'\b[a-zA-Z]+\w*:', Name.Function, '#pop'),
+            (r'\b[a-zA-Z]+\w*', Name.Function),
+            (r'\w+:?|[-+*/\\~<>=|&!?,@%]+', Name.Function, '#pop'),
+            (r'\.', Punctuation, '#pop'),
+            (r';', Punctuation),
+            (r'[\])}]', Text),
+            (r'[\[({]', Text, '#pop'),
+        ],
+        'squeak fileout': [
+            # Squeak fileout format (optional)
+            (r'^"(""|[^"])*"!', Keyword),
+            (r"^'(''|[^'])*'!", Keyword),
+            (r'^(!)(\w+)( commentStamp: )(.*?)( prior: .*?!\n)(.*?)(!)',
+                bygroups(Keyword, Name.Class, Keyword, String, Keyword, Text, Keyword)),
+            (r"^(!)(\w+(?: class)?)( methodsFor: )('(?:''|[^'])*')(.*?!)",
+                bygroups(Keyword, Name.Class, Keyword, String, Keyword)),
+            (r'^(\w+)( subclass: )(#\w+)'
+             r'(\s+instanceVariableNames: )(.*?)'
+             r'(\s+classVariableNames: )(.*?)'
+             r'(\s+poolDictionaries: )(.*?)'
+             r'(\s+category: )(.*?)(!)',
+                bygroups(Name.Class, Keyword, String.Symbol, Keyword, String, Keyword,
+                         String, Keyword, String, Keyword, String, Keyword)),
+            (r'^(\w+(?: class)?)(\s+instanceVariableNames: )(.*?)(!)',
+                bygroups(Name.Class, Keyword, String, Keyword)),
+            (r'(!\n)(\].*)(! !)$', bygroups(Keyword, Text, Keyword)),
+            (r'! !$', Keyword),
+        ],
+    }
+
+
+class NewspeakLexer(RegexLexer):
+    """
+    For Newspeak syntax.
+    """
+    name = 'Newspeak'
+    url = 'http://newspeaklanguage.org/'
+    filenames = ['*.ns2']
+    aliases = ['newspeak', ]
+    mimetypes = ['text/x-newspeak']
+    version_added = '1.1'
+
+    tokens = {
+        'root': [
+            (r'\b(Newsqueak2)\b', Keyword.Declaration),
+            (r"'[^']*'", String),
+            (r'\b(class)(\s+)(\w+)(\s*)',
+             bygroups(Keyword.Declaration, Text, Name.Class, Text)),
+            (r'\b(mixin|self|super|private|public|protected|nil|true|false)\b',
+             Keyword),
+            (r'(\w+\:)(\s*)([a-zA-Z_]\w+)',
+             bygroups(Name.Function, Text, Name.Variable)),
+            (r'(\w+)(\s*)(=)',
+             bygroups(Name.Attribute, Text, Operator)),
+            (r'<\w+>', Comment.Special),
+            include('expressionstat'),
+            include('whitespace')
+        ],
+
+        'expressionstat': [
+            (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
+            (r'\d+', Number.Integer),
+            (r':\w+', Name.Variable),
+            (r'(\w+)(::)', bygroups(Name.Variable, Operator)),
+            (r'\w+:', Name.Function),
+            (r'\w+', Name.Variable),
+            (r'\(|\)', Punctuation),
+            (r'\[|\]', Punctuation),
+            (r'\{|\}', Punctuation),
+
+            (r'(\^|\+|\/|~|\*|<|>|=|@|%|\||&|\?|!|,|-|:)', Operator),
+            (r'\.|;', Punctuation),
+            include('whitespace'),
+            include('literals'),
+        ],
+        'literals': [
+            (r'\$.', String),
+            (r"'[^']*'", String),
+            (r"#'[^']*'", String.Symbol),
+            (r"#\w+:?", String.Symbol),
+            (r"#(\+|\/|~|\*|<|>|=|@|%|\||&|\?|!|,|-)+", String.Symbol)
+        ],
+        'whitespace': [
+            (r'\s+', Text),
+            (r'"[^"]*"', Comment)
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/smithy.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/smithy.py
new file mode 100644
index 0000000000000000000000000000000000000000..2642bb879901595f88e605364703db92de9707bd
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/smithy.py
@@ -0,0 +1,77 @@
+"""
+    pygments.lexers.smithy
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Smithy IDL.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Keyword, Name, String, \
+    Number, Whitespace, Punctuation
+
+__all__ = ['SmithyLexer']
+
+
+class SmithyLexer(RegexLexer):
+    """
+    For Smithy IDL
+    """
+    name = 'Smithy'
+    url = 'https://awslabs.github.io/smithy/'
+    filenames = ['*.smithy']
+    aliases = ['smithy']
+    version_added = '2.10'
+
+    unquoted = r'[A-Za-z0-9_\.#$-]+'
+    identifier = r"[A-Za-z0-9_\.#$-]+"
+
+    simple_shapes = (
+        'use', 'byte', 'short', 'integer', 'long', 'float', 'document',
+        'double', 'bigInteger', 'bigDecimal', 'boolean', 'blob', 'string',
+        'timestamp',
+    )
+
+    aggregate_shapes = (
+       'apply', 'list', 'map', 'set', 'structure', 'union', 'resource',
+       'operation', 'service', 'trait'
+    )
+
+    tokens = {
+        'root': [
+            (r'///.*$', Comment.Multiline),
+            (r'//.*$', Comment),
+            (r'@[0-9a-zA-Z\.#-]*', Name.Decorator),
+            (r'(=)', Name.Decorator),
+            (r'^(\$version)(:)(.+)',
+                bygroups(Keyword.Declaration, Name.Decorator, Name.Class)),
+            (r'^(namespace)(\s+' + identifier + r')\b',
+                bygroups(Keyword.Declaration, Name.Class)),
+            (words(simple_shapes,
+                   prefix=r'^', suffix=r'(\s+' + identifier + r')\b'),
+                bygroups(Keyword.Declaration, Name.Class)),
+            (words(aggregate_shapes,
+                   prefix=r'^', suffix=r'(\s+' + identifier + r')'),
+                bygroups(Keyword.Declaration, Name.Class)),
+            (r'^(metadata)(\s+)((?:\S+)|(?:\"[^"]+\"))(\s*)(=)',
+                bygroups(Keyword.Declaration, Whitespace, Name.Class,
+                         Whitespace, Name.Decorator)),
+            (r"(true|false|null)", Keyword.Constant),
+            (r"(-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?)", Number),
+            (identifier + ":", Name.Label),
+            (identifier, Name.Variable.Class),
+            (r'\[', Text, "#push"),
+            (r'\]', Text, "#pop"),
+            (r'\(', Text, "#push"),
+            (r'\)', Text, "#pop"),
+            (r'\{', Text, "#push"),
+            (r'\}', Text, "#pop"),
+            (r'"{3}(\\\\|\n|\\")*"{3}', String.Doc),
+            (r'"(\\\\|\n|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\n|\\'|[^'])*'", String.Single),
+            (r'[:,]+', Punctuation),
+            (r'\s+', Whitespace),
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/smv.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/smv.py
new file mode 100644
index 0000000000000000000000000000000000000000..3d06f99f60b3af912d889faba24993a9b69ea5ca
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/smv.py
@@ -0,0 +1,78 @@
+"""
+    pygments.lexers.smv
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the SMV languages.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, \
+    Punctuation, Text
+
+__all__ = ['NuSMVLexer']
+
+
+class NuSMVLexer(RegexLexer):
+    """
+    Lexer for the NuSMV language.
+    """
+
+    name = 'NuSMV'
+    aliases = ['nusmv']
+    filenames = ['*.smv']
+    mimetypes = []
+    url = 'https://nusmv.fbk.eu'
+    version_added = '2.2'
+
+    tokens = {
+        'root': [
+            # Comments
+            (r'(?s)\/\-\-.*?\-\-/', Comment),
+            (r'--.*\n', Comment),
+
+            # Reserved
+            (words(('MODULE', 'DEFINE', 'MDEFINE', 'CONSTANTS', 'VAR', 'IVAR',
+                    'FROZENVAR', 'INIT', 'TRANS', 'INVAR', 'SPEC', 'CTLSPEC',
+                    'LTLSPEC', 'PSLSPEC', 'COMPUTE', 'NAME', 'INVARSPEC',
+                    'FAIRNESS', 'JUSTICE', 'COMPASSION', 'ISA', 'ASSIGN',
+                    'CONSTRAINT', 'SIMPWFF', 'CTLWFF', 'LTLWFF', 'PSLWFF',
+                    'COMPWFF', 'IN', 'MIN', 'MAX', 'MIRROR', 'PRED',
+                    'PREDICATES'), suffix=r'(?![\w$#-])'),
+             Keyword.Declaration),
+            (r'process(?![\w$#-])', Keyword),
+            (words(('array', 'of', 'boolean', 'integer', 'real', 'word'),
+                   suffix=r'(?![\w$#-])'), Keyword.Type),
+            (words(('case', 'esac'), suffix=r'(?![\w$#-])'), Keyword),
+            (words(('word1', 'bool', 'signed', 'unsigned', 'extend', 'resize',
+                    'sizeof', 'uwconst', 'swconst', 'init', 'self', 'count',
+                    'abs', 'max', 'min'), suffix=r'(?![\w$#-])'),
+             Name.Builtin),
+            (words(('EX', 'AX', 'EF', 'AF', 'EG', 'AG', 'E', 'F', 'O', 'G',
+                    'H', 'X', 'Y', 'Z', 'A', 'U', 'S', 'V', 'T', 'BU', 'EBF',
+                    'ABF', 'EBG', 'ABG', 'next', 'mod', 'union', 'in', 'xor',
+                    'xnor'), suffix=r'(?![\w$#-])'),
+                Operator.Word),
+            (words(('TRUE', 'FALSE'), suffix=r'(?![\w$#-])'), Keyword.Constant),
+
+            # Names
+            (r'[a-zA-Z_][\w$#-]*', Name.Variable),
+
+            # Operators
+            (r':=', Operator),
+            (r'[-&|+*/<>!=]', Operator),
+
+            # Literals
+            (r'\-?\d+\b', Number.Integer),
+            (r'0[su][bB]\d*_[01_]+', Number.Bin),
+            (r'0[su][oO]\d*_[0-7_]+', Number.Oct),
+            (r'0[su][dD]\d*_[\d_]+', Number.Decimal),
+            (r'0[su][hH]\d*_[\da-fA-F_]+', Number.Hex),
+
+            # Whitespace, punctuation and the rest
+            (r'\s+', Text.Whitespace),
+            (r'[()\[\]{};?:.,]', Punctuation),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/snobol.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/snobol.py
new file mode 100644
index 0000000000000000000000000000000000000000..7016c96a5fdbaa7ed5e531a78584faf903d99841
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/snobol.py
@@ -0,0 +1,82 @@
+"""
+    pygments.lexers.snobol
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the SNOBOL language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['SnobolLexer']
+
+
+class SnobolLexer(RegexLexer):
+    """
+    Lexer for the SNOBOL4 programming language.
+
+    Recognizes the common ASCII equivalents of the original SNOBOL4 operators.
+    Does not require spaces around binary operators.
+    """
+
+    name = "Snobol"
+    aliases = ["snobol"]
+    filenames = ['*.snobol']
+    mimetypes = ['text/x-snobol']
+    url = 'https://www.regressive.org/snobol4'
+    version_added = '1.5'
+
+    tokens = {
+        # root state, start of line
+        # comments, continuation lines, and directives start in column 1
+        # as do labels
+        'root': [
+            (r'\*.*\n', Comment),
+            (r'[+.] ', Punctuation, 'statement'),
+            (r'-.*\n', Comment),
+            (r'END\s*\n', Name.Label, 'heredoc'),
+            (r'[A-Za-z$][\w$]*', Name.Label, 'statement'),
+            (r'\s+', Text, 'statement'),
+        ],
+        # statement state, line after continuation or label
+        'statement': [
+            (r'\s*\n', Text, '#pop'),
+            (r'\s+', Text),
+            (r'(?<=[^\w.])(LT|LE|EQ|NE|GE|GT|INTEGER|IDENT|DIFFER|LGT|SIZE|'
+             r'REPLACE|TRIM|DUPL|REMDR|DATE|TIME|EVAL|APPLY|OPSYN|LOAD|UNLOAD|'
+             r'LEN|SPAN|BREAK|ANY|NOTANY|TAB|RTAB|REM|POS|RPOS|FAIL|FENCE|'
+             r'ABORT|ARB|ARBNO|BAL|SUCCEED|INPUT|OUTPUT|TERMINAL)(?=[^\w.])',
+             Name.Builtin),
+            (r'[A-Za-z][\w.]*', Name),
+            # ASCII equivalents of original operators
+            # | for the EBCDIC equivalent, ! likewise
+            # \ for EBCDIC negation
+            (r'\*\*|[?$.!%*/#+\-@|&\\=]', Operator),
+            (r'"[^"]*"', String),
+            (r"'[^']*'", String),
+            # Accept SPITBOL syntax for real numbers
+            # as well as Macro SNOBOL4
+            (r'[0-9]+(?=[^.EeDd])', Number.Integer),
+            (r'[0-9]+(\.[0-9]*)?([EDed][-+]?[0-9]+)?', Number.Float),
+            # Goto
+            (r':', Punctuation, 'goto'),
+            (r'[()<>,;]', Punctuation),
+        ],
+        # Goto block
+        'goto': [
+            (r'\s*\n', Text, "#pop:2"),
+            (r'\s+', Text),
+            (r'F|S', Keyword),
+            (r'(\()([A-Za-z][\w.]*)(\))',
+             bygroups(Punctuation, Name.Label, Punctuation))
+        ],
+        # everything after the END statement is basically one
+        # big heredoc.
+        'heredoc': [
+            (r'.*\n', String.Heredoc)
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/solidity.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/solidity.py
new file mode 100644
index 0000000000000000000000000000000000000000..3f2d1dd0ac19bd5d9eb90939da249d4505286803
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/solidity.py
@@ -0,0 +1,87 @@
+"""
+    pygments.lexers.solidity
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Solidity.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+
+__all__ = ['SolidityLexer']
+
+
+class SolidityLexer(RegexLexer):
+    """
+    For Solidity source code.
+    """
+
+    name = 'Solidity'
+    aliases = ['solidity']
+    filenames = ['*.sol']
+    mimetypes = []
+    url = 'https://soliditylang.org'
+    version_added = '2.5'
+
+    datatype = (
+        r'\b(address|bool|(?:(?:bytes|hash|int|string|uint)(?:8|16|24|32|40|48|56|64'
+        r'|72|80|88|96|104|112|120|128|136|144|152|160|168|176|184|192|200|208'
+        r'|216|224|232|240|248|256)?))\b'
+    )
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            include('comments'),
+            (r'\bpragma\s+solidity\b', Keyword, 'pragma'),
+            (r'\b(contract)(\s+)([a-zA-Z_]\w*)',
+             bygroups(Keyword, Whitespace, Name.Entity)),
+            (datatype + r'(\s+)((?:external|public|internal|private)\s+)?' +
+             r'([a-zA-Z_]\w*)',
+             bygroups(Keyword.Type, Whitespace, Keyword, Name.Variable)),
+            (r'\b(enum|event|function|struct)(\s+)([a-zA-Z_]\w*)',
+             bygroups(Keyword.Type, Whitespace, Name.Variable)),
+            (r'\b(msg|block|tx)\.([A-Za-z_][a-zA-Z0-9_]*)\b', Keyword),
+            (words((
+                'block', 'break', 'constant', 'constructor', 'continue',
+                'contract', 'do', 'else', 'external', 'false', 'for',
+                'function', 'if', 'import', 'inherited', 'internal', 'is',
+                'library', 'mapping', 'memory', 'modifier', 'msg', 'new',
+                'payable', 'private', 'public', 'require', 'return',
+                'returns', 'struct', 'suicide', 'throw', 'this', 'true',
+                'tx', 'var', 'while'), prefix=r'\b', suffix=r'\b'),
+             Keyword.Type),
+            (words(('keccak256',), prefix=r'\b', suffix=r'\b'), Name.Builtin),
+            (datatype, Keyword.Type),
+            include('constants'),
+            (r'[a-zA-Z_]\w*', Text),
+            (r'[~!%^&*+=|?:<>/-]', Operator),
+            (r'[.;{}(),\[\]]', Punctuation)
+        ],
+        'comments': [
+            (r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single),
+            (r'/(\\\n)?[*][\w\W]*?[*](\\\n)?/', Comment.Multiline),
+            (r'/(\\\n)?[*][\w\W]*', Comment.Multiline)
+        ],
+        'constants': [
+            (r'("(\\"|.)*?")', String.Double),
+            (r"('(\\'|.)*?')", String.Single),
+            (r'\b0[xX][0-9a-fA-F]+\b', Number.Hex),
+            (r'\b\d+\b', Number.Decimal),
+        ],
+        'pragma': [
+            include('whitespace'),
+            include('comments'),
+            (r'(\^|>=|<)(\s*)(\d+\.\d+\.\d+)',
+             bygroups(Operator, Whitespace, Keyword)),
+            (r';', Punctuation, '#pop')
+        ],
+        'whitespace': [
+            (r'\s+', Whitespace),
+            (r'\n', Whitespace)
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/soong.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/soong.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d7f12ccd592e71b6a866089a35fd2582c82f3d6
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/soong.py
@@ -0,0 +1,78 @@
+"""
+    pygments.lexers.soong
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Soong (Android.bp Blueprint) files.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, include
+from pygments.token import Comment, Name, Number, Operator, Punctuation, \
+        String, Whitespace
+
+__all__ = ['SoongLexer']
+
+class SoongLexer(RegexLexer):
+    name = 'Soong'
+    version_added = '2.18'
+    url = 'https://source.android.com/docs/setup/reference/androidbp'
+    aliases = ['androidbp', 'bp', 'soong']
+    filenames = ['Android.bp']
+
+    tokens = {
+        'root': [
+            # A variable assignment
+            (r'(\w*)(\s*)(\+?=)(\s*)',
+             bygroups(Name.Variable, Whitespace, Operator, Whitespace),
+             'assign-rhs'),
+
+            # A top-level module
+            (r'(\w*)(\s*)(\{)',
+             bygroups(Name.Function, Whitespace, Punctuation),
+             'in-rule'),
+
+            # Everything else
+            include('comments'),
+            (r'\s+', Whitespace),  # newlines okay
+        ],
+        'assign-rhs': [
+            include('expr'),
+            (r'\n', Whitespace, '#pop'),
+        ],
+        'in-list': [
+            include('expr'),
+            include('comments'),
+            (r'\s+', Whitespace),  # newlines okay in a list
+            (r',', Punctuation),
+            (r'\]', Punctuation, '#pop'),
+        ],
+        'in-map': [
+            # A map key
+            (r'(\w+)(:)(\s*)', bygroups(Name, Punctuation, Whitespace)),
+
+            include('expr'),
+            include('comments'),
+            (r'\s+', Whitespace),  # newlines okay in a map
+            (r',', Punctuation),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'in-rule': [
+            # Just re-use map syntax
+            include('in-map'),
+        ],
+        'comments': [
+            (r'//.*', Comment.Single),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+        ],
+        'expr': [
+            (r'(true|false)\b', Name.Builtin),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'".*?"', String),
+            (r'\{', Punctuation, 'in-map'),
+            (r'\[', Punctuation, 'in-list'),
+            (r'\w+', Name),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sophia.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sophia.py
new file mode 100644
index 0000000000000000000000000000000000000000..86435f3fe7e482cd069f2cdae33fe32a69f08434
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sophia.py
@@ -0,0 +1,102 @@
+"""
+    pygments.lexers.sophia
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Sophia.
+
+    Derived from pygments/lexers/reason.py.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, default, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, \
+    Punctuation, String, Text
+
+__all__ = ['SophiaLexer']
+
+class SophiaLexer(RegexLexer):
+    """
+    A Sophia lexer.
+    """
+
+    name = 'Sophia'
+    aliases = ['sophia']
+    filenames = ['*.aes']
+    mimetypes = []
+    url = 'https://docs.aeternity.com/aesophia'
+    version_added = '2.11'
+
+    keywords = (
+        'contract', 'include', 'let', 'switch', 'type', 'record', 'datatype',
+        'if', 'elif', 'else', 'function', 'stateful', 'payable', 'public',
+        'entrypoint', 'private', 'indexed', 'namespace', 'interface', 'main',
+        'using', 'as', 'for', 'hiding',
+    )
+
+    builtins = ('state', 'put', 'abort', 'require')
+
+    word_operators = ('mod', 'band', 'bor', 'bxor', 'bnot')
+
+    primitive_types = ('int', 'address', 'bool', 'bits', 'bytes', 'string',
+                       'list', 'option', 'char', 'unit', 'map', 'event',
+                       'hash', 'signature', 'oracle', 'oracle_query')
+
+    tokens = {
+        'escape-sequence': [
+            (r'\\[\\"\'ntbr]', String.Escape),
+            (r'\\[0-9]{3}', String.Escape),
+            (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        ],
+        'root': [
+            (r'\s+', Text.Whitespace),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Class, 'dotted'),
+            (r'\b([A-Z][\w\']*)', Name.Function),
+            (r'//.*?\n', Comment.Single),
+            (r'\/\*(?!/)', Comment.Multiline, 'comment'),
+
+            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+            (r'#[\da-fA-F][\da-fA-F_]*', Name.Label),
+            (r'\d[\d_]*', Number.Integer),
+
+            (words(keywords, suffix=r'\b'), Keyword),
+            (words(builtins, suffix=r'\b'), Name.Builtin),
+            (words(word_operators, prefix=r'\b', suffix=r'\b'), Operator.Word),
+            (words(primitive_types, prefix=r'\b', suffix=r'\b'), Keyword.Type),
+
+            (r'[=!<>+\\*/:&|?~@^-]', Operator.Word),
+            (r'[.;:{}(),\[\]]', Punctuation),
+
+            (r"(ak_|ok_|oq_|ct_)[\w']*", Name.Label),
+            (r"[^\W\d][\w']*", Name),
+
+            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+             String.Char),
+            (r"'.'", String.Char),
+            (r"'[a-z][\w]*", Name.Variable),
+
+            (r'"', String.Double, 'string')
+        ],
+        'comment': [
+            (r'[^/*]+', Comment.Multiline),
+            (r'\/\*', Comment.Multiline, '#push'),
+            (r'\*\/', Comment.Multiline, '#pop'),
+            (r'\*', Comment.Multiline),
+        ],
+        'string': [
+            (r'[^\\"]+', String.Double),
+            include('escape-sequence'),
+            (r'\\\n', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'dotted': [
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[A-Z][\w\']*(?=\s*\.)', Name.Function),
+            (r'[A-Z][\w\']*', Name.Function, '#pop'),
+            (r'[a-z_][\w\']*', Name, '#pop'),
+            default('#pop'),
+        ],
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/special.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/special.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea345226fc7beca9290514d90dfef58bc8512a3a
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/special.py
@@ -0,0 +1,122 @@
+"""
+    pygments.lexers.special
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Special lexers.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import ast
+
+from pygments.lexer import Lexer, line_re
+from pygments.token import Token, Error, Text, Generic
+from pygments.util import get_choice_opt
+
+
+__all__ = ['TextLexer', 'OutputLexer', 'RawTokenLexer']
+
+
+class TextLexer(Lexer):
+    """
+    "Null" lexer, doesn't highlight anything.
+    """
+    name = 'Text only'
+    aliases = ['text']
+    filenames = ['*.txt']
+    mimetypes = ['text/plain']
+    url = ""
+    version_added = ''
+
+    priority = 0.01
+
+    def get_tokens_unprocessed(self, text):
+        yield 0, Text, text
+
+    def analyse_text(text):
+        return TextLexer.priority
+
+
+class OutputLexer(Lexer):
+    """
+    Simple lexer that highlights everything as ``Token.Generic.Output``.
+    """
+    name = 'Text output'
+    aliases = ['output']
+    url = ""
+    version_added = '2.10'
+    _example = "output/output"
+
+    def get_tokens_unprocessed(self, text):
+        yield 0, Generic.Output, text
+
+
+_ttype_cache = {}
+
+
+class RawTokenLexer(Lexer):
+    """
+    Recreate a token stream formatted with the `RawTokenFormatter`.
+
+    Additional options accepted:
+
+    `compress`
+        If set to ``"gz"`` or ``"bz2"``, decompress the token stream with
+        the given compression algorithm before lexing (default: ``""``).
+    """
+    name = 'Raw token data'
+    aliases = []
+    filenames = []
+    mimetypes = ['application/x-pygments-tokens']
+    url = 'https://pygments.org/docs/formatters/#RawTokenFormatter'
+    version_added = ''
+
+    def __init__(self, **options):
+        self.compress = get_choice_opt(options, 'compress',
+                                       ['', 'none', 'gz', 'bz2'], '')
+        Lexer.__init__(self, **options)
+
+    def get_tokens(self, text):
+        if self.compress:
+            if isinstance(text, str):
+                text = text.encode('latin1')
+            try:
+                if self.compress == 'gz':
+                    import gzip
+                    text = gzip.decompress(text)
+                elif self.compress == 'bz2':
+                    import bz2
+                    text = bz2.decompress(text)
+            except OSError:
+                yield Error, text.decode('latin1')
+        if isinstance(text, bytes):
+            text = text.decode('latin1')
+
+        # do not call Lexer.get_tokens() because stripping is not optional.
+        text = text.strip('\n') + '\n'
+        for i, t, v in self.get_tokens_unprocessed(text):
+            yield t, v
+
+    def get_tokens_unprocessed(self, text):
+        length = 0
+        for match in line_re.finditer(text):
+            try:
+                ttypestr, val = match.group().rstrip().split('\t', 1)
+                ttype = _ttype_cache.get(ttypestr)
+                if not ttype:
+                    ttype = Token
+                    ttypes = ttypestr.split('.')[1:]
+                    for ttype_ in ttypes:
+                        if not ttype_ or not ttype_[0].isupper():
+                            raise ValueError('malformed token name')
+                        ttype = getattr(ttype, ttype_)
+                    _ttype_cache[ttypestr] = ttype
+                val = ast.literal_eval(val)
+                if not isinstance(val, str):
+                    raise ValueError('expected str')
+            except (SyntaxError, ValueError):
+                val = match.group()
+                ttype = Error
+            yield length, ttype, val
+            length += len(val)
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/spice.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/spice.py
new file mode 100644
index 0000000000000000000000000000000000000000..978f0a71fd2957c88060b51bb3bef863cff5769d
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/spice.py
@@ -0,0 +1,70 @@
+"""
+    pygments.lexers.spice
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Spice programming language.
+
+    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+
+__all__ = ['SpiceLexer']
+
+
+class SpiceLexer(RegexLexer):
+    """
+    For Spice source.
+    """
+    name = 'Spice'
+    url = 'https://www.spicelang.com'
+    filenames = ['*.spice']
+    aliases = ['spice', 'spicelang']
+    mimetypes = ['text/x-spice']
+    version_added = '2.11'
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'\s+', Whitespace),
+            (r'\\\n', Text),
+            # comments
+            (r'//(.*?)\n', Comment.Single),
+            (r'/(\\\n)?[*]{2}(.|\n)*?[*](\\\n)?/', String.Doc),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+            # keywords
+            (r'(import|as)\b', Keyword.Namespace),
+            (r'(f|p|type|struct|interface|enum|alias|operator)\b', Keyword.Declaration),
+            (words(('if', 'else', 'switch', 'case', 'default', 'for', 'foreach', 'do',
+                    'while', 'break', 'continue', 'fallthrough', 'return', 'assert',
+                    'unsafe', 'ext', 'cast'), suffix=r'\b'), Keyword),
+            (words(('const', 'signed', 'unsigned', 'inline', 'public', 'heap', 'compose'),
+                   suffix=r'\b'), Keyword.Pseudo),
+            (words(('new', 'yield', 'stash', 'pick', 'sync', 'class'), suffix=r'\b'),
+                   Keyword.Reserved),
+            (r'(true|false|nil)\b', Keyword.Constant),
+            (words(('double', 'int', 'short', 'long', 'byte', 'char', 'string',
+                    'bool', 'dyn'), suffix=r'\b'), Keyword.Type),
+            (words(('printf', 'sizeof', 'alignof', 'len', 'panic'), suffix=r'\b(\()'),
+             bygroups(Name.Builtin, Punctuation)),
+            # numeric literals
+            (r'[-]?[0-9]*[.][0-9]+([eE][+-]?[0-9]+)?', Number.Double),
+            (r'0[bB][01]+[slu]?', Number.Bin),
+            (r'0[oO][0-7]+[slu]?', Number.Oct),
+            (r'0[xXhH][0-9a-fA-F]+[slu]?', Number.Hex),
+            (r'(0[dD])?[0-9]+[slu]?', Number.Integer),
+            # string literal
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+            # char literal
+            (r'\'(\\\\|\\[^\\]|[^\'\\])\'', String.Char),
+            # tokens
+            (r'<<=|>>=|<<|>>|<=|>=|\+=|-=|\*=|/=|\%=|\|=|&=|\^=|&&|\|\||&|\||'
+             r'\+\+|--|\%|\^|\~|==|!=|->|::|[.]{3}|#!|#|[+\-*/&]', Operator),
+            (r'[|<>=!()\[\]{}.,;:\?]', Punctuation),
+            # identifiers
+            (r'[^\W\d]\w*', Name.Other),
+        ]
+    }
diff --git a/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sql.py b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sql.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d3458e9aefd10bb0ef89ea81f6c006e3f690ec3
--- /dev/null
+++ b/bioagent-bench/.venv/lib/python3.14/site-packages/pygments/lexers/sql.py
@@ -0,0 +1,1111 @@
+"""
+    pygments.lexers.sql
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for various SQL dialects and related interactive sessions.
+
+    Postgres specific lexers:
+
+    `PostgresLexer`
+        A SQL lexer for the PostgreSQL dialect. Differences w.r.t. the SQL
+        lexer are:
+
+        - keywords and data types list parsed from the PG docs (run the
+          `_postgres_builtins` module to update them);
+        - Content of $-strings parsed using a specific lexer, e.g. the content
+          of a PL/Python function is parsed using the Python lexer;
+        - parse PG specific constructs: E-strings, $-strings, U&-strings,
+          different operators and punctuation.
+
+    `PlPgsqlLexer`
+        A lexer for the PL/pgSQL language. Adds a few specific construct on
+        top of the PG SQL lexer (such as <

' : '\U0001d4ab', + '\\' : '\U0001d4ac', + '\\' : '\U0000211b', + '\\' : '\U0001d4ae', + '\\' : '\U0001d4af', + '\\' : '\U0001d4b0', + '\\' : '\U0001d4b1', + '\\' : '\U0001d4b2', + '\\' : '\U0001d4b3', + '\\' : '\U0001d4b4', + '\\' : '\U0001d4b5', + '\\' : '\U0001d5ba', + '\\' : '\U0001d5bb', + '\\' : '\U0001d5bc', + '\\' : '\U0001d5bd', + '\\' : '\U0001d5be', + '\\' : '\U0001d5bf', + '\\' : '\U0001d5c0', + '\\' : '\U0001d5c1', + '\\' : '\U0001d5c2', + '\\' : '\U0001d5c3', + '\\' : '\U0001d5c4', + '\\' : '\U0001d5c5', + '\\' : '\U0001d5c6', + '\\' : '\U0001d5c7', + '\\' : '\U0001d5c8', + '\\