diff --git a/.gitattributes b/.gitattributes index c10dbc585a852f98aad7ce2ef247df3bf9de5c51..f1265c26f63aacc95f72374a68ae4e7a5642a154 100644 --- a/.gitattributes +++ b/.gitattributes @@ -52,3 +52,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text .venv/lib/python3.13/site-packages/ada92cb5d92a588d1b93__mypyc.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text .venv/lib/python3.13/site-packages/cryptography/hazmat/bindings/_rust.abi3.so filter=lfs diff=lfs merge=lfs -text .venv/lib/python3.13/site-packages/hf_xet/hf_xet.abi3.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.13/site-packages/lxml/_elementpath.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.13/site-packages/lxml/builder.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.13/site-packages/lxml/etree.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.13/site-packages/lxml/html/_difflib.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.13/site-packages/lxml/html/diff.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.13/site-packages/lxml/objectify.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.13/site-packages/lxml/sax.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.13/site-packages/numpy/_core/_multiarray_tests.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.13/site-packages/numpy/_core/_multiarray_umath.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text +.venv/lib/python3.13/site-packages/numpy/_core/_simd.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/INSTALLER b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/METADATA b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..688fad4f58f2383aa1640eb846efa5d0b5d98bda --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/METADATA @@ -0,0 +1,373 @@ +Metadata-Version: 2.4 +Name: huggingface_hub +Version: 1.27.0 +Summary: Client library to download and publish models, datasets and other repos on the huggingface.co hub +Home-page: https://github.com/huggingface/huggingface_hub +Author: Hugging Face, Inc. +Author-email: julien@huggingface.co +License: Apache-2.0 +Keywords: model-hub machine-learning models natural-language-processing deep-learning pytorch pretrained-models +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Education +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence +Requires-Python: >=3.10.0 +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: click<9.0.0,>=8.4.2 +Requires-Dist: filelock>=3.10.0 +Requires-Dist: fsspec>=2023.5.0 +Requires-Dist: hf-xet<2.0.0,>=1.5.2; platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "arm64" or platform_machine == "aarch64" +Requires-Dist: httpx<1,>=0.23.0 +Requires-Dist: packaging>=20.9 +Requires-Dist: pyyaml>=5.1 +Requires-Dist: tqdm>=4.42.1 +Requires-Dist: typing-extensions>=4.1.0 +Provides-Extra: oauth +Requires-Dist: authlib>=1.3.2; extra == "oauth" +Requires-Dist: fastapi; extra == "oauth" +Requires-Dist: httpx; extra == "oauth" +Requires-Dist: itsdangerous; extra == "oauth" +Provides-Extra: torch +Requires-Dist: torch; extra == "torch" +Requires-Dist: safetensors[torch]; extra == "torch" +Provides-Extra: fastai +Requires-Dist: toml; extra == "fastai" +Requires-Dist: fastai>=2.4; extra == "fastai" +Requires-Dist: fastcore>=1.3.27; extra == "fastai" +Provides-Extra: hf-xet +Requires-Dist: hf-xet<2.0.0,>=1.5.2; extra == "hf-xet" +Provides-Extra: mcp +Requires-Dist: mcp>=1.8.0; extra == "mcp" +Provides-Extra: testing +Requires-Dist: authlib>=1.3.2; extra == "testing" +Requires-Dist: fastapi; extra == "testing" +Requires-Dist: httpx; extra == "testing" +Requires-Dist: itsdangerous; extra == "testing" +Requires-Dist: jedi; extra == "testing" +Requires-Dist: Jinja2; extra == "testing" +Requires-Dist: pytest>=8.4.2; extra == "testing" +Requires-Dist: pytest-cov; extra == "testing" +Requires-Dist: pytest-env; extra == "testing" +Requires-Dist: pytest-xdist; extra == "testing" +Requires-Dist: pytest-vcr; extra == "testing" +Requires-Dist: pytest-asyncio; extra == "testing" +Requires-Dist: pytest-rerunfailures>=16.2; extra == "testing" +Requires-Dist: pytest-mock; extra == "testing" +Requires-Dist: urllib3<2.0; extra == "testing" +Requires-Dist: soundfile; extra == "testing" +Requires-Dist: Pillow; extra == "testing" +Requires-Dist: numpy; extra == "testing" +Requires-Dist: duckdb; extra == "testing" +Requires-Dist: fastapi; extra == "testing" +Provides-Extra: gradio +Requires-Dist: gradio>=5.0.0; extra == "gradio" +Requires-Dist: requests; extra == "gradio" +Provides-Extra: typing +Requires-Dist: typing-extensions>=4.8.0; extra == "typing" +Requires-Dist: types-PyYAML; extra == "typing" +Requires-Dist: types-simplejson; extra == "typing" +Requires-Dist: types-toml; extra == "typing" +Requires-Dist: types-tqdm; extra == "typing" +Requires-Dist: types-urllib3; extra == "typing" +Provides-Extra: quality +Requires-Dist: ruff>=0.9.0; extra == "quality" +Requires-Dist: mypy==1.15.0; extra == "quality" +Requires-Dist: libcst>=1.4.0; extra == "quality" +Requires-Dist: ty; extra == "quality" +Provides-Extra: all +Requires-Dist: authlib>=1.3.2; extra == "all" +Requires-Dist: fastapi; extra == "all" +Requires-Dist: httpx; extra == "all" +Requires-Dist: itsdangerous; extra == "all" +Requires-Dist: jedi; extra == "all" +Requires-Dist: Jinja2; extra == "all" +Requires-Dist: pytest>=8.4.2; extra == "all" +Requires-Dist: pytest-cov; extra == "all" +Requires-Dist: pytest-env; extra == "all" +Requires-Dist: pytest-xdist; extra == "all" +Requires-Dist: pytest-vcr; extra == "all" +Requires-Dist: pytest-asyncio; extra == "all" +Requires-Dist: pytest-rerunfailures>=16.2; extra == "all" +Requires-Dist: pytest-mock; extra == "all" +Requires-Dist: urllib3<2.0; extra == "all" +Requires-Dist: soundfile; extra == "all" +Requires-Dist: Pillow; extra == "all" +Requires-Dist: numpy; extra == "all" +Requires-Dist: duckdb; extra == "all" +Requires-Dist: fastapi; extra == "all" +Requires-Dist: ruff>=0.9.0; extra == "all" +Requires-Dist: mypy==1.15.0; extra == "all" +Requires-Dist: libcst>=1.4.0; extra == "all" +Requires-Dist: ty; extra == "all" +Requires-Dist: typing-extensions>=4.8.0; extra == "all" +Requires-Dist: types-PyYAML; extra == "all" +Requires-Dist: types-simplejson; extra == "all" +Requires-Dist: types-toml; extra == "all" +Requires-Dist: types-tqdm; extra == "all" +Requires-Dist: types-urllib3; extra == "all" +Provides-Extra: dev +Requires-Dist: authlib>=1.3.2; extra == "dev" +Requires-Dist: fastapi; extra == "dev" +Requires-Dist: httpx; extra == "dev" +Requires-Dist: itsdangerous; extra == "dev" +Requires-Dist: jedi; extra == "dev" +Requires-Dist: Jinja2; extra == "dev" +Requires-Dist: pytest>=8.4.2; extra == "dev" +Requires-Dist: pytest-cov; extra == "dev" +Requires-Dist: pytest-env; extra == "dev" +Requires-Dist: pytest-xdist; extra == "dev" +Requires-Dist: pytest-vcr; extra == "dev" +Requires-Dist: pytest-asyncio; extra == "dev" +Requires-Dist: pytest-rerunfailures>=16.2; extra == "dev" +Requires-Dist: pytest-mock; extra == "dev" +Requires-Dist: urllib3<2.0; extra == "dev" +Requires-Dist: soundfile; extra == "dev" +Requires-Dist: Pillow; extra == "dev" +Requires-Dist: numpy; extra == "dev" +Requires-Dist: duckdb; extra == "dev" +Requires-Dist: fastapi; extra == "dev" +Requires-Dist: ruff>=0.9.0; extra == "dev" +Requires-Dist: mypy==1.15.0; extra == "dev" +Requires-Dist: libcst>=1.4.0; extra == "dev" +Requires-Dist: ty; extra == "dev" +Requires-Dist: typing-extensions>=4.8.0; extra == "dev" +Requires-Dist: types-PyYAML; extra == "dev" +Requires-Dist: types-simplejson; extra == "dev" +Requires-Dist: types-toml; extra == "dev" +Requires-Dist: types-tqdm; extra == "dev" +Requires-Dist: types-urllib3; extra == "dev" +Dynamic: author +Dynamic: author-email +Dynamic: classifier +Dynamic: description +Dynamic: description-content-type +Dynamic: home-page +Dynamic: keywords +Dynamic: license +Dynamic: license-file +Dynamic: provides-extra +Dynamic: requires-dist +Dynamic: requires-python +Dynamic: summary + +

+ + + + huggingface_hub library logo + +
+
+

+ +

+ The official CLI and Python client for the Hugging Face Hub. +
+ About + · + Documentation + · + Install + · + CLI Guide + · + Contributing +

+ +

+ Documentation + GitHub release + PyPi version + PyPI - Downloads + Code coverage +

+ +

+

+ English | + Deutsch | + Français | + हिंदी | + 한국어 | + 中文 (简体) | + ಕನ್ನಡ +

+

+ +## Quick start + +Install the [`hf` CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli) with the standalone installer: + +```bash +# On macOS and Linux. +curl -LsSf https://hf.co/cli/install.sh | bash +``` + +```powershell +# On Windows. +powershell -ExecutionPolicy ByPass -c "irm https://hf.co/cli/install.ps1 | iex" +``` + +Log in, then start working with the Hub: + +```bash +# Log in (use --token $HF_TOKEN in non-interactive environments) +hf auth login + +# Find models served by Inference Providers +hf models ls --warm + +# Download a model +hf download Qwen/Qwen3-0.6B + +# Upload files to your own repo +hf upload username/my-cool-model ./model.safetensors + +# Sync a local folder to a storage bucket +hf buckets sync ./checkpoints hf://buckets/username/my-bucket + +# Run a job on Hugging Face infrastructure +hf jobs run python:3.12 python -c "print('Hello from the cloud!')" + +# Discover everything else +hf --help +``` + +The Hub uses tokens to authenticate applications (see [docs](https://huggingface.co/docs/hub/security-tokens)). Check out the [CLI guide](https://huggingface.co/docs/huggingface_hub/en/guides/cli) for a tour of the main features. + +## What is `huggingface_hub`? + +The `huggingface_hub` library allows you to interact with the [Hugging Face Hub](https://huggingface.co/), a platform democratizing open-source Machine Learning for creators and collaborators. Discover pre-trained models and datasets for your projects, play with the thousands of machine learning apps hosted on the Hub, or create and share your own models, datasets and demos with the community. Everything ships in one package with two interfaces: the [`hf` CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli) for your terminal and the `huggingface_hub` library for Python — both designed to work well for humans and AI agents. Use them to: + +- [Download files](https://huggingface.co/docs/huggingface_hub/en/guides/download) from the Hub. +- [Upload files](https://huggingface.co/docs/huggingface_hub/en/guides/upload) to the Hub. +- [Manage your repositories](https://huggingface.co/docs/huggingface_hub/en/guides/repository). +- [Run Inference](https://huggingface.co/docs/huggingface_hub/en/guides/inference) on deployed models. +- [Run Jobs](https://huggingface.co/docs/huggingface_hub/en/guides/jobs) on Hugging Face infrastructure. +- [Search](https://huggingface.co/docs/huggingface_hub/en/guides/search) for models, datasets and Spaces. +- [Share Model Cards](https://huggingface.co/docs/huggingface_hub/en/guides/model-cards) to document your models. +- [Engage with the community](https://huggingface.co/docs/huggingface_hub/en/guides/community) through PRs and comments. +- Do all of the above from the terminal with the [`hf` CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli). + +## Built for humans and AI agents + +The `hf` CLI is designed for people and coding agents alike: the same commands adapt their output when run by an agent. If you use Claude Code, Codex, Cursor, or another coding agent, install the `hf` CLI Skill — a command reference generated from your installed CLI: + +```bash +# for Codex, Cursor, OpenCode, Pi and other agents that load skills from `.agents/skills` +hf skills add +# includes the above + Claude Code +hf skills add --claude +``` + +Learn more in the [Hugging Face CLI for AI agents guide](https://huggingface.co/docs/hub/agents-cli) and the [announcement blog post](https://huggingface.co/blog/hf-cli-for-agents). + +## Use the Python library + +Install the `huggingface_hub` package with [pip](https://pypi.org/project/huggingface-hub/) (this also installs the `hf` CLI): + +```bash +pip install huggingface_hub +``` + +We recommend using [`uv`](https://docs.astral.sh/uv/) for a fast and reliable install: + +```bash +uv pip install huggingface_hub +``` + +In order to keep the package minimal by default, `huggingface_hub` comes with optional dependencies useful for some use cases. For example, if you want to use the MCP module, run: + +```bash +pip install "huggingface_hub[mcp]" +``` + +To learn more about installation and optional dependencies, check out the [installation guide](https://huggingface.co/docs/huggingface_hub/en/installation). + +### Download files + +Download a single file + +```py +from huggingface_hub import hf_hub_download + +hf_hub_download(repo_id="zai-org/GLM-5.2", filename="config.json") +``` + +Or an entire repository + +```py +from huggingface_hub import snapshot_download + +snapshot_download("sentence-transformers/all-MiniLM-L6-v2") +``` + +Files will be downloaded in a local cache folder. More details in [this guide](https://huggingface.co/docs/huggingface_hub/en/guides/manage-cache). + +### Create a repository + +```py +from huggingface_hub import create_repo + +create_repo(repo_id="super-cool-model") +``` + +### Upload files + +Upload a single file + +```py +from huggingface_hub import upload_file + +upload_file( + path_or_fileobj="/home/lysandre/dummy-test/README.md", + path_in_repo="README.md", + repo_id="lysandre/test-model", +) +``` + +Or an entire folder + +```py +from huggingface_hub import upload_folder + +upload_folder( + folder_path="/path/to/local/space", + repo_id="username/my-cool-space", + repo_type="space", +) +``` + +More details in the [upload guide](https://huggingface.co/docs/huggingface_hub/en/guides/upload). + +## Integrating with the Hub. + +We're partnering with cool open source ML libraries to provide free model hosting and versioning. You can find the existing integrations [here](https://huggingface.co/docs/hub/libraries). + +The advantages are: + +- Free model or dataset hosting for libraries and their users. +- Built-in file versioning, even with very large files, made possible by [Xet](https://huggingface.co/docs/hub/xet/index), the Hub's chunk-deduplicated storage backend. +- In-browser widgets to play with the uploaded models. +- Anyone can upload a new model for your library, they just need to add the corresponding tag for the model to be discoverable. +- Fast downloads! We use Cloudfront (a CDN) to geo-replicate downloads so they're blazing fast from anywhere on the globe. +- Usage stats and more features to come. + +If you would like to integrate your library, feel free to open an issue to begin the discussion. We wrote a [step-by-step guide](https://huggingface.co/docs/hub/adding-a-library) with ❤️ showing how to do this integration. + +## Contributions (feature requests, bugs, etc.) are super welcome 💙💚💛💜🧡❤️ + +Everyone is welcome to contribute, and we value everybody's contribution. Code is not the only way to help the community. +Answering questions, helping others, reaching out and improving the documentations are immensely valuable to the community. +We wrote a [contribution guide](https://github.com/huggingface/huggingface_hub/blob/main/CONTRIBUTING.md) to summarize +how to get started to contribute to this repository. diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/RECORD b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..473855f937bb2c320d2250daacb913ecbaba5dd7 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/RECORD @@ -0,0 +1,380 @@ +../../../bin/hf,sha256=JQijLHciXBpOWGr6WRb8N0z29GfP0tPh2NA2HNavJvo,273 +../../../bin/huggingface-cli,sha256=nNpWnVatSJHRXWFCuXQb_WZFtQ_HulgkcuNiCSf64xo,285 +../../../bin/tiny-agents,sha256=0CQNfsvoFKWfo0ngJRPM94Fb2eOQaPFL-J1-Oblf3iA,283 +huggingface_hub-1.27.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +huggingface_hub-1.27.0.dist-info/METADATA,sha256=kdWT-K9bIzIoJpph_rdX1f5gcwxRWXkZ5PTijS2m2co,16314 +huggingface_hub-1.27.0.dist-info/RECORD,, +huggingface_hub-1.27.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +huggingface_hub-1.27.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91 +huggingface_hub-1.27.0.dist-info/entry_points.txt,sha256=zP7F_bBSdircPQFysHQZ9F3Lcn5_dCSOEZxVlGCsG0w,212 +huggingface_hub-1.27.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357 +huggingface_hub-1.27.0.dist-info/top_level.txt,sha256=8KzlQJAY4miUvjAssOAJodqKOw3harNzuiwGQ9qLSSk,16 +huggingface_hub/__init__.py,sha256=eZLm_W9SknOwodC709pz-SVyWudPEfO0-yrtIlL1FMc,60623 +huggingface_hub/__pycache__/__init__.cpython-313.pyc,, +huggingface_hub/__pycache__/_buckets.cpython-313.pyc,, +huggingface_hub/__pycache__/_commit_api.cpython-313.pyc,, +huggingface_hub/__pycache__/_commit_scheduler.cpython-313.pyc,, +huggingface_hub/__pycache__/_dataset_viewer.cpython-313.pyc,, +huggingface_hub/__pycache__/_eval_results.cpython-313.pyc,, +huggingface_hub/__pycache__/_inference_endpoints.cpython-313.pyc,, +huggingface_hub/__pycache__/_jobs_api.cpython-313.pyc,, +huggingface_hub/__pycache__/_local_folder.cpython-313.pyc,, +huggingface_hub/__pycache__/_login.cpython-313.pyc,, +huggingface_hub/__pycache__/_oauth.cpython-313.pyc,, +huggingface_hub/__pycache__/_oidc.cpython-313.pyc,, +huggingface_hub/__pycache__/_revision.cpython-313.pyc,, +huggingface_hub/__pycache__/_sandbox.cpython-313.pyc,, +huggingface_hub/__pycache__/_sandbox_cache.cpython-313.pyc,, +huggingface_hub/__pycache__/_snapshot_download.cpython-313.pyc,, +huggingface_hub/__pycache__/_space_api.cpython-313.pyc,, +huggingface_hub/__pycache__/_tensorboard_logger.cpython-313.pyc,, +huggingface_hub/__pycache__/_tree_cache.cpython-313.pyc,, +huggingface_hub/__pycache__/_upload_large_folder.cpython-313.pyc,, +huggingface_hub/__pycache__/_upload_pipeline.cpython-313.pyc,, +huggingface_hub/__pycache__/_webhooks_payload.cpython-313.pyc,, +huggingface_hub/__pycache__/_webhooks_server.cpython-313.pyc,, +huggingface_hub/__pycache__/community.cpython-313.pyc,, +huggingface_hub/__pycache__/constants.cpython-313.pyc,, +huggingface_hub/__pycache__/dataclasses.cpython-313.pyc,, +huggingface_hub/__pycache__/errors.cpython-313.pyc,, +huggingface_hub/__pycache__/fastai_utils.cpython-313.pyc,, +huggingface_hub/__pycache__/file_download.cpython-313.pyc,, +huggingface_hub/__pycache__/hf_api.cpython-313.pyc,, +huggingface_hub/__pycache__/hf_file_system.cpython-313.pyc,, +huggingface_hub/__pycache__/hub_mixin.cpython-313.pyc,, +huggingface_hub/__pycache__/lfs.cpython-313.pyc,, +huggingface_hub/__pycache__/repocard.cpython-313.pyc,, +huggingface_hub/__pycache__/repocard_data.cpython-313.pyc,, +huggingface_hub/_buckets.py,sha256=XyLbsLzS2fpQz0aAIjKzV-1YbQ4XYZ4-HXQTVq0TXoU,45311 +huggingface_hub/_commit_api.py,sha256=lalqhJKl8_RWVHgAdnzZKG_fy4E7fbyHIev6nDLrj4w,45378 +huggingface_hub/_commit_scheduler.py,sha256=VG3zjb7xb66NrFMq_pB9dWIsoqSPoF6CntovV7vUiWg,14684 +huggingface_hub/_dataset_viewer.py,sha256=LE9k4s0WJ0JNdriJ8L6qV_-xfbUVC1VEh7CBd0jOjLI,5675 +huggingface_hub/_eval_results.py,sha256=Og7p7bCl2HQG6ZWlSDyir-CRTXjNWPu3S3q65cTGffs,8126 +huggingface_hub/_hot_reload/__init__.py,sha256=nQEuyVZLAS_463Dff1JH5MorTpkYsBVFCyPhYWY3UyI,606 +huggingface_hub/_hot_reload/__pycache__/__init__.cpython-313.pyc,, +huggingface_hub/_hot_reload/__pycache__/client.cpython-313.pyc,, +huggingface_hub/_hot_reload/__pycache__/sse_client.cpython-313.pyc,, +huggingface_hub/_hot_reload/__pycache__/types.cpython-313.pyc,, +huggingface_hub/_hot_reload/client.py,sha256=Ax7qT4CUi5wrg3mDzqkAG8ZzDsCwdcIIS1FIXK0SqD8,4269 +huggingface_hub/_hot_reload/sse_client.py,sha256=49kmsB2P9sxgbZ16ed6F4aLDs3rsHrNpbp6iKlvF90c,4948 +huggingface_hub/_hot_reload/types.py,sha256=d3AOJCLUxT41Fhz4jUrFevm9bfpooe_3PQhpBdgNYSs,2725 +huggingface_hub/_inference_endpoints.py,sha256=FWSixNvBQlqLU4Z0yUC7Ch7a8V8Ci6eJBAD4pbTMKhk,18424 +huggingface_hub/_jobs_api.py,sha256=f7NhziUMH6jWpWClnLPFx4XIZ0xVGAdTGfBxIn2IVy0,23679 +huggingface_hub/_local_folder.py,sha256=KsetQw7pW0ZwFI2RLQPqVq0ZnORogPkBwRYcbRqNd5o,20654 +huggingface_hub/_login.py,sha256=wAMdhqMSSh3GBJDozodmk4IIsAvb9J6LaLY9KnXFoyY,21818 +huggingface_hub/_oauth.py,sha256=S8evEd8JLJHz-JiSjGH4WiIi_A7oDox_xH7E9rN1Jic,18628 +huggingface_hub/_oidc.py,sha256=3kcCLavWQ5FQOwizUPzLaBb1ft2FIJo0bSXL24S3r4U,7098 +huggingface_hub/_revision.py,sha256=NCmIVCR-_HjmxeYiDntcsTU-bLSfJLq8PLvvG8UhZtI,1696 +huggingface_hub/_sandbox.py,sha256=XEXZi5S3rYyw_o3td3rB0xOVdmhpAlnyUykBsAw9KP8,80148 +huggingface_hub/_sandbox_cache.py,sha256=UYpvHOWZj7CRhacYs5Ot-5N1wFcAapiAyniYdoX-jX8,5742 +huggingface_hub/_snapshot_download.py,sha256=fOte9xmpaok0LGbubJZhSRd7t0nn-p00mWMebCx56qc,31029 +huggingface_hub/_space_api.py,sha256=MF3ezhgl0Vpt3RP1wuwZ7VI9_rSyIlIJ5SmDzIwQlcQ,13424 +huggingface_hub/_tensorboard_logger.py,sha256=GA-LhoG4Z-0Zq7w5B9vTIDJbWUkAhiG2d8a-ktlwGDY,8377 +huggingface_hub/_tree_cache.py,sha256=M7CHMKhwY1cuhe9fb3WT1X8jmd9BXa2Sx_HS4TIXFgI,5843 +huggingface_hub/_upload_large_folder.py,sha256=BWA1zyOK9W_3YUerX9YWQa3wAYtHRe_ypgsO79JEkqo,31235 +huggingface_hub/_upload_pipeline.py,sha256=6YaQNtubAEaPyRoRBjPQrmsdSnCExg2KzfBuhjNqI5I,28994 +huggingface_hub/_webhooks_payload.py,sha256=B9zqljcm20DGeVxJKba570i6RP-cR9CEupMGBuNDHHA,3544 +huggingface_hub/_webhooks_server.py,sha256=VXOk3FHkbIc28phxknJfaFy5B_EoKOKihoCKyOaWZQk,15767 +huggingface_hub/cli/__init__.py,sha256=tWu3Is-m2tlJwXhM_vT2mPRgpnt9KhU9qvw5aRsnjU0,664 +huggingface_hub/cli/__pycache__/__init__.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/_city_game.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/_cli_utils.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/_completion.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/_cp.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/_errors.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/_file_listing.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/_framework.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/_help_formatter.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/_output.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/_skills.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/auth.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/buckets.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/cache.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/collections.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/datasets.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/deprecated_cli.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/discussions.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/download.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/extensions.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/hf.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/inference_endpoints.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/jobs.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/lfs.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/models.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/papers.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/repo_files.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/repos.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/sandbox.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/skills.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/spaces.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/system.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/upload.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/upload_large_folder.cpython-313.pyc,, +huggingface_hub/cli/__pycache__/webhooks.cpython-313.pyc,, +huggingface_hub/cli/_city_game.py,sha256=2qkC9dCtPwf4-aqrnbN-vApf0YqIcGGjn2v-etrTfqg,20585 +huggingface_hub/cli/_cli_utils.py,sha256=uFCEHDWmxTri9ei1kmN4t4ybdm4wO5Cj0e6DvLWWFao,41824 +huggingface_hub/cli/_completion.py,sha256=BDP6Vob9TLFp6K3lr0XOY98FyD-rm22KrjwsO_YGNbE,3759 +huggingface_hub/cli/_cp.py,sha256=Z-2Z_tBs1e4XBqRO5R9QnabU4qTi9HDlbA52jTBaCkY,10436 +huggingface_hub/cli/_errors.py,sha256=X0rUTO-3eCK5v_MLGEza1BxcuXGccPY-h7ShzmuQc34,5907 +huggingface_hub/cli/_file_listing.py,sha256=h_dTEphSqtvknzqJ_9cIK3Cjlz5M6cycYD3cqUW9Fn0,7986 +huggingface_hub/cli/_framework.py,sha256=HqfAEYGWRSjKABAZi2zVNgryyU9QKW8u-tRXfR_4nQ8,27364 +huggingface_hub/cli/_help_formatter.py,sha256=-vsa3HfHQFkKWms77EGyp9i22t44ETmVt2L-e8qP-KM,1288 +huggingface_hub/cli/_output.py,sha256=OxFrvAcDU1TRDV0AbNh4EI1P6tqifSeVT-yqfF92U_0,14754 +huggingface_hub/cli/_skills.py,sha256=s9Q4o4AIerDyLNuGzUB1ISXANQPgSiAs2CRcAY4NckA,15135 +huggingface_hub/cli/auth.py,sha256=Ew79M60pTMCnS1ewKVK3fd8ZyfHrNjREOmNWuFXy37M,6056 +huggingface_hub/cli/buckets.py,sha256=J8xMP88fQ6ri3f65Z8XEImI2GNA4KKWloLJJQCBezZw,21062 +huggingface_hub/cli/cache.py,sha256=2T0oTYXsrtcJxhQENSpXnYYNUkagsvvdrf5gbCACBjA,28574 +huggingface_hub/cli/collections.py,sha256=Ed41zJqrvK-AWkHdkeHWm_aJKR_ZeNIovCGvWG04itk,10120 +huggingface_hub/cli/datasets.py,sha256=X1xMJTFhHH5NJtmjTS6NN1g4ffai6Abh_obDEJM5mvo,10497 +huggingface_hub/cli/deprecated_cli.py,sha256=vAK_CR4NE9uzNJ0I59SyBnIln5nlypG4CPoohMoPMVg,1090 +huggingface_hub/cli/discussions.py,sha256=obvvN4mtsFnfMs67YTZaMvHkVaM6A-FYr5n-wGkF_jE,14029 +huggingface_hub/cli/download.py,sha256=o6fMKJoNQMxCm1RQUnX3blihJ17mantxkf0F68rZIxs,9669 +huggingface_hub/cli/extensions.py,sha256=qNM1DmzcQXt8kPWQfISc8Pvyku8BxAs-BOVP6sKRBOM,25572 +huggingface_hub/cli/hf.py,sha256=84sV7zWF5SB7Q48FMXpA9XwKsMFGaCAzNklrkV7l-vQ,5549 +huggingface_hub/cli/inference_endpoints.py,sha256=F-td5VCPrzgBZbkRwc6HhasJDrJDbLmreHoe4nTgktA,18820 +huggingface_hub/cli/jobs.py,sha256=EVGuxn2GZqkQolHib98PztJeJ64Opq-MjgIub2WTYb4,52840 +huggingface_hub/cli/lfs.py,sha256=MAccazVPX0n-ySRaPocaQUHP7oBnVrvWZt0RhmNIavs,5936 +huggingface_hub/cli/models.py,sha256=eG7H5yDjCVrdL7W_-wpcl70qXlv8bC-roSEu8AroPws,10148 +huggingface_hub/cli/papers.py,sha256=HvT-8zj2fi14ua6awWpHphlbJ362FEJIwSCBJRQZ_cM,4982 +huggingface_hub/cli/repo_files.py,sha256=sGbD5wND3miF71ThfOBJm4xzPRUpJBeLFdvyj229NAM,2443 +huggingface_hub/cli/repos.py,sha256=QS3xYKvspLhCfO5VvpORrSiF6XhSVFclmBxsacVDP2c,20192 +huggingface_hub/cli/sandbox.py,sha256=rTCZf6WhoDa7UiNhsTU_4Lh5uKqKYOs73xXGAAqJGdA,18379 +huggingface_hub/cli/skills.py,sha256=PcZsDiB8XC_-98c9Pw5upmof4FAAlMQPZUmRT0T8kCQ,19926 +huggingface_hub/cli/spaces.py,sha256=AU655GGrQmF1s1yO8yO6ml2Z8veE7zsh4UMuR5b13RI,40800 +huggingface_hub/cli/system.py,sha256=8_xaa-GLGp5tC-slv7cuFnpUQT3zozevsftYGlvFTgs,3071 +huggingface_hub/cli/upload.py,sha256=P6sq8Dj81a2wazUgZEqbliRleVAAmJrf8ajFDDzHrBE,11318 +huggingface_hub/cli/upload_large_folder.py,sha256=A-2qxAdLrYd8IogfZS2nDhSR1cpA_aaxhXst638WkAk,4081 +huggingface_hub/cli/webhooks.py,sha256=HdpPp01c6Ece_EuNNJr46rnRki3vF4j4o9-SqYRUHnM,8550 +huggingface_hub/community.py,sha256=N5OE5Rbc2BMwpNXCFOPQAwn0otqQunsXdEnSE_0J2Xo,12325 +huggingface_hub/constants.py,sha256=leJzcvH7fkoIpb8LLLd21gB_j71AWSonum0MRV18a6c,13792 +huggingface_hub/dataclasses.py,sha256=dkPHOKq1YVlto9TWMkeX8BnErmbZk6tVtqD1S_n6Cw8,25896 +huggingface_hub/errors.py,sha256=MsfOFq6Sxo2bZ8PTPWbQb250v7aqZJO5-mmWSnX_kDk,18695 +huggingface_hub/fastai_utils.py,sha256=jwKeVXH_VP0zxnOKVNQCVCkF-stH70l-bExbbrYkBKQ,16583 +huggingface_hub/file_download.py,sha256=3IPbCapgC-tvlh9ErLCzA-RerYXR_1XIhDKvUOjHvKA,89165 +huggingface_hub/hf_api.py,sha256=RNpf-DpBdBQQ5-lYjB8APlRaCOIcSK4lWUTMkvQiPcA,673285 +huggingface_hub/hf_file_system.py,sha256=3_tdhCz6IaLVbcfk_tpbQUd7VfndHyOmbI_6q4Lq36o,60869 +huggingface_hub/hub_mixin.py,sha256=ljKAtm3qQsY-ARtxY-PCFx0koFXLI37JytMrzpqfnQo,36912 +huggingface_hub/inference/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +huggingface_hub/inference/__pycache__/__init__.cpython-313.pyc,, +huggingface_hub/inference/__pycache__/_client.cpython-313.pyc,, +huggingface_hub/inference/__pycache__/_common.cpython-313.pyc,, +huggingface_hub/inference/_client.py,sha256=mENW3_gqgJR_9PuC7dwFZYjkQETPvrIrw8rU74PeusQ,157692 +huggingface_hub/inference/_common.py,sha256=X-fBP8s6xuNTAuQsGVIethE8dNQaa-mK7Ds54qVVDz8,14999 +huggingface_hub/inference/_generated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +huggingface_hub/inference/_generated/__pycache__/__init__.cpython-313.pyc,, +huggingface_hub/inference/_generated/__pycache__/_async_client.cpython-313.pyc,, +huggingface_hub/inference/_generated/_async_client.py,sha256=MQNKNIqcgqpGvL-HskuX1yJV6QDDuUEjM0mtZAZv8ss,160903 +huggingface_hub/inference/_generated/types/__init__.py,sha256=e625ENOmTUM4uUL1eAlsZOK7CovNoGBDH-TxSNQhaC4,6869 +huggingface_hub/inference/_generated/types/__pycache__/__init__.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/audio_classification.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/audio_to_audio.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/automatic_speech_recognition.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/base.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/chat_completion.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/depth_estimation.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/document_question_answering.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/feature_extraction.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/fill_mask.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/image_classification.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/image_segmentation.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/image_text_to_image.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/image_text_to_video.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/image_to_image.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/image_to_text.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/image_to_video.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/object_detection.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/question_answering.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/sentence_similarity.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/summarization.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/table_question_answering.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/text2text_generation.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/text_classification.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/text_generation.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/text_to_audio.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/text_to_image.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/text_to_speech.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/text_to_video.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/token_classification.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/translation.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/video_classification.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/visual_question_answering.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/zero_shot_classification.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/zero_shot_image_classification.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/__pycache__/zero_shot_object_detection.cpython-313.pyc,, +huggingface_hub/inference/_generated/types/audio_classification.py,sha256=DGWEvrRPX7xpXlbYY4LMojGcULdEZFTshhNR81rTNcQ,1567 +huggingface_hub/inference/_generated/types/audio_to_audio.py,sha256=2Ep4WkePL7oJwcp5nRJqApwviumGHbft9HhXE9XLHj4,891 +huggingface_hub/inference/_generated/types/automatic_speech_recognition.py,sha256=X0uCj9qwAcOmlKtV3vWH5cAgLGY0PuYIy2eSH7GE1qs,5439 +huggingface_hub/inference/_generated/types/base.py,sha256=EPWp1qudjeEimEAyQ-ptdxpAACoRM8tW342-Ds7pP3I,6938 +huggingface_hub/inference/_generated/types/chat_completion.py,sha256=3yc06ZI0NCZzsaeoxSMWyIHqGc8Vn_6aExncuXa1llE,11100 +huggingface_hub/inference/_generated/types/depth_estimation.py,sha256=5Kn4xNGJg235Cofr1lzO59eRGCuLTZun8z5t6a2do8Q,910 +huggingface_hub/inference/_generated/types/document_question_answering.py,sha256=aUZcrDGexjw8Fnm2rACm_3my77V15NnQHZxj-sa7bjo,3146 +huggingface_hub/inference/_generated/types/feature_extraction.py,sha256=DiO47PqwhdNta_KPDBsXfCPIL6BWNhyCLymxORlqUA0,1509 +huggingface_hub/inference/_generated/types/fill_mask.py,sha256=NhEskgoiMst7U_0cV_S-wryga4_2T5LgpbyQ23vHMp0,1680 +huggingface_hub/inference/_generated/types/image_classification.py,sha256=xWr4OY65EcIIhd7MI610a7xlaAVHQeydi8Gc7tlaPTU,1579 +huggingface_hub/inference/_generated/types/image_segmentation.py,sha256=lZKzbn_gkSi1sYGLb9ryeRr0aDAVrfYh0H5Iiv5JNsY,1935 +huggingface_hub/inference/_generated/types/image_text_to_image.py,sha256=pRQgYKXJbIpEjRBxuIE6NRDtk_km9DSTQ2s2qxkYJRM,2585 +huggingface_hub/inference/_generated/types/image_text_to_video.py,sha256=MVhP8D4eYjpT3hWYkkpuiME-hWY09o7C4rIn_wmnAmY,2450 +huggingface_hub/inference/_generated/types/image_to_image.py,sha256=P9TeMFSnZI_pa6DyinzDPWL0PNm_INJwsqFUEzpwIcU,2262 +huggingface_hub/inference/_generated/types/image_to_text.py,sha256=DCDS0Nqi-4AuATitvZm4EgrzYQuQEQ5lzBcaHuCmHnY,4740 +huggingface_hub/inference/_generated/types/image_to_video.py,sha256=iMCjiFTUza9AW_MyJ1videhYgvMY9QSzQ59kNdTE9QY,2206 +huggingface_hub/inference/_generated/types/object_detection.py,sha256=zNILlyZJh59rNNfqXM5x9uprDCPfEU-bvBtabeKSMzI,1965 +huggingface_hub/inference/_generated/types/question_answering.py,sha256=lIRf8DjRiAsjzWUBEsGt354SB4bvejLokf-8luHw6_o,2845 +huggingface_hub/inference/_generated/types/sentence_similarity.py,sha256=eNGiO4dR-Lft9yOLE0E97jfVyOT_ExDWZW5GeJGdatU,1027 +huggingface_hub/inference/_generated/types/summarization.py,sha256=5Dpt6LRteT-hO9esa3Ty_-xSL5U-zTtVZzy3iKwZRng,1472 +huggingface_hub/inference/_generated/types/table_question_answering.py,sha256=bwn616XJ-eLa5JKzn3m9rZZ3z63INGEUwRSOCWlEh-4,2269 +huggingface_hub/inference/_generated/types/text2text_generation.py,sha256=fotSlPWtIlp_LG26AQvNRXxU-jCUfda5PWsxCkbFeKM,1591 +huggingface_hub/inference/_generated/types/text_classification.py,sha256=uamBzz4YzWQ9mb6yuriykX8r47ZHf0K3eJpOLzB2qJw,1439 +huggingface_hub/inference/_generated/types/text_generation.py,sha256=FRUljypvCSQ2RaiylhlHeLJfHNy8qZvuty6SCB_zcaA,5813 +huggingface_hub/inference/_generated/types/text_to_audio.py,sha256=CiaTUSBdFtOs3gGTosqyQrjCwrN8mVJLRxDDWUtwZpQ,4677 +huggingface_hub/inference/_generated/types/text_to_image.py,sha256=Ju25WIDHE-1n8jlZ_udG4rpAaPYpmghR7-cDgVfuf48,1869 +huggingface_hub/inference/_generated/types/text_to_speech.py,sha256=WqDzWkhZsDcjapkoFrkyjF903SKs7aHhjypsYrn8oBw,4693 +huggingface_hub/inference/_generated/types/text_to_video.py,sha256=KjcDbtChwrIxIdTH6vcesUaMcjL2xjPf2nWEnHazj4Y,1756 +huggingface_hub/inference/_generated/types/token_classification.py,sha256=8KRBlqgFDszaUnPMvdRkZNQaZBtDfBhKZ6SggQldQMM,1894 +huggingface_hub/inference/_generated/types/translation.py,sha256=-j2PC1-qfoxvGMaKesqf5jXelqj0BDlEpfvZ7dsIEJM,1742 +huggingface_hub/inference/_generated/types/video_classification.py,sha256=vfaCqp8MuRMMtGvYCX0x4-AcwEd71kPHgSYJTNIOxLQ,1668 +huggingface_hub/inference/_generated/types/visual_question_answering.py,sha256=sWIB-SuEpW21oU6O5qQF3tt5PavUFsFOa8Xg4upTtek,1654 +huggingface_hub/inference/_generated/types/zero_shot_classification.py,sha256=gx-7WHad7xsNa89iSKSteKHjaHrKbIyzN_nSdJN3HCY,1697 +huggingface_hub/inference/_generated/types/zero_shot_image_classification.py,sha256=JZTQvcbSGrOQlWcPuJvAMT8YziDpBRn1O2RQSw507uY,1449 +huggingface_hub/inference/_generated/types/zero_shot_object_detection.py,sha256=sjdpVUN5zW9aYBymLVUs6i5HVk2qkUBO9ysEjHmsXVM,1605 +huggingface_hub/inference/_mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +huggingface_hub/inference/_mcp/__pycache__/__init__.cpython-313.pyc,, +huggingface_hub/inference/_mcp/__pycache__/_cli_hacks.cpython-313.pyc,, +huggingface_hub/inference/_mcp/__pycache__/agent.cpython-313.pyc,, +huggingface_hub/inference/_mcp/__pycache__/cli.cpython-313.pyc,, +huggingface_hub/inference/_mcp/__pycache__/constants.cpython-313.pyc,, +huggingface_hub/inference/_mcp/__pycache__/mcp_client.cpython-313.pyc,, +huggingface_hub/inference/_mcp/__pycache__/types.cpython-313.pyc,, +huggingface_hub/inference/_mcp/__pycache__/utils.cpython-313.pyc,, +huggingface_hub/inference/_mcp/_cli_hacks.py,sha256=kIA8FfRfOWS3SrZrMTbuVOjE6yxfKfMK75Co5egTWHA,3284 +huggingface_hub/inference/_mcp/agent.py,sha256=ufIzMGHore5n252hV5GZPM0ouDXIl6tv5Jl_5gHXnbg,4250 +huggingface_hub/inference/_mcp/cli.py,sha256=RKdVQ_VdEgf1AL_QBCvwrsUQ0Ux4R1yerrE46DrV5C0,9642 +huggingface_hub/inference/_mcp/constants.py,sha256=0Ih6ivEvG-pfF8NwErFngzP4Rof4lQoz6ms_eKSAkUI,2463 +huggingface_hub/inference/_mcp/mcp_client.py,sha256=dGp8PhN6aVw4bDnuSySFSiguHUiz-nzhgv89CVdO7pI,17243 +huggingface_hub/inference/_mcp/types.py,sha256=yHNfPsM9MhD06oeKdkbmrBsW-3WhUeqA26fyfRfx_bk,929 +huggingface_hub/inference/_mcp/utils.py,sha256=gxSB_rBjQ6VrkApKFsxk6-UzhijxkDVNuZrsjW5pL8k,4318 +huggingface_hub/inference/_providers/__init__.py,sha256=RSm8iJhLmkIvzUdAR9FGwOYuuWauK2RKVnZH_YHnK7s,9979 +huggingface_hub/inference/_providers/__pycache__/__init__.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/_common.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/baseten.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/cerebras.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/cohere.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/deepinfra.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/fal_ai.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/featherless_ai.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/fireworks_ai.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/groq.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/hf_inference.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/novita.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/nscale.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/openai.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/ovhcloud.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/publicai.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/replicate.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/scaleway.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/together.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/wavespeed.cpython-313.pyc,, +huggingface_hub/inference/_providers/__pycache__/zai_org.cpython-313.pyc,, +huggingface_hub/inference/_providers/_common.py,sha256=T3iKT6McNs7GeVfK3q-l7CBbuih2mTFcqkRluQn7GnQ,14308 +huggingface_hub/inference/_providers/baseten.py,sha256=1uucLZp6HiF8OjfeFOkiuJP6WqunP_xzwk3yWPF8vGE,213 +huggingface_hub/inference/_providers/cerebras.py,sha256=QOJ-1U-os7uE7p6eUnn_P_APq-yQhx28be7c3Tq2EuA,210 +huggingface_hub/inference/_providers/cohere.py,sha256=P9kbIuvQ2rXI1yNmgbw5VKFFTE0huLq2k-BCyDkDico,1226 +huggingface_hub/inference/_providers/deepinfra.py,sha256=4Behgf5V7FTv7CbN6vPEIYwv1z4gztdRRU-KVoOgxVE,4703 +huggingface_hub/inference/_providers/fal_ai.py,sha256=Y2vo5Dl3e8EoehplwNshu8LbDT1e7V4ZyhAMqs0wXws,11705 +huggingface_hub/inference/_providers/featherless_ai.py,sha256=C8OHdFpoFyA--pawLTekUmUSRq4sw_r0D-cSPekD4kE,1347 +huggingface_hub/inference/_providers/fireworks_ai.py,sha256=go6XPum8u0-g768HJ_r0S4Gb445gZJxnuY_acktz-9c,1188 +huggingface_hub/inference/_providers/groq.py,sha256=JTk2JV4ZOlaohho7zLAFQtk92kGVsPmLJ1hmzcwsqvQ,315 +huggingface_hub/inference/_providers/hf_inference.py,sha256=z-NoYyjppc2iBPh6uVGMDikIrhnW8-dD1CJHo665ko0,9803 +huggingface_hub/inference/_providers/novita.py,sha256=ATEoSdPAPLMfx3JpBc0sOyLh4upJYP6xHBuo4YEDYvg,2470 +huggingface_hub/inference/_providers/nscale.py,sha256=T1L2JLI9LqYT9_YEdW76YEtxfDNtdGkoFhH-p2LmSxg,1767 +huggingface_hub/inference/_providers/openai.py,sha256=wwjaaQ55xLmDsDnHpZk52xbuLoGZfWzJkFsE8AdFVaI,1054 +huggingface_hub/inference/_providers/ovhcloud.py,sha256=tdmymlkbddMJKV7NRZ-tH2wymbLPFDTqUSXpWJUXyDQ,314 +huggingface_hub/inference/_providers/publicai.py,sha256=1I2W6rORloB5QHSvky4njZO2XKLTwA-kPdNoauoT5rg,210 +huggingface_hub/inference/_providers/replicate.py,sha256=hte0ZB2RtGFwpAuLrFp2Gbgsn3EOlccCnJkWTDTW__A,6027 +huggingface_hub/inference/_providers/scaleway.py,sha256=MfIc7ZND1sPr__rOmNHZVg0VpECQo0bVyksgIm_32xQ,1174 +huggingface_hub/inference/_providers/together.py,sha256=lrymDZ86n5yoR_Zz7o3AzzSePyo0s7BOBU5digLxb5o,12220 +huggingface_hub/inference/_providers/wavespeed.py,sha256=MGM7Y7r2nQiH_EH0t5UE1o1fT-o3sS7RaQlicS9WHsg,5028 +huggingface_hub/inference/_providers/zai_org.py,sha256=gLJZOEmCPmUZvdM7VDn2nxm4ac3veHdoMgQNR63UeWE,4739 +huggingface_hub/lfs.py,sha256=8ymYNHRXgvgUUM9w0EAnC-vkSOOq_1EFJEJNoevoeE8,16588 +huggingface_hub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +huggingface_hub/repocard.py,sha256=7u7hiKjNuPYOwa3XPOPUnYPpuv4buya4WM47kVexklY,35602 +huggingface_hub/repocard_data.py,sha256=f834MUSXGqrVcC02668r3ACrPzKKOAg75y3aB_hXm1U,34308 +huggingface_hub/serialization/__init__.py,sha256=jCiw_vVQYW52gwVfWiqgocf2Q19kGTQlRGVpf-4SLP8,963 +huggingface_hub/serialization/__pycache__/__init__.cpython-313.pyc,, +huggingface_hub/serialization/__pycache__/_base.cpython-313.pyc,, +huggingface_hub/serialization/__pycache__/_dduf.cpython-313.pyc,, +huggingface_hub/serialization/__pycache__/_torch.cpython-313.pyc,, +huggingface_hub/serialization/_base.py,sha256=8vYeDIsqgyOO3I_uii44Bkb50LsMz3kkD4gwAPzhfWU,8436 +huggingface_hub/serialization/_dduf.py,sha256=FmGRg5wkXI5sNCgZgCNJFmqb_nVrrQYOHf4Lqq0d64I,15385 +huggingface_hub/serialization/_torch.py,sha256=BtaCP-G3oYRP6Pj9MhI-ygBsk3JbBPnVDW9huSOgtkk,46897 +huggingface_hub/templates/datasetcard_template.md,sha256=W-EMqR6wndbrnZorkVv56URWPG49l7MATGeI015kTvs,5503 +huggingface_hub/templates/modelcard_template.md,sha256=4AqArS3cqdtbit5Bo-DhjcnDFR-pza5hErLLTPM4Yuc,6870 +huggingface_hub/utils/__init__.py,sha256=pdVhPWWZkv0QM7kAQw7Bf259RZdPtYBLQqQwLQN-VC4,4045 +huggingface_hub/utils/__pycache__/__init__.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_auth.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_cache_assets.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_cache_manager.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_chunk_utils.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_datetime.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_deprecation.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_detect_agent.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_dotenv.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_experimental.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_fixes.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_git_credential.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_headers.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_hf_uris.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_http.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_lfs.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_oauth_device.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_pagination.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_parsing.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_paths.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_runtime.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_safetensors.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_subprocess.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_telemetry.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_terminal.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_typing.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_validators.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_verification.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_xet.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/_xet_progress_reporting.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/endpoint_helpers.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/insecure_hashlib.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/logging.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/sha.cpython-313.pyc,, +huggingface_hub/utils/__pycache__/tqdm.cpython-313.pyc,, +huggingface_hub/utils/_auth.py,sha256=Ykuq3yi67jxHmFdyLSc43xvGuCAIR7-h-dajX8IxpGQ,19958 +huggingface_hub/utils/_cache_assets.py,sha256=nnzHRtQAR50dQeIK6qKddsmjjTW9v9HZ7b9bq7PJqss,5691 +huggingface_hub/utils/_cache_manager.py,sha256=BFixsojrtG1K8rsD89EsGtXSktXbt0-VlJJkPRz5tso,35733 +huggingface_hub/utils/_chunk_utils.py,sha256=pTjy8Z-KLU4W_6D3OUh3E8lCodWDCd6aJwjNDU0C5U8,2121 +huggingface_hub/utils/_datetime.py,sha256=tbNyI0Dkh27oScPUtLIT_8apqIIkXZYbigjOn9S3aMw,2755 +huggingface_hub/utils/_deprecation.py,sha256=n4kNHbGipquSObJ-gxodcfd6lqoe_8s-VIsTuo3Oruk,4865 +huggingface_hub/utils/_detect_agent.py,sha256=Iw2KW9BUZWfIypn9CtiTmEibHwKK9xx5Ar48z1cMU-Q,7337 +huggingface_hub/utils/_dotenv.py,sha256=NdlEM8OXtzwlwmJCmdc6oGubFObrpgr408y8gnSs7ls,2604 +huggingface_hub/utils/_experimental.py,sha256=q9vUvc1JybVFRQ0GRREG-BLouZEJC_40MxcwbAlOud0,2464 +huggingface_hub/utils/_fixes.py,sha256=jTK1VLmc0ZC9ROSXjKoL5F6kOZFByzirRWIdXxhBfWU,4124 +huggingface_hub/utils/_git_credential.py,sha256=1BhjvIScCOAToDORLOKrR3Szs-m0E5AYHPmC0SD7Nrs,4548 +huggingface_hub/utils/_headers.py,sha256=Q9bu9mvOpm-9rfv-YRlr_Ywy_aMZkmRcN3MbVEcJAJA,8234 +huggingface_hub/utils/_hf_uris.py,sha256=RTHrVKLT5vVU-ca8roXHrKwbN3njowanvxHjx5B4sUg,27558 +huggingface_hub/utils/_http.py,sha256=zx1TItko5i0_CzL95zHyt36XXrtVLnn5F9fl6ODq8aw,46197 +huggingface_hub/utils/_lfs.py,sha256=xMU-ROgNAUpDkzbH6yRZsE-eVYUYTNMGrgVb_QdTv-k,3942 +huggingface_hub/utils/_oauth_device.py,sha256=o7ICmI01yDt1oAFNMUJvN1ThSfaCDpC8AHYSQgcnFGw,7983 +huggingface_hub/utils/_pagination.py,sha256=h-TJjFbX3FqkA1T0tGZ0EhxMvuRSCg-UfL_a-BMs54I,1943 +huggingface_hub/utils/_parsing.py,sha256=DVWuO6s_XMiGfXyc0jyX5hi_yGgBHpF9KLoeQM-VVK0,3586 +huggingface_hub/utils/_paths.py,sha256=kWpUMIK9rmKb2LHgHCkk2DnroDzucrwbj-urCJCgfGw,5962 +huggingface_hub/utils/_runtime.py,sha256=GVJ_Dt6y_48TZWl0ui9yPdW8M7rqGblRGyUqKaXxkCo,13493 +huggingface_hub/utils/_safetensors.py,sha256=qFE7OA-vjU8X0zBBtOeNRzLu147KF-PdFoaxkYfWB4M,4426 +huggingface_hub/utils/_subprocess.py,sha256=tFVBBNot_HLVqQ79y873TGb12C4PUTMwub-GhzDTemE,4542 +huggingface_hub/utils/_telemetry.py,sha256=Tpa3YmOLhK_MEnL2i2fRLQy8aqAEZWbj6MKC18Hs8BA,4824 +huggingface_hub/utils/_terminal.py,sha256=whHgs0Co-1LatoCmJE_ojqE_cWP6b5dWPWpU-uSFWUU,8969 +huggingface_hub/utils/_typing.py,sha256=1LeE785YedppXSR9a1fQDu5rhTxX5v9kBYBaHrP65rA,3542 +huggingface_hub/utils/_validators.py,sha256=tuC1U4yxB-4XKIajZMnKZUSKvZ6qPpydY8LItfa0g40,8419 +huggingface_hub/utils/_verification.py,sha256=ZSilnolSkYmHB3vmBlWuuVr53xz59P5uGnxC3H-Hjc0,5434 +huggingface_hub/utils/_xet.py,sha256=lVCUmCxzmBFYRSkJxFLtfhe7_HJWdLzJdENL-ln5Grc,6556 +huggingface_hub/utils/_xet_progress_reporting.py,sha256=MnNCxB9ieflArNNyz3R4kpkjOdt6mQxi8qp8xqzXQTM,14162 +huggingface_hub/utils/endpoint_helpers.py,sha256=9VtIAlxQ5H_4y30sjCAgbu7XCqAtNLC7aRYxaNn0hLI,2366 +huggingface_hub/utils/insecure_hashlib.py,sha256=z3dVUFvdBZ8kQI_8Vzvvlr3ims-EBiY-SYPdnzIKOkw,1008 +huggingface_hub/utils/logging.py,sha256=WaXk5gRa8Ml_LUIH34QCr8suYj8i_8Wot4IDnYJyWyM,4870 +huggingface_hub/utils/sha.py,sha256=h8wxheZpcv671RhtiIFcFmQTDSrogN3kwQx3ZaNEUHg,2121 +huggingface_hub/utils/tqdm.py,sha256=xXjKGy2y2gMqpKeMUqXcQWc66buW39NwkTr3hlWH6MY,16729 diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/REQUESTED b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/WHEEL b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..1d472b6c22838de58d1c3c0dd2c795a5cde9e415 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (83.0.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/entry_points.txt b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..62018f2f5622f61920179a20b80488f21dbb1686 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/entry_points.txt @@ -0,0 +1,7 @@ +[console_scripts] +hf = huggingface_hub.cli.hf:main +huggingface-cli = huggingface_hub.cli.deprecated_cli:main +tiny-agents = huggingface_hub.inference._mcp.cli:app + +[fsspec.specs] +hf = huggingface_hub.HfFileSystem diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/licenses/LICENSE b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/licenses/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/top_level.txt b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..6b964ccca3c1b6766042b3fe3b2707ba25372924 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/top_level.txt @@ -0,0 +1 @@ +huggingface_hub diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/novita.py b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/novita.py new file mode 100644 index 0000000000000000000000000000000000000000..af29e72032a7eb82f1ec20bc3df529612ab8730c --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/novita.py @@ -0,0 +1,69 @@ +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import ( + BaseConversationalTask, + BaseTextGenerationTask, + TaskProviderHelper, + filter_none, +) +from huggingface_hub.utils import get_session + + +_PROVIDER = "novita" +_BASE_URL = "https://api.novita.ai" + + +class NovitaTextGenerationTask(BaseTextGenerationTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + # there is no v1/ route for novita + return "/v3/openai/completions" + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + output = _as_dict(response)["choices"][0] + return { + "generated_text": output["text"], + "details": { + "finish_reason": output.get("finish_reason"), + "seed": output.get("seed"), + }, + } + + +class NovitaConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + # there is no v1/ route for novita + return "/v3/openai/chat/completions" + + +class NovitaTextToVideoTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task="text-to-video") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return f"/v3/hf/{mapped_model}" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + return {"prompt": inputs, **filter_none(parameters)} + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + if not ( + isinstance(response_dict, dict) + and "video" in response_dict + and isinstance(response_dict["video"], dict) + and "video_url" in response_dict["video"] + ): + raise ValueError("Expected response format: { 'video': { 'video_url': string } }") + + video_url = response_dict["video"]["video_url"] + return get_session().get(video_url).content diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/nscale.py b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/nscale.py new file mode 100644 index 0000000000000000000000000000000000000000..6a5a4c38e9f0f8ede459d36c584813cf5ef7dbac --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/nscale.py @@ -0,0 +1,44 @@ +import base64 +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict + +from ._common import BaseConversationalTask, TaskProviderHelper, filter_none + + +class NscaleConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="nscale", base_url="https://inference.api.nscale.com") + + +class NscaleTextToImageTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider="nscale", base_url="https://inference.api.nscale.com", task="text-to-image") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/images/generations" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + mapped_model = provider_mapping_info.provider_id + # Combine all parameters except inputs and parameters + parameters = filter_none(parameters) + if "width" in parameters and "height" in parameters: + parameters["size"] = f"{parameters.pop('width')}x{parameters.pop('height')}" + if "num_inference_steps" in parameters: + parameters.pop("num_inference_steps") + if "cfg_scale" in parameters: + parameters.pop("cfg_scale") + payload = { + "response_format": "b64_json", + "prompt": inputs, + "model": mapped_model, + **parameters, + } + return payload + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + return base64.b64decode(response_dict["data"][0]["b64_json"]) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/openai.py b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/openai.py new file mode 100644 index 0000000000000000000000000000000000000000..9f0f56002a6493c852836a1e5539900204fc0bb3 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/openai.py @@ -0,0 +1,23 @@ +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._providers._common import BaseConversationalTask + + +class OpenAIConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="openai", base_url="https://api.openai.com") + + def _prepare_api_key(self, api_key: str | None) -> str: + if api_key is None: + raise ValueError("You must provide an api_key to work with OpenAI API.") + if api_key.startswith("hf_"): + raise ValueError( + "OpenAI provider is not available through Hugging Face routing, please use your own OpenAI API key." + ) + return api_key + + def _prepare_mapping_info(self, model: str | None) -> InferenceProviderMapping: + if model is None: + raise ValueError("Please provide an OpenAI model ID, e.g. `gpt-4o` or `o1`.") + return InferenceProviderMapping( + provider="openai", providerId=model, task="conversational", status="live", hf_model_id=model + ) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/ovhcloud.py b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/ovhcloud.py new file mode 100644 index 0000000000000000000000000000000000000000..79be8d55089975b1ad9a0282d503cd2fc012127c --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/ovhcloud.py @@ -0,0 +1,10 @@ +from huggingface_hub.inference._providers._common import BaseConversationalTask + + +_PROVIDER = "ovhcloud" +_BASE_URL = "https://oai.endpoints.kepler.ai.cloud.ovh.net" + + +class OVHcloudConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/publicai.py b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/publicai.py new file mode 100644 index 0000000000000000000000000000000000000000..4c88528e4f1e2eefaf6be9315c490db19ff5ca1e --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/publicai.py @@ -0,0 +1,6 @@ +from ._common import BaseConversationalTask + + +class PublicAIConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="publicai", base_url="https://api.publicai.co") diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/replicate.py b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/replicate.py new file mode 100644 index 0000000000000000000000000000000000000000..d1aff75ec56460791b35b3acde7208d8c4a03bdb --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/replicate.py @@ -0,0 +1,160 @@ +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict, _as_url +from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none +from huggingface_hub.utils import get_session + + +_PROVIDER = "replicate" +_BASE_URL = "https://api.replicate.com" + + +class ReplicateTask(TaskProviderHelper): + def __init__(self, task: str): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task) + + def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + headers["Prefer"] = "wait" + return headers + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + if ":" in mapped_model: + return "/v1/predictions" + return f"/v1/models/{mapped_model}/predictions" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + mapped_model = provider_mapping_info.provider_id + payload: dict[str, Any] = {"input": {"prompt": inputs, **filter_none(parameters)}} + if ":" in mapped_model: + version = mapped_model.split(":", 1)[1] + payload["version"] = version + return payload + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + if response_dict.get("output") is None: + raise TimeoutError( + f"Inference request timed out after 60 seconds. No output generated for model {response_dict.get('model')}" + "The model might be in cold state or starting up. Please try again later." + ) + output_url = ( + response_dict["output"] if isinstance(response_dict["output"], str) else response_dict["output"][0] + ) + return get_session().get(output_url).content + + +class ReplicateTextToImageTask(ReplicateTask): + def __init__(self): + super().__init__("text-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + payload: dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore + if provider_mapping_info.adapter_weights_path is not None: + payload["input"]["lora_weights"] = f"https://huggingface.co/{provider_mapping_info.hf_model_id}" + return payload + + +class ReplicateTextToSpeechTask(ReplicateTask): + def __init__(self): + super().__init__("text-to-speech") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + payload: dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore + payload["input"]["text"] = payload["input"].pop("prompt") # rename "prompt" to "text" for TTS + return payload + + +class ReplicateAutomaticSpeechRecognitionTask(ReplicateTask): + def __init__(self) -> None: + super().__init__("automatic-speech-recognition") + + def _prepare_payload_as_dict( + self, + inputs: Any, + parameters: dict, + provider_mapping_info: InferenceProviderMapping, + ) -> dict | None: + mapped_model = provider_mapping_info.provider_id + audio_url = _as_url(inputs, default_mime_type="audio/wav") + + payload: dict[str, Any] = { + "input": { + **{"audio": audio_url}, + **filter_none(parameters), + } + } + + if ":" in mapped_model: + payload["version"] = mapped_model.split(":", 1)[1] + + return payload + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + output = response_dict.get("output") + + if isinstance(output, str): + return {"text": output} + + if isinstance(output, list) and output: + first_item = output[0] + if isinstance(first_item, str): + return {"text": first_item} + if isinstance(first_item, dict): + output = first_item + + text: str | None = None + if isinstance(output, dict): + transcription = output.get("transcription") + if isinstance(transcription, str): + text = transcription + + translation = output.get("translation") + if isinstance(translation, str): + text = translation + + txt_file = output.get("txt_file") + if isinstance(txt_file, str): + text_response = get_session().get(txt_file) + text_response.raise_for_status() + text = text_response.text + + if text is not None: + return {"text": text} + + raise ValueError("Received malformed response from Replicate automatic-speech-recognition API") + + +class ReplicateImageToImageTask(ReplicateTask): + def __init__(self): + super().__init__("image-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + image_url = _as_url(inputs, default_mime_type="image/jpeg") + + # Different Replicate models expect the image in different keys + payload: dict[str, Any] = { + "input": { + "image": image_url, + "images": [image_url], + "input_image": image_url, + "input_images": [image_url], + **filter_none(parameters), + } + } + + mapped_model = provider_mapping_info.provider_id + if ":" in mapped_model: + version = mapped_model.split(":", 1)[1] + payload["version"] = version + return payload diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/scaleway.py b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/scaleway.py new file mode 100644 index 0000000000000000000000000000000000000000..a64462075a091f4a9d9a664e09be229e0fb7bdf1 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/scaleway.py @@ -0,0 +1,28 @@ +from typing import Any + +from huggingface_hub.inference._common import RequestParameters, _as_dict + +from ._common import BaseConversationalTask, InferenceProviderMapping, TaskProviderHelper, filter_none + + +class ScalewayConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="scaleway", base_url="https://api.scaleway.ai") + + +class ScalewayFeatureExtractionTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider="scaleway", base_url="https://api.scaleway.ai", task="feature-extraction") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/embeddings" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + parameters = filter_none(parameters) + return {"input": inputs, "model": provider_mapping_info.provider_id, **parameters} + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + embeddings = _as_dict(response)["data"] + return [embedding["embedding"] for embedding in embeddings] diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/together.py b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/together.py new file mode 100644 index 0000000000000000000000000000000000000000..1f26595af3f919f675f14b5c4d10baa130e647d8 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/together.py @@ -0,0 +1,291 @@ +import base64 +import time +from abc import ABC +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import ( + RequestParameters, + _as_dict, + _as_url, +) +from huggingface_hub.inference._providers._common import ( + BaseConversationalTask, + BaseTextGenerationTask, + TaskProviderHelper, + filter_none, +) +from huggingface_hub.utils import get_session, hf_raise_for_status, logging + + +logger = logging.get_logger(__name__) + +_PROVIDER = "together" +_BASE_URL = "https://api.together.xyz" + +# Polling interval for async video generation (in seconds). +_VIDEO_POLLING_INTERVAL = 2.0 + +# Upper bound on status polls (initial response may already be terminal; each further poll is one attempt). +_VIDEO_MAX_POLL_ATTEMPTS = 150 # ~5 minutes at _VIDEO_POLLING_INTERVAL + +# Job statuses that mean "keep polling". Together returns "queued" before transitioning to +# "in_progress", so we must treat both as pending. +_VIDEO_PENDING_STATUSES = {"queued", "in_progress"} + + +class TogetherTask(TaskProviderHelper, ABC): + """Base class for Together API tasks.""" + + def __init__(self, task: str): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + match self.task: + case "text-to-image" | "image-to-image": + return "/v1/images/generations" + case "text-to-speech": + return "/v1/audio/speech" + case "feature-extraction": + return "/v1/embeddings" + case "text-to-video" | "image-to-video": + # Video creation lives under /v2 (see https://docs.together.ai/reference/create-videos). + return "/v2/videos" + raise ValueError(f"Unsupported task '{self.task}' for Together API.") + + +class TogetherTextGenerationTask(BaseTextGenerationTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + output = _as_dict(response)["choices"][0] + return { + "generated_text": output["text"], + "details": { + "finish_reason": output.get("finish_reason"), + "seed": output.get("seed"), + }, + } + + +class TogetherConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) + if payload is None: + return None + # Together accepts response_format `{type: "json_schema", schema: }` (flattened), + # so unwrap the OpenAI-style `{type: "json_schema", json_schema: {schema}}` envelope. + response_format = payload.get("response_format") + if ( + isinstance(response_format, dict) + and response_format.get("type") == "json_schema" + and isinstance(response_format.get("json_schema"), dict) + and "schema" in response_format["json_schema"] + ): + payload["response_format"] = { + "type": "json_schema", + "schema": response_format["json_schema"]["schema"], + } + return payload + + +class TogetherTextToImageTask(TogetherTask): + def __init__(self): + super().__init__("text-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + mapped_model = provider_mapping_info.provider_id + parameters = filter_none(parameters) + if "num_inference_steps" in parameters: + parameters["steps"] = parameters.pop("num_inference_steps") + + return {"prompt": inputs, "response_format": "base64", **parameters, "model": mapped_model} + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + return base64.b64decode(response_dict["data"][0]["b64_json"]) + + +class TogetherImageToImageTask(TogetherTask): + def __init__(self): + super().__init__("image-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + mapped_model = provider_mapping_info.provider_id + image_url = _as_url(inputs, default_mime_type="image/jpeg") + + # Filter `None` values first: the client always passes `"prompt": None` when the user + # omits the argument, so popping before filtering would yield `None` instead of the + # `""` default and send `"prompt": null` to Together (rejected by Flux Kontext). + parameters = filter_none(parameters) + prompt = parameters.pop("prompt", "") + if "num_inference_steps" in parameters: + parameters["steps"] = parameters.pop("num_inference_steps") + + # Together exposes two mutually-exclusive image inputs (see + # https://docs.together.ai/docs/image-to-image): FLUX.1 Kontext only accepts + # `image_url`; FLUX.2 [dev] and Google models (Gemini 3 Pro Image, Flash Image + # 2.5) only accept `reference_images`. FLUX.2 [pro]/[flex] accept either but + # `reference_images` is the documented default. Use `image_url` only for + # FLUX.1 Kontext models and `reference_images` for everything else. + lowered = mapped_model.lower() + use_image_url = "kontext" in lowered and "flux.1" in lowered + image_field: dict[str, Any] = {"image_url": image_url} if use_image_url else {"reference_images": [image_url]} + return { + "prompt": prompt, + **image_field, + "response_format": "base64", + **parameters, + "model": mapped_model, + } + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + return base64.b64decode(response_dict["data"][0]["b64_json"]) + + +class TogetherFeatureExtractionTask(TogetherTask): + def __init__(self): + super().__init__("feature-extraction") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + return { + "input": inputs, + "model": provider_mapping_info.provider_id, + **filter_none(parameters), + } + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + return [item["embedding"] for item in _as_dict(response)["data"]] + + +class TogetherTextToSpeechTask(TogetherTask): + def __init__(self): + super().__init__("text-to-speech") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + # `voice` is required by the Together API and is model-specific + # (see https://docs.together.ai/docs/text-to-speech#supported-voices), + # so we don't set a default and let the API surface a clear error if missing. + return { + "input": inputs, + "model": provider_mapping_info.provider_id, + **filter_none(parameters), + } + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + if isinstance(response, bytes): + return response + raise ValueError(f"Expected raw audio bytes for text-to-speech, got {type(response).__name__}.") + + +def _normalize_video_parameters(parameters: dict) -> dict: + """Map HF inference-client conventions onto Together's video API parameter names.""" + parameters = filter_none(parameters) + if "num_inference_steps" in parameters: + parameters["steps"] = parameters.pop("num_inference_steps") + if "target_size" in parameters: + target_size = parameters.pop("target_size") + if "width" in target_size: + parameters["width"] = target_size["width"] + if "height" in target_size: + parameters["height"] = target_size["height"] + return parameters + + +class TogetherVideoTask(TogetherTask, ABC): + """Base class for Together's asynchronous video generation tasks.""" + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + if request_params is None: + raise ValueError("A `RequestParameters` object is required to poll Together video jobs.") + + job = _as_dict(response) + job_id = job.get("id") + if not job_id: + raise ValueError("No job ID found in Together video generation response.") + + # Status polling lives at the same /v2/videos URL with the job ID appended. + status_url = f"{request_params.url}/{job_id}" + + logger.info("Generating video, polling for completion...") + # Together usually returns `status: "queued"` on the initial POST, but the field is + # optional per the spec — treat a missing status as "still pending" and poll, rather + # than falling through to the "unexpected status" error below. + status = job.get("status") + for _ in range(_VIDEO_MAX_POLL_ATTEMPTS): + if status is not None and status not in _VIDEO_PENDING_STATUSES: + break + time.sleep(_VIDEO_POLLING_INTERVAL) + status_response = get_session().get(status_url, headers=request_params.headers) + hf_raise_for_status(status_response) + job = status_response.json() + status = job.get("status") + if status is not None and status not in _VIDEO_PENDING_STATUSES: + break + else: + raise ValueError( + "Timed out while waiting for Together video generation " + f"— aborting after {_VIDEO_MAX_POLL_ATTEMPTS} status polls" + ) + + if status == "failed": + error = job.get("error") or {} + raise RuntimeError(f"Together video generation failed: {error.get('message') or 'Unknown error'}") + if status != "completed": + raise RuntimeError(f"Unexpected Together video job status: {status!r}") + + video_url = (job.get("outputs") or {}).get("video_url") + if not video_url: + raise ValueError("No video URL found in completed Together video job.") + + video_response = get_session().get(video_url) + hf_raise_for_status(video_response) + return video_response.content + + +class TogetherTextToVideoTask(TogetherVideoTask): + def __init__(self): + super().__init__("text-to-video") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + return { + "prompt": inputs, + "model": provider_mapping_info.provider_id, + **_normalize_video_parameters(parameters), + } + + +class TogetherImageToVideoTask(TogetherVideoTask): + def __init__(self): + super().__init__("image-to-video") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + # Together expects each keyframe as `{input_image, frame: "first" | "last"}` + # for i2v models. See https://docs.together.ai/docs/inference/videos/reference-and-keyframes. + # Note: `input_image` accepts a data URL or an HTTP(S) URL but the field is capped + # at ~60KB — users with larger inputs should host the image and pass `frame_images` + # directly via `extra_body`. + return { + "model": provider_mapping_info.provider_id, + "frame_images": [{"input_image": _as_url(inputs, default_mime_type="image/png"), "frame": "first"}], + **_normalize_video_parameters(parameters), + } diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/wavespeed.py b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/wavespeed.py new file mode 100644 index 0000000000000000000000000000000000000000..aa38eb9da2d47c491e746af6495f669147fe181e --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/wavespeed.py @@ -0,0 +1,138 @@ +import base64 +import time +from abc import ABC +from typing import Any +from urllib.parse import urlparse + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none +from huggingface_hub.utils import get_session, hf_raise_for_status +from huggingface_hub.utils.logging import get_logger + + +logger = get_logger(__name__) + +# Polling interval (in seconds) +_POLLING_INTERVAL = 0.5 + + +class WavespeedAITask(TaskProviderHelper, ABC): + def __init__(self, task: str): + super().__init__(provider="wavespeed", base_url="https://api.wavespeed.ai", task=task) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return f"/api/v3/{mapped_model}" + + def get_response( + self, + response: bytes | dict, + request_params: RequestParameters | None = None, + ) -> Any: + response_dict = _as_dict(response) + data = response_dict.get("data", {}) + result_path = data.get("urls", {}).get("get") + + if not result_path: + raise ValueError("No result URL found in the response") + if request_params is None: + raise ValueError("A `RequestParameters` object should be provided to get responses with WaveSpeed AI.") + + # Parse the request URL to determine base URL + parsed_url = urlparse(request_params.url) + # Add /wavespeed to base URL if going through HF router + if parsed_url.netloc == "router.huggingface.co": + base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/wavespeed" + else: + base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" + + # Extract path from result_path URL + if isinstance(result_path, str): + result_url_path = urlparse(result_path).path + else: + result_url_path = result_path + + result_url = f"{base_url}{result_url_path}" + + logger.info("Processing request, polling for results...") + + # Poll until task is completed + while True: + time.sleep(_POLLING_INTERVAL) + result_response = get_session().get(result_url, headers=request_params.headers) + hf_raise_for_status(result_response) + + result = result_response.json() + task_result = result.get("data", {}) + status = task_result.get("status") + + if status == "completed": + # Get content from the first output URL + if not task_result.get("outputs") or len(task_result["outputs"]) == 0: + raise ValueError("No output URL in completed response") + + output_url = task_result["outputs"][0] + return get_session().get(output_url).content + elif status == "failed": + error_msg = task_result.get("error", "Task failed with no specific error message") + raise ValueError(f"WaveSpeed AI task failed: {error_msg}") + elif status in ["processing", "created"]: + continue + else: + raise ValueError(f"Unknown status: {status}") + + +class WavespeedAITextToImageTask(WavespeedAITask): + def __init__(self): + super().__init__("text-to-image") + + def _prepare_payload_as_dict( + self, + inputs: Any, + parameters: dict, + provider_mapping_info: InferenceProviderMapping, + ) -> dict | None: + return {"prompt": inputs, **filter_none(parameters)} + + +class WavespeedAITextToVideoTask(WavespeedAITextToImageTask): + def __init__(self): + WavespeedAITask.__init__(self, "text-to-video") + + +class WavespeedAIImageToImageTask(WavespeedAITask): + def __init__(self): + super().__init__("image-to-image") + + def _prepare_payload_as_dict( + self, + inputs: Any, + parameters: dict, + provider_mapping_info: InferenceProviderMapping, + ) -> dict | None: + # Convert inputs to image (URL or base64) + if isinstance(inputs, str) and inputs.startswith(("http://", "https://")): + image = inputs + elif isinstance(inputs, str): + # If input is a file path, read it first + with open(inputs, "rb") as f: + file_content = f.read() + image_b64 = base64.b64encode(file_content).decode("utf-8") + image = f"data:image/jpeg;base64,{image_b64}" + else: + # If input is binary data + image_b64 = base64.b64encode(inputs).decode("utf-8") + image = f"data:image/jpeg;base64,{image_b64}" + + # Extract prompt from parameters if present + prompt = parameters.pop("prompt", None) + payload = {"image": image, **filter_none(parameters)} + if prompt is not None: + payload["prompt"] = prompt + + return payload + + +class WavespeedAIImageToVideoTask(WavespeedAIImageToImageTask): + def __init__(self): + WavespeedAITask.__init__(self, "image-to-video") diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/zai_org.py b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/zai_org.py new file mode 100644 index 0000000000000000000000000000000000000000..97225435084ab61860ef838747b5e0e7517f7bd1 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/zai_org.py @@ -0,0 +1,127 @@ +import time +from abc import ABC +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import BaseConversationalTask, TaskProviderHelper, filter_none +from huggingface_hub.utils import get_session + + +_PROVIDER = "zai-org" +_BASE_URL = "https://api.z.ai" +_POLLING_INTERVAL = 5 # seconds +_MAX_POLL_ATTEMPTS = 60 + + +class ZaiTask(TaskProviderHelper, ABC): + def __init__(self, task: str): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task) + + def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + headers["Accept-Language"] = "en-US,en" + headers["x-source-channel"] = "hugging_face" + return headers + + +class ZaiConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + headers["Accept-Language"] = "en-US,en" + headers["x-source-channel"] = "hugging_face" + return headers + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/api/paas/v4/chat/completions" + + +class ZaiTextToImageTask(ZaiTask): + """Text-to-image task for ZAI provider using async API.""" + + def __init__(self): + super().__init__("text-to-image") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/api/paas/v4/async/images/generations" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + width = parameters.pop("width", None) + height = parameters.pop("height", None) + size = None + if width is not None and height is not None: + size = f"{width}x{height}" + + payload: dict[str, Any] = { + "model": provider_mapping_info.provider_id, + "prompt": inputs, + } + if size is not None: + payload["size"] = size + + payload.update(filter_none(parameters)) + return payload + + def get_response( + self, + response: bytes | dict, + request_params: RequestParameters | None = None, + ) -> Any: + """Handle async response by polling for results.""" + response_dict = _as_dict(response) + + task_id = response_dict.get("id") + if task_id is None: + raise ValueError("No task_id in response from ZAI API") + + task_status = response_dict.get("task_status") + if task_status == "FAIL": + raise ValueError(f"ZAI image generation failed for request {task_id}") + + if task_status == "PROCESSING" and request_params is not None: + return self._poll_for_result(task_id, request_params) + + return self._extract_image(response_dict) + + def _poll_for_result(self, task_id: str, request_params: RequestParameters) -> bytes: + """Poll the async-result endpoint until completion.""" + session = get_session() + base_url = request_params.url.rsplit("/api/paas/v4/async/images/generations", 1)[0] + poll_url = f"{base_url}/api/paas/v4/async-result/{task_id}" + + for _ in range(_MAX_POLL_ATTEMPTS): + poll_response = session.get(poll_url, headers=request_params.headers) + poll_response.raise_for_status() + result = poll_response.json() + + task_status = result.get("task_status") + if task_status == "SUCCESS": + return self._extract_image(result) + elif task_status == "FAIL": + raise ValueError(f"Zai text-to-image generation failed for request {task_id}") + + time.sleep(_POLLING_INTERVAL) + + raise ValueError( + f"Timed out while waiting for the result from Zai API - aborting after {_MAX_POLL_ATTEMPTS} attempts" + ) + + def _extract_image(self, result: dict) -> bytes: + """Extract and download the image from the result.""" + image_result = result.get("image_result") + if not image_result or not isinstance(image_result, list) or len(image_result) == 0: + raise ValueError("No image_result in response from ZAI API") + + image_url = image_result[0].get("url") + if not image_url: + raise ValueError("No image URL in response from ZAI API") + + session = get_session() + image_response = session.get(image_url) + image_response.raise_for_status() + return image_response.content diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/lfs.py b/.venv/lib/python3.13/site-packages/huggingface_hub/lfs.py new file mode 100644 index 0000000000000000000000000000000000000000..9958a1c332f0ab938cb9769a6384e178fb5e7a96 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/lfs.py @@ -0,0 +1,442 @@ +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Git LFS related type definitions and utilities""" + +import io +import re +from collections.abc import Iterable +from math import ceil +from os.path import getsize +from typing import TYPE_CHECKING, BinaryIO, TypedDict +from urllib.parse import unquote + +from huggingface_hub import constants + +from .utils import ( + build_hf_headers, + fix_hf_endpoint_in_url, + hf_raise_for_status, + http_backoff, + logging, + validate_hf_hub_args, +) +from .utils._lfs import SliceFileObj +from .utils.sha import sha256, sha_fileobj + + +if TYPE_CHECKING: + from ._commit_api import CommitOperationAdd + +logger = logging.get_logger(__name__) + +OID_REGEX = re.compile(r"^[0-9a-f]{40}$") + +LFS_MULTIPART_UPLOAD_COMMAND = "lfs-multipart-upload" + +LFS_HEADERS = { + "Accept": "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json", +} + + +class UploadInfo: + """ + Data structure holding required information to determine whether a blob + should be uploaded to the hub using the LFS protocol or the regular protocol. + + The SHA256 of the blob is computed lazily: creating an `UploadInfo` from a local path only reads + the first 512 bytes of the file. The full file is read (and hashed) only if `sha256` is accessed + before it has been set. When a file is uploaded through the Xet protocol, the SHA256 is computed + during upload (single read pass) and set afterwards. + + Args: + size (`int`): + Size in bytes of the blob + sample (`bytes`): + First 512 bytes of the blob + sha256 (`bytes`, *optional*): + SHA256 hash of the blob, if already known. Otherwise computed lazily from `source_path`. + source_path (`str`, *optional*): + Path to the local file the blob comes from. Required to lazily compute `sha256` if not provided. + """ + + def __init__( + self, + size: int, + sample: bytes, + sha256: bytes | None = None, + source_path: str | None = None, + ): + if sha256 is None and source_path is None: + raise ValueError("Either `sha256` or `source_path` must be provided.") + self.size = size + self.sample = sample + self._sha256 = sha256 + self._source_path = source_path + + @property + def sha256(self) -> bytes: + """SHA256 of the blob. If not set yet, reads the whole file from `source_path` to compute it.""" + if self._sha256 is None: + assert self._source_path is not None # guaranteed by __init__ + with open(self._source_path, "rb") as file: + self._sha256 = sha_fileobj(file) + return self._sha256 + + @sha256.setter + def sha256(self, value: bytes) -> None: + self._sha256 = value + + @property + def is_hashed(self) -> bool: + """Whether the SHA256 is already known (accessing `sha256` will not trigger a file read).""" + return self._sha256 is not None + + def __repr__(self) -> str: + sha = self._sha256.hex() if self._sha256 is not None else "" + return f"UploadInfo(size={self.size}, sha256={sha})" + + @classmethod + def from_path(cls, path: str): + size = getsize(path) + with open(path, "rb") as file: + sample = file.peek(512)[:512] + return cls(size=size, sample=sample, source_path=path) + + @classmethod + def from_bytes(cls, data: bytes): + sha = sha256(data).digest() + return cls(size=len(data), sample=data[:512], sha256=sha) + + @classmethod + def from_fileobj(cls, fileobj: BinaryIO): + sample = fileobj.read(512) + fileobj.seek(0, io.SEEK_SET) + sha = sha_fileobj(fileobj) + size = fileobj.tell() + fileobj.seek(0, io.SEEK_SET) + return cls(size=size, sha256=sha, sample=sample) + + +@validate_hf_hub_args +def post_lfs_batch_info( + upload_infos: Iterable[UploadInfo], + token: str | None, + repo_type: str, + repo_id: str, + revision: str | None = None, + endpoint: str | None = None, + headers: dict[str, str] | None = None, + transfers: list[str] | None = None, +) -> tuple[list[dict], list[dict], str | None]: + """ + Requests the LFS batch endpoint to retrieve upload instructions + + Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md + + Args: + upload_infos (`Iterable` of `UploadInfo`): + `UploadInfo` for the files that are being uploaded, typically obtained + from `CommitOperationAdd.upload_info` + token (`str` or `None`): + An authentication token (see https://huggingface.co/settings/token). + Pass `None` to fall back to the local cached token (or no token if unauthenticated). + repo_type (`str`): + Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + revision (`str`, *optional*): + The git revision to upload to. + endpoint (`str`, *optional*): + The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`. + headers (`dict`, *optional*): + Additional headers to include in the request + transfers (`list`, *optional*): + List of transfer methods to use. Defaults to ["basic", "multipart"]. + + Returns: + `LfsBatchInfo`: 3-tuple: + - First element is the list of upload instructions from the server + - Second element is a list of errors, if any + - Third element is the chosen transfer adapter if provided by the server (e.g. "basic", "multipart", "xet") + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If an argument is invalid or the server response is malformed. + [`HfHubHTTPError`] + If the server returned an error. + """ + endpoint = endpoint if endpoint is not None else constants.ENDPOINT + url_prefix = "" + if repo_type in constants.REPO_TYPES_URL_PREFIXES: + url_prefix = constants.REPO_TYPES_URL_PREFIXES[repo_type] + batch_url = f"{endpoint}/{url_prefix}{repo_id}.git/info/lfs/objects/batch" + payload: dict = { + "operation": "upload", + "transfers": transfers if transfers is not None else ["basic", "multipart"], + "objects": [ + { + "oid": upload.sha256.hex(), + "size": upload.size, + } + for upload in upload_infos + ], + "hash_algo": "sha256", + } + if revision is not None: + payload["ref"] = {"name": unquote(revision)} # revision has been previously 'quoted' + + headers = { + **LFS_HEADERS, + **build_hf_headers(token=token), + **(headers or {}), + } + resp = http_backoff("POST", batch_url, headers=headers, json=payload) + hf_raise_for_status(resp) + batch_info = resp.json() + + objects = batch_info.get("objects", None) + if not isinstance(objects, list): + raise ValueError("Malformed response from server") + + chosen_transfer = batch_info.get("transfer") + chosen_transfer = chosen_transfer if isinstance(chosen_transfer, str) else None + + return ( + [_validate_batch_actions(obj) for obj in objects if "error" not in obj], + [_validate_batch_error(obj) for obj in objects if "error" in obj], + chosen_transfer, + ) + + +class PayloadPartT(TypedDict): + partNumber: int + etag: str + + +class CompletionPayloadT(TypedDict): + """Payload that will be sent to the Hub when uploading multi-part.""" + + oid: str + parts: list[PayloadPartT] + + +def lfs_upload( + operation: "CommitOperationAdd", + lfs_batch_action: dict, + token: str | None = None, + headers: dict[str, str] | None = None, + endpoint: str | None = None, +) -> None: + """ + Handles uploading a given object to the Hub with the LFS protocol. + + Can be a No-op if the content of the file is already present on the hub large file storage. + + Args: + operation (`CommitOperationAdd`): + The add operation triggering this upload. + lfs_batch_action (`dict`): + Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for + more details. + token (`str`, *optional*): + An authentication token (see https://huggingface.co/settings/token). Used to call the + optional LFS verify step at the end of the upload. If `None`, falls back to the local + cached token. + headers (`dict`, *optional*): + Headers to include in the request, including authentication and user agent headers. + endpoint (`str`, *optional*): + The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`. + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If `lfs_batch_action` is improperly formatted + [`HfHubHTTPError`] + If the upload resulted in an error + """ + # 0. If LFS file is already present, skip upload + _validate_batch_actions(lfs_batch_action) + actions = lfs_batch_action.get("actions") + if actions is None: + # The file was already uploaded + logger.debug(f"Content of file {operation.path_in_repo} is already present upstream - skipping upload") + return + + # 1. Validate server response (check required keys in dict) + upload_action = lfs_batch_action["actions"]["upload"] + _validate_lfs_action(upload_action) + verify_action = lfs_batch_action["actions"].get("verify") + if verify_action is not None: + _validate_lfs_action(verify_action) + + # 2. Upload file (either single part or multi-part) + header = upload_action.get("header", {}) + chunk_size = header.get("chunk_size") + upload_url = fix_hf_endpoint_in_url(upload_action["href"], endpoint=endpoint) + if chunk_size is not None: + try: + chunk_size = int(chunk_size) + except (ValueError, TypeError): + raise ValueError( + f"Malformed response from LFS batch endpoint: `chunk_size` should be an integer. Got '{chunk_size}'." + ) + _upload_multi_part(operation=operation, header=header, chunk_size=chunk_size, upload_url=upload_url) + else: + _upload_single_part(operation=operation, upload_url=upload_url) + + # 3. Verify upload went well + if verify_action is not None: + _validate_lfs_action(verify_action) + verify_url = fix_hf_endpoint_in_url(verify_action["href"], endpoint) + verify_resp = http_backoff( + "POST", + verify_url, + headers=build_hf_headers(token=token, headers=headers), + json={"oid": operation.upload_info.sha256.hex(), "size": operation.upload_info.size}, + ) + hf_raise_for_status(verify_resp) + logger.debug(f"{operation.path_in_repo}: Upload successful") + + +def _validate_lfs_action(lfs_action: dict): + """validates response from the LFS batch endpoint""" + if not ( + isinstance(lfs_action.get("href"), str) + and (lfs_action.get("header") is None or isinstance(lfs_action.get("header"), dict)) + ): + raise ValueError("lfs_action is improperly formatted") + return lfs_action + + +def _validate_batch_actions(lfs_batch_actions: dict): + """validates response from the LFS batch endpoint""" + if not (isinstance(lfs_batch_actions.get("oid"), str) and isinstance(lfs_batch_actions.get("size"), int)): + raise ValueError("lfs_batch_actions is improperly formatted") + + upload_action = lfs_batch_actions.get("actions", {}).get("upload") + verify_action = lfs_batch_actions.get("actions", {}).get("verify") + if upload_action is not None: + _validate_lfs_action(upload_action) + if verify_action is not None: + _validate_lfs_action(verify_action) + return lfs_batch_actions + + +def _validate_batch_error(lfs_batch_error: dict): + """validates response from the LFS batch endpoint""" + if not (isinstance(lfs_batch_error.get("oid"), str) and isinstance(lfs_batch_error.get("size"), int)): + raise ValueError("lfs_batch_error is improperly formatted") + error_info = lfs_batch_error.get("error") + if not ( + isinstance(error_info, dict) + and isinstance(error_info.get("message"), str) + and isinstance(error_info.get("code"), int) + ): + raise ValueError("lfs_batch_error is improperly formatted") + return lfs_batch_error + + +def _upload_single_part(operation: "CommitOperationAdd", upload_url: str) -> None: + """ + Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol) + + Args: + upload_url (`str`): + The URL to PUT the file to. + fileobj: + The file-like object holding the data to upload. + + Raises: + [`HfHubHTTPError`] + If the upload resulted in an error. + """ + with operation.as_file(with_tqdm=True) as fileobj: + # S3 might raise a transient 500 error -> let's retry if that happens + response = http_backoff("PUT", upload_url, data=fileobj) + hf_raise_for_status(response) + + +def _upload_multi_part(operation: "CommitOperationAdd", header: dict, chunk_size: int, upload_url: str) -> None: + """ + Uploads file using HF multipart LFS transfer protocol. + """ + # 1. Get upload URLs for each part + sorted_parts_urls = _get_sorted_parts_urls(header=header, upload_info=operation.upload_info, chunk_size=chunk_size) + + # 2. Upload parts (pure Python) + response_headers = _upload_parts_iteratively( + operation=operation, sorted_parts_urls=sorted_parts_urls, chunk_size=chunk_size + ) + + # 3. Send completion request + # NOTE: `upload_url` is the Hub completion endpoint (not the S3 upload URLs). + completion_res = http_backoff( + "POST", + upload_url, + json=_get_completion_payload(response_headers, operation.upload_info.sha256.hex()), + headers=LFS_HEADERS, + ) + hf_raise_for_status(completion_res) + + +def _get_sorted_parts_urls(header: dict, upload_info: UploadInfo, chunk_size: int) -> list[str]: + sorted_part_upload_urls = [ + upload_url + for _, upload_url in sorted( + [ + (int(part_num, 10), upload_url) + for part_num, upload_url in header.items() + if part_num.isdigit() and len(part_num) > 0 + ], + key=lambda t: t[0], + ) + ] + num_parts = len(sorted_part_upload_urls) + if num_parts != ceil(upload_info.size / chunk_size): + raise ValueError("Invalid server response to upload large LFS file") + return sorted_part_upload_urls + + +def _get_completion_payload(response_headers: list[dict], oid: str) -> CompletionPayloadT: + parts: list[PayloadPartT] = [] + for part_number, header in enumerate(response_headers): + etag = header.get("etag") + if etag is None or etag == "": + raise ValueError(f"Invalid etag (`{etag}`) returned for part {part_number + 1}") + parts.append( + { + "partNumber": part_number + 1, + "etag": etag, + } + ) + return {"oid": oid, "parts": parts} + + +def _upload_parts_iteratively( + operation: "CommitOperationAdd", sorted_parts_urls: list[str], chunk_size: int +) -> list[dict]: + headers = [] + with operation.as_file(with_tqdm=True) as fileobj: + for part_idx, part_upload_url in enumerate(sorted_parts_urls): + with SliceFileObj( + fileobj, + seek_from=chunk_size * part_idx, + read_limit=chunk_size, + ) as fileobj_slice: + # S3 might raise a transient 500 error -> let's retry if that happens + part_upload_res = http_backoff("PUT", part_upload_url, data=fileobj_slice) + hf_raise_for_status(part_upload_res) + headers.append(part_upload_res.headers) + return headers # type: ignore diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/py.typed b/.venv/lib/python3.13/site-packages/huggingface_hub/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/repocard.py b/.venv/lib/python3.13/site-packages/huggingface_hub/repocard.py new file mode 100644 index 0000000000000000000000000000000000000000..1bb03da924520ab87ad202b6032b2e4d85bae195 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/repocard.py @@ -0,0 +1,835 @@ +import os +import re +from pathlib import Path +from typing import Any, Literal + +import yaml + +from huggingface_hub.file_download import hf_hub_download +from huggingface_hub.hf_api import upload_file +from huggingface_hub.repocard_data import ( + CardData, + DatasetCardData, + EvalResult, + ModelCardData, + SpaceCardData, + eval_results_to_model_index, + model_index_to_eval_results, +) +from huggingface_hub.utils import HfHubHTTPError, get_session, hf_raise_for_status, is_jinja_available, yaml_dump + +from . import constants +from .errors import EntryNotFoundError +from .utils import SoftTemporaryDirectory, logging, validate_hf_hub_args + + +logger = logging.get_logger(__name__) + + +TEMPLATE_MODELCARD_PATH = Path(__file__).parent / "templates" / "modelcard_template.md" +TEMPLATE_DATASETCARD_PATH = Path(__file__).parent / "templates" / "datasetcard_template.md" + +# exact same regex as in the Hub server. Please keep in sync. +# See https://github.com/huggingface/moon-landing/blob/main/server/lib/ViewMarkdown.ts#L18 +REGEX_YAML_BLOCK = re.compile(r"^(\s*---(?:\r\n|\r|\n))([\S\s]*?)((?:\r\n|\r|\n)---[ \t]*(\r\n|\n|$))") + + +class RepoCard: + card_data_class = CardData + default_template_path = TEMPLATE_MODELCARD_PATH + repo_type = "model" + + def __init__(self, content: str, ignore_metadata_errors: bool = False): + """Initialize a RepoCard from string content. The content should be a + Markdown file with a YAML block at the beginning and a Markdown body. + + Args: + content (`str`): The content of the Markdown file. + + Example: + ```python + >>> from huggingface_hub.repocard import RepoCard + >>> text = ''' + ... --- + ... language: en + ... license: mit + ... --- + ... + ... # My repo + ... ''' + >>> card = RepoCard(text) + >>> card.data.to_dict() + {'language': 'en', 'license': 'mit'} + >>> card.text + '\\n# My repo\\n' + + ``` + > [!TIP] + > Raises the following error: + > + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > when the content of the repo card metadata is not a dictionary. + """ + + # Set the content of the RepoCard, as well as underlying .data and .text attributes. + # See the `content` property setter for more details. + self.ignore_metadata_errors = ignore_metadata_errors + self.content = content + + @property + def content(self): + """The content of the RepoCard, including the YAML block and the Markdown body.""" + line_break = _detect_line_ending(self._content) or "\n" + return f"---{line_break}{self.data.to_yaml(line_break=line_break, original_order=self._original_order)}{line_break}---{line_break}{self.text}" + + @content.setter + def content(self, content: str): + """Set the content of the RepoCard.""" + self._content = content + + match = REGEX_YAML_BLOCK.search(content) + if match: + # Metadata found in the YAML block + yaml_block = match.group(2) + self.text = content[match.end() :] + data_dict = yaml.safe_load(yaml_block) + + if data_dict is None: + data_dict = {} + + # The YAML block's data should be a dictionary + if not isinstance(data_dict, dict): + raise ValueError("repo card metadata block should be a dict") + else: + # Model card without metadata... create empty metadata + logger.warning("Repo card metadata block was not found. Setting CardData to empty.") + data_dict = {} + self.text = content + + self.data = self.card_data_class(**data_dict, ignore_metadata_errors=self.ignore_metadata_errors) + self._original_order = list(data_dict.keys()) + + def __str__(self): + return self.content + + def save(self, filepath: Path | str): + r"""Save a RepoCard to a file. + + Args: + filepath (`Union[Path, str]`): Filepath to the markdown file to save. + + Example: + ```python + >>> from huggingface_hub.repocard import RepoCard + >>> card = RepoCard("---\nlanguage: en\n---\n# This is a test repo card") + >>> card.save("/tmp/test.md") + + ``` + """ + filepath = Path(filepath) + filepath.parent.mkdir(parents=True, exist_ok=True) + # Preserve newlines as in the existing file. + with open(filepath, mode="w", newline="", encoding="utf-8") as f: + f.write(str(self)) + + @classmethod + def load( + cls, + repo_id_or_path: str | Path, + repo_type: str | None = None, + token: str | None = None, + ignore_metadata_errors: bool = False, + ): + """Initialize a RepoCard from a Hugging Face Hub repo's README.md or a local filepath. + + Args: + repo_id_or_path (`Union[str, Path]`): + The repo ID associated with a Hugging Face Hub repo or a local filepath. + repo_type (`str`, *optional*): + The type of Hugging Face repo to push to. Defaults to None, which will use "model". Other options + are "dataset" and "space". Not used when loading from a local filepath. If this is called from a child + class, the default value will be the child class's `repo_type`. + token (`str`, *optional*): + Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to the stored token. + ignore_metadata_errors (`str`): + If True, errors while parsing the metadata section will be ignored. Some information might be lost during + the process. Use it at your own risk. + + Returns: + [`huggingface_hub.repocard.RepoCard`]: The RepoCard (or subclass) initialized from the repo's + README.md file or filepath. + + Example: + ```python + >>> from huggingface_hub.repocard import RepoCard + >>> card = RepoCard.load("nateraw/food") + >>> assert card.data.tags == ["generated_from_trainer", "image-classification", "pytorch"] + + ``` + """ + + if Path(repo_id_or_path).is_file(): + card_path = Path(repo_id_or_path) + elif isinstance(repo_id_or_path, str): + card_path = Path( + hf_hub_download( + repo_id_or_path, + constants.REPOCARD_NAME, + repo_type=repo_type or cls.repo_type, + token=token, + ) + ) + else: + raise ValueError(f"Cannot load RepoCard: path not found on disk ({repo_id_or_path}).") + + # Preserve newlines in the existing file. + with card_path.open(mode="r", newline="", encoding="utf-8") as f: + return cls(f.read(), ignore_metadata_errors=ignore_metadata_errors) + + def validate(self, repo_type: str | None = None): + """Validates card against Hugging Face Hub's card validation logic. + Using this function requires access to the internet, so it is only called + internally by [`huggingface_hub.repocard.RepoCard.push_to_hub`]. + + Args: + repo_type (`str`, *optional*, defaults to "model"): + The type of Hugging Face repo to push to. Options are "model", "dataset", and "space". + If this function is called from a child class, the default will be the child class's `repo_type`. + + > [!TIP] + > Raises the following errors: + > + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if the card fails validation checks. + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the request to the Hub API fails for any other reason. + """ + + # If repo type is provided, otherwise, use the repo type of the card. + repo_type = repo_type or self.repo_type + + body = { + "repoType": repo_type, + "content": str(self), + } + headers = {"Accept": "text/plain"} + + try: + response = get_session().post("https://huggingface.co/api/validate-yaml", json=body, headers=headers) + hf_raise_for_status(response) + except HfHubHTTPError as exc: + if response.status_code == 400: + raise ValueError(response.text) + else: + raise exc + + def push_to_hub( + self, + repo_id: str, + token: str | None = None, + repo_type: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + revision: str | None = None, + create_pr: bool | None = None, + parent_commit: str | None = None, + ): + """Push a RepoCard to a Hugging Face Hub repo. + + Args: + repo_id (`str`): + The repo ID of the Hugging Face Hub repo to push to. Example: "nateraw/food". + token (`str`, *optional*): + Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to + the stored token. + repo_type (`str`, *optional*, defaults to "model"): + The type of Hugging Face repo to push to. Options are "model", "dataset", and "space". If this + function is called by a child class, it will default to the child class's `repo_type`. + commit_message (`str`, *optional*): + The summary / title / first line of the generated commit. + commit_description (`str`, *optional*) + The description of the generated commit. + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the `"main"` branch. + create_pr (`bool`, *optional*): + Whether or not to create a Pull Request with this commit. Defaults to `False`. + parent_commit (`str`, *optional*): + The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. + If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. + If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. + Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be + especially useful if the repo is updated / committed too concurrently. + Returns: + `str`: URL of the commit which updated the card metadata. + """ + + # If repo type is provided, otherwise, use the repo type of the card. + repo_type = repo_type or self.repo_type + + # Validate card before pushing to hub + self.validate(repo_type=repo_type) + + with SoftTemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) / constants.REPOCARD_NAME + tmp_path.write_text(str(self), encoding="utf-8") + url = upload_file( + path_or_fileobj=str(tmp_path), + path_in_repo=constants.REPOCARD_NAME, + repo_id=repo_id, + token=token, + repo_type=repo_type, + commit_message=commit_message, + commit_description=commit_description, + create_pr=create_pr, + revision=revision, + parent_commit=parent_commit, + ) + return url + + @classmethod + def from_template( + cls, + card_data: CardData, + template_path: str | None = None, + template_str: str | None = None, + **template_kwargs, + ): + """Initialize a RepoCard from a template. By default, it uses the default template. + + Templates are Jinja2 templates that can be customized by passing keyword arguments. + + Args: + card_data (`huggingface_hub.CardData`): + A huggingface_hub.CardData instance containing the metadata you want to include in the YAML + header of the repo card on the Hugging Face Hub. + template_path (`str`, *optional*): + A path to a markdown file with optional Jinja template variables that can be filled + in with `template_kwargs`. Defaults to the default template. + template_str (`str`, *optional*): + A raw Jinja template string with optional variables. Used when neither `template_path` + nor the default template is appropriate. Ignored if `template_path` is also provided. + + Returns: + [`huggingface_hub.repocard.RepoCard`]: A RepoCard instance with the specified card data and content from the + template. + """ + if is_jinja_available(): + import jinja2 + else: + raise ImportError( + "Using RepoCard.from_template requires Jinja2 to be installed. Please" + " install it with `pip install Jinja2`." + ) + + kwargs = card_data.to_dict().copy() + kwargs.update(template_kwargs) # Template_kwargs have priority + + if template_path is not None: + template_str = Path(template_path).read_text() + if template_str is None: + template_str = Path(cls.default_template_path).read_text() + template = jinja2.Template(template_str) + content = template.render(card_data=card_data.to_yaml(), **kwargs) + return cls(content) + + +class ModelCard(RepoCard): + card_data_class = ModelCardData # type: ignore[assignment] + default_template_path = TEMPLATE_MODELCARD_PATH + repo_type = "model" + + @classmethod + def from_template( # type: ignore # violates Liskov property but easier to use + cls, + card_data: ModelCardData, + template_path: str | None = None, + template_str: str | None = None, + **template_kwargs, + ): + """Initialize a ModelCard from a template. By default, it uses the default template, which can be found here: + https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/modelcard_template.md + + Templates are Jinja2 templates that can be customized by passing keyword arguments. + + Args: + card_data (`huggingface_hub.ModelCardData`): + A huggingface_hub.ModelCardData instance containing the metadata you want to include in the YAML + header of the model card on the Hugging Face Hub. + template_path (`str`, *optional*): + A path to a markdown file with optional Jinja template variables that can be filled + in with `template_kwargs`. Defaults to the default template. + template_str (`str`, *optional*): + A raw Jinja template string with optional variables. Used when neither `template_path` + nor the default template is appropriate. Ignored if `template_path` is also provided. + + Returns: + [`huggingface_hub.ModelCard`]: A ModelCard instance with the specified card data and content from the + template. + + Example: + ```python + >>> from huggingface_hub import ModelCard, ModelCardData, EvalResult + + >>> # Using the Default Template + >>> card_data = ModelCardData( + ... language='en', + ... license='mit', + ... library_name='timm', + ... tags=['image-classification', 'resnet'], + ... datasets=['beans'], + ... metrics=['accuracy'], + ... ) + >>> card = ModelCard.from_template( + ... card_data, + ... model_description='This model does x + y...' + ... ) + + >>> # Including Evaluation Results + >>> card_data = ModelCardData( + ... language='en', + ... tags=['image-classification', 'resnet'], + ... eval_results=[ + ... EvalResult( + ... task_type='image-classification', + ... dataset_type='beans', + ... dataset_name='Beans', + ... metric_type='accuracy', + ... metric_value=0.9, + ... ), + ... ], + ... model_name='my-cool-model', + ... ) + >>> card = ModelCard.from_template(card_data) + + >>> # Using a Custom Template + >>> card_data = ModelCardData( + ... language='en', + ... tags=['image-classification', 'resnet'] + ... ) + >>> card = ModelCard.from_template( + ... card_data=card_data, + ... template_path='./src/huggingface_hub/templates/modelcard_template.md', + ... custom_template_var='custom value', # will be replaced in template if it exists + ... ) + + ``` + """ + return super().from_template(card_data, template_path, template_str, **template_kwargs) + + +class DatasetCard(RepoCard): + card_data_class = DatasetCardData # type: ignore[assignment] + default_template_path = TEMPLATE_DATASETCARD_PATH + repo_type = "dataset" + + @classmethod + def from_template( # type: ignore # violates Liskov property but easier to use + cls, + card_data: DatasetCardData, + template_path: str | None = None, + template_str: str | None = None, + **template_kwargs, + ): + """Initialize a DatasetCard from a template. By default, it uses the default template, which can be found here: + https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/datasetcard_template.md + + Templates are Jinja2 templates that can be customized by passing keyword arguments. + + Args: + card_data (`huggingface_hub.DatasetCardData`): + A huggingface_hub.DatasetCardData instance containing the metadata you want to include in the YAML + header of the dataset card on the Hugging Face Hub. + template_path (`str`, *optional*): + A path to a markdown file with optional Jinja template variables that can be filled + in with `template_kwargs`. Defaults to the default template. + template_str (`str`, *optional*): + A raw Jinja template string with optional variables. Used when neither `template_path` + nor the default template is appropriate. Ignored if `template_path` is also provided. + + Returns: + [`huggingface_hub.DatasetCard`]: A DatasetCard instance with the specified card data and content from the + template. + + Example: + ```python + >>> from huggingface_hub import DatasetCard, DatasetCardData + + >>> # Using the Default Template + >>> card_data = DatasetCardData( + ... language='en', + ... license='mit', + ... annotations_creators='crowdsourced', + ... task_categories=['text-classification'], + ... task_ids=['sentiment-classification', 'text-scoring'], + ... multilinguality='monolingual', + ... pretty_name='My Text Classification Dataset', + ... ) + >>> card = DatasetCard.from_template( + ... card_data, + ... pretty_name=card_data.pretty_name, + ... ) + + >>> # Using a Custom Template + >>> card_data = DatasetCardData( + ... language='en', + ... license='mit', + ... ) + >>> card = DatasetCard.from_template( + ... card_data=card_data, + ... template_path='./src/huggingface_hub/templates/datasetcard_template.md', + ... custom_template_var='custom value', # will be replaced in template if it exists + ... ) + + ``` + """ + return super().from_template(card_data, template_path, template_str, **template_kwargs) + + +class SpaceCard(RepoCard): + card_data_class = SpaceCardData # type: ignore[assignment] + default_template_path = TEMPLATE_MODELCARD_PATH + repo_type = "space" + + +def _detect_line_ending(content: str) -> Literal["\r", "\n", "\r\n", None]: # noqa: F722 + """Detect the line ending of a string. Used by RepoCard to avoid making huge diff on newlines. + + Uses same implementation as in Hub server, keep it in sync. + + Returns: + str: The detected line ending of the string. + """ + cr = content.count("\r") + lf = content.count("\n") + crlf = content.count("\r\n") + if cr + lf == 0: + return None + if crlf == cr and crlf == lf: + return "\r\n" + if cr > lf: + return "\r" + else: + return "\n" + + +def metadata_load(local_path: str | Path) -> dict | None: + content = Path(local_path).read_text() + match = REGEX_YAML_BLOCK.search(content) + if match: + yaml_block = match.group(2) + data = yaml.safe_load(yaml_block) + if data is None or isinstance(data, dict): + return data + raise ValueError("repo card metadata block should be a dict") + else: + return None + + +def metadata_save(local_path: str | Path, data: dict) -> None: + """ + Save the metadata dict in the upper YAML part Trying to preserve newlines as + in the existing file. Docs about open() with newline="" parameter: + https://docs.python.org/3/library/functions.html?highlight=open#open Does + not work with "^M" linebreaks, which are replaced by \n + """ + line_break = "\n" + content = "" + # try to detect existing newline character + if os.path.exists(local_path): + with open(local_path, newline="", encoding="utf8") as readme: + content = readme.read() + if isinstance(readme.newlines, tuple): + line_break = readme.newlines[0] + elif isinstance(readme.newlines, str): + line_break = readme.newlines + + # creates a new file if it not + with open(local_path, "w", newline="", encoding="utf8") as readme: + data_yaml = yaml_dump(data, sort_keys=False, line_break=line_break) + # sort_keys: keep dict order + match = REGEX_YAML_BLOCK.search(content) + if match: + output = content[: match.start()] + f"---{line_break}{data_yaml}---{line_break}" + content[match.end() :] + else: + output = f"---{line_break}{data_yaml}---{line_break}{content}" + + readme.write(output) + readme.close() + + +def metadata_eval_result( + *, + model_pretty_name: str, + task_pretty_name: str, + task_id: str, + metrics_pretty_name: str, + metrics_id: str, + metrics_value: Any, + dataset_pretty_name: str, + dataset_id: str, + metrics_config: str | None = None, + metrics_verified: bool = False, + dataset_config: str | None = None, + dataset_split: str | None = None, + dataset_revision: str | None = None, + metrics_verification_token: str | None = None, +) -> dict: + """ + Creates a metadata dict with the result from a model evaluated on a dataset. + + Args: + model_pretty_name (`str`): + The name of the model in natural language. + task_pretty_name (`str`): + The name of a task in natural language. + task_id (`str`): + Example: automatic-speech-recognition. A task id. + metrics_pretty_name (`str`): + A name for the metric in natural language. Example: Test WER. + metrics_id (`str`): + Example: wer. A metric id from https://hf.co/metrics. + metrics_value (`Any`): + The value from the metric. Example: 20.0 or "20.0 ± 1.2". + dataset_pretty_name (`str`): + The name of the dataset in natural language. + dataset_id (`str`): + Example: common_voice. A dataset id from https://hf.co/datasets. + metrics_config (`str`, *optional*): + The name of the metric configuration used in `load_metric()`. + Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`. + metrics_verified (`bool`, *optional*, defaults to `False`): + Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set. + dataset_config (`str`, *optional*): + Example: fr. The name of the dataset configuration used in `load_dataset()`. + dataset_split (`str`, *optional*): + Example: test. The name of the dataset split used in `load_dataset()`. + dataset_revision (`str`, *optional*): + Example: 5503434ddd753f426f4b38109466949a1217c2bb. The name of the dataset dataset revision + used in `load_dataset()`. + metrics_verification_token (`bool`, *optional*): + A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. + + Returns: + `dict`: a metadata dict with the result from a model evaluated on a dataset. + + Example: + ```python + >>> from huggingface_hub import metadata_eval_result + >>> results = metadata_eval_result( + ... model_pretty_name="RoBERTa fine-tuned on ReactionGIF", + ... task_pretty_name="Text Classification", + ... task_id="text-classification", + ... metrics_pretty_name="Accuracy", + ... metrics_id="accuracy", + ... metrics_value=0.2662102282047272, + ... dataset_pretty_name="ReactionJPEG", + ... dataset_id="julien-c/reactionjpeg", + ... dataset_config="default", + ... dataset_split="test", + ... ) + >>> results == { + ... 'model-index': [ + ... { + ... 'name': 'RoBERTa fine-tuned on ReactionGIF', + ... 'results': [ + ... { + ... 'task': { + ... 'type': 'text-classification', + ... 'name': 'Text Classification' + ... }, + ... 'dataset': { + ... 'name': 'ReactionJPEG', + ... 'type': 'julien-c/reactionjpeg', + ... 'config': 'default', + ... 'split': 'test' + ... }, + ... 'metrics': [ + ... { + ... 'type': 'accuracy', + ... 'value': 0.2662102282047272, + ... 'name': 'Accuracy', + ... 'verified': False + ... } + ... ] + ... } + ... ] + ... } + ... ] + ... } + True + + ``` + """ + + return { + "model-index": eval_results_to_model_index( + model_name=model_pretty_name, + eval_results=[ + EvalResult( + task_name=task_pretty_name, + task_type=task_id, + metric_name=metrics_pretty_name, + metric_type=metrics_id, + metric_value=metrics_value, + dataset_name=dataset_pretty_name, + dataset_type=dataset_id, + metric_config=metrics_config, + verified=metrics_verified, + verify_token=metrics_verification_token, + dataset_config=dataset_config, + dataset_split=dataset_split, + dataset_revision=dataset_revision, + ) + ], + ) + } + + +@validate_hf_hub_args +def metadata_update( + repo_id: str, + metadata: dict, + *, + repo_type: str | None = None, + overwrite: bool = False, + token: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + revision: str | None = None, + create_pr: bool = False, + parent_commit: str | None = None, +) -> str: + """ + Updates the metadata in the README.md of a repository on the Hugging Face Hub. + If the README.md file doesn't exist yet, a new one is created with metadata and + the default ModelCard or DatasetCard template. For `space` repo, an error is thrown + as a Space cannot exist without a `README.md` file. + + Args: + repo_id (`str`): + The name of the repository. + metadata (`dict`): + A dictionary containing the metadata to be updated. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if updating to a dataset or space, + `None` or `"model"` if updating to a model. Default is `None`. + overwrite (`bool`, *optional*, defaults to `False`): + If set to `True` an existing field can be overwritten, otherwise + attempting to overwrite an existing field will cause an error. + token (`str`, *optional*): + The Hugging Face authentication token. + commit_message (`str`, *optional*): + The summary / title / first line of the generated commit. Defaults to + `f"Update metadata with huggingface_hub"` + commit_description (`str` *optional*) + The description of the generated commit + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the + `"main"` branch. + create_pr (`boolean`, *optional*): + Whether or not to create a Pull Request from `revision` with that commit. + Defaults to `False`. + parent_commit (`str`, *optional*): + The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. + If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. + If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. + Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be + especially useful if the repo is updated / committed too concurrently. + Returns: + `str`: URL of the commit which updated the card metadata. + + Example: + ```python + >>> from huggingface_hub import metadata_update + >>> metadata = {'model-index': [{'name': 'RoBERTa fine-tuned on ReactionGIF', + ... 'results': [{'dataset': {'name': 'ReactionGIF', + ... 'type': 'julien-c/reactiongif'}, + ... 'metrics': [{'name': 'Recall', + ... 'type': 'recall', + ... 'value': 0.7762102282047272}], + ... 'task': {'name': 'Text Classification', + ... 'type': 'text-classification'}}]}]} + >>> url = metadata_update("hf-internal-testing/reactiongif-roberta-card", metadata) + + ``` + """ + commit_message = commit_message if commit_message is not None else "Update metadata with huggingface_hub" + + # Card class given repo_type + card_class: type[RepoCard] + if repo_type is None or repo_type == "model": + card_class = ModelCard + elif repo_type == "dataset": + card_class = DatasetCard + elif repo_type == "space": + card_class = RepoCard + else: + raise ValueError(f"Unknown repo_type: {repo_type}") + + # Either load repo_card from the Hub or create an empty one. + # NOTE: Will not create the repo if it doesn't exist. + try: + card = card_class.load(repo_id, token=token, repo_type=repo_type) + except EntryNotFoundError: + if repo_type == "space": + raise ValueError("Cannot update metadata on a Space that doesn't contain a `README.md` file.") + + # Initialize a ModelCard or DatasetCard from default template and no data. + # Cast to the concrete expected card type to satisfy type checkers. + card = card_class.from_template(CardData()) # type: ignore + + for key, value in metadata.items(): + if key == "model-index": + # if the new metadata doesn't include a name, either use existing one or repo name + if "name" not in value[0]: + value[0]["name"] = getattr(card, "model_name", repo_id) + model_name, new_results = model_index_to_eval_results(value) + if card.data.eval_results is None: + card.data.eval_results = new_results + card.data.model_name = model_name + else: + existing_results = card.data.eval_results + + # Iterate over new results + # Iterate over existing results + # If both results describe the same metric but value is different: + # If overwrite=True: overwrite the metric value + # Else: raise ValueError + # Else: append new result to existing ones. + for new_result in new_results: + result_found = False + for existing_result in existing_results: + if new_result.is_equal_except_value(existing_result): + if new_result != existing_result and not overwrite: + raise ValueError( + "You passed a new value for the existing metric" + f" 'name: {new_result.metric_name}, type: " + f"{new_result.metric_type}'. Set `overwrite=True`" + " to overwrite existing metrics." + ) + result_found = True + existing_result.metric_value = new_result.metric_value + if existing_result.verified is True: + existing_result.verify_token = new_result.verify_token + if not result_found: + card.data.eval_results.append(new_result) + else: + # Any metadata that is not a result metric + if card.data.get(key) is not None and not overwrite and card.data.get(key) != value: + raise ValueError( + f"You passed a new value for the existing meta data field '{key}'." + " Set `overwrite=True` to overwrite existing metadata." + ) + else: + card.data[key] = value + + return card.push_to_hub( + repo_id, + token=token, + repo_type=repo_type, + commit_message=commit_message, + commit_description=commit_description, + create_pr=create_pr, + revision=revision, + parent_commit=parent_commit, + ) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/repocard_data.py b/.venv/lib/python3.13/site-packages/huggingface_hub/repocard_data.py new file mode 100644 index 0000000000000000000000000000000000000000..69539f809b7f38b54be76fe689153b3c3ec4ff7a --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/repocard_data.py @@ -0,0 +1,776 @@ +import copy +from collections import defaultdict +from dataclasses import dataclass +from typing import Any + +from huggingface_hub.utils import logging, yaml_dump + + +logger = logging.get_logger(__name__) + + +@dataclass +class EvalResult: + """ + Flattened representation of individual evaluation results found in model-index of Model Cards. + + For more information on the model-index spec, see https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1. + + Args: + task_type (`str`): + The task identifier. Example: "image-classification". + dataset_type (`str`): + The dataset identifier. Example: "common_voice". Use dataset id from https://hf.co/datasets. + dataset_name (`str`): + A pretty name for the dataset. Example: "Common Voice (French)". + metric_type (`str`): + The metric identifier. Example: "wer". Use metric id from https://hf.co/metrics. + metric_value (`Any`): + The metric value. Example: 0.9 or "20.0 ± 1.2". + task_name (`str`, *optional*): + A pretty name for the task. Example: "Speech Recognition". + dataset_config (`str`, *optional*): + The name of the dataset configuration used in `load_dataset()`. + Example: fr in `load_dataset("common_voice", "fr")`. See the `datasets` docs for more info: + https://hf.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name + dataset_split (`str`, *optional*): + The split used in `load_dataset()`. Example: "test". + dataset_revision (`str`, *optional*): + The revision (AKA Git Sha) of the dataset used in `load_dataset()`. + Example: 5503434ddd753f426f4b38109466949a1217c2bb + dataset_args (`dict[str, Any]`, *optional*): + The arguments passed during `Metric.compute()`. Example for `bleu`: `{"max_order": 4}` + metric_name (`str`, *optional*): + A pretty name for the metric. Example: "Test WER". + metric_config (`str`, *optional*): + The name of the metric configuration used in `load_metric()`. + Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`. + See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations + metric_args (`dict[str, Any]`, *optional*): + The arguments passed during `Metric.compute()`. Example for `bleu`: max_order: 4 + verified (`bool`, *optional*): + Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set. + verify_token (`str`, *optional*): + A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. + source_name (`str`, *optional*): + The name of the source of the evaluation result. Example: "Open LLM Leaderboard". + source_url (`str`, *optional*): + The URL of the source of the evaluation result. Example: "https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard". + """ + + # Required + + # The task identifier + # Example: automatic-speech-recognition + task_type: str + + # The dataset identifier + # Example: common_voice. Use dataset id from https://hf.co/datasets + dataset_type: str + + # A pretty name for the dataset. + # Example: Common Voice (French) + dataset_name: str + + # The metric identifier + # Example: wer. Use metric id from https://hf.co/metrics + metric_type: str + + # Value of the metric. + # Example: 20.0 or "20.0 ± 1.2" + metric_value: Any + + # Optional + + # A pretty name for the task. + # Example: Speech Recognition + task_name: str | None = None + + # The name of the dataset configuration used in `load_dataset()`. + # Example: fr in `load_dataset("common_voice", "fr")`. + # See the `datasets` docs for more info: + # https://huggingface.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name + dataset_config: str | None = None + + # The split used in `load_dataset()`. + # Example: test + dataset_split: str | None = None + + # The revision (AKA Git Sha) of the dataset used in `load_dataset()`. + # Example: 5503434ddd753f426f4b38109466949a1217c2bb + dataset_revision: str | None = None + + # The arguments passed during `Metric.compute()`. + # Example for `bleu`: max_order: 4 + dataset_args: dict[str, Any] | None = None + + # A pretty name for the metric. + # Example: Test WER + metric_name: str | None = None + + # The name of the metric configuration used in `load_metric()`. + # Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`. + # See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations + metric_config: str | None = None + + # The arguments passed during `Metric.compute()`. + # Example for `bleu`: max_order: 4 + metric_args: dict[str, Any] | None = None + + # Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set. + verified: bool | None = None + + # A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. + verify_token: str | None = None + + # The name of the source of the evaluation result. + # Example: Open LLM Leaderboard + source_name: str | None = None + + # The URL of the source of the evaluation result. + # Example: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard + source_url: str | None = None + + @property + def unique_identifier(self) -> tuple: + """Returns a tuple that uniquely identifies this evaluation.""" + return ( + self.task_type, + self.dataset_type, + self.dataset_config, + self.dataset_split, + self.dataset_revision, + ) + + def is_equal_except_value(self, other: "EvalResult") -> bool: + """ + Return True if `self` and `other` describe exactly the same metric but with a + different value. + """ + for key, _ in self.__dict__.items(): + if key == "metric_value": + continue + # For metrics computed by Hugging Face's evaluation service, `verify_token` is derived from `metric_value`, + # so we exclude it here in the comparison. + if key != "verify_token" and getattr(self, key) != getattr(other, key): + return False + return True + + def __post_init__(self) -> None: + if self.source_name is not None and self.source_url is None: + raise ValueError("If `source_name` is provided, `source_url` must also be provided.") + + +@dataclass +class CardData: + """Structure containing metadata from a RepoCard. + + [`CardData`] is the parent class of [`ModelCardData`] and [`DatasetCardData`]. + + Metadata can be exported as a dictionary or YAML. Export can be customized to alter the representation of the data + (example: flatten evaluation results). `CardData` behaves as a dictionary (can get, pop, set values) but do not + inherit from `dict` to allow this export step. + """ + + def __init__(self, ignore_metadata_errors: bool = False, **kwargs): + self.__dict__.update(kwargs) + + def to_dict(self): + """Converts CardData to a dict. + + Returns: + `dict`: CardData represented as a dictionary ready to be dumped to a YAML + block for inclusion in a README.md file. + """ + + data_dict = copy.deepcopy(self.__dict__) + self._to_dict(data_dict) + return {key: value for key, value in data_dict.items() if value is not None} + + def _to_dict(self, data_dict): + """Use this method in child classes to alter the dict representation of the data. Alter the dict in-place. + + Args: + data_dict (`dict`): The raw dict representation of the card data. + """ + pass + + def to_yaml(self, line_break=None, original_order: list[str] | None = None) -> str: + """Dumps CardData to a YAML block for inclusion in a README.md file. + + Args: + line_break (str, *optional*): + The line break to use when dumping to yaml. + original_order (`list[str]`, *optional*): + If provided, reorder the metadata fields to match this list before dumping. + Any keys not in `original_order` are appended after the listed keys, preserving + their existing relative order. Useful for round-tripping a YAML block without + shuffling its keys. + + Returns: + `str`: CardData represented as a YAML block. + """ + if original_order: + original_order_set = set(original_order) + self.__dict__ = { + k: self.__dict__[k] + for k in original_order + [k for k in self.__dict__ if k not in original_order_set] + if k in self.__dict__ + } + return yaml_dump(self.to_dict(), sort_keys=False, line_break=line_break).strip() + + def __repr__(self): + return repr(self.__dict__) + + def __str__(self): + return self.to_yaml() + + def get(self, key: str, default: Any = None) -> Any: + """Get value for a given metadata key.""" + value = self.__dict__.get(key) + return default if value is None else value + + def pop(self, key: str, default: Any = None) -> Any: + """Pop value for a given metadata key.""" + return self.__dict__.pop(key, default) + + def __getitem__(self, key: str) -> Any: + """Get value for a given metadata key.""" + return self.__dict__[key] + + def __setitem__(self, key: str, value: Any) -> None: + """Set value for a given metadata key.""" + self.__dict__[key] = value + + def __contains__(self, key: str) -> bool: + """Check if a given metadata key is set.""" + return key in self.__dict__ + + def __len__(self) -> int: + """Return the number of metadata keys set.""" + return len(self.__dict__) + + +def _validate_eval_results( + eval_results: EvalResult | list[EvalResult] | None, + model_name: str | None, +) -> list[EvalResult]: + if eval_results is None: + return [] + if isinstance(eval_results, EvalResult): + eval_results = [eval_results] + if not isinstance(eval_results, list) or not all(isinstance(r, EvalResult) for r in eval_results): + raise ValueError( + f"`eval_results` should be of type `EvalResult` or a list of `EvalResult`, got {type(eval_results)}." + ) + if model_name is None: + raise ValueError("Passing `eval_results` requires `model_name` to be set.") + return eval_results + + +class ModelCardData(CardData): + """Model Card Metadata that is used by Hugging Face Hub when included at the top of your README.md + + Args: + base_model (`str` or `list[str]`, *optional*): + The identifier of the base model from which the model derives. This is applicable for example if your model is a + fine-tune or adapter of an existing model. The value must be the ID of a model on the Hub (or a list of IDs + if your model derives from multiple models). Defaults to None. + datasets (`Union[str, list[str]]`, *optional*): + Dataset or list of datasets that were used to train this model. Should be a dataset ID + found on https://hf.co/datasets. Defaults to None. + eval_results (`Union[list[EvalResult], EvalResult]`, *optional*): + List of `huggingface_hub.EvalResult` that define evaluation results of the model. If provided, + `model_name` is used to as a name on PapersWithCode's leaderboards. Defaults to `None`. + language (`Union[str, list[str]]`, *optional*): + Language of model's training data or metadata. It must be an ISO 639-1, 639-2 or + 639-3 code (two/three letters), or a special value like "code", "multilingual". Defaults to `None`. + library_name (`str`, *optional*): + Name of library used by this model. Example: keras or any library from + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/model-libraries.ts. + Defaults to None. + license (`str`, *optional*): + License of this model. Example: apache-2.0 or any license from + https://huggingface.co/docs/hub/repositories-licenses. Defaults to None. + license_name (`str`, *optional*): + Name of the license of this model. Defaults to None. To be used in conjunction with `license_link`. + Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a name. In that case, use `license` instead. + license_link (`str`, *optional*): + Link to the license of this model. Defaults to None. To be used in conjunction with `license_name`. + Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a link. In that case, use `license` instead. + metrics (`list[str]`, *optional*): + List of metrics used to evaluate this model. Should be a metric name that can be found + at https://hf.co/metrics. Example: 'accuracy'. Defaults to None. + model_name (`str`, *optional*): + A name for this model. It is used along with + `eval_results` to construct the `model-index` within the card's metadata. The name + you supply here is what will be used on PapersWithCode's leaderboards. If None is provided + then the repo name is used as a default. Defaults to None. + pipeline_tag (`str`, *optional*): + The pipeline tag associated with the model. Example: "text-classification". + tags (`list[str]`, *optional*): + List of tags to add to your model that can be used when filtering on the Hugging + Face Hub. Defaults to None. + ignore_metadata_errors (`str`): + If True, errors while parsing the metadata section will be ignored. Some information might be lost during + the process. Use it at your own risk. + kwargs (`dict`, *optional*): + Additional metadata that will be added to the model card. Defaults to None. + + Example: + ```python + >>> from huggingface_hub import ModelCardData + >>> card_data = ModelCardData( + ... language="en", + ... license="mit", + ... library_name="timm", + ... tags=['image-classification', 'resnet'], + ... ) + >>> card_data.to_dict() + {'language': 'en', 'license': 'mit', 'library_name': 'timm', 'tags': ['image-classification', 'resnet']} + + ``` + """ + + def __init__( + self, + *, + base_model: str | list[str] | None = None, + datasets: str | list[str] | None = None, + eval_results: list[EvalResult] | None = None, + language: str | list[str] | None = None, + library_name: str | None = None, + license: str | None = None, + license_name: str | None = None, + license_link: str | None = None, + metrics: list[str] | None = None, + model_name: str | None = None, + pipeline_tag: str | None = None, + tags: list[str] | None = None, + ignore_metadata_errors: bool = False, + **kwargs, + ): + self.base_model = base_model + self.datasets = datasets + self.eval_results = eval_results + self.language = language + self.library_name = library_name + self.license = license + self.license_name = license_name + self.license_link = license_link + self.metrics = metrics + self.model_name = model_name + self.pipeline_tag = pipeline_tag + self.tags = _to_unique_list(tags) + + model_index = kwargs.pop("model-index", None) + if model_index: + try: + model_name, eval_results = model_index_to_eval_results(model_index) + self.model_name = model_name + self.eval_results = eval_results + except (KeyError, TypeError) as error: + if ignore_metadata_errors: + logger.warning("Invalid model-index. Not loading eval results into CardData.") + else: + raise ValueError( + f"Invalid `model_index` in metadata cannot be parsed: {error.__class__} {error}. Pass" + " `ignore_metadata_errors=True` to ignore this error while loading a Model Card. Warning:" + " some information will be lost. Use it at your own risk." + ) + + super().__init__(**kwargs) + + if self.eval_results: + try: + self.eval_results = _validate_eval_results(self.eval_results, self.model_name) + except Exception as e: + if ignore_metadata_errors: + logger.warning(f"Failed to validate eval_results: {e}. Not loading eval results into CardData.") + else: + raise ValueError(f"Failed to validate eval_results: {e}") from e + + def _to_dict(self, data_dict): + """Format the internal data dict. In this case, we convert eval results to a valid model index""" + if self.eval_results is not None: + data_dict["model-index"] = eval_results_to_model_index(self.model_name, self.eval_results) # type: ignore + del data_dict["eval_results"], data_dict["model_name"] + + +class DatasetCardData(CardData): + """Dataset Card Metadata that is used by Hugging Face Hub when included at the top of your README.md + + Args: + language (`list[str]`, *optional*): + Language of dataset's data or metadata. It must be an ISO 639-1, 639-2 or + 639-3 code (two/three letters), or a special value like "code", "multilingual". + license (`Union[str, list[str]]`, *optional*): + License(s) of this dataset. Example: apache-2.0 or any license from + https://huggingface.co/docs/hub/repositories-licenses. + annotations_creators (`Union[str, list[str]]`, *optional*): + How the annotations for the dataset were created. + Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'no-annotation', 'other'. + language_creators (`Union[str, list[str]]`, *optional*): + How the text-based data in the dataset was created. + Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'other' + multilinguality (`Union[str, list[str]]`, *optional*): + Whether the dataset is multilingual. + Options are: 'monolingual', 'multilingual', 'translation', 'other'. + size_categories (`Union[str, list[str]]`, *optional*): + The number of examples in the dataset. Options are: 'n<1K', '1K1T', and 'other'. + source_datasets (`list[str]]`, *optional*): + Indicates whether the dataset is an original dataset or extended from another existing dataset. + Options are: 'original' and 'extended'. + task_categories (`Union[str, list[str]]`, *optional*): + What categories of task does the dataset support? + task_ids (`Union[str, list[str]]`, *optional*): + What specific tasks does the dataset support? + paperswithcode_id (`str`, *optional*): + ID of the dataset on PapersWithCode. + pretty_name (`str`, *optional*): + A more human-readable name for the dataset. (ex. "Cats vs. Dogs") + train_eval_index (`dict`, *optional*): + A dictionary that describes the necessary spec for doing evaluation on the Hub. + If not provided, it will be gathered from the 'train-eval-index' key of the kwargs. + config_names (`Union[str, list[str]]`, *optional*): + A list of the available dataset configs for the dataset. + """ + + def __init__( + self, + *, + language: str | list[str] | None = None, + license: str | list[str] | None = None, + annotations_creators: str | list[str] | None = None, + language_creators: str | list[str] | None = None, + multilinguality: str | list[str] | None = None, + size_categories: str | list[str] | None = None, + source_datasets: list[str] | None = None, + task_categories: str | list[str] | None = None, + task_ids: str | list[str] | None = None, + paperswithcode_id: str | None = None, + pretty_name: str | None = None, + train_eval_index: dict | None = None, + config_names: str | list[str] | None = None, + ignore_metadata_errors: bool = False, + **kwargs, + ): + self.annotations_creators = annotations_creators + self.language_creators = language_creators + self.language = language + self.license = license + self.multilinguality = multilinguality + self.size_categories = size_categories + self.source_datasets = source_datasets + self.task_categories = task_categories + self.task_ids = task_ids + self.paperswithcode_id = paperswithcode_id + self.pretty_name = pretty_name + self.config_names = config_names + + # TODO - maybe handle this similarly to EvalResult? + self.train_eval_index = train_eval_index or kwargs.pop("train-eval-index", None) + super().__init__(**kwargs) + + def _to_dict(self, data_dict): + data_dict["train-eval-index"] = data_dict.pop("train_eval_index") + + +class SpaceCardData(CardData): + """Space Card Metadata that is used by Hugging Face Hub when included at the top of your README.md + + To get an exhaustive reference of Spaces configuration, please visit https://huggingface.co/docs/hub/spaces-config-reference#spaces-configuration-reference. + + Args: + title (`str`, *optional*) + Title of the Space. + sdk (`str`, *optional*) + SDK of the Space (one of `gradio`, `streamlit`, `docker`, or `static`). + sdk_version (`str`, *optional*) + Version of the used SDK (if Gradio/Streamlit sdk). + python_version (`str`, *optional*) + Python version used in the Space (if Gradio/Streamlit sdk). + app_file (`str`, *optional*) + Path to your main application file (which contains either gradio or streamlit Python code, or static html code). + Path is relative to the root of the repository. + app_port (`str`, *optional*) + Port on which your application is running. Used only if sdk is `docker`. + license (`str`, *optional*) + License of this model. Example: apache-2.0 or any license from + https://huggingface.co/docs/hub/repositories-licenses. + duplicated_from (`str`, *optional*) + ID of the original Space if this is a duplicated Space. + models (list[`str`], *optional*) + List of models related to this Space. Should be a dataset ID found on https://hf.co/models. + datasets (`list[str]`, *optional*) + List of datasets related to this Space. Should be a dataset ID found on https://hf.co/datasets. + tags (`list[str]`, *optional*) + List of tags to add to your Space that can be used when filtering on the Hub. + ignore_metadata_errors (`str`): + If True, errors while parsing the metadata section will be ignored. Some information might be lost during + the process. Use it at your own risk. + kwargs (`dict`, *optional*): + Additional metadata that will be added to the space card. + + Example: + ```python + >>> from huggingface_hub import SpaceCardData + >>> card_data = SpaceCardData( + ... title="Dreambooth Training", + ... license="mit", + ... sdk="gradio", + ... duplicated_from="multimodalart/dreambooth-training" + ... ) + >>> card_data.to_dict() + {'title': 'Dreambooth Training', 'sdk': 'gradio', 'license': 'mit', 'duplicated_from': 'multimodalart/dreambooth-training'} + ``` + """ + + def __init__( + self, + *, + title: str | None = None, + sdk: str | None = None, + sdk_version: str | None = None, + python_version: str | None = None, + app_file: str | None = None, + app_port: int | None = None, + license: str | None = None, + duplicated_from: str | None = None, + models: list[str] | None = None, + datasets: list[str] | None = None, + tags: list[str] | None = None, + ignore_metadata_errors: bool = False, + **kwargs, + ): + self.title = title + self.sdk = sdk + self.sdk_version = sdk_version + self.python_version = python_version + self.app_file = app_file + self.app_port = app_port + self.license = license + self.duplicated_from = duplicated_from + self.models = models + self.datasets = datasets + self.tags = _to_unique_list(tags) + super().__init__(**kwargs) + + +def model_index_to_eval_results(model_index: list[dict[str, Any]]) -> tuple[str, list[EvalResult]]: + """Takes in a model index and returns the model name and a list of `huggingface_hub.EvalResult` objects. + + A detailed spec of the model index can be found here: + https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 + + Args: + model_index (`list[dict[str, Any]]`): + A model index data structure, likely coming from a README.md file on the + Hugging Face Hub. + + Returns: + model_name (`str`): + The name of the model as found in the model index. This is used as the + identifier for the model on leaderboards like PapersWithCode. + eval_results (`list[EvalResult]`): + A list of `huggingface_hub.EvalResult` objects containing the metrics + reported in the provided model_index. + + Example: + ```python + >>> from huggingface_hub.repocard_data import model_index_to_eval_results + >>> # Define a minimal model index + >>> model_index = [ + ... { + ... "name": "my-cool-model", + ... "results": [ + ... { + ... "task": { + ... "type": "image-classification" + ... }, + ... "dataset": { + ... "type": "beans", + ... "name": "Beans" + ... }, + ... "metrics": [ + ... { + ... "type": "accuracy", + ... "value": 0.9 + ... } + ... ] + ... } + ... ] + ... } + ... ] + >>> model_name, eval_results = model_index_to_eval_results(model_index) + >>> model_name + 'my-cool-model' + >>> eval_results[0].task_type + 'image-classification' + >>> eval_results[0].metric_type + 'accuracy' + + ``` + """ + + eval_results = [] + for elem in model_index: + name = elem["name"] + results = elem["results"] + for result in results: + task_type = result["task"]["type"] + task_name = result["task"].get("name") + dataset_type = result["dataset"]["type"] + dataset_name = result["dataset"]["name"] + dataset_config = result["dataset"].get("config") + dataset_split = result["dataset"].get("split") + dataset_revision = result["dataset"].get("revision") + dataset_args = result["dataset"].get("args") + source_name = result.get("source", {}).get("name") + source_url = result.get("source", {}).get("url") + + for metric in result["metrics"]: + metric_type = metric["type"] + metric_value = metric["value"] + metric_name = metric.get("name") + metric_args = metric.get("args") + metric_config = metric.get("config") + verified = metric.get("verified") + verify_token = metric.get("verifyToken") + + eval_result = EvalResult( + task_type=task_type, # Required + dataset_type=dataset_type, # Required + dataset_name=dataset_name, # Required + metric_type=metric_type, # Required + metric_value=metric_value, # Required + task_name=task_name, + dataset_config=dataset_config, + dataset_split=dataset_split, + dataset_revision=dataset_revision, + dataset_args=dataset_args, + metric_name=metric_name, + metric_args=metric_args, + metric_config=metric_config, + verified=verified, + verify_token=verify_token, + source_name=source_name, + source_url=source_url, + ) + eval_results.append(eval_result) + return name, eval_results + + +def _remove_none(obj): + """ + Recursively remove `None` values from a dict. Borrowed from: https://stackoverflow.com/a/20558778 + """ + if isinstance(obj, (list, tuple, set)): + return type(obj)(_remove_none(x) for x in obj if x is not None) + elif isinstance(obj, dict): + return type(obj)((_remove_none(k), _remove_none(v)) for k, v in obj.items() if k is not None and v is not None) + else: + return obj + + +def eval_results_to_model_index(model_name: str, eval_results: list[EvalResult]) -> list[dict[str, Any]]: + """Takes in given model name and list of `huggingface_hub.EvalResult` and returns a + valid model-index that will be compatible with the format expected by the + Hugging Face Hub. + + Args: + model_name (`str`): + Name of the model (ex. "my-cool-model"). This is used as the identifier + for the model on leaderboards like PapersWithCode. + eval_results (`list[EvalResult]`): + List of `huggingface_hub.EvalResult` objects containing the metrics to be + reported in the model-index. + + Returns: + model_index (`list[dict[str, Any]]`): The eval_results converted to a model-index. + + Example: + ```python + >>> from huggingface_hub.repocard_data import eval_results_to_model_index, EvalResult + >>> # Define minimal eval_results + >>> eval_results = [ + ... EvalResult( + ... task_type="image-classification", # Required + ... dataset_type="beans", # Required + ... dataset_name="Beans", # Required + ... metric_type="accuracy", # Required + ... metric_value=0.9, # Required + ... ) + ... ] + >>> eval_results_to_model_index("my-cool-model", eval_results) + [{'name': 'my-cool-model', 'results': [{'task': {'type': 'image-classification'}, 'dataset': {'name': 'Beans', 'type': 'beans'}, 'metrics': [{'type': 'accuracy', 'value': 0.9}]}]}] + + ``` + """ + + # Metrics are reported on a unique task-and-dataset basis. + # Here, we make a map of those pairs and the associated EvalResults. + task_and_ds_types_map: dict[Any, list[EvalResult]] = defaultdict(list) + for eval_result in eval_results: + task_and_ds_types_map[eval_result.unique_identifier].append(eval_result) + + # Use the map from above to generate the model index data. + model_index_data: list[dict[str, Any]] = [] + for results in task_and_ds_types_map.values(): + # All items from `results` share same metadata + sample_result = results[0] + data: dict[str, Any] = { + "task": { + "type": sample_result.task_type, + "name": sample_result.task_name, + }, + "dataset": { + "name": sample_result.dataset_name, + "type": sample_result.dataset_type, + "config": sample_result.dataset_config, + "split": sample_result.dataset_split, + "revision": sample_result.dataset_revision, + "args": sample_result.dataset_args, + }, + "metrics": [ + { + "type": result.metric_type, + "value": result.metric_value, + "name": result.metric_name, + "config": result.metric_config, + "args": result.metric_args, + "verified": result.verified, + "verifyToken": result.verify_token, + } + for result in results + ], + } + if sample_result.source_url is not None: + source: dict[str, str] = { + "url": sample_result.source_url, + } + if sample_result.source_name is not None: + source["name"] = sample_result.source_name + data["source"] = source + model_index_data.append(data) + + # TODO - Check if there cases where this list is longer than one? + # Finally, the model index itself is list of dicts. + model_index = [ + { + "name": model_name, + "results": model_index_data, + } + ] + return _remove_none(model_index) + + +def _to_unique_list(tags: list[str] | None) -> list[str] | None: + if tags is None: + return tags + unique_tags = [] # make tags unique + keep order explicitly + for tag in tags: + if tag not in unique_tags: + unique_tags.append(tag) + return unique_tags diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/__init__.py b/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6e624a754176079eb63540513297099025a0685c --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ruff: noqa: F401 +"""Contains helpers to serialize tensors.""" + +from ._base import StateDictSplit, split_state_dict_into_shards_factory +from ._torch import ( + get_torch_storage_id, + get_torch_storage_size, + load_state_dict_from_file, + load_torch_model, + save_torch_model, + save_torch_state_dict, + split_torch_state_dict_into_shards, +) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_base.py b/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_base.py new file mode 100644 index 0000000000000000000000000000000000000000..f58bcb1f4ed57706be7d62bd39729d50dc98a5c1 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_base.py @@ -0,0 +1,212 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains helpers to split tensors into shards.""" + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any, TypeVar + +from .. import logging + + +TensorT = TypeVar("TensorT") +TensorSizeFn_T = Callable[[TensorT], int] +StorageIDFn_T = Callable[[TensorT], Any | None] + +MAX_SHARD_SIZE = "5GB" +SIZE_UNITS = { + "TB": 10**12, + "GB": 10**9, + "MB": 10**6, + "KB": 10**3, +} + + +logger = logging.get_logger(__file__) + + +@dataclass +class StateDictSplit: + is_sharded: bool = field(init=False) + metadata: dict[str, Any] + filename_to_tensors: dict[str, list[str]] + tensor_to_filename: dict[str, str] + + def __post_init__(self): + self.is_sharded = len(self.filename_to_tensors) > 1 + + +def split_state_dict_into_shards_factory( + state_dict: dict[str, TensorT], + *, + get_storage_size: TensorSizeFn_T, + filename_pattern: str, + get_storage_id: StorageIDFn_T = lambda tensor: None, + max_shard_size: int | str = MAX_SHARD_SIZE, +) -> StateDictSplit: + """ + Split a model state dictionary in shards so that each shard is smaller than a given size. + + The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization + made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we + have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not + [6+2+2GB], [6+2GB], [6GB]. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + Args: + state_dict (`dict[str, Tensor]`): + The state dictionary to save. + get_storage_size (`Callable[[Tensor], int]`): + A function that returns the size of a tensor when saved on disk in bytes. + get_storage_id (`Callable[[Tensor], Optional[Any]]`, *optional*): + A function that returns a unique identifier to a tensor storage. Multiple different tensors can share the + same underlying storage. This identifier is guaranteed to be unique and constant for this tensor's storage + during its lifetime. Two tensor storages with non-overlapping lifetimes may have the same id. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + + Returns: + [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them. + """ + storage_id_to_tensors: dict[Any, list[str]] = {} + + shard_list: list[dict[str, TensorT]] = [] + current_shard: dict[str, TensorT] = {} + current_shard_size = 0 + total_size = 0 + + if isinstance(max_shard_size, str): + max_shard_size = parse_size_to_int(max_shard_size) + + for key, tensor in state_dict.items(): + # when bnb serialization is used the weights in the state dict can be strings + # check: https://github.com/huggingface/transformers/pull/24416 for more details + if isinstance(tensor, str): + logger.info("Skipping tensor %s as it is a string (bnb serialization)", key) + continue + + # If a `tensor` shares the same underlying storage as another tensor, we put `tensor` in the same `block` + storage_id = get_storage_id(tensor) # type: ignore[invalid-argument-type] + if storage_id is not None: + if storage_id in storage_id_to_tensors: + # We skip this tensor for now and will reassign to correct shard later + storage_id_to_tensors[storage_id].append(key) + continue + else: + # This is the first tensor with this storage_id, we create a new entry + # in the storage_id_to_tensors dict => we will assign the shard id later + storage_id_to_tensors[storage_id] = [key] + + # Compute tensor size + tensor_size = get_storage_size(tensor) # type: ignore[invalid-argument-type] + + # If this tensor is bigger than the maximal size, we put it in its own shard + if tensor_size > max_shard_size: + total_size += tensor_size + shard_list.append({key: tensor}) + continue + + # If this tensor is going to tip up over the maximal size, we split. + # Current shard already has some tensors, we add it to the list of shards and create a new one. + if current_shard_size + tensor_size > max_shard_size: + shard_list.append(current_shard) + current_shard = {} + current_shard_size = 0 + + # Add the tensor to the current shard + current_shard[key] = tensor + current_shard_size += tensor_size + total_size += tensor_size + + # Add the last shard + if len(current_shard) > 0: + shard_list.append(current_shard) + nb_shards = len(shard_list) + + # Loop over the tensors that share the same storage and assign them together + for storage_id, keys in storage_id_to_tensors.items(): + # Let's try to find the shard where the first tensor of this storage is and put all tensors in the same shard + for shard in shard_list: + if keys[0] in shard: + for key in keys: + shard[key] = state_dict[key] + break + + # If we only have one shard, we return it => no need to build the index + if nb_shards == 1: + filename = filename_pattern.format(suffix="") + # Use the keys from the shard itself rather than `state_dict` directly, so that tensors + # skipped above (e.g. string tensors from bnb serialization) are excluded from the index, + # consistently with the multi-shard path below. + keys = list(shard_list[0].keys()) + return StateDictSplit( + metadata={"total_size": total_size}, + filename_to_tensors={filename: keys}, + tensor_to_filename={key: filename for key in keys}, + ) + + # Now that each tensor is assigned to a shard, let's assign a filename to each shard + tensor_name_to_filename = {} + filename_to_tensors = {} + for idx, shard in enumerate(shard_list): + filename = filename_pattern.format(suffix=f"-{idx + 1:05d}-of-{nb_shards:05d}") + for key in shard: + tensor_name_to_filename[key] = filename + filename_to_tensors[filename] = list(shard.keys()) + + # Build the index and return + return StateDictSplit( + metadata={"total_size": total_size}, + filename_to_tensors=filename_to_tensors, + tensor_to_filename=tensor_name_to_filename, + ) + + +def parse_size_to_int(size_as_str: str) -> int: + """ + Parse a size expressed as a string with digits and unit (like `"5MB"`) to an integer (in bytes). + + Supported units are "TB", "GB", "MB", "KB". + + Args: + size_as_str (`str`): The size to convert. Will be directly returned if an `int`. + + Example: + + ```py + >>> parse_size_to_int("5MB") + 5000000 + ``` + """ + size_as_str = size_as_str.strip() + + # Parse unit + unit = size_as_str[-2:].upper() + if unit not in SIZE_UNITS: + raise ValueError(f"Unit '{unit}' not supported. Supported units are TB, GB, MB, KB. Got '{size_as_str}'.") + multiplier = SIZE_UNITS[unit] + + # Parse value + try: + value = float(size_as_str[:-2].strip()) + except ValueError as e: + raise ValueError(f"Could not parse the size value from '{size_as_str}': {e}") from e + + return int(value * multiplier) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_dduf.py b/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_dduf.py new file mode 100644 index 0000000000000000000000000000000000000000..37e916d419d5a5934525bcb169fd5dae48d6389d --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_dduf.py @@ -0,0 +1,386 @@ +import json +import logging +import mmap +import os +import shutil +import zipfile +from collections.abc import Generator, Iterable +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from ..errors import DDUFCorruptedFileError, DDUFExportError, DDUFInvalidEntryNameError + + +logger = logging.getLogger(__name__) + +DDUF_ALLOWED_ENTRIES = { + # Allowed file extensions in a DDUF file + ".json", + ".model", + ".safetensors", + ".txt", +} + +DDUF_FOLDER_REQUIRED_ENTRIES = { + # Each folder must contain at least one of these entries + "config.json", + "tokenizer_config.json", + "preprocessor_config.json", + "scheduler_config.json", +} + + +@dataclass +class DDUFEntry: + """Object representing a file entry in a DDUF file. + + See [`read_dduf_file`] for how to read a DDUF file. + + Attributes: + filename (str): + The name of the file in the DDUF archive. + offset (int): + The offset of the file in the DDUF archive. + length (int): + The length of the file in the DDUF archive. + dduf_path (str): + The path to the DDUF archive (for internal use). + """ + + filename: str + length: int + offset: int + + dduf_path: Path = field(repr=False) + + @contextmanager + def as_mmap(self) -> Generator[bytes, None, None]: + """Open the file as a memory-mapped file. + + Useful to load safetensors directly from the file. + + Example: + ```py + >>> import safetensors.torch + >>> with entry.as_mmap() as mm: + ... tensors = safetensors.torch.load(mm) + ``` + """ + with self.dduf_path.open("rb") as f: + with mmap.mmap(f.fileno(), length=0, access=mmap.ACCESS_READ) as mm: + yield mm[self.offset : self.offset + self.length] + + def read_text(self, encoding: str = "utf-8") -> str: + """Read the file as text. + + Useful for '.txt' and '.json' entries. + + Example: + ```py + >>> import json + >>> index = json.loads(entry.read_text()) + ``` + """ + with self.dduf_path.open("rb") as f: + f.seek(self.offset) + return f.read(self.length).decode(encoding=encoding) + + +def read_dduf_file(dduf_path: os.PathLike | str) -> dict[str, DDUFEntry]: + """ + Read a DDUF file and return a dictionary of entries. + + Only the metadata is read, the data is not loaded in memory. + + Args: + dduf_path (`str` or `os.PathLike`): + The path to the DDUF file to read. + + Returns: + `dict[str, DDUFEntry]`: + A dictionary of [`DDUFEntry`] indexed by filename. + + Raises: + - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format). + + Example: + ```python + >>> import json + >>> import safetensors.torch + >>> from huggingface_hub import read_dduf_file + + # Read DDUF metadata + >>> dduf_entries = read_dduf_file("FLUX.1-dev.dduf") + + # Returns a mapping filename <> DDUFEntry + >>> dduf_entries["model_index.json"] + DDUFEntry(filename='model_index.json', offset=66, length=587) + + # Load model index as JSON + >>> json.loads(dduf_entries["model_index.json"].read_text()) + {'_class_name': 'FluxPipeline', '_diffusers_version': '0.32.0.dev0', '_name_or_path': 'black-forest-labs/FLUX.1-dev', ... + + # Load VAE weights using safetensors + >>> with dduf_entries["vae/diffusion_pytorch_model.safetensors"].as_mmap() as mm: + ... state_dict = safetensors.torch.load(mm) + ``` + """ + entries = {} + dduf_path = Path(dduf_path) + logger.info(f"Reading DDUF file {dduf_path}") + with zipfile.ZipFile(str(dduf_path), "r") as zf: + for info in zf.infolist(): + logger.debug(f"Reading entry {info.filename}") + if info.compress_type != zipfile.ZIP_STORED: + raise DDUFCorruptedFileError("Data must not be compressed in DDUF file.") + + try: + _validate_dduf_entry_name(info.filename) + except DDUFInvalidEntryNameError as e: + raise DDUFCorruptedFileError(f"Invalid entry name in DDUF file: {info.filename}") from e + + offset = _get_data_offset(zf, info) + + entries[info.filename] = DDUFEntry( + filename=info.filename, offset=offset, length=info.file_size, dduf_path=dduf_path + ) + + # Consistency checks on the DDUF file + if "model_index.json" not in entries: + raise DDUFCorruptedFileError("Missing required 'model_index.json' entry in DDUF file.") + index = json.loads(entries["model_index.json"].read_text()) + _validate_dduf_structure(index, entries.keys()) + + logger.info(f"Done reading DDUF file {dduf_path}. Found {len(entries)} entries") + return entries + + +def export_entries_as_dduf(dduf_path: str | os.PathLike, entries: Iterable[tuple[str, str | Path | bytes]]) -> None: + """Write a DDUF file from an iterable of entries. + + This is a lower-level helper than [`export_folder_as_dduf`] that allows more flexibility when serializing data. + In particular, you don't need to save the data on disk before exporting it in the DDUF file. + + Args: + dduf_path (`str` or `os.PathLike`): + The path to the DDUF file to write. + entries (`Iterable[tuple[str, Union[str, Path, bytes]]]`): + An iterable of entries to write in the DDUF file. Each entry is a tuple with the filename and the content. + The filename should be the path to the file in the DDUF archive. + The content can be a string or a pathlib.Path representing a path to a file on the local disk or directly the content as bytes. + + Raises: + - [`DDUFExportError`]: If anything goes wrong during the export (e.g. invalid entry name, missing 'model_index.json', etc.). + + Example: + ```python + # Export specific files from the local disk. + >>> from huggingface_hub import export_entries_as_dduf + >>> export_entries_as_dduf( + ... dduf_path="stable-diffusion-v1-4-FP16.dduf", + ... entries=[ # List entries to add to the DDUF file (here, only FP16 weights) + ... ("model_index.json", "path/to/model_index.json"), + ... ("vae/config.json", "path/to/vae/config.json"), + ... ("vae/diffusion_pytorch_model.fp16.safetensors", "path/to/vae/diffusion_pytorch_model.fp16.safetensors"), + ... ("text_encoder/config.json", "path/to/text_encoder/config.json"), + ... ("text_encoder/model.fp16.safetensors", "path/to/text_encoder/model.fp16.safetensors"), + ... # ... add more entries here + ... ] + ... ) + ``` + + ```python + # Export state_dicts one by one from a loaded pipeline + >>> from diffusers import DiffusionPipeline + >>> from typing import Generator, Tuple + >>> import safetensors.torch + >>> from huggingface_hub import export_entries_as_dduf + >>> pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + ... # ... do some work with the pipeline + + >>> def as_entries(pipe: DiffusionPipeline) -> Generator[tuple[str, bytes], None, None]: + ... # Build a generator that yields the entries to add to the DDUF file. + ... # The first element of the tuple is the filename in the DDUF archive (must use UNIX separator!). The second element is the content of the file. + ... # Entries will be evaluated lazily when the DDUF file is created (only 1 entry is loaded in memory at a time) + ... yield "vae/config.json", pipe.vae.to_json_string().encode() + ... yield "vae/diffusion_pytorch_model.safetensors", safetensors.torch.save(pipe.vae.state_dict()) + ... yield "text_encoder/config.json", pipe.text_encoder.config.to_json_string().encode() + ... yield "text_encoder/model.safetensors", safetensors.torch.save(pipe.text_encoder.state_dict()) + ... # ... add more entries here + + >>> export_entries_as_dduf(dduf_path="stable-diffusion-v1-4.dduf", entries=as_entries(pipe)) + ``` + """ + logger.info(f"Exporting DDUF file '{dduf_path}'") + filenames = set() + index = None + with zipfile.ZipFile(str(dduf_path), "w", zipfile.ZIP_STORED) as archive: + for filename, content in entries: + if filename in filenames: + raise DDUFExportError(f"Can't add duplicate entry: {filename}") + filenames.add(filename) + + if filename == "model_index.json": + try: + index = json.loads(_load_content(content).decode()) + except json.JSONDecodeError as e: + raise DDUFExportError("Failed to parse 'model_index.json'.") from e + + try: + filename = _validate_dduf_entry_name(filename) + except DDUFInvalidEntryNameError as e: + raise DDUFExportError(f"Invalid entry name: {filename}") from e + logger.debug(f"Adding entry '{filename}' to DDUF file") + _dump_content_in_archive(archive, filename, content) + + # Consistency checks on the DDUF file + if index is None: + raise DDUFExportError("Missing required 'model_index.json' entry in DDUF file.") + try: + _validate_dduf_structure(index, filenames) + except DDUFCorruptedFileError as e: + raise DDUFExportError("Invalid DDUF file structure.") from e + + logger.info(f"Done writing DDUF file {dduf_path}") + + +def export_folder_as_dduf(dduf_path: str | os.PathLike, folder_path: str | os.PathLike) -> None: + """ + Export a folder as a DDUF file. + + AUses [`export_entries_as_dduf`] under the hood. + + Args: + dduf_path (`str` or `os.PathLike`): + The path to the DDUF file to write. + folder_path (`str` or `os.PathLike`): + The path to the folder containing the diffusion model. + + Example: + ```python + >>> from huggingface_hub import export_folder_as_dduf + >>> export_folder_as_dduf(dduf_path="FLUX.1-dev.dduf", folder_path="path/to/FLUX.1-dev") + ``` + """ + folder_path = Path(folder_path) + + def _iterate_over_folder() -> Iterable[tuple[str, Path]]: + for path in Path(folder_path).glob("**/*"): + if not path.is_file(): + continue + if path.suffix not in DDUF_ALLOWED_ENTRIES: + logger.debug(f"Skipping file '{path}' (file type not allowed)") + continue + path_in_archive = path.relative_to(folder_path) + if len(path_in_archive.parts) >= 3: + logger.debug(f"Skipping file '{path}' (nested directories not allowed)") + continue + yield path_in_archive.as_posix(), path + + export_entries_as_dduf(dduf_path, _iterate_over_folder()) + + +def _dump_content_in_archive(archive: zipfile.ZipFile, filename: str, content: str | os.PathLike | bytes) -> None: + with archive.open(filename, "w", force_zip64=True) as archive_fh: + if isinstance(content, (str, Path)): + content_path = Path(content) + with content_path.open("rb") as content_fh: + shutil.copyfileobj(content_fh, archive_fh, 1024 * 1024 * 8) # type: ignore[misc] + elif isinstance(content, bytes): + archive_fh.write(content) + else: + raise DDUFExportError(f"Invalid content type for {filename}. Must be str, Path or bytes.") + + +def _load_content(content: str | Path | bytes) -> bytes: + """Load the content of an entry as bytes. + + Used only for small checks (not to dump content into archive). + """ + if isinstance(content, (str, Path)): + return Path(content).read_bytes() + elif isinstance(content, bytes): + return content + else: + raise DDUFExportError(f"Invalid content type. Must be str, Path or bytes. Got {type(content)}.") + + +def _validate_dduf_entry_name(entry_name: str) -> str: + if "." + entry_name.split(".")[-1] not in DDUF_ALLOWED_ENTRIES: + raise DDUFInvalidEntryNameError(f"File type not allowed: {entry_name}") + if "\\" in entry_name: + raise DDUFInvalidEntryNameError(f"Entry names must use UNIX separators ('/'). Got {entry_name}.") + entry_name = entry_name.strip("/") + if entry_name.count("/") > 1: + raise DDUFInvalidEntryNameError(f"DDUF only supports 1 level of directory. Got {entry_name}.") + return entry_name + + +def _validate_dduf_structure(index: Any, entry_names: Iterable[str]) -> None: + """ + Consistency checks on the DDUF file structure. + + Rules: + - The 'model_index.json' entry is required and must contain a dictionary. + - Each folder name must correspond to an entry in 'model_index.json'. + - Each folder must contain at least a config file ('config.json', 'tokenizer_config.json', 'preprocessor_config.json', 'scheduler_config.json'). + + Args: + index (Any): + The content of the 'model_index.json' entry. + entry_names (Iterable[str]): + The list of entry names in the DDUF file. + + Raises: + - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format). + """ + if not isinstance(index, dict): + raise DDUFCorruptedFileError(f"Invalid 'model_index.json' content. Must be a dictionary. Got {type(index)}.") + + dduf_folders = {entry.split("/")[0] for entry in entry_names if "/" in entry} + for folder in dduf_folders: + if folder not in index: + raise DDUFCorruptedFileError(f"Missing required entry '{folder}' in 'model_index.json'.") + if not any(f"{folder}/{required_entry}" in entry_names for required_entry in DDUF_FOLDER_REQUIRED_ENTRIES): + raise DDUFCorruptedFileError( + f"Missing required file in folder '{folder}'. Must contains at least one of {DDUF_FOLDER_REQUIRED_ENTRIES}." + ) + + +def _get_data_offset(zf: zipfile.ZipFile, info: zipfile.ZipInfo) -> int: + """ + Calculate the data offset for a file in a ZIP archive. + + Args: + zf (`zipfile.ZipFile`): + The opened ZIP file. Must be opened in read mode. + info (`zipfile.ZipInfo`): + The file info. + + Returns: + int: The offset of the file data in the ZIP archive. + """ + if zf.fp is None: + raise DDUFCorruptedFileError("ZipFile object must be opened in read mode.") + + # Step 1: Get the local file header offset + header_offset = info.header_offset + + # Step 2: Read the local file header + zf.fp.seek(header_offset) + local_file_header = zf.fp.read(30) # Fixed-size part of the local header + + if len(local_file_header) < 30: + raise DDUFCorruptedFileError("Incomplete local file header.") + + # Step 3: Parse the header fields to calculate the start of file data + # Local file header: https://en.wikipedia.org/wiki/ZIP_(file_format)#File_headers + filename_len = int.from_bytes(local_file_header[26:28], "little") + extra_field_len = int.from_bytes(local_file_header[28:30], "little") + + # Data offset is after the fixed header, filename, and extra fields + data_offset = header_offset + 30 + filename_len + extra_field_len + + return data_offset diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_torch.py b/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_torch.py new file mode 100644 index 0000000000000000000000000000000000000000..19516aa7feea7c206391dd3b9deecf6157b7e455 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_torch.py @@ -0,0 +1,1046 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains pytorch-specific helpers.""" + +import importlib +import importlib.util +import json +import os +import re +from collections import defaultdict, namedtuple +from collections.abc import Iterable +from functools import lru_cache +from pathlib import Path, PureWindowsPath +from typing import TYPE_CHECKING, Any, NamedTuple, Union + +from packaging import version + +from .. import constants, logging +from ._base import MAX_SHARD_SIZE, StateDictSplit, split_state_dict_into_shards_factory + + +logger = logging.get_logger(__file__) + +if TYPE_CHECKING: + import torch + +# SAVING + + +def save_torch_model( + model: "torch.nn.Module", + save_directory: str | Path, + *, + filename_pattern: str | None = None, + force_contiguous: bool = True, + max_shard_size: int | str = MAX_SHARD_SIZE, + metadata: dict[str, str] | None = None, + safe_serialization: bool = True, + is_main_process: bool = True, + shared_tensors_to_discard: list[str] | None = None, +): + """ + Saves a given torch model to disk, handling sharding and shared tensors issues. + + See also [`save_torch_state_dict`] to save a state dict with more flexibility. + + For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors). + + The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are + saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard, + an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses + [`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as + safetensors (the default). Otherwise, the shards are saved as pickle. + + Before saving the model, the `save_directory` is cleaned from any previous shard files. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + > [!WARNING] + > If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving. + + Args: + model (`torch.nn.Module`): + The model to save on disk. + save_directory (`str` or `Path`): + The directory in which the model will be saved. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization` + parameter. + force_contiguous (`boolean`, *optional*): + Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the + model, but it could potentially change performance if the layout of the tensor was chosen specifically for + that reason. Defaults to `True`. + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + metadata (`dict[str, str]`, *optional*): + Extra information to save along with the model. Some metadata will be added for each dropped tensors. + This information will not be enough to recover the entire shared structure but might help understanding + things. + safe_serialization (`bool`, *optional*): + Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle. + Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed + in a future version. + is_main_process (`bool`, *optional*): + Whether the process calling this is the main process or not. Useful when in distributed training like + TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on + the main process to avoid race conditions. Defaults to True. + shared_tensors_to_discard (`list[str]`, *optional*): + List of tensor names to drop when saving shared tensors. If not provided and shared tensors are + detected, it will drop the first name alphabetically. + + Example: + + ```py + >>> from huggingface_hub import save_torch_model + >>> model = ... # A PyTorch model + + # Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors. + >>> save_torch_model(model, "path/to/folder") + + # Load model back + >>> from huggingface_hub import load_torch_model # TODO + >>> load_torch_model(model, "path/to/folder") + >>> + ``` + """ + save_torch_state_dict( + state_dict=model.state_dict(), + filename_pattern=filename_pattern, + force_contiguous=force_contiguous, + max_shard_size=max_shard_size, + metadata=metadata, + safe_serialization=safe_serialization, + save_directory=save_directory, + is_main_process=is_main_process, + shared_tensors_to_discard=shared_tensors_to_discard, + ) + + +def save_torch_state_dict( + state_dict: dict[str, "torch.Tensor"], + save_directory: str | Path, + *, + filename_pattern: str | None = None, + force_contiguous: bool = True, + max_shard_size: int | str = MAX_SHARD_SIZE, + metadata: dict[str, str] | None = None, + safe_serialization: bool = True, + is_main_process: bool = True, + shared_tensors_to_discard: list[str] | None = None, +) -> None: + """ + Save a model state dictionary to the disk, handling sharding and shared tensors issues. + + See also [`save_torch_model`] to directly save a PyTorch model. + + For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors). + + The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are + saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard, + an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses + [`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as + safetensors (the default). Otherwise, the shards are saved as pickle. + + Before saving the model, the `save_directory` is cleaned from any previous shard files. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + > [!WARNING] + > If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving. + + Args: + state_dict (`dict[str, torch.Tensor]`): + The state dictionary to save. + save_directory (`str` or `Path`): + The directory in which the model will be saved. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization` + parameter. + force_contiguous (`boolean`, *optional*): + Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the + model, but it could potentially change performance if the layout of the tensor was chosen specifically for + that reason. Defaults to `True`. + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + metadata (`dict[str, str]`, *optional*): + Extra information to save along with the model. Some metadata will be added for each dropped tensors. + This information will not be enough to recover the entire shared structure but might help understanding + things. + safe_serialization (`bool`, *optional*): + Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle. + Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed + in a future version. + is_main_process (`bool`, *optional*): + Whether the process calling this is the main process or not. Useful when in distributed training like + TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on + the main process to avoid race conditions. Defaults to True. + shared_tensors_to_discard (`list[str]`, *optional*): + List of tensor names to drop when saving shared tensors. If not provided and shared tensors are + detected, it will drop the first name alphabetically. + + Example: + + ```py + >>> from huggingface_hub import save_torch_state_dict + >>> model = ... # A PyTorch model + + # Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors. + >>> state_dict = model_to_save.state_dict() + >>> save_torch_state_dict(state_dict, "path/to/folder") + ``` + """ + save_directory = str(save_directory) + + if filename_pattern is None: + filename_pattern = ( + constants.SAFETENSORS_WEIGHTS_FILE_PATTERN + if safe_serialization + else constants.PYTORCH_WEIGHTS_FILE_PATTERN + ) + + if metadata is None: + metadata = {} + if safe_serialization: + try: + from safetensors.torch import save_file as save_file_fn + except ImportError as e: + raise ImportError( + "Please install `safetensors` to use safe serialization. " + "You can install it with `pip install safetensors`." + ) from e + # Clean state dict for safetensors + state_dict = _clean_state_dict_for_safetensors( + state_dict, + metadata, + force_contiguous=force_contiguous, + shared_tensors_to_discard=shared_tensors_to_discard, + ) + else: + from torch import save as save_file_fn # type: ignore[assignment, no-redef] + + logger.warning( + "You are using unsafe serialization. Due to security reasons, it is recommended not to load " + "pickled models from untrusted sources. If you intend to share your model, we strongly recommend " + "using safe serialization by installing `safetensors` with `pip install safetensors`." + ) + # Split dict + state_dict_split = split_torch_state_dict_into_shards( + state_dict, filename_pattern=filename_pattern, max_shard_size=max_shard_size + ) + + # Only main process should clean up existing files to avoid race conditions in distributed environment + if is_main_process: + existing_files_regex = re.compile(filename_pattern.format(suffix=r"(-\d{5}-of-\d{5})?") + r"(\.index\.json)?") + for filename in os.listdir(save_directory): + if existing_files_regex.match(filename): + try: + logger.debug(f"Removing existing file '{filename}' from folder.") + os.remove(os.path.join(save_directory, filename)) + except Exception as e: + logger.warning( + f"Error when trying to remove existing '{filename}' from folder: {e}. Continuing..." + ) + + # Save each shard + per_file_metadata = {"format": "pt"} + if not state_dict_split.is_sharded: + per_file_metadata.update(metadata) + safe_file_kwargs = {"metadata": per_file_metadata} if safe_serialization else {} + for filename, tensors in state_dict_split.filename_to_tensors.items(): + shard = {tensor: state_dict[tensor] for tensor in tensors} + save_file_fn(shard, os.path.join(save_directory, filename), **safe_file_kwargs) # ty: ignore[invalid-argument-type] + logger.debug(f"Shard saved to {filename}") + + # Save the index (if any) + if state_dict_split.is_sharded: + index_path = filename_pattern.format(suffix="") + ".index.json" + index = { + "metadata": {**state_dict_split.metadata, **metadata}, + "weight_map": state_dict_split.tensor_to_filename, + } + with open(os.path.join(save_directory, index_path), "w") as f: + json.dump(index, f, indent=2) + logger.info( + f"The model is bigger than the maximum size per checkpoint ({max_shard_size}). " + f"Model weighs have been saved in {len(state_dict_split.filename_to_tensors)} checkpoint shards. " + f"You can find where each parameters has been saved in the index located at {index_path}." + ) + + logger.info(f"Model weights successfully saved to {save_directory}!") + + +def split_torch_state_dict_into_shards( + state_dict: dict[str, "torch.Tensor"], + *, + filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN, + max_shard_size: int | str = MAX_SHARD_SIZE, +) -> StateDictSplit: + """ + Split a model state dictionary in shards so that each shard is smaller than a given size. + + The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization + made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we + have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not + [6+2+2GB], [6+2GB], [6GB]. + + + > [!TIP] + > To save a model state dictionary to the disk, see [`save_torch_state_dict`]. This helper uses + > `split_torch_state_dict_into_shards` under the hood. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + Args: + state_dict (`dict[str, torch.Tensor]`): + The state dictionary to save. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"`. + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + + Returns: + [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them. + + Example: + ```py + >>> import json + >>> import os + >>> from safetensors.torch import save_file as safe_save_file + >>> from huggingface_hub import split_torch_state_dict_into_shards + + >>> def save_state_dict(state_dict: dict[str, torch.Tensor], save_directory: str): + ... state_dict_split = split_torch_state_dict_into_shards(state_dict) + ... for filename, tensors in state_dict_split.filename_to_tensors.items(): + ... shard = {tensor: state_dict[tensor] for tensor in tensors} + ... safe_save_file( + ... shard, + ... os.path.join(save_directory, filename), + ... metadata={"format": "pt"}, + ... ) + ... if state_dict_split.is_sharded: + ... index = { + ... "metadata": state_dict_split.metadata, + ... "weight_map": state_dict_split.tensor_to_filename, + ... } + ... with open(os.path.join(save_directory, "model.safetensors.index.json"), "w") as f: + ... f.write(json.dumps(index, indent=2)) + ``` + """ + return split_state_dict_into_shards_factory( + state_dict, + max_shard_size=max_shard_size, + filename_pattern=filename_pattern, + get_storage_size=get_torch_storage_size, + get_storage_id=get_torch_storage_id, + ) + + +# LOADING + + +def load_torch_model( + model: "torch.nn.Module", + checkpoint_path: str | os.PathLike, + *, + strict: bool = False, + safe: bool = True, + weights_only: bool = False, + map_location: Union[str, "torch.device"] | None = None, + mmap: bool = False, + filename_pattern: str | None = None, +) -> NamedTuple: + """ + Load a checkpoint into a model, handling both sharded and non-sharded checkpoints. + + Args: + model (`torch.nn.Module`): + The model in which to load the checkpoint. + checkpoint_path (`str` or `os.PathLike`): + Path to either the checkpoint file or directory containing the checkpoint(s). + strict (`bool`, *optional*, defaults to `False`): + Whether to strictly enforce that the keys in the model state dict match the keys in the checkpoint. + safe (`bool`, *optional*, defaults to `True`): + If `safe` is True, the safetensors files will be loaded. If `safe` is False, the function + will first attempt to load safetensors files if they are available, otherwise it will fall back to loading + pickle files. `filename_pattern` parameter takes precedence over `safe` parameter. + weights_only (`bool`, *optional*, defaults to `False`): + If True, only loads the model weights without optimizer states and other metadata. + Only supported in PyTorch >= 1.13. + map_location (`str` or `torch.device`, *optional*): + A `torch.device` object, string or a dict specifying how to remap storage locations. It + indicates the location where all tensors should be loaded. + mmap (`bool`, *optional*, defaults to `False`): + Whether to use memory-mapped file loading. Memory mapping can improve loading performance + for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints. + filename_pattern (`str`, *optional*): + The pattern to look for the index file. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"`. + Returns: + `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields. + - `missing_keys` is a list of str containing the missing keys, i.e. keys that are in the model but not in the checkpoint. + - `unexpected_keys` is a list of str containing the unexpected keys, i.e. keys that are in the checkpoint but not in the model. + + Raises: + [`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError) + If the checkpoint file or directory does not exist. + [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) + If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the checkpoint path is invalid or if the checkpoint format cannot be determined. + + Example: + ```python + >>> from huggingface_hub import load_torch_model + >>> model = ... # A PyTorch model + >>> load_torch_model(model, "path/to/checkpoint") + ``` + """ + checkpoint_path = Path(checkpoint_path) + + if not checkpoint_path.exists(): + raise ValueError(f"Checkpoint path {checkpoint_path} does not exist") + # 1. Check if checkpoint is a single file + if checkpoint_path.is_file(): + state_dict = load_state_dict_from_file( + checkpoint_file=checkpoint_path, + map_location=map_location, + weights_only=weights_only, + ) + return model.load_state_dict(state_dict, strict=strict) + + # 2. If not, checkpoint_path is a directory + if filename_pattern is None: + filename_pattern = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN + index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json") + # Only fallback to pickle format if safetensors index is not found and safe is False. + if not index_path.is_file() and not safe: + filename_pattern = constants.PYTORCH_WEIGHTS_FILE_PATTERN + + index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json") + + if index_path.is_file(): + return _load_sharded_checkpoint( + model=model, + save_directory=checkpoint_path, + strict=strict, + weights_only=weights_only, + filename_pattern=filename_pattern, + ) + + # Look for single model file + model_files = list(checkpoint_path.glob("*.safetensors" if safe else "*.bin")) + if len(model_files) == 1: + state_dict = load_state_dict_from_file( + checkpoint_file=model_files[0], + map_location=map_location, + weights_only=weights_only, + mmap=mmap, + ) + return model.load_state_dict(state_dict, strict=strict) + + raise ValueError( + f"Directory '{checkpoint_path}' does not contain a valid checkpoint. " + "Expected either a sharded checkpoint with an index file, or a single model file." + ) + + +def _load_sharded_checkpoint( + model: "torch.nn.Module", + save_directory: os.PathLike, + *, + strict: bool = False, + weights_only: bool = False, + filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN, +) -> NamedTuple: + """ + Loads a sharded checkpoint into a model. This is the same as + [`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict) + but for a sharded checkpoint. Each shard is loaded one by one and removed from memory after being loaded into the model. + + Args: + model (`torch.nn.Module`): + The model in which to load the checkpoint. + save_directory (`str` or `os.PathLike`): + A path to a folder containing the sharded checkpoint. + strict (`bool`, *optional*, defaults to `False`): + Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint. + weights_only (`bool`, *optional*, defaults to `False`): + If True, only loads the model weights without optimizer states and other metadata. + Only supported in PyTorch >= 1.13. + filename_pattern (`str`, *optional*, defaults to `"model{suffix}.safetensors"`): + The pattern to look for the index file. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"`. + + Returns: + `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields, + - `missing_keys` is a list of str containing the missing keys + - `unexpected_keys` is a list of str containing the unexpected keys + """ + + # 1. Load and validate index file + # The index file contains mapping of parameter names to shard files + index_path = filename_pattern.format(suffix="") + ".index.json" + index_file = os.path.join(save_directory, index_path) + with open(index_file, encoding="utf-8") as f: + index = json.load(f) + + # 2. Validate shard filenames from the index + # This prevents path traversal attacks and extension confusion attacks + # (e.g. a safetensors index referencing .bin pickle files) + expected_extension = Path(filename_pattern.format(suffix="")).suffix # e.g. ".safetensors" + shard_files = list(set(index["weight_map"].values())) + for shard_file in shard_files: + # Reject anything that could escape `save_directory` on any host OS: + # POSIX absolute ("/tmp/x"), Windows drive ("C:x", "C:\\x"), UNC + # ("\\\\server\\share\\x"), rooted-without-drive ("\\x", "/x"), or + # ".." traversal — including "..\\x" which `os.path.isabs` never caught on POSIX. + # + # We parse with `PureWindowsPath` *regardless of host OS*: it treats both "/" and + # "\\" as separators and exposes `drive` / `root`, so a single check rejects a + # malicious index file on Linux too (e.g. if it's later opened on Windows). The + # only over-strict case is a POSIX filename like "a:foo" which would be parsed as + # drive "a:" — such names are never produced for safetensors shards and would + # break on Windows anyway, so rejecting them is fine. + win_path = PureWindowsPath(shard_file) + if win_path.drive or win_path.root or ".." in win_path.parts: + raise ValueError( + f"Invalid shard filename '{shard_file}' in index file '{index_file}'. " + "Shard filenames must be relative paths without '..' components." + ) + # Reject extension mismatch (e.g. .bin shard in a .safetensors index) + if not shard_file.endswith(expected_extension): + raise ValueError( + f"Invalid shard filename '{shard_file}' in index file '{index_file}'. " + f"Expected '{expected_extension}' extension to match the index format." + ) + + # 3. Validate keys if in strict mode + # This is done before loading any shards to fail fast + if strict: + _validate_keys_for_strict_loading(model, index["weight_map"].keys()) + + # 4. Load each shard using `load_state_dict` + # Get unique shard files (multiple parameters can be in same shard) + for shard_file in shard_files: + # Load shard into memory + shard_path = os.path.join(save_directory, shard_file) + state_dict = load_state_dict_from_file( + shard_path, + map_location="cpu", + weights_only=weights_only, + ) + # Update model with parameters from this shard + model.load_state_dict(state_dict, strict=strict) + # Explicitly remove the state dict from memory + del state_dict + + # 5. Return compatibility info + loaded_keys = set(index["weight_map"].keys()) + model_keys = set(model.state_dict().keys()) + return _IncompatibleKeys( + missing_keys=list(model_keys - loaded_keys), unexpected_keys=list(loaded_keys - model_keys) + ) + + +def load_state_dict_from_file( + checkpoint_file: str | os.PathLike, + map_location: Union[str, "torch.device"] | None = None, + weights_only: bool = False, + mmap: bool = False, +) -> dict[str, "torch.Tensor"] | Any: + """ + Loads a checkpoint file, handling both safetensors and pickle checkpoint formats. + + Args: + checkpoint_file (`str` or `os.PathLike`): + Path to the checkpoint file to load. Can be either a safetensors or pickle (`.bin`) checkpoint. + map_location (`str` or `torch.device`, *optional*): + A `torch.device` object, string or a dict specifying how to remap storage locations. It + indicates the location where all tensors should be loaded. + weights_only (`bool`, *optional*, defaults to `False`): + If True, only loads the model weights without optimizer states and other metadata. + Only supported for pickle (`.bin`) checkpoints with PyTorch >= 1.13. Has no effect when + loading safetensors files. + mmap (`bool`, *optional*, defaults to `False`): + Whether to use memory-mapped file loading. Memory mapping can improve loading performance + for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints. Has no effect when + loading safetensors files, as the `safetensors` library uses memory mapping by default. + + Returns: + `Union[dict[str, "torch.Tensor"], Any]`: The loaded checkpoint. + - For safetensors files: always returns a dictionary mapping parameter names to tensors. + - For pickle files: returns any Python object that was pickled (commonly a state dict, but could be + an entire model, optimizer state, or any other Python object). + + Raises: + [`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError) + If the checkpoint file does not exist. + [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) + If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively. + [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) + If the checkpoint file format is invalid or if git-lfs files are not properly downloaded. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the checkpoint file path is empty or invalid. + + Example: + ```python + >>> from huggingface_hub import load_state_dict_from_file + + # Load a PyTorch checkpoint + >>> state_dict = load_state_dict_from_file("path/to/model.bin", map_location="cpu") + >>> model.load_state_dict(state_dict) + + # Load a safetensors checkpoint + >>> state_dict = load_state_dict_from_file("path/to/model.safetensors") + >>> model.load_state_dict(state_dict) + ``` + """ + checkpoint_path = Path(checkpoint_file) + + # Check if file exists and is a regular file (not a directory) + if not checkpoint_path.is_file(): + raise FileNotFoundError( + f"No checkpoint file found at '{checkpoint_path}'. Please verify the path is correct and " + "the file has been properly downloaded." + ) + + # Load safetensors checkpoint + if checkpoint_path.suffix == ".safetensors": + try: + from safetensors import safe_open + from safetensors.torch import load_file + except ImportError as e: + raise ImportError( + "Please install `safetensors` to load safetensors checkpoint. " + "You can install it with `pip install safetensors`." + ) from e + + # Check format of the archive + with safe_open(checkpoint_file, framework="pt") as f: # type: ignore[attr-defined] + metadata = f.metadata() + # see comment: https://github.com/huggingface/transformers/blob/3d213b57fe74302e5902d68ed9478c3ad1aaa713/src/transformers/modeling_utils.py#L3966 + if metadata is not None and metadata.get("format") not in ["pt", "mlx"]: + raise OSError( + f"The safetensors archive passed at {checkpoint_file} does not contain the valid metadata. Make sure " + "you save your model with the `save_torch_model` method." + ) + device = str(map_location.type) if map_location is not None and hasattr(map_location, "type") else map_location + # meta device is not supported with safetensors, falling back to CPU + if device == "meta": + logger.warning("Meta device is not supported with safetensors. Falling back to CPU device.") + device = "cpu" + return load_file(checkpoint_file, device=device) # type: ignore[arg-type] + # Otherwise, load from pickle + try: + import torch + from torch import load + except ImportError as e: + raise ImportError( + "Please install `torch` to load torch tensors. You can install it with `pip install torch`." + ) from e + # Add additional kwargs, mmap is only supported in torch >= 2.1.0 + additional_kwargs = {} + if version.parse(torch.__version__) >= version.parse("2.1.0"): + additional_kwargs["mmap"] = mmap + + # weights_only is only supported in torch >= 1.13.0 + if version.parse(torch.__version__) >= version.parse("1.13.0"): + additional_kwargs["weights_only"] = weights_only + + return load( + checkpoint_file, + map_location=map_location, + **additional_kwargs, + ) + + +# HELPERS + + +def _validate_keys_for_strict_loading( + model: "torch.nn.Module", + loaded_keys: Iterable[str], +) -> None: + """ + Validate that model keys match loaded keys when strict loading is enabled. + + Args: + model: The PyTorch model being loaded + loaded_keys: The keys present in the checkpoint + + Raises: + RuntimeError: If there are missing or unexpected keys in strict mode + """ + loaded_keys_set = set(loaded_keys) + model_keys = set(model.state_dict().keys()) + missing_keys = model_keys - loaded_keys_set # Keys in model but not in checkpoint + unexpected_keys = loaded_keys_set - model_keys # Keys in checkpoint but not in model + + if missing_keys or unexpected_keys: + error_message = f"Error(s) in loading state_dict for {model.__class__.__name__}" + if missing_keys: + str_missing_keys = ",".join([f'"{k}"' for k in sorted(missing_keys)]) + error_message += f"\nMissing key(s): {str_missing_keys}." + if unexpected_keys: + str_unexpected_keys = ",".join([f'"{k}"' for k in sorted(unexpected_keys)]) + error_message += f"\nUnexpected key(s): {str_unexpected_keys}." + raise RuntimeError(error_message) + + +def _get_unique_id(tensor: "torch.Tensor") -> int | tuple[Any, ...]: + """Returns a unique id for plain tensor + or a (potentially nested) Tuple of unique id for the flattened Tensor + if the input is a wrapper tensor subclass Tensor + """ + + try: + from torch.distributed.tensor import DTensor + + if isinstance(tensor, DTensor): + local_tensor = tensor.to_local() + return local_tensor.storage().data_ptr() + except ImportError: + pass + + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] + return tuple(_get_unique_id(getattr(tensor, attr)) for attr in attrs) + + except ImportError: + # for torch version less than 2.1, we can fall back to original implementation + pass + + if tensor.device.type == "xla" and is_torch_tpu_available(): + # NOTE: xla tensors don't have storage + # use some other unique id to distinguish. + # this is a XLA tensor, it must be created using torch_xla's + # device. So the following import is safe: + import torch_xla # type: ignore[import] + + unique_id = torch_xla._XLAC._xla_get_tensor_id(tensor) + else: + unique_id = storage_ptr(tensor) + + return unique_id + + +def get_torch_storage_id(tensor: "torch.Tensor") -> tuple["torch.device", int | tuple[Any, ...], int] | None: + """ + Return unique identifier to a tensor storage. + + Multiple different tensors can share the same underlying storage. This identifier is + guaranteed to be unique and constant for this tensor's storage during its lifetime. Two tensor storages with + non-overlapping lifetimes may have the same id. + In the case of meta tensors, we return None since we can't tell if they share the same storage. + + Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/pytorch_utils.py#L278. + """ + if tensor.device.type == "meta": + return None + else: + return tensor.device, _get_unique_id(tensor), get_torch_storage_size(tensor) + + +def get_torch_storage_size(tensor: "torch.Tensor") -> int: + """ + Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L31C1-L41C59 + """ + try: + from torch.distributed.tensor import DTensor + + if isinstance(tensor, DTensor): + # this returns the size of the FULL tensor in bytes + return tensor.nbytes + except ImportError: + pass + + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] + return sum(get_torch_storage_size(getattr(tensor, attr)) for attr in attrs) + except ImportError: + # for torch version less than 2.1, we can fall back to original implementation + pass + + try: + return tensor.untyped_storage().nbytes() + except AttributeError: + # Fallback for torch==1.10 + try: + return tensor.storage().size() * _get_dtype_size(tensor.dtype) + except NotImplementedError: + # Fallback for meta storage + # On torch >=2.0 this is the tensor size + return tensor.nelement() * _get_dtype_size(tensor.dtype) + + +@lru_cache +def is_torch_tpu_available(check_device=True): + """ + Checks if `torch_xla` is installed and potentially if a TPU is in the environment + + Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/utils/import_utils.py#L463. + """ + if importlib.util.find_spec("torch_xla") is not None: + if check_device: + # We need to check if `xla_device` can be found, will raise a RuntimeError if not + try: + import torch_xla.core.xla_model as xm # type: ignore[import] + + _ = xm.xla_device() + return True + except RuntimeError: + return False + return True + return False + + +def storage_ptr(tensor: "torch.Tensor") -> int | tuple[Any, ...]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L11. + """ + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + return _get_unique_id(tensor) # type: ignore + except ImportError: + # for torch version less than 2.1, we can fall back to original implementation + pass + + try: + return tensor.untyped_storage().data_ptr() + except Exception: + # Fallback for torch==1.10 + try: + return tensor.storage().data_ptr() + except NotImplementedError: + # Fallback for meta storage + return 0 + + +def _clean_state_dict_for_safetensors( + state_dict: dict[str, "torch.Tensor"], + metadata: dict[str, str], + force_contiguous: bool = True, + shared_tensors_to_discard: list[str] | None = None, +): + """Remove shared tensors from state_dict and update metadata accordingly (for reloading). + + Warning: `state_dict` and `metadata` are mutated in-place! + + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L155. + """ + to_removes = _remove_duplicate_names(state_dict, discard_names=shared_tensors_to_discard) + for kept_name, to_remove_group in to_removes.items(): + for to_remove in to_remove_group: + if metadata is None: + metadata = {} + + if to_remove not in metadata: + # Do not override user data + metadata[to_remove] = kept_name + del state_dict[to_remove] + if force_contiguous: + state_dict = {k: v.contiguous() for k, v in state_dict.items()} + return state_dict + + +def _end_ptr(tensor: "torch.Tensor") -> int: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L23. + """ + if tensor.nelement(): + stop = tensor.view(-1)[-1].data_ptr() + _get_dtype_size(tensor.dtype) + else: + stop = tensor.data_ptr() + return stop + + +def _filter_shared_not_shared(tensors: list[set[str]], state_dict: dict[str, "torch.Tensor"]) -> list[set[str]]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L44 + """ + filtered_tensors = [] + for shared in tensors: + if len(shared) < 2: + filtered_tensors.append(shared) + continue + + areas = [] + for name in shared: + tensor = state_dict[name] + areas.append((tensor.data_ptr(), _end_ptr(tensor), name)) + areas.sort() + + _, last_stop, last_name = areas[0] + filtered_tensors.append({last_name}) + for start, stop, name in areas[1:]: + if start >= last_stop: + filtered_tensors.append({name}) + else: + filtered_tensors[-1].add(name) + last_stop = stop + + return filtered_tensors + + +def _find_shared_tensors(state_dict: dict[str, "torch.Tensor"]) -> list[set[str]]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L69. + """ + import torch + + tensors_dict = defaultdict(set) + for k, v in state_dict.items(): + if v.device != torch.device("meta") and storage_ptr(v) != 0 and get_torch_storage_size(v) != 0: + # Need to add device as key because of multiple GPU. + tensors_dict[(v.device, storage_ptr(v), get_torch_storage_size(v))].add(k) + tensors = list(sorted(tensors_dict.values())) + tensors = _filter_shared_not_shared(tensors, state_dict) + return tensors + + +def _is_complete(tensor: "torch.Tensor") -> bool: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80 + """ + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] + return all(_is_complete(getattr(tensor, attr)) for attr in attrs) + except ImportError: + # for torch version less than 2.1, we can fall back to original implementation + pass + + return tensor.data_ptr() == storage_ptr(tensor) and tensor.nelement() * _get_dtype_size( + tensor.dtype + ) == get_torch_storage_size(tensor) + + +def _remove_duplicate_names( + state_dict: dict[str, "torch.Tensor"], + *, + preferred_names: list[str] | None = None, + discard_names: list[str] | None = None, +) -> dict[str, list[str]]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80 + """ + if preferred_names is None: + preferred_names = [] + unique_preferred_names = set(preferred_names) + if discard_names is None: + discard_names = [] + unique_discard_names = set(discard_names) + + shareds = _find_shared_tensors(state_dict) + to_remove = defaultdict(list) + for shared in shareds: + complete_names = {name for name in shared if _is_complete(state_dict[name])} + if not complete_names: + raise RuntimeError( + "Error while trying to find names to remove to save state dict, but found no suitable name to keep" + f" for saving amongst: {shared}. None is covering the entire storage. Refusing to save/load the model" + " since you could be storing much more memory than needed. Please refer to" + " https://huggingface.co/docs/safetensors/torch_shared_tensors for more information. Or open an" + " issue." + ) + + keep_name = sorted(list(complete_names))[0] + + # Mechanism to preferentially select keys to keep + # coming from the on-disk file to allow + # loading models saved with a different choice + # of keep_name + preferred = complete_names.difference(unique_discard_names) + if preferred: + keep_name = sorted(list(preferred))[0] + + if unique_preferred_names: + preferred = unique_preferred_names.intersection(complete_names) + if preferred: + keep_name = sorted(list(preferred))[0] + for name in sorted(shared): + if name != keep_name: + to_remove[keep_name].append(name) + return to_remove + + +@lru_cache +def _get_dtype_size(dtype: "torch.dtype") -> int: + """ + Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L344 + """ + import torch + + # torch.float8 formats require 2.1; we do not support these dtypes on earlier versions + _float8_e4m3fn = getattr(torch, "float8_e4m3fn", None) + _float8_e5m2 = getattr(torch, "float8_e5m2", None) + _SIZE = { + torch.int64: 8, + torch.float32: 4, + torch.int32: 4, + torch.bfloat16: 2, + torch.float16: 2, + torch.int16: 2, + torch.uint8: 1, + torch.int8: 1, + torch.bool: 1, + torch.float64: 8, + _float8_e4m3fn: 1, + _float8_e5m2: 1, + } + return _SIZE[dtype] + + +class _IncompatibleKeys(namedtuple("IncompatibleKeys", ["missing_keys", "unexpected_keys"])): + """ + This is used to report missing and unexpected keys in the state dict. + Taken from https://github.com/pytorch/pytorch/blob/main/torch/nn/modules/module.py#L52. + + """ + + def __repr__(self) -> str: + if not self.missing_keys and not self.unexpected_keys: + return "" + return super().__repr__() + + __str__ = __repr__ diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/templates/datasetcard_template.md b/.venv/lib/python3.13/site-packages/huggingface_hub/templates/datasetcard_template.md new file mode 100644 index 0000000000000000000000000000000000000000..9af29ebbed93653ec74a8952e314e7554323ef15 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/templates/datasetcard_template.md @@ -0,0 +1,143 @@ +--- +# For reference on dataset card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/datasetcard.md?plain=1 +# Doc / guide: https://huggingface.co/docs/hub/datasets-cards +{{ card_data }} +--- + +# Dataset Card for {{ pretty_name | default("Dataset Name", true) }} + + + +{{ dataset_summary | default("", true) }} + +## Dataset Details + +### Dataset Description + + + +{{ dataset_description | default("", true) }} + +- **Curated by:** {{ curators | default("[More Information Needed]", true)}} +- **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}} +- **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}} +- **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}} +- **License:** {{ license | default("[More Information Needed]", true)}} + +### Dataset Sources [optional] + + + +- **Repository:** {{ repo | default("[More Information Needed]", true)}} +- **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}} +- **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}} + +## Uses + + + +### Direct Use + + + +{{ direct_use | default("[More Information Needed]", true)}} + +### Out-of-Scope Use + + + +{{ out_of_scope_use | default("[More Information Needed]", true)}} + +## Dataset Structure + + + +{{ dataset_structure | default("[More Information Needed]", true)}} + +## Dataset Creation + +### Curation Rationale + + + +{{ curation_rationale_section | default("[More Information Needed]", true)}} + +### Source Data + + + +#### Data Collection and Processing + + + +{{ data_collection_and_processing_section | default("[More Information Needed]", true)}} + +#### Who are the source data producers? + + + +{{ source_data_producers_section | default("[More Information Needed]", true)}} + +### Annotations [optional] + + + +#### Annotation process + + + +{{ annotation_process_section | default("[More Information Needed]", true)}} + +#### Who are the annotators? + + + +{{ who_are_annotators_section | default("[More Information Needed]", true)}} + +#### Personal and Sensitive Information + + + +{{ personal_and_sensitive_information | default("[More Information Needed]", true)}} + +## Bias, Risks, and Limitations + + + +{{ bias_risks_limitations | default("[More Information Needed]", true)}} + +### Recommendations + + + +{{ bias_recommendations | default("Users should be made aware of the risks, biases and limitations of the dataset. More information needed for further recommendations.", true)}} + +## Citation [optional] + + + +**BibTeX:** + +{{ citation_bibtex | default("[More Information Needed]", true)}} + +**APA:** + +{{ citation_apa | default("[More Information Needed]", true)}} + +## Glossary [optional] + + + +{{ glossary | default("[More Information Needed]", true)}} + +## More Information [optional] + +{{ more_information | default("[More Information Needed]", true)}} + +## Dataset Card Authors [optional] + +{{ dataset_card_authors | default("[More Information Needed]", true)}} + +## Dataset Card Contact + +{{ dataset_card_contact | default("[More Information Needed]", true)}} diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/templates/modelcard_template.md b/.venv/lib/python3.13/site-packages/huggingface_hub/templates/modelcard_template.md new file mode 100644 index 0000000000000000000000000000000000000000..79ca15e4547debac763b390ef8e4b715e6f6403f --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/templates/modelcard_template.md @@ -0,0 +1,200 @@ +--- +# For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 +# Doc / guide: https://huggingface.co/docs/hub/model-cards +{{ card_data }} +--- + +# Model Card for {{ model_id | default("Model ID", true) }} + + + +{{ model_summary | default("", true) }} + +## Model Details + +### Model Description + + + +{{ model_description | default("", true) }} + +- **Developed by:** {{ developers | default("[More Information Needed]", true)}} +- **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}} +- **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}} +- **Model type:** {{ model_type | default("[More Information Needed]", true)}} +- **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}} +- **License:** {{ license | default("[More Information Needed]", true)}} +- **Finetuned from model [optional]:** {{ base_model | default("[More Information Needed]", true)}} + +### Model Sources [optional] + + + +- **Repository:** {{ repo | default("[More Information Needed]", true)}} +- **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}} +- **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}} + +## Uses + + + +### Direct Use + + + +{{ direct_use | default("[More Information Needed]", true)}} + +### Downstream Use [optional] + + + +{{ downstream_use | default("[More Information Needed]", true)}} + +### Out-of-Scope Use + + + +{{ out_of_scope_use | default("[More Information Needed]", true)}} + +## Bias, Risks, and Limitations + + + +{{ bias_risks_limitations | default("[More Information Needed]", true)}} + +### Recommendations + + + +{{ bias_recommendations | default("Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.", true)}} + +## How to Get Started with the Model + +Use the code below to get started with the model. + +{{ get_started_code | default("[More Information Needed]", true)}} + +## Training Details + +### Training Data + + + +{{ training_data | default("[More Information Needed]", true)}} + +### Training Procedure + + + +#### Preprocessing [optional] + +{{ preprocessing | default("[More Information Needed]", true)}} + + +#### Training Hyperparameters + +- **Training regime:** {{ training_regime | default("[More Information Needed]", true)}} + +#### Speeds, Sizes, Times [optional] + + + +{{ speeds_sizes_times | default("[More Information Needed]", true)}} + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +{{ testing_data | default("[More Information Needed]", true)}} + +#### Factors + + + +{{ testing_factors | default("[More Information Needed]", true)}} + +#### Metrics + + + +{{ testing_metrics | default("[More Information Needed]", true)}} + +### Results + +{{ results | default("[More Information Needed]", true)}} + +#### Summary + +{{ results_summary | default("", true) }} + +## Model Examination [optional] + + + +{{ model_examination | default("[More Information Needed]", true)}} + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** {{ hardware_type | default("[More Information Needed]", true)}} +- **Hours used:** {{ hours_used | default("[More Information Needed]", true)}} +- **Cloud Provider:** {{ cloud_provider | default("[More Information Needed]", true)}} +- **Compute Region:** {{ cloud_region | default("[More Information Needed]", true)}} +- **Carbon Emitted:** {{ co2_emitted | default("[More Information Needed]", true)}} + +## Technical Specifications [optional] + +### Model Architecture and Objective + +{{ model_specs | default("[More Information Needed]", true)}} + +### Compute Infrastructure + +{{ compute_infrastructure | default("[More Information Needed]", true)}} + +#### Hardware + +{{ hardware_requirements | default("[More Information Needed]", true)}} + +#### Software + +{{ software | default("[More Information Needed]", true)}} + +## Citation [optional] + + + +**BibTeX:** + +{{ citation_bibtex | default("[More Information Needed]", true)}} + +**APA:** + +{{ citation_apa | default("[More Information Needed]", true)}} + +## Glossary [optional] + + + +{{ glossary | default("[More Information Needed]", true)}} + +## More Information [optional] + +{{ more_information | default("[More Information Needed]", true)}} + +## Model Card Authors [optional] + +{{ model_card_authors | default("[More Information Needed]", true)}} + +## Model Card Contact + +{{ model_card_contact | default("[More Information Needed]", true)}} diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/__init__.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9b96202748ec537c0a7d7a3edf5cbca11ade591 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/__init__.py @@ -0,0 +1,134 @@ +# Copyright 2021 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# ruff: noqa: F401 +from huggingface_hub.errors import ( + BadRequestError, + BucketNotFoundError, + CacheNotFound, + CorruptedCacheException, + DisabledRepoError, + EntryNotFoundError, + FileMetadataError, + GatedRepoError, + HfHubHTTPError, + HFValidationError, + JobNotFoundError, + LocalEntryNotFoundError, + LocalTokenNotFoundError, + NotASafetensorsRepoError, + OfflineModeIsEnabled, + RepositoryNotFoundError, + RevisionNotFoundError, + SafetensorsParsingError, +) + +from . import tqdm as _tqdm # _tqdm is the module +from ._auth import get_stored_tokens, get_token +from ._cache_assets import cached_assets_path +from ._cache_manager import ( + CachedFileInfo, + CachedIncompleteFileInfo, + CachedRepoInfo, + CachedRevisionInfo, + DeleteCacheStrategy, + HFCacheInfo, + _format_size, + scan_cache_dir, +) +from ._chunk_utils import chunk_iterable +from ._datetime import parse_datetime +from ._detect_agent import detect_agent, is_agent +from ._experimental import experimental +from ._fixes import SoftTemporaryDirectory, WeakFileLock, yaml_dump +from ._git_credential import list_credential_helpers, set_git_credential, unset_git_credential +from ._headers import build_hf_headers, get_token_to_send +from ._hf_uris import HfMount, HfUri, is_hf_uri, parse_hf_mount, parse_hf_uri +from ._http import ( + ASYNC_CLIENT_FACTORY_T, + CLIENT_FACTORY_T, + RateLimitInfo, + close_session, + fix_hf_endpoint_in_url, + get_async_session, + get_session, + hf_raise_for_status, + http_backoff, + http_stream_backoff, + parse_ratelimit_headers, + set_async_client_factory, + set_client_factory, +) +from ._pagination import paginate +from ._paths import DEFAULT_IGNORE_PATTERNS, FORBIDDEN_FOLDERS, filter_repo_objects +from ._runtime import ( + dump_environment_info, + get_aiohttp_version, + get_fastai_version, + get_fastapi_version, + get_fastcore_version, + get_gradio_version, + get_graphviz_version, + get_hf_hub_version, + get_jinja_version, + get_numpy_version, + get_pillow_version, + get_pydantic_version, + get_pydot_version, + get_python_version, + get_tensorboard_version, + get_tf_version, + get_torch_version, + installation_method, + is_aiohttp_available, + is_colab_enterprise, + is_fastai_available, + is_fastapi_available, + is_fastcore_available, + is_google_colab, + is_gradio_available, + is_graphviz_available, + is_jinja_available, + is_notebook, + is_numpy_available, + is_package_available, + is_pillow_available, + is_pydantic_available, + is_pydot_available, + is_safetensors_available, + is_tensorboard_available, + is_tf_available, + is_torch_available, +) +from ._safetensors import SafetensorsFileMetadata, SafetensorsRepoMetadata, TensorInfo +from ._subprocess import capture_output, run_interactive_subprocess, run_subprocess +from ._telemetry import send_telemetry +from ._terminal import ANSI, StatusLine, select_choice, tabulate +from ._typing import is_jsonable, is_simple_optional_type, unwrap_simple_optional_type +from ._validators import validate_hf_hub_args, validate_repo_id +from ._xet import ( + XetFileData, + XetTokenType, + parse_xet_file_data_from_response, +) +from .tqdm import ( + are_progress_bars_disabled, + disable_progress_bars, + enable_progress_bars, + hf_thread_map, + is_tqdm_disabled, + silent_tqdm, + tqdm, + tqdm_stream_file, +) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_auth.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_auth.py new file mode 100644 index 0000000000000000000000000000000000000000..d1dba82be8e198693cca84b5bdd51c7c91d907d3 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_auth.py @@ -0,0 +1,456 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains a helper to get the token from machine (env variable, secret or config file).""" + +import configparser +import io +import logging +import os +import time +import warnings +from pathlib import Path +from threading import Lock +from typing import TypedDict + +from .. import constants +from ..errors import DeviceCodeError, OAuthErrorCode, OIDCError +from ._fixes import WeakFileLock +from ._oauth_device import refresh_access_token +from ._runtime import is_colab_enterprise, is_google_colab + + +_SECRET_FILE_MODE = 0o600 +_SECRET_DIR_MODE = 0o700 + + +def _write_secret(path: Path, content: str) -> None: + """Write content to file, restricting both the file and its parent directory to owner-only on POSIX systems.""" + path.parent.mkdir(parents=True, exist_ok=True, mode=_SECRET_DIR_MODE) + fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _SECRET_FILE_MODE) + with os.fdopen(fd, "w") as f: + f.write(content) + try: + path.chmod(_SECRET_FILE_MODE) + path.parent.chmod(_SECRET_DIR_MODE) + except (OSError, NotImplementedError): + # Windows does not support POSIX modes; chmod() will raise. Best-effort. + pass + + +_IS_GOOGLE_COLAB_CHECKED = False +_GOOGLE_COLAB_SECRET_LOCK = Lock() +_GOOGLE_COLAB_SECRET: str | None = None + +logger = logging.getLogger(__name__) + + +def get_token() -> str | None: + """ + Get token if user is logged in. + + Note: in most cases, you should use [`huggingface_hub.utils.build_hf_headers`] instead. This method is only useful + if you want to retrieve the token for other purposes than sending an HTTP request. + + If `HF_OIDC_RESOURCE` is set (Trusted Publishers, typically in CI), a short-lived token obtained via OIDC token + exchange takes precedence. Otherwise the token is retrieved from the `HF_TOKEN` environment variable, then from the + token file in the Hugging Face home folder. Returns None if user is not logged in. To log in, use [`login`] or + `hf auth login`. + + OAuth tokens obtained with the browser-based login come with a refresh token: when such a token is close to + expiry, it is transparently refreshed and persisted before being returned. + + Note: if `HF_OIDC_RESOURCE` is set but the OIDC token exchange fails, this raises instead of returning `None`, + opting into OIDC is explicit, so a failure surfaces as a clear error rather than a silent fallback. + + Returns: + `str` or `None`: The token, `None` if it doesn't exist. + """ + return ( + _get_token_from_oidc() + or _get_token_from_environment() + or _get_token_from_file_refreshed() + or _get_token_from_google_colab() + ) + + +def _get_token_from_google_colab() -> str | None: + """Get token from Google Colab secrets vault using `google.colab.userdata.get(...)`. + + Token is read from the vault only once per session and then stored in a global variable to avoid re-requesting + access to the vault. + """ + # If it's not a Google Colab or it's Colab Enterprise, fallback to environment variable or token file authentication + if not is_google_colab() or is_colab_enterprise(): + return None + + # `google.colab.userdata` is not thread-safe + # This can lead to a deadlock if multiple threads try to access it at the same time + # (typically when using `snapshot_download`) + # => use a lock + # See https://github.com/huggingface/huggingface_hub/issues/1952 for more details. + with _GOOGLE_COLAB_SECRET_LOCK: + global _GOOGLE_COLAB_SECRET + global _IS_GOOGLE_COLAB_CHECKED + + if _IS_GOOGLE_COLAB_CHECKED: # request access only once + return _GOOGLE_COLAB_SECRET + + try: + from google.colab import userdata # type: ignore + from google.colab.errors import Error as ColabError # type: ignore + except ImportError: + return None + + try: + token = userdata.get("HF_TOKEN") + _GOOGLE_COLAB_SECRET = _clean_token(token) + except userdata.NotebookAccessError: + # Means the user has a secret call `HF_TOKEN` and got a popup "please grand access to HF_TOKEN" and refused it + # => warn user but ignore error => do not re-request access to user + warnings.warn( + "\nAccess to the secret `HF_TOKEN` has not been granted on this notebook." + "\nYou will not be requested again." + "\nPlease restart the session if you want to be prompted again." + ) + _GOOGLE_COLAB_SECRET = None + except userdata.SecretNotFoundError: + # No `HF_TOKEN` secret defined: simply not logged in via the Colab vault. Not worth a + # warning now that `login()` is the primary flow (it would even fire during `login()` + # itself, telling the user to set up a secret while they are busy authenticating). + logger.info( + "The secret `HF_TOKEN` does not exist in your Colab secrets. Run `huggingface_hub.login()` to" + " authenticate (recommended but still optional to access public models or datasets)." + ) + _GOOGLE_COLAB_SECRET = None + except ColabError as e: + # Something happen but we don't know what => recommend to open a GitHub issue + warnings.warn(f"\nError while fetching `HF_TOKEN` secret value from your vault: '{str(e)}'.") + _GOOGLE_COLAB_SECRET = None + + _IS_GOOGLE_COLAB_CHECKED = True + return _GOOGLE_COLAB_SECRET + + +def _get_token_from_environment() -> str | None: + # `HF_TOKEN` has priority (keep `HUGGING_FACE_HUB_TOKEN` for backward compatibility) + return _clean_token(os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")) + + +def _get_token_from_file() -> str | None: + try: + return _clean_token(Path(constants.HF_TOKEN_PATH).read_text()) + except FileNotFoundError: + return None + + +class _OidcTokenCache(TypedDict): + resource: str + token: str + expires_at: float # monotonic clock value after which the cached token must be re-exchanged + + +# Cache for the OIDC-exchanged token: re-exchanging on every `get_token()` call would be wasteful, +# and re-exchanging shortly before expiry transparently keeps long-running jobs authenticated. +_OIDC_TOKEN_LOCK = Lock() +_OIDC_TOKEN_CACHE: _OidcTokenCache | None = None +_OIDC_REFRESH_MARGIN = 300 # re-exchange this many seconds before the token actually expires + + +def _get_token_from_oidc() -> str | None: + """Get a short-lived OIDC token in CI (Trusted Publishers). + + Enabled by setting `HF_OIDC_RESOURCE`, which scopes the token to a repo or user. + The ID token is read from `HF_OIDC_ID_TOKEN` if available, or minted from a supported CI provider (e.g. GitHub Actions). + + Returns `None` when OIDC is not enabled. + If enabled, any failure is raised explicitly rather than falling back silently. + + See `huggingface_hub._oidc` and https://huggingface.co/docs/hub/trusted-publishers. + """ + resource = os.environ.get("HF_OIDC_RESOURCE") + if not resource: + return None + + from .._oidc import detect_provider, oidc_login + + global _OIDC_TOKEN_CACHE + with _OIDC_TOKEN_LOCK: + now = time.monotonic() + if ( + _OIDC_TOKEN_CACHE is not None + and _OIDC_TOKEN_CACHE["resource"] == resource + and now < _OIDC_TOKEN_CACHE["expires_at"] + ): + return _OIDC_TOKEN_CACHE["token"] + + # An explicit id token (any provider) takes precedence; otherwise mint from a detected one. + subject_token = os.environ.get("HF_OIDC_ID_TOKEN") or None + if subject_token is None and detect_provider() is None: + raise OIDCError( + "HF_OIDC_RESOURCE is set but no OIDC id token is available: not running in a supported " + "CI provider (github) and HF_OIDC_ID_TOKEN is not set. Set HF_OIDC_ID_TOKEN to the id " + "token minted by your CI provider, or unset HF_OIDC_RESOURCE." + ) + + result = oidc_login(resource=resource, subject_token=subject_token) + token = result["access_token"] + expires_in = int(result.get("expires_in", 3600)) + # A pre-supplied HF_OIDC_ID_TOKEN can't be re-minted, so refreshing early is pointless (the id + # token is likely already expired by then): cache for the full lifetime. Only the auto-minted + # path can refresh, so only it gets the safety margin. + margin = 0 if subject_token is not None else _OIDC_REFRESH_MARGIN + _OIDC_TOKEN_CACHE = { + "resource": resource, + "token": token, + "expires_at": now + max(expires_in - margin, 0), + } + return token + + +class _OAuthRefreshCache(TypedDict): + file_token: str # token as read from HF_TOKEN_PATH (cache key) + resolved_token: str # token to return (refreshed, or identical if no refresh was needed) + recheck_at: float # wall-clock timestamp after which the expiry must be re-evaluated + + +# Cache the refresh decision in-process: `get_token()` is called on every HTTP request and must not +# re-read the stored tokens file (let alone hit the network) each time. +_OAUTH_REFRESH_LOCK = Lock() +_OAUTH_REFRESH_CACHE: _OAuthRefreshCache | None = None +_OAUTH_REFRESH_MARGIN = 24 * 3600 # refresh when less than 1 day of validity remains +_OAUTH_RECHECK_INTERVAL = 300 # re-check interval when there is no metadata or the refresh failed +_OAUTH_REFRESH_WARNED = False # warn at most once per process on refresh failure + + +def _get_token_from_file_refreshed() -> str | None: + """Get the token from `HF_TOKEN_PATH`, transparently refreshing it if close to expiry.""" + token = _get_token_from_file() + if token is None: + return None + return _refresh_oauth_token_if_needed(token) + + +def _refresh_oauth_token_if_needed(token: str) -> str: + """Refresh an OAuth access token if it is close to expiry. Best-effort: never raises. + + OAuth tokens obtained with the browser-based login are stored with a `refresh_token` and an + `expires_at` timestamp (see `_save_token`). When the active token is one of them and about to + expire, exchange the refresh token for a new access token and persist it. Any other token is + returned unchanged. + """ + global _OAUTH_REFRESH_CACHE + with _OAUTH_REFRESH_LOCK: + now = time.time() + cache = _OAUTH_REFRESH_CACHE + if cache is not None and cache["file_token"] == token and now < cache["recheck_at"]: + return cache["resolved_token"] + + token_name, fields = next( + ((name, fields) for name, fields in _read_stored_tokens_full().items() if fields.get("hf_token") == token), + (None, {}), + ) + refresh_token = fields.get("refresh_token") + expires_at = _parse_expires_at(fields) + if token_name is None or refresh_token is None or expires_at is None: + # `token` may have just been replaced by a concurrent refresh (in which case it no + # longer appears in the stored tokens): serve the fresh file token without caching. + current_file_token = _get_token_from_file() + if current_file_token is not None and current_file_token != token: + return current_file_token + # Not a refreshable OAuth token (or its metadata was lost): nothing to do. + _OAUTH_REFRESH_CACHE = { + "file_token": token, + "resolved_token": token, + "recheck_at": now + _OAUTH_RECHECK_INTERVAL, + } + return token + + if expires_at - _OAUTH_REFRESH_MARGIN > now: + _OAUTH_REFRESH_CACHE = { + "file_token": token, + "resolved_token": token, + "recheck_at": expires_at - _OAUTH_REFRESH_MARGIN, + } + return token + + try: + # Cross-process file lock: if the server rotates refresh tokens, two processes + # refreshing concurrently would invalidate each other's refresh token. + with WeakFileLock(constants.HF_STORED_TOKENS_PATH + ".lock", timeout=30): + # Re-read under the lock: another process may have refreshed in the meantime. + fields = _read_stored_tokens_full().get(token_name, {}) + if fields.get("hf_token") != token: + # Another process already refreshed this token: adopt its result. + new_token = fields.get("hf_token") or token + new_expires_at = _parse_expires_at(fields) + else: + response = refresh_access_token(refresh_token) + new_token = response["access_token"] + new_expires_at = int(now) + int(response["expires_in"]) if "expires_in" in response else None + _save_token( + token=new_token, + token_name=token_name, + # The server may rotate the refresh token; keep the old one if it doesn't. + refresh_token=response.get("refresh_token") or refresh_token, + expires_at=new_expires_at, + ) + # Update the active token file, unless another process switched to a different token meanwhile. + if _get_token_from_file() == token: + _write_secret(Path(constants.HF_TOKEN_PATH), new_token) + logger.info(f"Access token `{token_name}` has been refreshed.") + except Exception as e: + if isinstance(e, DeviceCodeError) and e.error_code == OAuthErrorCode.INVALID_GRANT: + # Refresh token expired or revoked: retrying is pointless, a re-login is required. + # Warned unconditionally (the inf recheck guarantees it fires at most once): an + # earlier transient warning must not suppress this actionable message. + logger.warning( + "Your Hugging Face access token has expired and could not be refreshed " + f"(session expired or revoked). Run `hf auth login` to re-authenticate. ({e})" + ) + recheck_at = float("inf") + else: + # Transient failure (offline, server error, ...): retry later. + _warn_refresh_failure_once(f"Could not refresh your Hugging Face access token: {e}. Will retry later.") + recheck_at = now + _OAUTH_RECHECK_INTERVAL + # Return the existing token: if it's truly expired, the API will reject it with a clear error. + _OAUTH_REFRESH_CACHE = {"file_token": token, "resolved_token": token, "recheck_at": recheck_at} + return token + + _OAUTH_REFRESH_CACHE = { + "file_token": new_token, + "resolved_token": new_token, + # The floor guards against a token lifetime shorter than the refresh margin, which + # would otherwise put `recheck_at` in the past and trigger a refresh on every call. + "recheck_at": max( + now + _OAUTH_RECHECK_INTERVAL, + new_expires_at - _OAUTH_REFRESH_MARGIN if new_expires_at else 0, + ), + } + return new_token + + +def _warn_refresh_failure_once(message: str) -> None: + global _OAUTH_REFRESH_WARNED + if not _OAUTH_REFRESH_WARNED: + logger.warning(message) + _OAUTH_REFRESH_WARNED = True + + +def _parse_expires_at(fields: dict[str, str]) -> int | None: + """Parse the `expires_at` field of a stored-tokens section, `None` if missing or corrupt.""" + try: + return int(fields["expires_at"]) + except (KeyError, ValueError): + return None + + +def get_stored_tokens() -> dict[str, str]: + """ + Returns the parsed INI file containing the access tokens. + The file is located at `HF_STORED_TOKENS_PATH`, defaulting to `~/.cache/huggingface/stored_tokens`. + If the file does not exist, an empty dictionary is returned. + + Returns: `dict[str, str]` + Key is the token name and value is the token. + """ + return {token_name: fields.get("hf_token", "") for token_name, fields in _read_stored_tokens_full().items()} + + +def _read_stored_tokens_full() -> dict[str, dict[str, str]]: + """Read all sections of the stored tokens INI file, with all their fields. + + Beside `hf_token`, sections for OAuth tokens also carry `refresh_token` and `expires_at` + (unix timestamp), used by [`get_token`] to transparently refresh them. + """ + tokens_path = Path(constants.HF_STORED_TOKENS_PATH) + if not tokens_path.exists(): + return {} + # interpolation=None: token values are opaque strings, a `%` must not be interpreted. + config = configparser.ConfigParser(interpolation=None) + try: + config.read(tokens_path) + return {token_name: dict(config.items(token_name)) for token_name in config.sections()} + except configparser.Error as e: + logger.error(f"Error parsing stored tokens file: {e}") + return {} + + +def _save_stored_tokens_full(stored_tokens: dict[str, dict[str, str]]) -> None: + """Write all sections and their fields to the stored tokens INI file.""" + config = configparser.ConfigParser(interpolation=None) + for token_name in sorted(stored_tokens.keys()): + config.add_section(token_name) + for key, value in stored_tokens[token_name].items(): + config.set(token_name, key, value) + + buf = io.StringIO() + config.write(buf) + _write_secret(Path(constants.HF_STORED_TOKENS_PATH), buf.getvalue()) + + +def _get_token_by_name(token_name: str) -> str | None: + """ + Get the token by name. + + Args: + token_name (`str`): + The name of the token to get. + + Returns: + `str` or `None`: The token, `None` if it doesn't exist. + + """ + stored_tokens = get_stored_tokens() + if token_name not in stored_tokens: + return None + return _clean_token(stored_tokens[token_name]) + + +def _save_token( + token: str, token_name: str, *, refresh_token: str | None = None, expires_at: int | None = None +) -> None: + """ + Save the given token. + + If the stored tokens file does not exist, it will be created. + Args: + token (`str`): + The token to save. + token_name (`str`): + The name of the token. + refresh_token (`str`, *optional*): + OAuth refresh token used to renew the access token when it expires. + expires_at (`int`, *optional*): + Unix timestamp at which the access token expires. + """ + stored_tokens = _read_stored_tokens_full() + fields = {"hf_token": token} + if refresh_token is not None: + fields["refresh_token"] = refresh_token + if expires_at is not None: + fields["expires_at"] = str(expires_at) + # Replace the whole section: re-logging in under the same name must drop stale metadata. + stored_tokens[token_name] = fields + _save_stored_tokens_full(stored_tokens) + logger.info(f"The token `{token_name}` has been saved to {constants.HF_STORED_TOKENS_PATH}") + + +def _clean_token(token: str | None) -> str | None: + """Clean token by removing trailing and leading spaces and newlines. + + If token is an empty string, return None. + """ + if token is None: + return None + return token.replace("\r", "").replace("\n", "").strip() or None diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_assets.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_assets.py new file mode 100644 index 0000000000000000000000000000000000000000..01263873715c56d8331200b47a4608a1b1135ab7 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_assets.py @@ -0,0 +1,133 @@ +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pathlib import Path + +from ..constants import HF_ASSETS_CACHE + + +def cached_assets_path( + library_name: str, + namespace: str = "default", + subfolder: str = "default", + *, + assets_dir: str | Path | None = None, +) -> Path: + """Return a folder path to cache arbitrary files. + + `huggingface_hub` provides a canonical folder path to store assets. This is the + recommended way to integrate cache in a downstream library as it will benefit from + the builtins tools to scan and delete the cache properly. + + The distinction is made between files cached from the Hub and assets. Files from the + Hub are cached in a git-aware manner and entirely managed by `huggingface_hub`. See + [related documentation](https://huggingface.co/docs/huggingface_hub/how-to-cache). + All other files that a downstream library caches are considered to be "assets" + (files downloaded from external sources, extracted from a .tar archive, preprocessed + for training,...). + + Once the folder path is generated, it is guaranteed to exist and to be a directory. + The path is based on 3 levels of depth: the library name, a namespace and a + subfolder. Those 3 levels grants flexibility while allowing `huggingface_hub` to + expect folders when scanning/deleting parts of the assets cache. Within a library, + it is expected that all namespaces share the same subset of subfolder names but this + is not a mandatory rule. The downstream library has then full control on which file + structure to adopt within its cache. Namespace and subfolder are optional (would + default to a `"default/"` subfolder) but library name is mandatory as we want every + downstream library to manage its own cache. + + Expected tree: + ```text + assets/ + └── datasets/ + │ ├── SQuAD/ + │ │ ├── downloaded/ + │ │ ├── extracted/ + │ │ └── processed/ + │ ├── Helsinki-NLP--tatoeba_mt/ + │ ├── downloaded/ + │ ├── extracted/ + │ └── processed/ + └── transformers/ + ├── default/ + │ ├── something/ + ├── bert-base-cased/ + │ ├── default/ + │ └── training/ + hub/ + └── models--julien-c--EsperBERTo-small/ + ├── blobs/ + │ ├── (...) + │ ├── (...) + ├── refs/ + │ └── (...) + └── [ 128] snapshots/ + ├── 2439f60ef33a0d46d85da5001d52aeda5b00ce9f/ + │ ├── (...) + └── bbc77c8132af1cc5cf678da3f1ddf2de43606d48/ + └── (...) + ``` + + + Args: + library_name (`str`): + Name of the library that will manage the cache folder. Example: `"dataset"`. + namespace (`str`, *optional*, defaults to "default"): + Namespace to which the data belongs. Example: `"SQuAD"`. + subfolder (`str`, *optional*, defaults to "default"): + Subfolder in which the data will be stored. Example: `extracted`. + assets_dir (`str`, `Path`, *optional*): + Path to the folder where assets are cached. This must not be the same folder + where Hub files are cached. Defaults to `HF_HOME / "assets"` if not provided. + Can also be set with `HF_ASSETS_CACHE` environment variable. + + Returns: + Path to the cache folder (`Path`). + + Example: + ```py + >>> from huggingface_hub import cached_assets_path + + >>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="download") + PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/download') + + >>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="extracted") + PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/extracted') + + >>> cached_assets_path(library_name="datasets", namespace="Helsinki-NLP/tatoeba_mt") + PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/Helsinki-NLP--tatoeba_mt/default') + + >>> cached_assets_path(library_name="datasets", assets_dir="/tmp/tmp123456") + PosixPath('/tmp/tmp123456/datasets/default/default') + ``` + """ + # Resolve assets_dir + if assets_dir is None: + assets_dir = HF_ASSETS_CACHE + assets_dir = Path(assets_dir).expanduser().resolve() + + # Avoid names that could create path issues + for part in (" ", "/", "\\"): + library_name = library_name.replace(part, "--") + namespace = namespace.replace(part, "--") + subfolder = subfolder.replace(part, "--") + + # Path to subfolder is created + path = assets_dir / library_name / namespace / subfolder + try: + path.mkdir(exist_ok=True, parents=True) + except (FileExistsError, NotADirectoryError): + raise ValueError(f"Corrupted assets folder: cannot create directory because of an existing file ({path}).") + + # Return + return path diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_manager.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..8676537e8ab88357c026bf00ed01e7ebf7994305 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_manager.py @@ -0,0 +1,901 @@ +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to manage the HF cache directory.""" + +import os +import shutil +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from huggingface_hub.errors import CacheNotFound, CorruptedCacheException + +from ..constants import HF_HUB_CACHE +from . import logging +from ._parsing import format_timesince +from ._terminal import tabulate + + +logger = logging.get_logger(__name__) + +REPO_TYPE_T = Literal["model", "dataset", "space"] + +# List of OS-created helper files that need to be ignored +FILES_TO_IGNORE = [".DS_Store", "Thumbs.db", "desktop.ini"] + + +@dataclass(frozen=True) +class CachedFileInfo: + """Frozen data structure holding information about a single cached file. + + Args: + file_name (`str`): + Name of the file. Example: `config.json`. + file_path (`Path`): + Path of the file in the `snapshots` directory. The file path is a symlink + referring to a blob in the `blobs` folder. + blob_path (`Path`): + Path of the blob file. This is equivalent to `file_path.resolve()`. + size_on_disk (`int`): + Size of the blob file in bytes. + blob_last_accessed (`float`): + Timestamp of the last time the blob file has been accessed (from any + revision). + blob_last_modified (`float`): + Timestamp of the last time the blob file has been modified/created. + + > [!WARNING] + > `blob_last_accessed` and `blob_last_modified` reliability can depend on the OS you + > are using. See [python documentation](https://docs.python.org/3/library/os.html#os.stat_result) + > for more details. + """ + + file_name: str + file_path: Path + blob_path: Path + size_on_disk: int + + blob_last_accessed: float + blob_last_modified: float + + @property + def blob_last_accessed_str(self) -> str: + """ + (property) Timestamp of the last time the blob file has been accessed (from any + revision), returned as a human-readable string. + + Example: "2 weeks ago". + """ + return format_timesince(self.blob_last_accessed) + + @property + def blob_last_modified_str(self) -> str: + """ + (property) Timestamp of the last time the blob file has been modified, returned + as a human-readable string. + + Example: "2 weeks ago". + """ + return format_timesince(self.blob_last_modified) + + @property + def size_on_disk_str(self) -> str: + """ + (property) Size of the blob file as a human-readable string. + + Example: "42.2K". + """ + return _format_size(self.size_on_disk) + + +@dataclass(frozen=True) +class CachedRevisionInfo: + """Frozen data structure holding information about a revision. + + A revision correspond to a folder in the `snapshots` folder and is populated with + the exact tree structure as the repo on the Hub but contains only symlinks. A + revision can be either referenced by 1 or more `refs` or be "detached" (no refs). + + Args: + commit_hash (`str`): + Hash of the revision (unique). + Example: `"9338f7b671827df886678df2bdd7cc7b4f36dffd"`. + snapshot_path (`Path`): + Path to the revision directory in the `snapshots` folder. It contains the + exact tree structure as the repo on the Hub. + files: (`frozenset[CachedFileInfo]`): + Set of [`~CachedFileInfo`] describing all files contained in the snapshot. + refs (`frozenset[str]`): + Set of `refs` pointing to this revision. If the revision has no `refs`, it + is considered detached. + Example: `{"main", "2.4.0"}` or `{"refs/pr/1"}`. + size_on_disk (`int`): + Sum of the blob file sizes that are symlink-ed by the revision. + last_modified (`float`): + Timestamp of the last time the revision has been created/modified. + + > [!WARNING] + > `last_accessed` cannot be determined correctly on a single revision as blob files + > are shared across revisions. + + > [!WARNING] + > `size_on_disk` is not necessarily the sum of all file sizes because of possible + > duplicated files. Besides, only blobs are taken into account, not the (negligible) + > size of folders and symlinks. + """ + + commit_hash: str + snapshot_path: Path + size_on_disk: int + files: frozenset[CachedFileInfo] + refs: frozenset[str] + + last_modified: float + + @property + def last_modified_str(self) -> str: + """ + (property) Timestamp of the last time the revision has been modified, returned + as a human-readable string. + + Example: "2 weeks ago". + """ + return format_timesince(self.last_modified) + + @property + def size_on_disk_str(self) -> str: + """ + (property) Sum of the blob file sizes as a human-readable string. + + Example: "42.2K". + """ + return _format_size(self.size_on_disk) + + @property + def nb_files(self) -> int: + """ + (property) Total number of files in the revision. + """ + return len(self.files) + + +@dataclass(frozen=True) +class CachedRepoInfo: + """Frozen data structure holding information about a cached repository. + + Args: + repo_id (`str`): + Repo id of the repo on the Hub. Example: `"google/fleurs"`. + repo_type (`Literal["dataset", "model", "space"]`): + Type of the cached repo. + repo_path (`Path`): + Local path to the cached repo. + size_on_disk (`int`): + Sum of the blob file sizes in the cached repo. + nb_files (`int`): + Total number of blob files in the cached repo. + revisions (`frozenset[CachedRevisionInfo]`): + Set of [`~CachedRevisionInfo`] describing all revisions cached in the repo. + last_accessed (`float`): + Timestamp of the last time a blob file of the repo has been accessed. + last_modified (`float`): + Timestamp of the last time a blob file of the repo has been modified/created. + + > [!WARNING] + > `size_on_disk` is not necessarily the sum of all revisions sizes because of + > duplicated files. Besides, only blobs are taken into account, not the (negligible) + > size of folders and symlinks. + + > [!WARNING] + > `last_accessed` and `last_modified` reliability can depend on the OS you are using. + > See [python documentation](https://docs.python.org/3/library/os.html#os.stat_result) + > for more details. + """ + + repo_id: str + repo_type: REPO_TYPE_T + repo_path: Path + size_on_disk: int + nb_files: int + revisions: frozenset[CachedRevisionInfo] + + last_accessed: float + last_modified: float + + @property + def last_accessed_str(self) -> str: + """ + (property) Last time a blob file of the repo has been accessed, returned as a + human-readable string. + + Example: "2 weeks ago". + """ + return format_timesince(self.last_accessed) + + @property + def last_modified_str(self) -> str: + """ + (property) Last time a blob file of the repo has been modified, returned as a + human-readable string. + + Example: "2 weeks ago". + """ + return format_timesince(self.last_modified) + + @property + def size_on_disk_str(self) -> str: + """ + (property) Sum of the blob file sizes as a human-readable string. + + Example: "42.2K". + """ + return _format_size(self.size_on_disk) + + @property + def cache_id(self) -> str: + """Canonical `type/id` identifier used across cache tooling.""" + return f"{self.repo_type}/{self.repo_id}" + + @property + def refs(self) -> dict[str, CachedRevisionInfo]: + """ + (property) Mapping between `refs` and revision data structures. + """ + return {ref: revision for revision in self.revisions for ref in revision.refs} + + +@dataclass(frozen=True) +class DeleteCacheStrategy: + """Frozen data structure holding the strategy to delete cached revisions. + + This object is not meant to be instantiated programmatically but to be returned by + [`~utils.HFCacheInfo.delete_revisions`]. See documentation for usage example. + + Args: + expected_freed_size (`float`): + Expected freed size once strategy is executed. + blobs (`frozenset[Path]`): + Set of blob file paths to be deleted. + refs (`frozenset[Path]`): + Set of reference file paths to be deleted. + repos (`frozenset[Path]`): + Set of entire repo paths to be deleted. + snapshots (`frozenset[Path]`): + Set of snapshots to be deleted (directory of symlinks). + """ + + expected_freed_size: int + blobs: frozenset[Path] + refs: frozenset[Path] + repos: frozenset[Path] + snapshots: frozenset[Path] + + @property + def expected_freed_size_str(self) -> str: + """ + (property) Expected size that will be freed as a human-readable string. + + Example: "42.2K". + """ + return _format_size(self.expected_freed_size) + + def execute(self) -> None: + """Execute the defined strategy. + + > [!WARNING] + > If this method is interrupted, the cache might get corrupted. Deletion order is + > implemented so that references and symlinks are deleted before the actual blob + > files. + + > [!WARNING] + > This method is irreversible. If executed, cached files are erased and must be + > downloaded again. + """ + # Deletion order matters. Blobs are deleted in last so that the user can't end + # up in a state where a `ref`` refers to a missing snapshot or a snapshot + # symlink refers to a deleted blob. + + # Delete entire repos + for path in self.repos: + _try_delete_path(path, path_type="repo") + + # Delete snapshot directories + for path in self.snapshots: + _try_delete_path(path, path_type="snapshot") + + # Delete refs files + for path in self.refs: + _try_delete_path(path, path_type="ref") + + # Delete blob files + for path in self.blobs: + _try_delete_path(path, path_type="blob") + + logger.info(f"Cache deletion done. Saved {self.expected_freed_size_str}.") + + +@dataclass(frozen=True) +class CachedIncompleteFileInfo: + """Frozen data structure holding information about a single incomplete download. + + Interrupted downloads leave `//blobs/.incomplete` files behind. + These are not part of any committed revision, so they are surfaced separately by + [`scan_cache_dir`]. + + Args: + file_path (`Path`): + Path of the `.incomplete` file in the `blobs` folder. + size_on_disk (`int`): + Size of the partially-downloaded file in bytes. + """ + + file_path: Path + size_on_disk: int + + +@dataclass(frozen=True) +class HFCacheInfo: + """Frozen data structure holding information about the entire cache-system. + + This data structure is returned by [`scan_cache_dir`] and is immutable. + + Args: + size_on_disk (`int`): + Sum of all valid repo sizes in the cache-system. + repos (`frozenset[CachedRepoInfo]`): + Set of [`~CachedRepoInfo`] describing all valid cached repos found on the + cache-system while scanning. + incomplete_files (`frozenset[CachedIncompleteFileInfo]`): + Set of [`~CachedIncompleteFileInfo`] describing orphaned `*.incomplete` + files left behind by interrupted downloads. + warnings (`list[CorruptedCacheException]`): + List of [`~CorruptedCacheException`] that occurred while scanning the cache. + Those exceptions are captured so that the scan can continue. Corrupted repos + are skipped from the scan. + + > [!WARNING] + > Here `size_on_disk` is equal to the sum of all repo sizes (only blobs). However if + > some cached repos are corrupted, their sizes are not taken into account. + """ + + size_on_disk: int + repos: frozenset[CachedRepoInfo] + incomplete_files: frozenset[CachedIncompleteFileInfo] + warnings: list[CorruptedCacheException] + + @property + def size_on_disk_str(self) -> str: + """ + (property) Sum of all valid repo sizes in the cache-system as a human-readable + string. + + Example: "42.2K". + """ + return _format_size(self.size_on_disk) + + @property + def incomplete_size_on_disk(self) -> int: + """(property) Sum of all incomplete download sizes in bytes.""" + return sum(file.size_on_disk for file in self.incomplete_files) + + def delete_revisions(self, *revisions: str) -> DeleteCacheStrategy: + """Prepare the strategy to delete one or more revisions cached locally. + + Input revisions can be any revision hash. If a revision hash is not found in the + local cache, a warning is thrown but no error is raised. Revisions can be from + different cached repos since hashes are unique across repos, + + Examples: + ```py + >>> from huggingface_hub import scan_cache_dir + >>> cache_info = scan_cache_dir() + >>> delete_strategy = cache_info.delete_revisions( + ... "81fd1d6e7847c99f5862c9fb81387956d99ec7aa" + ... ) + >>> print(f"Will free {delete_strategy.expected_freed_size_str}.") + Will free 7.9K. + >>> delete_strategy.execute() + Cache deletion done. Saved 7.9K. + ``` + + ```py + >>> from huggingface_hub import scan_cache_dir + >>> scan_cache_dir().delete_revisions( + ... "81fd1d6e7847c99f5862c9fb81387956d99ec7aa", + ... "e2983b237dccf3ab4937c97fa717319a9ca1a96d", + ... "6c0e6080953db56375760c0471a8c5f2929baf11", + ... ).execute() + Cache deletion done. Saved 8.6G. + ``` + + > [!WARNING] + > `delete_revisions` returns a [`~utils.DeleteCacheStrategy`] object that needs to + > be executed. The [`~utils.DeleteCacheStrategy`] is not meant to be modified but + > allows having a dry run before actually executing the deletion. + """ + hashes_to_delete: set[str] = set(revisions) + + repos_with_revisions: dict[CachedRepoInfo, set[CachedRevisionInfo]] = defaultdict(set) + + for repo in self.repos: + for revision in repo.revisions: + if revision.commit_hash in hashes_to_delete: + repos_with_revisions[repo].add(revision) + hashes_to_delete.remove(revision.commit_hash) + + if len(hashes_to_delete) > 0: + logger.warning(f"Revision(s) not found - cannot delete them: {', '.join(hashes_to_delete)}") + + delete_strategy_blobs: set[Path] = set() + delete_strategy_refs: set[Path] = set() + delete_strategy_repos: set[Path] = set() + delete_strategy_snapshots: set[Path] = set() + delete_strategy_expected_freed_size = 0 + + for affected_repo, revisions_to_delete in repos_with_revisions.items(): + other_revisions = affected_repo.revisions - revisions_to_delete + + # If no other revisions, it means all revisions are deleted + # -> delete the entire cached repo + if len(other_revisions) == 0: + delete_strategy_repos.add(affected_repo.repo_path) + delete_strategy_expected_freed_size += affected_repo.size_on_disk + continue + + # Some revisions of the repo will be deleted but not all. We need to filter + # which blob files will not be linked anymore. + for revision_to_delete in revisions_to_delete: + # Snapshot dir + delete_strategy_snapshots.add(revision_to_delete.snapshot_path) + + # Refs dir + for ref in revision_to_delete.refs: + delete_strategy_refs.add(affected_repo.repo_path / "refs" / ref) + + # Blobs dir + for file in revision_to_delete.files: + # Some files are not symlinks to the blobs dir: files copied instead of symlinked + # (Windows, filesystems without symlink support) or files created by the user + # inside the snapshot dir (e.g. build artifacts from a `pip install`). They are + # already removed when deleting the snapshot dir -> count them but don't delete + # them a second time. + if revision_to_delete.snapshot_path in file.blob_path.parents: + delete_strategy_expected_freed_size += file.size_on_disk + continue + + if file.blob_path not in delete_strategy_blobs: + is_file_alone = True + for revision in other_revisions: + for rev_file in revision.files: + if file.blob_path == rev_file.blob_path: + is_file_alone = False + break + if not is_file_alone: + break + + # Blob file not referenced by remaining revisions -> delete + if is_file_alone: + delete_strategy_blobs.add(file.blob_path) + delete_strategy_expected_freed_size += file.size_on_disk + + # Return the strategy instead of executing it. + return DeleteCacheStrategy( + blobs=frozenset(delete_strategy_blobs), + refs=frozenset(delete_strategy_refs), + repos=frozenset(delete_strategy_repos), + snapshots=frozenset(delete_strategy_snapshots), + expected_freed_size=delete_strategy_expected_freed_size, + ) + + def export_as_table(self, *, verbosity: int = 0) -> str: + """Generate a table from the [`HFCacheInfo`] object. + + Pass `verbosity=0` to get a table with a single row per repo, with columns + "repo_id", "repo_type", "size_on_disk", "nb_files", "last_accessed", "last_modified", "refs", "local_path". + + Pass `verbosity=1` to get a table with a row per repo and revision (thus multiple rows can appear for a single repo), with columns + "repo_id", "repo_type", "revision", "size_on_disk", "nb_files", "last_modified", "refs", "local_path". + + Example: + ```py + >>> from huggingface_hub.utils import scan_cache_dir + + >>> hf_cache_info = scan_cache_dir() + HFCacheInfo(...) + + >>> print(hf_cache_info.export_as_table()) + REPO ID REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED LAST_MODIFIED REFS LOCAL PATH + --------------------------------------------------- --------- ------------ -------- ------------- ------------- ---- -------------------------------------------------------------------------------------------------- + roberta-base model 2.7M 5 1 day ago 1 week ago main ~/.cache/huggingface/hub/models--roberta-base + suno/bark model 8.8K 1 1 week ago 1 week ago main ~/.cache/huggingface/hub/models--suno--bark + t5-base model 893.8M 4 4 days ago 7 months ago main ~/.cache/huggingface/hub/models--t5-base + t5-large model 3.0G 4 5 weeks ago 5 months ago main ~/.cache/huggingface/hub/models--t5-large + + >>> print(hf_cache_info.export_as_table(verbosity=1)) + REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED REFS LOCAL PATH + --------------------------------------------------- --------- ---------------------------------------- ------------ -------- ------------- ---- ----------------------------------------------------------------------------------------------------------------------------------------------------- + roberta-base model e2da8e2f811d1448a5b465c236feacd80ffbac7b 2.7M 5 1 week ago main ~/.cache/huggingface/hub/models--roberta-base/snapshots/e2da8e2f811d1448a5b465c236feacd80ffbac7b + suno/bark model 70a8a7d34168586dc5d028fa9666aceade177992 8.8K 1 1 week ago main ~/.cache/huggingface/hub/models--suno--bark/snapshots/70a8a7d34168586dc5d028fa9666aceade177992 + t5-base model a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 893.8M 4 7 months ago main ~/.cache/huggingface/hub/models--t5-base/snapshots/a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 + t5-large model 150ebc2c4b72291e770f58e6057481c8d2ed331a 3.0G 4 5 months ago main ~/.cache/huggingface/hub/models--t5-large/snapshots/150ebc2c4b72291e770f58e6057481c8d2ed331a + ``` + + Args: + verbosity (`int`, *optional*): + The verbosity level. Defaults to 0. + + Returns: + `str`: The table as a string. + """ + if verbosity == 0: + return tabulate( + rows=[ + [ + repo.repo_id, + repo.repo_type, + f"{repo.size_on_disk_str:>12}", + repo.nb_files, + repo.last_accessed_str, + repo.last_modified_str, + ", ".join(sorted(repo.refs)), + str(repo.repo_path), + ] + for repo in sorted(self.repos, key=lambda repo: repo.repo_path) + ], + headers=[ + "REPO ID", + "REPO TYPE", + "SIZE ON DISK", + "NB FILES", + "LAST_ACCESSED", + "LAST_MODIFIED", + "REFS", + "LOCAL PATH", + ], + ) + else: + return tabulate( + rows=[ + [ + repo.repo_id, + repo.repo_type, + revision.commit_hash, + f"{revision.size_on_disk_str:>12}", + revision.nb_files, + revision.last_modified_str, + ", ".join(sorted(revision.refs)), + str(revision.snapshot_path), + ] + for repo in sorted(self.repos, key=lambda repo: repo.repo_path) + for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash) + ], + headers=[ + "REPO ID", + "REPO TYPE", + "REVISION", + "SIZE ON DISK", + "NB FILES", + "LAST_MODIFIED", + "REFS", + "LOCAL PATH", + ], + ) + + +def scan_cache_dir(cache_dir: str | Path | None = None) -> HFCacheInfo: + """Scan the entire HF cache-system and return a [`~HFCacheInfo`] structure. + + Use `scan_cache_dir` in order to programmatically scan your cache-system. The cache + will be scanned repo by repo. If a repo is corrupted, a [`~CorruptedCacheException`] + will be thrown internally but captured and returned in the [`~HFCacheInfo`] + structure. Only valid repos get a proper report. + + ```py + >>> from huggingface_hub import scan_cache_dir + + >>> hf_cache_info = scan_cache_dir() + HFCacheInfo( + size_on_disk=3398085269, + repos=frozenset({ + CachedRepoInfo( + repo_id='t5-small', + repo_type='model', + repo_path=PosixPath(...), + size_on_disk=970726914, + nb_files=11, + revisions=frozenset({ + CachedRevisionInfo( + commit_hash='d78aea13fa7ecd06c29e3e46195d6341255065d5', + size_on_disk=970726339, + snapshot_path=PosixPath(...), + files=frozenset({ + CachedFileInfo( + file_name='config.json', + size_on_disk=1197 + file_path=PosixPath(...), + blob_path=PosixPath(...), + ), + CachedFileInfo(...), + ... + }), + ), + CachedRevisionInfo(...), + ... + }), + ), + CachedRepoInfo(...), + ... + }), + warnings=[ + CorruptedCacheException("Snapshots dir doesn't exist in cached repo: ..."), + CorruptedCacheException(...), + ... + ], + ) + ``` + + You can also print a detailed report directly from the `hf` command line using: + ```text + > hf cache ls + ID SIZE LAST_ACCESSED LAST_MODIFIED REFS + --------------------------- -------- ------------- ------------- ----------- + dataset/nyu-mll/glue 157.4M 2 days ago 2 days ago main script + model/LiquidAI/LFM2-VL-1.6B 3.2G 4 days ago 4 days ago main + model/microsoft/UserLM-8b 32.1G 4 days ago 4 days ago main + + Done in 0.0s. Scanned 6 repo(s) for a total of 3.4G. + Got 1 warning(s) while scanning. Use -vvv to print details. + ``` + + Args: + cache_dir (`str` or `Path`, `optional`): + Cache directory to cache. Defaults to the default HF cache directory. + + > [!WARNING] + > Raises: + > + > `CacheNotFound` + > If the cache directory does not exist. + > + > [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > If the cache directory is a file, instead of a directory. + + Returns: a [`~HFCacheInfo`] object. + """ + if cache_dir is None: + cache_dir = HF_HUB_CACHE + + cache_dir = Path(cache_dir).expanduser().resolve() + if not cache_dir.exists(): + raise CacheNotFound( + f"Cache directory not found: {cache_dir}. Please use `cache_dir` argument or set `HF_HUB_CACHE` environment variable.", + cache_dir=cache_dir, + ) + + if cache_dir.is_file(): + raise ValueError( + f"Scan cache expects a directory but found a file: {cache_dir}. Please use `cache_dir` argument or set `HF_HUB_CACHE` environment variable." + ) + + repos: set[CachedRepoInfo] = set() + warnings: list[CorruptedCacheException] = [] + for repo_path in cache_dir.iterdir(): + if repo_path.name in FILES_TO_IGNORE: + continue + if repo_path.name == ".locks": # skip './.locks/' folder + continue + if repo_path.name == "CACHEDIR.TAG": # skip CACHEDIR.TAG file + continue + try: + repos.add(_scan_cached_repo(repo_path)) + except CorruptedCacheException as e: + warnings.append(e) + + return HFCacheInfo( + repos=frozenset(repos), + size_on_disk=sum(repo.size_on_disk for repo in repos), + incomplete_files=_scan_incomplete_files(cache_dir), + warnings=warnings, + ) + + +def _scan_incomplete_files(cache_dir: Path) -> frozenset[CachedIncompleteFileInfo]: + """Find orphaned `*.incomplete` partial-download files in the cache. + + Interrupted downloads leave `//blobs/.incomplete` files behind. + These are not part of any committed revision, so they are reported separately in the + [`~HFCacheInfo`] returned by [`scan_cache_dir`]. + """ + files: set[CachedIncompleteFileInfo] = set() + for path in cache_dir.glob("*/blobs/*.incomplete"): + try: + size_on_disk = path.stat().st_size + except OSError: + continue + files.add(CachedIncompleteFileInfo(file_path=path, size_on_disk=size_on_disk)) + return frozenset(files) + + +def _scan_cached_repo(repo_path: Path) -> CachedRepoInfo: + """Scan a single cache repo and return information about it. + + Any unexpected behavior will raise a [`~CorruptedCacheException`]. + """ + if not repo_path.is_dir(): + raise CorruptedCacheException(f"Repo path is not a directory: {repo_path}") + + if "--" not in repo_path.name: + raise CorruptedCacheException(f"Repo path is not a valid HuggingFace cache directory: {repo_path}") + + repo_type, repo_id = repo_path.name.split("--", maxsplit=1) + repo_type = repo_type[:-1] # "models" -> "model" + repo_id = repo_id.replace("--", "/") # google/fleurs -> "google/fleurs" + + if repo_type not in {"dataset", "model", "space"}: + raise CorruptedCacheException( + f"Repo type must be `dataset`, `model` or `space`, found `{repo_type}` ({repo_path})." + ) + + blob_stats: dict[Path, os.stat_result] = {} # Key is blob_path, value is blob stats + + snapshots_path = repo_path / "snapshots" + refs_path = repo_path / "refs" + + if not snapshots_path.exists() or not snapshots_path.is_dir(): + raise CorruptedCacheException(f"Snapshots dir doesn't exist in cached repo: {snapshots_path}") + + # Scan over `refs` directory + + # key is revision hash, value is set of refs + refs_by_hash: dict[str, set[str]] = defaultdict(set) + if refs_path.exists(): + # Example of `refs` directory + # ── refs + # ├── main + # └── refs + # └── pr + # └── 1 + if refs_path.is_file(): + raise CorruptedCacheException(f"Refs directory cannot be a file: {refs_path}") + + for ref_path in refs_path.glob("**/*"): + # glob("**/*") iterates over all files and directories -> skip directories + if ref_path.is_dir() or ref_path.name in FILES_TO_IGNORE: + continue + + ref_name = str(ref_path.relative_to(refs_path)) + with ref_path.open() as f: + commit_hash = f.read() + + refs_by_hash[commit_hash].add(ref_name) + + # Scan snapshots directory + cached_revisions: set[CachedRevisionInfo] = set() + for revision_path in snapshots_path.iterdir(): + # Ignore OS-created helper files + if revision_path.name in FILES_TO_IGNORE: + continue + if revision_path.is_file(): + raise CorruptedCacheException(f"Snapshots folder corrupted. Found a file: {revision_path}") + + cached_files = set() + for file_path in revision_path.glob("**/*"): + # glob("**/*") iterates over all files and directories -> skip directories + if file_path.is_dir(): + continue + + blob_path = Path(file_path).resolve() + if not blob_path.exists(): + raise CorruptedCacheException(f"Blob missing (broken symlink): {blob_path}") + + if blob_path not in blob_stats: + blob_stats[blob_path] = blob_path.stat() + + cached_files.add( + CachedFileInfo( + file_name=file_path.name, + file_path=file_path, + size_on_disk=blob_stats[blob_path].st_size, + blob_path=blob_path, + blob_last_accessed=blob_stats[blob_path].st_atime, + blob_last_modified=blob_stats[blob_path].st_mtime, + ) + ) + + # Last modified is either the last modified blob file or the revision folder + # itself if it is empty + if len(cached_files) > 0: + revision_last_modified = max(blob_stats[file.blob_path].st_mtime for file in cached_files) + else: + revision_last_modified = revision_path.stat().st_mtime + + cached_revisions.add( + CachedRevisionInfo( + commit_hash=revision_path.name, + files=frozenset(cached_files), + refs=frozenset(refs_by_hash.pop(revision_path.name, set())), + size_on_disk=sum( + blob_stats[blob_path].st_size for blob_path in {file.blob_path for file in cached_files} + ), + snapshot_path=revision_path, + last_modified=revision_last_modified, + ) + ) + + # Check that all refs referred to an existing revision + if len(refs_by_hash) > 0: + raise CorruptedCacheException( + f"Reference(s) refer to missing commit hashes: {dict(refs_by_hash)} ({repo_path})." + ) + + # Last modified is either the last modified blob file or the repo folder itself if + # no blob files has been found. Same for last accessed. + if len(blob_stats) > 0: + repo_last_accessed = max(stat.st_atime for stat in blob_stats.values()) + repo_last_modified = max(stat.st_mtime for stat in blob_stats.values()) + else: + repo_stats = repo_path.stat() + repo_last_accessed = repo_stats.st_atime + repo_last_modified = repo_stats.st_mtime + + # Build and return frozen structure + return CachedRepoInfo( + nb_files=len(blob_stats), + repo_id=repo_id, + repo_path=repo_path, + repo_type=repo_type, # type: ignore + revisions=frozenset(cached_revisions), + size_on_disk=sum(stat.st_size for stat in blob_stats.values()), + last_accessed=repo_last_accessed, + last_modified=repo_last_modified, + ) + + +def _format_size(num: int) -> str: + """Format size in bytes into a human-readable string. + + Taken from https://stackoverflow.com/a/1094933 + """ + num_f = float(num) + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: + if abs(num_f) < 1000.0: + return f"{num_f:3.1f}{unit}" + num_f /= 1000.0 + return f"{num_f:.1f}Y" + + +def _try_delete_path(path: Path, path_type: str) -> None: + """Try to delete a local file or folder. + + If the path does not exist, error is logged as a warning and then ignored. + + Args: + path (`Path`) + Path to delete. Can be a file or a folder. + path_type (`str`) + What path are we deleting ? Only for logging purposes. Example: "snapshot". + """ + logger.debug(f"Delete {path_type}: {path}") + try: + if path.is_file(): + os.remove(path) + else: + shutil.rmtree(path) + except FileNotFoundError: + # Nothing to delete: the path is already gone. Log without traceback to avoid flooding the + # terminal when deleting many paths at once. + logger.warning(f"Couldn't delete {path_type}: file not found ({path})") + except PermissionError: + logger.warning(f"Couldn't delete {path_type}: permission denied ({path})", exc_info=True) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_chunk_utils.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_chunk_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..081d6e32b0ccc1e123b8f0719ae307eeb7b46133 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_chunk_utils.py @@ -0,0 +1,64 @@ +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains a utility to iterate by chunks over an iterator.""" + +import itertools +from collections.abc import Iterable +from typing import TypeVar + + +T = TypeVar("T") + + +def chunk_iterable(iterable: Iterable[T], chunk_size: int) -> Iterable[Iterable[T]]: + """Iterates over an iterator chunk by chunk. + + Taken from https://stackoverflow.com/a/8998040. + See also https://github.com/huggingface/huggingface_hub/pull/920#discussion_r938793088. + + Args: + iterable (`Iterable`): + The iterable on which we want to iterate. + chunk_size (`int`): + Size of the chunks. Must be a strictly positive integer (e.g. >0). + + Example: + + ```python + >>> from huggingface_hub.utils import chunk_iterable + + >>> for items in chunk_iterable(range(17), chunk_size=8): + ... print(items) + # [0, 1, 2, 3, 4, 5, 6, 7] + # [8, 9, 10, 11, 12, 13, 14, 15] + # [16] # smaller last chunk + ``` + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If `chunk_size` <= 0. + + > [!WARNING] + > The last chunk can be smaller than `chunk_size`. + """ + if not isinstance(chunk_size, int) or chunk_size <= 0: + raise ValueError("`chunk_size` must be a strictly positive integer (>0).") + + iterator = iter(iterable) + while True: + try: + next_item = next(iterator) + except StopIteration: + return + yield itertools.chain((next_item,), itertools.islice(iterator, chunk_size - 1)) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_datetime.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_datetime.py new file mode 100644 index 0000000000000000000000000000000000000000..44961712a959d8be370b7e3adcd2ef298f27c108 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_datetime.py @@ -0,0 +1,66 @@ +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to handle datetimes in Huggingface Hub.""" + +from datetime import datetime, timezone + + +def parse_datetime(date_string: str) -> datetime: + """ + Parses a date_string returned from the server to a datetime object. + + This parser is a weak-parser is the sense that it handles only a single format of + date_string. It is expected that the server format will never change. The + implementation depends only on the standard lib to avoid an external dependency + (python-dateutil). See full discussion about this decision on PR: + https://github.com/huggingface/huggingface_hub/pull/999. + + Example: + ```py + > parse_datetime('2022-08-19T07:19:38.123Z') + datetime.datetime(2022, 8, 19, 7, 19, 38, 123000, tzinfo=timezone.utc) + ``` + + Args: + date_string (`str`): + A string representing a datetime returned by the Hub server. + String is expected to follow '%Y-%m-%dT%H:%M:%S.%fZ' pattern. + + Returns: + A python datetime object. + + Raises: + :class:`ValueError`: + If `date_string` cannot be parsed. + """ + try: + # Normalize the string to always have 6 digits of fractional seconds + if date_string.endswith("Z"): + # Case 1: No decimal point (e.g., "2024-11-16T00:27:02Z") + if "." not in date_string: + # No fractional seconds - insert .000000 + date_string = date_string[:-1] + ".000000Z" + # Case 2: Has decimal point (e.g., "2022-08-19T07:19:38.123456789Z") + else: + # Get the fractional and base parts + base, fraction = date_string[:-1].split(".") + # fraction[:6] takes first 6 digits and :0<6 pads with zeros if less than 6 digits + date_string = f"{base}.{fraction[:6]:0<6}Z" + + return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc) + except ValueError as e: + raise ValueError( + f"Cannot parse '{date_string}' as a datetime. Date string is expected to" + " follow '%Y-%m-%dT%H:%M:%S.%fZ' pattern." + ) from e diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_deprecation.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_deprecation.py new file mode 100644 index 0000000000000000000000000000000000000000..99df8405958f565e6bdf8aaa61e0042d685c4e77 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_deprecation.py @@ -0,0 +1,136 @@ +import warnings +from collections.abc import Iterable +from functools import wraps +from inspect import Parameter, signature + + +def _deprecate_positional_args(*, version: str): + """Decorator for methods that issues warnings for positional arguments. + Using the keyword-only argument syntax in pep 3102, arguments after the + * will issue a warning when passed as a positional argument. + + Args: + version (`str`): + The version when positional arguments will result in error. + """ + + def _inner_deprecate_positional_args(f): + sig = signature(f) + kwonly_args = [] + all_args = [] + for name, param in sig.parameters.items(): + if param.kind == Parameter.POSITIONAL_OR_KEYWORD: + all_args.append(name) + elif param.kind == Parameter.KEYWORD_ONLY: + kwonly_args.append(name) + + @wraps(f) + def inner_f(*args, **kwargs): + extra_args = len(args) - len(all_args) + if extra_args <= 0: + return f(*args, **kwargs) + # extra_args > 0 + args_msg = [ + f"{name}='{arg}'" if isinstance(arg, str) else f"{name}={arg}" + for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:]) + ] + args_msg = ", ".join(args_msg) + warnings.warn( + f"Deprecated positional argument(s) used in '{f.__name__}': pass" + f" {args_msg} as keyword args. From version {version} passing these" + " as positional arguments will result in an error,", + FutureWarning, + ) + kwargs.update(zip(sig.parameters, args)) + return f(**kwargs) + + return inner_f + + return _inner_deprecate_positional_args + + +def _deprecate_arguments( + *, + version: str, + deprecated_args: Iterable[str], + custom_message: str | None = None, +): + """Decorator to issue warnings when using deprecated arguments. + + TODO: could be useful to be able to set a custom error message. + + Args: + version (`str`): + The version when deprecated arguments will result in error. + deprecated_args (`list[str]`): + List of the arguments to be deprecated. + custom_message (`str`, *optional*): + Warning message that is raised. If not passed, a default warning message + will be created. + """ + + def _inner_deprecate_positional_args(f): + sig = signature(f) + + @wraps(f) + def inner_f(*args, **kwargs): + # Check for used deprecated arguments + used_deprecated_args = [] + for _, parameter in zip(args, sig.parameters.values()): + if parameter.name in deprecated_args: + used_deprecated_args.append(parameter.name) + for kwarg_name, kwarg_value in kwargs.items(): + if ( + # If argument is deprecated but still used + kwarg_name in deprecated_args + # And then the value is not the default value + and kwarg_value != sig.parameters[kwarg_name].default + ): + used_deprecated_args.append(kwarg_name) + + # Warn and proceed + if len(used_deprecated_args) > 0: + message = ( + f"Deprecated argument(s) used in '{f.__name__}':" + f" {', '.join(used_deprecated_args)}. Will not be supported from" + f" version '{version}'." + ) + if custom_message is not None: + message += "\n\n" + custom_message + warnings.warn(message, FutureWarning) + return f(*args, **kwargs) + + return inner_f + + return _inner_deprecate_positional_args + + +def _deprecate_method(*, version: str, message: str | None = None): + """Decorator to issue warnings when using a deprecated method. + + Args: + version (`str`): + The version when deprecated arguments will result in error. + message (`str`, *optional*): + Warning message that is raised. If not passed, a default warning message + will be created. + """ + + def _inner_deprecate_method(f): + name = f.__name__ + if name == "__init__": + name = f.__qualname__.split(".")[0] # class name instead of method name + + @wraps(f) + def inner_f(*args, **kwargs): + warning_message = ( + f"'{name}' (from '{f.__module__}') is deprecated and will be removed from version '{version}'." + ) + if message is not None: + warning_message += " " + message + warnings.warn(warning_message, FutureWarning) + return f(*args, **kwargs) + + return inner_f + + return _inner_deprecate_method diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_detect_agent.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_detect_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..fb2661bc2288fdbcea44de7023b2dd8268c6d215 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_detect_agent.py @@ -0,0 +1,204 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Detect whether the process is being invoked by an AI coding agent. + +Detection is based on environment variables that AI agents set in their shell +sessions. `AI_AGENT` and `AGENT` are treated as a universal standard (any +tool can set its harness id there); the remaining checks are tool-specific and +ordered by priority (first match wins). + +The list of known harnesses is maintained on the Hub and exposed at +`{ENDPOINT}/api/agent-harnesses`. We fetch it at most once a day and cache it +locally so the list can be updated without requiring a new client release. + +Detection is entirely best-effort: there is no hardcoded list of harnesses. When +the registry cannot be fetched (and no cached copy is available), detection simply +reports "no agent". Any error while fetching/reading the registry is swallowed — +detection must never make a process fail. + +More details: https://huggingface.co/docs/hub/agents-overview#register-your-agent-harness +""" + +import json +import os +import time +from pathlib import Path +from typing import Optional, TypedDict + +from .. import constants +from . import logging + + +logger = logging.get_logger(__name__) + +# Refresh the cached registry at most once every 24 hours. +_REGISTRY_TTL_SECONDS = 24 * 3600 + +# Short timeout: fetching the registry is best-effort telemetry, never block the caller for long. +_REGISTRY_FETCH_TIMEOUT = 3 + + +class HarnessInfo(TypedDict, total=False): + """A single harness entry. `envVars` maps an env var name to a match pattern (see `_env_vars_match`).""" + + envVars: dict[str, str] + + +class Registry(TypedDict): + """The agent harness registry, as served by `{ENDPOINT}/api/agent-harnesses`.""" + + standardEnvVars: list[str] + harnesses: dict[str, HarnessInfo] + + +# Empty registry: detection is disabled (no agent ever detected). Used when the +# Hub is unreachable and no cached copy is available. +_EMPTY_REGISTRY: Registry = {"standardEnvVars": [], "harnesses": {}} + +# In-process cache of the resolved registry. Populated lazily on first detection. +_registry: Registry | None = None + + +def detect_agent() -> Optional[str]: + """Return the id of the detected AI agent harness or `None`. + + Harnesses are checked in registry order; for each one we match its env var + pattern(s) and, failing that, the standard `AI_AGENT` / `AGENT` vars + against the harness id. The first match wins. When a standard var is set to + an unrecognized value, `"unknown"` is returned. + """ + registry = _get_registry() + standard_vars = registry.get("standardEnvVars") or [] + harnesses = registry.get("harnesses") or {} + + for harness_id, info in harnesses.items(): + env_vars = (info or {}).get("envVars") + if env_vars and _env_vars_match(env_vars): + return harness_id + for var in standard_vars: + if os.environ.get(var, "").strip() == harness_id: + return harness_id + + # No harness matched but a standard var is set => unrecognized agent. + lowercased_harnesses = {k.lower() for k in harnesses.keys()} + for var in standard_vars: + if value := os.environ.get(var, "").strip().lower(): + if value in lowercased_harnesses: + return value + return "unknown" + + return None + + +def is_agent() -> bool: + """Return `True` if the process is being invoked by an AI coding agent.""" + return detect_agent() is not None + + +def _env_vars_match(env_vars: dict[str, str]) -> bool: + """Return `True` if any `(var, pattern)` from the harness matches the environment. + + Supported patterns: + - `"*"`: the variable is set to any non-empty value + - `""`: the variable equals this exact value + """ + for var, pattern in env_vars.items(): + value = os.environ.get(var) + if not value: + continue + if pattern == "*": + return True + if value == pattern: + return True + return False + + +def _get_registry() -> Registry: + """Return the harness registry, loading (and caching in-process) on first call. + + Best-effort: any unexpected error degrades to an empty registry so detection + never raises. + """ + global _registry + if _registry is None: + try: + _registry = _load_registry() + except Exception: + logger.debug("Could not resolve agent harnesses registry.", exc_info=True) + _registry = _EMPTY_REGISTRY + return _registry + + +def _load_registry() -> Registry: + """Resolve the registry from the local cache or the Hub. + + No hardcoded list: if the Hub is unreachable and no cached copy exists, an + empty registry is returned (i.e. no agent is detected). + """ + path = constants.AGENT_HARNESSES_PATH + + # 1. Use the cached file if it was refreshed within the last 24 hours. + if cached := _read_cached_registry(path, max_age=_REGISTRY_TTL_SECONDS): + return cached + + # 2. Otherwise refresh it from the Hub and persist it for next time. + if (fetched := _fetch_registry()) is not None: + _write_cached_registry(path, fetched) + return fetched + + # 3. Fetch failed: reuse a stale cache if available, else give up (no detection). + if stale := _read_cached_registry(path, max_age=None): + return stale + return _EMPTY_REGISTRY + + +def _read_cached_registry(path: str, max_age: int | None) -> Registry | None: + """Return the cached registry, or `None` if missing/stale/unreadable.""" + try: + if not os.path.exists(path): + return None + if max_age is not None and (time.time() - os.path.getmtime(path)) >= max_age: + return None + with open(path, encoding="utf-8") as f: + return json.load(f) + except Exception: + logger.debug("Could not read cached agent harnesses registry.", exc_info=True) + return None + + +def _write_cached_registry(path: str, registry: Registry) -> None: + try: + Path(path).parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(registry, f) + except Exception: + logger.debug("Could not cache agent harnesses registry.", exc_info=True) + + +def _fetch_registry() -> Registry | None: + """Fetch the registry from the Hub. Returns `None` when offline or on any error.""" + if constants.HF_HUB_OFFLINE: + return None + try: + from ._http import get_session + + response = get_session().get( + f"{constants.ENDPOINT}/api/agent-harnesses", + timeout=_REGISTRY_FETCH_TIMEOUT, + ) + response.raise_for_status() + return response.json() + except Exception: + logger.debug("Could not fetch agent harnesses registry from the Hub.", exc_info=True) + return None diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_dotenv.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_dotenv.py new file mode 100644 index 0000000000000000000000000000000000000000..dd6de0ce23f9c24137a65c5e9ac7728da3073a27 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_dotenv.py @@ -0,0 +1,68 @@ +# AI-generated module (ChatGPT) +import re + + +# Escape sequences expanded inside quoted values. Double-quoted values additionally +# expand "\$" to "$"; single-quoted values keep it verbatim. +_ESCAPES = {"n": "\n", "t": "\t", '"': '"', "\\": "\\"} +_DOUBLE_QUOTE_ESCAPES = {**_ESCAPES, "$": "$"} + + +def _unescape(value: str, escapes: dict[str, str]) -> str: + r"""Expand backslash escapes in a single left-to-right pass. + + Processing in one pass (rather than chained `str.replace` calls) ensures an escaped + backslash (`\\`) is consumed as a unit and cannot merge with the following character, + e.g. `\\n` is a backslash followed by `n`, not a newline. Unknown escapes are kept as-is. + """ + return re.sub(r"\\(.)", lambda match: escapes.get(match.group(1), match.group(0)), value) + + +def load_dotenv(dotenv_str: str, environ: dict[str, str] | None = None) -> dict[str, str]: + """ + Parse a DOTENV-format string and return a dictionary of key-value pairs. + Handles quoted values, comments, export keyword, and blank lines. + """ + env: dict[str, str] = {} + line_pattern = re.compile( + r""" + ^\s* + (?:export[^\S\n]+)? # optional export + ([A-Za-z_][A-Za-z0-9_]*) # key + [^\S\n]*(=)?[^\S\n]* + ( # value group + (?: + '(?:\\'|[^'])*' # single-quoted value + | \"(?:\\\"|[^\"])*\" # double-quoted value + | [^#\n\r]+? # unquoted value + ) + )? + [^\S\n]*(?:\#.*)?$ # optional inline comment + """, + re.VERBOSE, + ) + + for line in dotenv_str.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue # Skip comments and empty lines + + match = line_pattern.match(line) + if match: + key = match.group(1) + val = None + if match.group(2): # if there is '=' + raw_val = match.group(3) or "" + val = raw_val.strip() + # Remove surrounding quotes if quoted + if (val.startswith('"') and val.endswith('"')) or (val.startswith("'") and val.endswith("'")): + escapes = _DOUBLE_QUOTE_ESCAPES if raw_val.startswith('"') else _ESCAPES + val = _unescape(val[1:-1], escapes) + elif environ is not None: + # Get it from the current environment + val = environ.get(key) + + if val is not None: + env[key] = val + + return env diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_experimental.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_experimental.py new file mode 100644 index 0000000000000000000000000000000000000000..82753e1fe052639ee9f4cc7adcbbe2ffb595c88b --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_experimental.py @@ -0,0 +1,67 @@ +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to flag a feature as "experimental" in Huggingface Hub.""" + +import warnings +from collections.abc import Callable +from functools import wraps + +from .. import constants + + +def experimental(fn: Callable) -> Callable: + """Decorator to flag a feature as experimental. + + An experimental feature triggers a warning when used as it might be subject to breaking changes without prior notice + in the future. + + Warnings can be disabled by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment variable. + + Args: + fn (`Callable`): + The function to flag as experimental. + + Returns: + `Callable`: The decorated function. + + Example: + + ```python + >>> from huggingface_hub.utils import experimental + + >>> @experimental + ... def my_function(): + ... print("Hello world!") + + >>> my_function() + UserWarning: 'my_function' is experimental and might be subject to breaking changes in the future without prior + notice. You can disable this warning by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment variable. + Hello world! + ``` + """ + # For classes, put the "experimental" around the "__new__" method => __new__ will be removed in warning message + name = fn.__qualname__[: -len(".__new__")] if fn.__qualname__.endswith(".__new__") else fn.__qualname__ + + @wraps(fn) + def _inner_fn(*args, **kwargs): + if not constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING: + warnings.warn( + f"'{name}' is experimental and might be subject to breaking changes in the future without prior notice." + " You can disable this warning by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment" + " variable.", + UserWarning, + ) + return fn(*args, **kwargs) + + return _inner_fn diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_fixes.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_fixes.py new file mode 100644 index 0000000000000000000000000000000000000000..b4f755fcc0aebc229dbf75c7d0ea51a50c91afff --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_fixes.py @@ -0,0 +1,123 @@ +import contextlib +import os +import shutil +import stat +import tempfile +import time +from collections.abc import Callable, Generator +from functools import partial +from pathlib import Path + +import yaml +from filelock import BaseFileLock, FileLock, SoftFileLock, Timeout + +from .. import constants +from . import logging + + +logger = logging.get_logger(__name__) + +# Wrap `yaml.dump` to set `allow_unicode=True` by default. +# +# Example: +# ```py +# >>> yaml.dump({"emoji": "👀", "some unicode": "日本か"}) +# 'emoji: "\\U0001F440"\nsome unicode: "\\u65E5\\u672C\\u304B"\n' +# +# >>> yaml_dump({"emoji": "👀", "some unicode": "日本か"}) +# 'emoji: "👀"\nsome unicode: "日本か"\n' +# ``` +yaml_dump: Callable[..., str] = partial(yaml.dump, stream=None, allow_unicode=True) # type: ignore + + +@contextlib.contextmanager +def SoftTemporaryDirectory( + suffix: str | None = None, + prefix: str | None = None, + dir: Path | str | None = None, + **kwargs, +) -> Generator[Path, None, None]: + """ + Context manager to create a temporary directory and safely delete it. + + If tmp directory cannot be deleted normally, we set the WRITE permission and retry. + If cleanup still fails, we give up but don't raise an exception. This is equivalent + to `tempfile.TemporaryDirectory(..., ignore_cleanup_errors=True)` introduced in + Python 3.10. + + See https://www.scivision.dev/python-tempfile-permission-error-windows/. + """ + tmpdir = tempfile.TemporaryDirectory(prefix=prefix, suffix=suffix, dir=dir, **kwargs) + yield Path(tmpdir.name).resolve() + + try: + # First once with normal cleanup + shutil.rmtree(tmpdir.name) + except Exception: + # If failed, try to set write permission and retry + try: + shutil.rmtree(tmpdir.name, onerror=_set_write_permission_and_retry) + except Exception: + pass + + # And finally, cleanup the tmpdir. + # If it fails again, give up but do not throw error + try: + tmpdir.cleanup() + except Exception: + pass + + +def _set_write_permission_and_retry(func, path, excinfo): + os.chmod(path, stat.S_IWRITE) + func(path) + + +@contextlib.contextmanager +def WeakFileLock(lock_file: str | Path, *, timeout: float | None = None) -> Generator[BaseFileLock, None, None]: + """A filelock with some custom logic. + + This filelock is weaker than the default filelock in that: + 1. It won't raise an exception if release fails. + 2. It will default to a SoftFileLock if the filesystem does not support flock. + 3. Lock files are created with mode 0o664 (group-writable) instead of the default 0o644. + This allows multiple users sharing a cache directory to wait for locks. + + An INFO log message is emitted every 10 seconds if the lock is not acquired immediately. + If a timeout is provided, a `filelock.Timeout` exception is raised if the lock is not acquired within the timeout. + """ + log_interval = constants.FILELOCK_LOG_EVERY_SECONDS + lock = FileLock(lock_file, timeout=log_interval, mode=0o664) + start_time = time.time() + + while True: + elapsed_time = time.time() - start_time + if timeout is not None and elapsed_time >= timeout: + raise Timeout(str(lock_file)) + + try: + lock.acquire(timeout=min(log_interval, timeout - elapsed_time) if timeout else log_interval) + except Timeout: + logger.info( + f"Still waiting to acquire lock on {lock_file} (elapsed: {time.time() - start_time:.1f} seconds)" + ) + except NotImplementedError as e: + if "use SoftFileLock instead" in str(e): + logger.warning( + "FileSystem does not appear to support flock. Falling back to SoftFileLock for %s", lock_file + ) + lock = SoftFileLock(lock_file, timeout=log_interval) + continue + else: + break + + try: + yield lock + finally: + try: + lock.release() + except OSError: + try: + Path(lock_file).unlink() + except OSError: + pass diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_git_credential.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_git_credential.py new file mode 100644 index 0000000000000000000000000000000000000000..8ef2b2dce55c1274d89dc9e34beaf92d0989f8e3 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_git_credential.py @@ -0,0 +1,119 @@ +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to manage Git credentials.""" + +import re +import subprocess + +from ..constants import ENDPOINT +from ._subprocess import run_interactive_subprocess, run_subprocess + + +GIT_CREDENTIAL_REGEX = re.compile( + r""" + ^\s* # start of line + credential\.helper # credential.helper value + \s*=\s* # separator + ([\w\-\/]+) # the helper name or absolute path (group 1) + (\s|$) # whitespace or end of line + """, + flags=re.MULTILINE | re.IGNORECASE | re.VERBOSE, +) + + +def list_credential_helpers(folder: str | None = None) -> list[str]: + """Return the list of git credential helpers configured. + + See https://git-scm.com/docs/gitcredentials. + + Credentials are saved in all configured helpers (store, cache, macOS keychain,...). + Calls "`git credential approve`" internally. See https://git-scm.com/docs/git-credential. + + Args: + folder (`str`, *optional*): + The folder in which to check the configured helpers. + """ + try: + output = run_subprocess("git config --list", folder=folder).stdout + parsed = _parse_credential_output(output) + return parsed + except subprocess.CalledProcessError as exc: + raise OSError(exc.stderr) + + +def set_git_credential(token: str, username: str = "hf_user", folder: str | None = None) -> None: + """Save a username/token pair in git credential for HF Hub registry. + + Credentials are saved in all configured helpers (store, cache, macOS keychain,...). + Calls "`git credential approve`" internally. See https://git-scm.com/docs/git-credential. + + Args: + username (`str`, defaults to `"hf_user"`): + A git username. Defaults to `"hf_user"`, the default user used in the Hub. + token (`str`, defaults to `"hf_user"`): + A git password. In practice, the User Access Token for the Hub. + See https://huggingface.co/settings/tokens. + folder (`str`, *optional*): + The folder in which to check the configured helpers. + """ + with run_interactive_subprocess("git credential approve", folder=folder) as ( + stdin, + _, + ): + stdin.write(f"url={ENDPOINT}\nusername={username.lower()}\npassword={token}\n\n") + stdin.flush() + + +def unset_git_credential(username: str = "hf_user", folder: str | None = None) -> None: + """Erase credentials from git credential for HF Hub registry. + + Credentials are erased from the configured helpers (store, cache, macOS + keychain,...), if any. If `username` is not provided, any credential configured for + HF Hub endpoint is erased. + Calls "`git credential erase`" internally. See https://git-scm.com/docs/git-credential. + + Args: + username (`str`, defaults to `"hf_user"`): + A git username. Defaults to `"hf_user"`, the default user used in the Hub. + folder (`str`, *optional*): + The folder in which to check the configured helpers. + """ + with run_interactive_subprocess("git credential reject", folder=folder) as ( + stdin, + _, + ): + standard_input = f"url={ENDPOINT}\n" + if username is not None: + standard_input += f"username={username.lower()}\n" + standard_input += "\n" + + stdin.write(standard_input) + stdin.flush() + + +def _parse_credential_output(output: str) -> list[str]: + """Parse the output of `git credential fill` to extract the password. + + Args: + output (`str`): + The output of `git credential fill`. + """ + # NOTE: If user has set a helper for a custom URL, it will not be caught here. + # Example: `credential.https://huggingface.co.helper=store` + # See: https://github.com/huggingface/huggingface_hub/pull/1138#discussion_r1013324508 + return sorted( # Sort for nice printing + { # Might have some duplicates + match[0] for match in GIT_CREDENTIAL_REGEX.findall(output) + } + ) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_headers.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_headers.py new file mode 100644 index 0000000000000000000000000000000000000000..aa6867401da47688c66cc1183e04d3facb708262 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_headers.py @@ -0,0 +1,208 @@ +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to handle headers to send in calls to Huggingface Hub.""" + +from huggingface_hub.errors import LocalTokenNotFoundError + +from .. import constants +from ._auth import get_token +from ._detect_agent import detect_agent +from ._runtime import ( + get_hf_hub_version, + get_python_version, + get_torch_version, + is_torch_available, +) +from ._validators import validate_hf_hub_args + + +@validate_hf_hub_args +def build_hf_headers( + *, + token: bool | str | None = None, + library_name: str | None = None, + library_version: str | None = None, + user_agent: dict | str | None = None, + headers: dict[str, str] | None = None, +) -> dict[str, str]: + """ + Build headers dictionary to send in a HF Hub call. + + By default, authorization token is always provided either from argument (explicit + use) or retrieved from the cache (implicit use). To explicitly avoid sending the + token to the Hub, set `token=False` or set the `HF_HUB_DISABLE_IMPLICIT_TOKEN` + environment variable. + + In case of an API call that requires write access, an error is thrown if token is + `None` or token is an organization token (starting with `"api_org***"`). + + In addition to the auth header, a user-agent is added to provide information about + the installed packages (versions of python, huggingface_hub, torch). + + Args: + token (`str`, `bool`, *optional*): + The token to be sent in authorization header for the Hub call: + - if a string, it is used as the Hugging Face token + - if `True`, the token is read from the machine (cache or env variable) + - if `False`, authorization header is not set + - if `None`, the token is read from the machine only except if + `HF_HUB_DISABLE_IMPLICIT_TOKEN` env variable is set. + library_name (`str`, *optional*): + The name of the library that is making the HTTP request. Will be added to + the user-agent header. + library_version (`str`, *optional*): + The version of the library that is making the HTTP request. Will be added + to the user-agent header. + user_agent (`str`, `dict`, *optional*): + The user agent info in the form of a dictionary or a single string. It will + be completed with information about the installed packages. + headers (`dict`, *optional*): + Additional headers to include in the request. Those headers take precedence + over the ones generated by this function. + + Returns: + A `dict` of headers to pass in your API call. + + Example: + ```py + >>> build_hf_headers(token="hf_***") # explicit token + {"authorization": "Bearer hf_***", "user-agent": ""} + + >>> build_hf_headers(token=True) # explicitly use cached token + {"authorization": "Bearer hf_***",...} + + >>> build_hf_headers(token=False) # explicitly don't use cached token + {"user-agent": ...} + + >>> build_hf_headers() # implicit use of the cached token + {"authorization": "Bearer hf_***",...} + + # HF_HUB_DISABLE_IMPLICIT_TOKEN=True # to set as env variable + >>> build_hf_headers() # token is not sent + {"user-agent": ...} + + >>> build_hf_headers(library_name="transformers", library_version="1.2.3") + {"authorization": ..., "user-agent": "transformers/1.2.3; hf_hub/0.10.2; python/3.10.4; tensorflow/1.55"} + ``` + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If organization token is passed and "write" access is required. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If "write" access is required but token is not passed and not saved locally. + [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) + If `token=True` but token is not saved locally. + """ + # Get auth token to send + token_to_send = get_token_to_send(token) + + # Combine headers + hf_headers = { + "user-agent": _http_user_agent( + library_name=library_name, + library_version=library_version, + user_agent=user_agent, + ) + } + if token_to_send is not None: + hf_headers["authorization"] = f"Bearer {token_to_send}" + if headers is not None: + hf_headers.update(headers) + return hf_headers + + +def get_token_to_send(token: bool | str | None) -> str | None: + """Select the token to send from either `token` or the cache.""" + # Case token is explicitly provided + if isinstance(token, str): + return token + + # Case token is explicitly forbidden + if token is False: + return None + + # Case token is explicitly required + if token is True: + cached_token = get_token() + if cached_token is None: + raise LocalTokenNotFoundError( + "Token is required (`token=True`), but no token found. You" + " need to provide a token or be logged in to Hugging Face with" + " `hf auth login` or `huggingface_hub.login`. See" + " https://huggingface.co/settings/tokens." + ) + return cached_token + + # Case implicit use of the token is forbidden by env variable. Checked before resolving the + # cached token: `get_token()` may refresh an OAuth token (network call + file writes), which + # must not happen when the resolved token wouldn't be used anyway. + if constants.HF_HUB_DISABLE_IMPLICIT_TOKEN: + return None + + # Otherwise: we use the cached token as the user has not explicitly forbidden it + return get_token() + + +def _http_user_agent( + *, + library_name: str | None = None, + library_version: str | None = None, + user_agent: dict | str | None = None, +) -> str: + """Format a user-agent string containing information about the installed packages. + + Args: + library_name (`str`, *optional*): + The name of the library that is making the HTTP request. + library_version (`str`, *optional*): + The version of the library that is making the HTTP request. + user_agent (`str`, `dict`, *optional*): + The user agent info in the form of a dictionary or a single string. + + Returns: + The formatted user-agent string. + """ + if library_name is not None: + ua = f"{library_name}/{library_version}" + else: + ua = "unknown/None" + ua += f"; hf_hub/{get_hf_hub_version()}" + ua += f"; python/{get_python_version()}" + + if not constants.HF_HUB_DISABLE_TELEMETRY: + if is_torch_available(): + ua += f"; torch/{get_torch_version()}" + + agent = detect_agent() + if agent: + ua += f"; agent/{agent}" + + if isinstance(user_agent, dict): + ua += "; " + "; ".join(f"{k}/{v}" for k, v in user_agent.items()) + elif isinstance(user_agent, str): + ua += "; " + user_agent + + # Retrieve user-agent origin headers from environment variable + origin = constants.HF_HUB_USER_AGENT_ORIGIN + if origin is not None: + ua += "; origin/" + origin + + return _deduplicate_user_agent(ua) + + +def _deduplicate_user_agent(user_agent: str) -> str: + """Deduplicate redundant information in the generated user-agent.""" + # Split around ";" > Strip whitespaces > Store as dict keys (ensure unicity) > format back as string + # Order is implicitly preserved by dictionary structure (see https://stackoverflow.com/a/53657523). + return "; ".join({key.strip(): None for key in user_agent.split(";")}.keys()) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_hf_uris.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_hf_uris.py new file mode 100644 index 0000000000000000000000000000000000000000..a678c3e1c753ea97ee42794e1b7d6d00040089d7 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_hf_uris.py @@ -0,0 +1,642 @@ +# Copyright 2026-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Centralized parser for Hugging Face Hub URIs ('hf://...') and mount specifications. + +A HF URI is a URI-like string that identifies a location on the Hugging Face +Hub: a model/dataset/space/kernel repository, a bucket, optionally a revision, +and optionally a path inside the repo or bucket. + +Canonical syntax: + +``` +hf://[/][@][/] +``` + +For convenience, [`parse_hf_uri`] also accepts Hugging Face **web URLs** (the +ones you copy-paste from your browser), e.g. +'https://huggingface.co/datasets/my-org/my-dataset/blob/main/train.csv'. They are +normalized to the canonical 'hf://' form before parsing. Only unambiguous URLs +(repository / bucket pages and file/folder viewer routes) are accepted; any other +route is rejected rather than guessed. + +A HF mount wraps a HF URI with a local mount path and an optional ':ro'/':rw' +flag (used by Spaces and Jobs volumes): + +``` +hf://[/][@][/]:[:ro|:rw] +``` + +See 'docs/source/en/package_reference/hf_uris.md' for the full grammar and examples. +""" + +import functools +import re +from dataclasses import dataclass, field +from urllib.parse import quote, unquote, urlsplit + +from huggingface_hub import constants +from huggingface_hub.errors import HfUriError, HFValidationError + +from ._validators import validate_repo_id + + +# Inverse map (singular -> plural URI prefix). Built once from the canonical +# 'constants.HF_URI_TYPE_PREFIXES' and used to render URIs. +_TYPE_TO_PREFIX: dict[str, str] = {v: k for k, v in constants.HF_URI_TYPE_PREFIXES.items()} + +# Special revisions that contain a '/'. They take precedence when splitting +# the part after '@' into '/'. Matches 'refs/pr/N' +# (Pull Request refs) and 'refs/convert/' (e.g. parquet conversions). +# The conversion name allows the typical git ref characters '[a-zA-Z0-9_.-]' +# so names like 'parquet-v2' or 'duckdb.v1' round-trip correctly. +_SPECIAL_REFS_REVISION_REGEX = re.compile(r"^refs/(?:convert/[\w.-]+|pr/\d+)") + +# Same as constants.HfUriType, but as a set of strings for easy lookup.) +_VALID_URI_TYPES: frozenset[str] = frozenset(constants.HF_URI_TYPE_PREFIXES.values()) + + +# Web-viewer routes that point at a file or folder and that map cleanly onto a +# '/' pair. Other routes (commit, commits, discussions, settings, +# edit, ...) do not identify a Hub location and are rejected by the URL parser. +_URL_REPO_LOCATION_ACTIONS: frozenset[str] = frozenset({"blame", "blob", "raw", "resolve", "tree"}) +# Bucket web routes that point at a file or folder. Buckets are not versioned, so +# these are followed directly by '' (no revision segment). +_URL_BUCKET_LOCATION_ACTIONS: frozenset[str] = frozenset({"resolve", "tree"}) + + +@dataclass(frozen=True) +class HfUri: + """Parsed representation of a Hugging Face Hub URI ('hf://...'). + + Attributes: + type (`str`): + One of 'model', 'dataset', 'space', 'kernel' or 'bucket'. + id (`str`): + The repository id ('namespace/name', e.g. 'my-org/my-model') for repo URIs, or the bucket id ('namespace/name') for bucket URIs. + revision (`str`, *optional*): + The revision specified after '@' in the URI, URL-decoded. 'None' if no revision was specified, or for bucket URIs (which + never carry a revision). Special refs like 'refs/pr/10' and 'refs/convert/parquet' are preserved as-is. + path_in_repo (`str`): + The path inside the repo or bucket. Empty string if the URI points at the root. + """ + + type: constants.HfUriType + id: str + revision: str | None = None + path_in_repo: str = "" + _raw: str | None = field(repr=False, hash=False, compare=False, default=None) + + def __post_init__(self) -> None: + uri = self._raw or "" # For error messages + + # Check valid URI type + if self.type not in _VALID_URI_TYPES: + raise HfUriError(uri=uri, msg=f"Invalid type '{self.type}'. Must be one of {sorted(_VALID_URI_TYPES)}.") + + # Check valid ID + if not self.id or self.id.count("/") != 1: + raise HfUriError(uri=uri, msg=f"Id must be 'namespace/name', got '{self.id}'.") + if self.type != "bucket": + try: + validate_repo_id(self.id) + except HFValidationError as e: + raise HfUriError(uri=uri, msg=str(e)) from e + + # Check valid revision + if self.revision is not None and not self.revision: + raise HfUriError(uri=uri, msg="Revision must not be an empty string.") + if self.type == "bucket" and self.revision is not None: + raise HfUriError(uri=uri, msg="Bucket URIs do not support a revision.") + + # Check valid path in repo + if self.path_in_repo: + if self.path_in_repo.startswith("/") or "//" in self.path_in_repo: + raise HfUriError(uri=uri, msg=f"Path must not contain empty segments (got '{self.path_in_repo}').") + + @property + def is_bucket(self) -> bool: + """True if this URI points at a bucket.""" + return self.type == "bucket" + + @property + def is_repo(self) -> bool: + """True if this URI points at a repository (model, dataset, space or kernel).""" + return self.type != "bucket" + + def to_uri(self) -> str: + """Render the URI as a canonical 'hf://' string. + + The type prefix is always written explicitly (e.g. 'hf://models/my-org/my-model'). + """ + parts: list[str] = [constants.HF_PROTOCOL, _TYPE_TO_PREFIX[self.type], "/", self.id] + if self.revision is not None: + # Encode '/' as '%2F' for revisions that would otherwise be split as '/' + # at parse time. Special refs ('refs/pr/N', 'refs/convert/') are kept verbatim + # because the parser matches them eagerly. + revision = self.revision + if "/" in revision and _SPECIAL_REFS_REVISION_REGEX.fullmatch(revision) is None: + revision = revision.replace("/", "%2F") + parts.append(f"@{revision}") + if self.path_in_repo: + parts.append(f"/{self.path_in_repo}") + return "".join(parts) + + def to_url(self, endpoint: str | None = None) -> str: + """Render the URI as a Hugging Face **web URL** (the kind you open in a browser). + + This is the inverse of parsing a URL with [`parse_hf_uri`]. The returned URL points at: + + - the repository / bucket landing page when no path or revision is set; + - the folder viewer ('/tree/') when only a revision is set; + - the file viewer ('/blob//') for repository files (revision defaults to 'main'); + - the tree route ('/tree/') for bucket files (buckets are not versioned). + + Args: + endpoint (`str`, *optional*): + Base endpoint to use. Defaults to 'constants.ENDPOINT' (i.e. 'https://huggingface.co'). + + Returns: + `str`: the web URL. + + Example: + ```py + >>> from huggingface_hub import parse_hf_uri + >>> parse_hf_uri("hf://datasets/my-org/my-dataset@v1/train.csv").to_url() + 'https://huggingface.co/datasets/my-org/my-dataset/blob/v1/train.csv' + ``` + """ + base = (endpoint or constants.ENDPOINT).rstrip("/") + # Percent-encode characters that would otherwise break the URL (spaces, '#', '?', ...), + # keeping '/' as the path separator. This is the inverse of the decoding done when parsing. + path = quote(self.path_in_repo, safe="/") + + if self.type == "bucket": + url = f"{base}/buckets/{self.id}" + if path: + url += f"/tree/{path}" + return url + + # Models live at the root ('hf.co/'); other repos are namespaced by their plural prefix. + url = f"{base}/{self.id}" if self.type == "model" else f"{base}/{_TYPE_TO_PREFIX[self.type]}/{self.id}" + revision = self.revision + # Percent-encode the branch/tag name so it stays a single, URL-safe segment: a '/' would + # otherwise open a new path segment and '#'/'?' would be read as a fragment/query when the + # URL is opened or parsed back. This mirrors the decoding done when parsing. Special refs + # ('refs/pr/N', 'refs/convert/') are used verbatim by the Hub web routes. + if revision is not None and _SPECIAL_REFS_REVISION_REGEX.fullmatch(revision) is None: + revision = quote(revision, safe="") + if path: + url += f"/blob/{revision or constants.DEFAULT_REVISION}/{path}" + elif revision is not None: + url += f"/tree/{revision}" + return url + + +@dataclass(frozen=True) +class HfMount: + """A HF URI paired with a local mount path and optional read-only flag. + + Used by Spaces and Jobs to describe volume mounts. The full syntax is: + + ``` + hf://[/][@][/]:[:ro|:rw] + ``` + + Attributes: + source ([`HfUri`]): + The parsed HF URI identifying the Hub resource to mount. + mount_path (`str`): + The local mount path (always starts with '/'). + read_only (`bool`, *optional*): + True if the mount ends with ':ro', False if it ends with ':rw', 'None' if no flag was provided. + """ + + source: HfUri + mount_path: str + read_only: bool | None = None + _raw: str | None = field(repr=False, hash=False, compare=False, default=None) + + def __post_init__(self) -> None: + raw = self._raw or "" + if not self.mount_path.startswith("/") or self.mount_path == "/": + raise HfUriError( + uri=raw, + msg=f"Mount path must be a non-empty absolute path starting with '/', got '{self.mount_path}'.", + ) + + def to_uri(self) -> str: + """Render the mount as a canonical 'hf://' string. + + Example: 'hf://models/my-org/my-model:/data:ro' + """ + parts = [self.source.to_uri(), ":", self.mount_path] + if self.read_only is not None: + parts.append(":ro" if self.read_only else ":rw") + return "".join(parts) + + +def is_hf_uri(uri: str) -> bool: + """Check if a string is a valid HF URI ('hf://...') or a recognized Hugging Face web URL.""" + try: + parse_hf_uri(uri) + return True + except HfUriError: + return False + + +@functools.lru_cache +def parse_hf_uri(uri: str, endpoint: str | None = None) -> HfUri: + """Parse a Hugging Face Hub URI ('hf://...') or a Hugging Face web URL. + + A HF URI is a URI-like string identifying a location on the Hugging Face Hub. The full grammar is: + + ``` + hf://[/][@][/] + ``` + + For convenience, Hugging Face **web URLs** (the ones you copy-paste from the website) are also + accepted and normalized to the canonical 'hf://' form, e.g. + 'https://huggingface.co/datasets/my-org/my-dataset/blob/main/train.csv'. Only unambiguous URLs + (repository / bucket pages and file/folder viewer routes) are accepted; any other route is rejected. + + See 'docs/source/en/package_reference/hf_uris.md' for the full specification. + + Args: + uri (`str`): + The URI to parse. Must start with 'hf://', or be a Hugging Face URL (e.g. 'https://huggingface.co/...'). + endpoint (`str`, *optional*): + A custom Hub endpoint (e.g. a self-hosted or proxied Hub like 'https://hub.my-company.com' or + 'http://localhost:8080/hf'). When provided, web URLs on that endpoint are recognized in addition to + the default Hugging Face hosts. Has no effect on 'hf://' URIs. + + Returns: + [`HfUri`]: the parsed URI. + + Raises: + [`HfUriError`]: + If the URI is malformed (missing prefix, invalid type, missing id, unsupported URL route, etc.). + + Examples: + ```py + >>> from huggingface_hub.utils import parse_hf_uri + >>> parse_hf_uri("hf://my-org/my-model") + HfUri(type='model', id='my-org/my-model', revision=None, path_in_repo='') + >>> parse_hf_uri("hf://datasets/my-org/my-dataset@refs/pr/3/train.json") + HfUri(type='dataset', id='my-org/my-dataset', revision='refs/pr/3', path_in_repo='train.json') + >>> parse_hf_uri("https://huggingface.co/datasets/my-org/my-dataset/blob/main/train.csv") + HfUri(type='dataset', id='my-org/my-dataset', revision='main', path_in_repo='train.csv') + ``` + """ + raw = uri + if uri.startswith(constants.HF_PROTOCOL): + body = uri[len(constants.HF_PROTOCOL) :] + if not body: + raise HfUriError(uri, f"Empty body after '{constants.HF_PROTOCOL}'.") + elif _looks_like_hf_url(uri, endpoint=endpoint): + body = _url_to_uri_body(uri, endpoint=endpoint) + else: + raise HfUriError( + uri, + f"Must start with '{constants.HF_PROTOCOL}' or be a Hugging Face URL (e.g. 'https://huggingface.co/...'). " + f"Expected format: {constants.HF_PROTOCOL}[/][@][/]", + ) + + type_, location = _split_type(body, raw=raw) + + if type_ == "bucket": + return _parse_bucket_body(location, type_, raw=raw) + return _parse_repo_body(location, type_, raw=raw) + + +def _endpoint_host_and_path(endpoint: str | None) -> tuple[str | None, str]: + """Return the lowercased host and stripped path prefix of a custom Hub 'endpoint'. + + E.g. 'https://hub.my-company.com' -> ('hub.my-company.com', '') and a self-hosted + 'http://localhost:8080/hf' -> ('localhost', 'hf'). Returns '(None, "")' when 'endpoint' is None. + """ + if endpoint is None: + return None, "" + # Prefix '//' for scheme-less endpoints so 'urlsplit' populates 'netloc' instead of 'path'. + parsed = urlsplit(endpoint if "://" in endpoint else "//" + endpoint) + host = parsed.hostname.lower() if parsed.hostname else None + return host, parsed.path.strip("/") + + +def _recognized_hosts(endpoint: str | None) -> frozenset[str]: + """The set of hosts whose web URLs can be parsed: the default Hugging Face hosts plus 'endpoint'.""" + host, _ = _endpoint_host_and_path(endpoint) + return constants.HF_URL_HOSTS | {host} if host else constants.HF_URL_HOSTS + + +def _looks_like_hf_url(uri: str, endpoint: str | None = None) -> bool: + """Return True if 'uri' looks like a (possibly scheme-less) Hugging Face web URL.""" + lowered = uri.lower() + if lowered.startswith(("http://", "https://")): + return True + # Scheme-less host (e.g. 'huggingface.co/org/model'). + return any(lowered == host or lowered.startswith(host + "/") for host in _recognized_hosts(endpoint)) + + +def _decode_url_path_segment(segment: str) -> str: + """Percent-decode a single URL path segment (e.g. 'file%20name.txt' -> 'file name.txt'). + + A decoded '/' is re-encoded as '%2F' so the segment stays atomic when the normalized body is + re-split by the shared parser. This decodes ordinary path characters (spaces, '#', ...) that + browsers encode, while keeping '%2F'-encoded revisions (e.g. 'feature%2Ffoo') intact. + """ + return unquote(segment).replace("/", "%2F") + + +def _url_to_uri_body(url: str, endpoint: str | None = None) -> str: + """Normalize a Hugging Face web URL into the body of a 'hf://' URI (everything after 'hf://'). + + The returned string is fed back into the regular URI parsing logic, so all validation + (repo id, revision, empty path segments, ...) is shared with the canonical 'hf://' path. + Only unambiguous URLs are accepted: any unrecognized route raises [`HfUriError`]. When 'endpoint' + is provided, URLs on that custom Hub host are recognized too (and its path prefix is stripped). + """ + raw = url + # Prefix '//' for scheme-less inputs so 'urlsplit' populates 'netloc' instead of 'path'. + parsed = urlsplit(url if "://" in url else "//" + url) + host = (parsed.hostname or "").lower() + if host not in _recognized_hosts(endpoint): + raise HfUriError( + uri=raw, + msg=f"Unrecognized host '{host or url}'. Expected a Hugging Face URL (e.g. 'https://huggingface.co/...').", + ) + + # Query string and fragment are intentionally dropped (e.g. '?download=true'). + path = parsed.path + # For a self-hosted endpoint with a path prefix (e.g. 'http://localhost:8080/hf'), drop it so the + # remaining segments are '[/]/[/...]' just like on the public Hub. + endpoint_host, endpoint_path = _endpoint_host_and_path(endpoint) + if endpoint_path and host == endpoint_host: + prefix = "/" + endpoint_path + if path == prefix or path.startswith(prefix + "/"): + path = path[len(prefix) :] + segments = [segment for segment in path.split("/") if segment] + if not segments: + raise HfUriError(uri=raw, msg=f"Missing repository or bucket identifier in URL '{url}'.") + + # Optional type prefix ('datasets', 'spaces', 'kernels', 'buckets', 'models'). + type_prefix: str | None = None + if segments[0] in constants.HF_URI_TYPE_PREFIXES: + type_prefix = segments[0] + segments = segments[1:] + + # Everything in the web UI is namespaced ('/'); a single segment is a user or + # organization page (or a listing page), which we cannot map to a repository -> reject. + if len(segments) < 2: + raise HfUriError( + uri=raw, + msg=( + f"Cannot parse URL '{url}': expected a '/' repository or bucket. " + "User/organization pages and single-segment URLs are not supported." + ), + ) + repo_id = f"{segments[0]}/{segments[1]}" + rest = segments[2:] + + if type_prefix == "buckets": + if not rest: + return f"buckets/{repo_id}" + action, *tail = rest + if action not in _URL_BUCKET_LOCATION_ACTIONS: + raise HfUriError(uri=raw, msg=f"Cannot parse bucket URL '{url}': unsupported '/{action}/' route.") + path = "/".join(_decode_url_path_segment(segment) for segment in tail) + return f"buckets/{repo_id}/{path}" if path else f"buckets/{repo_id}" + + prefix = f"{type_prefix}/" if type_prefix else "" + if not rest: + return f"{prefix}{repo_id}" + action, *tail = rest + if action not in _URL_REPO_LOCATION_ACTIONS: + raise HfUriError( + uri=raw, + msg=( + f"Cannot parse URL '{url}': unsupported '/{action}/' route. " + "Only repository pages and file/folder viewer routes (blob, resolve, raw, tree, ...) can be parsed." + ), + ) + if not tail: + # e.g. '.../tree' with nothing after -> repository root. + return f"{prefix}{repo_id}" + # 'tail' is '/'; reuse the canonical '@/' splitting logic + # (special refs, URL-encoded slashes, ...) by handing it back to the URI parser. Each segment + # is percent-decoded first so file names with spaces, '#', ... resolve correctly; the revision + # segment's '%2F' survives (re-encoded by '_decode_url_path_segment') and is decoded downstream. + decoded = "/".join(_decode_url_path_segment(segment) for segment in tail) + return f"{prefix}{repo_id}@{decoded}" + + +def parse_hf_mount(mount_str: str) -> HfMount: + """Parse a HF mount specification ('hf://...:[:ro|:rw]'). + + A mount specification is a HF URI followed by a local mount path and an optional read-only/read-write flag. + The full grammar is: + + ``` + hf://[/][@][/]:[:ro|:rw] + ``` + + See 'docs/source/en/package_reference/hf_uris.md' for the full specification. + + Args: + mount_str (`str`): + The mount string to parse. Must start with 'hf://' and contain a ':' segment. + + Returns: + [`HfMount`]: the parsed mount. + + Raises: + [`HfUriError`]: + If the mount string is malformed (missing mount path, invalid URI, etc.). + + Examples: + ```py + >>> from huggingface_hub.utils import parse_hf_mount + >>> parse_hf_mount("hf://my-org/my-model:/data:ro") + HfMount(source=HfUri(type='model', id='my-org/my-model', revision=None, path_in_repo=''), mount_path='/data', read_only=True) + >>> parse_hf_mount("hf://buckets/my-org/my-bucket/sub/dir:/mnt:rw") + HfMount(source=HfUri(type='bucket', id='my-org/my-bucket', revision=None, path_in_repo='sub/dir'), mount_path='/mnt', read_only=False) + ``` + """ + if not mount_str.startswith(constants.HF_PROTOCOL): + raise HfUriError( + uri=mount_str, + msg=f"Must start with '{constants.HF_PROTOCOL}'.", + ) + + raw = mount_str + body = mount_str[len(constants.HF_PROTOCOL) :] + if not body: + raise HfUriError(uri=raw, msg=f"Empty body after '{constants.HF_PROTOCOL}'.") + + location, mount_path, read_only = _split_mount(body, raw=raw) + + if mount_path is None: + raise HfUriError(uri=raw, msg="Missing mount path. Expected ':' (e.g. 'hf://org/model:/data').") + + # Re-assemble the URI part and parse it + uri_str = constants.HF_PROTOCOL + location + try: + source = parse_hf_uri(uri_str) + except HfUriError as e: + raise HfUriError(uri=raw, msg=e.msg) from e + + return HfMount(source=source, mount_path=mount_path, read_only=read_only, _raw=raw) + + +def _split_mount(body: str, *, raw: str) -> tuple[str, str | None, bool | None]: + """Split the ':[:ro|:rw]' suffix from 'body'. + + Returns '(location, mount_path, read_only)' where 'mount_path' is 'None' if no mount segment is present. + """ + if body.endswith(":ro"): + read_only, body = True, body.removesuffix(":ro") + elif body.endswith(":rw"): + read_only, body = False, body.removesuffix(":rw") + else: + read_only = None + + # Mount paths always start with '/', so the delimiter is ':/'. + # We use rfind() because the mount segment is always trailing + idx = body.rfind(":/") + if idx == -1: + if read_only is not None: + raise HfUriError( + uri=raw, + msg="':ro'/':rw' suffix is only valid when a mount path is provided (e.g. 'hf://...:/:ro').", + ) + return body, None, None + + location = body[:idx] + mount_path = body[idx + 1 :] # includes the leading '/' + if not location: + raise HfUriError(uri=raw, msg="Missing location before mount path.") + return location, mount_path, read_only + + +def _split_type(location: str, *, raw: str) -> tuple[constants.HfUriType, str]: + """Detect the (optional) type prefix and return '(type, remaining_location)'. + + A missing type prefix defaults to 'model'. Singular forms ('model/', 'dataset/', etc.) are explicitly rejected with a helpful error. + """ + slash_idx = location.find("/") + if slash_idx == -1: + # Single segment, no prefix. Reject if it looks like a bare type name. + if location in constants.HF_URI_TYPE_PREFIXES: + raise HfUriError( + uri=raw, + msg=f"Missing identifier after '{location}'. Expected '{constants.HF_PROTOCOL}{location}/'.", + ) + if (singular_plural := _TYPE_TO_PREFIX.get(location)) is not None: + raise HfUriError( + uri=raw, + msg=f"Type prefix must be plural. Did you mean '{constants.HF_PROTOCOL}{singular_plural}/...'?", + ) + return "model", location + + first = location[:slash_idx] + rest = location[slash_idx + 1 :] + if first in constants.HF_URI_TYPE_PREFIXES: + return constants.HF_URI_TYPE_PREFIXES[first], rest + if (singular_plural := _TYPE_TO_PREFIX.get(first)) is not None: + raise HfUriError( + uri=raw, msg=f"Type prefix must be plural, got '{first}/'. Did you mean '{singular_plural}/'?" + ) + return "model", location + + +def _parse_bucket_body( + location: str, + type_: constants.HfUriType, + *, + raw: str, +) -> HfUri: + """Parse the body of a bucket URI: 'namespace/name[/path]'.""" + location = location.strip("/") + parts = location.split("/", 2) + if len(parts) < 2 or not parts[0] or not parts[1]: + raise HfUriError(uri=raw, msg=f"Bucket id must be 'namespace/name', got '{location}'.") + bucket_id = f"{parts[0]}/{parts[1]}" + if "@" in bucket_id: + raise HfUriError(uri=raw, msg="Bucket URIs do not support a revision marker ('@').") + path_in_bucket = parts[2] if len(parts) >= 3 else "" + return HfUri( + type=type_, + id=bucket_id, + revision=None, + path_in_repo=path_in_bucket, + _raw=raw, + ) + + +def _parse_repo_body( + location: str, + type_: constants.HfUriType, + *, + raw: str, +) -> HfUri: + """Parse the body of a repo URI: '[@][/]'.""" + location = location.strip("/") + if not location: + raise HfUriError(uri=raw, msg="Missing repository id.") + + # The '@' separates the repo_id from the revision, but only when it + # appears right after 'namespace/name' (at most one '/' before it). + # An '@' deeper in the path (e.g. in a filename like 'file@1.txt') is literal. + at_idx = location.find("@") + revision: str | None + + if at_idx == -1 or location[:at_idx].count("/") > 1: + # No '@' at all, or the '@' is past the repo_id portion (in a filename). + revision = None + parts = location.split("/", 2) + if len(parts) < 2: + raise HfUriError(uri=raw, msg=f"Repository id must be 'namespace/name', got '{location}'. ") + repo_id = f"{parts[0]}/{parts[1]}" + path_in_repo = parts[2] if len(parts) > 2 else "" + else: + repo_id = location[:at_idx] + rev_and_path = location[at_idx + 1 :] + if not repo_id: + raise HfUriError(uri=raw, msg="Missing repository id before '@'.") + if repo_id.count("/") != 1: + raise HfUriError(uri=raw, msg=f"Repository id must be 'namespace/name', got '{repo_id}'.") + # Special refs like 'refs/pr/10' contain '/' and must be matched eagerly, + # otherwise we would split them at the first '/' and treat the rest as a path. + match = _SPECIAL_REFS_REVISION_REGEX.match(rev_and_path) + if match is not None: + revision = match.group() + path_in_repo = rev_and_path[len(revision) :].removeprefix("/") + else: + slash_idx = rev_and_path.find("/") + if slash_idx == -1: + revision = rev_and_path + path_in_repo = "" + else: + revision = rev_and_path[:slash_idx] + path_in_repo = rev_and_path[slash_idx + 1 :] + revision = unquote(revision) + if not revision: + raise HfUriError(uri=raw, msg="Empty revision after '@'.") + + return HfUri( + type=type_, + id=repo_id, + revision=revision, + path_in_repo=path_in_repo, + _raw=raw, + ) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_http.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_http.py new file mode 100644 index 0000000000000000000000000000000000000000..0251ec82dd91e8e50b70ed18e728999c7202ebf9 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_http.py @@ -0,0 +1,1163 @@ +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to handle HTTP requests in huggingface_hub.""" + +import atexit +import io +import json +import os +import re +import threading +import time +import uuid +from collections.abc import Callable, Generator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from shlex import quote +from typing import Any, TypeVar +from urllib.parse import urlparse + +import httpx + +from huggingface_hub.errors import OfflineModeIsEnabled + +from .. import constants +from ..errors import ( + BadRequestError, + BucketNotFoundError, + DisabledRepoError, + GatedRepoError, + HfHubHTTPError, + JobNotFoundError, + RemoteEntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, +) +from . import logging +from ._lfs import SliceFileObj +from ._typing import HTTP_METHOD_T + + +logger = logging.get_logger(__name__) + + +@dataclass(frozen=True) +class RateLimitInfo: + """ + Parsed rate limit information from HTTP response headers. + + Attributes: + resource_type (`str`): The type of resource being rate limited. + remaining (`int`): The number of requests remaining in the current window. + reset_in_seconds (`int`): The number of seconds until the rate limit resets. + limit (`int`, *optional*): The maximum number of requests allowed in the current window. + window_seconds (`int`, *optional*): The number of seconds in the current window. + + """ + + resource_type: str + remaining: int + reset_in_seconds: int + limit: int | None = None + window_seconds: int | None = None + + +# Regex patterns for parsing rate limit headers +# e.g.: "api";r=0;t=55 --> resource_type="api", r=0, t=55 +_RATELIMIT_REGEX = re.compile(r"\"(?P\w+)\"\s*;\s*r\s*=\s*(?P\d+)\s*;\s*t\s*=\s*(?P\d+)") +# e.g.: "fixed window";"api";q=500;w=300 --> q=500, w=300 +_RATELIMIT_POLICY_REGEX = re.compile(r"q\s*=\s*(?P\d+).*?w\s*=\s*(?P\d+)") + + +def parse_ratelimit_headers(headers: Mapping[str, str]) -> RateLimitInfo | None: + """Parse rate limit information from HTTP response headers. + + Follows IETF draft: https://www.ietf.org/archive/id/draft-ietf-httpapi-ratelimit-headers-09.html + Only a subset is implemented. + + Example: + ```python + >>> from huggingface_hub.utils import parse_ratelimit_headers + >>> headers = { + ... "ratelimit": '"api";r=0;t=55', + ... "ratelimit-policy": '"fixed window";"api";q=500;w=300', + ... } + >>> info = parse_ratelimit_headers(headers) + >>> info.remaining + 0 + >>> info.reset_in_seconds + 55 + ``` + """ + + ratelimit: str | None = None + policy: str | None = None + for key in headers: + lower_key = key.lower() + if lower_key == "ratelimit": + ratelimit = headers[key] + elif lower_key == "ratelimit-policy": + policy = headers[key] + + if not ratelimit: + return None + + match = _RATELIMIT_REGEX.search(ratelimit) + if not match: + return None + + resource_type = match.group("resource_type") + remaining = int(match.group("r")) + reset_in_seconds = int(match.group("t")) + + limit: int | None = None + window_seconds: int | None = None + + if policy: + policy_match = _RATELIMIT_POLICY_REGEX.search(policy) + if policy_match: + limit = int(policy_match.group("q")) + window_seconds = int(policy_match.group("w")) + + return RateLimitInfo( + resource_type=resource_type, + remaining=remaining, + reset_in_seconds=reset_in_seconds, + limit=limit, + window_seconds=window_seconds, + ) + + +def _parse_retry_after(headers: Mapping[str, str]) -> int | None: + """Parse the standard `Retry-After` HTTP header into a number of seconds to wait. + + The `Retry-After` header can be either a non-negative number of seconds (delay-seconds) + or an HTTP-date after which to retry. We handle only the delay-seconds case. + + See https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After. + """ + value: str | None = None + for key in headers: + if key.lower() == "retry-after": + value = headers[key] + break + + if value is None: + return None + value = value.strip() + if not value: + return None + + if value.isdigit(): + return int(value) # e.g. "Retry-After: 120" + return None # e.g. "Retry-After: Wed, 21 Oct 2015 07:28:00 GMT" - not supported + + +# When raising an error, we include the request id in the error message for easier debugging. +# Request ID is sourced from headers in order of precedence: "X-Request-Id", "X-Amzn-Trace-Id", "X-Amz-Cf-Id". +X_REQUEST_ID = "x-request-id" +X_AMZN_TRACE_ID = "X-Amzn-Trace-Id" +X_AMZ_CF_ID = "x-amz-cf-id" + +REPO_API_REGEX = re.compile( + r""" + # staging or production endpoint + ^https://[^/]+ + ( + # on /api/repo_type/repo_id + /api/(models|datasets|spaces)/(.+) + | + # or /repo_id/resolve/revision/... + /(.+)/resolve/(.+) + ) + """, + flags=re.VERBOSE, +) + +BUCKET_API_REGEX = re.compile( + r""" + # staging or production endpoint + ^https?://[^/]+ + # on /api/buckets/... + /api/buckets/ + """, + flags=re.VERBOSE, +) + +# Regex to extract the job_id from a (scheduled) job API URL. +# Matches /api/jobs/{namespace}/{job_id}[/...] and /api/scheduled-jobs/{namespace}/{job_id}[/...]. +_JOB_ID_FROM_URL_REGEX = re.compile(r"^https?://[^/]+/api/(?:scheduled-jobs|jobs)/[^/]+/([^/?]+)") + +# Regex to extract repo_type and repo_id from API URLs. +# Captures: group(1) = repo_type plural (models/datasets/spaces), group(2) = first path segment, group(3) = optional second segment. +_REPO_ID_FROM_URL_REGEX = re.compile(r"^https?://[^/]+/api/(models|datasets|spaces)/([^/]+)(?:/([^/]+))?") + +# Regex to extract bucket_id (namespace/name) from bucket API URLs. +_BUCKET_ID_FROM_URL_REGEX = re.compile(r"^https?://[^/]+/api/buckets/([^/]+/[^/]+)") + +# Sub-paths that follow a repo_id in API URLs (not part of the repo name). +_REPO_URL_SUBPATHS = {"resolve", "tree", "blob", "raw", "refs", "commit", "discussions", "settings", "revision"} + + +def _parse_repo_info_from_url(url: str) -> tuple[str | None, str | None]: + """Extract (repo_type, repo_id) from an API URL. + + Returns canonical repo_type values: "model", "dataset", "space" (or None). + + Examples: + >>> _parse_repo_info_from_url("https://huggingface.co/api/models/user/repo") + ("model", "user/repo") + >>> _parse_repo_info_from_url("https://huggingface.co/api/datasets/user/repo/resolve/main/data.csv") + ("dataset", "user/repo") + >>> _parse_repo_info_from_url("https://huggingface.co/api/models/bert-base-cased/resolve/main/config.json") + ("model", "bert-base-cased") + """ + match = _REPO_ID_FROM_URL_REGEX.search(url) + if not match: + return None, None + repo_type = constants.REPO_TYPES_MAPPING.get(match.group(1)) + first, second = match.group(2), match.group(3) + if second and second not in _REPO_URL_SUBPATHS: + repo_id = f"{first}/{second}" + else: + repo_id = first + return repo_type, repo_id + + +def _parse_bucket_id_from_url(url: str) -> str | None: + """Extract bucket_id (namespace/name) from a bucket API URL.""" + match = _BUCKET_ID_FROM_URL_REGEX.search(url) + return match.group(1) if match else None + + +def _parse_job_id_from_url(url: str) -> str | None: + """Extract the job_id from a (scheduled) job API URL, if present.""" + match = _JOB_ID_FROM_URL_REGEX.search(url) + return match.group(1) if match else None + + +def hf_request_event_hook(request: httpx.Request) -> None: + """ + Event hook that will be used to make HTTP requests to the Hugging Face Hub. + + What it does: + - Block requests if offline mode is enabled + - Add a request ID to the request headers + - Log the request if debug mode is enabled + """ + if constants.is_offline_mode(): + raise OfflineModeIsEnabled( + f"Cannot reach {request.url}: offline mode is enabled. To disable it, please unset the `HF_HUB_OFFLINE` environment variable." + ) + + # Add random request ID => easier for server-side debugging + if X_AMZN_TRACE_ID not in request.headers: + request.headers[X_AMZN_TRACE_ID] = request.headers.get(X_REQUEST_ID) or str(uuid.uuid4()) + request_id = request.headers.get(X_AMZN_TRACE_ID) + + # Debug log + logger.debug( + "Request %s: %s %s (authenticated: %s)", + request_id, + request.method, + request.url, + request.headers.get("authorization") is not None, + ) + if constants.HF_DEBUG: + logger.debug("Send: %s", _curlify(request)) + + return request_id + + +async def async_hf_request_event_hook(request: httpx.Request) -> None: + """ + Async version of `hf_request_event_hook`. + """ + return hf_request_event_hook(request) + + +async def async_hf_response_event_hook(response: httpx.Response) -> None: + if response.status_code >= 400: + # If response will raise, read content from stream to have it available when raising the exception + # If content-length is not set or is too large, skip reading the content to avoid OOM + if "Content-length" in response.headers: + try: + length = int(response.headers["Content-length"]) + except ValueError: + return + + if length < 1_000_000: + await response.aread() + + +def default_client_factory() -> httpx.Client: + """ + Factory function to create a `httpx.Client` with the default transport. + """ + return httpx.Client( + event_hooks={"request": [hf_request_event_hook]}, + follow_redirects=True, + timeout=None, + ) + + +def default_async_client_factory() -> httpx.AsyncClient: + """ + Factory function to create a `httpx.AsyncClient` with the default transport. + """ + return httpx.AsyncClient( + event_hooks={"request": [async_hf_request_event_hook], "response": [async_hf_response_event_hook]}, + follow_redirects=True, + timeout=None, + ) + + +CLIENT_FACTORY_T = Callable[[], httpx.Client] +ASYNC_CLIENT_FACTORY_T = Callable[[], httpx.AsyncClient] + +_CLIENT_LOCK = threading.Lock() +_GLOBAL_CLIENT_FACTORY: CLIENT_FACTORY_T = default_client_factory +_GLOBAL_ASYNC_CLIENT_FACTORY: ASYNC_CLIENT_FACTORY_T = default_async_client_factory +_GLOBAL_CLIENT: httpx.Client | None = None + + +def set_client_factory(client_factory: CLIENT_FACTORY_T) -> None: + """ + Set the HTTP client factory to be used by `huggingface_hub`. + + The client factory is a method that returns a `httpx.Client` object. On the first call to [`get_session`] the client factory + will be used to create a new `httpx.Client` object that will be shared between all calls made by `huggingface_hub`. + + This can be useful if you are running your scripts in a specific environment requiring custom configuration (e.g. custom proxy or certifications). + + Use [`get_session`] to get a correctly configured `httpx.Client`. + """ + global _GLOBAL_CLIENT_FACTORY + with _CLIENT_LOCK: + close_session() + _GLOBAL_CLIENT_FACTORY = client_factory + + +def set_async_client_factory(async_client_factory: ASYNC_CLIENT_FACTORY_T) -> None: + """ + Set the HTTP async client factory to be used by `huggingface_hub`. + + The async client factory is a method that returns a `httpx.AsyncClient` object. + This can be useful if you are running your scripts in a specific environment requiring custom configuration (e.g. custom proxy or certifications). + Use [`get_async_client`] to get a correctly configured `httpx.AsyncClient`. + + > [!WARNING] + > Contrary to the `httpx.Client` that is shared between all calls made by `huggingface_hub`, the `httpx.AsyncClient` is not shared. + > It is recommended to use an async context manager to ensure the client is properly closed when the context is exited. + """ + global _GLOBAL_ASYNC_CLIENT_FACTORY + _GLOBAL_ASYNC_CLIENT_FACTORY = async_client_factory + + +def get_session() -> httpx.Client: + """ + Get a `httpx.Client` object, using the transport factory from the user. + + This client is shared between all calls made by `huggingface_hub`. Therefore you should not close it manually. + + Use [`set_client_factory`] to customize the `httpx.Client`. + """ + global _GLOBAL_CLIENT + if _GLOBAL_CLIENT is None: + with _CLIENT_LOCK: + _GLOBAL_CLIENT = _GLOBAL_CLIENT_FACTORY() + return _GLOBAL_CLIENT + + +def get_async_session() -> httpx.AsyncClient: + """ + Return a `httpx.AsyncClient` object, using the transport factory from the user. + + Use [`set_async_client_factory`] to customize the `httpx.AsyncClient`. + + > [!WARNING] + > Contrary to the `httpx.Client` that is shared between all calls made by `huggingface_hub`, the `httpx.AsyncClient` is not shared. + > It is recommended to use an async context manager to ensure the client is properly closed when the context is exited. + """ + return _GLOBAL_ASYNC_CLIENT_FACTORY() + + +def close_session() -> None: + """ + Close the global `httpx.Client` used by `huggingface_hub`. + + If a Client is closed, it will be recreated on the next call to [`get_session`]. + + Can be useful if e.g. an SSL certificate has been updated. + """ + global _GLOBAL_CLIENT + client = _GLOBAL_CLIENT + + # First, set global client to None + _GLOBAL_CLIENT = None + + # Then, close the clients + if client is not None: + try: + client.close() + except Exception as e: + logger.warning(f"Error closing client: {e}") + + +atexit.register(close_session) +if hasattr(os, "register_at_fork"): + os.register_at_fork(after_in_child=close_session) + + +_DEFAULT_RETRY_ON_EXCEPTIONS: tuple[type[Exception], ...] = ( + httpx.TimeoutException, + httpx.NetworkError, + httpx.RemoteProtocolError, +) +_DEFAULT_RETRY_ON_STATUS_CODES: tuple[int, ...] = (408, 429, 500, 502, 503, 504) + + +def _http_backoff_base( + method: HTTP_METHOD_T, + url: str, + *, + max_retries: int = 5, + base_wait_time: float = 1, + max_wait_time: float = 8, + retry_on_exceptions: type[Exception] | tuple[type[Exception], ...] = _DEFAULT_RETRY_ON_EXCEPTIONS, + retry_on_status_codes: int | tuple[int, ...] = _DEFAULT_RETRY_ON_STATUS_CODES, + stream: bool = False, + **kwargs, +) -> Generator[httpx.Response, None, None]: + """Internal implementation of HTTP backoff logic shared between `http_backoff` and `http_stream_backoff`.""" + if isinstance(retry_on_exceptions, type): # Tuple from single exception type + retry_on_exceptions = (retry_on_exceptions,) + + if isinstance(retry_on_status_codes, int): # Tuple from single status code + retry_on_status_codes = (retry_on_status_codes,) + + nb_tries = 0 + sleep_time = base_wait_time + ratelimit_reset: int | None = None # seconds to wait for rate limit reset if 429 response + + # If `data` is used and is a file object (or any IO), it will be consumed on the + # first HTTP request. We need to save the initial position so that the full content + # of the file is re-sent on http backoff. See warning tip in docstring. + io_obj_initial_pos = None + if "data" in kwargs and isinstance(kwargs["data"], (io.IOBase, SliceFileObj)): + io_obj_initial_pos = kwargs["data"].tell() + + while True: + nb_tries += 1 + ratelimit_reset = None + # Fetched on each attempt: a previous attempt may have closed the shared client (see `close_session` below), + # and closed `httpx.Client` objects cannot be reused. + client = get_session() + try: + # If `data` is used and is a file object (or any IO), set back cursor to + # initial position. + if io_obj_initial_pos is not None: + kwargs["data"].seek(io_obj_initial_pos) + + # Perform request and handle response + def _should_retry(response: httpx.Response) -> bool: + """Handle response and return True if should retry, False if should return/yield.""" + nonlocal ratelimit_reset + + if response.status_code not in retry_on_status_codes: + return False # Success, don't retry + + # Wrong status code returned (HTTP 503 for instance) + logger.warning(f"HTTP Error {response.status_code} thrown while requesting {method} {url}") + if nb_tries > max_retries: + hf_raise_for_status(response) # Will raise uncaught exception + # Return/yield response to avoid infinite loop in the corner case where the + # user ask for retry on a status code that doesn't raise_for_status. + return False # Don't retry, return/yield response + + # Check 'ratelimit' and `Retry-After` headers. + if ( + response.status_code == 429 + and (ratelimit_info := parse_ratelimit_headers(response.headers)) is not None + ): + ratelimit_reset = ratelimit_info.reset_in_seconds + elif (retry_after := _parse_retry_after(response.headers)) is not None: + ratelimit_reset = retry_after + + return True # Should retry + + if stream: + with client.stream(method=method, url=url, **kwargs) as response: + if not _should_retry(response): + yield response + return + else: + response = client.request(method=method, url=url, **kwargs) + if not _should_retry(response): + yield response + return + + except retry_on_exceptions as err: + logger.warning(f"'{err}' thrown while requesting {method} {url}") + + if isinstance(err, httpx.ConnectError): + close_session() # In case of SSLError it's best to close the shared httpx.Client objects + + if nb_tries > max_retries: + raise err + + if ratelimit_reset is not None: + actual_sleep = float(ratelimit_reset) + 1 # +1s to avoid rounding issues + logger.warning(f"Rate limited. Waiting {actual_sleep}s before retry [Retry {nb_tries}/{max_retries}].") + else: + actual_sleep = sleep_time + logger.warning(f"Retrying in {actual_sleep}s [Retry {nb_tries}/{max_retries}].") + + time.sleep(actual_sleep) + + # Update sleep time for next retry + sleep_time = min(max_wait_time, sleep_time * 2) # Exponential backoff + + +def http_backoff( + method: HTTP_METHOD_T, + url: str, + *, + max_retries: int = 5, + base_wait_time: float = 1, + max_wait_time: float = 8, + retry_on_exceptions: type[Exception] | tuple[type[Exception], ...] = _DEFAULT_RETRY_ON_EXCEPTIONS, + retry_on_status_codes: int | tuple[int, ...] = _DEFAULT_RETRY_ON_STATUS_CODES, + **kwargs, +) -> httpx.Response: + """Wrapper around httpx to retry calls on an endpoint, with exponential backoff. + + Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...) + and/or on specific status codes (ex: service unavailable). If the call failed more + than `max_retries`, the exception is thrown or `raise_for_status` is called on the + response object. + + Re-implement mechanisms from the `backoff` library to avoid adding an external + dependencies to `hugging_face_hub`. See https://github.com/litl/backoff. + + Args: + method (`Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]`): + HTTP method to perform. + url (`str`): + The URL of the resource to fetch. + max_retries (`int`, *optional*, defaults to `5`): + Maximum number of retries, defaults to 5. Set to `0` to disable retries. + base_wait_time (`float`, *optional*, defaults to `1`): + Duration (in seconds) to wait before retrying the first time. + Wait time between retries then grows exponentially, capped by + `max_wait_time`. + max_wait_time (`float`, *optional*, defaults to `8`): + Maximum duration (in seconds) to wait before retrying. + retry_on_exceptions (`type[Exception]` or `tuple[type[Exception]]`, *optional*): + Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types. + By default, retry on `httpx.TimeoutException`, `httpx.NetworkError` and `httpx.RemoteProtocolError`. + retry_on_status_codes (`int` or `tuple[int]`, *optional*, defaults to `(429, 500, 502, 503, 504)`): + Define on which status codes the request must be retried. By default, retries + on rate limit (429) and server errors (5xx). + **kwargs (`dict`, *optional*): + kwargs to pass to `httpx.request`. + + Example: + ``` + >>> from huggingface_hub.utils import http_backoff + + # Same usage as "httpx.request". + >>> response = http_backoff("GET", "https://www.google.com") + >>> response.raise_for_status() + + # If you expect a Gateway Timeout from time to time + >>> http_backoff("PUT", upload_url, data=data, retry_on_status_codes=504) + >>> response.raise_for_status() + ``` + + > [!WARNING] + > When using `requests` it is possible to stream data by passing an iterator to the + > `data` argument. On http backoff this is a problem as the iterator is not reset + > after a failed call. This issue is mitigated for file objects or any IO streams + > by saving the initial position of the cursor (with `data.tell()`) and resetting the + > cursor between each call (with `data.seek()`). For arbitrary iterators, http backoff + > will fail. If this is a hard constraint for you, please let us know by opening an + > issue on [Github](https://github.com/huggingface/huggingface_hub). + """ + return next( + _http_backoff_base( + method=method, + url=url, + max_retries=max_retries, + base_wait_time=base_wait_time, + max_wait_time=max_wait_time, + retry_on_exceptions=retry_on_exceptions, + retry_on_status_codes=retry_on_status_codes, + stream=False, + **kwargs, + ) + ) + + +@contextmanager +def http_stream_backoff( + method: HTTP_METHOD_T, + url: str, + *, + max_retries: int = 5, + base_wait_time: float = 1, + max_wait_time: float = 8, + retry_on_exceptions: type[Exception] | tuple[type[Exception], ...] = _DEFAULT_RETRY_ON_EXCEPTIONS, + retry_on_status_codes: int | tuple[int, ...] = _DEFAULT_RETRY_ON_STATUS_CODES, + **kwargs, +) -> Generator[httpx.Response, None, None]: + """Wrapper around httpx to retry calls on an endpoint, with exponential backoff. + + Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...) + and/or on specific status codes (ex: service unavailable). If the call failed more + than `max_retries`, the exception is thrown or `raise_for_status` is called on the + response object. + + Re-implement mechanisms from the `backoff` library to avoid adding an external + dependencies to `hugging_face_hub`. See https://github.com/litl/backoff. + + Args: + method (`Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]`): + HTTP method to perform. + url (`str`): + The URL of the resource to fetch. + max_retries (`int`, *optional*, defaults to `5`): + Maximum number of retries, defaults to 5. Set to `0` to disable retries. + base_wait_time (`float`, *optional*, defaults to `1`): + Duration (in seconds) to wait before retrying the first time. + Wait time between retries then grows exponentially, capped by + `max_wait_time`. + max_wait_time (`float`, *optional*, defaults to `8`): + Maximum duration (in seconds) to wait before retrying. + retry_on_exceptions (`type[Exception]` or `tuple[type[Exception]]`, *optional*): + Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types. + By default, retry on `httpx.TimeoutException`, `httpx.NetworkError` and `httpx.RemoteProtocolError`. + retry_on_status_codes (`int` or `tuple[int]`, *optional*, defaults to `(429, 500, 502, 503, 504)`): + Define on which status codes the request must be retried. By default, retries + on rate limit (429) and server errors (5xx). + **kwargs (`dict`, *optional*): + kwargs to pass to `httpx.request`. + + Example: + ``` + >>> from huggingface_hub.utils import http_stream_backoff + + # Same usage as "httpx.stream". + >>> with http_stream_backoff("GET", "https://www.google.com") as response: + ... for chunk in response.iter_bytes(): + ... print(chunk) + + # If you expect a Gateway Timeout from time to time + >>> with http_stream_backoff("PUT", upload_url, data=data, retry_on_status_codes=504) as response: + ... response.raise_for_status() + ``` + + > [!WARNING] + > When using `httpx` it is possible to stream data by passing an iterator to the + > `data` argument. On http backoff this is a problem as the iterator is not reset + > after a failed call. This issue is mitigated for file objects or any IO streams + > by saving the initial position of the cursor (with `data.tell()`) and resetting the + > cursor between each call (with `data.seek()`). For arbitrary iterators, http backoff + > will fail. If this is a hard constraint for you, please let us know by opening an + > issue on [Github](https://github.com/huggingface/huggingface_hub). + """ + yield from _http_backoff_base( + method=method, + url=url, + max_retries=max_retries, + base_wait_time=base_wait_time, + max_wait_time=max_wait_time, + retry_on_exceptions=retry_on_exceptions, + retry_on_status_codes=retry_on_status_codes, + stream=True, + **kwargs, + ) + + +def _httpx_follow_relative_redirects_with_backoff( + method: HTTP_METHOD_T, url: str, *, retry_on_errors: bool = False, **httpx_kwargs +) -> httpx.Response: + """Perform an HTTP request with backoff and follow relative redirects only. + + Used to fetch HEAD /resolve on repo or bucket files. + + This is useful to follow a redirection to a renamed repository without following redirection to a CDN. + + A backoff mechanism retries the HTTP call on errors (429, 5xx, timeout, network errors). + + Args: + method (`str`): + HTTP method, such as 'GET' or 'HEAD'. + url (`str`): + The URL of the resource to fetch. + retry_on_errors (`bool`, *optional*, defaults to `False`): + Whether to retry on errors. If False, no retry is performed (fast fallback to local cache). + If True, uses default retry behavior (429, 5xx, timeout, network errors). + **httpx_kwargs (`dict`, *optional*): + Params to pass to `httpx.request`. + """ + # if `retry_on_errors=False`, disable all retries for fast fallback to cache + no_retry_kwargs: dict[str, Any] = ( + {} if retry_on_errors else {"retry_on_exceptions": (), "retry_on_status_codes": ()} + ) + + while True: + response = http_backoff( + method=method, + url=url, + **httpx_kwargs, + follow_redirects=False, + **no_retry_kwargs, + ) + hf_raise_for_status(response) + + # Check if response is a relative redirect + if 300 <= response.status_code <= 399: + parsed_target = urlparse(response.headers["Location"]) + if parsed_target.netloc == "": + # Relative redirect -> update URL and retry + url = urlparse(url)._replace(path=parsed_target.path).geturl() + continue + + # Break if no relative redirect + break + + return response + + +def fix_hf_endpoint_in_url(url: str, endpoint: str | None) -> str: + """Replace the default endpoint in a URL by a custom one. + + This is useful when using a proxy and the Hugging Face Hub returns a URL with the default endpoint. + """ + endpoint = endpoint.rstrip("/") if endpoint else constants.ENDPOINT + # check if a proxy has been set => if yes, update the returned URL to use the proxy + if endpoint not in (constants._HF_DEFAULT_ENDPOINT, constants._HF_DEFAULT_STAGING_ENDPOINT): + url = url.replace(constants._HF_DEFAULT_ENDPOINT, endpoint) + url = url.replace(constants._HF_DEFAULT_STAGING_ENDPOINT, endpoint) + return url + + +def hf_raise_for_status(response: httpx.Response, endpoint_name: str | None = None) -> None: + """ + Internal version of `response.raise_for_status()` that will refine a potential HTTPError. + Raised exception will be an instance of [`~errors.HfHubHTTPError`]. + + This helper is meant to be the unique method to raise_for_status when making a call to the Hugging Face Hub. + + Args: + response (`Response`): + Response from the server. + endpoint_name (`str`, *optional*): + Name of the endpoint that has been called. If provided, the error message will be more complete. + + > [!WARNING] + > Raises when the request has failed: + > + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it + > doesn't exist, because `repo_type` is not set correctly, or because the repo + > is `private` and you do not have access. + > - [`~utils.GatedRepoError`] + > If the repository exists but is gated and the user is not on the authorized + > list. + > - [`~utils.RevisionNotFoundError`] + > If the repository exists but the revision couldn't be found. + > - [`~utils.EntryNotFoundError`] + > If the repository exists but the entry (e.g. the requested file) couldn't be + > find. + > - [`~utils.BadRequestError`] + > If request failed with a HTTP 400 BadRequest error. + > - [`~utils.HfHubHTTPError`] + > If request failed for a reason not listed above. + """ + try: + _warn_on_warning_headers(response) + except Exception: + # Never raise on warning parsing + logger.debug("Failed to parse warning headers", exc_info=True) + + try: + response.raise_for_status() + except httpx.HTTPStatusError as e: + if response.status_code // 100 == 3: + return # Do not raise on redirects to stay consistent with `requests` + + error_code = response.headers.get("X-Error-Code") + error_message = response.headers.get("X-Error-Message") + + # Parse repo info from request URL (used to enrich errors below) + request_url = ( + str(response.request.url) if response.request is not None and response.request.url is not None else None + ) + repo_type, repo_id = _parse_repo_info_from_url(request_url) if request_url else (None, None) + + if error_code == "RevisionNotFound": + message = f"{response.status_code} Client Error." + "\n\n" + f"Revision Not Found for url: {response.url}." + raise _format(RevisionNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e + + elif error_code == "EntryNotFound": + message = f"{response.status_code} Client Error." + "\n\n" + f"Entry Not Found for url: {response.url}." + raise _format(RemoteEntryNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e + + elif error_code == "GatedRepo": + message = ( + f"{response.status_code} Client Error." + "\n\n" + f"Cannot access gated repo for url {response.url}." + ) + raise _format(GatedRepoError, message, response, repo_type=repo_type, repo_id=repo_id) from e + + elif error_message == "Access to this resource is disabled.": + message = ( + f"{response.status_code} Client Error." + + "\n\n" + + f"Cannot access repository for url {response.url}." + + "\n" + + "Access to this resource is disabled." + ) + raise _format(DisabledRepoError, message, response) from e + + elif ( + error_code == "RepoNotFound" + and request_url is not None + and BUCKET_API_REGEX.search(request_url) is not None + ): + message = ( + f"{response.status_code} Client Error." + + "\n\n" + + f"Bucket Not Found for url: {response.url}." + + "\nPlease make sure you specified the correct bucket id (namespace/name)." + + "\nIf the bucket is private, make sure you are authenticated and your token has the required permissions." + ) + raise _format( + BucketNotFoundError, message, response, bucket_id=_parse_bucket_id_from_url(request_url) + ) from e + + elif ( + response.status_code == 404 + and request_url is not None + and (job_id := _parse_job_id_from_url(request_url)) is not None + ): + message = ( + f"{response.status_code} Client Error." + + "\n\n" + + f"Job Not Found for url: {response.url}." + + "\nPlease make sure you specified the correct job ID and namespace." + ) + raise _format(JobNotFoundError, message, response, job_id=job_id) from e + + elif error_code == "RepoNotFound" or ( + response.status_code == 401 + and error_message != "Invalid credentials in Authorization header" + and request_url is not None + and REPO_API_REGEX.search(request_url) is not None + ): + # 401 is misleading as it is returned for: + # - private and gated repos if user is not authenticated + # - missing repos + # => for now, we process them as `RepoNotFound` anyway. + # See https://gist.github.com/Wauplin/46c27ad266b15998ce56a6603796f0b9 + message = ( + f"{response.status_code} Client Error." + + "\n\n" + + f"Repository Not Found for url: {response.url}." + + "\nPlease make sure you specified the correct `repo_id` and" + " `repo_type`.\nIf you are trying to access a private or gated repo," + " make sure you are authenticated and your token has the required permissions." + + "\nFor more details, see https://huggingface.co/docs/huggingface_hub/authentication" + ) + raise _format(RepositoryNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e + + elif response.status_code == 400: + message = ( + f"\n\nBad request for {endpoint_name} endpoint:" if endpoint_name is not None else "\n\nBad request:" + ) + raise _format(BadRequestError, message, response) from e + + elif response.status_code == 403: + message = ( + f"\n\n{response.status_code} Forbidden: {error_message}." + + f"\nCannot access content at: {response.url}." + + "\nMake sure your token has the correct permissions." + ) + raise _format(HfHubHTTPError, message, response) from e + + elif response.status_code == 429: + ratelimit_info = parse_ratelimit_headers(response.headers) + if ratelimit_info is not None: + message = ( + f"\n\n429 Too Many Requests: you have reached your '{ratelimit_info.resource_type}' rate limit." + ) + message += f"\nRetry after {ratelimit_info.reset_in_seconds} seconds" + if ratelimit_info.limit is not None and ratelimit_info.window_seconds is not None: + message += ( + f" ({ratelimit_info.remaining}/{ratelimit_info.limit} requests remaining" + f" in current {ratelimit_info.window_seconds}s window)." + ) + else: + message += "." + message += f"\nUrl: {response.url}." + else: + message = f"\n\n429 Too Many Requests for url: {response.url}." + raise _format(HfHubHTTPError, message, response) from e + + elif response.status_code == 416: + range_header = response.request.headers.get("Range") + message = f"{e}. Requested range: {range_header}. Content-Range: {response.headers.get('Content-Range')}." + raise _format(HfHubHTTPError, message, response) from e + + # Convert `HTTPError` into a `HfHubHTTPError` to display request information + # as well (request id and/or server error message) + raise _format(HfHubHTTPError, str(e), response) from e + + +_WARNED_TOPICS = set() + + +def _warn_on_warning_headers(response: httpx.Response) -> None: + """ + Emit warnings if warning headers are present in the HTTP response. + + Expected header format: 'X-HF-Warning: topic; message' + + Only the first warning for each topic will be shown. Topic is optional and can be empty. Note that several warning + headers can be present in a single response. + + Args: + response (`httpx.Response`): + The HTTP response to check for warning headers. + """ + server_warnings = response.headers.get_list("X-HF-Warning") + for server_warning in server_warnings: + topic, message = server_warning.split(";", 1) if ";" in server_warning else ("", server_warning) + topic = topic.strip() + if topic not in _WARNED_TOPICS: + message = message.strip() + if message: + _WARNED_TOPICS.add(topic) + logger.warning(message) + + +_HfHubHTTPErrorT = TypeVar("_HfHubHTTPErrorT", bound=HfHubHTTPError) + + +def _format( + error_type: type[_HfHubHTTPErrorT], custom_message: str, response: httpx.Response, **attrs: Any +) -> _HfHubHTTPErrorT: + server_errors = [] + + # Retrieve server error from header + from_headers = response.headers.get("X-Error-Message") + if from_headers is not None: + server_errors.append(from_headers) + + # Retrieve server error from body + try: + # Case errors are returned in a JSON format + try: + data = response.json() + except httpx.ResponseNotRead: + try: + response.read() # In case of streaming response, we need to read the response first + data = response.json() + except RuntimeError: + # In case of async streaming response, we can't read the stream here. + # In practice if user is using the default async client from `get_async_client`, the stream will have + # already been read in the async event hook `async_hf_response_event_hook`. + # + # Here, we are skipping reading the response to avoid RuntimeError but it happens only if async + stream + used httpx.AsyncClient directly. + data = {} + + error = data.get("error") + error_description = data.get("error_description") + if error is not None: + if isinstance(error, list): + # Case {'error': ['my error 1', 'my error 2']} + server_errors.extend(error) + elif error_description is not None: + # OAuth-style case {'error': 'invalid_grant', 'error_description': 'my description'} + server_errors.append(f"{error}: {error_description}") + else: + # Case {'error': 'my error'} + server_errors.append(error) + elif error_description is not None: + # Case {'error_description': 'my description'} (no 'error' field) + server_errors.append(error_description) + + errors = data.get("errors") + if errors is not None: + # Case {'errors': [{'message': 'my error 1'}, {'message': 'my error 2'}]} + for error in errors: + if "message" in error: + server_errors.append(error["message"]) + + except json.JSONDecodeError: + # If content is not JSON and not HTML, append the text + content_type = response.headers.get("Content-Type", "") + if response.text and "html" not in content_type.lower(): + server_errors.append(response.text) + + # Strip all server messages + server_errors = [str(line).strip() for line in server_errors if str(line).strip()] + + # Deduplicate server messages (keep order) + # taken from https://stackoverflow.com/a/17016257 + server_errors = list(dict.fromkeys(server_errors)) + + # Format server error + server_message = "\n".join(server_errors) + + # Add server error to custom message + final_error_message = custom_message + if server_message and server_message.lower() not in custom_message.lower(): + if "\n\n" in custom_message: + final_error_message += "\n" + server_message + else: + final_error_message += "\n\n" + server_message + + # Prepare Request ID message + request_id = "" + request_id_message = "" + for header, label in ( + (X_REQUEST_ID, "Request ID"), + (X_AMZN_TRACE_ID, "Amzn Trace ID"), + (X_AMZ_CF_ID, "Amz CF ID"), + ): + value = response.headers.get(header) + if value: + request_id = str(value) + request_id_message = f" ({label}: {value})" + break + + # Add Request ID + if request_id and request_id.lower() not in final_error_message.lower(): + if "\n" in final_error_message: + newline_index = final_error_message.index("\n") + final_error_message = ( + final_error_message[:newline_index] + request_id_message + final_error_message[newline_index:] + ) + else: + final_error_message += request_id_message + + # Return + err = error_type(final_error_message.strip(), response=response, server_message=server_message or None) + for k, v in attrs.items(): + setattr(err, k, v) + return err + + +# Request-body fields that carry credentials; redacted from debug-log curl commands (HF_DEBUG). +_SENSITIVE_BODY_KEYS = ("subject_token", "access_token", "refresh_token", "client_secret", "device_code") +_SENSITIVE_BODY_PATTERNS = [ + pattern + for key in _SENSITIVE_BODY_KEYS + for pattern in ( + (re.compile(rf'("{key}"\s*:\s*")[^"]*(")'), r"\1\2"), # JSON + (re.compile(rf"(^|&)({key}=)[^&]*"), r"\1\2"), # application/x-www-form-urlencoded + ) +] + + +def _redact_sensitive_body(body: str) -> str: + """Redact OAuth credential values from a JSON or form-urlencoded request body string.""" + for pattern, replacement in _SENSITIVE_BODY_PATTERNS: + body = pattern.sub(replacement, body) + return body + + +def _curlify(request: httpx.Request) -> str: + """Convert a `httpx.Request` into a curl command (str). + + Used for debug purposes only. + + Implementation vendored from https://github.com/ofw/curlify/blob/master/curlify.py. + MIT License Copyright (c) 2016 Egor. + """ + parts: list[tuple[Any, Any]] = [ + ("curl", None), + ("-X", request.method), + ] + + for k, v in sorted(request.headers.items()): + if k.lower() == "authorization": + v = "" # Hide authorization header, no matter its value (can be Bearer, Key, etc.) + parts += [("-H", f"{k}: {v}")] + + body: str | None = None + try: + if request.content is not None: + body = request.content.decode("utf-8", errors="ignore") + if len(body) > 1000: + body = f"{body[:1000]} ... [truncated]" + body = _redact_sensitive_body(body) + except httpx.RequestNotRead: + body = "" + if body is not None: + parts += [("-d", body.replace("\n", ""))] + + parts += [(None, request.url)] + + flat_parts = [] + for k, v in parts: + if k: + flat_parts.append(quote(str(k))) + if v: + flat_parts.append(quote(str(v))) + + return " ".join(flat_parts) + + +# Regex to parse HTTP Range header +RANGE_REGEX = re.compile(r"^\s*bytes\s*=\s*(\d*)\s*-\s*(\d*)\s*$", re.IGNORECASE) + + +def _adjust_range_header(original_range: str | None, resume_size: int) -> str | None: + """ + Adjust HTTP Range header to account for resume position. + """ + if not original_range: + return f"bytes={resume_size}-" + + if "," in original_range: + raise ValueError(f"Multiple ranges detected - {original_range!r}, not supported yet.") + + match = RANGE_REGEX.match(original_range) + if not match: + raise RuntimeError(f"Invalid range format - {original_range!r}.") + start, end = match.groups() + + if not start: + if not end: + raise RuntimeError(f"Invalid range format - {original_range!r}.") + + new_suffix = int(end) - resume_size + new_range = f"bytes=-{new_suffix}" + if new_suffix <= 0: + raise RuntimeError(f"Empty new range - {new_range!r}.") + return new_range + + start = int(start) + new_start = start + resume_size + if end: + end = int(end) + new_range = f"bytes={new_start}-{end}" + if new_start > end: + raise RuntimeError(f"Empty new range - {new_range!r}.") + return new_range + + return f"bytes={new_start}-" diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_lfs.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_lfs.py new file mode 100644 index 0000000000000000000000000000000000000000..78c5d49b23f4363904600ddc808a8b29095a65f8 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_lfs.py @@ -0,0 +1,109 @@ +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Git LFS related utilities""" + +import io +import os +from contextlib import AbstractContextManager +from typing import BinaryIO + + +class SliceFileObj(AbstractContextManager): + """ + Utility context manager to read a *slice* of a seekable file-like object as a seekable, file-like object. + + This is NOT thread safe + + Inspired by stackoverflow.com/a/29838711/593036 + + Credits to @julien-c + + Args: + fileobj (`BinaryIO`): + A file-like object to slice. MUST implement `tell()` and `seek()` (and `read()` of course). + `fileobj` will be reset to its original position when exiting the context manager. + seek_from (`int`): + The start of the slice (offset from position 0 in bytes). + read_limit (`int`): + The maximum number of bytes to read from the slice. + + Attributes: + previous_position (`int`): + The previous position + + Examples: + + Reading 200 bytes with an offset of 128 bytes from a file (ie bytes 128 to 327): + ```python + >>> with open("path/to/file", "rb") as file: + ... with SliceFileObj(file, seek_from=128, read_limit=200) as fslice: + ... fslice.read(...) + ``` + + Reading a file in chunks of 512 bytes + ```python + >>> import os + >>> chunk_size = 512 + >>> file_size = os.getsize("path/to/file") + >>> with open("path/to/file", "rb") as file: + ... for chunk_idx in range(ceil(file_size / chunk_size)): + ... with SliceFileObj(file, seek_from=chunk_idx * chunk_size, read_limit=chunk_size) as fslice: + ... chunk = fslice.read(...) + + ``` + """ + + def __init__(self, fileobj: BinaryIO, seek_from: int, read_limit: int): + self.fileobj = fileobj + self.seek_from = seek_from + self.read_limit = read_limit + + def __enter__(self): + self._previous_position = self.fileobj.tell() + end_of_stream = self.fileobj.seek(0, os.SEEK_END) + self._len = min(self.read_limit, end_of_stream - self.seek_from) + # ^^ The actual number of bytes that can be read from the slice + self.fileobj.seek(self.seek_from, io.SEEK_SET) + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.fileobj.seek(self._previous_position, io.SEEK_SET) + + def read(self, n: int = -1): + pos = self.tell() + if pos >= self._len: + return b"" + remaining_amount = self._len - pos + data = self.fileobj.read(remaining_amount if n < 0 else min(n, remaining_amount)) + return data + + def tell(self) -> int: + return self.fileobj.tell() - self.seek_from + + def seek(self, offset: int, whence: int = os.SEEK_SET) -> int: + start = self.seek_from + end = start + self._len + if whence in (os.SEEK_SET, os.SEEK_END): + offset = start + offset if whence == os.SEEK_SET else end + offset + offset = max(start, min(offset, end)) + whence = os.SEEK_SET + elif whence == os.SEEK_CUR: + cur_pos = self.fileobj.tell() + offset = max(start - cur_pos, min(offset, end - cur_pos)) + else: + raise ValueError(f"whence value {whence} is not supported") + return self.fileobj.seek(offset, whence) - self.seek_from + + def __iter__(self): + yield self.read(n=4 * 1024 * 1024) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_oauth_device.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_oauth_device.py new file mode 100644 index 0000000000000000000000000000000000000000..e626d0c28a34170ce6035ed26b732d4c733e1f0a --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_oauth_device.py @@ -0,0 +1,197 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Device Code OAuth (RFC 8628) for browser-based login, plus access token refresh. + +The flow: the client requests a device code, displays a URL and a short user code, the user +authorizes in a browser, and the client polls ``POST {ENDPOINT}/oauth/token`` until a token is +issued. Access tokens may come with a refresh token, used to renew them transparently (see +``utils/_auth.py::get_token``). + +This module is self-contained protocol logic: no printing, no persistence. Interactive flows +live in ``_login.py`` (human/library) and ``cli/auth.py`` (machine-readable event stream). +""" + +import time +from collections.abc import Callable +from typing import TypedDict, cast + +import httpx + +from .. import constants +from ..errors import DeviceCodeError, OAuthErrorCode +from ._http import get_session, hf_raise_for_status + + +_DEVICE_CODE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:device_code" +_REFRESH_TOKEN_GRANT_TYPE = "refresh_token" + + +class DeviceCodeInfo(TypedDict): + device_code: str + user_code: str + verification_uri: str + verification_uri_complete: str # falls back to verification_uri if the server omits it + interval: int + expires_in: int + + +class OAuthTokenResponse(TypedDict, total=False): + access_token: str + refresh_token: str + expires_in: int + token_type: str + + +def request_device_code() -> DeviceCodeInfo: + """Request a device code from the Hub's OAuth device authorization endpoint. + + The returned dict is normalized: `interval`, `expires_in` and `verification_uri_complete` + are always set (server values, or sensible defaults). + + Raises: + [`DeviceCodeError`]: If the request fails. + """ + try: + response = get_session().post( + f"{constants.ENDPOINT}/oauth/device", + data={"client_id": constants.DEVICE_CODE_OAUTH_CLIENT_ID}, + timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT, + ) + hf_raise_for_status(response) + except httpx.HTTPError as e: + raise DeviceCodeError(f"Failed to request device code from {constants.ENDPOINT}/oauth/device: {e}") from e + info = response.json() + # `interval` is optional per RFC 8628 (5s is the spec-mandated fallback); `expires_in` is + # required but defaulted defensively so polling stays bounded if a server omits it. + info.setdefault("interval", 5) + info.setdefault("expires_in", 900) + if not info.get("verification_uri_complete"): + info["verification_uri_complete"] = info["verification_uri"] + return cast(DeviceCodeInfo, info) + + +def poll_device_token( + device_info: DeviceCodeInfo, *, on_pending: Callable[[], None] | None = None +) -> OAuthTokenResponse: + """Poll the token endpoint until the user authorizes the device. + + Args: + device_info (`DeviceCodeInfo`): + The device authorization response from [`request_device_code`]. + on_pending (`Callable`, *optional*): + Called after each "authorization pending" response (e.g. to print a progress dot). + + Returns: + `OAuthTokenResponse`: the full token response: `access_token`, and optionally + `refresh_token` and `expires_in`. + + Raises: + [`DeviceCodeError`]: If authorization is denied, the device code expires, or the server + returns an unexpected OAuth error. + """ + interval = device_info["interval"] + deadline = time.monotonic() + device_info["expires_in"] + while time.monotonic() < deadline: + # Inconclusive responses (network blip, 5xx, gateway error page, rate limiting) must not + # abort the login: keep polling until the device code expires (RFC 8628 section 3.5). + # The deadline bounds the total wait even if the endpoint is genuinely broken. + data = None + try: + response = get_session().post( + f"{constants.ENDPOINT}/oauth/token", + data={ + "grant_type": _DEVICE_CODE_GRANT_TYPE, + "device_code": device_info["device_code"], + "client_id": constants.DEVICE_CODE_OAUTH_CLIENT_ID, + }, + timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT, + ) + if response.status_code < 500: + data = response.json() + except (httpx.HTTPError, ValueError): + pass + + if data is not None: + if "access_token" in data: + return cast(OAuthTokenResponse, data) + + match data.get("error"): + case None: + pass # JSON without an OAuth `error` field (proxy error page, ...): transient + case OAuthErrorCode.AUTHORIZATION_PENDING: + if on_pending is not None: + on_pending() + case OAuthErrorCode.SLOW_DOWN: + interval += 5 + case OAuthErrorCode.EXPIRED_TOKEN: + raise DeviceCodeError( + "Device code expired. Please try again.", error_code=OAuthErrorCode.EXPIRED_TOKEN + ) + case OAuthErrorCode.ACCESS_DENIED: + raise DeviceCodeError( + "Authorization was denied. Please try again.", error_code=OAuthErrorCode.ACCESS_DENIED + ) + case error: + raise DeviceCodeError( + f"OAuth error: {error} - {data.get('error_description', '')}", error_code=error + ) + + time.sleep(interval) + + raise DeviceCodeError("Device code expired (timeout). Please try again.", error_code=OAuthErrorCode.EXPIRED_TOKEN) + + +def refresh_access_token(refresh_token: str) -> OAuthTokenResponse: + """Exchange a refresh token for a new access token. + + Returns: + `OAuthTokenResponse`: the full token response: `access_token`, and optionally a rotated + `refresh_token` and `expires_in`. + + Raises: + [`DeviceCodeError`]: If the server rejects the refresh (`error_code="invalid_grant"` when + the refresh token is expired or revoked) or returns an unexpected response. + """ + try: + response = get_session().post( + f"{constants.ENDPOINT}/oauth/token", + data={ + "grant_type": _REFRESH_TOKEN_GRANT_TYPE, + "refresh_token": refresh_token, + "client_id": constants.DEVICE_CODE_OAUTH_CLIENT_ID, + }, + # An explicit timeout is critical here: this runs inside `get_token()`, so a hung + # request would otherwise block every Hub call in the process. + timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT, + ) + except httpx.HTTPError as e: + raise DeviceCodeError(f"Failed to refresh access token: {e}") from e + data = _parse_token_response(response) + if "access_token" in data: + return cast(OAuthTokenResponse, data) + error = data.get("error") + raise DeviceCodeError( + f"Failed to refresh access token: {error or response.status_code} - {data.get('error_description', '')}", + error_code=error, + ) + + +def _parse_token_response(response: httpx.Response) -> dict: + try: + return response.json() + except ValueError as e: + raise DeviceCodeError( + f"Failed to parse response from {constants.ENDPOINT}/oauth/token " + f"(status {response.status_code}): {response.text[:500]}" + ) from e diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_pagination.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_pagination.py new file mode 100644 index 0000000000000000000000000000000000000000..4e04f664edee574fc72b935a4472c4b102019045 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_pagination.py @@ -0,0 +1,52 @@ +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to handle pagination on Huggingface Hub.""" + +from collections.abc import Iterable +from typing import Any + +import httpx + +from . import get_session, hf_raise_for_status, http_backoff, logging + + +logger = logging.get_logger(__name__) + + +def paginate(path: str, params: dict | list[tuple[str, Any]], headers: dict, timeout: float | None = None) -> Iterable: + """Fetch a list of models/datasets/spaces and paginate through results. + + This is using the same "Link" header format as GitHub. + See: + - https://requests.readthedocs.io/en/latest/api/#requests.Response.links + - https://docs.github.com/en/rest/guides/traversing-with-pagination#link-header + """ + session = get_session() + r = session.get(path, params=params, headers=headers, timeout=timeout) + hf_raise_for_status(r) + yield from r.json() + + # Follow pages + # Next link already contains query params + next_page = _get_next_page(r) + while next_page is not None: + logger.debug(f"Pagination detected. Requesting next page: {next_page}") + r = http_backoff("GET", next_page, headers=headers, timeout=timeout) + hf_raise_for_status(r) + yield from r.json() + next_page = _get_next_page(r) + + +def _get_next_page(response: httpx.Response) -> str | None: + return response.links.get("next", {}).get("url") diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_parsing.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_parsing.py new file mode 100644 index 0000000000000000000000000000000000000000..eeddee70f6416da25abaef8cb78ca2a5905b89a9 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_parsing.py @@ -0,0 +1,116 @@ +# Copyright 2025-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Parsing helpers shared across modules.""" + +import re +import time + + +RE_NUMBER_WITH_UNIT = re.compile(r"(\d+)([a-z]+)", re.IGNORECASE) + +BYTE_UNITS: dict[str, int] = { + "k": 1_000, + "kb": 1_000, + "m": 1_000_000, + "mb": 1_000_000, + "g": 1_000_000_000, + "gb": 1_000_000_000, + "t": 1_000_000_000_000, + "tb": 1_000_000_000_000, + "p": 1_000_000_000_000_000, + "pb": 1_000_000_000_000_000, +} + +TIME_UNITS: dict[str, int] = { + "s": 1, + "m": 60, + "h": 60 * 60, + "d": 24 * 60 * 60, + "w": 7 * 24 * 60 * 60, + "mo": 30 * 24 * 60 * 60, + "y": 365 * 24 * 60 * 60, +} + + +def parse_size(value: str) -> int: + """Parse a size expressed as a string with digits and unit (like `"10MB"`) to an integer (in bytes).""" + return _parse_with_unit(value, BYTE_UNITS) + + +def parse_duration(value: str) -> int: + """Parse a duration expressed as a string with digits and unit (like `"10s"`) to an integer (in seconds).""" + return _parse_with_unit(value, TIME_UNITS) + + +def _parse_with_unit(value: str, units: dict[str, int]) -> int: + """Parse a numeric value with optional unit.""" + stripped = value.strip() + if not stripped: + raise ValueError("Value cannot be empty.") + try: + return int(value) + except ValueError: + pass + + match = RE_NUMBER_WITH_UNIT.fullmatch(stripped) + if not match: + raise ValueError(f"Invalid value '{value}'. Must match pattern '\\d+[a-z]+' or be a plain number.") + + number = int(match.group(1)) + unit = match.group(2).lower() + + if unit not in units: + raise ValueError(f"Unknown unit '{unit}'. Must be one of {list(units.keys())}.") + + return number * units[unit] + + +def format_duration(secs: int | None) -> str: + """Format a duration in seconds as a short human-readable string (e.g. `"1m 32s"`, `"2h 15m"`, `"45s"`). + + Returns `"--"` when `secs` is `None` so it can be used directly as a CLI table cell. + """ + if secs is None: + return "--" + secs = int(secs) + if secs < 60: + return f"{secs}s" + if secs < 3600: + return f"{secs // 60}m {secs % 60}s" + return f"{secs // 3600}h {(secs % 3600) // 60}m" + + +def format_timesince(ts: float) -> str: + """Format timestamp in seconds into a human-readable string, relative to now. + + Vaguely inspired by Django's `timesince` formatter. + """ + _TIMESINCE_CHUNKS = ( + # Label, divider, max value + ("second", 1, 60), + ("minute", 60, 60), + ("hour", 60 * 60, 24), + ("day", 60 * 60 * 24, 6), + ("week", 60 * 60 * 24 * 7, 6), + ("month", 60 * 60 * 24 * 30, 11), + ("year", 60 * 60 * 24 * 365, None), + ) + delta = time.time() - ts + if delta < 20: + return "a few seconds ago" + for label, divider, max_value in _TIMESINCE_CHUNKS: # noqa: B007 + value = round(delta / divider) + if max_value is not None and value <= max_value: + break + return f"{value} {label}{'s' if value > 1 else ''} ago" diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_paths.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_paths.py new file mode 100644 index 0000000000000000000000000000000000000000..f1420f98642515d6394e72bbab15b7c0ccecbd7b --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_paths.py @@ -0,0 +1,155 @@ +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to handle paths in Huggingface Hub.""" + +from collections.abc import Callable, Generator, Iterable +from fnmatch import fnmatchcase +from pathlib import Path +from typing import TypeVar + + +T = TypeVar("T") + +# Always ignore `.git` and `.cache/huggingface` folders in commits +DEFAULT_IGNORE_PATTERNS = [ + ".git", + ".git/*", + "*/.git", + "**/.git/**", + ".cache/huggingface", + ".cache/huggingface/*", + "*/.cache/huggingface", + "**/.cache/huggingface/**", +] +# Forbidden to commit these folders +FORBIDDEN_FOLDERS = [".git", ".cache"] + + +def filter_repo_objects( + items: Iterable[T], + *, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + key: Callable[[T], str] | None = None, +) -> Generator[T, None, None]: + """Filter repo objects based on an allowlist and a denylist. + + Input must be a list of paths (`str` or `Path`) or a list of arbitrary objects. + In the later case, `key` must be provided and specifies a function of one argument + that is used to extract a path from each element in iterable. + + Patterns are Standard Wildcards (globbing patterns), NOT regular expressions. The pattern matching is based on + Python's `fnmatch.fnmatchcase`, so it is case-sensitive on every platform. Backslashes are treated as path separators + and normalized to forward slashes in both patterns and paths before matching, so patterns built with `os.path.join` + on Windows work as expected. + + Note that it matches `*` across path boundaries, unlike traditional Unix shell globbing. For example, `"data/*.json"` + will match both `data/file.json` and `data/subdir/file.json`. + + See https://docs.python.org/3/library/fnmatch.html for more details. + + Args: + items (`Iterable`): + List of items to filter. + allow_patterns (`str` or `list[str]`, *optional*): + Patterns constituting the allowlist. If provided, item paths must match at + least one pattern from the allowlist. + ignore_patterns (`str` or `list[str]`, *optional*): + Patterns constituting the denylist. If provided, item paths must not match + any patterns from the denylist. + key (`Callable[[T], str]`, *optional*): + Single-argument function to extract a path from each item. If not provided, + the `items` must already be `str` or `Path`. + + Returns: + Filtered list of objects, as a generator. + + Raises: + :class:`ValueError`: + If `key` is not provided and items are not `str` or `Path`. + + Example usage with paths: + ```python + >>> # Filter only PDFs that are not hidden. + >>> list(filter_repo_objects( + ... ["aaa.pdf", "bbb.jpg", ".ccc.pdf", ".ddd.png"], + ... allow_patterns=["*.pdf"], + ... ignore_patterns=[".*"], + ... )) + ["aaa.pdf"] + ``` + + Example usage with objects: + ```python + >>> list(filter_repo_objects( + ... [ + ... CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf") + ... CommitOperationAdd(path_or_fileobj="/tmp/bbb.jpg", path_in_repo="bbb.jpg") + ... CommitOperationAdd(path_or_fileobj="/tmp/.ccc.pdf", path_in_repo=".ccc.pdf") + ... CommitOperationAdd(path_or_fileobj="/tmp/.ddd.png", path_in_repo=".ddd.png") + ... ], + ... allow_patterns=["*.pdf"], + ... ignore_patterns=[".*"], + ... key=lambda x: x.path_in_repo + ... )) + [CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf")] + ``` + """ + if isinstance(allow_patterns, str): + allow_patterns = [allow_patterns] + + if isinstance(ignore_patterns, str): + ignore_patterns = [ignore_patterns] + + if allow_patterns is not None: + allow_patterns = [_add_wildcard_to_directories(_normalize_separators(p)) for p in allow_patterns] + if ignore_patterns is not None: + ignore_patterns = [_add_wildcard_to_directories(_normalize_separators(p)) for p in ignore_patterns] + + if key is None: + + def _identity(item: T) -> str: + if isinstance(item, str): + return item + if isinstance(item, Path): + return str(item) + raise ValueError(f"Please provide `key` argument in `filter_repo_objects`: `{item}` is not a string.") + + key = _identity # Items must be `str` or `Path`, otherwise raise ValueError + + for item in items: + path = _normalize_separators(key(item)) + + # Skip if there's an allowlist and path doesn't match any + if allow_patterns is not None and not any(fnmatchcase(path, r) for r in allow_patterns): + continue + + # Skip if there's a denylist and path matches any + if ignore_patterns is not None and any(fnmatchcase(path, r) for r in ignore_patterns): + continue + + yield item + + +def _normalize_separators(value: str | Path) -> str: + # Repo paths always use `/` and `\` is not an fnmatch escape character, so treating backslashes as path separators + # (e.g. patterns built with `os.path.join` on Windows) is safe. + # `value` can be a `Path` if a custom `key` returns one; coerce to `str` first. + return str(value).replace("\\", "/") + + +def _add_wildcard_to_directories(pattern: str) -> str: + if pattern.endswith("/"): + return pattern + "*" + return pattern diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_runtime.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..f29de42c998b94e60c167018e9a27eb3f50f8c57 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_runtime.py @@ -0,0 +1,445 @@ +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Check presence of installed packages at runtime.""" + +import importlib.metadata +import importlib.util +import os +import platform +import sys +import warnings +from pathlib import Path +from typing import Any, Literal + +from .. import __version__, constants + + +_PY_VERSION: str = sys.version.split()[0].rstrip("+") + +_package_versions = {} + +_CANDIDATES = { + "aiohttp": {"aiohttp"}, + "fastai": {"fastai"}, + "fastapi": {"fastapi"}, + "fastcore": {"fastcore"}, + "gradio": {"gradio"}, + "graphviz": {"graphviz"}, + "hf_xet": {"hf_xet"}, + "jinja": {"Jinja2"}, + "httpx": {"httpx"}, + "keras": {"keras"}, + "numpy": {"numpy"}, + "pillow": {"Pillow"}, + "pydantic": {"pydantic"}, + "pydot": {"pydot"}, + "safetensors": {"safetensors"}, + "tensorboard": {"tensorboardX"}, + "tensorflow": ( + "tensorflow", + "tensorflow-cpu", + "tensorflow-gpu", + "tf-nightly", + "tf-nightly-cpu", + "tf-nightly-gpu", + "intel-tensorflow", + "intel-tensorflow-avx512", + "tensorflow-rocm", + "tensorflow-macos", + ), + "torch": {"torch"}, +} + +# Check once at runtime +for candidate_name, package_names in _CANDIDATES.items(): + _package_versions[candidate_name] = "N/A" + for name in package_names: + try: + _package_versions[candidate_name] = importlib.metadata.version(name) + break + except importlib.metadata.PackageNotFoundError: + pass + + +def _get_version(package_name: str) -> str: + return _package_versions.get(package_name, "N/A") + + +def is_package_available(package_name: str) -> bool: + return _get_version(package_name) != "N/A" + + +# Python +def get_python_version() -> str: + return _PY_VERSION + + +# Huggingface Hub +def get_hf_hub_version() -> str: + return __version__ + + +# aiohttp +def is_aiohttp_available() -> bool: + return is_package_available("aiohttp") + + +def get_aiohttp_version() -> str: + return _get_version("aiohttp") + + +# FastAI +def is_fastai_available() -> bool: + return is_package_available("fastai") + + +def get_fastai_version() -> str: + return _get_version("fastai") + + +# FastAPI +def is_fastapi_available() -> bool: + return is_package_available("fastapi") + + +def get_fastapi_version() -> str: + return _get_version("fastapi") + + +# Fastcore +def is_fastcore_available() -> bool: + return is_package_available("fastcore") + + +def get_fastcore_version() -> str: + return _get_version("fastcore") + + +# FastAI +def is_gradio_available() -> bool: + return is_package_available("gradio") + + +def get_gradio_version() -> str: + return _get_version("gradio") + + +# Graphviz +def is_graphviz_available() -> bool: + return is_package_available("graphviz") + + +def get_graphviz_version() -> str: + return _get_version("graphviz") + + +# httpx +def is_httpx_available() -> bool: + return is_package_available("httpx") + + +def get_httpx_version() -> str: + return _get_version("httpx") + + +# xet +def is_xet_available() -> bool: + # since hf_xet is automatically used if available, allow explicit disabling via environment variable + if constants.HF_HUB_DISABLE_XET: + return False + + return is_package_available("hf_xet") + + +def get_xet_version() -> str: + return _get_version("hf_xet") + + +# keras +def is_keras_available() -> bool: + return is_package_available("keras") + + +def get_keras_version() -> str: + return _get_version("keras") + + +# Numpy +def is_numpy_available() -> bool: + return is_package_available("numpy") + + +def get_numpy_version() -> str: + return _get_version("numpy") + + +# Jinja +def is_jinja_available() -> bool: + return is_package_available("jinja") + + +def get_jinja_version() -> str: + return _get_version("jinja") + + +# Pillow +def is_pillow_available() -> bool: + return is_package_available("pillow") + + +def get_pillow_version() -> str: + return _get_version("pillow") + + +# Pydantic +def is_pydantic_available() -> bool: + if not is_package_available("pydantic"): + return False + # For Pydantic, we add an extra check to test whether it is correctly installed or not. If both pydantic 2.x and + # typing_extensions<=4.5.0 are installed, then pydantic will fail at import time. This should not happen when + # it is installed with `pip install huggingface_hub[inference]` but it can happen when it is installed manually + # by the user in an environment that we don't control. + # + # Usually we won't need to do this kind of check on optional dependencies. However, pydantic is a special case + # as it is automatically imported when doing `from huggingface_hub import ...` even if the user doesn't use it. + # + # See https://github.com/huggingface/huggingface_hub/pull/1829 for more details. + try: + from pydantic import validator # noqa: F401 + except ImportError: + # Example: "ImportError: cannot import name 'TypeAliasType' from 'typing_extensions'" + warnings.warn( + "Pydantic is installed but cannot be imported. Please check your installation. `huggingface_hub` will " + "default to not using Pydantic. Error message: '{e}'" + ) + return False + return True + + +def get_pydantic_version() -> str: + return _get_version("pydantic") + + +# Pydot +def is_pydot_available() -> bool: + return is_package_available("pydot") + + +def get_pydot_version() -> str: + return _get_version("pydot") + + +# Tensorboard +def is_tensorboard_available() -> bool: + return is_package_available("tensorboard") + + +def get_tensorboard_version() -> str: + return _get_version("tensorboard") + + +# Tensorflow +def is_tf_available() -> bool: + return is_package_available("tensorflow") + + +def get_tf_version() -> str: + return _get_version("tensorflow") + + +# Torch +def is_torch_available() -> bool: + return is_package_available("torch") + + +def get_torch_version() -> str: + return _get_version("torch") + + +# Safetensors +def is_safetensors_available() -> bool: + return is_package_available("safetensors") + + +# Shell-related helpers +try: + # Set to `True` if script is running in a Google Colab notebook. + # If running in Google Colab, git credential store is set globally which makes the + # warning disappear. See https://github.com/huggingface/huggingface_hub/issues/1043 + # + # Taken from https://stackoverflow.com/a/63519730. + _is_google_colab = "google.colab" in str(get_ipython()) # type: ignore # noqa: F821 +except NameError: + _is_google_colab = False + + +def is_notebook() -> bool: + """Return `True` if code is executed in a notebook (Jupyter, Colab, QTconsole). + + Taken from https://stackoverflow.com/a/39662359. + Adapted to make it work with Google colab as well. + """ + try: + shell_class = get_ipython().__class__ # type: ignore # noqa: F821 + for parent_class in shell_class.__mro__: # e.g. "is subclass of" + if parent_class.__name__ == "ZMQInteractiveShell": + return True # Jupyter notebook, Google colab or qtconsole + return False + except NameError: + return False # Probably standard Python interpreter + + +def is_google_colab() -> bool: + """Return `True` if code is executed in a Google colab. + + Taken from https://stackoverflow.com/a/63519730. + """ + return _is_google_colab + + +def is_colab_enterprise() -> bool: + """Return `True` if code is executed in a Google Colab Enterprise environment.""" + return os.environ.get("VERTEX_PRODUCT") == "COLAB_ENTERPRISE" + + +# Check how huggingface_hub has been installed + + +def installation_method() -> Literal["brew", "hf_installer", "pip", "unknown"]: + """Return the installation method of the current environment. + + - "hf_installer" if installed via the official installer script + - "brew" if installed via Homebrew + - "pip" if pip is available (default fallback for standard Python environments) + - "unknown" otherwise + """ + # hf_installer check must come first: the installer creates a venv using the + # system Python, which may be Homebrew's. Checking brew first would false-positive + # when the resolved sys.executable points to /opt/homebrew/... inside a venv. + if _is_hf_installer_installation(): + return "hf_installer" + if _is_brew_installation(): + return "brew" + if _is_pip_available(): + return "pip" + return "unknown" + + +def _is_brew_installation() -> bool: + """Check if running from a Homebrew installation. + + Homebrew installs the `hf` formula into a Cellar directory and creates a + libexec virtualenv at e.g. /opt/homebrew/Cellar/hf/0.30.0/libexec/. + We check `sys.prefix` (the venv/prefix root) for "/Cellar/hf/" rather + than checking `sys.executable` — the latter resolves to Homebrew's Python + (e.g. /opt/homebrew/Cellar/python@3.12/...) even for non-brew installs + when the system Python happens to come from Homebrew. + """ + prefix = str(Path(sys.prefix).resolve()) + return "/Cellar/hf/" in prefix + + +def _is_hf_installer_installation() -> bool: + """Return `True` if the current environment was set up via the official hf installer script. + + i.e. using one of + curl -LsSf https://hf.co/cli/install.sh | bash + powershell -ExecutionPolicy ByPass -c "irm https://hf.co/cli/install.ps1 | iex" + """ + venv = sys.prefix # points to venv root if active + marker = Path(venv) / ".hf_installer_marker" + return marker.exists() + + +def _is_pip_available() -> bool: + """Return `True` if pip is importable in the current environment.""" + return importlib.util.find_spec("pip") is not None + + +def dump_environment_info() -> dict[str, Any]: + """Dump information about the machine to help debugging issues. + + Similar helper exist in: + - `datasets` (https://github.com/huggingface/datasets/blob/main/src/datasets/commands/env.py) + - `diffusers` (https://github.com/huggingface/diffusers/blob/main/src/diffusers/commands/env.py) + - `transformers` (https://github.com/huggingface/transformers/blob/main/src/transformers/commands/env.py) + """ + from huggingface_hub import get_token, whoami + from huggingface_hub.utils import is_agent, list_credential_helpers + + token = get_token() + + # Generic machine info + info: dict[str, Any] = { + "huggingface_hub version": get_hf_hub_version(), + "Platform": platform.platform(), + "Python version": get_python_version(), + } + + # Interpreter info + try: + shell_class = get_ipython().__class__ # type: ignore # noqa: F821 + info["Running in iPython ?"] = "Yes" + info["iPython shell"] = shell_class.__name__ + except NameError: + info["Running in iPython ?"] = "No" + info["Running in notebook ?"] = "Yes" if is_notebook() else "No" + info["Running in Google Colab ?"] = "Yes" if is_google_colab() else "No" + info["Running in Google Colab Enterprise ?"] = "Yes" if is_colab_enterprise() else "No" + # Login info + info["Token path ?"] = constants.HF_TOKEN_PATH + info["Has saved token ?"] = token is not None + if token is not None: + try: + info["Who am I ?"] = whoami()["name"] + except Exception: + pass + + try: + info["Configured git credential helpers"] = ", ".join(list_credential_helpers()) + except Exception: + pass + + info["Run by AI agent ?"] = "Yes" if is_agent() else "No" + + # How huggingface_hub has been installed? + info["Installation method"] = installation_method() + + # Installed dependencies + info["httpx"] = get_httpx_version() + info["hf_xet"] = get_xet_version() + info["gradio"] = get_gradio_version() + info["tensorboard"] = get_tensorboard_version() + + # Environment variables + info["ENDPOINT"] = constants.ENDPOINT + info["HF_HUB_CACHE"] = constants.HF_HUB_CACHE + info["HF_ASSETS_CACHE"] = constants.HF_ASSETS_CACHE + info["HF_TOKEN_PATH"] = constants.HF_TOKEN_PATH + info["HF_STORED_TOKENS_PATH"] = constants.HF_STORED_TOKENS_PATH + info["HF_HUB_OFFLINE"] = constants.HF_HUB_OFFLINE + info["HF_HUB_DISABLE_TELEMETRY"] = constants.HF_HUB_DISABLE_TELEMETRY + info["HF_HUB_DISABLE_PROGRESS_BARS"] = constants.HF_HUB_DISABLE_PROGRESS_BARS + info["HF_HUB_DISABLE_SYMLINKS"] = constants.HF_HUB_DISABLE_SYMLINKS + info["HF_HUB_DISABLE_SYMLINKS_WARNING"] = constants.HF_HUB_DISABLE_SYMLINKS_WARNING + info["HF_HUB_DISABLE_EXPERIMENTAL_WARNING"] = constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING + info["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = constants.HF_HUB_DISABLE_IMPLICIT_TOKEN + info["HF_HUB_DISABLE_XET"] = constants.HF_HUB_DISABLE_XET + info["HF_HUB_ETAG_TIMEOUT"] = constants.HF_HUB_ETAG_TIMEOUT + info["HF_HUB_DOWNLOAD_TIMEOUT"] = constants.HF_HUB_DOWNLOAD_TIMEOUT + info["HF_XET_HIGH_PERFORMANCE"] = constants.HF_XET_HIGH_PERFORMANCE + + print("\nCopy-and-paste the text below in your GitHub issue.\n") + print("\n".join([f"- {prop}: {val}" for prop, val in info.items()]) + "\n") + return info diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_safetensors.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_safetensors.py new file mode 100644 index 0000000000000000000000000000000000000000..8e4d0adbc1907cc9a9be6aafef1e9826bd6f97be --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_safetensors.py @@ -0,0 +1,111 @@ +import functools +import operator +from collections import defaultdict +from dataclasses import dataclass, field +from typing import Literal + + +FILENAME_T = str +TENSOR_NAME_T = str +DTYPE_T = Literal["F64", "F32", "F16", "BF16", "I64", "I32", "I16", "I8", "U8", "BOOL"] + + +@dataclass +class TensorInfo: + """Information about a tensor. + + For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. + + Attributes: + dtype (`str`): + The data type of the tensor ("F64", "F32", "F16", "BF16", "I64", "I32", "I16", "I8", "U8", "BOOL"). + shape (`list[int]`): + The shape of the tensor. + data_offsets (`tuple[int, int]`): + The offsets of the data in the file as a tuple `[BEGIN, END]`. + parameter_count (`int`): + The number of parameters in the tensor. + """ + + dtype: DTYPE_T + shape: list[int] + data_offsets: tuple[int, int] + parameter_count: int = field(init=False) + + def __post_init__(self) -> None: + # Taken from https://stackoverflow.com/a/13840436 + try: + self.parameter_count = functools.reduce(operator.mul, self.shape) + except TypeError: + self.parameter_count = 1 # scalar value has no shape + + +@dataclass +class SafetensorsFileMetadata: + """Metadata for a Safetensors file hosted on the Hub. + + This class is returned by [`parse_safetensors_file_metadata`]. + + For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. + + Attributes: + metadata (`dict`): + The metadata contained in the file. + tensors (`dict[str, TensorInfo]`): + A map of all tensors. Keys are tensor names and values are information about the corresponding tensor, as a + [`TensorInfo`] object. + parameter_count (`dict[str, int]`): + A map of the number of parameters per data type. Keys are data types and values are the number of parameters + of that data type. + """ + + metadata: dict[str, str] + tensors: dict[TENSOR_NAME_T, TensorInfo] + parameter_count: dict[DTYPE_T, int] = field(init=False) + + def __post_init__(self) -> None: + parameter_count: dict[DTYPE_T, int] = defaultdict(int) + for tensor in self.tensors.values(): + parameter_count[tensor.dtype] += tensor.parameter_count + self.parameter_count = dict(parameter_count) + + +@dataclass +class SafetensorsRepoMetadata: + """Metadata for a Safetensors repo. + + A repo is considered to be a Safetensors repo if it contains either a 'model.safetensors' weight file (non-shared + model) or a 'model.safetensors.index.json' index file (sharded model) at its root. + + This class is returned by [`get_safetensors_metadata`]. + + For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. + + Attributes: + metadata (`dict`, *optional*): + The metadata contained in the 'model.safetensors.index.json' file, if it exists. Only populated for sharded + models. + sharded (`bool`): + Whether the repo contains a sharded model or not. + weight_map (`dict[str, str]`): + A map of all weights. Keys are tensor names and values are filenames of the files containing the tensors. + files_metadata (`dict[str, SafetensorsFileMetadata]`): + A map of all files metadata. Keys are filenames and values are the metadata of the corresponding file, as + a [`SafetensorsFileMetadata`] object. + parameter_count (`dict[str, int]`): + A map of the number of parameters per data type. Keys are data types and values are the number of parameters + of that data type. + """ + + metadata: dict | None + sharded: bool + weight_map: dict[TENSOR_NAME_T, FILENAME_T] # tensor name -> filename + files_metadata: dict[FILENAME_T, SafetensorsFileMetadata] # filename -> metadata + parameter_count: dict[DTYPE_T, int] = field(init=False) + + def __post_init__(self) -> None: + parameter_count: dict[DTYPE_T, int] = defaultdict(int) + for file_metadata in self.files_metadata.values(): + for dtype, nb_parameters_ in file_metadata.parameter_count.items(): + parameter_count[dtype] += nb_parameters_ + self.parameter_count = dict(parameter_count) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_subprocess.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_subprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..e648e11146ecaa18bdb736bafedaf6cb6d8c15ff --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_subprocess.py @@ -0,0 +1,143 @@ +# Copyright 2021 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +"""Contains utilities to easily handle subprocesses in `huggingface_hub`.""" + +import os +import subprocess +import sys +from collections.abc import Generator +from contextlib import contextmanager +from io import StringIO +from pathlib import Path +from typing import IO + +from .logging import get_logger + + +logger = get_logger(__name__) + + +@contextmanager +def capture_output() -> Generator[StringIO, None, None]: + """Capture output that is printed to terminal. + + Taken from https://stackoverflow.com/a/34738440 + + Example: + ```py + >>> with capture_output() as output: + ... print("hello world") + >>> assert output.getvalue() == "hello world\n" + ``` + """ + output = StringIO() + previous_output = sys.stdout + sys.stdout = output + try: + yield output + finally: + sys.stdout = previous_output + + +def run_subprocess( + command: str | list[str], + folder: str | Path | None = None, + check=True, + **kwargs, +) -> subprocess.CompletedProcess: + """ + Method to run subprocesses. Calling this will capture the `stderr` and `stdout`, + please call `subprocess.run` manually in case you would like for them not to + be captured. + + Args: + command (`str` or `list[str]`): + The command to execute as a string or list of strings. + folder (`str`, *optional*): + The folder in which to run the command. Defaults to current working + directory (from `os.getcwd()`). + check (`bool`, *optional*, defaults to `True`): + Setting `check` to `True` will raise a `subprocess.CalledProcessError` + when the subprocess has a non-zero exit code. + kwargs (`dict[str]`): + Keyword arguments to be passed to the `subprocess.run` underlying command. + + Returns: + `subprocess.CompletedProcess`: The completed process. + """ + if isinstance(command, str): + command = command.split() + + if isinstance(folder, Path): + folder = str(folder) + + return subprocess.run( + command, + capture_output=True, + check=check, + encoding="utf-8", + errors="replace", # if not utf-8, replace char by � + cwd=folder or os.getcwd(), + **kwargs, + ) + + +@contextmanager +def run_interactive_subprocess( + command: str | list[str], + folder: str | Path | None = None, + **kwargs, +) -> Generator[tuple[IO[str], IO[str]], None, None]: + """Run a subprocess in an interactive mode in a context manager. + + Args: + command (`str` or `list[str]`): + The command to execute as a string or list of strings. + folder (`str`, *optional*): + The folder in which to run the command. Defaults to current working + directory (from `os.getcwd()`). + kwargs (`dict[str]`): + Keyword arguments to be passed to the `subprocess.run` underlying command. + + Returns: + `tuple[IO[str], IO[str]]`: A tuple with `stdin` and `stdout` to interact + with the process (input and output are utf-8 encoded). + + Example: + ```python + with _interactive_subprocess("git credential-store get") as (stdin, stdout): + # Write to stdin + stdin.write("url=hf.co\nusername=obama\n".encode("utf-8")) + stdin.flush() + + # Read from stdout + output = stdout.read().decode("utf-8") + ``` + """ + if isinstance(command, str): + command = command.split() + + with subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + errors="replace", # if not utf-8, replace char by � + cwd=folder or os.getcwd(), + **kwargs, + ) as process: + assert process.stdin is not None, "subprocess is opened as subprocess.PIPE" + assert process.stdout is not None, "subprocess is opened as subprocess.PIPE" + yield process.stdin, process.stdout diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_telemetry.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_telemetry.py new file mode 100644 index 0000000000000000000000000000000000000000..a97bae52111902e0b1ffb5fdcdb91ae07396f3f2 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_telemetry.py @@ -0,0 +1,125 @@ +from queue import Queue +from threading import Lock, Thread +from urllib.parse import quote + +from .. import constants, logging +from . import build_hf_headers, get_session, hf_raise_for_status + + +logger = logging.get_logger(__name__) + +# Telemetry is sent by a separate thread to avoid blocking the main thread. +# A daemon thread is started once and consume tasks from the _TELEMETRY_QUEUE. +# If the thread stops for some reason -shouldn't happen-, we restart a new one. +_TELEMETRY_THREAD: Thread | None = None +_TELEMETRY_THREAD_LOCK = Lock() # Lock to avoid starting multiple threads in parallel +_TELEMETRY_QUEUE: Queue = Queue() + + +def send_telemetry( + topic: str, + *, + library_name: str | None = None, + library_version: str | None = None, + user_agent: dict | str | None = None, +) -> None: + """ + Sends telemetry that helps track usage of different HF libraries. + + This usage data helps us debug issues and prioritize new features. However, we understand that not everyone wants + to share additional information, and we respect your privacy. You can disable telemetry collection by setting the + `HF_HUB_DISABLE_TELEMETRY=1` as environment variable. Telemetry is also disabled in offline mode (i.e. when setting + `HF_HUB_OFFLINE=1`). + + Telemetry collection is run in a separate thread to minimize impact for the user. + + Args: + topic (`str`): + Name of the topic that is monitored. The topic is directly used to build the URL. If you want to monitor + subtopics, just use "/" separation. Examples: "gradio", "transformers/examples",... + library_name (`str`, *optional*): + The name of the library that is making the HTTP request. Will be added to the user-agent header. + library_version (`str`, *optional*): + The version of the library that is making the HTTP request. Will be added to the user-agent header. + user_agent (`str`, `dict`, *optional*): + The user agent info in the form of a dictionary or a single string. It will be completed with information about the installed packages. + + Example: + ```py + >>> from huggingface_hub.utils import send_telemetry + + # Send telemetry without library information + >>> send_telemetry("ping") + + # Send telemetry to subtopic with library information + >>> send_telemetry("gradio/local_link", library_name="gradio", library_version="3.22.1") + + # Send telemetry with additional data + >>> send_telemetry( + ... topic="examples", + ... library_name="transformers", + ... library_version="4.26.0", + ... user_agent={"pipeline": "text_classification", "framework": "flax"}, + ... ) + ``` + """ + if constants.is_offline_mode() or constants.HF_HUB_DISABLE_TELEMETRY: + return + + _start_telemetry_thread() # starts thread only if doesn't exist yet + _TELEMETRY_QUEUE.put( + {"topic": topic, "library_name": library_name, "library_version": library_version, "user_agent": user_agent} + ) + + +def _start_telemetry_thread(): + """Start a daemon thread to consume tasks from the telemetry queue. + + If the thread is interrupted, start a new one. + """ + with _TELEMETRY_THREAD_LOCK: # avoid to start multiple threads if called concurrently + global _TELEMETRY_THREAD + if _TELEMETRY_THREAD is None or not _TELEMETRY_THREAD.is_alive(): + _TELEMETRY_THREAD = Thread(target=_telemetry_worker, daemon=True) + _TELEMETRY_THREAD.start() + + +def _telemetry_worker(): + """Wait for a task and consume it.""" + while True: + kwargs = _TELEMETRY_QUEUE.get() + _send_telemetry_in_thread(**kwargs) + _TELEMETRY_QUEUE.task_done() + + +def _send_telemetry_in_thread( + topic: str, + *, + library_name: str | None = None, + library_version: str | None = None, + user_agent: dict | str | None = None, +) -> None: + """Contains the actual data sending data to the Hub. + + This function is called directly in gradio's analytics because + it is not possible to send telemetry from a daemon thread. + + See here: https://github.com/gradio-app/gradio/pull/8180 + + Please do not rename or remove this function. + """ + path = "/".join(quote(part) for part in topic.split("/") if len(part) > 0) + try: + r = get_session().head( + f"{constants.ENDPOINT}/api/telemetry/{path}", + headers=build_hf_headers( + token=False, # no need to send a token for telemetry + library_name=library_name, + library_version=library_version, + user_agent=user_agent, + ), + ) + hf_raise_for_status(r) + except Exception as e: + # We don't want to error in case of connection errors of any kind. + logger.debug(f"Error while sending telemetry: {e}") diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_terminal.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_terminal.py new file mode 100644 index 0000000000000000000000000000000000000000..051713fb8bfafb8654efc5952b844bb218757173 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_terminal.py @@ -0,0 +1,262 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to print stuff to the terminal (styling, helpers).""" + +import ctypes +import os +import shutil +import sys +from contextlib import contextmanager + +from ._detect_agent import is_agent + + +if sys.platform == "win32": + import msvcrt +else: + import select + import termios + import tty + + +class StatusLine: + """Minimal TTY status line for sync progress (stderr, single-line overwrite).""" + + def __init__(self, enabled: bool = True): + self._active = enabled and sys.stderr.isatty() + + def update(self, msg: str) -> None: + if not self._active: + return + width = shutil.get_terminal_size().columns + if len(msg) > width - 1: + msg = msg[: width - 4] + "..." + sys.stderr.write(f"\r\033[K\033[90m{msg}\033[0m") + sys.stderr.flush() + + def done(self, msg: str) -> None: + if not self._active: + return + width = shutil.get_terminal_size().columns + if len(msg) > width - 1: + msg = msg[: width - 4] + "..." + sys.stderr.write(f"\r\033[K\033[90m{msg}\033[0m\n") + sys.stderr.flush() + + +class ANSI: + """ + Helper for en.wikipedia.org/wiki/ANSI_escape_code + """ + + _blue = "\u001b[34m" + _bold = "\u001b[1m" + _gray = "\u001b[90m" + _green = "\u001b[32m" + _red = "\u001b[31m" + _reset = "\u001b[0m" + _underline = "\u001b[4m" + _yellow = "\u001b[33m" + + @classmethod + def blue(cls, s: str) -> str: + return cls._format(s, cls._blue) + + @classmethod + def bold(cls, s: str) -> str: + return cls._format(s, cls._bold) + + @classmethod + def gray(cls, s: str) -> str: + return cls._format(s, cls._gray) + + @classmethod + def green(cls, s: str) -> str: + return cls._format(s, cls._green) + + @classmethod + def red(cls, s: str) -> str: + return cls._format(s, cls._bold + cls._red) + + @classmethod + def underline(cls, s: str) -> str: + return cls._format(s, cls._underline) + + @classmethod + def yellow(cls, s: str) -> str: + return cls._format(s, cls._yellow) + + @classmethod + def _format(cls, s: str, code: str) -> str: + if os.environ.get("NO_COLOR") or is_agent(): + # See https://no-color.org/ + return s + return f"{code}{s}{cls._reset}" + + +def select_choice(prompt: str, choices: list[str]) -> int: + """Single-choice interactive prompt. Returns the index of the selected choice. + + On a TTY, renders an arrow-key menu (Up/Down to move, Enter to confirm, 1-9 to pick + directly, Ctrl+C to abort). Falls back to a numbered `input()` prompt when raw + keyboard input is not available. Callers are responsible for not prompting at all in + non-interactive contexts. + """ + if not choices: + raise ValueError("select_choice() requires at least one choice.") + if _supports_raw_keyboard(): + return _select_with_arrows(prompt, choices) + return _select_with_numbers(prompt, choices) + + +def _supports_raw_keyboard() -> bool: + if not (sys.stdin and sys.stdin.isatty() and sys.stdout.isatty()): + return False + if sys.platform == "win32": + return _enable_windows_vt_processing() + return True + + +def _enable_windows_vt_processing() -> bool: + """Enable VT escape-sequence processing on the Windows console (off by default in legacy + cmd.exe/PowerShell windows, where the menu would otherwise render as literal `←[K` garbage).""" + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 + STD_OUTPUT_HANDLE = -11 + try: + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] + handle = kernel32.GetStdHandle(STD_OUTPUT_HANDLE) + mode = ctypes.c_uint32() + if not kernel32.GetConsoleMode(handle, ctypes.byref(mode)): + return False + return bool(kernel32.SetConsoleMode(handle, mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING)) + except Exception: + return False + + +@contextmanager +def _raw_terminal(): + """Put the terminal in cbreak mode (read keypresses without Enter). No-op on Windows.""" + if sys.platform == "win32": + yield + return + + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + try: + tty.setcbreak(fd) + yield + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + + +def _read_key() -> str: + """Read one keypress, normalizing arrow keys to "up"/"down".""" + if sys.platform == "win32": + char = msvcrt.getwch() + if char in ("\x00", "\xe0"): # arrow keys come as a two-character sequence + return {"H": "up", "P": "down"}.get(msvcrt.getwch(), "") + return char + + # Read the file descriptor directly: `sys.stdin.read(1)` would buffer the whole escape + # sequence internally, making the fd look empty to `select()` below. + fd = sys.stdin.fileno() + char = os.read(fd, 1) + # Disambiguate a bare Escape from an escape sequence (e.g. "\x1b[A" for Up). + if char == b"\x1b" and select.select([fd], [], [], 0.05)[0]: + if os.read(fd, 1) == b"[" and select.select([fd], [], [], 0.05)[0]: + return {b"A": "up", b"B": "down"}.get(os.read(fd, 1), "") + return "" + return char.decode(errors="replace") + + +def _select_with_arrows(prompt: str, choices: list[str]) -> int: + selected = 0 + print(ANSI.bold(f"? {prompt}") + ANSI.gray(" [Use arrows, Enter to confirm]")) + + def render() -> None: + for i, choice in enumerate(choices): + line = ANSI.green("> ") + ANSI.bold(choice) if i == selected else " " + choice + sys.stdout.write(f"\r\x1b[K{line}\n") + sys.stdout.flush() + + try: + sys.stdout.write("\x1b[?25l") # hide cursor + render() + with _raw_terminal(): + while True: + key = _read_key() + if key == "\x03": + # Ctrl+C: POSIX cbreak keeps ISIG so it never reaches here, but on Windows + # msvcrt.getwch() returns the raw character instead of raising. + raise KeyboardInterrupt + if key == "up": + selected = (selected - 1) % len(choices) + elif key == "down": + selected = (selected + 1) % len(choices) + elif key.isdecimal() and 1 <= int(key) <= len(choices): + selected = int(key) - 1 + break + elif key in ("\r", "\n"): + break + else: + continue + sys.stdout.write(f"\x1b[{len(choices)}A") # move back to the first option line + render() + finally: + sys.stdout.write("\x1b[?25h") # show cursor + sys.stdout.flush() + + # Collapse the menu into a single "? prompt answer" summary line, like gh does. + sys.stdout.write(f"\x1b[{len(choices) + 1}A\r\x1b[J") + print(ANSI.bold(f"? {prompt} ") + ANSI.green(choices[selected])) + return selected + + +def _select_with_numbers(prompt: str, choices: list[str]) -> int: + print(f"? {prompt}") + for i, choice in enumerate(choices, start=1): + print(f" {i}. {choice}") + while True: + raw = input("Choice [1]: ").strip() + if not raw: + return 0 + if raw.isdecimal() and 1 <= int(raw) <= len(choices): + return int(raw) - 1 + print(f"Invalid choice. Enter a number between 1 and {len(choices)}.") + + +def tabulate( + rows: list[list[str | int]], + headers: list[str], + alignments: dict[str, str] | None = None, +) -> str: + """ + Inspired by: + + - stackoverflow.com/a/8356620/593036 + - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data + """ + _ALIGN_MAP = {"left": "<", "right": ">"} + for row in rows: + if len(row) < len(headers): + raise IndexError(f"Row has {len(row)} values but expected {len(headers)} (headers: {headers})") + col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] + col_aligns = [_ALIGN_MAP.get((alignments or {}).get(h, "left"), "<") for h in headers] + row_format = " ".join(f"{{:{a}{w}}}" for a, w in zip(col_aligns, col_widths)) + lines = [] + lines.append(row_format.format(*headers)) + lines.append(row_format.format(*["-" * w for w in col_widths])) + for row in rows: + lines.append(row_format.format(*row)) + return "\n".join(lines) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_typing.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_typing.py new file mode 100644 index 0000000000000000000000000000000000000000..20db0e2d3ae9ae30a2a8a641e82a7869738fe2a9 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_typing.py @@ -0,0 +1,91 @@ +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Handle typing imports based on system compatibility.""" + +from collections.abc import Callable +from types import UnionType +from typing import Any, Literal, TypeVar, Union, get_args, get_origin + + +UNION_TYPES: list[Any] = [Union, UnionType] + + +HTTP_METHOD_T = Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"] + +# type hint meaning "function signature not changed by decorator" +CallableT = TypeVar("CallableT", bound=Callable) + +_JSON_SERIALIZABLE_TYPES = (int, float, str, bool, type(None)) + + +def is_jsonable(obj: Any, _visited: set[int] | None = None) -> bool: + """Check if an object is JSON serializable. + + This is a weak check, as it does not check for the actual JSON serialization, but only for the types of the object. + It works correctly for basic use cases but do not guarantee an exhaustive check. + + Object is considered to be recursively json serializable if: + - it is an instance of int, float, str, bool, or NoneType + - it is a list or tuple and all its items are json serializable + - it is a dict and all its keys are strings and all its values are json serializable + + Uses a visited set to avoid infinite recursion on circular references. If object has already been visited, it is + considered not json serializable. + """ + # Initialize visited set to track object ids and detect circular references + if _visited is None: + _visited = set() + + # Detect circular reference + obj_id = id(obj) + if obj_id in _visited: + return False + + # Add current object to visited before recursive checks + _visited.add(obj_id) + try: + if isinstance(obj, _JSON_SERIALIZABLE_TYPES): + return True + if isinstance(obj, (list, tuple)): + return all(is_jsonable(item, _visited) for item in obj) + if isinstance(obj, dict): + return all( + isinstance(key, _JSON_SERIALIZABLE_TYPES) and is_jsonable(value, _visited) + for key, value in obj.items() + ) + if hasattr(obj, "__json__"): + return True + return False + except RecursionError: + return False + finally: + # Remove the object id from visited to avoid side‑effects for other branches + _visited.discard(obj_id) + + +def is_simple_optional_type(type_: type) -> bool: + """Check if a type is optional, i.e. Optional[Type] or Union[Type, None] or Type | None, where Type is a non-composite type.""" + if get_origin(type_) in UNION_TYPES: + union_args = get_args(type_) + if len(union_args) == 2 and type(None) in union_args: + return True + return False + + +def unwrap_simple_optional_type(optional_type: type) -> type: + """Unwraps a simple optional type, i.e. returns Type from Optional[Type].""" + for arg in get_args(optional_type): + if arg is not type(None): + return arg + raise ValueError(f"'{optional_type}' is not an optional type") diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_validators.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_validators.py new file mode 100644 index 0000000000000000000000000000000000000000..59c02dad4d08d8838b84ed2cda7634108a5ff1a2 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_validators.py @@ -0,0 +1,210 @@ +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to validate argument values in `huggingface_hub`.""" + +import inspect +import re +import warnings +from functools import wraps +from itertools import chain +from typing import Any + +from huggingface_hub.errors import HFValidationError + +from ._typing import CallableT + + +REPO_ID_REGEX = re.compile( + r""" + ^ + (\b[\w\-.]+\b/)? # optional namespace (username or organization) + \b # starts with a word boundary + [\w\-.]{1,96} # repo_name: alphanumeric + . _ - + \b # ends with a word boundary + $ + """, + flags=re.VERBOSE, +) + + +def validate_hf_hub_args(fn: CallableT) -> CallableT: + """Validate values received as argument for any public method of `huggingface_hub`. + + The goal of this decorator is to harmonize validation of arguments reused + everywhere. By default, all defined validators are tested. + + Validators: + - [`~utils.validate_repo_id`]: `repo_id` must be `"repo_name"` + or `"namespace/repo_name"`. Namespace is a username or an organization. + - [`~utils.smoothly_deprecate_legacy_arguments`]: Ignore `proxies` when downloading files (should be set globally). + + Example: + ```py + >>> from huggingface_hub.utils import validate_hf_hub_args + + >>> @validate_hf_hub_args + ... def my_cool_method(repo_id: str): + ... print(repo_id) + + >>> my_cool_method(repo_id="valid_repo_id") + valid_repo_id + + >>> my_cool_method("other..repo..id") + huggingface_hub.utils._validators.HFValidationError: Cannot have -- or .. in repo_id: 'other..repo..id'. + + >>> my_cool_method(repo_id="other..repo..id") + huggingface_hub.utils._validators.HFValidationError: Cannot have -- or .. in repo_id: 'other..repo..id'. + ``` + + Raises: + [`~utils.HFValidationError`]: + If an input is not valid. + """ + # TODO: add an argument to opt-out validation for specific argument? + signature = inspect.signature(fn) + + @wraps(fn) + def _inner_fn(*args, **kwargs): + for arg_name, arg_value in chain( + zip(signature.parameters, args), # Args values + kwargs.items(), # Kwargs values + ): + if arg_name in ["repo_id", "from_id", "to_id"]: + validate_repo_id(arg_value) + + kwargs = smoothly_deprecate_legacy_arguments(fn_name=fn.__name__, kwargs=kwargs) + + return fn(*args, **kwargs) + + return _inner_fn # type: ignore + + +def validate_repo_id(repo_id: str | None) -> None: + """Validate `repo_id` is valid. + + This is not meant to replace the proper validation made on the Hub but rather to + avoid local inconsistencies whenever possible (example: passing `repo_type` in the + `repo_id` is forbidden). + + Rules: + - Between 1 and 96 characters. + - Either "repo_name" or "namespace/repo_name" + - [a-zA-Z0-9] or "-", "_", "." + - "--" and ".." are forbidden + + Valid: `"foo"`, `"foo/bar"`, `"123"`, `"Foo-BAR_foo.bar123"` + + Not valid: `"datasets/foo/bar"`, `".repo_id"`, `"foo--bar"`, `"foo.git"` + + Example: + ```py + >>> from huggingface_hub.utils import validate_repo_id + >>> validate_repo_id(repo_id="valid_repo_id") + >>> validate_repo_id(repo_id="other..repo..id") + huggingface_hub.utils._validators.HFValidationError: Cannot have -- or .. in repo_id: 'other..repo..id'. + ``` + + Discussed in https://github.com/huggingface/huggingface_hub/issues/1008. + In moon-landing (internal repository): + - https://github.com/huggingface/moon-landing/blob/main/server/lib/Names.ts#L27 + - https://github.com/huggingface/moon-landing/blob/main/server/views/components/NewRepoForm/NewRepoForm.svelte#L138 + """ + if repo_id is None: + # Repo id is not always required + return + + if not isinstance(repo_id, str): + # Typically, a Path is not a repo_id + raise HFValidationError(f"Repo id must be a string, not {type(repo_id)}: '{repo_id}'.") + + if repo_id.count("/") > 1: + raise HFValidationError( + "Repo id must be in the form 'repo_name' or 'namespace/repo_name':" + f" '{repo_id}'. Use `repo_type` argument if needed." + ) + + if not REPO_ID_REGEX.match(repo_id): + raise HFValidationError( + "Repo id must use alphanumeric chars, '-', '_' or '.'." + " The name cannot start or end with '-' or '.' and the maximum length is 96:" + f" '{repo_id}'." + ) + + if "--" in repo_id or ".." in repo_id: + raise HFValidationError(f"Cannot have -- or .. in repo_id: '{repo_id}'.") + + if repo_id.endswith(".git"): + raise HFValidationError(f"Repo_id cannot end by '.git': '{repo_id}'.") + + +def smoothly_deprecate_legacy_arguments(fn_name: str, kwargs: dict[str, Any]) -> dict[str, Any]: + """Smoothly deprecate legacy arguments in the `huggingface_hub` codebase. + + This function ignores some deprecated arguments from the kwargs and warns the user they are ignored. + The goal is to avoid breaking existing code while guiding the user to the new way of doing things. + + List of deprecated arguments: + - `proxies`: + To set up proxies, user must either use the HTTP_PROXY environment variable or configure the `httpx.Client` + manually using the [`set_client_factory`] function. + + In huggingface_hub 0.x, `proxies` was a dictionary directly passed to `requests.request`. + In huggingface_hub 1.x, we migrated to `httpx` which does not support `proxies` the same way. + In particular, it is not possible to configure proxies on a per-request basis. The solution is to configure + it globally using the [`set_client_factory`] function or using the HTTP_PROXY environment variable. + + For more details, see: + - https://www.python-httpx.org/advanced/proxies/ + - https://www.python-httpx.org/compatibility/#proxy-keys. + + - `resume_download`: deprecated without replacement. `huggingface_hub` always resumes downloads whenever possible. + - `force_filename`: deprecated without replacement. Filename is always the same as on the Hub. + - `local_dir_use_symlinks`: deprecated without replacement. Downloading to a local directory does not use symlinks anymore. + """ + new_kwargs = kwargs.copy() # do not mutate input ! + + # proxies + proxies = new_kwargs.pop("proxies", None) # remove from kwargs + if proxies is not None: + warnings.warn( + f"The `proxies` argument is ignored in `{fn_name}`. To set up proxies, use the HTTP_PROXY / HTTPS_PROXY" + " environment variables or configure the `httpx.Client` manually using `huggingface_hub.set_client_factory`." + " See https://www.python-httpx.org/advanced/proxies/ for more details." + ) + + # resume_download + resume_download = new_kwargs.pop("resume_download", None) # remove from kwargs + if resume_download is not None: + warnings.warn( + f"The `resume_download` argument is deprecated and ignored in `{fn_name}`. Downloads always resume" + " whenever possible." + ) + + # force_filename + force_filename = new_kwargs.pop("force_filename", None) # remove from kwargs + if force_filename is not None: + warnings.warn( + f"The `force_filename` argument is deprecated and ignored in `{fn_name}`. Filename is always the same " + "as on the Hub." + ) + + # local_dir_use_symlinks + local_dir_use_symlinks = new_kwargs.pop("local_dir_use_symlinks", None) # remove from kwargs + if local_dir_use_symlinks is not None: + warnings.warn( + f"The `local_dir_use_symlinks` argument is deprecated and ignored in `{fn_name}`. Downloading to a local" + " directory does not use symlinks anymore." + ) + + return new_kwargs diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_verification.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_verification.py new file mode 100644 index 0000000000000000000000000000000000000000..1c01f97a4cb79959a628b95b2b03624bff364485 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_verification.py @@ -0,0 +1,167 @@ +import re +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Literal, TypedDict + +from .. import constants +from ..file_download import repo_folder_name +from .sha import git_hash, sha_fileobj + + +if TYPE_CHECKING: + from ..hf_api import RepoFile + +# using fullmatch for clarity and strictness +_REGEX_COMMIT_HASH = re.compile(r"^[0-9a-f]{40}$") + + +# Typed structure describing a checksum mismatch +class Mismatch(TypedDict): + path: str + expected: str + actual: str + algorithm: str + + +HashAlgo = Literal["sha256", "git-sha1"] + + +@dataclass(frozen=True) +class FolderVerification: + revision: str + checked_count: int + mismatches: list[Mismatch] + missing_paths: list[str] + extra_paths: list[str] + verified_path: Path + + +def collect_local_files(root: Path) -> dict[str, Path]: + """ + Return a mapping of repo-relative path -> absolute path for all files under `root`. + """ + return {p.relative_to(root).as_posix(): p for p in root.rglob("*") if p.is_file()} + + +def _resolve_commit_hash_from_cache(storage_folder: Path, revision: str | None) -> str: + """ + Resolve a commit hash from a cache repo folder and an optional revision. + """ + if revision and _REGEX_COMMIT_HASH.fullmatch(revision): + return revision + + refs_dir = storage_folder / "refs" + snapshots_dir = storage_folder / "snapshots" + + if revision: + ref_path = refs_dir / revision + if ref_path.is_file(): + return ref_path.read_text(encoding="utf-8").strip() + raise ValueError(f"Revision '{revision}' could not be resolved in cache (expected file '{ref_path}').") + + # No revision provided: try common defaults + main_ref = refs_dir / "main" + if main_ref.is_file(): + return main_ref.read_text(encoding="utf-8").strip() + + if not snapshots_dir.is_dir(): + raise ValueError(f"Cache repo is missing snapshots directory: {snapshots_dir}. Provide --revision explicitly.") + + candidates = [p.name for p in snapshots_dir.iterdir() if p.is_dir() and _REGEX_COMMIT_HASH.fullmatch(p.name)] + if len(candidates) == 1: + return candidates[0] + + raise ValueError( + "Ambiguous cached revision: multiple snapshots found and no refs to disambiguate. Please pass --revision." + ) + + +def compute_file_hash(path: Path, algorithm: HashAlgo) -> str: + """ + Compute the checksum of a local file using the requested algorithm. + """ + + with path.open("rb") as stream: + if algorithm == "sha256": + return sha_fileobj(stream).hex() + if algorithm == "git-sha1": + return git_hash(stream.read()) + raise ValueError(f"Unsupported hash algorithm: {algorithm}") + + +def verify_maps( + *, + remote_by_path: dict[str, "RepoFile"], + local_by_path: dict[str, Path], + revision: str, + verified_path: Path, +) -> FolderVerification: + """Compare remote entries and local files and return a verification result.""" + remote_paths = set(remote_by_path) + local_paths = set(local_by_path) + + missing = sorted(remote_paths - local_paths) + extra = sorted(local_paths - remote_paths) + both = sorted(remote_paths & local_paths) + + mismatches: list[Mismatch] = [] + + for rel_path in both: + remote_entry = remote_by_path[rel_path] + local_path = local_by_path[rel_path] + + lfs = getattr(remote_entry, "lfs", None) + lfs_sha = getattr(lfs, "sha256", None) if lfs is not None else None + if lfs_sha is None and isinstance(lfs, dict): + lfs_sha = lfs.get("sha256") + if lfs_sha: + algorithm: HashAlgo = "sha256" + expected = str(lfs_sha).lower() + else: + blob_id = remote_entry.blob_id # type: ignore + algorithm = "git-sha1" + expected = str(blob_id).lower() + + actual = compute_file_hash(local_path, algorithm) + + if actual != expected: + mismatches.append(Mismatch(path=rel_path, expected=expected, actual=actual, algorithm=algorithm)) + + return FolderVerification( + revision=revision, + checked_count=len(both), + mismatches=mismatches, + missing_paths=missing, + extra_paths=extra, + verified_path=verified_path, + ) + + +def resolve_local_root( + *, + repo_id: str, + repo_type: str, + revision: str | None, + cache_dir: Path | None, + local_dir: Path | None, +) -> tuple[Path, str]: + """ + Resolve the root directory to scan locally and the remote revision to verify. + """ + if local_dir is not None: + root = Path(local_dir).expanduser().resolve() + if not root.is_dir(): + raise ValueError(f"Local directory does not exist or is not a directory: {root}") + return root, (revision or constants.DEFAULT_REVISION) + + cache_root = Path(cache_dir or constants.HF_HUB_CACHE).expanduser().resolve() + storage_folder = cache_root / repo_folder_name(repo_id=repo_id, repo_type=repo_type) + if not storage_folder.exists(): + raise ValueError( + f"Repo is not present in cache: {storage_folder}. Use 'hf download' first or pass --local-dir." + ) + commit = _resolve_commit_hash_from_cache(storage_folder, revision) + snapshot_dir = storage_folder / "snapshots" / commit + if not snapshot_dir.is_dir(): + raise ValueError(f"Snapshot directory does not exist for revision '{commit}': {snapshot_dir}.") + return snapshot_dir, commit diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_xet.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_xet.py new file mode 100644 index 0000000000000000000000000000000000000000..e2361c02abf29dac8af132961ab4f65a177df010 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_xet.py @@ -0,0 +1,181 @@ +import os +import re +import threading +from dataclasses import dataclass +from enum import Enum + +import httpx + +from .. import constants +from . import validate_hf_hub_args + + +_REGEX_XET_HASH = re.compile(r"^[0-9a-fA-F]{64}$") + + +def is_valid_xet_hash(xet_hash: str) -> bool: + """Return whether `xet_hash` looks like a real Xet content hash (64 hex chars).""" + return bool(_REGEX_XET_HASH.fullmatch(xet_hash)) + + +class XetTokenType(str, Enum): + READ = "read" + WRITE = "write" + + +@dataclass(frozen=True) +class XetFileData: + file_hash: str + refresh_route: str + + +def parse_xet_file_data_from_response(response: httpx.Response, endpoint: str | None = None) -> XetFileData | None: + """ + Parse XET file metadata from an HTTP response. + + This function extracts XET file metadata from the HTTP headers or HTTP links + of a given response object. If the required metadata is not found, it returns `None`. + + Args: + response (`httpx.Response`): + The HTTP response object containing headers dict and links dict to extract the XET metadata from. + Returns: + `Optional[XetFileData]`: + An instance of `XetFileData` containing the file hash and refresh route if the metadata + is found. Returns `None` if the required metadata is missing. + """ + if response is None: + return None + try: + file_hash = response.headers[constants.HUGGINGFACE_HEADER_X_XET_HASH] + + if constants.HUGGINGFACE_HEADER_LINK_XET_AUTH_KEY in response.links: + refresh_route = response.links[constants.HUGGINGFACE_HEADER_LINK_XET_AUTH_KEY]["url"] + else: + refresh_route = response.headers[constants.HUGGINGFACE_HEADER_X_XET_REFRESH_ROUTE] + except KeyError: + return None + endpoint = endpoint if endpoint is not None else constants.ENDPOINT + if refresh_route.startswith(constants.HUGGINGFACE_CO_URL_HOME): + refresh_route = refresh_route.replace(constants.HUGGINGFACE_CO_URL_HOME.rstrip("/"), endpoint.rstrip("/")) + return XetFileData( + file_hash=file_hash, + refresh_route=refresh_route, + ) + + +@validate_hf_hub_args +def xet_connection_info_refresh_url( + *, + token_type: XetTokenType, + repo_id: str, + repo_type: str, + revision: str | None = None, + endpoint: str | None = None, +) -> str: + """ + Build the URL used to fetch or refresh a Xet access token for a given repo. + Args: + token_type (`XetTokenType`): + Type of the token to request: `"read"` or `"write"`. + repo_id (`str`): + A namespace (user or an organization) and a repo name separated by a `/`. + repo_type (`str`): + Type of the repo (e.g. `"model"`, `"dataset"`, `"space"`, `"bucket"`). + revision (`str`, `optional`): + The revision of the repo to get the token for. + endpoint (`str`, `optional`): + The endpoint to use for the request. Defaults to the Hub endpoint. + Returns: + `str`: + The fully-qualified URL of the token refresh endpoint. + """ + endpoint = endpoint if endpoint is not None else constants.ENDPOINT + url = f"{endpoint}/api/{repo_type}s/{repo_id}/xet-{token_type.value}-token" + if repo_type != "bucket" or revision is not None: + # On "bucket" repo type, the revision never needed => don't use it + # Otherwise, use the revision. + # Note: when creating a PR on a git-based repo, user needs write access but they don't know the revision in advance. + # => pass "/None" in URL and server will return a token for PR refs. + url += f"/{revision}" + return url + + +class XetSessionHolder: + """Holds an optional XetSession; supports safe re-creation after sigint_abort or fork. + + Thread-safe: a ``threading.Lock`` guards all state mutations, which matters + for free-threaded Python (3.14t) where multiple threads can race on ``get()`` + or ``sigint_abort()`` without the GIL serialising them. + """ + + def __init__(self): + self._lock = threading.Lock() + self._session = None + self._session_pid: int | None = None + + def get(self): + """Return the current session, creating one if needed. + + Fork-safe: if the current process PID differs from the PID that created + the session (i.e. we are in a forked child), the old session is discarded + and a fresh session is created for this process. + """ + with self._lock: + current_pid = os.getpid() + + if self._session is not None and self._session_pid != current_pid: + # Fork detected. Discard the parent's session; the Rust Drop will + # call discard_runtime() (std::mem::forget) rather than the normal + # shutdown path, so this returns immediately without blocking. + self._session = None + + if self._session is None: + from hf_xet import XetSession + + self._session = XetSession() + self._session_pid = current_pid + + return self._session + + def sigint_abort(self): + """Abort the current session and clear it so the next get() creates a fresh one.""" + with self._lock: + if self._session is not None: + try: + self._session.sigint_abort() + except Exception: + pass + self._session = None + self._session_pid = None + + +_GLOBAL_XET_HOLDER = XetSessionHolder() + + +def get_xet_session(): + """Return the global :class:`hf_xet.XetSession`, creating it on first call. + + The session is shared across all calls within a process, just as the HTTP + client returned by :func:`~huggingface_hub.utils._http.get_session` is shared. + It is created lazily and is fork-safe and thread-safe. + """ + return _GLOBAL_XET_HOLDER.get() + + +def xet_headers_without_auth(headers: dict[str, str]) -> dict[str, str]: + """Return a copy of headers with the authorization header removed. + + Xet storage requests use a short-lived xet access token for auth, so the + Hub authorization header must not be forwarded to xet storage endpoints. + """ + return {key: value for key, value in headers.items() if key.lower() != "authorization"} + + +def abort_xet_session(): + """Abort the global xet session after a KeyboardInterrupt. + + Cancels any in-flight Rust operation and clears the session so the next + call to :func:`get_xet_session` starts fresh (notebook-friendly). + """ + _GLOBAL_XET_HOLDER.sigint_abort() diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_xet_progress_reporting.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_xet_progress_reporting.py new file mode 100644 index 0000000000000000000000000000000000000000..d60162eb2973d24c24e18f4970fb03b251e386d9 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/_xet_progress_reporting.py @@ -0,0 +1,353 @@ +from collections import OrderedDict +from typing import Any + +from . import is_google_colab, is_notebook +from .tqdm import _create_progress_bar, tqdm + + +def _format_speed_postfix(speed: float | None) -> str: + s = tqdm.format_sizeof(speed) if speed is not None else "???" + return f"{s}B/s ".rjust(10, " ") + + +# Transfer byte count is hard to predict (dedup/compression), so we omit a total and show bytes only. +XET_TRANSFER_BAR_FORMAT = "{desc}: {bar}| {n_fmt:>5}B{postfix:>12}" +XET_BYTES_BAR_FORMAT = "{l_bar}{bar}| {n_fmt:>5}B / {total_fmt:>5}B{postfix:>12}" + + +def _set_monotonic_total(bar, total: int | None) -> None: + if total is None or not hasattr(bar, "total"): + return + bar.total = max(bar.total or 0, total) + + +def _update_transfer_bar(bar, inc: int) -> None: + """Update the transfer bar and grow its hidden total so the bar graphic advances. + + Network bytes are hard to predict (dedup/compression), so the display omits a denominator. + tqdm still needs an internal total for the bar width — seeded from file size when known, + then expanded here if bytes received exceed that estimate. + """ + n_after = getattr(bar, "n", 0) + inc + current_total = getattr(bar, "total", 0) or 0 + if n_after > 0 and current_total < n_after: + bar.total = max(current_total, int(n_after * 1.25) + 1) + bar.update(inc) + + +def _finish_transfer_bar(bar) -> None: + """Snap the transfer bar to 100% when downloading stops. + + Transfer totals are seeded from file size, but actual network bytes are often lower. + Set ``total = n`` so the bar fills completely instead of stopping partway. + """ + n = getattr(bar, "n", 0) + if n > 0 and hasattr(bar, "total") and bar.total != n: + bar.total = n + bar.refresh() + + +def _set_aggregate_rate_postfix(bar) -> None: + """Show a shared bar's own throughput as its rate. + + When many files feed one bar (snapshot download), each file only knows its own per-item rate — a + fraction of the total. Deriving the rate from the shared bar's aggregated byte count reports the + true combined speed instead of whichever single file reported last. + """ + bar.set_postfix_str(_format_speed_postfix(bar.format_dict.get("rate")), refresh=False) + + +class XetDownloadProgressReporter: + """Dual progress bars for Xet downloads: network transfer and file reconstruction. + + ``total_transfer_bytes_completed`` tracks bytes received from the network (updated continuously). + ``total_bytes_completed`` tracks bytes written to disk (updated after buffered chunks are flushed). + Showing both bars gives responsive feedback on slow connections where reconstruction lags behind transfer. + """ + + def __init__( + self, + *, + reconstruction_desc: str, + transfer_desc: str = "Downloading bytes", + total: int | None = None, + log_level: int, + name: str | None = None, + tqdm_class: type | None = None, + external_reconstruction_bar: Any | None = None, + position: int = 0, + ): + self._prev_bytes_completed = 0 + self._prev_transfer_bytes_completed = 0 + + cls = tqdm_class or tqdm + routes_transfer_via_reconstruction = external_reconstruction_bar is not None and callable( + getattr(external_reconstruction_bar, "update_transfer", None) + ) + uses_aggregated_tqdm_class = external_reconstruction_bar is None and callable( + getattr(cls, "update_transfer", None) + ) + + if external_reconstruction_bar is not None: + self.reconstruction_bar = external_reconstruction_bar + self._owns_reconstruction_bar = False + else: + self.reconstruction_bar = _create_progress_bar( + cls=cls, # ty: ignore[invalid-argument-type] + log_level=log_level, + name=name, + desc=reconstruction_desc, + total=total, + unit="B", + unit_scale=True, + position=position + 1, + bar_format=XET_BYTES_BAR_FORMAT, + leave=True, + ) + self._owns_reconstruction_bar = True + + if routes_transfer_via_reconstruction or uses_aggregated_tqdm_class: + self.transfer_bar = self.reconstruction_bar + self._owns_transfer_bar = False + elif external_reconstruction_bar is not None: + self.transfer_bar = None + self._owns_transfer_bar = False + else: + self.transfer_bar = _create_progress_bar( + cls=tqdm, + log_level=log_level, + name=f"{name}.transfer" if name else None, + desc=transfer_desc, + total=total, + unit="B", + unit_scale=True, + position=position, + bar_format=XET_TRANSFER_BAR_FORMAT, + leave=True, + ) + self._owns_transfer_bar = True + + @property + def _aggregated(self) -> bool: + return self.transfer_bar is not None and self.transfer_bar is self.reconstruction_bar + + def update_progress(self, group_report, _item_reports: dict | None = None) -> None: + bytes_inc = max(0, group_report.total_bytes_completed - self._prev_bytes_completed) + transfer_inc = max(0, group_report.total_transfer_bytes_completed - self._prev_transfer_bytes_completed) + self._prev_bytes_completed = group_report.total_bytes_completed + self._prev_transfer_bytes_completed = group_report.total_transfer_bytes_completed + + if bytes_inc > 0: + self.reconstruction_bar.update(bytes_inc) + self.reconstruction_bar.set_postfix_str( + _format_speed_postfix(group_report.total_bytes_completion_rate), refresh=False + ) + + if transfer_inc > 0 and self.transfer_bar is not None: + if self._aggregated: + self.reconstruction_bar.update_transfer(transfer_inc) + self.reconstruction_bar.set_transfer_postfix_str( + _format_speed_postfix(group_report.total_transfer_bytes_completion_rate), refresh=False + ) + else: + _update_transfer_bar(self.transfer_bar, transfer_inc) + self.transfer_bar.set_postfix_str( + _format_speed_postfix(group_report.total_transfer_bytes_completion_rate), refresh=False + ) + + if group_report.total_bytes: + _set_monotonic_total(self.reconstruction_bar, group_report.total_bytes) + + def close(self) -> None: + """Close bars owned by this reporter. + + Standalone downloads finish the transfer bar first (snap hidden total to ``n``), then close it. + Aggregated and external bars (e.g. snapshot ``_AggregatedTqdm`` or a reused ``_tqdm_bar``) are left + open — their parent caller owns their lifecycle. + """ + if self.transfer_bar is not None and self._owns_transfer_bar: + _finish_transfer_bar(self.transfer_bar) + if hasattr(self.transfer_bar, "close"): + self.transfer_bar.close() + if self._owns_reconstruction_bar and hasattr(self.reconstruction_bar, "close"): + self.reconstruction_bar.close() + + def __enter__(self) -> "XetDownloadProgressReporter": + return self + + def __exit__(self, *args) -> None: + self.close() + + +class XetUploadProgressReporter: + """ + Reports on progress for Xet uploads. + + Shows summary progress bars when running in notebooks or GUIs, and detailed per-file progress in console environments. + """ + + def __init__(self, n_lines: int = 10, description_width: int = 30, total_files: int | None = None): + self.n_lines = n_lines + self.description_width = description_width + self.total_files = total_files + + self.per_file_progress = is_google_colab() or not is_notebook() + + self.tqdm_settings = { + "unit": "B", + "unit_scale": True, + "leave": True, + "unit_divisor": 1000, + "nrows": n_lines + 3 if self.per_file_progress else 3, + "miniters": 1, + "bar_format": XET_BYTES_BAR_FORMAT, + } + + # Overall progress bars + self.data_processing_bar = tqdm( + total=0, desc=self.format_desc("Processing Files (0 / 0)", False), position=0, **self.tqdm_settings + ) + + self.upload_bar = tqdm( + total=0, desc=self.format_desc("New Data Upload", False), position=1, **self.tqdm_settings + ) + + self.known_items: set[str] = set() + self.completed_items: set[str] = set() + + # Track previous absolute values to compute increments + self._prev_bytes_completed: int = 0 + self._prev_transfer_bytes_completed: int = 0 + + # Item bars (scrolling view) + self.item_state: OrderedDict[str, Any] = OrderedDict() + self.current_bars: list = [None] * self.n_lines + + def format_desc(self, name: str, indent: bool) -> str: + """ + if name is longer than width characters, prints ... at the start and then the last width-3 characters of the name, otherwise + the whole name right justified into description_width characters. Also adds some padding. + """ + + if not self.per_file_progress: + # Here we just use the defaults. + return name + + padding = " " if indent else "" + width = self.description_width - len(padding) + + if len(name) > width: + name = f"...{name[-(width - 3) :]}" + + return f"{padding}{name.ljust(width)}" + + def reset_for_next_commit(self): + """Reset per-commit state so the reporter can be reused across multiple upload commits.""" + self._prev_bytes_completed = 0 + self._prev_transfer_bytes_completed = 0 + self.known_items.clear() + self.completed_items.clear() + self.item_state.clear() + + def update_progress(self, group_report, item_reports: dict): + # Update all the per-item values. + for item in item_reports.values(): + item_name = item.item_name + + self.known_items.add(item_name) + + # Only care about items where the processing has already started. + if item.bytes_completed == 0: + continue + + # Overwrite the existing value in there. + self.item_state[item_name] = item + + bar_idx = 0 + new_completed = [] + + # Now, go through and update all the bars + for name, item in self.item_state.items(): + # Is this ready to be removed on the next update? + if item.bytes_completed == item.total_bytes: + self.completed_items.add(name) + new_completed.append(name) + + # If we're only showing summary information, then don't update the individual bars + if not self.per_file_progress: + continue + + # If we've run out of bars to use, then collapse the last ones together. + if bar_idx >= len(self.current_bars): + bar = self.current_bars[-1] + in_final_bar_mode = True + final_bar_aggregation_count = bar_idx + 1 - len(self.current_bars) + else: + bar = self.current_bars[bar_idx] + in_final_bar_mode = False + + if bar is None: + self.current_bars[bar_idx] = tqdm( + desc=self.format_desc(name, True), + position=2 + bar_idx, # Set to the position past the initial bars. + total=item.total_bytes, + initial=item.bytes_completed, + **self.tqdm_settings, + ) + + elif in_final_bar_mode: + bar.n += item.bytes_completed + bar.total += item.total_bytes + bar.set_description(self.format_desc(f"[+ {final_bar_aggregation_count} files]", True), refresh=False) + else: + bar.set_description(self.format_desc(name, True), refresh=False) + bar.n = item.bytes_completed + bar.total = item.total_bytes + + bar_idx += 1 + + # Remove all the completed ones from the ordered dictionary + for name in new_completed: + # Only remove ones from consideration to make room for more items coming in. + if len(self.item_state) <= self.n_lines: + break + + del self.item_state[name] + + if self.per_file_progress: + # Now manually refresh each of the bars + for bar in self.current_bars: + if bar: + bar.refresh() + + # Update overall bars + bytes_inc = max(0, group_report.total_bytes_completed - self._prev_bytes_completed) + transfer_inc = max(0, group_report.total_transfer_bytes_completed - self._prev_transfer_bytes_completed) + self._prev_bytes_completed = group_report.total_bytes_completed + self._prev_transfer_bytes_completed = group_report.total_transfer_bytes_completed + + self.data_processing_bar.total = group_report.total_bytes + total_files_count = self.total_files if self.total_files is not None else len(self.known_items) + self.data_processing_bar.set_description( + self.format_desc(f"Processing Files ({len(self.completed_items)} / {total_files_count})", False), + refresh=False, + ) + self.data_processing_bar.set_postfix_str( + _format_speed_postfix(group_report.total_bytes_completion_rate), refresh=False + ) + self.data_processing_bar.update(bytes_inc) + + self.upload_bar.total = group_report.total_transfer_bytes + self.upload_bar.set_postfix_str( + _format_speed_postfix(group_report.total_transfer_bytes_completion_rate), refresh=False + ) + self.upload_bar.update(transfer_inc) + + def close(self): + self.data_processing_bar.close() + self.upload_bar.close() + + if self.per_file_progress: + for bar in self.current_bars: + if bar: + bar.close() diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/endpoint_helpers.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/endpoint_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..85cd86011b78bcdc57034aeebc3c01e9e721ab50 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/endpoint_helpers.py @@ -0,0 +1,66 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Helpful utility functions and classes in relation to exploring API endpoints +with the aim for a user-friendly interface. +""" + +import math +import re +from typing import TYPE_CHECKING + +from ..repocard_data import ModelCardData + + +if TYPE_CHECKING: + from ..hf_api import ModelInfo + + +def _is_emission_within_threshold(model_info: "ModelInfo", minimum_threshold: float, maximum_threshold: float) -> bool: + """Checks if a model's emission is within a given threshold. + + Args: + model_info (`ModelInfo`): + A model info object containing the model's emission information. + minimum_threshold (`float`): + A minimum carbon threshold to filter by, such as 1. + maximum_threshold (`float`): + A maximum carbon threshold to filter by, such as 10. + + Returns: + `bool`: Whether the model's emission is within the given threshold. + """ + if minimum_threshold is None and maximum_threshold is None: + raise ValueError("Both `minimum_threshold` and `maximum_threshold` cannot both be `None`") + if minimum_threshold is None: + minimum_threshold = -1 + if maximum_threshold is None: + maximum_threshold = math.inf + + card_data = getattr(model_info, "card_data", None) + if card_data is None or not isinstance(card_data, (dict, ModelCardData)): + return False + + # Get CO2 emission metadata + emission = card_data.get("co2_eq_emissions", None) + if isinstance(emission, dict): + emission = emission["emissions"] + if not emission: + return False + + # Filter out if value is missing or out of range + matched = re.search(r"\d+\.\d+|\d+", str(emission)) + if matched is None: + return False + + emission_value = float(matched.group(0)) + return minimum_threshold <= emission_value <= maximum_threshold diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/insecure_hashlib.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/insecure_hashlib.py new file mode 100644 index 0000000000000000000000000000000000000000..639e04460b0338d4642ea522490bedf038bfb557 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/insecure_hashlib.py @@ -0,0 +1,32 @@ +# Taken from https://github.com/mlflow/mlflow/pull/10119 +# +# DO NOT use this function for security purposes (e.g., password hashing). +# +# In Python >= 3.9, insecure hashing algorithms such as MD5 fail in FIPS-compliant +# environments unless `usedforsecurity=False` is explicitly passed. +# +# References: +# - https://github.com/mlflow/mlflow/issues/9905 +# - https://github.com/mlflow/mlflow/pull/10119 +# - https://docs.python.org/3/library/hashlib.html +# - https://github.com/huggingface/transformers/pull/27038 +# +# Usage: +# ```python +# # Use +# from huggingface_hub.utils.insecure_hashlib import sha256 +# # instead of +# from hashlib import sha256 +# +# # Use +# from huggingface_hub.utils import insecure_hashlib +# # instead of +# import hashlib +# ``` +import functools +import hashlib + + +md5 = functools.partial(hashlib.md5, usedforsecurity=False) +sha1 = functools.partial(hashlib.sha1, usedforsecurity=False) +sha256 = functools.partial(hashlib.sha256, usedforsecurity=False) diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/logging.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..48d6317fa7e6ad6083c6a4d486f0cab456d0c1da --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/logging.py @@ -0,0 +1,183 @@ +# Copyright 2020 Optuna, Hugging Face +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Logging utilities.""" + +import logging +import os +from logging import ( + CRITICAL, # NOQA + DEBUG, # NOQA + ERROR, # NOQA + FATAL, # NOQA + INFO, # NOQA + NOTSET, # NOQA + WARN, # NOQA + WARNING, # NOQA +) + +from .. import constants + + +log_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +_default_log_level = logging.WARNING + + +def _get_library_name() -> str: + return __name__.split(".")[0] + + +def _get_library_root_logger() -> logging.Logger: + return logging.getLogger(_get_library_name()) + + +def _get_default_logging_level(): + """ + If `HF_HUB_VERBOSITY` env var is set to one of the valid choices return that as the new default level. If it is not + - fall back to `_default_log_level` + """ + env_level_str = os.getenv("HF_HUB_VERBOSITY", None) + if env_level_str: + if env_level_str in log_levels: + return log_levels[env_level_str] + else: + logging.getLogger().warning( + f"Unknown option HF_HUB_VERBOSITY={env_level_str}, has to be one of: {', '.join(log_levels.keys())}" + ) + return _default_log_level + + +def _configure_library_root_logger() -> None: + library_root_logger = _get_library_root_logger() + library_root_logger.addHandler(logging.StreamHandler()) + library_root_logger.setLevel(_get_default_logging_level()) + + +def _reset_library_root_logger() -> None: + library_root_logger = _get_library_root_logger() + library_root_logger.setLevel(logging.NOTSET) + + +def get_logger(name: str | None = None) -> logging.Logger: + """ + Returns a logger with the specified name. This function is not supposed + to be directly accessed by library users. + + Args: + name (`str`, *optional*): + The name of the logger to get, usually the filename + + Example: + + ```python + >>> from huggingface_hub import get_logger + + >>> logger = get_logger(__file__) + >>> logger.set_verbosity_info() + ``` + """ + + if name is None: + name = _get_library_name() + + return logging.getLogger(name) + + +def get_verbosity() -> int: + """Return the current level for the HuggingFace Hub's root logger. + + Returns: + Logging level, e.g., `huggingface_hub.logging.DEBUG` and + `huggingface_hub.logging.INFO`. + + > [!TIP] + > HuggingFace Hub has following logging levels: + > + > - `huggingface_hub.logging.CRITICAL`, `huggingface_hub.logging.FATAL` + > - `huggingface_hub.logging.ERROR` + > - `huggingface_hub.logging.WARNING`, `huggingface_hub.logging.WARN` + > - `huggingface_hub.logging.INFO` + > - `huggingface_hub.logging.DEBUG` + """ + return _get_library_root_logger().getEffectiveLevel() + + +def set_verbosity(verbosity: int) -> None: + """ + Sets the level for the HuggingFace Hub's root logger. + + Args: + verbosity (`int`): + Logging level, e.g., `huggingface_hub.logging.DEBUG` and + `huggingface_hub.logging.INFO`. + """ + _get_library_root_logger().setLevel(verbosity) + + +def set_verbosity_info(): + """ + Sets the verbosity to `logging.INFO`. + """ + return set_verbosity(INFO) + + +def set_verbosity_warning(): + """ + Sets the verbosity to `logging.WARNING`. + """ + return set_verbosity(WARNING) + + +def set_verbosity_debug(): + """ + Sets the verbosity to `logging.DEBUG`. + """ + return set_verbosity(DEBUG) + + +def set_verbosity_error(): + """ + Sets the verbosity to `logging.ERROR`. + """ + return set_verbosity(ERROR) + + +def disable_propagation() -> None: + """ + Disable propagation of the library log outputs. Note that log propagation is + disabled by default. + """ + _get_library_root_logger().propagate = False + + +def enable_propagation() -> None: + """ + Enable propagation of the library log outputs. Please disable the + HuggingFace Hub's default handler to prevent double logging if the root + logger has been configured. + """ + _get_library_root_logger().propagate = True + + +_configure_library_root_logger() + +if constants.HF_DEBUG: + # If `HF_DEBUG` environment variable is set, set the verbosity of `huggingface_hub` logger to `DEBUG`. + set_verbosity_debug() diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/sha.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/sha.py new file mode 100644 index 0000000000000000000000000000000000000000..dc2f8f614c9ec02579153eb3308603f7ff24adef --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/sha.py @@ -0,0 +1,64 @@ +"""Utilities to efficiently compute the SHA 256 hash of a bunch of bytes.""" + +from typing import BinaryIO + +from .insecure_hashlib import sha1, sha256 + + +def sha_fileobj(fileobj: BinaryIO, chunk_size: int | None = None) -> bytes: + """ + Computes the sha256 hash of the given file object, by chunks of size `chunk_size`. + + Args: + fileobj (file-like object): + The File object to compute sha256 for, typically obtained with `open(path, "rb")` + chunk_size (`int`, *optional*): + The number of bytes to read from `fileobj` at once, defaults to 1MB. + + Returns: + `bytes`: `fileobj`'s sha256 hash as bytes + """ + chunk_size = chunk_size if chunk_size is not None else 1024 * 1024 + + sha = sha256() + while True: + chunk = fileobj.read(chunk_size) + sha.update(chunk) + if not chunk: + break + return sha.digest() + + +def git_hash(data: bytes) -> str: + """ + Computes the git-sha1 hash of the given bytes, using the same algorithm as git. + + This is equivalent to running `git hash-object`. See https://git-scm.com/docs/git-hash-object + for more details. + + Note: this method is valid for regular files. For LFS files, the proper git hash is supposed to be computed on the + pointer file content, not the actual file content. However, for simplicity, we directly compare the sha256 of + the LFS file content when we want to compare LFS files. + + Args: + data (`bytes`): + The data to compute the git-hash for. + + Returns: + `str`: the git-hash of `data` as an hexadecimal string. + + Example: + ```python + >>> from huggingface_hub.utils.sha import git_hash + >>> git_hash(b"Hello, World!") + 'b45ef6fec89518d314f546fd6c3025367b721684' + ``` + """ + # Taken from https://gist.github.com/msabramo/763200 + # Note: no need to optimize by reading the file in chunks as we're not supposed to hash huge files (5MB maximum). + sha = sha1() + sha.update(b"blob ") + sha.update(str(len(data)).encode()) + sha.update(b"\0") + sha.update(data) + return sha.hexdigest() diff --git a/.venv/lib/python3.13/site-packages/huggingface_hub/utils/tqdm.py b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/tqdm.py new file mode 100644 index 0000000000000000000000000000000000000000..cc111f02da0abe1ea864ad64534dd38d8ee8f4b8 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/huggingface_hub/utils/tqdm.py @@ -0,0 +1,449 @@ +# Copyright 2021 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +"""Utility helpers to handle progress bars in `huggingface_hub`. + +Example: + 1. Use `huggingface_hub.utils.tqdm` as you would use `tqdm.tqdm` or `tqdm.auto.tqdm`. + 2. To disable progress bars, either use `disable_progress_bars()` helper or set the + environment variable `HF_HUB_DISABLE_PROGRESS_BARS` to 1. + 3. To re-enable progress bars, use `enable_progress_bars()`. + 4. To check whether progress bars are disabled, use `are_progress_bars_disabled()`. + +NOTE: Environment variable `HF_HUB_DISABLE_PROGRESS_BARS` has the priority. + +Example: + ```py + >>> from huggingface_hub.utils import are_progress_bars_disabled, disable_progress_bars, enable_progress_bars, tqdm + + # Disable progress bars globally + >>> disable_progress_bars() + + # Use as normal `tqdm` + >>> for _ in tqdm(range(5)): + ... pass + + # Still not showing progress bars, as `disable=False` is overwritten to `True`. + >>> for _ in tqdm(range(5), disable=False): + ... pass + + >>> are_progress_bars_disabled() + True + + # Re-enable progress bars globally + >>> enable_progress_bars() + + # Progress bar will be shown ! + >>> for _ in tqdm(range(5)): + ... pass + 100%|███████████████████████████████████████| 5/5 [00:00<00:00, 117817.53it/s] + ``` + +Group-based control: + ```python + # Disable progress bars for a specific group + >>> disable_progress_bars("peft.foo") + + # Check state of different groups + >>> assert not are_progress_bars_disabled("peft")) + >>> assert not are_progress_bars_disabled("peft.something") + >>> assert are_progress_bars_disabled("peft.foo")) + >>> assert are_progress_bars_disabled("peft.foo.bar")) + + # Enable progress bars for a subgroup + >>> enable_progress_bars("peft.foo.bar") + + # Check if enabling a subgroup affects the parent group + >>> assert are_progress_bars_disabled("peft.foo")) + >>> assert not are_progress_bars_disabled("peft.foo.bar")) + + # No progress bar for `name="peft.foo"` + >>> for _ in tqdm(range(5), name="peft.foo"): + ... pass + + # Progress bar will be shown for `name="peft.foo.bar"` + >>> for _ in tqdm(range(5), name="peft.foo.bar"): + ... pass + 100%|███████████████████████████████████████| 5/5 [00:00<00:00, 117817.53it/s] + + ``` +""" + +import io +import logging +import os +import threading +import warnings +from collections.abc import Iterable, Iterator +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager, nullcontext +from pathlib import Path +from typing import Callable, ContextManager, TypeVar, cast + +from tqdm.auto import tqdm as old_tqdm + +from ..constants import HF_HUB_DISABLE_PROGRESS_BARS + + +# The `HF_HUB_DISABLE_PROGRESS_BARS` environment variable can be True, False, or not set (None), +# allowing for control over progress bar visibility. When set, this variable takes precedence +# over programmatic settings, dictating whether progress bars should be shown or hidden globally. +# Essentially, the environment variable's setting overrides any code-based configurations. +# +# If `HF_HUB_DISABLE_PROGRESS_BARS` is not defined (None), it implies that users can manage +# progress bar visibility through code. By default, progress bars are turned on. + +progress_bar_states: dict[str, bool] = {} + + +class disable_progress_bars: + """ + Disable progress bars either globally or for a specified group. + + This function updates the state of progress bars based on a group name. + If no group name is provided, all progress bars are disabled. The operation + respects the `HF_HUB_DISABLE_PROGRESS_BARS` environment variable's setting. + + Works as both a regular call and a context manager: + disable_progress_bars() # disables until enable_progress_bars() + with disable_progress_bars(): # disables for the block, re-enables on exit + ... + + Args: + name (`str`, *optional*): + The name of the group for which to disable the progress bars. If None, + progress bars are disabled globally. + + Raises: + Warning: If the environment variable precludes changes. + """ + + def __init__(self, name: str | None = None) -> None: + self.name = name + + if HF_HUB_DISABLE_PROGRESS_BARS is False: + warnings.warn( + "Cannot disable progress bars: environment variable `HF_HUB_DISABLE_PROGRESS_BARS=0` is set and has priority." + ) + self._should_reenable = False + return + + self._should_reenable = not are_progress_bars_disabled(name) + if name is None: + progress_bar_states.clear() + progress_bar_states["_global"] = False + else: + keys_to_remove = [key for key in progress_bar_states if key.startswith(f"{name}.")] + for key in keys_to_remove: + del progress_bar_states[key] + progress_bar_states[name] = False + + def __enter__(self) -> "disable_progress_bars": + return self + + def __exit__(self, *exc) -> None: + if self._should_reenable: + enable_progress_bars(self.name) + + +def enable_progress_bars(name: str | None = None) -> None: + """ + Enable progress bars either globally or for a specified group. + + This function sets the progress bars to enabled for the specified group or globally + if no group is specified. The operation is subject to the `HF_HUB_DISABLE_PROGRESS_BARS` + environment setting. + + Args: + name (`str`, *optional*): + The name of the group for which to enable the progress bars. If None, + progress bars are enabled globally. + + Raises: + Warning: If the environment variable precludes changes. + """ + if HF_HUB_DISABLE_PROGRESS_BARS is True: + warnings.warn( + "Cannot enable progress bars: environment variable `HF_HUB_DISABLE_PROGRESS_BARS=1` is set and has priority." + ) + return + + if name is None: + progress_bar_states.clear() + progress_bar_states["_global"] = True + else: + keys_to_remove = [key for key in progress_bar_states if key.startswith(f"{name}.")] + for key in keys_to_remove: + del progress_bar_states[key] + progress_bar_states[name] = True + + +def are_progress_bars_disabled(name: str | None = None) -> bool: + """ + Check if progress bars are disabled globally or for a specific group. + + This function returns whether progress bars are disabled for a given group or globally. + It checks the `HF_HUB_DISABLE_PROGRESS_BARS` environment variable first, then the programmatic + settings. + + Args: + name (`str`, *optional*): + The group name to check; if None, checks the global setting. + + Returns: + `bool`: True if progress bars are disabled, False otherwise. + """ + if HF_HUB_DISABLE_PROGRESS_BARS is True: + return True + + if name is None: + return not progress_bar_states.get("_global", True) + + while name: + if name in progress_bar_states: + return not progress_bar_states[name] + name = ".".join(name.split(".")[:-1]) + + return not progress_bar_states.get("_global", True) + + +def is_tqdm_disabled(log_level: int) -> bool | None: + """ + Determine if tqdm progress bars should be disabled based on logging level and environment settings. + + see https://github.com/huggingface/huggingface_hub/pull/2000 and https://github.com/huggingface/huggingface_hub/pull/2698. + """ + if log_level == logging.NOTSET: + return True + if os.getenv("TQDM_POSITION") == "-1": + return False + return None + + +class tqdm(old_tqdm): + """ + Class to override `disable` argument in case progress bars are globally disabled. + + Taken from https://github.com/tqdm/tqdm/issues/619#issuecomment-619639324. + """ + + def __init__(self, *args, **kwargs): + name = kwargs.pop("name", None) # do not pass `name` to `tqdm` + if are_progress_bars_disabled(name): + kwargs["disable"] = True + super().__init__(*args, **kwargs) + + def __delattr__(self, attr: str) -> None: + """Fix for https://github.com/huggingface/huggingface_hub/issues/1603""" + try: + super().__delattr__(attr) + except AttributeError: + if attr != "_lock": + raise + + +# Prevent tqdm's default multiprocessing write-lock from spawning a resource +# tracker subprocess via fork_exec(). That path fails when stderr has an invalid +# fd (e.g. Textual TUIs that return -1 from sys.stderr.fileno()). Inter-process +# bar coordination on the HF subclass is not a supported use case. See #4065. +tqdm.set_lock(threading.RLock()) + + +class silent_tqdm: + """Fake tqdm object that does nothing.""" + + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + pass + + def update(self, n: int | float | None = 1) -> None: + pass + + +@contextmanager +def tqdm_stream_file(path: Path | str) -> Iterator[io.BufferedReader]: + """ + Open a file as binary and wrap the `read` method to display a progress bar when it's streamed. + + First implemented in `transformers` in 2019 but removed when switched to git-lfs. Used in `huggingface_hub` to show + progress bar when uploading an LFS file to the Hub. See github.com/huggingface/transformers/pull/2078#discussion_r354739608 + for implementation details. + + Note: currently implementation handles only files stored on disk as it is the most common use case. Could be + extended to stream any `BinaryIO` object but we might have to debug some corner cases. + + Example: + ```py + >>> with tqdm_stream_file("config.json") as f: + >>> httpx.put(url, data=f) + config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s] + ``` + """ + if isinstance(path, str): + path = Path(path) + + with path.open("rb") as f: + total_size = path.stat().st_size + pbar = tqdm( + unit="B", + unit_scale=True, + total=total_size, + initial=0, + desc=path.name, + ) + + f_read = f.read + + def _inner_read(size: int | None = -1) -> bytes: + data = f_read(size) + pbar.update(len(data)) + return data + + f.read = _inner_read # type: ignore + + yield f + + pbar.close() + + +def _create_progress_bar(*, cls: type[old_tqdm], log_level: int, name: str | None = None, **kwargs) -> old_tqdm: + """Create a progress bar. + + For our `tqdm` subclass (or subclasses of it): respects all disable signals + (`HF_HUB_DISABLE_PROGRESS_BARS`, `disable_progress_bars()`, log level) and uses + `disable=None` for TTY auto-detection (see https://github.com/huggingface/huggingface_hub/pull/2000), + unless `TQDM_POSITION=-1` forces bars on (https://github.com/huggingface/huggingface_hub/pull/2698). + + For other classes: does not inject `disable` or `name`. the custom class is fully + responsible for its own behavior. Vanilla tqdm defaults to `disable=False` (bar shows). + Omits `name` which vanilla tqdm rejects with `TqdmKeyError`. See https://github.com/huggingface/huggingface_hub/issues/4050. + """ + # issubclass() crashes on non-class callables (e.g. functools.partial), guard with isinstance. + if not (isinstance(cls, type) and issubclass(cls, tqdm)): + return cls(**kwargs) # type: ignore[return-value] + + # HF subclass: keep the historical log-level / TTY behavior. Group-based + # disabling is already handled in `tqdm.__init__`. + disable = is_tqdm_disabled(log_level) + return cls(disable=disable, name=name, **kwargs) # type: ignore[return-value] + + +def _get_progress_bar_context( + *, + desc: str, + log_level: int, + total: int | None = None, + initial: int = 0, + unit: str = "B", + unit_scale: bool = True, + name: str | None = None, + tqdm_class: type[old_tqdm] | None = None, + _tqdm_bar: tqdm | None = None, +) -> ContextManager[tqdm]: + if _tqdm_bar is not None: + return nullcontext(_tqdm_bar) + # ^ `contextlib.nullcontext` mimics a context manager that does nothing + # Makes it easier to use the same code path for both cases but in the later + # case, the progress bar is not closed when exiting the context manager. + + return _create_progress_bar( # type: ignore + cls=tqdm_class or tqdm, + log_level=log_level, + name=name, + unit=unit, + unit_scale=unit_scale, + total=total, + initial=initial, + desc=desc, + ) + + +T = TypeVar("T") +R = TypeVar("R") + + +def _make_thread_pool_executor(*, max_workers: int | None, bar_class: type[old_tqdm]) -> ThreadPoolExecutor: + """Create a `ThreadPoolExecutor` that shares the tqdm lock with its worker threads. + + Like `tqdm.contrib.concurrent.thread_map`, we make sure the bar class has a lock and pass it to the + worker threads, so that bars created inside the mapped function (e.g. the per-file download bars in + `snapshot_download`) don't race on concurrent `update()` calls. `get_lock()` creates the shared + class-level lock if needed. Classes that don't implement tqdm's locking API (e.g. `silent_tqdm`) + render nothing anyway, so no lock is needed. + """ + if hasattr(bar_class, "get_lock") and hasattr(bar_class, "set_lock"): + return ThreadPoolExecutor( + max_workers=max_workers, initializer=bar_class.set_lock, initargs=(bar_class.get_lock(),) + ) + return ThreadPoolExecutor(max_workers=max_workers) + + +def hf_thread_map( + fn: Callable[[T], R], + iterable: Iterable[T], + *, + max_workers: int | None = None, + tqdm_class: type[old_tqdm] | None = None, + **tqdm_kwargs, +) -> list[R]: + """Drop-in replacement for `tqdm.contrib.concurrent.thread_map`. + + The version shipped by `tqdm` consumes results in submission order and the progress bar only advances when the "next + item in input order" finishes. A single slow early item pins the bar near zero while later items + complete, with everything flushed at once at the end. This helper avoids that problem by consuming items in completion + order. + + See https://github.com/huggingface/huggingface_hub/issues/4518 for more details. + + Args: + fn (`Callable`): + Function to apply to each item. + iterable (`Iterable`): + Items to process. + max_workers (`int`, *optional*): + Maximum number of worker threads. Defaults to tqdm's own default. + tqdm_class (`type`, *optional*): + Progress bar class to use. Defaults to `huggingface_hub`'s `tqdm`. + **tqdm_kwargs: + Additional keyword arguments forwarded to the progress bar (e.g. `desc`). + + Returns: + `list`: Results in the same order as `iterable`. + """ + items = list(iterable) + if max_workers is None: + max_workers = min(32, (os.cpu_count() or 1) + 4) + tqdm_kwargs.setdefault("total", len(items)) + bar_class = tqdm_class or tqdm + results: list[R | None] = [None] * len(items) + with ( + bar_class(**tqdm_kwargs) as pbar, + _make_thread_pool_executor(max_workers=max_workers, bar_class=bar_class) as executor, + ): + future_to_index = {executor.submit(fn, item): index for index, item in enumerate(items)} + try: + for future in as_completed(future_to_index): + results[future_to_index[future]] = future.result() + pbar.update(1) + except BaseException: + # Cancel not-yet-started tasks so a single failure doesn't keep the whole queue running + # (mirrors `tqdm.contrib.concurrent.thread_map`, which relies on `Executor.map` for this). + for future in future_to_index: + future.cancel() + raise + return cast("list[R]", results) diff --git a/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/INSTALLER b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/METADATA b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..6c4bf89b25c79af2e9ecdc22c5ddb802effa85bc --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/METADATA @@ -0,0 +1,155 @@ +Metadata-Version: 2.4 +Name: idna +Version: 3.18 +Summary: Internationalized Domain Names in Applications (IDNA) +Author-email: Kim Davies +Requires-Python: >=3.9 +Description-Content-Type: text/markdown +License-Expression: BSD-3-Clause +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: System Administrators +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Internet :: Name Service (DNS) +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Utilities +License-File: LICENSE.md +Requires-Dist: ruff >= 0.6.2 ; extra == "all" +Requires-Dist: mypy >= 1.11.2 ; extra == "all" +Requires-Dist: pytest >= 8.3.2 ; extra == "all" +Project-URL: Changelog, https://github.com/kjd/idna/blob/master/HISTORY.md +Project-URL: Issue tracker, https://github.com/kjd/idna/issues +Project-URL: Source, https://github.com/kjd/idna +Provides-Extra: all + +# Internationalized Domain Names in Applications (IDNA) + +Support for [Internationalized Domain Names in Applications +(IDNA)](https://tools.ietf.org/html/rfc5891) and [Unicode IDNA +Compatibility Processing](https://unicode.org/reports/tr46/). It +supersedes the standard library's `encodings.idna`, which only +implements the 2003 specification, offering broader script coverage and +limiting domains with known security vulnerabilities. + +## Usage + +Package may be installed from [PyPI](https://pypi.org/project/idna/) via +the typical methods (e.g. `python3 -m pip install idna`) + +For typical usage, the `encode` and `decode` functions will take a +domain name argument and perform a conversion to ASCII-compatible encoding +(known as A-labels), or to Unicode strings (known as U-labels) +respectively. + +```pycon +>>> import idna +>>> idna.encode('ドメイン.テスト') +b'xn--eckwd4c7c.xn--zckzah' +>>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) +ドメイン.テスト +``` + +Conversions can be applied at a per-label basis using the `ulabel` or +`alabel` functions for specialized use cases. + + +### Compatibility Mapping (UTS #46) + +This library provides support for [Unicode IDNA Compatibility +Processing](https://unicode.org/reports/tr46/) which normalizes input from +different potential ways a user may input a domain prior to performing the IDNA +conversion operations. This functionality, known as a +[mapping](https://tools.ietf.org/html/rfc5895), is considered by the +specification to be a local user-interface issue distinct from IDNA +conversion functionality. + +For example, "Königsgäßchen" is not a permissible label as capital letters +are not allowed. UTS 46 will convert this into lower case prior to applying +the IDNA conversion. + +```pycon +>>> import idna +>>> idna.encode('Königsgäßchen') +... +idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed +>>> idna.encode('Königsgäßchen', uts46=True) +b'xn--knigsgchen-b4a3dun' +>>> idna.decode('xn--knigsgchen-b4a3dun') +'königsgäßchen' +``` + +When performing a decode operation for display purposes, `decode()` +accepts a `display=True` argument that leaves any `xn--` label that +fails to decode unchanged. This is useful for user interface display +where a domain is in use, the A-label form can be presented when it +is not a valid IDN. + + +## Exceptions + +All errors raised during conversion derive from the `idna.IDNAError` +base class. The more specific exceptions are: + +* `idna.IDNABidiError` — raised when a label contains an illegal + combination of left-to-right and right-to-left characters. +* `idna.InvalidCodepoint` — raised when a label contains a codepoint + that is INVALID for IDNA. +* `idna.InvalidCodepointContext` — raised when a CONTEXTO or CONTEXTJ + codepoint appears in a position whose contextual requirements are + not satisfied. + + +## Command-line tool + +The package supports command-line usage to convert domain names +between their Unicode and ASCII-compatible forms. It can be run either +as a module (`python3 -m idna`) or, once installed (such as with `uv +tool` or `pipx`), via the `idna` script: + +```bash +$ uv tool install idna +$ idna xn--e1afmkfd.xn--p1ai +пример.рф +$ idna пример.рф +xn--e1afmkfd.xn--p1ai +``` + +Mode can be specified with `-e`/`--encode` or `-d`/`--decode`, otherwise +it will be chosen automatically based on the first input. Multiple +domains can be supplied either as arguments or through standard input. +UTS #46 mapping is applied by default, which lets the tool accept +inputs that aren't strictly valid IDNA 2008 by normalising them first, +pass `--strict` to disable UTS #46. + +Conversion failures are reported on stderr together with the +offending input; processing continues with the remaining domains and +the tool exits with a non-zero status if any conversion failed. + + +## Additional Notes + +* **Version support**. This library supports Python 3.9 and higher. + As this library serves as a low-level toolkit for a variety of + applications, we strive to support all versions of Python that are + not beyond end-of-life. + +* **Emoji**. It is an occasional request to support emoji domains in + this library. Encoding of symbols like emoji is expressly prohibited by + the IDNA technical standard, and emoji domains are broadly phased + out across the domain industry due to associated security risks. + +* **Regenerating lookup tables**. The IDNA and UTS 46 functionality + relies upon pre-calculated lookup tables, generated using the + `idna-data` script in [`tools/`](tools/README.md). + diff --git a/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/RECORD b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..6259e38a95e9958a88a0074b44db57ccfbb1e157 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/RECORD @@ -0,0 +1,28 @@ +../../../bin/idna,sha256=nqgOBnFP3X5l32Q4ip-Y-zlzBeHU7uM_9DX2_4nb33Q,259 +idna-3.18.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +idna-3.18.dist-info/METADATA,sha256=Rt_m5axGLQ9oDs2avPZugptqIzSCS02eOXmzETXK8oE,6119 +idna-3.18.dist-info/RECORD,, +idna-3.18.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 +idna-3.18.dist-info/entry_points.txt,sha256=7H3nGOHap3jnLE5e7q7Ywr9Vq8axB7WIj5-C_4N2vhw,38 +idna-3.18.dist-info/licenses/LICENSE.md,sha256=GppPDj1HmickDd1ZqRN6ZqtKD539yMphiMwL_YUYfwQ,1541 +idna/__init__.py,sha256=MPqNDLZbXqGaNdXxAFhiqFPKEQXju2jNQhCey6-5eJM,868 +idna/__main__.py,sha256=4JMK66Wj4uLZTKbF-sT3LAxOsr6buig77PmOkJCRRxw,83 +idna/__pycache__/__init__.cpython-313.pyc,, +idna/__pycache__/__main__.cpython-313.pyc,, +idna/__pycache__/cli.cpython-313.pyc,, +idna/__pycache__/codec.cpython-313.pyc,, +idna/__pycache__/compat.cpython-313.pyc,, +idna/__pycache__/core.cpython-313.pyc,, +idna/__pycache__/idnadata.cpython-313.pyc,, +idna/__pycache__/intranges.cpython-313.pyc,, +idna/__pycache__/package_data.cpython-313.pyc,, +idna/__pycache__/uts46data.cpython-313.pyc,, +idna/cli.py,sha256=swqJLMNc8Uzs60KziNpbWnHuqlG3WRQwJSbo4n8xDAo,4139 +idna/codec.py,sha256=JRbo-f7pEkLdWeiH89Z72UR4VBYhvKDFrQBeNX6sRDE,5040 +idna/compat.py,sha256=AepA39ceRHxkfHP41-FvKW5Ki-f4PfUZ90RUMlCNdmo,1353 +idna/core.py,sha256=SfOr1xO3PoE0RDYx7bMciAnjiyjJPbPw_93AB5IUYOw,24685 +idna/idnadata.py,sha256=Af-mo8WBmkhAK6TyXKOQH88OX0mQNDKtdL7UWtQpppk,44862 +idna/intranges.py,sha256=g49scLSkqJtAhLmOODa7hVHriSjmb60tiTsEoocJdBI,1851 +idna/package_data.py,sha256=TeI94EqAFAFaXfBJwsOPUMLn2969uirPa-DaeaceAyU,21 +idna/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +idna/uts46data.py,sha256=jujNz5QqWMcJf-XYLv4X1jBvb5FlI0t6-e1mILsgbPk,234325 diff --git a/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/WHEEL b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..d8b9936dad9ab2513fa6979f411560d3b6b57e37 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: flit 3.12.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/entry_points.txt b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..59ca7ac02ea0a40938ca867305e020981f52a077 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +idna=idna.cli:main + diff --git a/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/licenses/LICENSE.md b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/licenses/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..f706835ab3a5dd709a6e1f57aee0a94ae1415df6 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna-3.18.dist-info/licenses/LICENSE.md @@ -0,0 +1,31 @@ +BSD 3-Clause License + +Copyright (c) 2013-2026, Kim Davies and contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/.venv/lib/python3.13/site-packages/idna/__init__.py b/.venv/lib/python3.13/site-packages/idna/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cfdc030a751b089fc7e38fc88093b791605d501d --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna/__init__.py @@ -0,0 +1,45 @@ +from .core import ( + IDNABidiError, + IDNAError, + InvalidCodepoint, + InvalidCodepointContext, + alabel, + check_bidi, + check_hyphen_ok, + check_initial_combiner, + check_label, + check_nfc, + decode, + encode, + ulabel, + uts46_remap, + valid_contextj, + valid_contexto, + valid_label_length, + valid_string_length, +) +from .intranges import intranges_contain +from .package_data import __version__ + +__all__ = [ + "__version__", + "IDNABidiError", + "IDNAError", + "InvalidCodepoint", + "InvalidCodepointContext", + "alabel", + "check_bidi", + "check_hyphen_ok", + "check_initial_combiner", + "check_label", + "check_nfc", + "decode", + "encode", + "intranges_contain", + "ulabel", + "uts46_remap", + "valid_contextj", + "valid_contexto", + "valid_label_length", + "valid_string_length", +] diff --git a/.venv/lib/python3.13/site-packages/idna/__main__.py b/.venv/lib/python3.13/site-packages/idna/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..dbdd066172160ef2d47b29f0caa00e27c62ebc20 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.venv/lib/python3.13/site-packages/idna/cli.py b/.venv/lib/python3.13/site-packages/idna/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..4acda2c0f03561ee7e2a80611bdcb35900bb34e0 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna/cli.py @@ -0,0 +1,128 @@ +"""Command-line interface for the :mod:`idna` package. + +Invoked via ``python -m idna``. See :func:`main` for the entry point. +""" + +import argparse +import sys +from collections.abc import Iterable +from itertools import chain +from typing import IO, Optional + +from . import IDNAError, decode, encode +from .core import _alabel_prefix, _unicode_dots_re +from .package_data import __version__ + + +def _looks_like_alabel(s: str) -> bool: + """Return True if any label in ``s`` carries the ``xn--`` ACE prefix.""" + prefix = _alabel_prefix.decode("ascii") + return any(label.lower().startswith(prefix) for label in _unicode_dots_re.split(s)) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m idna", + description=( + "Convert a domain name between its Unicode (U-label) and " + "ASCII-compatible (A-label) forms. With no mode flag, the " + "direction is chosen from the first input — if it contains " + "an xn-- label the stream is decoded, otherwise it is " + "encoded — and the same mode is applied to every remaining " + "input. UTS #46 mapping is applied by default; pass " + "--strict to disable it. When no domains are given on the " + "command line and stdin is piped, one domain per line is " + "read from stdin." + ), + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + "-e", + "--encode", + dest="mode", + action="store_const", + const="encode", + help="Encode the input to its ASCII A-label form.", + ) + mode.add_argument( + "-d", + "--decode", + dest="mode", + action="store_const", + const="decode", + help="Decode the input from its ASCII A-label form.", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Disable the default UTS #46 mapping and apply IDNA 2008 rules verbatim.", + ) + parser.add_argument( + "--version", + action="version", + version=f"idna {__version__}", + ) + parser.add_argument( + "domain", + nargs="*", + help="One or more domain names to convert. Omit to read from stdin.", + ) + return parser + + +def _iter_stdin(stream: IO[str]) -> Iterable[str]: + """Yield non-empty stripped lines from ``stream``, ignoring blanks.""" + for line in stream: + stripped = line.strip() + if stripped: + yield stripped + + +def _convert_one(domain: str, mode: str, uts46: bool) -> bool: + """Convert ``domain`` and write the result; return ``False`` on failure.""" + try: + if mode == "decode": + print(decode(domain, uts46=uts46)) + else: + print(encode(domain, uts46=uts46).decode("ascii")) + except IDNAError as err: + print(f"idna: {mode} failed for {domain!r}: {err}", file=sys.stderr) + return False + return True + + +def main(argv: Optional[list[str]] = None) -> int: + """Entry point for ``python -m idna``. + + When more than one domain is supplied (via positional arguments or + piped stdin) and no mode flag is given, the first input determines + the direction and that mode is applied uniformly to the rest. + + :param argv: Argument list excluding the program name. Defaults to + :data:`sys.argv` when ``None``. + :returns: ``0`` on success, ``1`` if any conversion fails. + """ + parser = _build_parser() + args = parser.parse_args(argv) + uts46 = not args.strict + + if args.domain: + domains: Iterable[str] = args.domain + elif not sys.stdin.isatty(): + domains = _iter_stdin(sys.stdin) + else: + parser.error("a domain argument is required when stdin is a terminal") + + iterator = iter(domains) + first = next(iterator, None) + if first is None: + return 0 + + mode = args.mode or ("decode" if _looks_like_alabel(first) else "encode") + + results = [_convert_one(domain, mode, uts46) for domain in chain([first], iterator)] + return 0 if all(results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.venv/lib/python3.13/site-packages/idna/codec.py b/.venv/lib/python3.13/site-packages/idna/codec.py new file mode 100644 index 0000000000000000000000000000000000000000..83b42fe42b51adbe1cf9dbea79c4c8d8dbf8a0b9 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna/codec.py @@ -0,0 +1,159 @@ +import codecs +from typing import Any, Optional + +from .core import IDNAError, _unicode_dots_re, alabel, decode, encode, ulabel + + +class Codec(codecs.Codec): + """Stateless IDNA 2008 codec. + + Implements the :class:`codecs.Codec` protocol so that the whole-domain + encoder (:func:`idna.encode`) and decoder (:func:`idna.decode`) are + accessible through the standard codec machinery as ``"idna2008"``. + + Only the ``"strict"`` error handler is supported; any other handler + raises :exc:`~idna.IDNAError`. + """ + + def encode(self, data: str, errors: str = "strict") -> tuple[bytes, int]: # ty: ignore[invalid-method-override] + if errors != "strict": + raise IDNAError(f'Unsupported error handling "{errors}"') + + if not data: + return b"", 0 + + return encode(data), len(data) + + def decode(self, data: bytes, errors: str = "strict") -> tuple[str, int]: # ty: ignore[invalid-method-override] + if errors != "strict": + raise IDNAError(f'Unsupported error handling "{errors}"') + + if not data: + return "", 0 + + return decode(data), len(data) + + +class IncrementalEncoder(codecs.BufferedIncrementalEncoder): + """Incremental IDNA 2008 encoder. + + Buffers a partial trailing label across calls until either the next + label separator is seen or ``final=True``, so that streamed input is + encoded one whole label at a time. Any of the four Unicode label + separators (``U+002E``, ``U+3002``, ``U+FF0E``, ``U+FF61``) ends a + label; the result always uses ``U+002E`` as the separator. + + Only the ``"strict"`` error handler is supported. + """ + + def _buffer_encode(self, data: str, errors: str, final: bool) -> tuple[bytes, int]: # ty: ignore[invalid-method-override] + if errors != "strict": + raise IDNAError(f'Unsupported error handling "{errors}"') + + if not data: + return b"", 0 + + labels = _unicode_dots_re.split(data) + trailing_dot = b"" + if labels: + if not labels[-1]: + trailing_dot = b"." + del labels[-1] + elif not final: + # Keep potentially unfinished label until the next call + del labels[-1] + if labels: + trailing_dot = b"." + + result = [] + size = 0 + for label in labels: + result.append(alabel(label)) + if size: + size += 1 + size += len(label) + + # Join with U+002E + result_bytes = b".".join(result) + trailing_dot + size += len(trailing_dot) + return result_bytes, size + + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + """Incremental IDNA 2008 decoder. + + Buffers a partial trailing label across calls until either the next + label separator is seen or ``final=True``, so that streamed input is + decoded one whole label at a time. + + Only the ``"strict"`` error handler is supported. + """ + + def _buffer_decode(self, data: Any, errors: str, final: bool) -> tuple[str, int]: # ty: ignore[invalid-method-override] + if errors != "strict": + raise IDNAError(f'Unsupported error handling "{errors}"') + + if not data: + return ("", 0) + + if not isinstance(data, str): + data = str(data, "ascii") + + labels = _unicode_dots_re.split(data) + trailing_dot = "" + if labels: + if not labels[-1]: + trailing_dot = "." + del labels[-1] + elif not final: + # Keep potentially unfinished label until the next call + del labels[-1] + if labels: + trailing_dot = "." + + result = [] + size = 0 + for label in labels: + result.append(ulabel(label)) + if size: + size += 1 + size += len(label) + + result_str = ".".join(result) + trailing_dot + size += len(trailing_dot) + return (result_str, size) + + +class StreamWriter(Codec, codecs.StreamWriter): + pass + + +class StreamReader(Codec, codecs.StreamReader): + pass + + +def search_function(name: str) -> Optional[codecs.CodecInfo]: + """Codec search function registered with :mod:`codecs`. + + Returns a :class:`codecs.CodecInfo` for the ``"idna2008"`` codec name + so that ``str.encode("idna2008")`` and ``bytes.decode("idna2008")`` + invoke the IDNA 2008 codec defined in this module. + + :param name: The codec name being looked up. + :returns: A :class:`codecs.CodecInfo` instance if ``name`` is + ``"idna2008"``, otherwise ``None``. + """ + if name != "idna2008": + return None + return codecs.CodecInfo( + name=name, + encode=Codec().encode, + decode=Codec().decode, # type: ignore + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamwriter=StreamWriter, + streamreader=StreamReader, + ) + + +codecs.register(search_function) diff --git a/.venv/lib/python3.13/site-packages/idna/compat.py b/.venv/lib/python3.13/site-packages/idna/compat.py new file mode 100644 index 0000000000000000000000000000000000000000..1d01e3d9750860e6ec87fb7c0955cee6bcd9fcc1 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna/compat.py @@ -0,0 +1,41 @@ +from typing import Any, Union + +from .core import decode, encode + + +def ToASCII(label: str) -> bytes: + """Compatibility shim for :rfc:`3490` ``ToASCII``. + + Delegates to :func:`idna.encode` (IDNA 2008). Provided to ease porting + of code written against the legacy :mod:`encodings.idna` API; new code + should call :func:`idna.encode` directly. + + :param label: The label or domain to encode. + :returns: The encoded form as ASCII :class:`bytes`. + """ + return encode(label) + + +def ToUnicode(label: Union[bytes, bytearray]) -> str: + """Compatibility shim for :rfc:`3490` ``ToUnicode``. + + Delegates to :func:`idna.decode` (IDNA 2008). Provided to ease porting + of code written against the legacy :mod:`encodings.idna` API; new code + should call :func:`idna.decode` directly. + + :param label: The label or domain to decode. + :returns: The decoded Unicode form. + """ + return decode(label) + + +def nameprep(s: Any) -> None: + """Stub for :rfc:`3491` Nameprep, which is not used by IDNA 2008. + + IDNA 2008 (:rfc:`5891`) replaces Nameprep with the per-codepoint + validity classes from :rfc:`5892`; this function exists only to + return a clear error if legacy code attempts to call it. + + :raises NotImplementedError: Always. + """ + raise NotImplementedError("IDNA 2008 does not utilise nameprep protocol") diff --git a/.venv/lib/python3.13/site-packages/idna/core.py b/.venv/lib/python3.13/site-packages/idna/core.py new file mode 100644 index 0000000000000000000000000000000000000000..1ccbd1f3356e4b9f2aeb36b00fd5591d85459464 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna/core.py @@ -0,0 +1,648 @@ +import bisect +import re +import unicodedata +import warnings +from typing import Optional, Union + +from . import idnadata +from .intranges import intranges_contain + +_virama_combining_class = 9 +_alabel_prefix = b"xn--" +_max_input_length = 1024 +_unicode_dots_re = re.compile("[\u002e\u3002\uff0e\uff61]") + + +# Bidi category sets from RFC 5893, hoisted out of the per-codepoint loop +_bidi_rtl_first = frozenset({"R", "AL"}) +_bidi_rtl_categories = frozenset({"R", "AL", "AN"}) +_bidi_rtl_allowed = frozenset({"R", "AL", "AN", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"}) +_bidi_rtl_valid_ending = frozenset({"R", "AL", "EN", "AN"}) +_bidi_rtl_numeric = frozenset({"AN", "EN"}) +_bidi_ltr_allowed = frozenset({"L", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"}) +_bidi_ltr_valid_ending = frozenset({"L", "EN"}) +_bidi_joiner_l_or_d = frozenset({"L", "D"}) +_bidi_joiner_r_or_d = frozenset({"R", "D"}) + + +def _joining_type(cp: int) -> Optional[str]: + for jt, ranges in idnadata.joining_types.items(): + if intranges_contain(cp, ranges): + return jt + return None + + +class IDNAError(UnicodeError): + """Base exception for all IDNA-encoding related problems""" + + +class IDNABidiError(IDNAError): + """Exception when bidirectional requirements are not satisfied""" + + +class InvalidCodepoint(IDNAError): + """Exception when a disallowed or unallocated codepoint is used""" + + +class InvalidCodepointContext(IDNAError): + """Exception when the codepoint is not valid in the context it is used""" + + +def _combining_class(cp: int) -> int: + v = unicodedata.combining(chr(cp)) + if v == 0 and not unicodedata.name(chr(cp)): + raise ValueError("Unknown character in unicodedata") + return v + + +def _is_script(cp: str, script: str) -> bool: + return intranges_contain(ord(cp), idnadata.scripts[script]) + + +def _punycode(s: str) -> bytes: + return s.encode("punycode") + + +def _unot(s: int) -> str: + return f"U+{s:04X}" + + +def valid_label_length(label: Union[bytes, str]) -> bool: + """Check that a label does not exceed the maximum permitted length. + + Per :rfc:`1035` (and :rfc:`5891` §4.2.4) a DNS label must not exceed + 63 octets. The argument may be either a :class:`str` (a U-label, where + length is measured in characters) or :class:`bytes` (an A-label, where + length is measured in octets). + + :param label: The label to check. + :returns: ``True`` if the label is within the length limit, otherwise + ``False``. + """ + return len(label) <= 63 + + +def valid_string_length(domain: Union[bytes, str], trailing_dot: bool) -> bool: + """Check that a full domain name does not exceed the maximum length. + + Per :rfc:`1035`, a domain name is limited to 253 octets when no trailing + dot is present, or 254 octets when one is included. + + :param domain: The full (possibly multi-label) domain name. + :param trailing_dot: ``True`` if ``domain`` includes a trailing ``.``. + :returns: ``True`` if the domain is within the length limit, otherwise + ``False``. + """ + return len(domain) <= (254 if trailing_dot else 253) + + +def check_bidi(label: str, check_ltr: bool = False) -> bool: + """Validate the Bidi Rule from :rfc:`5893` for a single label. + + The Bidi Rule constrains how bidirectional characters (Hebrew, Arabic, + etc.) may appear within a label. By default the check is only applied + when the label contains at least one right-to-left character (Unicode + bidirectional categories ``R``, ``AL``, or ``AN``); set ``check_ltr`` + to ``True`` to apply it to LTR-only labels as well. + + :param label: The label to validate, as a Unicode string. + :param check_ltr: If ``True``, apply the rules even when the label + contains no RTL characters. + :returns: ``True`` if the label satisfies the Bidi Rule. + :raises IDNABidiError: If any of Bidi Rule conditions 1-6 are violated, + or if the directional category of a codepoint cannot be determined. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") + # Bidi rules should only be applied if string contains RTL characters + bidi_label = False + for idx, cp in enumerate(label, 1): + direction = unicodedata.bidirectional(cp) + if direction == "": + # String likely comes from a newer version of Unicode + raise IDNABidiError(f"Unknown directionality in label {label!r} at position {idx}") + if direction in _bidi_rtl_categories: + bidi_label = True + if not bidi_label and not check_ltr: + return True + + # Bidi rule 1 + direction = unicodedata.bidirectional(label[0]) + if direction in _bidi_rtl_first: + rtl = True + elif direction == "L": + rtl = False + else: + raise IDNABidiError(f"First codepoint in label {label!r} must be directionality L, R or AL") + + valid_ending = False + number_type: Optional[str] = None + for idx, cp in enumerate(label, 1): + direction = unicodedata.bidirectional(cp) + + if rtl: + # Bidi rule 2 + if direction not in _bidi_rtl_allowed: + raise IDNABidiError(f"Invalid direction for codepoint at position {idx} in a right-to-left label") + # Bidi rule 3 + if direction in _bidi_rtl_valid_ending: + valid_ending = True + elif direction != "NSM": + valid_ending = False + # Bidi rule 4 + if direction in _bidi_rtl_numeric: + if not number_type: + number_type = direction + elif number_type != direction: + raise IDNABidiError("Can not mix numeral types in a right-to-left label") + else: + # Bidi rule 5 + if direction not in _bidi_ltr_allowed: + raise IDNABidiError(f"Invalid direction for codepoint at position {idx} in a left-to-right label") + # Bidi rule 6 + if direction in _bidi_ltr_valid_ending: + valid_ending = True + elif direction != "NSM": + valid_ending = False + + if not valid_ending: + raise IDNABidiError("Label ends with illegal codepoint directionality") + + return True + + +def check_initial_combiner(label: str) -> bool: + """Reject labels that begin with a combining mark. + + Per :rfc:`5891` §4.2.3.2 a label must not start with a character of + Unicode general category ``M`` (Mark). + + :param label: The label to check. + :returns: ``True`` if the first character is not a combining mark. + :raises IDNAError: If the label begins with a combining character. + """ + if unicodedata.category(label[0])[0] == "M": + raise IDNAError("Label begins with an illegal combining character") + return True + + +def check_hyphen_ok(label: str) -> bool: + """Validate the hyphen restrictions for a label. + + Per :rfc:`5891` §4.2.3.1 a label must not start or end with a hyphen + (``U+002D``), and must not have hyphens in both the third and fourth + positions (the prefix reserved for A-labels). + + :param label: The label to check. + :returns: ``True`` if the hyphen restrictions are satisfied. + :raises IDNAError: If any of the hyphen restrictions are violated. + """ + if label[2:4] == "--": + raise IDNAError("Label has disallowed hyphens in 3rd and 4th position") + if label[0] == "-" or label[-1] == "-": + raise IDNAError("Label must not start or end with a hyphen") + return True + + +def check_nfc(label: str) -> None: + """Require that a label is in Unicode Normalization Form C. + + :param label: The label to check. + :raises IDNAError: If ``label`` differs from its NFC normalisation. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") + if unicodedata.normalize("NFC", label) != label: + raise IDNAError("Label must be in Normalization Form C") + + +def valid_contextj(label: str, pos: int) -> bool: + """Validate the CONTEXTJ rules from :rfc:`5892` Appendix A. + + These rules govern the contextual use of the joiner codepoints + ``U+200C`` (ZERO WIDTH NON-JOINER, Appendix A.1) and ``U+200D`` + (ZERO WIDTH JOINER, Appendix A.2) within a label. + + :param label: The label containing the codepoint. + :param pos: Index of the joiner codepoint within ``label``. + :returns: ``True`` if the codepoint at ``pos`` satisfies its CONTEXTJ + rule, ``False`` otherwise (including when the codepoint at + ``pos`` is not a recognised joiner). + :raises ValueError: If an adjacent codepoint has no Unicode name when + determining its combining class. + :raises IDNAError: If ``label`` exceeds the defensive input length limit. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") + cp_value = ord(label[pos]) + + if cp_value == 0x200C: + if pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class: + return True + + ok = False + for i in range(pos - 1, -1, -1): + joining_type = _joining_type(ord(label[i])) + if joining_type == "T": + continue + if joining_type in _bidi_joiner_l_or_d: + ok = True + break + break + + if not ok: + return False + + ok = False + for i in range(pos + 1, len(label)): + joining_type = _joining_type(ord(label[i])) + if joining_type == "T": + continue + if joining_type in _bidi_joiner_r_or_d: + ok = True + break + break + return ok + + if cp_value == 0x200D: + return pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class + + return False + + +def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: + """Validate the CONTEXTO rules from :rfc:`5892` Appendix A. + + Covers the contextual rules for codepoints such as MIDDLE DOT + (``U+00B7``), Greek lower numeral sign, Hebrew punctuation, Katakana + middle dot, and the Arabic-Indic / Extended Arabic-Indic digit ranges. + + :param label: The label containing the codepoint. + :param pos: Index of the codepoint within ``label``. + :param exception: Reserved for forward compatibility; currently unused. + :returns: ``True`` if the codepoint at ``pos`` satisfies its CONTEXTO + rule, ``False`` otherwise (including when the codepoint is not a + recognised CONTEXTO codepoint). + :raises IDNAError: If ``label`` exceeds the defensive input length limit. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") + cp_value = ord(label[pos]) + + if cp_value == 0x00B7: + return 0 < pos < len(label) - 1 and ord(label[pos - 1]) == 0x006C and ord(label[pos + 1]) == 0x006C + + if cp_value == 0x0375: + if pos < len(label) - 1 and len(label) > 1: + return _is_script(label[pos + 1], "Greek") + return False + + if cp_value in {0x05F3, 0x05F4}: + if pos > 0: + return _is_script(label[pos - 1], "Hebrew") + return False + + if cp_value == 0x30FB: + for cp in label: + if cp == "\u30fb": + continue + if _is_script(cp, "Hiragana") or _is_script(cp, "Katakana") or _is_script(cp, "Han"): + return True + return False + + if 0x660 <= cp_value <= 0x669: + return not any(0x6F0 <= ord(cp) <= 0x06F9 for cp in label) + + if 0x6F0 <= cp_value <= 0x6F9: + return not any(0x660 <= ord(cp) <= 0x0669 for cp in label) + + return False + + +def check_label(label: Union[str, bytes, bytearray]) -> None: + """Run the full set of IDNA 2008 validity checks on a single label. + + Applies, in order: NFC normalisation (:func:`check_nfc`), hyphen + restrictions (:func:`check_hyphen_ok`), the no-leading-combiner rule + (:func:`check_initial_combiner`), per-codepoint validity (PVALID, + CONTEXTJ, CONTEXTO classes from :rfc:`5892`), and the Bidi Rule + (:func:`check_bidi`). + + :param label: The label to validate. ``bytes`` or ``bytearray`` input + is decoded as UTF-8 first. + :raises IDNAError: If the label is empty or fails a structural rule. + :raises InvalidCodepoint: If the label contains a DISALLOWED or + UNASSIGNED codepoint. + :raises InvalidCodepointContext: If a CONTEXTJ or CONTEXTO codepoint + is not valid in its context. + :raises IDNABidiError: If the Bidi Rule is violated. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") + if isinstance(label, (bytes, bytearray)): + label = label.decode("utf-8") + if len(label) == 0: + raise IDNAError("Empty Label") + + # Reject on domain length rather than label length so support some UTS 46 + # use cases, still reducing processing of label contextual rules + if not valid_string_length(label, trailing_dot=True): + raise IDNAError("Label too long") + + check_nfc(label) + check_hyphen_ok(label) + check_initial_combiner(label) + + for pos, cp in enumerate(label): + cp_value = ord(cp) + if intranges_contain(cp_value, idnadata.codepoint_classes["PVALID"]): + continue + if intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]): + try: + if not valid_contextj(label, pos): + raise InvalidCodepointContext(f"Joiner {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") + except ValueError as err: + raise IDNAError( + f"Unknown codepoint adjacent to joiner {_unot(cp_value)} at position {pos + 1} in {label!r}" + ) from err + elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTO"]): + if not valid_contexto(label, pos): + raise InvalidCodepointContext(f"Codepoint {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") + else: + raise InvalidCodepoint(f"Codepoint {_unot(cp_value)} at position {pos + 1} of {label!r} not allowed") + + check_bidi(label) + + +def alabel(label: str) -> bytes: + """Convert a single U-label into its A-label form. + + The result is the ASCII-Compatible Encoding (ACE) form per :rfc:`5891` + §4: the label is validated, Punycode-encoded, and prefixed with + ``xn--``. Pure ASCII labels that are already valid IDNA labels are + returned unchanged (as :class:`bytes`). + + :param label: The label to convert, as a Unicode string. + :returns: The A-label as ASCII-encoded :class:`bytes`. + :raises IDNAError: If the label is invalid or the resulting A-label + exceeds 63 octets. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") + try: + label_bytes = label.encode("ascii") + except UnicodeEncodeError: + pass + else: + ulabel(label_bytes) + if not valid_label_length(label_bytes): + raise IDNAError("Label too long") + return label_bytes + + check_label(label) + label_bytes = _alabel_prefix + _punycode(label) + + if not valid_label_length(label_bytes): + raise IDNAError("Label too long") + + return label_bytes + + +def ulabel(label: Union[str, bytes, bytearray]) -> str: + """Convert a single A-label into its U-label form. + + Performs the inverse of :func:`alabel`: an ``xn--``-prefixed label is + Punycode-decoded and validated. Labels that are already Unicode (or + plain ASCII without the ACE prefix) are validated and returned as a + Unicode string. + + :param label: The label to convert. ``bytes`` or ``bytearray`` input + is treated as ASCII. + :returns: The U-label as a Unicode string. + :raises IDNAError: If the label is malformed or fails validation. + """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") + if not isinstance(label, (bytes, bytearray)): + try: + label_bytes = label.encode("ascii") + except UnicodeEncodeError: + check_label(label) + return label + else: + label_bytes = bytes(label) + + label_bytes = label_bytes.lower() + if label_bytes.startswith(_alabel_prefix): + label_bytes = label_bytes[len(_alabel_prefix) :] + if not label_bytes: + raise IDNAError("Malformed A-label, no Punycode eligible content found") + if label_bytes.endswith(b"-"): + raise IDNAError("A-label must not end with a hyphen") + else: + check_label(label_bytes) + return label_bytes.decode("ascii") + + try: + label = label_bytes.decode("punycode") + except UnicodeError as err: + raise IDNAError("Invalid A-label") from err + check_label(label) + return label + + +def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False) -> str: + """Apply the UTS #46 character mapping to a domain string. + + Implements the mapping table from `UTS #46 §4 + `_: each character is kept, + replaced, or rejected based on its status (``V``, ``M``, ``D``, ``3``, + ``I``). The result is returned in Normalisation Form C. + + :param domain: The full domain name to remap. + :param std3_rules: If ``True``, apply the stricter STD3 ASCII rules + (status ``3`` codepoints raise instead of being kept or mapped). + :param transitional: If ``True``, use transitional processing (status + ``D`` codepoints are mapped instead of kept). Transitional + processing has been removed from UTS #46 and this option is + retained only for backwards compatibility. + :returns: The remapped domain, in Normalisation Form C. + :raises InvalidCodepoint: If the domain contains a disallowed + codepoint under the chosen rules. + :raises IDNAError: If ``domain`` exceeds the defensive input length limit. + """ + if len(domain) > _max_input_length: + raise IDNAError("Domain too long") + from .uts46data import uts46_replacements, uts46_starts, uts46_statuses + + output = "" + + for pos, char in enumerate(domain): + code_point = ord(char) + i = code_point if code_point < 256 else bisect.bisect_right(uts46_starts, code_point) - 1 + status = chr(uts46_statuses[i]) + replacement: Optional[str] = uts46_replacements[i] + + # UTS #46 §4: V is always valid, D is deviation (kept unless transitional), + # 3 is disallowed-STD3 (kept unmapped if std3_rules is off and no mapping). + keep_as_is = ( + status == "V" or (status == "D" and not transitional) or (status == "3" and not std3_rules and replacement is None) + ) + # M is mapped, 3-with-replacement and transitional D fall through to the + # same replacement output path. + use_replacement = replacement is not None and ( + status == "M" or (status == "3" and not std3_rules) or (status == "D" and transitional) + ) + + if keep_as_is: + output += char + elif use_replacement: + assert replacement is not None # narrowed by use_replacement + output += replacement + elif status == "I": + continue + else: + raise InvalidCodepoint(f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {domain!r}") + + return unicodedata.normalize("NFC", output) + + +def encode( + s: Union[str, bytes, bytearray], + strict: bool = False, + uts46: bool = False, + std3_rules: bool = False, + transitional: bool = False, +) -> bytes: + """Encode a Unicode domain name into its ASCII (A-label) form. + + Splits the input on label separators (only ``U+002E`` if ``strict`` is + set; otherwise also IDEOGRAPHIC FULL STOP ``U+3002``, FULLWIDTH FULL + STOP ``U+FF0E``, and HALFWIDTH IDEOGRAPHIC FULL STOP ``U+FF61``), + encodes each label with :func:`alabel`, and rejoins them with ``.``. + Optionally pre-processes the input through :func:`uts46_remap`. + + :param s: The domain name to encode. + :param strict: If ``True``, only ``U+002E`` is recognised as a label + separator. + :param uts46: If ``True``, apply UTS #46 mapping before encoding. + :param std3_rules: Forwarded to :func:`uts46_remap` when ``uts46`` is + ``True``. + :param transitional: Forwarded to :func:`uts46_remap` when ``uts46`` + is ``True``. Deprecated: emits a :class:`DeprecationWarning` and + will be removed in a future version. + :returns: The encoded domain as ASCII :class:`bytes`. + :raises IDNAError: If the domain is empty, contains an invalid label, + or exceeds the maximum domain length. + """ + if transitional: + warnings.warn( + "Transitional processing has been removed from UTS #46. " + "The transitional argument will be removed in a future version.", + DeprecationWarning, + stacklevel=2, + ) + if not isinstance(s, str): + try: + s = str(s, "ascii") + except (UnicodeDecodeError, TypeError) as err: + raise IDNAError("should pass a unicode string to the function rather than a byte string.") from err + if len(s) > _max_input_length: + raise IDNAError("Domain too long") + if uts46: + s = uts46_remap(s, std3_rules, transitional) + + # Reject inputs that exceed the maximum DNS domain length up-front + # to avoid expensive computation on long inputs. + if not valid_string_length(s, trailing_dot=True): + raise IDNAError("Domain too long") + + trailing_dot = False + result = [] + labels = s.split(".") if strict else _unicode_dots_re.split(s) + if not labels or labels == [""]: + raise IDNAError("Empty domain") + if labels[-1] == "": + del labels[-1] + trailing_dot = True + for label in labels: + s = alabel(label) + if s: + result.append(s) + else: + raise IDNAError("Empty label") + if trailing_dot: + result.append(b"") + s = b".".join(result) + if not valid_string_length(s, trailing_dot): + raise IDNAError("Domain too long") + return s + + +def decode( + s: Union[str, bytes, bytearray], + strict: bool = False, + uts46: bool = False, + std3_rules: bool = False, + display: bool = False, +) -> str: + """Decode an A-label-encoded domain name back to Unicode. + + Splits the input on label separators (see :func:`encode` for the + rules), decodes each label with :func:`ulabel`, and rejoins them + with ``.``. Optionally pre-processes the input through + :func:`uts46_remap`. + + :param s: The domain name to decode. + :param strict: If ``True``, only ``U+002E`` is recognised as a label + separator. + :param uts46: If ``True``, apply UTS #46 mapping before decoding. + :param std3_rules: Forwarded to :func:`uts46_remap` when ``uts46`` is + ``True``. + :param display: If ``True``, any ``xn--`` label that fails IDNA + validation is passed through unchanged (lowercased) rather than + aborting the whole call. Intended for "decode for display" + consumers (e.g. URL libraries, HTTP clients) that want to show + the user the label as it appears on the wire when it cannot be + rendered as Unicode. Matches the per-label recovery prescribed + by UTS #46 §4 and the WHATWG URL "domain to Unicode" algorithm. + :returns: The decoded domain as a Unicode string. + :raises IDNAError: If the input is not valid ASCII, contains an + invalid label, or is empty. + """ + if not isinstance(s, str): + try: + s = str(s, "ascii") + except (UnicodeDecodeError, TypeError) as err: + raise IDNAError("Invalid ASCII in A-label") from err + if len(s) > _max_input_length: + raise IDNAError("Domain too long") + if uts46: + s = uts46_remap(s, std3_rules, False) + # Reject inputs that exceed the maximum DNS domain length up-front + # to avoid expensive computation on long inputs. + if not valid_string_length(s, trailing_dot=True): + raise IDNAError("Domain too long") + trailing_dot = False + result = [] + labels = s.split(".") if strict else _unicode_dots_re.split(s) + if not labels or labels == [""]: + raise IDNAError("Empty domain") + if not labels[-1]: + del labels[-1] + trailing_dot = True + for label in labels: + try: + u = ulabel(label) + except IDNAError: + if display and label[:4].lower() == "xn--": + u = label.lower() + else: + raise + if u: + result.append(u) + else: + raise IDNAError("Empty label") + if trailing_dot: + result.append("") + return ".".join(result) diff --git a/.venv/lib/python3.13/site-packages/idna/idnadata.py b/.venv/lib/python3.13/site-packages/idna/idnadata.py new file mode 100644 index 0000000000000000000000000000000000000000..f2ab38897bf1f8fc6f0db50f237edec68ca89295 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna/idnadata.py @@ -0,0 +1,1897 @@ +# This file is automatically generated by tools/idna-data + +__version__ = "17.0.0" + +scripts = { + "Greek": ( + 0x37000000374, + 0x37500000378, + 0x37A0000037E, + 0x37F00000380, + 0x38400000385, + 0x38600000387, + 0x3880000038B, + 0x38C0000038D, + 0x38E000003A2, + 0x3A3000003E2, + 0x3F000000400, + 0x1D2600001D2B, + 0x1D5D00001D62, + 0x1D6600001D6B, + 0x1DBF00001DC0, + 0x1F0000001F16, + 0x1F1800001F1E, + 0x1F2000001F46, + 0x1F4800001F4E, + 0x1F5000001F58, + 0x1F5900001F5A, + 0x1F5B00001F5C, + 0x1F5D00001F5E, + 0x1F5F00001F7E, + 0x1F8000001FB5, + 0x1FB600001FC5, + 0x1FC600001FD4, + 0x1FD600001FDC, + 0x1FDD00001FF0, + 0x1FF200001FF5, + 0x1FF600001FFF, + 0x212600002127, + 0xAB650000AB66, + 0x101400001018F, + 0x101A0000101A1, + 0x1D2000001D246, + ), + "Han": ( + 0x2E8000002E9A, + 0x2E9B00002EF4, + 0x2F0000002FD6, + 0x300500003006, + 0x300700003008, + 0x30210000302A, + 0x30380000303C, + 0x340000004DC0, + 0x4E000000A000, + 0xF9000000FA6E, + 0xFA700000FADA, + 0x16FE200016FE4, + 0x16FF000016FF7, + 0x200000002A6E0, + 0x2A7000002B81E, + 0x2B8200002CEAE, + 0x2CEB00002EBE1, + 0x2EBF00002EE5E, + 0x2F8000002FA1E, + 0x300000003134B, + 0x313500003347A, + ), + "Hebrew": ( + 0x591000005C8, + 0x5D0000005EB, + 0x5EF000005F5, + 0xFB1D0000FB37, + 0xFB380000FB3D, + 0xFB3E0000FB3F, + 0xFB400000FB42, + 0xFB430000FB45, + 0xFB460000FB50, + ), + "Hiragana": ( + 0x304100003097, + 0x309D000030A0, + 0x1B0010001B120, + 0x1B1320001B133, + 0x1B1500001B153, + 0x1F2000001F201, + ), + "Katakana": ( + 0x30A1000030FB, + 0x30FD00003100, + 0x31F000003200, + 0x32D0000032FF, + 0x330000003358, + 0xFF660000FF70, + 0xFF710000FF9E, + 0x1AFF00001AFF4, + 0x1AFF50001AFFC, + 0x1AFFD0001AFFF, + 0x1B0000001B001, + 0x1B1200001B123, + 0x1B1550001B156, + 0x1B1640001B168, + ), +} + + +joining_types = { + "C": ( + 0x64000000641, + 0x7FA000007FB, + 0x88300000886, + 0x180A0000180B, + 0x200D0000200E, + ), + "D": ( + 0x62000000621, + 0x62600000627, + 0x62800000629, + 0x62A0000062F, + 0x63300000640, + 0x64100000648, + 0x6490000064B, + 0x66E00000670, + 0x67800000688, + 0x69A000006C0, + 0x6C1000006C3, + 0x6CC000006CD, + 0x6CE000006CF, + 0x6D0000006D2, + 0x6FA000006FD, + 0x6FF00000700, + 0x71200000715, + 0x71A0000071E, + 0x71F00000728, + 0x7290000072A, + 0x72B0000072C, + 0x72D0000072F, + 0x74E00000759, + 0x75C0000076B, + 0x76D00000771, + 0x77200000773, + 0x77500000778, + 0x77A00000780, + 0x7CA000007EB, + 0x84100000846, + 0x84800000849, + 0x84A00000854, + 0x85500000856, + 0x86000000861, + 0x86200000866, + 0x86800000869, + 0x88600000887, + 0x8890000088E, + 0x88F00000890, + 0x8A0000008AA, + 0x8AF000008B1, + 0x8B3000008B9, + 0x8BA000008C9, + 0x180700001808, + 0x182000001879, + 0x1887000018A9, + 0x18AA000018AB, + 0xA8400000A872, + 0x10AC000010AC5, + 0x10AD300010AD7, + 0x10AD800010ADD, + 0x10ADE00010AE1, + 0x10AEB00010AEF, + 0x10B8000010B81, + 0x10B8200010B83, + 0x10B8600010B89, + 0x10B8A00010B8C, + 0x10B8D00010B8E, + 0x10B9000010B91, + 0x10BAD00010BAF, + 0x10D0100010D22, + 0x10D2300010D24, + 0x10EC300010EC5, + 0x10EC600010EC8, + 0x10F3000010F33, + 0x10F3400010F45, + 0x10F5100010F54, + 0x10F7000010F74, + 0x10F7600010F82, + 0x10FB000010FB1, + 0x10FB200010FB4, + 0x10FB800010FB9, + 0x10FBB00010FBD, + 0x10FBE00010FC0, + 0x10FC100010FC2, + 0x10FC400010FC5, + 0x10FCA00010FCB, + 0x1E9000001E944, + ), + "L": ( + 0xA8720000A873, + 0x10ACD00010ACE, + 0x10AD700010AD8, + 0x10D0000010D01, + 0x10FCB00010FCC, + ), + "R": ( + 0x62200000626, + 0x62700000628, + 0x6290000062A, + 0x62F00000633, + 0x64800000649, + 0x67100000674, + 0x67500000678, + 0x6880000069A, + 0x6C0000006C1, + 0x6C3000006CC, + 0x6CD000006CE, + 0x6CF000006D0, + 0x6D2000006D4, + 0x6D5000006D6, + 0x6EE000006F0, + 0x71000000711, + 0x7150000071A, + 0x71E0000071F, + 0x72800000729, + 0x72A0000072B, + 0x72C0000072D, + 0x72F00000730, + 0x74D0000074E, + 0x7590000075C, + 0x76B0000076D, + 0x77100000772, + 0x77300000775, + 0x7780000077A, + 0x84000000841, + 0x84600000848, + 0x8490000084A, + 0x85400000855, + 0x85600000859, + 0x86700000868, + 0x8690000086B, + 0x87000000883, + 0x88E0000088F, + 0x8AA000008AD, + 0x8AE000008AF, + 0x8B1000008B3, + 0x8B9000008BA, + 0x10AC500010AC6, + 0x10AC700010AC8, + 0x10AC900010ACB, + 0x10ACE00010AD3, + 0x10ADD00010ADE, + 0x10AE100010AE2, + 0x10AE400010AE5, + 0x10AEF00010AF0, + 0x10B8100010B82, + 0x10B8300010B86, + 0x10B8900010B8A, + 0x10B8C00010B8D, + 0x10B8E00010B90, + 0x10B9100010B92, + 0x10BA900010BAD, + 0x10D2200010D23, + 0x10EC200010EC3, + 0x10F3300010F34, + 0x10F5400010F55, + 0x10F7400010F76, + 0x10FB400010FB7, + 0x10FB900010FBB, + 0x10FBD00010FBE, + 0x10FC200010FC4, + 0x10FC900010FCA, + ), + "T": ( + 0xAD000000AE, + 0x30000000370, + 0x4830000048A, + 0x591000005BE, + 0x5BF000005C0, + 0x5C1000005C3, + 0x5C4000005C6, + 0x5C7000005C8, + 0x6100000061B, + 0x61C0000061D, + 0x64B00000660, + 0x67000000671, + 0x6D6000006DD, + 0x6DF000006E5, + 0x6E7000006E9, + 0x6EA000006EE, + 0x70F00000710, + 0x71100000712, + 0x7300000074B, + 0x7A6000007B1, + 0x7EB000007F4, + 0x7FD000007FE, + 0x8160000081A, + 0x81B00000824, + 0x82500000828, + 0x8290000082E, + 0x8590000085C, + 0x897000008A0, + 0x8CA000008E2, + 0x8E300000903, + 0x93A0000093B, + 0x93C0000093D, + 0x94100000949, + 0x94D0000094E, + 0x95100000958, + 0x96200000964, + 0x98100000982, + 0x9BC000009BD, + 0x9C1000009C5, + 0x9CD000009CE, + 0x9E2000009E4, + 0x9FE000009FF, + 0xA0100000A03, + 0xA3C00000A3D, + 0xA4100000A43, + 0xA4700000A49, + 0xA4B00000A4E, + 0xA5100000A52, + 0xA7000000A72, + 0xA7500000A76, + 0xA8100000A83, + 0xABC00000ABD, + 0xAC100000AC6, + 0xAC700000AC9, + 0xACD00000ACE, + 0xAE200000AE4, + 0xAFA00000B00, + 0xB0100000B02, + 0xB3C00000B3D, + 0xB3F00000B40, + 0xB4100000B45, + 0xB4D00000B4E, + 0xB5500000B57, + 0xB6200000B64, + 0xB8200000B83, + 0xBC000000BC1, + 0xBCD00000BCE, + 0xC0000000C01, + 0xC0400000C05, + 0xC3C00000C3D, + 0xC3E00000C41, + 0xC4600000C49, + 0xC4A00000C4E, + 0xC5500000C57, + 0xC6200000C64, + 0xC8100000C82, + 0xCBC00000CBD, + 0xCBF00000CC0, + 0xCC600000CC7, + 0xCCC00000CCE, + 0xCE200000CE4, + 0xD0000000D02, + 0xD3B00000D3D, + 0xD4100000D45, + 0xD4D00000D4E, + 0xD6200000D64, + 0xD8100000D82, + 0xDCA00000DCB, + 0xDD200000DD5, + 0xDD600000DD7, + 0xE3100000E32, + 0xE3400000E3B, + 0xE4700000E4F, + 0xEB100000EB2, + 0xEB400000EBD, + 0xEC800000ECF, + 0xF1800000F1A, + 0xF3500000F36, + 0xF3700000F38, + 0xF3900000F3A, + 0xF7100000F7F, + 0xF8000000F85, + 0xF8600000F88, + 0xF8D00000F98, + 0xF9900000FBD, + 0xFC600000FC7, + 0x102D00001031, + 0x103200001038, + 0x10390000103B, + 0x103D0000103F, + 0x10580000105A, + 0x105E00001061, + 0x107100001075, + 0x108200001083, + 0x108500001087, + 0x108D0000108E, + 0x109D0000109E, + 0x135D00001360, + 0x171200001715, + 0x173200001734, + 0x175200001754, + 0x177200001774, + 0x17B4000017B6, + 0x17B7000017BE, + 0x17C6000017C7, + 0x17C9000017D4, + 0x17DD000017DE, + 0x180B0000180E, + 0x180F00001810, + 0x188500001887, + 0x18A9000018AA, + 0x192000001923, + 0x192700001929, + 0x193200001933, + 0x19390000193C, + 0x1A1700001A19, + 0x1A1B00001A1C, + 0x1A5600001A57, + 0x1A5800001A5F, + 0x1A6000001A61, + 0x1A6200001A63, + 0x1A6500001A6D, + 0x1A7300001A7D, + 0x1A7F00001A80, + 0x1AB000001ADE, + 0x1AE000001AEC, + 0x1B0000001B04, + 0x1B3400001B35, + 0x1B3600001B3B, + 0x1B3C00001B3D, + 0x1B4200001B43, + 0x1B6B00001B74, + 0x1B8000001B82, + 0x1BA200001BA6, + 0x1BA800001BAA, + 0x1BAB00001BAE, + 0x1BE600001BE7, + 0x1BE800001BEA, + 0x1BED00001BEE, + 0x1BEF00001BF2, + 0x1C2C00001C34, + 0x1C3600001C38, + 0x1CD000001CD3, + 0x1CD400001CE1, + 0x1CE200001CE9, + 0x1CED00001CEE, + 0x1CF400001CF5, + 0x1CF800001CFA, + 0x1DC000001E00, + 0x200B0000200C, + 0x200E00002010, + 0x202A0000202F, + 0x206000002065, + 0x206A00002070, + 0x20D0000020F1, + 0x2CEF00002CF2, + 0x2D7F00002D80, + 0x2DE000002E00, + 0x302A0000302E, + 0x30990000309B, + 0xA66F0000A673, + 0xA6740000A67E, + 0xA69E0000A6A0, + 0xA6F00000A6F2, + 0xA8020000A803, + 0xA8060000A807, + 0xA80B0000A80C, + 0xA8250000A827, + 0xA82C0000A82D, + 0xA8C40000A8C6, + 0xA8E00000A8F2, + 0xA8FF0000A900, + 0xA9260000A92E, + 0xA9470000A952, + 0xA9800000A983, + 0xA9B30000A9B4, + 0xA9B60000A9BA, + 0xA9BC0000A9BE, + 0xA9E50000A9E6, + 0xAA290000AA2F, + 0xAA310000AA33, + 0xAA350000AA37, + 0xAA430000AA44, + 0xAA4C0000AA4D, + 0xAA7C0000AA7D, + 0xAAB00000AAB1, + 0xAAB20000AAB5, + 0xAAB70000AAB9, + 0xAABE0000AAC0, + 0xAAC10000AAC2, + 0xAAEC0000AAEE, + 0xAAF60000AAF7, + 0xABE50000ABE6, + 0xABE80000ABE9, + 0xABED0000ABEE, + 0xFB1E0000FB1F, + 0xFE000000FE10, + 0xFE200000FE30, + 0xFEFF0000FF00, + 0xFFF90000FFFC, + 0x101FD000101FE, + 0x102E0000102E1, + 0x103760001037B, + 0x10A0100010A04, + 0x10A0500010A07, + 0x10A0C00010A10, + 0x10A3800010A3B, + 0x10A3F00010A40, + 0x10AE500010AE7, + 0x10D2400010D28, + 0x10D6900010D6E, + 0x10EAB00010EAD, + 0x10EFA00010F00, + 0x10F4600010F51, + 0x10F8200010F86, + 0x1100100011002, + 0x1103800011047, + 0x1107000011071, + 0x1107300011075, + 0x1107F00011082, + 0x110B3000110B7, + 0x110B9000110BB, + 0x110C2000110C3, + 0x1110000011103, + 0x111270001112C, + 0x1112D00011135, + 0x1117300011174, + 0x1118000011182, + 0x111B6000111BF, + 0x111C9000111CD, + 0x111CF000111D0, + 0x1122F00011232, + 0x1123400011235, + 0x1123600011238, + 0x1123E0001123F, + 0x1124100011242, + 0x112DF000112E0, + 0x112E3000112EB, + 0x1130000011302, + 0x1133B0001133D, + 0x1134000011341, + 0x113660001136D, + 0x1137000011375, + 0x113BB000113C1, + 0x113CE000113CF, + 0x113D0000113D1, + 0x113D2000113D3, + 0x113E1000113E3, + 0x1143800011440, + 0x1144200011445, + 0x1144600011447, + 0x1145E0001145F, + 0x114B3000114B9, + 0x114BA000114BB, + 0x114BF000114C1, + 0x114C2000114C4, + 0x115B2000115B6, + 0x115BC000115BE, + 0x115BF000115C1, + 0x115DC000115DE, + 0x116330001163B, + 0x1163D0001163E, + 0x1163F00011641, + 0x116AB000116AC, + 0x116AD000116AE, + 0x116B0000116B6, + 0x116B7000116B8, + 0x1171D0001171E, + 0x1171F00011720, + 0x1172200011726, + 0x117270001172C, + 0x1182F00011838, + 0x118390001183B, + 0x1193B0001193D, + 0x1193E0001193F, + 0x1194300011944, + 0x119D4000119D8, + 0x119DA000119DC, + 0x119E0000119E1, + 0x11A0100011A0B, + 0x11A3300011A39, + 0x11A3B00011A3F, + 0x11A4700011A48, + 0x11A5100011A57, + 0x11A5900011A5C, + 0x11A8A00011A97, + 0x11A9800011A9A, + 0x11B6000011B61, + 0x11B6200011B65, + 0x11B6600011B67, + 0x11C3000011C37, + 0x11C3800011C3E, + 0x11C3F00011C40, + 0x11C9200011CA8, + 0x11CAA00011CB1, + 0x11CB200011CB4, + 0x11CB500011CB7, + 0x11D3100011D37, + 0x11D3A00011D3B, + 0x11D3C00011D3E, + 0x11D3F00011D46, + 0x11D4700011D48, + 0x11D9000011D92, + 0x11D9500011D96, + 0x11D9700011D98, + 0x11EF300011EF5, + 0x11F0000011F02, + 0x11F3600011F3B, + 0x11F4000011F41, + 0x11F4200011F43, + 0x11F5A00011F5B, + 0x1343000013441, + 0x1344700013456, + 0x1611E0001612A, + 0x1612D00016130, + 0x16AF000016AF5, + 0x16B3000016B37, + 0x16F4F00016F50, + 0x16F8F00016F93, + 0x16FE400016FE5, + 0x1BC9D0001BC9F, + 0x1BCA00001BCA4, + 0x1CF000001CF2E, + 0x1CF300001CF47, + 0x1D1670001D16A, + 0x1D1730001D183, + 0x1D1850001D18C, + 0x1D1AA0001D1AE, + 0x1D2420001D245, + 0x1DA000001DA37, + 0x1DA3B0001DA6D, + 0x1DA750001DA76, + 0x1DA840001DA85, + 0x1DA9B0001DAA0, + 0x1DAA10001DAB0, + 0x1E0000001E007, + 0x1E0080001E019, + 0x1E01B0001E022, + 0x1E0230001E025, + 0x1E0260001E02B, + 0x1E08F0001E090, + 0x1E1300001E137, + 0x1E2AE0001E2AF, + 0x1E2EC0001E2F0, + 0x1E4EC0001E4F0, + 0x1E5EE0001E5F0, + 0x1E6E30001E6E4, + 0x1E6E60001E6E7, + 0x1E6EE0001E6F0, + 0x1E6F50001E6F6, + 0x1E8D00001E8D7, + 0x1E9440001E94C, + 0xE0001000E0002, + 0xE0020000E0080, + 0xE0100000E01F0, + ), +} + + +codepoint_classes = { + "PVALID": ( + 0x2D0000002E, + 0x300000003A, + 0x610000007B, + 0xDF000000F7, + 0xF800000100, + 0x10100000102, + 0x10300000104, + 0x10500000106, + 0x10700000108, + 0x1090000010A, + 0x10B0000010C, + 0x10D0000010E, + 0x10F00000110, + 0x11100000112, + 0x11300000114, + 0x11500000116, + 0x11700000118, + 0x1190000011A, + 0x11B0000011C, + 0x11D0000011E, + 0x11F00000120, + 0x12100000122, + 0x12300000124, + 0x12500000126, + 0x12700000128, + 0x1290000012A, + 0x12B0000012C, + 0x12D0000012E, + 0x12F00000130, + 0x13100000132, + 0x13500000136, + 0x13700000139, + 0x13A0000013B, + 0x13C0000013D, + 0x13E0000013F, + 0x14200000143, + 0x14400000145, + 0x14600000147, + 0x14800000149, + 0x14B0000014C, + 0x14D0000014E, + 0x14F00000150, + 0x15100000152, + 0x15300000154, + 0x15500000156, + 0x15700000158, + 0x1590000015A, + 0x15B0000015C, + 0x15D0000015E, + 0x15F00000160, + 0x16100000162, + 0x16300000164, + 0x16500000166, + 0x16700000168, + 0x1690000016A, + 0x16B0000016C, + 0x16D0000016E, + 0x16F00000170, + 0x17100000172, + 0x17300000174, + 0x17500000176, + 0x17700000178, + 0x17A0000017B, + 0x17C0000017D, + 0x17E0000017F, + 0x18000000181, + 0x18300000184, + 0x18500000186, + 0x18800000189, + 0x18C0000018E, + 0x19200000193, + 0x19500000196, + 0x1990000019C, + 0x19E0000019F, + 0x1A1000001A2, + 0x1A3000001A4, + 0x1A5000001A6, + 0x1A8000001A9, + 0x1AA000001AC, + 0x1AD000001AE, + 0x1B0000001B1, + 0x1B4000001B5, + 0x1B6000001B7, + 0x1B9000001BC, + 0x1BD000001C4, + 0x1CE000001CF, + 0x1D0000001D1, + 0x1D2000001D3, + 0x1D4000001D5, + 0x1D6000001D7, + 0x1D8000001D9, + 0x1DA000001DB, + 0x1DC000001DE, + 0x1DF000001E0, + 0x1E1000001E2, + 0x1E3000001E4, + 0x1E5000001E6, + 0x1E7000001E8, + 0x1E9000001EA, + 0x1EB000001EC, + 0x1ED000001EE, + 0x1EF000001F1, + 0x1F5000001F6, + 0x1F9000001FA, + 0x1FB000001FC, + 0x1FD000001FE, + 0x1FF00000200, + 0x20100000202, + 0x20300000204, + 0x20500000206, + 0x20700000208, + 0x2090000020A, + 0x20B0000020C, + 0x20D0000020E, + 0x20F00000210, + 0x21100000212, + 0x21300000214, + 0x21500000216, + 0x21700000218, + 0x2190000021A, + 0x21B0000021C, + 0x21D0000021E, + 0x21F00000220, + 0x22100000222, + 0x22300000224, + 0x22500000226, + 0x22700000228, + 0x2290000022A, + 0x22B0000022C, + 0x22D0000022E, + 0x22F00000230, + 0x23100000232, + 0x2330000023A, + 0x23C0000023D, + 0x23F00000241, + 0x24200000243, + 0x24700000248, + 0x2490000024A, + 0x24B0000024C, + 0x24D0000024E, + 0x24F000002B0, + 0x2B9000002C2, + 0x2C6000002D2, + 0x2EC000002ED, + 0x2EE000002EF, + 0x30000000340, + 0x34200000343, + 0x3460000034F, + 0x35000000370, + 0x37100000372, + 0x37300000374, + 0x37700000378, + 0x37B0000037E, + 0x39000000391, + 0x3AC000003CF, + 0x3D7000003D8, + 0x3D9000003DA, + 0x3DB000003DC, + 0x3DD000003DE, + 0x3DF000003E0, + 0x3E1000003E2, + 0x3E3000003E4, + 0x3E5000003E6, + 0x3E7000003E8, + 0x3E9000003EA, + 0x3EB000003EC, + 0x3ED000003EE, + 0x3EF000003F0, + 0x3F3000003F4, + 0x3F8000003F9, + 0x3FB000003FD, + 0x43000000460, + 0x46100000462, + 0x46300000464, + 0x46500000466, + 0x46700000468, + 0x4690000046A, + 0x46B0000046C, + 0x46D0000046E, + 0x46F00000470, + 0x47100000472, + 0x47300000474, + 0x47500000476, + 0x47700000478, + 0x4790000047A, + 0x47B0000047C, + 0x47D0000047E, + 0x47F00000480, + 0x48100000482, + 0x48300000488, + 0x48B0000048C, + 0x48D0000048E, + 0x48F00000490, + 0x49100000492, + 0x49300000494, + 0x49500000496, + 0x49700000498, + 0x4990000049A, + 0x49B0000049C, + 0x49D0000049E, + 0x49F000004A0, + 0x4A1000004A2, + 0x4A3000004A4, + 0x4A5000004A6, + 0x4A7000004A8, + 0x4A9000004AA, + 0x4AB000004AC, + 0x4AD000004AE, + 0x4AF000004B0, + 0x4B1000004B2, + 0x4B3000004B4, + 0x4B5000004B6, + 0x4B7000004B8, + 0x4B9000004BA, + 0x4BB000004BC, + 0x4BD000004BE, + 0x4BF000004C0, + 0x4C2000004C3, + 0x4C4000004C5, + 0x4C6000004C7, + 0x4C8000004C9, + 0x4CA000004CB, + 0x4CC000004CD, + 0x4CE000004D0, + 0x4D1000004D2, + 0x4D3000004D4, + 0x4D5000004D6, + 0x4D7000004D8, + 0x4D9000004DA, + 0x4DB000004DC, + 0x4DD000004DE, + 0x4DF000004E0, + 0x4E1000004E2, + 0x4E3000004E4, + 0x4E5000004E6, + 0x4E7000004E8, + 0x4E9000004EA, + 0x4EB000004EC, + 0x4ED000004EE, + 0x4EF000004F0, + 0x4F1000004F2, + 0x4F3000004F4, + 0x4F5000004F6, + 0x4F7000004F8, + 0x4F9000004FA, + 0x4FB000004FC, + 0x4FD000004FE, + 0x4FF00000500, + 0x50100000502, + 0x50300000504, + 0x50500000506, + 0x50700000508, + 0x5090000050A, + 0x50B0000050C, + 0x50D0000050E, + 0x50F00000510, + 0x51100000512, + 0x51300000514, + 0x51500000516, + 0x51700000518, + 0x5190000051A, + 0x51B0000051C, + 0x51D0000051E, + 0x51F00000520, + 0x52100000522, + 0x52300000524, + 0x52500000526, + 0x52700000528, + 0x5290000052A, + 0x52B0000052C, + 0x52D0000052E, + 0x52F00000530, + 0x5590000055A, + 0x56000000587, + 0x58800000589, + 0x591000005BE, + 0x5BF000005C0, + 0x5C1000005C3, + 0x5C4000005C6, + 0x5C7000005C8, + 0x5D0000005EB, + 0x5EF000005F3, + 0x6100000061B, + 0x62000000640, + 0x64100000660, + 0x66E00000675, + 0x679000006D4, + 0x6D5000006DD, + 0x6DF000006E9, + 0x6EA000006F0, + 0x6FA00000700, + 0x7100000074B, + 0x74D000007B2, + 0x7C0000007F6, + 0x7FD000007FE, + 0x8000000082E, + 0x8400000085C, + 0x8600000086B, + 0x87000000888, + 0x88900000890, + 0x897000008E2, + 0x8E300000958, + 0x96000000964, + 0x96600000970, + 0x97100000984, + 0x9850000098D, + 0x98F00000991, + 0x993000009A9, + 0x9AA000009B1, + 0x9B2000009B3, + 0x9B6000009BA, + 0x9BC000009C5, + 0x9C7000009C9, + 0x9CB000009CF, + 0x9D7000009D8, + 0x9E0000009E4, + 0x9E6000009F2, + 0x9FC000009FD, + 0x9FE000009FF, + 0xA0100000A04, + 0xA0500000A0B, + 0xA0F00000A11, + 0xA1300000A29, + 0xA2A00000A31, + 0xA3200000A33, + 0xA3500000A36, + 0xA3800000A3A, + 0xA3C00000A3D, + 0xA3E00000A43, + 0xA4700000A49, + 0xA4B00000A4E, + 0xA5100000A52, + 0xA5C00000A5D, + 0xA6600000A76, + 0xA8100000A84, + 0xA8500000A8E, + 0xA8F00000A92, + 0xA9300000AA9, + 0xAAA00000AB1, + 0xAB200000AB4, + 0xAB500000ABA, + 0xABC00000AC6, + 0xAC700000ACA, + 0xACB00000ACE, + 0xAD000000AD1, + 0xAE000000AE4, + 0xAE600000AF0, + 0xAF900000B00, + 0xB0100000B04, + 0xB0500000B0D, + 0xB0F00000B11, + 0xB1300000B29, + 0xB2A00000B31, + 0xB3200000B34, + 0xB3500000B3A, + 0xB3C00000B45, + 0xB4700000B49, + 0xB4B00000B4E, + 0xB5500000B58, + 0xB5F00000B64, + 0xB6600000B70, + 0xB7100000B72, + 0xB8200000B84, + 0xB8500000B8B, + 0xB8E00000B91, + 0xB9200000B96, + 0xB9900000B9B, + 0xB9C00000B9D, + 0xB9E00000BA0, + 0xBA300000BA5, + 0xBA800000BAB, + 0xBAE00000BBA, + 0xBBE00000BC3, + 0xBC600000BC9, + 0xBCA00000BCE, + 0xBD000000BD1, + 0xBD700000BD8, + 0xBE600000BF0, + 0xC0000000C0D, + 0xC0E00000C11, + 0xC1200000C29, + 0xC2A00000C3A, + 0xC3C00000C45, + 0xC4600000C49, + 0xC4A00000C4E, + 0xC5500000C57, + 0xC5800000C5B, + 0xC5C00000C5E, + 0xC6000000C64, + 0xC6600000C70, + 0xC8000000C84, + 0xC8500000C8D, + 0xC8E00000C91, + 0xC9200000CA9, + 0xCAA00000CB4, + 0xCB500000CBA, + 0xCBC00000CC5, + 0xCC600000CC9, + 0xCCA00000CCE, + 0xCD500000CD7, + 0xCDC00000CDF, + 0xCE000000CE4, + 0xCE600000CF0, + 0xCF100000CF4, + 0xD0000000D0D, + 0xD0E00000D11, + 0xD1200000D45, + 0xD4600000D49, + 0xD4A00000D4F, + 0xD5400000D58, + 0xD5F00000D64, + 0xD6600000D70, + 0xD7A00000D80, + 0xD8100000D84, + 0xD8500000D97, + 0xD9A00000DB2, + 0xDB300000DBC, + 0xDBD00000DBE, + 0xDC000000DC7, + 0xDCA00000DCB, + 0xDCF00000DD5, + 0xDD600000DD7, + 0xDD800000DE0, + 0xDE600000DF0, + 0xDF200000DF4, + 0xE0100000E33, + 0xE3400000E3B, + 0xE4000000E4F, + 0xE5000000E5A, + 0xE8100000E83, + 0xE8400000E85, + 0xE8600000E8B, + 0xE8C00000EA4, + 0xEA500000EA6, + 0xEA700000EB3, + 0xEB400000EBE, + 0xEC000000EC5, + 0xEC600000EC7, + 0xEC800000ECF, + 0xED000000EDA, + 0xEDE00000EE0, + 0xF0000000F01, + 0xF0B00000F0C, + 0xF1800000F1A, + 0xF2000000F2A, + 0xF3500000F36, + 0xF3700000F38, + 0xF3900000F3A, + 0xF3E00000F43, + 0xF4400000F48, + 0xF4900000F4D, + 0xF4E00000F52, + 0xF5300000F57, + 0xF5800000F5C, + 0xF5D00000F69, + 0xF6A00000F6D, + 0xF7100000F73, + 0xF7400000F75, + 0xF7A00000F81, + 0xF8200000F85, + 0xF8600000F93, + 0xF9400000F98, + 0xF9900000F9D, + 0xF9E00000FA2, + 0xFA300000FA7, + 0xFA800000FAC, + 0xFAD00000FB9, + 0xFBA00000FBD, + 0xFC600000FC7, + 0x10000000104A, + 0x10500000109E, + 0x10D0000010FB, + 0x10FD00001100, + 0x120000001249, + 0x124A0000124E, + 0x125000001257, + 0x125800001259, + 0x125A0000125E, + 0x126000001289, + 0x128A0000128E, + 0x1290000012B1, + 0x12B2000012B6, + 0x12B8000012BF, + 0x12C0000012C1, + 0x12C2000012C6, + 0x12C8000012D7, + 0x12D800001311, + 0x131200001316, + 0x13180000135B, + 0x135D00001360, + 0x138000001390, + 0x13A0000013F6, + 0x14010000166D, + 0x166F00001680, + 0x16810000169B, + 0x16A0000016EB, + 0x16F1000016F9, + 0x170000001716, + 0x171F00001735, + 0x174000001754, + 0x17600000176D, + 0x176E00001771, + 0x177200001774, + 0x1780000017B4, + 0x17B6000017D4, + 0x17D7000017D8, + 0x17DC000017DE, + 0x17E0000017EA, + 0x18100000181A, + 0x182000001879, + 0x1880000018AB, + 0x18B0000018F6, + 0x19000000191F, + 0x19200000192C, + 0x19300000193C, + 0x19460000196E, + 0x197000001975, + 0x1980000019AC, + 0x19B0000019CA, + 0x19D0000019DA, + 0x1A0000001A1C, + 0x1A2000001A5F, + 0x1A6000001A7D, + 0x1A7F00001A8A, + 0x1A9000001A9A, + 0x1AA700001AA8, + 0x1AB000001ABE, + 0x1ABF00001ADE, + 0x1AE000001AEC, + 0x1B0000001B4D, + 0x1B5000001B5A, + 0x1B6B00001B74, + 0x1B8000001BF4, + 0x1C0000001C38, + 0x1C4000001C4A, + 0x1C4D00001C7E, + 0x1C8A00001C8B, + 0x1CD000001CD3, + 0x1CD400001CFB, + 0x1D0000001D2C, + 0x1D2F00001D30, + 0x1D3B00001D3C, + 0x1D4E00001D4F, + 0x1D6B00001D78, + 0x1D7900001D9B, + 0x1DC000001E00, + 0x1E0100001E02, + 0x1E0300001E04, + 0x1E0500001E06, + 0x1E0700001E08, + 0x1E0900001E0A, + 0x1E0B00001E0C, + 0x1E0D00001E0E, + 0x1E0F00001E10, + 0x1E1100001E12, + 0x1E1300001E14, + 0x1E1500001E16, + 0x1E1700001E18, + 0x1E1900001E1A, + 0x1E1B00001E1C, + 0x1E1D00001E1E, + 0x1E1F00001E20, + 0x1E2100001E22, + 0x1E2300001E24, + 0x1E2500001E26, + 0x1E2700001E28, + 0x1E2900001E2A, + 0x1E2B00001E2C, + 0x1E2D00001E2E, + 0x1E2F00001E30, + 0x1E3100001E32, + 0x1E3300001E34, + 0x1E3500001E36, + 0x1E3700001E38, + 0x1E3900001E3A, + 0x1E3B00001E3C, + 0x1E3D00001E3E, + 0x1E3F00001E40, + 0x1E4100001E42, + 0x1E4300001E44, + 0x1E4500001E46, + 0x1E4700001E48, + 0x1E4900001E4A, + 0x1E4B00001E4C, + 0x1E4D00001E4E, + 0x1E4F00001E50, + 0x1E5100001E52, + 0x1E5300001E54, + 0x1E5500001E56, + 0x1E5700001E58, + 0x1E5900001E5A, + 0x1E5B00001E5C, + 0x1E5D00001E5E, + 0x1E5F00001E60, + 0x1E6100001E62, + 0x1E6300001E64, + 0x1E6500001E66, + 0x1E6700001E68, + 0x1E6900001E6A, + 0x1E6B00001E6C, + 0x1E6D00001E6E, + 0x1E6F00001E70, + 0x1E7100001E72, + 0x1E7300001E74, + 0x1E7500001E76, + 0x1E7700001E78, + 0x1E7900001E7A, + 0x1E7B00001E7C, + 0x1E7D00001E7E, + 0x1E7F00001E80, + 0x1E8100001E82, + 0x1E8300001E84, + 0x1E8500001E86, + 0x1E8700001E88, + 0x1E8900001E8A, + 0x1E8B00001E8C, + 0x1E8D00001E8E, + 0x1E8F00001E90, + 0x1E9100001E92, + 0x1E9300001E94, + 0x1E9500001E9A, + 0x1E9C00001E9E, + 0x1E9F00001EA0, + 0x1EA100001EA2, + 0x1EA300001EA4, + 0x1EA500001EA6, + 0x1EA700001EA8, + 0x1EA900001EAA, + 0x1EAB00001EAC, + 0x1EAD00001EAE, + 0x1EAF00001EB0, + 0x1EB100001EB2, + 0x1EB300001EB4, + 0x1EB500001EB6, + 0x1EB700001EB8, + 0x1EB900001EBA, + 0x1EBB00001EBC, + 0x1EBD00001EBE, + 0x1EBF00001EC0, + 0x1EC100001EC2, + 0x1EC300001EC4, + 0x1EC500001EC6, + 0x1EC700001EC8, + 0x1EC900001ECA, + 0x1ECB00001ECC, + 0x1ECD00001ECE, + 0x1ECF00001ED0, + 0x1ED100001ED2, + 0x1ED300001ED4, + 0x1ED500001ED6, + 0x1ED700001ED8, + 0x1ED900001EDA, + 0x1EDB00001EDC, + 0x1EDD00001EDE, + 0x1EDF00001EE0, + 0x1EE100001EE2, + 0x1EE300001EE4, + 0x1EE500001EE6, + 0x1EE700001EE8, + 0x1EE900001EEA, + 0x1EEB00001EEC, + 0x1EED00001EEE, + 0x1EEF00001EF0, + 0x1EF100001EF2, + 0x1EF300001EF4, + 0x1EF500001EF6, + 0x1EF700001EF8, + 0x1EF900001EFA, + 0x1EFB00001EFC, + 0x1EFD00001EFE, + 0x1EFF00001F08, + 0x1F1000001F16, + 0x1F2000001F28, + 0x1F3000001F38, + 0x1F4000001F46, + 0x1F5000001F58, + 0x1F6000001F68, + 0x1F7000001F71, + 0x1F7200001F73, + 0x1F7400001F75, + 0x1F7600001F77, + 0x1F7800001F79, + 0x1F7A00001F7B, + 0x1F7C00001F7D, + 0x1FB000001FB2, + 0x1FB600001FB7, + 0x1FC600001FC7, + 0x1FD000001FD3, + 0x1FD600001FD8, + 0x1FE000001FE3, + 0x1FE400001FE8, + 0x1FF600001FF7, + 0x214E0000214F, + 0x218400002185, + 0x2C3000002C60, + 0x2C6100002C62, + 0x2C6500002C67, + 0x2C6800002C69, + 0x2C6A00002C6B, + 0x2C6C00002C6D, + 0x2C7100002C72, + 0x2C7300002C75, + 0x2C7600002C7C, + 0x2C8100002C82, + 0x2C8300002C84, + 0x2C8500002C86, + 0x2C8700002C88, + 0x2C8900002C8A, + 0x2C8B00002C8C, + 0x2C8D00002C8E, + 0x2C8F00002C90, + 0x2C9100002C92, + 0x2C9300002C94, + 0x2C9500002C96, + 0x2C9700002C98, + 0x2C9900002C9A, + 0x2C9B00002C9C, + 0x2C9D00002C9E, + 0x2C9F00002CA0, + 0x2CA100002CA2, + 0x2CA300002CA4, + 0x2CA500002CA6, + 0x2CA700002CA8, + 0x2CA900002CAA, + 0x2CAB00002CAC, + 0x2CAD00002CAE, + 0x2CAF00002CB0, + 0x2CB100002CB2, + 0x2CB300002CB4, + 0x2CB500002CB6, + 0x2CB700002CB8, + 0x2CB900002CBA, + 0x2CBB00002CBC, + 0x2CBD00002CBE, + 0x2CBF00002CC0, + 0x2CC100002CC2, + 0x2CC300002CC4, + 0x2CC500002CC6, + 0x2CC700002CC8, + 0x2CC900002CCA, + 0x2CCB00002CCC, + 0x2CCD00002CCE, + 0x2CCF00002CD0, + 0x2CD100002CD2, + 0x2CD300002CD4, + 0x2CD500002CD6, + 0x2CD700002CD8, + 0x2CD900002CDA, + 0x2CDB00002CDC, + 0x2CDD00002CDE, + 0x2CDF00002CE0, + 0x2CE100002CE2, + 0x2CE300002CE5, + 0x2CEC00002CED, + 0x2CEE00002CF2, + 0x2CF300002CF4, + 0x2D0000002D26, + 0x2D2700002D28, + 0x2D2D00002D2E, + 0x2D3000002D68, + 0x2D7F00002D97, + 0x2DA000002DA7, + 0x2DA800002DAF, + 0x2DB000002DB7, + 0x2DB800002DBF, + 0x2DC000002DC7, + 0x2DC800002DCF, + 0x2DD000002DD7, + 0x2DD800002DDF, + 0x2DE000002E00, + 0x2E2F00002E30, + 0x300500003008, + 0x302A0000302E, + 0x303C0000303D, + 0x304100003097, + 0x30990000309B, + 0x309D0000309F, + 0x30A1000030FB, + 0x30FC000030FF, + 0x310500003130, + 0x31A0000031C0, + 0x31F000003200, + 0x340000004DC0, + 0x4E000000A48D, + 0xA4D00000A4FE, + 0xA5000000A60D, + 0xA6100000A62C, + 0xA6410000A642, + 0xA6430000A644, + 0xA6450000A646, + 0xA6470000A648, + 0xA6490000A64A, + 0xA64B0000A64C, + 0xA64D0000A64E, + 0xA64F0000A650, + 0xA6510000A652, + 0xA6530000A654, + 0xA6550000A656, + 0xA6570000A658, + 0xA6590000A65A, + 0xA65B0000A65C, + 0xA65D0000A65E, + 0xA65F0000A660, + 0xA6610000A662, + 0xA6630000A664, + 0xA6650000A666, + 0xA6670000A668, + 0xA6690000A66A, + 0xA66B0000A66C, + 0xA66D0000A670, + 0xA6740000A67E, + 0xA67F0000A680, + 0xA6810000A682, + 0xA6830000A684, + 0xA6850000A686, + 0xA6870000A688, + 0xA6890000A68A, + 0xA68B0000A68C, + 0xA68D0000A68E, + 0xA68F0000A690, + 0xA6910000A692, + 0xA6930000A694, + 0xA6950000A696, + 0xA6970000A698, + 0xA6990000A69A, + 0xA69B0000A69C, + 0xA69E0000A6E6, + 0xA6F00000A6F2, + 0xA7170000A720, + 0xA7230000A724, + 0xA7250000A726, + 0xA7270000A728, + 0xA7290000A72A, + 0xA72B0000A72C, + 0xA72D0000A72E, + 0xA72F0000A732, + 0xA7330000A734, + 0xA7350000A736, + 0xA7370000A738, + 0xA7390000A73A, + 0xA73B0000A73C, + 0xA73D0000A73E, + 0xA73F0000A740, + 0xA7410000A742, + 0xA7430000A744, + 0xA7450000A746, + 0xA7470000A748, + 0xA7490000A74A, + 0xA74B0000A74C, + 0xA74D0000A74E, + 0xA74F0000A750, + 0xA7510000A752, + 0xA7530000A754, + 0xA7550000A756, + 0xA7570000A758, + 0xA7590000A75A, + 0xA75B0000A75C, + 0xA75D0000A75E, + 0xA75F0000A760, + 0xA7610000A762, + 0xA7630000A764, + 0xA7650000A766, + 0xA7670000A768, + 0xA7690000A76A, + 0xA76B0000A76C, + 0xA76D0000A76E, + 0xA76F0000A770, + 0xA7710000A779, + 0xA77A0000A77B, + 0xA77C0000A77D, + 0xA77F0000A780, + 0xA7810000A782, + 0xA7830000A784, + 0xA7850000A786, + 0xA7870000A789, + 0xA78C0000A78D, + 0xA78E0000A790, + 0xA7910000A792, + 0xA7930000A796, + 0xA7970000A798, + 0xA7990000A79A, + 0xA79B0000A79C, + 0xA79D0000A79E, + 0xA79F0000A7A0, + 0xA7A10000A7A2, + 0xA7A30000A7A4, + 0xA7A50000A7A6, + 0xA7A70000A7A8, + 0xA7A90000A7AA, + 0xA7AF0000A7B0, + 0xA7B50000A7B6, + 0xA7B70000A7B8, + 0xA7B90000A7BA, + 0xA7BB0000A7BC, + 0xA7BD0000A7BE, + 0xA7BF0000A7C0, + 0xA7C10000A7C2, + 0xA7C30000A7C4, + 0xA7C80000A7C9, + 0xA7CA0000A7CB, + 0xA7CD0000A7CE, + 0xA7CF0000A7D0, + 0xA7D10000A7D2, + 0xA7D30000A7D4, + 0xA7D50000A7D6, + 0xA7D70000A7D8, + 0xA7D90000A7DA, + 0xA7DB0000A7DC, + 0xA7F60000A7F8, + 0xA7FA0000A828, + 0xA82C0000A82D, + 0xA8400000A874, + 0xA8800000A8C6, + 0xA8D00000A8DA, + 0xA8E00000A8F8, + 0xA8FB0000A8FC, + 0xA8FD0000A92E, + 0xA9300000A954, + 0xA9800000A9C1, + 0xA9CF0000A9DA, + 0xA9E00000A9FF, + 0xAA000000AA37, + 0xAA400000AA4E, + 0xAA500000AA5A, + 0xAA600000AA77, + 0xAA7A0000AAC3, + 0xAADB0000AADE, + 0xAAE00000AAF0, + 0xAAF20000AAF7, + 0xAB010000AB07, + 0xAB090000AB0F, + 0xAB110000AB17, + 0xAB200000AB27, + 0xAB280000AB2F, + 0xAB300000AB5B, + 0xAB600000AB69, + 0xABC00000ABEB, + 0xABEC0000ABEE, + 0xABF00000ABFA, + 0xAC000000D7A4, + 0xFA0E0000FA10, + 0xFA110000FA12, + 0xFA130000FA15, + 0xFA1F0000FA20, + 0xFA210000FA22, + 0xFA230000FA25, + 0xFA270000FA2A, + 0xFB1E0000FB1F, + 0xFE200000FE30, + 0xFE730000FE74, + 0x100000001000C, + 0x1000D00010027, + 0x100280001003B, + 0x1003C0001003E, + 0x1003F0001004E, + 0x100500001005E, + 0x10080000100FB, + 0x101FD000101FE, + 0x102800001029D, + 0x102A0000102D1, + 0x102E0000102E1, + 0x1030000010320, + 0x1032D00010341, + 0x103420001034A, + 0x103500001037B, + 0x103800001039E, + 0x103A0000103C4, + 0x103C8000103D0, + 0x104280001049E, + 0x104A0000104AA, + 0x104D8000104FC, + 0x1050000010528, + 0x1053000010564, + 0x10597000105A2, + 0x105A3000105B2, + 0x105B3000105BA, + 0x105BB000105BD, + 0x105C0000105F4, + 0x1060000010737, + 0x1074000010756, + 0x1076000010768, + 0x1078000010781, + 0x1080000010806, + 0x1080800010809, + 0x1080A00010836, + 0x1083700010839, + 0x1083C0001083D, + 0x1083F00010856, + 0x1086000010877, + 0x108800001089F, + 0x108E0000108F3, + 0x108F4000108F6, + 0x1090000010916, + 0x109200001093A, + 0x109400001095A, + 0x10980000109B8, + 0x109BE000109C0, + 0x10A0000010A04, + 0x10A0500010A07, + 0x10A0C00010A14, + 0x10A1500010A18, + 0x10A1900010A36, + 0x10A3800010A3B, + 0x10A3F00010A40, + 0x10A6000010A7D, + 0x10A8000010A9D, + 0x10AC000010AC8, + 0x10AC900010AE7, + 0x10B0000010B36, + 0x10B4000010B56, + 0x10B6000010B73, + 0x10B8000010B92, + 0x10C0000010C49, + 0x10CC000010CF3, + 0x10D0000010D28, + 0x10D3000010D3A, + 0x10D4000010D50, + 0x10D6900010D6E, + 0x10D6F00010D86, + 0x10E8000010EAA, + 0x10EAB00010EAD, + 0x10EB000010EB2, + 0x10EC200010EC8, + 0x10EFA00010F1D, + 0x10F2700010F28, + 0x10F3000010F51, + 0x10F7000010F86, + 0x10FB000010FC5, + 0x10FE000010FF7, + 0x1100000011047, + 0x1106600011076, + 0x1107F000110BB, + 0x110C2000110C3, + 0x110D0000110E9, + 0x110F0000110FA, + 0x1110000011135, + 0x1113600011140, + 0x1114400011148, + 0x1115000011174, + 0x1117600011177, + 0x11180000111C5, + 0x111C9000111CD, + 0x111CE000111DB, + 0x111DC000111DD, + 0x1120000011212, + 0x1121300011238, + 0x1123E00011242, + 0x1128000011287, + 0x1128800011289, + 0x1128A0001128E, + 0x1128F0001129E, + 0x1129F000112A9, + 0x112B0000112EB, + 0x112F0000112FA, + 0x1130000011304, + 0x113050001130D, + 0x1130F00011311, + 0x1131300011329, + 0x1132A00011331, + 0x1133200011334, + 0x113350001133A, + 0x1133B00011345, + 0x1134700011349, + 0x1134B0001134E, + 0x1135000011351, + 0x1135700011358, + 0x1135D00011364, + 0x113660001136D, + 0x1137000011375, + 0x113800001138A, + 0x1138B0001138C, + 0x1138E0001138F, + 0x11390000113B6, + 0x113B7000113C1, + 0x113C2000113C3, + 0x113C5000113C6, + 0x113C7000113CB, + 0x113CC000113D4, + 0x113E1000113E3, + 0x114000001144B, + 0x114500001145A, + 0x1145E00011462, + 0x11480000114C6, + 0x114C7000114C8, + 0x114D0000114DA, + 0x11580000115B6, + 0x115B8000115C1, + 0x115D8000115DE, + 0x1160000011641, + 0x1164400011645, + 0x116500001165A, + 0x11680000116B9, + 0x116C0000116CA, + 0x116D0000116E4, + 0x117000001171B, + 0x1171D0001172C, + 0x117300001173A, + 0x1174000011747, + 0x118000001183B, + 0x118C0000118EA, + 0x118FF00011907, + 0x119090001190A, + 0x1190C00011914, + 0x1191500011917, + 0x1191800011936, + 0x1193700011939, + 0x1193B00011944, + 0x119500001195A, + 0x119A0000119A8, + 0x119AA000119D8, + 0x119DA000119E2, + 0x119E3000119E5, + 0x11A0000011A3F, + 0x11A4700011A48, + 0x11A5000011A9A, + 0x11A9D00011A9E, + 0x11AB000011AF9, + 0x11B6000011B68, + 0x11BC000011BE1, + 0x11BF000011BFA, + 0x11C0000011C09, + 0x11C0A00011C37, + 0x11C3800011C41, + 0x11C5000011C5A, + 0x11C7200011C90, + 0x11C9200011CA8, + 0x11CA900011CB7, + 0x11D0000011D07, + 0x11D0800011D0A, + 0x11D0B00011D37, + 0x11D3A00011D3B, + 0x11D3C00011D3E, + 0x11D3F00011D48, + 0x11D5000011D5A, + 0x11D6000011D66, + 0x11D6700011D69, + 0x11D6A00011D8F, + 0x11D9000011D92, + 0x11D9300011D99, + 0x11DA000011DAA, + 0x11DB000011DDC, + 0x11DE000011DEA, + 0x11EE000011EF7, + 0x11F0000011F11, + 0x11F1200011F3B, + 0x11F3E00011F43, + 0x11F5000011F5B, + 0x11FB000011FB1, + 0x120000001239A, + 0x1248000012544, + 0x12F9000012FF1, + 0x1300000013430, + 0x1344000013456, + 0x13460000143FB, + 0x1440000014647, + 0x161000001613A, + 0x1680000016A39, + 0x16A4000016A5F, + 0x16A6000016A6A, + 0x16A7000016ABF, + 0x16AC000016ACA, + 0x16AD000016AEE, + 0x16AF000016AF5, + 0x16B0000016B37, + 0x16B4000016B44, + 0x16B5000016B5A, + 0x16B6300016B78, + 0x16B7D00016B90, + 0x16D4000016D6D, + 0x16D7000016D7A, + 0x16E6000016E80, + 0x16EBB00016ED4, + 0x16F0000016F4B, + 0x16F4F00016F88, + 0x16F8F00016FA0, + 0x16FE000016FE2, + 0x16FE300016FE5, + 0x16FF000016FF4, + 0x1700000018CD6, + 0x18CFF00018D1F, + 0x18D8000018DF3, + 0x1AFF00001AFF4, + 0x1AFF50001AFFC, + 0x1AFFD0001AFFF, + 0x1B0000001B123, + 0x1B1320001B133, + 0x1B1500001B153, + 0x1B1550001B156, + 0x1B1640001B168, + 0x1B1700001B2FC, + 0x1BC000001BC6B, + 0x1BC700001BC7D, + 0x1BC800001BC89, + 0x1BC900001BC9A, + 0x1BC9D0001BC9F, + 0x1CF000001CF2E, + 0x1CF300001CF47, + 0x1DA000001DA37, + 0x1DA3B0001DA6D, + 0x1DA750001DA76, + 0x1DA840001DA85, + 0x1DA9B0001DAA0, + 0x1DAA10001DAB0, + 0x1DF000001DF1F, + 0x1DF250001DF2B, + 0x1E0000001E007, + 0x1E0080001E019, + 0x1E01B0001E022, + 0x1E0230001E025, + 0x1E0260001E02B, + 0x1E08F0001E090, + 0x1E1000001E12D, + 0x1E1300001E13E, + 0x1E1400001E14A, + 0x1E14E0001E14F, + 0x1E2900001E2AF, + 0x1E2C00001E2FA, + 0x1E4D00001E4FA, + 0x1E5D00001E5FB, + 0x1E6C00001E6DF, + 0x1E6E00001E6F6, + 0x1E6FE0001E700, + 0x1E7E00001E7E7, + 0x1E7E80001E7EC, + 0x1E7ED0001E7EF, + 0x1E7F00001E7FF, + 0x1E8000001E8C5, + 0x1E8D00001E8D7, + 0x1E9220001E94C, + 0x1E9500001E95A, + 0x200000002A6E0, + 0x2A7000002B81E, + 0x2B8200002CEAE, + 0x2CEB00002EBE1, + 0x2EBF00002EE5E, + 0x300000003134B, + 0x313500003347A, + ), + "CONTEXTJ": (0x200C0000200E,), + "CONTEXTO": ( + 0xB7000000B8, + 0x37500000376, + 0x5F3000005F5, + 0x6600000066A, + 0x6F0000006FA, + 0x30FB000030FC, + ), +} diff --git a/.venv/lib/python3.13/site-packages/idna/intranges.py b/.venv/lib/python3.13/site-packages/idna/intranges.py new file mode 100644 index 0000000000000000000000000000000000000000..19d77810caf332a1f310bd3b734dab7bbd82f64c --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna/intranges.py @@ -0,0 +1,55 @@ +""" +Given a list of integers, made up of (hopefully) a small number of long runs +of consecutive integers, compute a representation of the form +((start1, end1), (start2, end2) ...). Then answer the question "was x present +in the original list?" in time O(log(# runs)). +""" + +import bisect + + +def intranges_from_list(list_: list[int]) -> tuple[int, ...]: + """Represent a list of integers as a sequence of ranges: + ((start_0, end_0), (start_1, end_1), ...), such that the original + integers are exactly those x such that start_i <= x < end_i for some i. + + Ranges are encoded as single integers (start << 32 | end), not as tuples. + """ + + sorted_list = sorted(list_) + ranges = [] + last_write = -1 + for i in range(len(sorted_list)): + if i + 1 < len(sorted_list) and sorted_list[i] == sorted_list[i + 1] - 1: + continue + current_range = sorted_list[last_write + 1 : i + 1] + ranges.append(_encode_range(current_range[0], current_range[-1] + 1)) + last_write = i + + return tuple(ranges) + + +def _encode_range(start: int, end: int) -> int: + return (start << 32) | end + + +def _decode_range(r: int) -> tuple[int, int]: + return (r >> 32), (r & ((1 << 32) - 1)) + + +def intranges_contain(int_: int, ranges: tuple[int, ...]) -> bool: + """Determine if `int_` falls into one of the ranges in `ranges`.""" + tuple_ = _encode_range(int_, 0) + pos = bisect.bisect_left(ranges, tuple_) + # we could be immediately ahead of a tuple (start, end) + # with start < int_ <= end + if pos > 0: + left, right = _decode_range(ranges[pos - 1]) + if left <= int_ < right: + return True + # or we could be immediately behind a tuple (int_, end) + if pos < len(ranges): + left, _ = _decode_range(ranges[pos]) + if left == int_: + return True + return False diff --git a/.venv/lib/python3.13/site-packages/idna/package_data.py b/.venv/lib/python3.13/site-packages/idna/package_data.py new file mode 100644 index 0000000000000000000000000000000000000000..94e40398cda6322e7c348ce8c2c44b80a904e5c7 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna/package_data.py @@ -0,0 +1 @@ +__version__ = "3.18" diff --git a/.venv/lib/python3.13/site-packages/idna/py.typed b/.venv/lib/python3.13/site-packages/idna/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.13/site-packages/idna/uts46data.py b/.venv/lib/python3.13/site-packages/idna/uts46data.py new file mode 100644 index 0000000000000000000000000000000000000000..f2d931fe5d23c043a47687c480cffffd069e7aa6 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/idna/uts46data.py @@ -0,0 +1,16896 @@ +# This file is automatically generated by tools/idna-data + +from array import array +from typing import Optional + +"""IDNA Mapping Table from UTS46.""" + + +__version__ = "17.0.0" + +uts46_starts: "array[int]" = array( + "I", + ( + 0x0, + 0x1, + 0x2, + 0x3, + 0x4, + 0x5, + 0x6, + 0x7, + 0x8, + 0x9, + 0xA, + 0xB, + 0xC, + 0xD, + 0xE, + 0xF, + 0x10, + 0x11, + 0x12, + 0x13, + 0x14, + 0x15, + 0x16, + 0x17, + 0x18, + 0x19, + 0x1A, + 0x1B, + 0x1C, + 0x1D, + 0x1E, + 0x1F, + 0x20, + 0x21, + 0x22, + 0x23, + 0x24, + 0x25, + 0x26, + 0x27, + 0x28, + 0x29, + 0x2A, + 0x2B, + 0x2C, + 0x2D, + 0x2E, + 0x2F, + 0x30, + 0x31, + 0x32, + 0x33, + 0x34, + 0x35, + 0x36, + 0x37, + 0x38, + 0x39, + 0x3A, + 0x3B, + 0x3C, + 0x3D, + 0x3E, + 0x3F, + 0x40, + 0x41, + 0x42, + 0x43, + 0x44, + 0x45, + 0x46, + 0x47, + 0x48, + 0x49, + 0x4A, + 0x4B, + 0x4C, + 0x4D, + 0x4E, + 0x4F, + 0x50, + 0x51, + 0x52, + 0x53, + 0x54, + 0x55, + 0x56, + 0x57, + 0x58, + 0x59, + 0x5A, + 0x5B, + 0x5C, + 0x5D, + 0x5E, + 0x5F, + 0x60, + 0x61, + 0x62, + 0x63, + 0x64, + 0x65, + 0x66, + 0x67, + 0x68, + 0x69, + 0x6A, + 0x6B, + 0x6C, + 0x6D, + 0x6E, + 0x6F, + 0x70, + 0x71, + 0x72, + 0x73, + 0x74, + 0x75, + 0x76, + 0x77, + 0x78, + 0x79, + 0x7A, + 0x7B, + 0x7C, + 0x7D, + 0x7E, + 0x7F, + 0x80, + 0x81, + 0x82, + 0x83, + 0x84, + 0x85, + 0x86, + 0x87, + 0x88, + 0x89, + 0x8A, + 0x8B, + 0x8C, + 0x8D, + 0x8E, + 0x8F, + 0x90, + 0x91, + 0x92, + 0x93, + 0x94, + 0x95, + 0x96, + 0x97, + 0x98, + 0x99, + 0x9A, + 0x9B, + 0x9C, + 0x9D, + 0x9E, + 0x9F, + 0xA0, + 0xA1, + 0xA2, + 0xA3, + 0xA4, + 0xA5, + 0xA6, + 0xA7, + 0xA8, + 0xA9, + 0xAA, + 0xAB, + 0xAC, + 0xAD, + 0xAE, + 0xAF, + 0xB0, + 0xB1, + 0xB2, + 0xB3, + 0xB4, + 0xB5, + 0xB6, + 0xB7, + 0xB8, + 0xB9, + 0xBA, + 0xBB, + 0xBC, + 0xBD, + 0xBE, + 0xBF, + 0xC0, + 0xC1, + 0xC2, + 0xC3, + 0xC4, + 0xC5, + 0xC6, + 0xC7, + 0xC8, + 0xC9, + 0xCA, + 0xCB, + 0xCC, + 0xCD, + 0xCE, + 0xCF, + 0xD0, + 0xD1, + 0xD2, + 0xD3, + 0xD4, + 0xD5, + 0xD6, + 0xD7, + 0xD8, + 0xD9, + 0xDA, + 0xDB, + 0xDC, + 0xDD, + 0xDE, + 0xDF, + 0xE0, + 0xE1, + 0xE2, + 0xE3, + 0xE4, + 0xE5, + 0xE6, + 0xE7, + 0xE8, + 0xE9, + 0xEA, + 0xEB, + 0xEC, + 0xED, + 0xEE, + 0xEF, + 0xF0, + 0xF1, + 0xF2, + 0xF3, + 0xF4, + 0xF5, + 0xF6, + 0xF7, + 0xF8, + 0xF9, + 0xFA, + 0xFB, + 0xFC, + 0xFD, + 0xFE, + 0xFF, + 0x100, + 0x101, + 0x102, + 0x103, + 0x104, + 0x105, + 0x106, + 0x107, + 0x108, + 0x109, + 0x10A, + 0x10B, + 0x10C, + 0x10D, + 0x10E, + 0x10F, + 0x110, + 0x111, + 0x112, + 0x113, + 0x114, + 0x115, + 0x116, + 0x117, + 0x118, + 0x119, + 0x11A, + 0x11B, + 0x11C, + 0x11D, + 0x11E, + 0x11F, + 0x120, + 0x121, + 0x122, + 0x123, + 0x124, + 0x125, + 0x126, + 0x127, + 0x128, + 0x129, + 0x12A, + 0x12B, + 0x12C, + 0x12D, + 0x12E, + 0x12F, + 0x130, + 0x131, + 0x132, + 0x134, + 0x135, + 0x136, + 0x137, + 0x139, + 0x13A, + 0x13B, + 0x13C, + 0x13D, + 0x13E, + 0x13F, + 0x141, + 0x142, + 0x143, + 0x144, + 0x145, + 0x146, + 0x147, + 0x148, + 0x149, + 0x14A, + 0x14B, + 0x14C, + 0x14D, + 0x14E, + 0x14F, + 0x150, + 0x151, + 0x152, + 0x153, + 0x154, + 0x155, + 0x156, + 0x157, + 0x158, + 0x159, + 0x15A, + 0x15B, + 0x15C, + 0x15D, + 0x15E, + 0x15F, + 0x160, + 0x161, + 0x162, + 0x163, + 0x164, + 0x165, + 0x166, + 0x167, + 0x168, + 0x169, + 0x16A, + 0x16B, + 0x16C, + 0x16D, + 0x16E, + 0x16F, + 0x170, + 0x171, + 0x172, + 0x173, + 0x174, + 0x175, + 0x176, + 0x177, + 0x178, + 0x179, + 0x17A, + 0x17B, + 0x17C, + 0x17D, + 0x17E, + 0x17F, + 0x180, + 0x181, + 0x182, + 0x183, + 0x184, + 0x185, + 0x186, + 0x187, + 0x188, + 0x189, + 0x18A, + 0x18B, + 0x18C, + 0x18E, + 0x18F, + 0x190, + 0x191, + 0x192, + 0x193, + 0x194, + 0x195, + 0x196, + 0x197, + 0x198, + 0x199, + 0x19C, + 0x19D, + 0x19E, + 0x19F, + 0x1A0, + 0x1A1, + 0x1A2, + 0x1A3, + 0x1A4, + 0x1A5, + 0x1A6, + 0x1A7, + 0x1A8, + 0x1A9, + 0x1AA, + 0x1AC, + 0x1AD, + 0x1AE, + 0x1AF, + 0x1B0, + 0x1B1, + 0x1B2, + 0x1B3, + 0x1B4, + 0x1B5, + 0x1B6, + 0x1B7, + 0x1B8, + 0x1B9, + 0x1BC, + 0x1BD, + 0x1C4, + 0x1C7, + 0x1CA, + 0x1CD, + 0x1CE, + 0x1CF, + 0x1D0, + 0x1D1, + 0x1D2, + 0x1D3, + 0x1D4, + 0x1D5, + 0x1D6, + 0x1D7, + 0x1D8, + 0x1D9, + 0x1DA, + 0x1DB, + 0x1DC, + 0x1DE, + 0x1DF, + 0x1E0, + 0x1E1, + 0x1E2, + 0x1E3, + 0x1E4, + 0x1E5, + 0x1E6, + 0x1E7, + 0x1E8, + 0x1E9, + 0x1EA, + 0x1EB, + 0x1EC, + 0x1ED, + 0x1EE, + 0x1EF, + 0x1F1, + 0x1F4, + 0x1F5, + 0x1F6, + 0x1F7, + 0x1F8, + 0x1F9, + 0x1FA, + 0x1FB, + 0x1FC, + 0x1FD, + 0x1FE, + 0x1FF, + 0x200, + 0x201, + 0x202, + 0x203, + 0x204, + 0x205, + 0x206, + 0x207, + 0x208, + 0x209, + 0x20A, + 0x20B, + 0x20C, + 0x20D, + 0x20E, + 0x20F, + 0x210, + 0x211, + 0x212, + 0x213, + 0x214, + 0x215, + 0x216, + 0x217, + 0x218, + 0x219, + 0x21A, + 0x21B, + 0x21C, + 0x21D, + 0x21E, + 0x21F, + 0x220, + 0x221, + 0x222, + 0x223, + 0x224, + 0x225, + 0x226, + 0x227, + 0x228, + 0x229, + 0x22A, + 0x22B, + 0x22C, + 0x22D, + 0x22E, + 0x22F, + 0x230, + 0x231, + 0x232, + 0x233, + 0x23A, + 0x23B, + 0x23C, + 0x23D, + 0x23E, + 0x23F, + 0x241, + 0x242, + 0x243, + 0x244, + 0x245, + 0x246, + 0x247, + 0x248, + 0x249, + 0x24A, + 0x24B, + 0x24C, + 0x24D, + 0x24E, + 0x24F, + 0x2B0, + 0x2B1, + 0x2B2, + 0x2B3, + 0x2B4, + 0x2B5, + 0x2B6, + 0x2B7, + 0x2B8, + 0x2B9, + 0x2D8, + 0x2D9, + 0x2DA, + 0x2DB, + 0x2DC, + 0x2DD, + 0x2DE, + 0x2E0, + 0x2E1, + 0x2E2, + 0x2E3, + 0x2E4, + 0x2E5, + 0x340, + 0x341, + 0x342, + 0x343, + 0x344, + 0x345, + 0x346, + 0x34F, + 0x350, + 0x370, + 0x371, + 0x372, + 0x373, + 0x374, + 0x375, + 0x376, + 0x377, + 0x378, + 0x37A, + 0x37B, + 0x37E, + 0x37F, + 0x380, + 0x384, + 0x385, + 0x386, + 0x387, + 0x388, + 0x389, + 0x38A, + 0x38B, + 0x38C, + 0x38D, + 0x38E, + 0x38F, + 0x390, + 0x391, + 0x392, + 0x393, + 0x394, + 0x395, + 0x396, + 0x397, + 0x398, + 0x399, + 0x39A, + 0x39B, + 0x39C, + 0x39D, + 0x39E, + 0x39F, + 0x3A0, + 0x3A1, + 0x3A2, + 0x3A3, + 0x3A4, + 0x3A5, + 0x3A6, + 0x3A7, + 0x3A8, + 0x3A9, + 0x3AA, + 0x3AB, + 0x3AC, + 0x3C2, + 0x3C3, + 0x3CF, + 0x3D0, + 0x3D1, + 0x3D2, + 0x3D3, + 0x3D4, + 0x3D5, + 0x3D6, + 0x3D7, + 0x3D8, + 0x3D9, + 0x3DA, + 0x3DB, + 0x3DC, + 0x3DD, + 0x3DE, + 0x3DF, + 0x3E0, + 0x3E1, + 0x3E2, + 0x3E3, + 0x3E4, + 0x3E5, + 0x3E6, + 0x3E7, + 0x3E8, + 0x3E9, + 0x3EA, + 0x3EB, + 0x3EC, + 0x3ED, + 0x3EE, + 0x3EF, + 0x3F0, + 0x3F1, + 0x3F2, + 0x3F3, + 0x3F4, + 0x3F5, + 0x3F6, + 0x3F7, + 0x3F8, + 0x3F9, + 0x3FA, + 0x3FB, + 0x3FD, + 0x3FE, + 0x3FF, + 0x400, + 0x401, + 0x402, + 0x403, + 0x404, + 0x405, + 0x406, + 0x407, + 0x408, + 0x409, + 0x40A, + 0x40B, + 0x40C, + 0x40D, + 0x40E, + 0x40F, + 0x410, + 0x411, + 0x412, + 0x413, + 0x414, + 0x415, + 0x416, + 0x417, + 0x418, + 0x419, + 0x41A, + 0x41B, + 0x41C, + 0x41D, + 0x41E, + 0x41F, + 0x420, + 0x421, + 0x422, + 0x423, + 0x424, + 0x425, + 0x426, + 0x427, + 0x428, + 0x429, + 0x42A, + 0x42B, + 0x42C, + 0x42D, + 0x42E, + 0x42F, + 0x430, + 0x460, + 0x461, + 0x462, + 0x463, + 0x464, + 0x465, + 0x466, + 0x467, + 0x468, + 0x469, + 0x46A, + 0x46B, + 0x46C, + 0x46D, + 0x46E, + 0x46F, + 0x470, + 0x471, + 0x472, + 0x473, + 0x474, + 0x475, + 0x476, + 0x477, + 0x478, + 0x479, + 0x47A, + 0x47B, + 0x47C, + 0x47D, + 0x47E, + 0x47F, + 0x480, + 0x481, + 0x48A, + 0x48B, + 0x48C, + 0x48D, + 0x48E, + 0x48F, + 0x490, + 0x491, + 0x492, + 0x493, + 0x494, + 0x495, + 0x496, + 0x497, + 0x498, + 0x499, + 0x49A, + 0x49B, + 0x49C, + 0x49D, + 0x49E, + 0x49F, + 0x4A0, + 0x4A1, + 0x4A2, + 0x4A3, + 0x4A4, + 0x4A5, + 0x4A6, + 0x4A7, + 0x4A8, + 0x4A9, + 0x4AA, + 0x4AB, + 0x4AC, + 0x4AD, + 0x4AE, + 0x4AF, + 0x4B0, + 0x4B1, + 0x4B2, + 0x4B3, + 0x4B4, + 0x4B5, + 0x4B6, + 0x4B7, + 0x4B8, + 0x4B9, + 0x4BA, + 0x4BB, + 0x4BC, + 0x4BD, + 0x4BE, + 0x4BF, + 0x4C0, + 0x4C1, + 0x4C2, + 0x4C3, + 0x4C4, + 0x4C5, + 0x4C6, + 0x4C7, + 0x4C8, + 0x4C9, + 0x4CA, + 0x4CB, + 0x4CC, + 0x4CD, + 0x4CE, + 0x4D0, + 0x4D1, + 0x4D2, + 0x4D3, + 0x4D4, + 0x4D5, + 0x4D6, + 0x4D7, + 0x4D8, + 0x4D9, + 0x4DA, + 0x4DB, + 0x4DC, + 0x4DD, + 0x4DE, + 0x4DF, + 0x4E0, + 0x4E1, + 0x4E2, + 0x4E3, + 0x4E4, + 0x4E5, + 0x4E6, + 0x4E7, + 0x4E8, + 0x4E9, + 0x4EA, + 0x4EB, + 0x4EC, + 0x4ED, + 0x4EE, + 0x4EF, + 0x4F0, + 0x4F1, + 0x4F2, + 0x4F3, + 0x4F4, + 0x4F5, + 0x4F6, + 0x4F7, + 0x4F8, + 0x4F9, + 0x4FA, + 0x4FB, + 0x4FC, + 0x4FD, + 0x4FE, + 0x4FF, + 0x500, + 0x501, + 0x502, + 0x503, + 0x504, + 0x505, + 0x506, + 0x507, + 0x508, + 0x509, + 0x50A, + 0x50B, + 0x50C, + 0x50D, + 0x50E, + 0x50F, + 0x510, + 0x511, + 0x512, + 0x513, + 0x514, + 0x515, + 0x516, + 0x517, + 0x518, + 0x519, + 0x51A, + 0x51B, + 0x51C, + 0x51D, + 0x51E, + 0x51F, + 0x520, + 0x521, + 0x522, + 0x523, + 0x524, + 0x525, + 0x526, + 0x527, + 0x528, + 0x529, + 0x52A, + 0x52B, + 0x52C, + 0x52D, + 0x52E, + 0x52F, + 0x530, + 0x531, + 0x532, + 0x533, + 0x534, + 0x535, + 0x536, + 0x537, + 0x538, + 0x539, + 0x53A, + 0x53B, + 0x53C, + 0x53D, + 0x53E, + 0x53F, + 0x540, + 0x541, + 0x542, + 0x543, + 0x544, + 0x545, + 0x546, + 0x547, + 0x548, + 0x549, + 0x54A, + 0x54B, + 0x54C, + 0x54D, + 0x54E, + 0x54F, + 0x550, + 0x551, + 0x552, + 0x553, + 0x554, + 0x555, + 0x556, + 0x557, + 0x559, + 0x587, + 0x588, + 0x58B, + 0x58D, + 0x590, + 0x591, + 0x5C8, + 0x5D0, + 0x5EB, + 0x5EF, + 0x5F5, + 0x606, + 0x61C, + 0x61D, + 0x675, + 0x676, + 0x677, + 0x678, + 0x679, + 0x6DD, + 0x6DE, + 0x70E, + 0x710, + 0x74B, + 0x74D, + 0x7B2, + 0x7C0, + 0x7FB, + 0x7FD, + 0x82E, + 0x830, + 0x83F, + 0x840, + 0x85C, + 0x85E, + 0x85F, + 0x860, + 0x86B, + 0x870, + 0x890, + 0x897, + 0x8E2, + 0x8E3, + 0x958, + 0x959, + 0x95A, + 0x95B, + 0x95C, + 0x95D, + 0x95E, + 0x95F, + 0x960, + 0x984, + 0x985, + 0x98D, + 0x98F, + 0x991, + 0x993, + 0x9A9, + 0x9AA, + 0x9B1, + 0x9B2, + 0x9B3, + 0x9B6, + 0x9BA, + 0x9BC, + 0x9C5, + 0x9C7, + 0x9C9, + 0x9CB, + 0x9CF, + 0x9D7, + 0x9D8, + 0x9DC, + 0x9DD, + 0x9DE, + 0x9DF, + 0x9E0, + 0x9E4, + 0x9E6, + 0x9FF, + 0xA01, + 0xA04, + 0xA05, + 0xA0B, + 0xA0F, + 0xA11, + 0xA13, + 0xA29, + 0xA2A, + 0xA31, + 0xA32, + 0xA33, + 0xA34, + 0xA35, + 0xA36, + 0xA37, + 0xA38, + 0xA3A, + 0xA3C, + 0xA3D, + 0xA3E, + 0xA43, + 0xA47, + 0xA49, + 0xA4B, + 0xA4E, + 0xA51, + 0xA52, + 0xA59, + 0xA5A, + 0xA5B, + 0xA5C, + 0xA5D, + 0xA5E, + 0xA5F, + 0xA66, + 0xA77, + 0xA81, + 0xA84, + 0xA85, + 0xA8E, + 0xA8F, + 0xA92, + 0xA93, + 0xAA9, + 0xAAA, + 0xAB1, + 0xAB2, + 0xAB4, + 0xAB5, + 0xABA, + 0xABC, + 0xAC6, + 0xAC7, + 0xACA, + 0xACB, + 0xACE, + 0xAD0, + 0xAD1, + 0xAE0, + 0xAE4, + 0xAE6, + 0xAF2, + 0xAF9, + 0xB00, + 0xB01, + 0xB04, + 0xB05, + 0xB0D, + 0xB0F, + 0xB11, + 0xB13, + 0xB29, + 0xB2A, + 0xB31, + 0xB32, + 0xB34, + 0xB35, + 0xB3A, + 0xB3C, + 0xB45, + 0xB47, + 0xB49, + 0xB4B, + 0xB4E, + 0xB55, + 0xB58, + 0xB5C, + 0xB5D, + 0xB5E, + 0xB5F, + 0xB64, + 0xB66, + 0xB78, + 0xB82, + 0xB84, + 0xB85, + 0xB8B, + 0xB8E, + 0xB91, + 0xB92, + 0xB96, + 0xB99, + 0xB9B, + 0xB9C, + 0xB9D, + 0xB9E, + 0xBA0, + 0xBA3, + 0xBA5, + 0xBA8, + 0xBAB, + 0xBAE, + 0xBBA, + 0xBBE, + 0xBC3, + 0xBC6, + 0xBC9, + 0xBCA, + 0xBCE, + 0xBD0, + 0xBD1, + 0xBD7, + 0xBD8, + 0xBE6, + 0xBFB, + 0xC00, + 0xC0D, + 0xC0E, + 0xC11, + 0xC12, + 0xC29, + 0xC2A, + 0xC3A, + 0xC3C, + 0xC45, + 0xC46, + 0xC49, + 0xC4A, + 0xC4E, + 0xC55, + 0xC57, + 0xC58, + 0xC5B, + 0xC5C, + 0xC5E, + 0xC60, + 0xC64, + 0xC66, + 0xC70, + 0xC77, + 0xC8D, + 0xC8E, + 0xC91, + 0xC92, + 0xCA9, + 0xCAA, + 0xCB4, + 0xCB5, + 0xCBA, + 0xCBC, + 0xCC5, + 0xCC6, + 0xCC9, + 0xCCA, + 0xCCE, + 0xCD5, + 0xCD7, + 0xCDC, + 0xCDF, + 0xCE0, + 0xCE4, + 0xCE6, + 0xCF0, + 0xCF1, + 0xCF4, + 0xD00, + 0xD0D, + 0xD0E, + 0xD11, + 0xD12, + 0xD45, + 0xD46, + 0xD49, + 0xD4A, + 0xD50, + 0xD54, + 0xD64, + 0xD66, + 0xD80, + 0xD81, + 0xD84, + 0xD85, + 0xD97, + 0xD9A, + 0xDB2, + 0xDB3, + 0xDBC, + 0xDBD, + 0xDBE, + 0xDC0, + 0xDC7, + 0xDCA, + 0xDCB, + 0xDCF, + 0xDD5, + 0xDD6, + 0xDD7, + 0xDD8, + 0xDE0, + 0xDE6, + 0xDF0, + 0xDF2, + 0xDF5, + 0xE01, + 0xE33, + 0xE34, + 0xE3B, + 0xE3F, + 0xE5C, + 0xE81, + 0xE83, + 0xE84, + 0xE85, + 0xE86, + 0xE8B, + 0xE8C, + 0xEA4, + 0xEA5, + 0xEA6, + 0xEA7, + 0xEB3, + 0xEB4, + 0xEBE, + 0xEC0, + 0xEC5, + 0xEC6, + 0xEC7, + 0xEC8, + 0xECF, + 0xED0, + 0xEDA, + 0xEDC, + 0xEDD, + 0xEDE, + 0xEE0, + 0xF00, + 0xF0C, + 0xF0D, + 0xF43, + 0xF44, + 0xF48, + 0xF49, + 0xF4D, + 0xF4E, + 0xF52, + 0xF53, + 0xF57, + 0xF58, + 0xF5C, + 0xF5D, + 0xF69, + 0xF6A, + 0xF6D, + 0xF71, + 0xF73, + 0xF74, + 0xF75, + 0xF76, + 0xF77, + 0xF78, + 0xF79, + 0xF7A, + 0xF81, + 0xF82, + 0xF93, + 0xF94, + 0xF98, + 0xF99, + 0xF9D, + 0xF9E, + 0xFA2, + 0xFA3, + 0xFA7, + 0xFA8, + 0xFAC, + 0xFAD, + 0xFB9, + 0xFBA, + 0xFBD, + 0xFBE, + 0xFCD, + 0xFCE, + 0xFDB, + 0x1000, + 0x10A0, + 0x10A1, + 0x10A2, + 0x10A3, + 0x10A4, + 0x10A5, + 0x10A6, + 0x10A7, + 0x10A8, + 0x10A9, + 0x10AA, + 0x10AB, + 0x10AC, + 0x10AD, + 0x10AE, + 0x10AF, + 0x10B0, + 0x10B1, + 0x10B2, + 0x10B3, + 0x10B4, + 0x10B5, + 0x10B6, + 0x10B7, + 0x10B8, + 0x10B9, + 0x10BA, + 0x10BB, + 0x10BC, + 0x10BD, + 0x10BE, + 0x10BF, + 0x10C0, + 0x10C1, + 0x10C2, + 0x10C3, + 0x10C4, + 0x10C5, + 0x10C6, + 0x10C7, + 0x10C8, + 0x10CD, + 0x10CE, + 0x10D0, + 0x10FC, + 0x10FD, + 0x115F, + 0x1161, + 0x1249, + 0x124A, + 0x124E, + 0x1250, + 0x1257, + 0x1258, + 0x1259, + 0x125A, + 0x125E, + 0x1260, + 0x1289, + 0x128A, + 0x128E, + 0x1290, + 0x12B1, + 0x12B2, + 0x12B6, + 0x12B8, + 0x12BF, + 0x12C0, + 0x12C1, + 0x12C2, + 0x12C6, + 0x12C8, + 0x12D7, + 0x12D8, + 0x1311, + 0x1312, + 0x1316, + 0x1318, + 0x135B, + 0x135D, + 0x137D, + 0x1380, + 0x139A, + 0x13A0, + 0x13F6, + 0x13F8, + 0x13F9, + 0x13FA, + 0x13FB, + 0x13FC, + 0x13FD, + 0x13FE, + 0x1400, + 0x1680, + 0x1681, + 0x169D, + 0x16A0, + 0x16F9, + 0x1700, + 0x1716, + 0x171F, + 0x1737, + 0x1740, + 0x1754, + 0x1760, + 0x176D, + 0x176E, + 0x1771, + 0x1772, + 0x1774, + 0x1780, + 0x17B4, + 0x17B6, + 0x17DE, + 0x17E0, + 0x17EA, + 0x17F0, + 0x17FA, + 0x1800, + 0x180B, + 0x1810, + 0x181A, + 0x1820, + 0x1879, + 0x1880, + 0x18AB, + 0x18B0, + 0x18F6, + 0x1900, + 0x191F, + 0x1920, + 0x192C, + 0x1930, + 0x193C, + 0x1940, + 0x1941, + 0x1944, + 0x196E, + 0x1970, + 0x1975, + 0x1980, + 0x19AC, + 0x19B0, + 0x19CA, + 0x19D0, + 0x19DB, + 0x19DE, + 0x1A1C, + 0x1A1E, + 0x1A5F, + 0x1A60, + 0x1A7D, + 0x1A7F, + 0x1A8A, + 0x1A90, + 0x1A9A, + 0x1AA0, + 0x1AAE, + 0x1AB0, + 0x1ADE, + 0x1AE0, + 0x1AEC, + 0x1B00, + 0x1B4D, + 0x1B4E, + 0x1BF4, + 0x1BFC, + 0x1C38, + 0x1C3B, + 0x1C4A, + 0x1C4D, + 0x1C80, + 0x1C81, + 0x1C82, + 0x1C83, + 0x1C84, + 0x1C86, + 0x1C87, + 0x1C88, + 0x1C89, + 0x1C8A, + 0x1C8B, + 0x1C90, + 0x1C91, + 0x1C92, + 0x1C93, + 0x1C94, + 0x1C95, + 0x1C96, + 0x1C97, + 0x1C98, + 0x1C99, + 0x1C9A, + 0x1C9B, + 0x1C9C, + 0x1C9D, + 0x1C9E, + 0x1C9F, + 0x1CA0, + 0x1CA1, + 0x1CA2, + 0x1CA3, + 0x1CA4, + 0x1CA5, + 0x1CA6, + 0x1CA7, + 0x1CA8, + 0x1CA9, + 0x1CAA, + 0x1CAB, + 0x1CAC, + 0x1CAD, + 0x1CAE, + 0x1CAF, + 0x1CB0, + 0x1CB1, + 0x1CB2, + 0x1CB3, + 0x1CB4, + 0x1CB5, + 0x1CB6, + 0x1CB7, + 0x1CB8, + 0x1CB9, + 0x1CBA, + 0x1CBB, + 0x1CBD, + 0x1CBE, + 0x1CBF, + 0x1CC0, + 0x1CC8, + 0x1CD0, + 0x1CFB, + 0x1D00, + 0x1D2C, + 0x1D2D, + 0x1D2E, + 0x1D2F, + 0x1D30, + 0x1D31, + 0x1D32, + 0x1D33, + 0x1D34, + 0x1D35, + 0x1D36, + 0x1D37, + 0x1D38, + 0x1D39, + 0x1D3A, + 0x1D3B, + 0x1D3C, + 0x1D3D, + 0x1D3E, + 0x1D3F, + 0x1D40, + 0x1D41, + 0x1D42, + 0x1D43, + 0x1D44, + 0x1D45, + 0x1D46, + 0x1D47, + 0x1D48, + 0x1D49, + 0x1D4A, + 0x1D4B, + 0x1D4C, + 0x1D4D, + 0x1D4E, + 0x1D4F, + 0x1D50, + 0x1D51, + 0x1D52, + 0x1D53, + 0x1D54, + 0x1D55, + 0x1D56, + 0x1D57, + 0x1D58, + 0x1D59, + 0x1D5A, + 0x1D5B, + 0x1D5C, + 0x1D5D, + 0x1D5E, + 0x1D5F, + 0x1D60, + 0x1D61, + 0x1D62, + 0x1D63, + 0x1D64, + 0x1D65, + 0x1D66, + 0x1D67, + 0x1D68, + 0x1D69, + 0x1D6A, + 0x1D6B, + 0x1D78, + 0x1D79, + 0x1D9B, + 0x1D9C, + 0x1D9D, + 0x1D9E, + 0x1D9F, + 0x1DA0, + 0x1DA1, + 0x1DA2, + 0x1DA3, + 0x1DA4, + 0x1DA5, + 0x1DA6, + 0x1DA7, + 0x1DA8, + 0x1DA9, + 0x1DAA, + 0x1DAB, + 0x1DAC, + 0x1DAD, + 0x1DAE, + 0x1DAF, + 0x1DB0, + 0x1DB1, + 0x1DB2, + 0x1DB3, + 0x1DB4, + 0x1DB5, + 0x1DB6, + 0x1DB7, + 0x1DB8, + 0x1DB9, + 0x1DBA, + 0x1DBB, + 0x1DBC, + 0x1DBD, + 0x1DBE, + 0x1DBF, + 0x1DC0, + 0x1E00, + 0x1E01, + 0x1E02, + 0x1E03, + 0x1E04, + 0x1E05, + 0x1E06, + 0x1E07, + 0x1E08, + 0x1E09, + 0x1E0A, + 0x1E0B, + 0x1E0C, + 0x1E0D, + 0x1E0E, + 0x1E0F, + 0x1E10, + 0x1E11, + 0x1E12, + 0x1E13, + 0x1E14, + 0x1E15, + 0x1E16, + 0x1E17, + 0x1E18, + 0x1E19, + 0x1E1A, + 0x1E1B, + 0x1E1C, + 0x1E1D, + 0x1E1E, + 0x1E1F, + 0x1E20, + 0x1E21, + 0x1E22, + 0x1E23, + 0x1E24, + 0x1E25, + 0x1E26, + 0x1E27, + 0x1E28, + 0x1E29, + 0x1E2A, + 0x1E2B, + 0x1E2C, + 0x1E2D, + 0x1E2E, + 0x1E2F, + 0x1E30, + 0x1E31, + 0x1E32, + 0x1E33, + 0x1E34, + 0x1E35, + 0x1E36, + 0x1E37, + 0x1E38, + 0x1E39, + 0x1E3A, + 0x1E3B, + 0x1E3C, + 0x1E3D, + 0x1E3E, + 0x1E3F, + 0x1E40, + 0x1E41, + 0x1E42, + 0x1E43, + 0x1E44, + 0x1E45, + 0x1E46, + 0x1E47, + 0x1E48, + 0x1E49, + 0x1E4A, + 0x1E4B, + 0x1E4C, + 0x1E4D, + 0x1E4E, + 0x1E4F, + 0x1E50, + 0x1E51, + 0x1E52, + 0x1E53, + 0x1E54, + 0x1E55, + 0x1E56, + 0x1E57, + 0x1E58, + 0x1E59, + 0x1E5A, + 0x1E5B, + 0x1E5C, + 0x1E5D, + 0x1E5E, + 0x1E5F, + 0x1E60, + 0x1E61, + 0x1E62, + 0x1E63, + 0x1E64, + 0x1E65, + 0x1E66, + 0x1E67, + 0x1E68, + 0x1E69, + 0x1E6A, + 0x1E6B, + 0x1E6C, + 0x1E6D, + 0x1E6E, + 0x1E6F, + 0x1E70, + 0x1E71, + 0x1E72, + 0x1E73, + 0x1E74, + 0x1E75, + 0x1E76, + 0x1E77, + 0x1E78, + 0x1E79, + 0x1E7A, + 0x1E7B, + 0x1E7C, + 0x1E7D, + 0x1E7E, + 0x1E7F, + 0x1E80, + 0x1E81, + 0x1E82, + 0x1E83, + 0x1E84, + 0x1E85, + 0x1E86, + 0x1E87, + 0x1E88, + 0x1E89, + 0x1E8A, + 0x1E8B, + 0x1E8C, + 0x1E8D, + 0x1E8E, + 0x1E8F, + 0x1E90, + 0x1E91, + 0x1E92, + 0x1E93, + 0x1E94, + 0x1E95, + 0x1E9A, + 0x1E9B, + 0x1E9C, + 0x1E9E, + 0x1E9F, + 0x1EA0, + 0x1EA1, + 0x1EA2, + 0x1EA3, + 0x1EA4, + 0x1EA5, + 0x1EA6, + 0x1EA7, + 0x1EA8, + 0x1EA9, + 0x1EAA, + 0x1EAB, + 0x1EAC, + 0x1EAD, + 0x1EAE, + 0x1EAF, + 0x1EB0, + 0x1EB1, + 0x1EB2, + 0x1EB3, + 0x1EB4, + 0x1EB5, + 0x1EB6, + 0x1EB7, + 0x1EB8, + 0x1EB9, + 0x1EBA, + 0x1EBB, + 0x1EBC, + 0x1EBD, + 0x1EBE, + 0x1EBF, + 0x1EC0, + 0x1EC1, + 0x1EC2, + 0x1EC3, + 0x1EC4, + 0x1EC5, + 0x1EC6, + 0x1EC7, + 0x1EC8, + 0x1EC9, + 0x1ECA, + 0x1ECB, + 0x1ECC, + 0x1ECD, + 0x1ECE, + 0x1ECF, + 0x1ED0, + 0x1ED1, + 0x1ED2, + 0x1ED3, + 0x1ED4, + 0x1ED5, + 0x1ED6, + 0x1ED7, + 0x1ED8, + 0x1ED9, + 0x1EDA, + 0x1EDB, + 0x1EDC, + 0x1EDD, + 0x1EDE, + 0x1EDF, + 0x1EE0, + 0x1EE1, + 0x1EE2, + 0x1EE3, + 0x1EE4, + 0x1EE5, + 0x1EE6, + 0x1EE7, + 0x1EE8, + 0x1EE9, + 0x1EEA, + 0x1EEB, + 0x1EEC, + 0x1EED, + 0x1EEE, + 0x1EEF, + 0x1EF0, + 0x1EF1, + 0x1EF2, + 0x1EF3, + 0x1EF4, + 0x1EF5, + 0x1EF6, + 0x1EF7, + 0x1EF8, + 0x1EF9, + 0x1EFA, + 0x1EFB, + 0x1EFC, + 0x1EFD, + 0x1EFE, + 0x1EFF, + 0x1F08, + 0x1F09, + 0x1F0A, + 0x1F0B, + 0x1F0C, + 0x1F0D, + 0x1F0E, + 0x1F0F, + 0x1F10, + 0x1F16, + 0x1F18, + 0x1F19, + 0x1F1A, + 0x1F1B, + 0x1F1C, + 0x1F1D, + 0x1F1E, + 0x1F20, + 0x1F28, + 0x1F29, + 0x1F2A, + 0x1F2B, + 0x1F2C, + 0x1F2D, + 0x1F2E, + 0x1F2F, + 0x1F30, + 0x1F38, + 0x1F39, + 0x1F3A, + 0x1F3B, + 0x1F3C, + 0x1F3D, + 0x1F3E, + 0x1F3F, + 0x1F40, + 0x1F46, + 0x1F48, + 0x1F49, + 0x1F4A, + 0x1F4B, + 0x1F4C, + 0x1F4D, + 0x1F4E, + 0x1F50, + 0x1F58, + 0x1F59, + 0x1F5A, + 0x1F5B, + 0x1F5C, + 0x1F5D, + 0x1F5E, + 0x1F5F, + 0x1F60, + 0x1F68, + 0x1F69, + 0x1F6A, + 0x1F6B, + 0x1F6C, + 0x1F6D, + 0x1F6E, + 0x1F6F, + 0x1F70, + 0x1F71, + 0x1F72, + 0x1F73, + 0x1F74, + 0x1F75, + 0x1F76, + 0x1F77, + 0x1F78, + 0x1F79, + 0x1F7A, + 0x1F7B, + 0x1F7C, + 0x1F7D, + 0x1F7E, + 0x1F80, + 0x1F81, + 0x1F82, + 0x1F83, + 0x1F84, + 0x1F85, + 0x1F86, + 0x1F87, + 0x1F88, + 0x1F89, + 0x1F8A, + 0x1F8B, + 0x1F8C, + 0x1F8D, + 0x1F8E, + 0x1F8F, + 0x1F90, + 0x1F91, + 0x1F92, + 0x1F93, + 0x1F94, + 0x1F95, + 0x1F96, + 0x1F97, + 0x1F98, + 0x1F99, + 0x1F9A, + 0x1F9B, + 0x1F9C, + 0x1F9D, + 0x1F9E, + 0x1F9F, + 0x1FA0, + 0x1FA1, + 0x1FA2, + 0x1FA3, + 0x1FA4, + 0x1FA5, + 0x1FA6, + 0x1FA7, + 0x1FA8, + 0x1FA9, + 0x1FAA, + 0x1FAB, + 0x1FAC, + 0x1FAD, + 0x1FAE, + 0x1FAF, + 0x1FB0, + 0x1FB2, + 0x1FB3, + 0x1FB4, + 0x1FB5, + 0x1FB6, + 0x1FB7, + 0x1FB8, + 0x1FB9, + 0x1FBA, + 0x1FBB, + 0x1FBC, + 0x1FBD, + 0x1FBE, + 0x1FBF, + 0x1FC0, + 0x1FC1, + 0x1FC2, + 0x1FC3, + 0x1FC4, + 0x1FC5, + 0x1FC6, + 0x1FC7, + 0x1FC8, + 0x1FC9, + 0x1FCA, + 0x1FCB, + 0x1FCC, + 0x1FCD, + 0x1FCE, + 0x1FCF, + 0x1FD0, + 0x1FD3, + 0x1FD4, + 0x1FD6, + 0x1FD8, + 0x1FD9, + 0x1FDA, + 0x1FDB, + 0x1FDC, + 0x1FDD, + 0x1FDE, + 0x1FDF, + 0x1FE0, + 0x1FE3, + 0x1FE4, + 0x1FE8, + 0x1FE9, + 0x1FEA, + 0x1FEB, + 0x1FEC, + 0x1FED, + 0x1FEE, + 0x1FEF, + 0x1FF0, + 0x1FF2, + 0x1FF3, + 0x1FF4, + 0x1FF5, + 0x1FF6, + 0x1FF7, + 0x1FF8, + 0x1FF9, + 0x1FFA, + 0x1FFB, + 0x1FFC, + 0x1FFD, + 0x1FFE, + 0x1FFF, + 0x2000, + 0x200B, + 0x200C, + 0x200E, + 0x2010, + 0x2011, + 0x2012, + 0x2017, + 0x2018, + 0x2024, + 0x2027, + 0x2028, + 0x202F, + 0x2030, + 0x2033, + 0x2034, + 0x2035, + 0x2036, + 0x2037, + 0x2038, + 0x203C, + 0x203D, + 0x203E, + 0x203F, + 0x2047, + 0x2048, + 0x2049, + 0x204A, + 0x2057, + 0x2058, + 0x205F, + 0x2060, + 0x2065, + 0x206A, + 0x2070, + 0x2071, + 0x2072, + 0x2074, + 0x2075, + 0x2076, + 0x2077, + 0x2078, + 0x2079, + 0x207A, + 0x207B, + 0x207C, + 0x207D, + 0x207E, + 0x207F, + 0x2080, + 0x2081, + 0x2082, + 0x2083, + 0x2084, + 0x2085, + 0x2086, + 0x2087, + 0x2088, + 0x2089, + 0x208A, + 0x208B, + 0x208C, + 0x208D, + 0x208E, + 0x208F, + 0x2090, + 0x2091, + 0x2092, + 0x2093, + 0x2094, + 0x2095, + 0x2096, + 0x2097, + 0x2098, + 0x2099, + 0x209A, + 0x209B, + 0x209C, + 0x209D, + 0x20A0, + 0x20A8, + 0x20A9, + 0x20C2, + 0x20D0, + 0x20F1, + 0x2100, + 0x2101, + 0x2102, + 0x2103, + 0x2104, + 0x2105, + 0x2106, + 0x2107, + 0x2108, + 0x2109, + 0x210A, + 0x210B, + 0x210F, + 0x2110, + 0x2112, + 0x2114, + 0x2115, + 0x2116, + 0x2117, + 0x2119, + 0x211A, + 0x211B, + 0x211E, + 0x2120, + 0x2121, + 0x2122, + 0x2123, + 0x2124, + 0x2125, + 0x2126, + 0x2127, + 0x2128, + 0x2129, + 0x212A, + 0x212B, + 0x212C, + 0x212D, + 0x212E, + 0x212F, + 0x2131, + 0x2132, + 0x2133, + 0x2134, + 0x2135, + 0x2136, + 0x2137, + 0x2138, + 0x2139, + 0x213A, + 0x213B, + 0x213C, + 0x213D, + 0x213F, + 0x2140, + 0x2141, + 0x2145, + 0x2147, + 0x2148, + 0x2149, + 0x214A, + 0x2150, + 0x2151, + 0x2152, + 0x2153, + 0x2154, + 0x2155, + 0x2156, + 0x2157, + 0x2158, + 0x2159, + 0x215A, + 0x215B, + 0x215C, + 0x215D, + 0x215E, + 0x215F, + 0x2160, + 0x2161, + 0x2162, + 0x2163, + 0x2164, + 0x2165, + 0x2166, + 0x2167, + 0x2168, + 0x2169, + 0x216A, + 0x216B, + 0x216C, + 0x216D, + 0x216E, + 0x216F, + 0x2170, + 0x2171, + 0x2172, + 0x2173, + 0x2174, + 0x2175, + 0x2176, + 0x2177, + 0x2178, + 0x2179, + 0x217A, + 0x217B, + 0x217C, + 0x217D, + 0x217E, + 0x217F, + 0x2180, + 0x2183, + 0x2184, + 0x2189, + 0x218A, + 0x218C, + 0x2190, + 0x222C, + 0x222D, + 0x222E, + 0x222F, + 0x2230, + 0x2231, + 0x2329, + 0x232A, + 0x232B, + 0x242A, + 0x2440, + 0x244B, + 0x2460, + 0x2461, + 0x2462, + 0x2463, + 0x2464, + 0x2465, + 0x2466, + 0x2467, + 0x2468, + 0x2469, + 0x246A, + 0x246B, + 0x246C, + 0x246D, + 0x246E, + 0x246F, + 0x2470, + 0x2471, + 0x2472, + 0x2473, + 0x2474, + 0x2475, + 0x2476, + 0x2477, + 0x2478, + 0x2479, + 0x247A, + 0x247B, + 0x247C, + 0x247D, + 0x247E, + 0x247F, + 0x2480, + 0x2481, + 0x2482, + 0x2483, + 0x2484, + 0x2485, + 0x2486, + 0x2487, + 0x2488, + 0x249C, + 0x249D, + 0x249E, + 0x249F, + 0x24A0, + 0x24A1, + 0x24A2, + 0x24A3, + 0x24A4, + 0x24A5, + 0x24A6, + 0x24A7, + 0x24A8, + 0x24A9, + 0x24AA, + 0x24AB, + 0x24AC, + 0x24AD, + 0x24AE, + 0x24AF, + 0x24B0, + 0x24B1, + 0x24B2, + 0x24B3, + 0x24B4, + 0x24B5, + 0x24B6, + 0x24B7, + 0x24B8, + 0x24B9, + 0x24BA, + 0x24BB, + 0x24BC, + 0x24BD, + 0x24BE, + 0x24BF, + 0x24C0, + 0x24C1, + 0x24C2, + 0x24C3, + 0x24C4, + 0x24C5, + 0x24C6, + 0x24C7, + 0x24C8, + 0x24C9, + 0x24CA, + 0x24CB, + 0x24CC, + 0x24CD, + 0x24CE, + 0x24CF, + 0x24D0, + 0x24D1, + 0x24D2, + 0x24D3, + 0x24D4, + 0x24D5, + 0x24D6, + 0x24D7, + 0x24D8, + 0x24D9, + 0x24DA, + 0x24DB, + 0x24DC, + 0x24DD, + 0x24DE, + 0x24DF, + 0x24E0, + 0x24E1, + 0x24E2, + 0x24E3, + 0x24E4, + 0x24E5, + 0x24E6, + 0x24E7, + 0x24E8, + 0x24E9, + 0x24EA, + 0x24EB, + 0x2A0C, + 0x2A0D, + 0x2A74, + 0x2A75, + 0x2A76, + 0x2A77, + 0x2ADC, + 0x2ADD, + 0x2B74, + 0x2B76, + 0x2C00, + 0x2C01, + 0x2C02, + 0x2C03, + 0x2C04, + 0x2C05, + 0x2C06, + 0x2C07, + 0x2C08, + 0x2C09, + 0x2C0A, + 0x2C0B, + 0x2C0C, + 0x2C0D, + 0x2C0E, + 0x2C0F, + 0x2C10, + 0x2C11, + 0x2C12, + 0x2C13, + 0x2C14, + 0x2C15, + 0x2C16, + 0x2C17, + 0x2C18, + 0x2C19, + 0x2C1A, + 0x2C1B, + 0x2C1C, + 0x2C1D, + 0x2C1E, + 0x2C1F, + 0x2C20, + 0x2C21, + 0x2C22, + 0x2C23, + 0x2C24, + 0x2C25, + 0x2C26, + 0x2C27, + 0x2C28, + 0x2C29, + 0x2C2A, + 0x2C2B, + 0x2C2C, + 0x2C2D, + 0x2C2E, + 0x2C2F, + 0x2C30, + 0x2C60, + 0x2C61, + 0x2C62, + 0x2C63, + 0x2C64, + 0x2C65, + 0x2C67, + 0x2C68, + 0x2C69, + 0x2C6A, + 0x2C6B, + 0x2C6C, + 0x2C6D, + 0x2C6E, + 0x2C6F, + 0x2C70, + 0x2C71, + 0x2C72, + 0x2C73, + 0x2C75, + 0x2C76, + 0x2C7C, + 0x2C7D, + 0x2C7E, + 0x2C7F, + 0x2C80, + 0x2C81, + 0x2C82, + 0x2C83, + 0x2C84, + 0x2C85, + 0x2C86, + 0x2C87, + 0x2C88, + 0x2C89, + 0x2C8A, + 0x2C8B, + 0x2C8C, + 0x2C8D, + 0x2C8E, + 0x2C8F, + 0x2C90, + 0x2C91, + 0x2C92, + 0x2C93, + 0x2C94, + 0x2C95, + 0x2C96, + 0x2C97, + 0x2C98, + 0x2C99, + 0x2C9A, + 0x2C9B, + 0x2C9C, + 0x2C9D, + 0x2C9E, + 0x2C9F, + 0x2CA0, + 0x2CA1, + 0x2CA2, + 0x2CA3, + 0x2CA4, + 0x2CA5, + 0x2CA6, + 0x2CA7, + 0x2CA8, + 0x2CA9, + 0x2CAA, + 0x2CAB, + 0x2CAC, + 0x2CAD, + 0x2CAE, + 0x2CAF, + 0x2CB0, + 0x2CB1, + 0x2CB2, + 0x2CB3, + 0x2CB4, + 0x2CB5, + 0x2CB6, + 0x2CB7, + 0x2CB8, + 0x2CB9, + 0x2CBA, + 0x2CBB, + 0x2CBC, + 0x2CBD, + 0x2CBE, + 0x2CBF, + 0x2CC0, + 0x2CC1, + 0x2CC2, + 0x2CC3, + 0x2CC4, + 0x2CC5, + 0x2CC6, + 0x2CC7, + 0x2CC8, + 0x2CC9, + 0x2CCA, + 0x2CCB, + 0x2CCC, + 0x2CCD, + 0x2CCE, + 0x2CCF, + 0x2CD0, + 0x2CD1, + 0x2CD2, + 0x2CD3, + 0x2CD4, + 0x2CD5, + 0x2CD6, + 0x2CD7, + 0x2CD8, + 0x2CD9, + 0x2CDA, + 0x2CDB, + 0x2CDC, + 0x2CDD, + 0x2CDE, + 0x2CDF, + 0x2CE0, + 0x2CE1, + 0x2CE2, + 0x2CE3, + 0x2CEB, + 0x2CEC, + 0x2CED, + 0x2CEE, + 0x2CF2, + 0x2CF3, + 0x2CF4, + 0x2CF9, + 0x2D26, + 0x2D27, + 0x2D28, + 0x2D2D, + 0x2D2E, + 0x2D30, + 0x2D68, + 0x2D6F, + 0x2D70, + 0x2D71, + 0x2D7F, + 0x2D97, + 0x2DA0, + 0x2DA7, + 0x2DA8, + 0x2DAF, + 0x2DB0, + 0x2DB7, + 0x2DB8, + 0x2DBF, + 0x2DC0, + 0x2DC7, + 0x2DC8, + 0x2DCF, + 0x2DD0, + 0x2DD7, + 0x2DD8, + 0x2DDF, + 0x2DE0, + 0x2E5E, + 0x2E80, + 0x2E9A, + 0x2E9B, + 0x2E9F, + 0x2EA0, + 0x2EF3, + 0x2EF4, + 0x2F00, + 0x2F01, + 0x2F02, + 0x2F03, + 0x2F04, + 0x2F05, + 0x2F06, + 0x2F07, + 0x2F08, + 0x2F09, + 0x2F0A, + 0x2F0B, + 0x2F0C, + 0x2F0D, + 0x2F0E, + 0x2F0F, + 0x2F10, + 0x2F11, + 0x2F12, + 0x2F13, + 0x2F14, + 0x2F15, + 0x2F16, + 0x2F17, + 0x2F18, + 0x2F19, + 0x2F1A, + 0x2F1B, + 0x2F1C, + 0x2F1D, + 0x2F1E, + 0x2F1F, + 0x2F20, + 0x2F21, + 0x2F22, + 0x2F23, + 0x2F24, + 0x2F25, + 0x2F26, + 0x2F27, + 0x2F28, + 0x2F29, + 0x2F2A, + 0x2F2B, + 0x2F2C, + 0x2F2D, + 0x2F2E, + 0x2F2F, + 0x2F30, + 0x2F31, + 0x2F32, + 0x2F33, + 0x2F34, + 0x2F35, + 0x2F36, + 0x2F37, + 0x2F38, + 0x2F39, + 0x2F3A, + 0x2F3B, + 0x2F3C, + 0x2F3D, + 0x2F3E, + 0x2F3F, + 0x2F40, + 0x2F41, + 0x2F42, + 0x2F43, + 0x2F44, + 0x2F45, + 0x2F46, + 0x2F47, + 0x2F48, + 0x2F49, + 0x2F4A, + 0x2F4B, + 0x2F4C, + 0x2F4D, + 0x2F4E, + 0x2F4F, + 0x2F50, + 0x2F51, + 0x2F52, + 0x2F53, + 0x2F54, + 0x2F55, + 0x2F56, + 0x2F57, + 0x2F58, + 0x2F59, + 0x2F5A, + 0x2F5B, + 0x2F5C, + 0x2F5D, + 0x2F5E, + 0x2F5F, + 0x2F60, + 0x2F61, + 0x2F62, + 0x2F63, + 0x2F64, + 0x2F65, + 0x2F66, + 0x2F67, + 0x2F68, + 0x2F69, + 0x2F6A, + 0x2F6B, + 0x2F6C, + 0x2F6D, + 0x2F6E, + 0x2F6F, + 0x2F70, + 0x2F71, + 0x2F72, + 0x2F73, + 0x2F74, + 0x2F75, + 0x2F76, + 0x2F77, + 0x2F78, + 0x2F79, + 0x2F7A, + 0x2F7B, + 0x2F7C, + 0x2F7D, + 0x2F7E, + 0x2F7F, + 0x2F80, + 0x2F81, + 0x2F82, + 0x2F83, + 0x2F84, + 0x2F85, + 0x2F86, + 0x2F87, + 0x2F88, + 0x2F89, + 0x2F8A, + 0x2F8B, + 0x2F8C, + 0x2F8D, + 0x2F8E, + 0x2F8F, + 0x2F90, + 0x2F91, + 0x2F92, + 0x2F93, + 0x2F94, + 0x2F95, + 0x2F96, + 0x2F97, + 0x2F98, + 0x2F99, + 0x2F9A, + 0x2F9B, + 0x2F9C, + 0x2F9D, + 0x2F9E, + 0x2F9F, + 0x2FA0, + 0x2FA1, + 0x2FA2, + 0x2FA3, + 0x2FA4, + 0x2FA5, + 0x2FA6, + 0x2FA7, + 0x2FA8, + 0x2FA9, + 0x2FAA, + 0x2FAB, + 0x2FAC, + 0x2FAD, + 0x2FAE, + 0x2FAF, + 0x2FB0, + 0x2FB1, + 0x2FB2, + 0x2FB3, + 0x2FB4, + 0x2FB5, + 0x2FB6, + 0x2FB7, + 0x2FB8, + 0x2FB9, + 0x2FBA, + 0x2FBB, + 0x2FBC, + 0x2FBD, + 0x2FBE, + 0x2FBF, + 0x2FC0, + 0x2FC1, + 0x2FC2, + 0x2FC3, + 0x2FC4, + 0x2FC5, + 0x2FC6, + 0x2FC7, + 0x2FC8, + 0x2FC9, + 0x2FCA, + 0x2FCB, + 0x2FCC, + 0x2FCD, + 0x2FCE, + 0x2FCF, + 0x2FD0, + 0x2FD1, + 0x2FD2, + 0x2FD3, + 0x2FD4, + 0x2FD5, + 0x2FD6, + 0x3000, + 0x3001, + 0x3002, + 0x3003, + 0x3036, + 0x3037, + 0x3038, + 0x3039, + 0x303A, + 0x303B, + 0x3040, + 0x3041, + 0x3097, + 0x3099, + 0x309B, + 0x309C, + 0x309D, + 0x309F, + 0x30A0, + 0x30FF, + 0x3100, + 0x3105, + 0x3130, + 0x3131, + 0x3132, + 0x3133, + 0x3134, + 0x3135, + 0x3136, + 0x3137, + 0x3138, + 0x3139, + 0x313A, + 0x313B, + 0x313C, + 0x313D, + 0x313E, + 0x313F, + 0x3140, + 0x3141, + 0x3142, + 0x3143, + 0x3144, + 0x3145, + 0x3146, + 0x3147, + 0x3148, + 0x3149, + 0x314A, + 0x314B, + 0x314C, + 0x314D, + 0x314E, + 0x314F, + 0x3150, + 0x3151, + 0x3152, + 0x3153, + 0x3154, + 0x3155, + 0x3156, + 0x3157, + 0x3158, + 0x3159, + 0x315A, + 0x315B, + 0x315C, + 0x315D, + 0x315E, + 0x315F, + 0x3160, + 0x3161, + 0x3162, + 0x3163, + 0x3164, + 0x3165, + 0x3166, + 0x3167, + 0x3168, + 0x3169, + 0x316A, + 0x316B, + 0x316C, + 0x316D, + 0x316E, + 0x316F, + 0x3170, + 0x3171, + 0x3172, + 0x3173, + 0x3174, + 0x3175, + 0x3176, + 0x3177, + 0x3178, + 0x3179, + 0x317A, + 0x317B, + 0x317C, + 0x317D, + 0x317E, + 0x317F, + 0x3180, + 0x3181, + 0x3182, + 0x3183, + 0x3184, + 0x3185, + 0x3186, + 0x3187, + 0x3188, + 0x3189, + 0x318A, + 0x318B, + 0x318C, + 0x318D, + 0x318E, + 0x318F, + 0x3190, + 0x3192, + 0x3193, + 0x3194, + 0x3195, + 0x3196, + 0x3197, + 0x3198, + 0x3199, + 0x319A, + 0x319B, + 0x319C, + 0x319D, + 0x319E, + 0x319F, + 0x31A0, + 0x31E6, + 0x31F0, + 0x3200, + 0x3201, + 0x3202, + 0x3203, + 0x3204, + 0x3205, + 0x3206, + 0x3207, + 0x3208, + 0x3209, + 0x320A, + 0x320B, + 0x320C, + 0x320D, + 0x320E, + 0x320F, + 0x3210, + 0x3211, + 0x3212, + 0x3213, + 0x3214, + 0x3215, + 0x3216, + 0x3217, + 0x3218, + 0x3219, + 0x321A, + 0x321B, + 0x321C, + 0x321D, + 0x321E, + 0x321F, + 0x3220, + 0x3221, + 0x3222, + 0x3223, + 0x3224, + 0x3225, + 0x3226, + 0x3227, + 0x3228, + 0x3229, + 0x322A, + 0x322B, + 0x322C, + 0x322D, + 0x322E, + 0x322F, + 0x3230, + 0x3231, + 0x3232, + 0x3233, + 0x3234, + 0x3235, + 0x3236, + 0x3237, + 0x3238, + 0x3239, + 0x323A, + 0x323B, + 0x323C, + 0x323D, + 0x323E, + 0x323F, + 0x3240, + 0x3241, + 0x3242, + 0x3243, + 0x3244, + 0x3245, + 0x3246, + 0x3247, + 0x3248, + 0x3250, + 0x3251, + 0x3252, + 0x3253, + 0x3254, + 0x3255, + 0x3256, + 0x3257, + 0x3258, + 0x3259, + 0x325A, + 0x325B, + 0x325C, + 0x325D, + 0x325E, + 0x325F, + 0x3260, + 0x3261, + 0x3262, + 0x3263, + 0x3264, + 0x3265, + 0x3266, + 0x3267, + 0x3268, + 0x3269, + 0x326A, + 0x326B, + 0x326C, + 0x326D, + 0x326E, + 0x326F, + 0x3270, + 0x3271, + 0x3272, + 0x3273, + 0x3274, + 0x3275, + 0x3276, + 0x3277, + 0x3278, + 0x3279, + 0x327A, + 0x327B, + 0x327C, + 0x327D, + 0x327E, + 0x327F, + 0x3280, + 0x3281, + 0x3282, + 0x3283, + 0x3284, + 0x3285, + 0x3286, + 0x3287, + 0x3288, + 0x3289, + 0x328A, + 0x328B, + 0x328C, + 0x328D, + 0x328E, + 0x328F, + 0x3290, + 0x3291, + 0x3292, + 0x3293, + 0x3294, + 0x3295, + 0x3296, + 0x3297, + 0x3298, + 0x3299, + 0x329A, + 0x329B, + 0x329C, + 0x329D, + 0x329E, + 0x329F, + 0x32A0, + 0x32A1, + 0x32A2, + 0x32A3, + 0x32A4, + 0x32A5, + 0x32A6, + 0x32A7, + 0x32A8, + 0x32A9, + 0x32AA, + 0x32AB, + 0x32AC, + 0x32AD, + 0x32AE, + 0x32AF, + 0x32B0, + 0x32B1, + 0x32B2, + 0x32B3, + 0x32B4, + 0x32B5, + 0x32B6, + 0x32B7, + 0x32B8, + 0x32B9, + 0x32BA, + 0x32BB, + 0x32BC, + 0x32BD, + 0x32BE, + 0x32BF, + 0x32C0, + 0x32C1, + 0x32C2, + 0x32C3, + 0x32C4, + 0x32C5, + 0x32C6, + 0x32C7, + 0x32C8, + 0x32C9, + 0x32CA, + 0x32CB, + 0x32CC, + 0x32CD, + 0x32CE, + 0x32CF, + 0x32D0, + 0x32D1, + 0x32D2, + 0x32D3, + 0x32D4, + 0x32D5, + 0x32D6, + 0x32D7, + 0x32D8, + 0x32D9, + 0x32DA, + 0x32DB, + 0x32DC, + 0x32DD, + 0x32DE, + 0x32DF, + 0x32E0, + 0x32E1, + 0x32E2, + 0x32E3, + 0x32E4, + 0x32E5, + 0x32E6, + 0x32E7, + 0x32E8, + 0x32E9, + 0x32EA, + 0x32EB, + 0x32EC, + 0x32ED, + 0x32EE, + 0x32EF, + 0x32F0, + 0x32F1, + 0x32F2, + 0x32F3, + 0x32F4, + 0x32F5, + 0x32F6, + 0x32F7, + 0x32F8, + 0x32F9, + 0x32FA, + 0x32FB, + 0x32FC, + 0x32FD, + 0x32FE, + 0x32FF, + 0x3300, + 0x3301, + 0x3302, + 0x3303, + 0x3304, + 0x3305, + 0x3306, + 0x3307, + 0x3308, + 0x3309, + 0x330A, + 0x330B, + 0x330C, + 0x330D, + 0x330E, + 0x330F, + 0x3310, + 0x3311, + 0x3312, + 0x3313, + 0x3314, + 0x3315, + 0x3316, + 0x3317, + 0x3318, + 0x3319, + 0x331A, + 0x331B, + 0x331C, + 0x331D, + 0x331E, + 0x331F, + 0x3320, + 0x3321, + 0x3322, + 0x3323, + 0x3324, + 0x3325, + 0x3326, + 0x3327, + 0x3328, + 0x3329, + 0x332A, + 0x332B, + 0x332C, + 0x332D, + 0x332E, + 0x332F, + 0x3330, + 0x3331, + 0x3332, + 0x3333, + 0x3334, + 0x3335, + 0x3336, + 0x3337, + 0x3338, + 0x3339, + 0x333A, + 0x333B, + 0x333C, + 0x333D, + 0x333E, + 0x333F, + 0x3340, + 0x3341, + 0x3342, + 0x3343, + 0x3344, + 0x3345, + 0x3346, + 0x3347, + 0x3348, + 0x3349, + 0x334A, + 0x334B, + 0x334C, + 0x334D, + 0x334E, + 0x334F, + 0x3350, + 0x3351, + 0x3352, + 0x3353, + 0x3354, + 0x3355, + 0x3356, + 0x3357, + 0x3358, + 0x3359, + 0x335A, + 0x335B, + 0x335C, + 0x335D, + 0x335E, + 0x335F, + 0x3360, + 0x3361, + 0x3362, + 0x3363, + 0x3364, + 0x3365, + 0x3366, + 0x3367, + 0x3368, + 0x3369, + 0x336A, + 0x336B, + 0x336C, + 0x336D, + 0x336E, + 0x336F, + 0x3370, + 0x3371, + 0x3372, + 0x3373, + 0x3374, + 0x3375, + 0x3376, + 0x3377, + 0x3378, + 0x3379, + 0x337A, + 0x337B, + 0x337C, + 0x337D, + 0x337E, + 0x337F, + 0x3380, + 0x3381, + 0x3382, + 0x3383, + 0x3384, + 0x3385, + 0x3386, + 0x3387, + 0x3388, + 0x3389, + 0x338A, + 0x338B, + 0x338C, + 0x338D, + 0x338E, + 0x338F, + 0x3390, + 0x3391, + 0x3392, + 0x3393, + 0x3394, + 0x3395, + 0x3396, + 0x3397, + 0x3398, + 0x3399, + 0x339A, + 0x339B, + 0x339C, + 0x339D, + 0x339E, + 0x339F, + 0x33A0, + 0x33A1, + 0x33A2, + 0x33A3, + 0x33A4, + 0x33A5, + 0x33A6, + 0x33A7, + 0x33A8, + 0x33A9, + 0x33AA, + 0x33AB, + 0x33AC, + 0x33AD, + 0x33AE, + 0x33AF, + 0x33B0, + 0x33B1, + 0x33B2, + 0x33B3, + 0x33B4, + 0x33B5, + 0x33B6, + 0x33B7, + 0x33B8, + 0x33B9, + 0x33BA, + 0x33BB, + 0x33BC, + 0x33BD, + 0x33BE, + 0x33BF, + 0x33C0, + 0x33C1, + 0x33C2, + 0x33C3, + 0x33C4, + 0x33C5, + 0x33C6, + 0x33C7, + 0x33C8, + 0x33C9, + 0x33CA, + 0x33CB, + 0x33CC, + 0x33CD, + 0x33CE, + 0x33CF, + 0x33D0, + 0x33D1, + 0x33D2, + 0x33D3, + 0x33D4, + 0x33D5, + 0x33D6, + 0x33D7, + 0x33D8, + 0x33D9, + 0x33DA, + 0x33DB, + 0x33DC, + 0x33DD, + 0x33DE, + 0x33DF, + 0x33E0, + 0x33E1, + 0x33E2, + 0x33E3, + 0x33E4, + 0x33E5, + 0x33E6, + 0x33E7, + 0x33E8, + 0x33E9, + 0x33EA, + 0x33EB, + 0x33EC, + 0x33ED, + 0x33EE, + 0x33EF, + 0x33F0, + 0x33F1, + 0x33F2, + 0x33F3, + 0x33F4, + 0x33F5, + 0x33F6, + 0x33F7, + 0x33F8, + 0x33F9, + 0x33FA, + 0x33FB, + 0x33FC, + 0x33FD, + 0x33FE, + 0x33FF, + 0x3400, + 0xA48D, + 0xA490, + 0xA4C7, + 0xA4D0, + 0xA62C, + 0xA640, + 0xA641, + 0xA642, + 0xA643, + 0xA644, + 0xA645, + 0xA646, + 0xA647, + 0xA648, + 0xA649, + 0xA64A, + 0xA64B, + 0xA64C, + 0xA64D, + 0xA64E, + 0xA64F, + 0xA650, + 0xA651, + 0xA652, + 0xA653, + 0xA654, + 0xA655, + 0xA656, + 0xA657, + 0xA658, + 0xA659, + 0xA65A, + 0xA65B, + 0xA65C, + 0xA65D, + 0xA65E, + 0xA65F, + 0xA660, + 0xA661, + 0xA662, + 0xA663, + 0xA664, + 0xA665, + 0xA666, + 0xA667, + 0xA668, + 0xA669, + 0xA66A, + 0xA66B, + 0xA66C, + 0xA66D, + 0xA680, + 0xA681, + 0xA682, + 0xA683, + 0xA684, + 0xA685, + 0xA686, + 0xA687, + 0xA688, + 0xA689, + 0xA68A, + 0xA68B, + 0xA68C, + 0xA68D, + 0xA68E, + 0xA68F, + 0xA690, + 0xA691, + 0xA692, + 0xA693, + 0xA694, + 0xA695, + 0xA696, + 0xA697, + 0xA698, + 0xA699, + 0xA69A, + 0xA69B, + 0xA69C, + 0xA69D, + 0xA69E, + 0xA6F8, + 0xA700, + 0xA722, + 0xA723, + 0xA724, + 0xA725, + 0xA726, + 0xA727, + 0xA728, + 0xA729, + 0xA72A, + 0xA72B, + 0xA72C, + 0xA72D, + 0xA72E, + 0xA72F, + 0xA732, + 0xA733, + 0xA734, + 0xA735, + 0xA736, + 0xA737, + 0xA738, + 0xA739, + 0xA73A, + 0xA73B, + 0xA73C, + 0xA73D, + 0xA73E, + 0xA73F, + 0xA740, + 0xA741, + 0xA742, + 0xA743, + 0xA744, + 0xA745, + 0xA746, + 0xA747, + 0xA748, + 0xA749, + 0xA74A, + 0xA74B, + 0xA74C, + 0xA74D, + 0xA74E, + 0xA74F, + 0xA750, + 0xA751, + 0xA752, + 0xA753, + 0xA754, + 0xA755, + 0xA756, + 0xA757, + 0xA758, + 0xA759, + 0xA75A, + 0xA75B, + 0xA75C, + 0xA75D, + 0xA75E, + 0xA75F, + 0xA760, + 0xA761, + 0xA762, + 0xA763, + 0xA764, + 0xA765, + 0xA766, + 0xA767, + 0xA768, + 0xA769, + 0xA76A, + 0xA76B, + 0xA76C, + 0xA76D, + 0xA76E, + 0xA76F, + 0xA770, + 0xA771, + 0xA779, + 0xA77A, + 0xA77B, + 0xA77C, + 0xA77D, + 0xA77E, + 0xA77F, + 0xA780, + 0xA781, + 0xA782, + 0xA783, + 0xA784, + 0xA785, + 0xA786, + 0xA787, + 0xA78B, + 0xA78C, + 0xA78D, + 0xA78E, + 0xA790, + 0xA791, + 0xA792, + 0xA793, + 0xA796, + 0xA797, + 0xA798, + 0xA799, + 0xA79A, + 0xA79B, + 0xA79C, + 0xA79D, + 0xA79E, + 0xA79F, + 0xA7A0, + 0xA7A1, + 0xA7A2, + 0xA7A3, + 0xA7A4, + 0xA7A5, + 0xA7A6, + 0xA7A7, + 0xA7A8, + 0xA7A9, + 0xA7AA, + 0xA7AB, + 0xA7AC, + 0xA7AD, + 0xA7AE, + 0xA7AF, + 0xA7B0, + 0xA7B1, + 0xA7B2, + 0xA7B3, + 0xA7B4, + 0xA7B5, + 0xA7B6, + 0xA7B7, + 0xA7B8, + 0xA7B9, + 0xA7BA, + 0xA7BB, + 0xA7BC, + 0xA7BD, + 0xA7BE, + 0xA7BF, + 0xA7C0, + 0xA7C1, + 0xA7C2, + 0xA7C3, + 0xA7C4, + 0xA7C5, + 0xA7C6, + 0xA7C7, + 0xA7C8, + 0xA7C9, + 0xA7CA, + 0xA7CB, + 0xA7CC, + 0xA7CD, + 0xA7CE, + 0xA7CF, + 0xA7D0, + 0xA7D1, + 0xA7D2, + 0xA7D3, + 0xA7D4, + 0xA7D5, + 0xA7D6, + 0xA7D7, + 0xA7D8, + 0xA7D9, + 0xA7DA, + 0xA7DB, + 0xA7DC, + 0xA7DD, + 0xA7F1, + 0xA7F2, + 0xA7F3, + 0xA7F4, + 0xA7F5, + 0xA7F6, + 0xA7F8, + 0xA7F9, + 0xA7FA, + 0xA82D, + 0xA830, + 0xA83A, + 0xA840, + 0xA878, + 0xA880, + 0xA8C6, + 0xA8CE, + 0xA8DA, + 0xA8E0, + 0xA954, + 0xA95F, + 0xA97D, + 0xA980, + 0xA9CE, + 0xA9CF, + 0xA9DA, + 0xA9DE, + 0xA9FF, + 0xAA00, + 0xAA37, + 0xAA40, + 0xAA4E, + 0xAA50, + 0xAA5A, + 0xAA5C, + 0xAAC3, + 0xAADB, + 0xAAF7, + 0xAB01, + 0xAB07, + 0xAB09, + 0xAB0F, + 0xAB11, + 0xAB17, + 0xAB20, + 0xAB27, + 0xAB28, + 0xAB2F, + 0xAB30, + 0xAB5C, + 0xAB5D, + 0xAB5E, + 0xAB5F, + 0xAB60, + 0xAB69, + 0xAB6A, + 0xAB6C, + 0xAB70, + 0xAB71, + 0xAB72, + 0xAB73, + 0xAB74, + 0xAB75, + 0xAB76, + 0xAB77, + 0xAB78, + 0xAB79, + 0xAB7A, + 0xAB7B, + 0xAB7C, + 0xAB7D, + 0xAB7E, + 0xAB7F, + 0xAB80, + 0xAB81, + 0xAB82, + 0xAB83, + 0xAB84, + 0xAB85, + 0xAB86, + 0xAB87, + 0xAB88, + 0xAB89, + 0xAB8A, + 0xAB8B, + 0xAB8C, + 0xAB8D, + 0xAB8E, + 0xAB8F, + 0xAB90, + 0xAB91, + 0xAB92, + 0xAB93, + 0xAB94, + 0xAB95, + 0xAB96, + 0xAB97, + 0xAB98, + 0xAB99, + 0xAB9A, + 0xAB9B, + 0xAB9C, + 0xAB9D, + 0xAB9E, + 0xAB9F, + 0xABA0, + 0xABA1, + 0xABA2, + 0xABA3, + 0xABA4, + 0xABA5, + 0xABA6, + 0xABA7, + 0xABA8, + 0xABA9, + 0xABAA, + 0xABAB, + 0xABAC, + 0xABAD, + 0xABAE, + 0xABAF, + 0xABB0, + 0xABB1, + 0xABB2, + 0xABB3, + 0xABB4, + 0xABB5, + 0xABB6, + 0xABB7, + 0xABB8, + 0xABB9, + 0xABBA, + 0xABBB, + 0xABBC, + 0xABBD, + 0xABBE, + 0xABBF, + 0xABC0, + 0xABEE, + 0xABF0, + 0xABFA, + 0xAC00, + 0xD7A4, + 0xD7B0, + 0xD7C7, + 0xD7CB, + 0xD7FC, + 0xF900, + 0xF901, + 0xF902, + 0xF903, + 0xF904, + 0xF905, + 0xF906, + 0xF907, + 0xF909, + 0xF90A, + 0xF90B, + 0xF90C, + 0xF90D, + 0xF90E, + 0xF90F, + 0xF910, + 0xF911, + 0xF912, + 0xF913, + 0xF914, + 0xF915, + 0xF916, + 0xF917, + 0xF918, + 0xF919, + 0xF91A, + 0xF91B, + 0xF91C, + 0xF91D, + 0xF91E, + 0xF91F, + 0xF920, + 0xF921, + 0xF922, + 0xF923, + 0xF924, + 0xF925, + 0xF926, + 0xF927, + 0xF928, + 0xF929, + 0xF92A, + 0xF92B, + 0xF92C, + 0xF92D, + 0xF92E, + 0xF92F, + 0xF930, + 0xF931, + 0xF932, + 0xF933, + 0xF934, + 0xF935, + 0xF936, + 0xF937, + 0xF938, + 0xF939, + 0xF93A, + 0xF93B, + 0xF93C, + 0xF93D, + 0xF93E, + 0xF93F, + 0xF940, + 0xF941, + 0xF942, + 0xF943, + 0xF944, + 0xF945, + 0xF946, + 0xF947, + 0xF948, + 0xF949, + 0xF94A, + 0xF94B, + 0xF94C, + 0xF94D, + 0xF94E, + 0xF94F, + 0xF950, + 0xF951, + 0xF952, + 0xF953, + 0xF954, + 0xF955, + 0xF956, + 0xF957, + 0xF958, + 0xF959, + 0xF95A, + 0xF95B, + 0xF95C, + 0xF95D, + 0xF95E, + 0xF95F, + 0xF960, + 0xF961, + 0xF962, + 0xF963, + 0xF964, + 0xF965, + 0xF966, + 0xF967, + 0xF968, + 0xF969, + 0xF96A, + 0xF96B, + 0xF96C, + 0xF96D, + 0xF96E, + 0xF96F, + 0xF970, + 0xF971, + 0xF972, + 0xF973, + 0xF974, + 0xF975, + 0xF976, + 0xF977, + 0xF978, + 0xF979, + 0xF97A, + 0xF97B, + 0xF97C, + 0xF97D, + 0xF97E, + 0xF97F, + 0xF980, + 0xF981, + 0xF982, + 0xF983, + 0xF984, + 0xF985, + 0xF986, + 0xF987, + 0xF988, + 0xF989, + 0xF98A, + 0xF98B, + 0xF98C, + 0xF98D, + 0xF98E, + 0xF98F, + 0xF990, + 0xF991, + 0xF992, + 0xF993, + 0xF994, + 0xF995, + 0xF996, + 0xF997, + 0xF998, + 0xF999, + 0xF99A, + 0xF99B, + 0xF99C, + 0xF99D, + 0xF99E, + 0xF99F, + 0xF9A0, + 0xF9A1, + 0xF9A2, + 0xF9A3, + 0xF9A4, + 0xF9A5, + 0xF9A6, + 0xF9A7, + 0xF9A8, + 0xF9A9, + 0xF9AA, + 0xF9AB, + 0xF9AC, + 0xF9AD, + 0xF9AE, + 0xF9AF, + 0xF9B0, + 0xF9B1, + 0xF9B2, + 0xF9B3, + 0xF9B4, + 0xF9B5, + 0xF9B6, + 0xF9B7, + 0xF9B8, + 0xF9B9, + 0xF9BA, + 0xF9BB, + 0xF9BC, + 0xF9BD, + 0xF9BE, + 0xF9BF, + 0xF9C0, + 0xF9C1, + 0xF9C2, + 0xF9C3, + 0xF9C4, + 0xF9C5, + 0xF9C6, + 0xF9C7, + 0xF9C8, + 0xF9C9, + 0xF9CA, + 0xF9CB, + 0xF9CC, + 0xF9CD, + 0xF9CE, + 0xF9CF, + 0xF9D0, + 0xF9D1, + 0xF9D2, + 0xF9D3, + 0xF9D4, + 0xF9D5, + 0xF9D6, + 0xF9D7, + 0xF9D8, + 0xF9D9, + 0xF9DA, + 0xF9DB, + 0xF9DC, + 0xF9DD, + 0xF9DE, + 0xF9DF, + 0xF9E0, + 0xF9E1, + 0xF9E2, + 0xF9E3, + 0xF9E4, + 0xF9E5, + 0xF9E6, + 0xF9E7, + 0xF9E8, + 0xF9E9, + 0xF9EA, + 0xF9EB, + 0xF9EC, + 0xF9ED, + 0xF9EE, + 0xF9EF, + 0xF9F0, + 0xF9F1, + 0xF9F2, + 0xF9F3, + 0xF9F4, + 0xF9F5, + 0xF9F6, + 0xF9F7, + 0xF9F8, + 0xF9F9, + 0xF9FA, + 0xF9FB, + 0xF9FC, + 0xF9FD, + 0xF9FE, + 0xF9FF, + 0xFA00, + 0xFA01, + 0xFA02, + 0xFA03, + 0xFA04, + 0xFA05, + 0xFA06, + 0xFA07, + 0xFA08, + 0xFA09, + 0xFA0A, + 0xFA0B, + 0xFA0C, + 0xFA0D, + 0xFA0E, + 0xFA10, + 0xFA11, + 0xFA12, + 0xFA13, + 0xFA15, + 0xFA16, + 0xFA17, + 0xFA18, + 0xFA19, + 0xFA1A, + 0xFA1B, + 0xFA1C, + 0xFA1D, + 0xFA1E, + 0xFA1F, + 0xFA20, + 0xFA21, + 0xFA22, + 0xFA23, + 0xFA25, + 0xFA26, + 0xFA27, + 0xFA2A, + 0xFA2B, + 0xFA2C, + 0xFA2D, + 0xFA2E, + 0xFA2F, + 0xFA30, + 0xFA31, + 0xFA32, + 0xFA33, + 0xFA34, + 0xFA35, + 0xFA36, + 0xFA37, + 0xFA38, + 0xFA39, + 0xFA3A, + 0xFA3B, + 0xFA3C, + 0xFA3D, + 0xFA3E, + 0xFA3F, + 0xFA40, + 0xFA41, + 0xFA42, + 0xFA43, + 0xFA44, + 0xFA45, + 0xFA46, + 0xFA47, + 0xFA48, + 0xFA49, + 0xFA4A, + 0xFA4B, + 0xFA4C, + 0xFA4D, + 0xFA4E, + 0xFA4F, + 0xFA50, + 0xFA51, + 0xFA52, + 0xFA53, + 0xFA54, + 0xFA55, + 0xFA56, + 0xFA57, + 0xFA58, + 0xFA59, + 0xFA5A, + 0xFA5B, + 0xFA5C, + 0xFA5D, + 0xFA5F, + 0xFA60, + 0xFA61, + 0xFA62, + 0xFA63, + 0xFA64, + 0xFA65, + 0xFA66, + 0xFA67, + 0xFA68, + 0xFA69, + 0xFA6A, + 0xFA6B, + 0xFA6C, + 0xFA6D, + 0xFA6E, + 0xFA70, + 0xFA71, + 0xFA72, + 0xFA73, + 0xFA74, + 0xFA75, + 0xFA76, + 0xFA77, + 0xFA78, + 0xFA79, + 0xFA7A, + 0xFA7B, + 0xFA7C, + 0xFA7D, + 0xFA7E, + 0xFA7F, + 0xFA80, + 0xFA81, + 0xFA82, + 0xFA83, + 0xFA84, + 0xFA85, + 0xFA86, + 0xFA87, + 0xFA88, + 0xFA89, + 0xFA8A, + 0xFA8B, + 0xFA8C, + 0xFA8D, + 0xFA8E, + 0xFA8F, + 0xFA90, + 0xFA91, + 0xFA92, + 0xFA93, + 0xFA94, + 0xFA95, + 0xFA96, + 0xFA97, + 0xFA98, + 0xFA99, + 0xFA9A, + 0xFA9B, + 0xFA9C, + 0xFA9D, + 0xFA9E, + 0xFA9F, + 0xFAA0, + 0xFAA1, + 0xFAA2, + 0xFAA3, + 0xFAA4, + 0xFAA5, + 0xFAA6, + 0xFAA7, + 0xFAA8, + 0xFAA9, + 0xFAAA, + 0xFAAB, + 0xFAAC, + 0xFAAD, + 0xFAAE, + 0xFAAF, + 0xFAB0, + 0xFAB1, + 0xFAB2, + 0xFAB3, + 0xFAB4, + 0xFAB5, + 0xFAB6, + 0xFAB7, + 0xFAB8, + 0xFAB9, + 0xFABA, + 0xFABB, + 0xFABC, + 0xFABD, + 0xFABE, + 0xFABF, + 0xFAC0, + 0xFAC1, + 0xFAC2, + 0xFAC3, + 0xFAC4, + 0xFAC5, + 0xFAC6, + 0xFAC7, + 0xFAC8, + 0xFAC9, + 0xFACA, + 0xFACB, + 0xFACC, + 0xFACD, + 0xFACE, + 0xFACF, + 0xFAD0, + 0xFAD1, + 0xFAD2, + 0xFAD3, + 0xFAD4, + 0xFAD5, + 0xFAD6, + 0xFAD7, + 0xFAD8, + 0xFAD9, + 0xFADA, + 0xFB00, + 0xFB01, + 0xFB02, + 0xFB03, + 0xFB04, + 0xFB05, + 0xFB07, + 0xFB13, + 0xFB14, + 0xFB15, + 0xFB16, + 0xFB17, + 0xFB18, + 0xFB1D, + 0xFB1E, + 0xFB1F, + 0xFB20, + 0xFB21, + 0xFB22, + 0xFB23, + 0xFB24, + 0xFB25, + 0xFB26, + 0xFB27, + 0xFB28, + 0xFB29, + 0xFB2A, + 0xFB2B, + 0xFB2C, + 0xFB2D, + 0xFB2E, + 0xFB2F, + 0xFB30, + 0xFB31, + 0xFB32, + 0xFB33, + 0xFB34, + 0xFB35, + 0xFB36, + 0xFB37, + 0xFB38, + 0xFB39, + 0xFB3A, + 0xFB3B, + 0xFB3C, + 0xFB3D, + 0xFB3E, + 0xFB3F, + 0xFB40, + 0xFB41, + 0xFB42, + 0xFB43, + 0xFB44, + 0xFB45, + 0xFB46, + 0xFB47, + 0xFB48, + 0xFB49, + 0xFB4A, + 0xFB4B, + 0xFB4C, + 0xFB4D, + 0xFB4E, + 0xFB4F, + 0xFB50, + 0xFB52, + 0xFB56, + 0xFB5A, + 0xFB5E, + 0xFB62, + 0xFB66, + 0xFB6A, + 0xFB6E, + 0xFB72, + 0xFB76, + 0xFB7A, + 0xFB7E, + 0xFB82, + 0xFB84, + 0xFB86, + 0xFB88, + 0xFB8A, + 0xFB8C, + 0xFB8E, + 0xFB92, + 0xFB96, + 0xFB9A, + 0xFB9E, + 0xFBA0, + 0xFBA4, + 0xFBA6, + 0xFBAA, + 0xFBAE, + 0xFBB0, + 0xFBB2, + 0xFBD3, + 0xFBD7, + 0xFBD9, + 0xFBDB, + 0xFBDD, + 0xFBDE, + 0xFBE0, + 0xFBE2, + 0xFBE4, + 0xFBE8, + 0xFBEA, + 0xFBEC, + 0xFBEE, + 0xFBF0, + 0xFBF2, + 0xFBF4, + 0xFBF6, + 0xFBF9, + 0xFBFC, + 0xFC00, + 0xFC01, + 0xFC02, + 0xFC03, + 0xFC04, + 0xFC05, + 0xFC06, + 0xFC07, + 0xFC08, + 0xFC09, + 0xFC0A, + 0xFC0B, + 0xFC0C, + 0xFC0D, + 0xFC0E, + 0xFC0F, + 0xFC10, + 0xFC11, + 0xFC12, + 0xFC13, + 0xFC14, + 0xFC15, + 0xFC16, + 0xFC17, + 0xFC18, + 0xFC19, + 0xFC1A, + 0xFC1B, + 0xFC1C, + 0xFC1D, + 0xFC1E, + 0xFC1F, + 0xFC20, + 0xFC21, + 0xFC22, + 0xFC23, + 0xFC24, + 0xFC25, + 0xFC26, + 0xFC27, + 0xFC28, + 0xFC29, + 0xFC2A, + 0xFC2B, + 0xFC2C, + 0xFC2D, + 0xFC2E, + 0xFC2F, + 0xFC30, + 0xFC31, + 0xFC32, + 0xFC33, + 0xFC34, + 0xFC35, + 0xFC36, + 0xFC37, + 0xFC38, + 0xFC39, + 0xFC3A, + 0xFC3B, + 0xFC3C, + 0xFC3D, + 0xFC3E, + 0xFC3F, + 0xFC40, + 0xFC41, + 0xFC42, + 0xFC43, + 0xFC44, + 0xFC45, + 0xFC46, + 0xFC47, + 0xFC48, + 0xFC49, + 0xFC4A, + 0xFC4B, + 0xFC4C, + 0xFC4D, + 0xFC4E, + 0xFC4F, + 0xFC50, + 0xFC51, + 0xFC52, + 0xFC53, + 0xFC54, + 0xFC55, + 0xFC56, + 0xFC57, + 0xFC58, + 0xFC59, + 0xFC5A, + 0xFC5B, + 0xFC5C, + 0xFC5D, + 0xFC5E, + 0xFC5F, + 0xFC60, + 0xFC61, + 0xFC62, + 0xFC63, + 0xFC64, + 0xFC65, + 0xFC66, + 0xFC67, + 0xFC68, + 0xFC69, + 0xFC6A, + 0xFC6B, + 0xFC6C, + 0xFC6D, + 0xFC6E, + 0xFC6F, + 0xFC70, + 0xFC71, + 0xFC72, + 0xFC73, + 0xFC74, + 0xFC75, + 0xFC76, + 0xFC77, + 0xFC78, + 0xFC79, + 0xFC7A, + 0xFC7B, + 0xFC7C, + 0xFC7D, + 0xFC7E, + 0xFC7F, + 0xFC80, + 0xFC81, + 0xFC82, + 0xFC83, + 0xFC84, + 0xFC85, + 0xFC86, + 0xFC87, + 0xFC88, + 0xFC89, + 0xFC8A, + 0xFC8B, + 0xFC8C, + 0xFC8D, + 0xFC8E, + 0xFC8F, + 0xFC90, + 0xFC91, + 0xFC92, + 0xFC93, + 0xFC94, + 0xFC95, + 0xFC96, + 0xFC97, + 0xFC98, + 0xFC99, + 0xFC9A, + 0xFC9B, + 0xFC9C, + 0xFC9D, + 0xFC9E, + 0xFC9F, + 0xFCA0, + 0xFCA1, + 0xFCA2, + 0xFCA3, + 0xFCA4, + 0xFCA5, + 0xFCA6, + 0xFCA7, + 0xFCA8, + 0xFCA9, + 0xFCAA, + 0xFCAB, + 0xFCAC, + 0xFCAD, + 0xFCAE, + 0xFCAF, + 0xFCB0, + 0xFCB1, + 0xFCB2, + 0xFCB3, + 0xFCB4, + 0xFCB5, + 0xFCB6, + 0xFCB7, + 0xFCB8, + 0xFCB9, + 0xFCBA, + 0xFCBB, + 0xFCBC, + 0xFCBD, + 0xFCBE, + 0xFCBF, + 0xFCC0, + 0xFCC1, + 0xFCC2, + 0xFCC3, + 0xFCC4, + 0xFCC5, + 0xFCC6, + 0xFCC7, + 0xFCC8, + 0xFCC9, + 0xFCCA, + 0xFCCB, + 0xFCCC, + 0xFCCD, + 0xFCCE, + 0xFCCF, + 0xFCD0, + 0xFCD1, + 0xFCD2, + 0xFCD3, + 0xFCD4, + 0xFCD5, + 0xFCD6, + 0xFCD7, + 0xFCD8, + 0xFCD9, + 0xFCDA, + 0xFCDB, + 0xFCDC, + 0xFCDD, + 0xFCDE, + 0xFCDF, + 0xFCE0, + 0xFCE1, + 0xFCE2, + 0xFCE3, + 0xFCE4, + 0xFCE5, + 0xFCE6, + 0xFCE7, + 0xFCE8, + 0xFCE9, + 0xFCEA, + 0xFCEB, + 0xFCEC, + 0xFCED, + 0xFCEE, + 0xFCEF, + 0xFCF0, + 0xFCF1, + 0xFCF2, + 0xFCF3, + 0xFCF4, + 0xFCF5, + 0xFCF6, + 0xFCF7, + 0xFCF8, + 0xFCF9, + 0xFCFA, + 0xFCFB, + 0xFCFC, + 0xFCFD, + 0xFCFE, + 0xFCFF, + 0xFD00, + 0xFD01, + 0xFD02, + 0xFD03, + 0xFD04, + 0xFD05, + 0xFD06, + 0xFD07, + 0xFD08, + 0xFD09, + 0xFD0A, + 0xFD0B, + 0xFD0C, + 0xFD0D, + 0xFD0E, + 0xFD0F, + 0xFD10, + 0xFD11, + 0xFD12, + 0xFD13, + 0xFD14, + 0xFD15, + 0xFD16, + 0xFD17, + 0xFD18, + 0xFD19, + 0xFD1A, + 0xFD1B, + 0xFD1C, + 0xFD1D, + 0xFD1E, + 0xFD1F, + 0xFD20, + 0xFD21, + 0xFD22, + 0xFD23, + 0xFD24, + 0xFD25, + 0xFD26, + 0xFD27, + 0xFD28, + 0xFD29, + 0xFD2A, + 0xFD2B, + 0xFD2C, + 0xFD2D, + 0xFD2E, + 0xFD2F, + 0xFD30, + 0xFD31, + 0xFD32, + 0xFD33, + 0xFD34, + 0xFD35, + 0xFD36, + 0xFD37, + 0xFD38, + 0xFD39, + 0xFD3A, + 0xFD3B, + 0xFD3C, + 0xFD3E, + 0xFD50, + 0xFD51, + 0xFD53, + 0xFD54, + 0xFD55, + 0xFD56, + 0xFD57, + 0xFD58, + 0xFD5A, + 0xFD5B, + 0xFD5C, + 0xFD5D, + 0xFD5E, + 0xFD5F, + 0xFD61, + 0xFD62, + 0xFD64, + 0xFD66, + 0xFD67, + 0xFD69, + 0xFD6A, + 0xFD6C, + 0xFD6E, + 0xFD6F, + 0xFD71, + 0xFD73, + 0xFD74, + 0xFD75, + 0xFD76, + 0xFD78, + 0xFD79, + 0xFD7A, + 0xFD7B, + 0xFD7C, + 0xFD7E, + 0xFD7F, + 0xFD80, + 0xFD81, + 0xFD82, + 0xFD83, + 0xFD85, + 0xFD87, + 0xFD89, + 0xFD8A, + 0xFD8B, + 0xFD8C, + 0xFD8D, + 0xFD8E, + 0xFD8F, + 0xFD90, + 0xFD92, + 0xFD93, + 0xFD94, + 0xFD95, + 0xFD96, + 0xFD97, + 0xFD99, + 0xFD9A, + 0xFD9B, + 0xFD9C, + 0xFD9E, + 0xFD9F, + 0xFDA0, + 0xFDA1, + 0xFDA2, + 0xFDA3, + 0xFDA4, + 0xFDA5, + 0xFDA6, + 0xFDA7, + 0xFDA8, + 0xFDA9, + 0xFDAA, + 0xFDAB, + 0xFDAC, + 0xFDAD, + 0xFDAE, + 0xFDAF, + 0xFDB0, + 0xFDB1, + 0xFDB2, + 0xFDB3, + 0xFDB4, + 0xFDB5, + 0xFDB6, + 0xFDB7, + 0xFDB8, + 0xFDB9, + 0xFDBA, + 0xFDBB, + 0xFDBC, + 0xFDBD, + 0xFDBE, + 0xFDBF, + 0xFDC0, + 0xFDC1, + 0xFDC2, + 0xFDC3, + 0xFDC4, + 0xFDC5, + 0xFDC6, + 0xFDC7, + 0xFDC8, + 0xFDD0, + 0xFDF0, + 0xFDF1, + 0xFDF2, + 0xFDF3, + 0xFDF4, + 0xFDF5, + 0xFDF6, + 0xFDF7, + 0xFDF8, + 0xFDF9, + 0xFDFA, + 0xFDFB, + 0xFDFC, + 0xFDFD, + 0xFE00, + 0xFE10, + 0xFE11, + 0xFE12, + 0xFE13, + 0xFE14, + 0xFE15, + 0xFE16, + 0xFE17, + 0xFE18, + 0xFE19, + 0xFE20, + 0xFE30, + 0xFE31, + 0xFE32, + 0xFE33, + 0xFE35, + 0xFE36, + 0xFE37, + 0xFE38, + 0xFE39, + 0xFE3A, + 0xFE3B, + 0xFE3C, + 0xFE3D, + 0xFE3E, + 0xFE3F, + 0xFE40, + 0xFE41, + 0xFE42, + 0xFE43, + 0xFE44, + 0xFE45, + 0xFE47, + 0xFE48, + 0xFE49, + 0xFE4D, + 0xFE50, + 0xFE51, + 0xFE52, + 0xFE54, + 0xFE55, + 0xFE56, + 0xFE57, + 0xFE58, + 0xFE59, + 0xFE5A, + 0xFE5B, + 0xFE5C, + 0xFE5D, + 0xFE5E, + 0xFE5F, + 0xFE60, + 0xFE61, + 0xFE62, + 0xFE63, + 0xFE64, + 0xFE65, + 0xFE66, + 0xFE67, + 0xFE68, + 0xFE69, + 0xFE6A, + 0xFE6B, + 0xFE6C, + 0xFE70, + 0xFE71, + 0xFE72, + 0xFE73, + 0xFE74, + 0xFE75, + 0xFE76, + 0xFE77, + 0xFE78, + 0xFE79, + 0xFE7A, + 0xFE7B, + 0xFE7C, + 0xFE7D, + 0xFE7E, + 0xFE7F, + 0xFE80, + 0xFE81, + 0xFE83, + 0xFE85, + 0xFE87, + 0xFE89, + 0xFE8D, + 0xFE8F, + 0xFE93, + 0xFE95, + 0xFE99, + 0xFE9D, + 0xFEA1, + 0xFEA5, + 0xFEA9, + 0xFEAB, + 0xFEAD, + 0xFEAF, + 0xFEB1, + 0xFEB5, + 0xFEB9, + 0xFEBD, + 0xFEC1, + 0xFEC5, + 0xFEC9, + 0xFECD, + 0xFED1, + 0xFED5, + 0xFED9, + 0xFEDD, + 0xFEE1, + 0xFEE5, + 0xFEE9, + 0xFEED, + 0xFEEF, + 0xFEF1, + 0xFEF5, + 0xFEF7, + 0xFEF9, + 0xFEFB, + 0xFEFD, + 0xFEFF, + 0xFF00, + 0xFF01, + 0xFF02, + 0xFF03, + 0xFF04, + 0xFF05, + 0xFF06, + 0xFF07, + 0xFF08, + 0xFF09, + 0xFF0A, + 0xFF0B, + 0xFF0C, + 0xFF0D, + 0xFF0E, + 0xFF0F, + 0xFF10, + 0xFF11, + 0xFF12, + 0xFF13, + 0xFF14, + 0xFF15, + 0xFF16, + 0xFF17, + 0xFF18, + 0xFF19, + 0xFF1A, + 0xFF1B, + 0xFF1C, + 0xFF1D, + 0xFF1E, + 0xFF1F, + 0xFF20, + 0xFF21, + 0xFF22, + 0xFF23, + 0xFF24, + 0xFF25, + 0xFF26, + 0xFF27, + 0xFF28, + 0xFF29, + 0xFF2A, + 0xFF2B, + 0xFF2C, + 0xFF2D, + 0xFF2E, + 0xFF2F, + 0xFF30, + 0xFF31, + 0xFF32, + 0xFF33, + 0xFF34, + 0xFF35, + 0xFF36, + 0xFF37, + 0xFF38, + 0xFF39, + 0xFF3A, + 0xFF3B, + 0xFF3C, + 0xFF3D, + 0xFF3E, + 0xFF3F, + 0xFF40, + 0xFF41, + 0xFF42, + 0xFF43, + 0xFF44, + 0xFF45, + 0xFF46, + 0xFF47, + 0xFF48, + 0xFF49, + 0xFF4A, + 0xFF4B, + 0xFF4C, + 0xFF4D, + 0xFF4E, + 0xFF4F, + 0xFF50, + 0xFF51, + 0xFF52, + 0xFF53, + 0xFF54, + 0xFF55, + 0xFF56, + 0xFF57, + 0xFF58, + 0xFF59, + 0xFF5A, + 0xFF5B, + 0xFF5C, + 0xFF5D, + 0xFF5E, + 0xFF5F, + 0xFF60, + 0xFF61, + 0xFF62, + 0xFF63, + 0xFF64, + 0xFF65, + 0xFF66, + 0xFF67, + 0xFF68, + 0xFF69, + 0xFF6A, + 0xFF6B, + 0xFF6C, + 0xFF6D, + 0xFF6E, + 0xFF6F, + 0xFF70, + 0xFF71, + 0xFF72, + 0xFF73, + 0xFF74, + 0xFF75, + 0xFF76, + 0xFF77, + 0xFF78, + 0xFF79, + 0xFF7A, + 0xFF7B, + 0xFF7C, + 0xFF7D, + 0xFF7E, + 0xFF7F, + 0xFF80, + 0xFF81, + 0xFF82, + 0xFF83, + 0xFF84, + 0xFF85, + 0xFF86, + 0xFF87, + 0xFF88, + 0xFF89, + 0xFF8A, + 0xFF8B, + 0xFF8C, + 0xFF8D, + 0xFF8E, + 0xFF8F, + 0xFF90, + 0xFF91, + 0xFF92, + 0xFF93, + 0xFF94, + 0xFF95, + 0xFF96, + 0xFF97, + 0xFF98, + 0xFF99, + 0xFF9A, + 0xFF9B, + 0xFF9C, + 0xFF9D, + 0xFF9E, + 0xFF9F, + 0xFFA0, + 0xFFA1, + 0xFFA2, + 0xFFA3, + 0xFFA4, + 0xFFA5, + 0xFFA6, + 0xFFA7, + 0xFFA8, + 0xFFA9, + 0xFFAA, + 0xFFAB, + 0xFFAC, + 0xFFAD, + 0xFFAE, + 0xFFAF, + 0xFFB0, + 0xFFB1, + 0xFFB2, + 0xFFB3, + 0xFFB4, + 0xFFB5, + 0xFFB6, + 0xFFB7, + 0xFFB8, + 0xFFB9, + 0xFFBA, + 0xFFBB, + 0xFFBC, + 0xFFBD, + 0xFFBE, + 0xFFBF, + 0xFFC2, + 0xFFC3, + 0xFFC4, + 0xFFC5, + 0xFFC6, + 0xFFC7, + 0xFFC8, + 0xFFCA, + 0xFFCB, + 0xFFCC, + 0xFFCD, + 0xFFCE, + 0xFFCF, + 0xFFD0, + 0xFFD2, + 0xFFD3, + 0xFFD4, + 0xFFD5, + 0xFFD6, + 0xFFD7, + 0xFFD8, + 0xFFDA, + 0xFFDB, + 0xFFDC, + 0xFFDD, + 0xFFE0, + 0xFFE1, + 0xFFE2, + 0xFFE3, + 0xFFE4, + 0xFFE5, + 0xFFE6, + 0xFFE7, + 0xFFE8, + 0xFFE9, + 0xFFEA, + 0xFFEB, + 0xFFEC, + 0xFFED, + 0xFFEE, + 0xFFEF, + 0x10000, + 0x1000C, + 0x1000D, + 0x10027, + 0x10028, + 0x1003B, + 0x1003C, + 0x1003E, + 0x1003F, + 0x1004E, + 0x10050, + 0x1005E, + 0x10080, + 0x100FB, + 0x10100, + 0x10103, + 0x10107, + 0x10134, + 0x10137, + 0x1018F, + 0x10190, + 0x1019D, + 0x101A0, + 0x101A1, + 0x101D0, + 0x101FE, + 0x10280, + 0x1029D, + 0x102A0, + 0x102D1, + 0x102E0, + 0x102FC, + 0x10300, + 0x10324, + 0x1032D, + 0x1034B, + 0x10350, + 0x1037B, + 0x10380, + 0x1039E, + 0x1039F, + 0x103C4, + 0x103C8, + 0x103D6, + 0x10400, + 0x10401, + 0x10402, + 0x10403, + 0x10404, + 0x10405, + 0x10406, + 0x10407, + 0x10408, + 0x10409, + 0x1040A, + 0x1040B, + 0x1040C, + 0x1040D, + 0x1040E, + 0x1040F, + 0x10410, + 0x10411, + 0x10412, + 0x10413, + 0x10414, + 0x10415, + 0x10416, + 0x10417, + 0x10418, + 0x10419, + 0x1041A, + 0x1041B, + 0x1041C, + 0x1041D, + 0x1041E, + 0x1041F, + 0x10420, + 0x10421, + 0x10422, + 0x10423, + 0x10424, + 0x10425, + 0x10426, + 0x10427, + 0x10428, + 0x1049E, + 0x104A0, + 0x104AA, + 0x104B0, + 0x104B1, + 0x104B2, + 0x104B3, + 0x104B4, + 0x104B5, + 0x104B6, + 0x104B7, + 0x104B8, + 0x104B9, + 0x104BA, + 0x104BB, + 0x104BC, + 0x104BD, + 0x104BE, + 0x104BF, + 0x104C0, + 0x104C1, + 0x104C2, + 0x104C3, + 0x104C4, + 0x104C5, + 0x104C6, + 0x104C7, + 0x104C8, + 0x104C9, + 0x104CA, + 0x104CB, + 0x104CC, + 0x104CD, + 0x104CE, + 0x104CF, + 0x104D0, + 0x104D1, + 0x104D2, + 0x104D3, + 0x104D4, + 0x104D8, + 0x104FC, + 0x10500, + 0x10528, + 0x10530, + 0x10564, + 0x1056F, + 0x10570, + 0x10571, + 0x10572, + 0x10573, + 0x10574, + 0x10575, + 0x10576, + 0x10577, + 0x10578, + 0x10579, + 0x1057A, + 0x1057B, + 0x1057C, + 0x1057D, + 0x1057E, + 0x1057F, + 0x10580, + 0x10581, + 0x10582, + 0x10583, + 0x10584, + 0x10585, + 0x10586, + 0x10587, + 0x10588, + 0x10589, + 0x1058A, + 0x1058B, + 0x1058C, + 0x1058D, + 0x1058E, + 0x1058F, + 0x10590, + 0x10591, + 0x10592, + 0x10593, + 0x10594, + 0x10595, + 0x10596, + 0x10597, + 0x105A2, + 0x105A3, + 0x105B2, + 0x105B3, + 0x105BA, + 0x105BB, + 0x105BD, + 0x105C0, + 0x105F4, + 0x10600, + 0x10737, + 0x10740, + 0x10756, + 0x10760, + 0x10768, + 0x10780, + 0x10781, + 0x10782, + 0x10783, + 0x10784, + 0x10785, + 0x10786, + 0x10787, + 0x10788, + 0x10789, + 0x1078A, + 0x1078B, + 0x1078C, + 0x1078D, + 0x1078E, + 0x1078F, + 0x10790, + 0x10791, + 0x10792, + 0x10793, + 0x10794, + 0x10795, + 0x10796, + 0x10797, + 0x10798, + 0x10799, + 0x1079A, + 0x1079B, + 0x1079C, + 0x1079D, + 0x1079E, + 0x1079F, + 0x107A0, + 0x107A1, + 0x107A2, + 0x107A3, + 0x107A4, + 0x107A5, + 0x107A6, + 0x107A7, + 0x107A8, + 0x107A9, + 0x107AA, + 0x107AB, + 0x107AC, + 0x107AD, + 0x107AE, + 0x107AF, + 0x107B0, + 0x107B1, + 0x107B2, + 0x107B3, + 0x107B4, + 0x107B5, + 0x107B6, + 0x107B7, + 0x107B8, + 0x107B9, + 0x107BA, + 0x107BB, + 0x10800, + 0x10806, + 0x10808, + 0x10809, + 0x1080A, + 0x10836, + 0x10837, + 0x10839, + 0x1083C, + 0x1083D, + 0x1083F, + 0x10856, + 0x10857, + 0x1089F, + 0x108A7, + 0x108B0, + 0x108E0, + 0x108F3, + 0x108F4, + 0x108F6, + 0x108FB, + 0x1091C, + 0x1091F, + 0x1093A, + 0x1093F, + 0x1095A, + 0x10980, + 0x109B8, + 0x109BC, + 0x109D0, + 0x109D2, + 0x10A04, + 0x10A05, + 0x10A07, + 0x10A0C, + 0x10A14, + 0x10A15, + 0x10A18, + 0x10A19, + 0x10A36, + 0x10A38, + 0x10A3B, + 0x10A3F, + 0x10A49, + 0x10A50, + 0x10A59, + 0x10A60, + 0x10AA0, + 0x10AC0, + 0x10AE7, + 0x10AEB, + 0x10AF7, + 0x10B00, + 0x10B36, + 0x10B39, + 0x10B56, + 0x10B58, + 0x10B73, + 0x10B78, + 0x10B92, + 0x10B99, + 0x10B9D, + 0x10BA9, + 0x10BB0, + 0x10C00, + 0x10C49, + 0x10C80, + 0x10C81, + 0x10C82, + 0x10C83, + 0x10C84, + 0x10C85, + 0x10C86, + 0x10C87, + 0x10C88, + 0x10C89, + 0x10C8A, + 0x10C8B, + 0x10C8C, + 0x10C8D, + 0x10C8E, + 0x10C8F, + 0x10C90, + 0x10C91, + 0x10C92, + 0x10C93, + 0x10C94, + 0x10C95, + 0x10C96, + 0x10C97, + 0x10C98, + 0x10C99, + 0x10C9A, + 0x10C9B, + 0x10C9C, + 0x10C9D, + 0x10C9E, + 0x10C9F, + 0x10CA0, + 0x10CA1, + 0x10CA2, + 0x10CA3, + 0x10CA4, + 0x10CA5, + 0x10CA6, + 0x10CA7, + 0x10CA8, + 0x10CA9, + 0x10CAA, + 0x10CAB, + 0x10CAC, + 0x10CAD, + 0x10CAE, + 0x10CAF, + 0x10CB0, + 0x10CB1, + 0x10CB2, + 0x10CB3, + 0x10CC0, + 0x10CF3, + 0x10CFA, + 0x10D28, + 0x10D30, + 0x10D3A, + 0x10D40, + 0x10D50, + 0x10D51, + 0x10D52, + 0x10D53, + 0x10D54, + 0x10D55, + 0x10D56, + 0x10D57, + 0x10D58, + 0x10D59, + 0x10D5A, + 0x10D5B, + 0x10D5C, + 0x10D5D, + 0x10D5E, + 0x10D5F, + 0x10D60, + 0x10D61, + 0x10D62, + 0x10D63, + 0x10D64, + 0x10D65, + 0x10D66, + 0x10D69, + 0x10D86, + 0x10D8E, + 0x10D90, + 0x10E60, + 0x10E7F, + 0x10E80, + 0x10EAA, + 0x10EAB, + 0x10EAE, + 0x10EB0, + 0x10EB2, + 0x10EC2, + 0x10EC8, + 0x10ED0, + 0x10ED9, + 0x10EFA, + 0x10F28, + 0x10F30, + 0x10F5A, + 0x10F70, + 0x10F8A, + 0x10FB0, + 0x10FCC, + 0x10FE0, + 0x10FF7, + 0x11000, + 0x1104E, + 0x11052, + 0x11076, + 0x1107F, + 0x110BD, + 0x110BE, + 0x110C3, + 0x110D0, + 0x110E9, + 0x110F0, + 0x110FA, + 0x11100, + 0x11135, + 0x11136, + 0x11148, + 0x11150, + 0x11177, + 0x11180, + 0x111E0, + 0x111E1, + 0x111F5, + 0x11200, + 0x11212, + 0x11213, + 0x11242, + 0x11280, + 0x11287, + 0x11288, + 0x11289, + 0x1128A, + 0x1128E, + 0x1128F, + 0x1129E, + 0x1129F, + 0x112AA, + 0x112B0, + 0x112EB, + 0x112F0, + 0x112FA, + 0x11300, + 0x11304, + 0x11305, + 0x1130D, + 0x1130F, + 0x11311, + 0x11313, + 0x11329, + 0x1132A, + 0x11331, + 0x11332, + 0x11334, + 0x11335, + 0x1133A, + 0x1133B, + 0x11345, + 0x11347, + 0x11349, + 0x1134B, + 0x1134E, + 0x11350, + 0x11351, + 0x11357, + 0x11358, + 0x1135D, + 0x11364, + 0x11366, + 0x1136D, + 0x11370, + 0x11375, + 0x11380, + 0x1138A, + 0x1138B, + 0x1138C, + 0x1138E, + 0x1138F, + 0x11390, + 0x113B6, + 0x113B7, + 0x113C1, + 0x113C2, + 0x113C3, + 0x113C5, + 0x113C6, + 0x113C7, + 0x113CB, + 0x113CC, + 0x113D6, + 0x113D7, + 0x113D9, + 0x113E1, + 0x113E3, + 0x11400, + 0x1145C, + 0x1145D, + 0x11462, + 0x11480, + 0x114C8, + 0x114D0, + 0x114DA, + 0x11580, + 0x115B6, + 0x115B8, + 0x115DE, + 0x11600, + 0x11645, + 0x11650, + 0x1165A, + 0x11660, + 0x1166D, + 0x11680, + 0x116BA, + 0x116C0, + 0x116CA, + 0x116D0, + 0x116E4, + 0x11700, + 0x1171B, + 0x1171D, + 0x1172C, + 0x11730, + 0x11747, + 0x11800, + 0x1183C, + 0x118A0, + 0x118A1, + 0x118A2, + 0x118A3, + 0x118A4, + 0x118A5, + 0x118A6, + 0x118A7, + 0x118A8, + 0x118A9, + 0x118AA, + 0x118AB, + 0x118AC, + 0x118AD, + 0x118AE, + 0x118AF, + 0x118B0, + 0x118B1, + 0x118B2, + 0x118B3, + 0x118B4, + 0x118B5, + 0x118B6, + 0x118B7, + 0x118B8, + 0x118B9, + 0x118BA, + 0x118BB, + 0x118BC, + 0x118BD, + 0x118BE, + 0x118BF, + 0x118C0, + 0x118F3, + 0x118FF, + 0x11907, + 0x11909, + 0x1190A, + 0x1190C, + 0x11914, + 0x11915, + 0x11917, + 0x11918, + 0x11936, + 0x11937, + 0x11939, + 0x1193B, + 0x11947, + 0x11950, + 0x1195A, + 0x119A0, + 0x119A8, + 0x119AA, + 0x119D8, + 0x119DA, + 0x119E5, + 0x11A00, + 0x11A48, + 0x11A50, + 0x11AA3, + 0x11AB0, + 0x11AF9, + 0x11B00, + 0x11B0A, + 0x11B60, + 0x11B68, + 0x11BC0, + 0x11BE2, + 0x11BF0, + 0x11BFA, + 0x11C00, + 0x11C09, + 0x11C0A, + 0x11C37, + 0x11C38, + 0x11C46, + 0x11C50, + 0x11C6D, + 0x11C70, + 0x11C90, + 0x11C92, + 0x11CA8, + 0x11CA9, + 0x11CB7, + 0x11D00, + 0x11D07, + 0x11D08, + 0x11D0A, + 0x11D0B, + 0x11D37, + 0x11D3A, + 0x11D3B, + 0x11D3C, + 0x11D3E, + 0x11D3F, + 0x11D48, + 0x11D50, + 0x11D5A, + 0x11D60, + 0x11D66, + 0x11D67, + 0x11D69, + 0x11D6A, + 0x11D8F, + 0x11D90, + 0x11D92, + 0x11D93, + 0x11D99, + 0x11DA0, + 0x11DAA, + 0x11DB0, + 0x11DDC, + 0x11DE0, + 0x11DEA, + 0x11EE0, + 0x11EF9, + 0x11F00, + 0x11F11, + 0x11F12, + 0x11F3B, + 0x11F3E, + 0x11F5B, + 0x11FB0, + 0x11FB1, + 0x11FC0, + 0x11FF2, + 0x11FFF, + 0x1239A, + 0x12400, + 0x1246F, + 0x12470, + 0x12475, + 0x12480, + 0x12544, + 0x12F90, + 0x12FF3, + 0x13000, + 0x13430, + 0x13440, + 0x13456, + 0x13460, + 0x143FB, + 0x14400, + 0x14647, + 0x16100, + 0x1613A, + 0x16800, + 0x16A39, + 0x16A40, + 0x16A5F, + 0x16A60, + 0x16A6A, + 0x16A6E, + 0x16ABF, + 0x16AC0, + 0x16ACA, + 0x16AD0, + 0x16AEE, + 0x16AF0, + 0x16AF6, + 0x16B00, + 0x16B46, + 0x16B50, + 0x16B5A, + 0x16B5B, + 0x16B62, + 0x16B63, + 0x16B78, + 0x16B7D, + 0x16B90, + 0x16D40, + 0x16D7A, + 0x16E40, + 0x16E41, + 0x16E42, + 0x16E43, + 0x16E44, + 0x16E45, + 0x16E46, + 0x16E47, + 0x16E48, + 0x16E49, + 0x16E4A, + 0x16E4B, + 0x16E4C, + 0x16E4D, + 0x16E4E, + 0x16E4F, + 0x16E50, + 0x16E51, + 0x16E52, + 0x16E53, + 0x16E54, + 0x16E55, + 0x16E56, + 0x16E57, + 0x16E58, + 0x16E59, + 0x16E5A, + 0x16E5B, + 0x16E5C, + 0x16E5D, + 0x16E5E, + 0x16E5F, + 0x16E60, + 0x16E9B, + 0x16EA0, + 0x16EA1, + 0x16EA2, + 0x16EA3, + 0x16EA4, + 0x16EA5, + 0x16EA6, + 0x16EA7, + 0x16EA8, + 0x16EA9, + 0x16EAA, + 0x16EAB, + 0x16EAC, + 0x16EAD, + 0x16EAE, + 0x16EAF, + 0x16EB0, + 0x16EB1, + 0x16EB2, + 0x16EB3, + 0x16EB4, + 0x16EB5, + 0x16EB6, + 0x16EB7, + 0x16EB8, + 0x16EB9, + 0x16EBB, + 0x16ED4, + 0x16F00, + 0x16F4B, + 0x16F4F, + 0x16F88, + 0x16F8F, + 0x16FA0, + 0x16FE0, + 0x16FE5, + 0x16FF0, + 0x16FF7, + 0x17000, + 0x18CD6, + 0x18CFF, + 0x18D1F, + 0x18D80, + 0x18DF3, + 0x1AFF0, + 0x1AFF4, + 0x1AFF5, + 0x1AFFC, + 0x1AFFD, + 0x1AFFF, + 0x1B000, + 0x1B123, + 0x1B132, + 0x1B133, + 0x1B150, + 0x1B153, + 0x1B155, + 0x1B156, + 0x1B164, + 0x1B168, + 0x1B170, + 0x1B2FC, + 0x1BC00, + 0x1BC6B, + 0x1BC70, + 0x1BC7D, + 0x1BC80, + 0x1BC89, + 0x1BC90, + 0x1BC9A, + 0x1BC9C, + 0x1BCA0, + 0x1BCA4, + 0x1CC00, + 0x1CCD6, + 0x1CCD7, + 0x1CCD8, + 0x1CCD9, + 0x1CCDA, + 0x1CCDB, + 0x1CCDC, + 0x1CCDD, + 0x1CCDE, + 0x1CCDF, + 0x1CCE0, + 0x1CCE1, + 0x1CCE2, + 0x1CCE3, + 0x1CCE4, + 0x1CCE5, + 0x1CCE6, + 0x1CCE7, + 0x1CCE8, + 0x1CCE9, + 0x1CCEA, + 0x1CCEB, + 0x1CCEC, + 0x1CCED, + 0x1CCEE, + 0x1CCEF, + 0x1CCF0, + 0x1CCF1, + 0x1CCF2, + 0x1CCF3, + 0x1CCF4, + 0x1CCF5, + 0x1CCF6, + 0x1CCF7, + 0x1CCF8, + 0x1CCF9, + 0x1CCFA, + 0x1CCFD, + 0x1CD00, + 0x1CEB4, + 0x1CEBA, + 0x1CED1, + 0x1CEE0, + 0x1CEF1, + 0x1CF00, + 0x1CF2E, + 0x1CF30, + 0x1CF47, + 0x1CF50, + 0x1CFC4, + 0x1D000, + 0x1D0F6, + 0x1D100, + 0x1D127, + 0x1D129, + 0x1D15E, + 0x1D15F, + 0x1D160, + 0x1D161, + 0x1D162, + 0x1D163, + 0x1D164, + 0x1D165, + 0x1D173, + 0x1D17B, + 0x1D1BB, + 0x1D1BC, + 0x1D1BD, + 0x1D1BE, + 0x1D1BF, + 0x1D1C0, + 0x1D1C1, + 0x1D1EB, + 0x1D200, + 0x1D246, + 0x1D2C0, + 0x1D2D4, + 0x1D2E0, + 0x1D2F4, + 0x1D300, + 0x1D357, + 0x1D360, + 0x1D379, + 0x1D400, + 0x1D401, + 0x1D402, + 0x1D403, + 0x1D404, + 0x1D405, + 0x1D406, + 0x1D407, + 0x1D408, + 0x1D409, + 0x1D40A, + 0x1D40B, + 0x1D40C, + 0x1D40D, + 0x1D40E, + 0x1D40F, + 0x1D410, + 0x1D411, + 0x1D412, + 0x1D413, + 0x1D414, + 0x1D415, + 0x1D416, + 0x1D417, + 0x1D418, + 0x1D419, + 0x1D41A, + 0x1D41B, + 0x1D41C, + 0x1D41D, + 0x1D41E, + 0x1D41F, + 0x1D420, + 0x1D421, + 0x1D422, + 0x1D423, + 0x1D424, + 0x1D425, + 0x1D426, + 0x1D427, + 0x1D428, + 0x1D429, + 0x1D42A, + 0x1D42B, + 0x1D42C, + 0x1D42D, + 0x1D42E, + 0x1D42F, + 0x1D430, + 0x1D431, + 0x1D432, + 0x1D433, + 0x1D434, + 0x1D435, + 0x1D436, + 0x1D437, + 0x1D438, + 0x1D439, + 0x1D43A, + 0x1D43B, + 0x1D43C, + 0x1D43D, + 0x1D43E, + 0x1D43F, + 0x1D440, + 0x1D441, + 0x1D442, + 0x1D443, + 0x1D444, + 0x1D445, + 0x1D446, + 0x1D447, + 0x1D448, + 0x1D449, + 0x1D44A, + 0x1D44B, + 0x1D44C, + 0x1D44D, + 0x1D44E, + 0x1D44F, + 0x1D450, + 0x1D451, + 0x1D452, + 0x1D453, + 0x1D454, + 0x1D455, + 0x1D456, + 0x1D457, + 0x1D458, + 0x1D459, + 0x1D45A, + 0x1D45B, + 0x1D45C, + 0x1D45D, + 0x1D45E, + 0x1D45F, + 0x1D460, + 0x1D461, + 0x1D462, + 0x1D463, + 0x1D464, + 0x1D465, + 0x1D466, + 0x1D467, + 0x1D468, + 0x1D469, + 0x1D46A, + 0x1D46B, + 0x1D46C, + 0x1D46D, + 0x1D46E, + 0x1D46F, + 0x1D470, + 0x1D471, + 0x1D472, + 0x1D473, + 0x1D474, + 0x1D475, + 0x1D476, + 0x1D477, + 0x1D478, + 0x1D479, + 0x1D47A, + 0x1D47B, + 0x1D47C, + 0x1D47D, + 0x1D47E, + 0x1D47F, + 0x1D480, + 0x1D481, + 0x1D482, + 0x1D483, + 0x1D484, + 0x1D485, + 0x1D486, + 0x1D487, + 0x1D488, + 0x1D489, + 0x1D48A, + 0x1D48B, + 0x1D48C, + 0x1D48D, + 0x1D48E, + 0x1D48F, + 0x1D490, + 0x1D491, + 0x1D492, + 0x1D493, + 0x1D494, + 0x1D495, + 0x1D496, + 0x1D497, + 0x1D498, + 0x1D499, + 0x1D49A, + 0x1D49B, + 0x1D49C, + 0x1D49D, + 0x1D49E, + 0x1D49F, + 0x1D4A0, + 0x1D4A2, + 0x1D4A3, + 0x1D4A5, + 0x1D4A6, + 0x1D4A7, + 0x1D4A9, + 0x1D4AA, + 0x1D4AB, + 0x1D4AC, + 0x1D4AD, + 0x1D4AE, + 0x1D4AF, + 0x1D4B0, + 0x1D4B1, + 0x1D4B2, + 0x1D4B3, + 0x1D4B4, + 0x1D4B5, + 0x1D4B6, + 0x1D4B7, + 0x1D4B8, + 0x1D4B9, + 0x1D4BA, + 0x1D4BB, + 0x1D4BC, + 0x1D4BD, + 0x1D4BE, + 0x1D4BF, + 0x1D4C0, + 0x1D4C1, + 0x1D4C2, + 0x1D4C3, + 0x1D4C4, + 0x1D4C5, + 0x1D4C6, + 0x1D4C7, + 0x1D4C8, + 0x1D4C9, + 0x1D4CA, + 0x1D4CB, + 0x1D4CC, + 0x1D4CD, + 0x1D4CE, + 0x1D4CF, + 0x1D4D0, + 0x1D4D1, + 0x1D4D2, + 0x1D4D3, + 0x1D4D4, + 0x1D4D5, + 0x1D4D6, + 0x1D4D7, + 0x1D4D8, + 0x1D4D9, + 0x1D4DA, + 0x1D4DB, + 0x1D4DC, + 0x1D4DD, + 0x1D4DE, + 0x1D4DF, + 0x1D4E0, + 0x1D4E1, + 0x1D4E2, + 0x1D4E3, + 0x1D4E4, + 0x1D4E5, + 0x1D4E6, + 0x1D4E7, + 0x1D4E8, + 0x1D4E9, + 0x1D4EA, + 0x1D4EB, + 0x1D4EC, + 0x1D4ED, + 0x1D4EE, + 0x1D4EF, + 0x1D4F0, + 0x1D4F1, + 0x1D4F2, + 0x1D4F3, + 0x1D4F4, + 0x1D4F5, + 0x1D4F6, + 0x1D4F7, + 0x1D4F8, + 0x1D4F9, + 0x1D4FA, + 0x1D4FB, + 0x1D4FC, + 0x1D4FD, + 0x1D4FE, + 0x1D4FF, + 0x1D500, + 0x1D501, + 0x1D502, + 0x1D503, + 0x1D504, + 0x1D505, + 0x1D506, + 0x1D507, + 0x1D508, + 0x1D509, + 0x1D50A, + 0x1D50B, + 0x1D50D, + 0x1D50E, + 0x1D50F, + 0x1D510, + 0x1D511, + 0x1D512, + 0x1D513, + 0x1D514, + 0x1D515, + 0x1D516, + 0x1D517, + 0x1D518, + 0x1D519, + 0x1D51A, + 0x1D51B, + 0x1D51C, + 0x1D51D, + 0x1D51E, + 0x1D51F, + 0x1D520, + 0x1D521, + 0x1D522, + 0x1D523, + 0x1D524, + 0x1D525, + 0x1D526, + 0x1D527, + 0x1D528, + 0x1D529, + 0x1D52A, + 0x1D52B, + 0x1D52C, + 0x1D52D, + 0x1D52E, + 0x1D52F, + 0x1D530, + 0x1D531, + 0x1D532, + 0x1D533, + 0x1D534, + 0x1D535, + 0x1D536, + 0x1D537, + 0x1D538, + 0x1D539, + 0x1D53A, + 0x1D53B, + 0x1D53C, + 0x1D53D, + 0x1D53E, + 0x1D53F, + 0x1D540, + 0x1D541, + 0x1D542, + 0x1D543, + 0x1D544, + 0x1D545, + 0x1D546, + 0x1D547, + 0x1D54A, + 0x1D54B, + 0x1D54C, + 0x1D54D, + 0x1D54E, + 0x1D54F, + 0x1D550, + 0x1D551, + 0x1D552, + 0x1D553, + 0x1D554, + 0x1D555, + 0x1D556, + 0x1D557, + 0x1D558, + 0x1D559, + 0x1D55A, + 0x1D55B, + 0x1D55C, + 0x1D55D, + 0x1D55E, + 0x1D55F, + 0x1D560, + 0x1D561, + 0x1D562, + 0x1D563, + 0x1D564, + 0x1D565, + 0x1D566, + 0x1D567, + 0x1D568, + 0x1D569, + 0x1D56A, + 0x1D56B, + 0x1D56C, + 0x1D56D, + 0x1D56E, + 0x1D56F, + 0x1D570, + 0x1D571, + 0x1D572, + 0x1D573, + 0x1D574, + 0x1D575, + 0x1D576, + 0x1D577, + 0x1D578, + 0x1D579, + 0x1D57A, + 0x1D57B, + 0x1D57C, + 0x1D57D, + 0x1D57E, + 0x1D57F, + 0x1D580, + 0x1D581, + 0x1D582, + 0x1D583, + 0x1D584, + 0x1D585, + 0x1D586, + 0x1D587, + 0x1D588, + 0x1D589, + 0x1D58A, + 0x1D58B, + 0x1D58C, + 0x1D58D, + 0x1D58E, + 0x1D58F, + 0x1D590, + 0x1D591, + 0x1D592, + 0x1D593, + 0x1D594, + 0x1D595, + 0x1D596, + 0x1D597, + 0x1D598, + 0x1D599, + 0x1D59A, + 0x1D59B, + 0x1D59C, + 0x1D59D, + 0x1D59E, + 0x1D59F, + 0x1D5A0, + 0x1D5A1, + 0x1D5A2, + 0x1D5A3, + 0x1D5A4, + 0x1D5A5, + 0x1D5A6, + 0x1D5A7, + 0x1D5A8, + 0x1D5A9, + 0x1D5AA, + 0x1D5AB, + 0x1D5AC, + 0x1D5AD, + 0x1D5AE, + 0x1D5AF, + 0x1D5B0, + 0x1D5B1, + 0x1D5B2, + 0x1D5B3, + 0x1D5B4, + 0x1D5B5, + 0x1D5B6, + 0x1D5B7, + 0x1D5B8, + 0x1D5B9, + 0x1D5BA, + 0x1D5BB, + 0x1D5BC, + 0x1D5BD, + 0x1D5BE, + 0x1D5BF, + 0x1D5C0, + 0x1D5C1, + 0x1D5C2, + 0x1D5C3, + 0x1D5C4, + 0x1D5C5, + 0x1D5C6, + 0x1D5C7, + 0x1D5C8, + 0x1D5C9, + 0x1D5CA, + 0x1D5CB, + 0x1D5CC, + 0x1D5CD, + 0x1D5CE, + 0x1D5CF, + 0x1D5D0, + 0x1D5D1, + 0x1D5D2, + 0x1D5D3, + 0x1D5D4, + 0x1D5D5, + 0x1D5D6, + 0x1D5D7, + 0x1D5D8, + 0x1D5D9, + 0x1D5DA, + 0x1D5DB, + 0x1D5DC, + 0x1D5DD, + 0x1D5DE, + 0x1D5DF, + 0x1D5E0, + 0x1D5E1, + 0x1D5E2, + 0x1D5E3, + 0x1D5E4, + 0x1D5E5, + 0x1D5E6, + 0x1D5E7, + 0x1D5E8, + 0x1D5E9, + 0x1D5EA, + 0x1D5EB, + 0x1D5EC, + 0x1D5ED, + 0x1D5EE, + 0x1D5EF, + 0x1D5F0, + 0x1D5F1, + 0x1D5F2, + 0x1D5F3, + 0x1D5F4, + 0x1D5F5, + 0x1D5F6, + 0x1D5F7, + 0x1D5F8, + 0x1D5F9, + 0x1D5FA, + 0x1D5FB, + 0x1D5FC, + 0x1D5FD, + 0x1D5FE, + 0x1D5FF, + 0x1D600, + 0x1D601, + 0x1D602, + 0x1D603, + 0x1D604, + 0x1D605, + 0x1D606, + 0x1D607, + 0x1D608, + 0x1D609, + 0x1D60A, + 0x1D60B, + 0x1D60C, + 0x1D60D, + 0x1D60E, + 0x1D60F, + 0x1D610, + 0x1D611, + 0x1D612, + 0x1D613, + 0x1D614, + 0x1D615, + 0x1D616, + 0x1D617, + 0x1D618, + 0x1D619, + 0x1D61A, + 0x1D61B, + 0x1D61C, + 0x1D61D, + 0x1D61E, + 0x1D61F, + 0x1D620, + 0x1D621, + 0x1D622, + 0x1D623, + 0x1D624, + 0x1D625, + 0x1D626, + 0x1D627, + 0x1D628, + 0x1D629, + 0x1D62A, + 0x1D62B, + 0x1D62C, + 0x1D62D, + 0x1D62E, + 0x1D62F, + 0x1D630, + 0x1D631, + 0x1D632, + 0x1D633, + 0x1D634, + 0x1D635, + 0x1D636, + 0x1D637, + 0x1D638, + 0x1D639, + 0x1D63A, + 0x1D63B, + 0x1D63C, + 0x1D63D, + 0x1D63E, + 0x1D63F, + 0x1D640, + 0x1D641, + 0x1D642, + 0x1D643, + 0x1D644, + 0x1D645, + 0x1D646, + 0x1D647, + 0x1D648, + 0x1D649, + 0x1D64A, + 0x1D64B, + 0x1D64C, + 0x1D64D, + 0x1D64E, + 0x1D64F, + 0x1D650, + 0x1D651, + 0x1D652, + 0x1D653, + 0x1D654, + 0x1D655, + 0x1D656, + 0x1D657, + 0x1D658, + 0x1D659, + 0x1D65A, + 0x1D65B, + 0x1D65C, + 0x1D65D, + 0x1D65E, + 0x1D65F, + 0x1D660, + 0x1D661, + 0x1D662, + 0x1D663, + 0x1D664, + 0x1D665, + 0x1D666, + 0x1D667, + 0x1D668, + 0x1D669, + 0x1D66A, + 0x1D66B, + 0x1D66C, + 0x1D66D, + 0x1D66E, + 0x1D66F, + 0x1D670, + 0x1D671, + 0x1D672, + 0x1D673, + 0x1D674, + 0x1D675, + 0x1D676, + 0x1D677, + 0x1D678, + 0x1D679, + 0x1D67A, + 0x1D67B, + 0x1D67C, + 0x1D67D, + 0x1D67E, + 0x1D67F, + 0x1D680, + 0x1D681, + 0x1D682, + 0x1D683, + 0x1D684, + 0x1D685, + 0x1D686, + 0x1D687, + 0x1D688, + 0x1D689, + 0x1D68A, + 0x1D68B, + 0x1D68C, + 0x1D68D, + 0x1D68E, + 0x1D68F, + 0x1D690, + 0x1D691, + 0x1D692, + 0x1D693, + 0x1D694, + 0x1D695, + 0x1D696, + 0x1D697, + 0x1D698, + 0x1D699, + 0x1D69A, + 0x1D69B, + 0x1D69C, + 0x1D69D, + 0x1D69E, + 0x1D69F, + 0x1D6A0, + 0x1D6A1, + 0x1D6A2, + 0x1D6A3, + 0x1D6A4, + 0x1D6A5, + 0x1D6A6, + 0x1D6A8, + 0x1D6A9, + 0x1D6AA, + 0x1D6AB, + 0x1D6AC, + 0x1D6AD, + 0x1D6AE, + 0x1D6AF, + 0x1D6B0, + 0x1D6B1, + 0x1D6B2, + 0x1D6B3, + 0x1D6B4, + 0x1D6B5, + 0x1D6B6, + 0x1D6B7, + 0x1D6B8, + 0x1D6B9, + 0x1D6BA, + 0x1D6BB, + 0x1D6BC, + 0x1D6BD, + 0x1D6BE, + 0x1D6BF, + 0x1D6C0, + 0x1D6C1, + 0x1D6C2, + 0x1D6C3, + 0x1D6C4, + 0x1D6C5, + 0x1D6C6, + 0x1D6C7, + 0x1D6C8, + 0x1D6C9, + 0x1D6CA, + 0x1D6CB, + 0x1D6CC, + 0x1D6CD, + 0x1D6CE, + 0x1D6CF, + 0x1D6D0, + 0x1D6D1, + 0x1D6D2, + 0x1D6D3, + 0x1D6D5, + 0x1D6D6, + 0x1D6D7, + 0x1D6D8, + 0x1D6D9, + 0x1D6DA, + 0x1D6DB, + 0x1D6DC, + 0x1D6DD, + 0x1D6DE, + 0x1D6DF, + 0x1D6E0, + 0x1D6E1, + 0x1D6E2, + 0x1D6E3, + 0x1D6E4, + 0x1D6E5, + 0x1D6E6, + 0x1D6E7, + 0x1D6E8, + 0x1D6E9, + 0x1D6EA, + 0x1D6EB, + 0x1D6EC, + 0x1D6ED, + 0x1D6EE, + 0x1D6EF, + 0x1D6F0, + 0x1D6F1, + 0x1D6F2, + 0x1D6F3, + 0x1D6F4, + 0x1D6F5, + 0x1D6F6, + 0x1D6F7, + 0x1D6F8, + 0x1D6F9, + 0x1D6FA, + 0x1D6FB, + 0x1D6FC, + 0x1D6FD, + 0x1D6FE, + 0x1D6FF, + 0x1D700, + 0x1D701, + 0x1D702, + 0x1D703, + 0x1D704, + 0x1D705, + 0x1D706, + 0x1D707, + 0x1D708, + 0x1D709, + 0x1D70A, + 0x1D70B, + 0x1D70C, + 0x1D70D, + 0x1D70F, + 0x1D710, + 0x1D711, + 0x1D712, + 0x1D713, + 0x1D714, + 0x1D715, + 0x1D716, + 0x1D717, + 0x1D718, + 0x1D719, + 0x1D71A, + 0x1D71B, + 0x1D71C, + 0x1D71D, + 0x1D71E, + 0x1D71F, + 0x1D720, + 0x1D721, + 0x1D722, + 0x1D723, + 0x1D724, + 0x1D725, + 0x1D726, + 0x1D727, + 0x1D728, + 0x1D729, + 0x1D72A, + 0x1D72B, + 0x1D72C, + 0x1D72D, + 0x1D72E, + 0x1D72F, + 0x1D730, + 0x1D731, + 0x1D732, + 0x1D733, + 0x1D734, + 0x1D735, + 0x1D736, + 0x1D737, + 0x1D738, + 0x1D739, + 0x1D73A, + 0x1D73B, + 0x1D73C, + 0x1D73D, + 0x1D73E, + 0x1D73F, + 0x1D740, + 0x1D741, + 0x1D742, + 0x1D743, + 0x1D744, + 0x1D745, + 0x1D746, + 0x1D747, + 0x1D749, + 0x1D74A, + 0x1D74B, + 0x1D74C, + 0x1D74D, + 0x1D74E, + 0x1D74F, + 0x1D750, + 0x1D751, + 0x1D752, + 0x1D753, + 0x1D754, + 0x1D755, + 0x1D756, + 0x1D757, + 0x1D758, + 0x1D759, + 0x1D75A, + 0x1D75B, + 0x1D75C, + 0x1D75D, + 0x1D75E, + 0x1D75F, + 0x1D760, + 0x1D761, + 0x1D762, + 0x1D763, + 0x1D764, + 0x1D765, + 0x1D766, + 0x1D767, + 0x1D768, + 0x1D769, + 0x1D76A, + 0x1D76B, + 0x1D76C, + 0x1D76D, + 0x1D76E, + 0x1D76F, + 0x1D770, + 0x1D771, + 0x1D772, + 0x1D773, + 0x1D774, + 0x1D775, + 0x1D776, + 0x1D777, + 0x1D778, + 0x1D779, + 0x1D77A, + 0x1D77B, + 0x1D77C, + 0x1D77D, + 0x1D77E, + 0x1D77F, + 0x1D780, + 0x1D781, + 0x1D783, + 0x1D784, + 0x1D785, + 0x1D786, + 0x1D787, + 0x1D788, + 0x1D789, + 0x1D78A, + 0x1D78B, + 0x1D78C, + 0x1D78D, + 0x1D78E, + 0x1D78F, + 0x1D790, + 0x1D791, + 0x1D792, + 0x1D793, + 0x1D794, + 0x1D795, + 0x1D796, + 0x1D797, + 0x1D798, + 0x1D799, + 0x1D79A, + 0x1D79B, + 0x1D79C, + 0x1D79D, + 0x1D79E, + 0x1D79F, + 0x1D7A0, + 0x1D7A1, + 0x1D7A2, + 0x1D7A3, + 0x1D7A4, + 0x1D7A5, + 0x1D7A6, + 0x1D7A7, + 0x1D7A8, + 0x1D7A9, + 0x1D7AA, + 0x1D7AB, + 0x1D7AC, + 0x1D7AD, + 0x1D7AE, + 0x1D7AF, + 0x1D7B0, + 0x1D7B1, + 0x1D7B2, + 0x1D7B3, + 0x1D7B4, + 0x1D7B5, + 0x1D7B6, + 0x1D7B7, + 0x1D7B8, + 0x1D7B9, + 0x1D7BA, + 0x1D7BB, + 0x1D7BD, + 0x1D7BE, + 0x1D7BF, + 0x1D7C0, + 0x1D7C1, + 0x1D7C2, + 0x1D7C3, + 0x1D7C4, + 0x1D7C5, + 0x1D7C6, + 0x1D7C7, + 0x1D7C8, + 0x1D7C9, + 0x1D7CA, + 0x1D7CC, + 0x1D7CE, + 0x1D7CF, + 0x1D7D0, + 0x1D7D1, + 0x1D7D2, + 0x1D7D3, + 0x1D7D4, + 0x1D7D5, + 0x1D7D6, + 0x1D7D7, + 0x1D7D8, + 0x1D7D9, + 0x1D7DA, + 0x1D7DB, + 0x1D7DC, + 0x1D7DD, + 0x1D7DE, + 0x1D7DF, + 0x1D7E0, + 0x1D7E1, + 0x1D7E2, + 0x1D7E3, + 0x1D7E4, + 0x1D7E5, + 0x1D7E6, + 0x1D7E7, + 0x1D7E8, + 0x1D7E9, + 0x1D7EA, + 0x1D7EB, + 0x1D7EC, + 0x1D7ED, + 0x1D7EE, + 0x1D7EF, + 0x1D7F0, + 0x1D7F1, + 0x1D7F2, + 0x1D7F3, + 0x1D7F4, + 0x1D7F5, + 0x1D7F6, + 0x1D7F7, + 0x1D7F8, + 0x1D7F9, + 0x1D7FA, + 0x1D7FB, + 0x1D7FC, + 0x1D7FD, + 0x1D7FE, + 0x1D7FF, + 0x1D800, + 0x1DA8C, + 0x1DA9B, + 0x1DAA0, + 0x1DAA1, + 0x1DAB0, + 0x1DF00, + 0x1DF1F, + 0x1DF25, + 0x1DF2B, + 0x1E000, + 0x1E007, + 0x1E008, + 0x1E019, + 0x1E01B, + 0x1E022, + 0x1E023, + 0x1E025, + 0x1E026, + 0x1E02B, + 0x1E030, + 0x1E031, + 0x1E032, + 0x1E033, + 0x1E034, + 0x1E035, + 0x1E036, + 0x1E037, + 0x1E038, + 0x1E039, + 0x1E03A, + 0x1E03B, + 0x1E03C, + 0x1E03D, + 0x1E03E, + 0x1E03F, + 0x1E040, + 0x1E041, + 0x1E042, + 0x1E043, + 0x1E044, + 0x1E045, + 0x1E046, + 0x1E047, + 0x1E048, + 0x1E049, + 0x1E04A, + 0x1E04B, + 0x1E04C, + 0x1E04D, + 0x1E04E, + 0x1E04F, + 0x1E050, + 0x1E051, + 0x1E052, + 0x1E053, + 0x1E054, + 0x1E055, + 0x1E056, + 0x1E057, + 0x1E058, + 0x1E059, + 0x1E05A, + 0x1E05B, + 0x1E05C, + 0x1E05D, + 0x1E05E, + 0x1E05F, + 0x1E060, + 0x1E061, + 0x1E062, + 0x1E063, + 0x1E064, + 0x1E065, + 0x1E066, + 0x1E067, + 0x1E068, + 0x1E069, + 0x1E06A, + 0x1E06B, + 0x1E06C, + 0x1E06D, + 0x1E06E, + 0x1E08F, + 0x1E090, + 0x1E100, + 0x1E12D, + 0x1E130, + 0x1E13E, + 0x1E140, + 0x1E14A, + 0x1E14E, + 0x1E150, + 0x1E290, + 0x1E2AF, + 0x1E2C0, + 0x1E2FA, + 0x1E2FF, + 0x1E300, + 0x1E4D0, + 0x1E4FA, + 0x1E5D0, + 0x1E5FB, + 0x1E5FF, + 0x1E600, + 0x1E6C0, + 0x1E6DF, + 0x1E6E0, + 0x1E6F6, + 0x1E6FE, + 0x1E700, + 0x1E7E0, + 0x1E7E7, + 0x1E7E8, + 0x1E7EC, + 0x1E7ED, + 0x1E7EF, + 0x1E7F0, + 0x1E7FF, + 0x1E800, + 0x1E8C5, + 0x1E8C7, + 0x1E8D7, + 0x1E900, + 0x1E901, + 0x1E902, + 0x1E903, + 0x1E904, + 0x1E905, + 0x1E906, + 0x1E907, + 0x1E908, + 0x1E909, + 0x1E90A, + 0x1E90B, + 0x1E90C, + 0x1E90D, + 0x1E90E, + 0x1E90F, + 0x1E910, + 0x1E911, + 0x1E912, + 0x1E913, + 0x1E914, + 0x1E915, + 0x1E916, + 0x1E917, + 0x1E918, + 0x1E919, + 0x1E91A, + 0x1E91B, + 0x1E91C, + 0x1E91D, + 0x1E91E, + 0x1E91F, + 0x1E920, + 0x1E921, + 0x1E922, + 0x1E94C, + 0x1E950, + 0x1E95A, + 0x1E95E, + 0x1E960, + 0x1EC71, + 0x1ECB5, + 0x1ED01, + 0x1ED3E, + 0x1EE00, + 0x1EE01, + 0x1EE02, + 0x1EE03, + 0x1EE04, + 0x1EE05, + 0x1EE06, + 0x1EE07, + 0x1EE08, + 0x1EE09, + 0x1EE0A, + 0x1EE0B, + 0x1EE0C, + 0x1EE0D, + 0x1EE0E, + 0x1EE0F, + 0x1EE10, + 0x1EE11, + 0x1EE12, + 0x1EE13, + 0x1EE14, + 0x1EE15, + 0x1EE16, + 0x1EE17, + 0x1EE18, + 0x1EE19, + 0x1EE1A, + 0x1EE1B, + 0x1EE1C, + 0x1EE1D, + 0x1EE1E, + 0x1EE1F, + 0x1EE20, + 0x1EE21, + 0x1EE22, + 0x1EE23, + 0x1EE24, + 0x1EE25, + 0x1EE27, + 0x1EE28, + 0x1EE29, + 0x1EE2A, + 0x1EE2B, + 0x1EE2C, + 0x1EE2D, + 0x1EE2E, + 0x1EE2F, + 0x1EE30, + 0x1EE31, + 0x1EE32, + 0x1EE33, + 0x1EE34, + 0x1EE35, + 0x1EE36, + 0x1EE37, + 0x1EE38, + 0x1EE39, + 0x1EE3A, + 0x1EE3B, + 0x1EE3C, + 0x1EE42, + 0x1EE43, + 0x1EE47, + 0x1EE48, + 0x1EE49, + 0x1EE4A, + 0x1EE4B, + 0x1EE4C, + 0x1EE4D, + 0x1EE4E, + 0x1EE4F, + 0x1EE50, + 0x1EE51, + 0x1EE52, + 0x1EE53, + 0x1EE54, + 0x1EE55, + 0x1EE57, + 0x1EE58, + 0x1EE59, + 0x1EE5A, + 0x1EE5B, + 0x1EE5C, + 0x1EE5D, + 0x1EE5E, + 0x1EE5F, + 0x1EE60, + 0x1EE61, + 0x1EE62, + 0x1EE63, + 0x1EE64, + 0x1EE65, + 0x1EE67, + 0x1EE68, + 0x1EE69, + 0x1EE6A, + 0x1EE6B, + 0x1EE6C, + 0x1EE6D, + 0x1EE6E, + 0x1EE6F, + 0x1EE70, + 0x1EE71, + 0x1EE72, + 0x1EE73, + 0x1EE74, + 0x1EE75, + 0x1EE76, + 0x1EE77, + 0x1EE78, + 0x1EE79, + 0x1EE7A, + 0x1EE7B, + 0x1EE7C, + 0x1EE7D, + 0x1EE7E, + 0x1EE7F, + 0x1EE80, + 0x1EE81, + 0x1EE82, + 0x1EE83, + 0x1EE84, + 0x1EE85, + 0x1EE86, + 0x1EE87, + 0x1EE88, + 0x1EE89, + 0x1EE8A, + 0x1EE8B, + 0x1EE8C, + 0x1EE8D, + 0x1EE8E, + 0x1EE8F, + 0x1EE90, + 0x1EE91, + 0x1EE92, + 0x1EE93, + 0x1EE94, + 0x1EE95, + 0x1EE96, + 0x1EE97, + 0x1EE98, + 0x1EE99, + 0x1EE9A, + 0x1EE9B, + 0x1EE9C, + 0x1EEA1, + 0x1EEA2, + 0x1EEA3, + 0x1EEA4, + 0x1EEA5, + 0x1EEA6, + 0x1EEA7, + 0x1EEA8, + 0x1EEA9, + 0x1EEAA, + 0x1EEAB, + 0x1EEAC, + 0x1EEAD, + 0x1EEAE, + 0x1EEAF, + 0x1EEB0, + 0x1EEB1, + 0x1EEB2, + 0x1EEB3, + 0x1EEB4, + 0x1EEB5, + 0x1EEB6, + 0x1EEB7, + 0x1EEB8, + 0x1EEB9, + 0x1EEBA, + 0x1EEBB, + 0x1EEBC, + 0x1EEF0, + 0x1EEF2, + 0x1F000, + 0x1F02C, + 0x1F030, + 0x1F094, + 0x1F0A0, + 0x1F0AF, + 0x1F0B1, + 0x1F0C0, + 0x1F0C1, + 0x1F0D0, + 0x1F0D1, + 0x1F0F6, + 0x1F101, + 0x1F102, + 0x1F103, + 0x1F104, + 0x1F105, + 0x1F106, + 0x1F107, + 0x1F108, + 0x1F109, + 0x1F10A, + 0x1F10B, + 0x1F110, + 0x1F111, + 0x1F112, + 0x1F113, + 0x1F114, + 0x1F115, + 0x1F116, + 0x1F117, + 0x1F118, + 0x1F119, + 0x1F11A, + 0x1F11B, + 0x1F11C, + 0x1F11D, + 0x1F11E, + 0x1F11F, + 0x1F120, + 0x1F121, + 0x1F122, + 0x1F123, + 0x1F124, + 0x1F125, + 0x1F126, + 0x1F127, + 0x1F128, + 0x1F129, + 0x1F12A, + 0x1F12B, + 0x1F12C, + 0x1F12D, + 0x1F12E, + 0x1F12F, + 0x1F130, + 0x1F131, + 0x1F132, + 0x1F133, + 0x1F134, + 0x1F135, + 0x1F136, + 0x1F137, + 0x1F138, + 0x1F139, + 0x1F13A, + 0x1F13B, + 0x1F13C, + 0x1F13D, + 0x1F13E, + 0x1F13F, + 0x1F140, + 0x1F141, + 0x1F142, + 0x1F143, + 0x1F144, + 0x1F145, + 0x1F146, + 0x1F147, + 0x1F148, + 0x1F149, + 0x1F14A, + 0x1F14B, + 0x1F14C, + 0x1F14D, + 0x1F14E, + 0x1F14F, + 0x1F150, + 0x1F16A, + 0x1F16B, + 0x1F16C, + 0x1F16D, + 0x1F190, + 0x1F191, + 0x1F1AE, + 0x1F1E6, + 0x1F200, + 0x1F201, + 0x1F202, + 0x1F203, + 0x1F210, + 0x1F211, + 0x1F212, + 0x1F213, + 0x1F214, + 0x1F215, + 0x1F216, + 0x1F217, + 0x1F218, + 0x1F219, + 0x1F21A, + 0x1F21B, + 0x1F21C, + 0x1F21D, + 0x1F21E, + 0x1F21F, + 0x1F220, + 0x1F221, + 0x1F222, + 0x1F223, + 0x1F224, + 0x1F225, + 0x1F226, + 0x1F227, + 0x1F228, + 0x1F229, + 0x1F22A, + 0x1F22B, + 0x1F22C, + 0x1F22D, + 0x1F22E, + 0x1F22F, + 0x1F230, + 0x1F231, + 0x1F232, + 0x1F233, + 0x1F234, + 0x1F235, + 0x1F236, + 0x1F237, + 0x1F238, + 0x1F239, + 0x1F23A, + 0x1F23B, + 0x1F23C, + 0x1F240, + 0x1F241, + 0x1F242, + 0x1F243, + 0x1F244, + 0x1F245, + 0x1F246, + 0x1F247, + 0x1F248, + 0x1F249, + 0x1F250, + 0x1F251, + 0x1F252, + 0x1F260, + 0x1F266, + 0x1F300, + 0x1F6D9, + 0x1F6DC, + 0x1F6ED, + 0x1F6F0, + 0x1F6FD, + 0x1F700, + 0x1F7DA, + 0x1F7E0, + 0x1F7EC, + 0x1F7F0, + 0x1F7F1, + 0x1F800, + 0x1F80C, + 0x1F810, + 0x1F848, + 0x1F850, + 0x1F85A, + 0x1F860, + 0x1F888, + 0x1F890, + 0x1F8AE, + 0x1F8B0, + 0x1F8BC, + 0x1F8C0, + 0x1F8C2, + 0x1F8D0, + 0x1F8D9, + 0x1F900, + 0x1FA58, + 0x1FA60, + 0x1FA6E, + 0x1FA70, + 0x1FA7D, + 0x1FA80, + 0x1FA8B, + 0x1FA8E, + 0x1FAC7, + 0x1FAC8, + 0x1FAC9, + 0x1FACD, + 0x1FADD, + 0x1FADF, + 0x1FAEB, + 0x1FAEF, + 0x1FAF9, + 0x1FB00, + 0x1FB93, + 0x1FB94, + 0x1FBF0, + 0x1FBF1, + 0x1FBF2, + 0x1FBF3, + 0x1FBF4, + 0x1FBF5, + 0x1FBF6, + 0x1FBF7, + 0x1FBF8, + 0x1FBF9, + 0x1FBFA, + 0x1FBFB, + 0x20000, + 0x2A6E0, + 0x2A700, + 0x2B81E, + 0x2B820, + 0x2CEAE, + 0x2CEB0, + 0x2EBE1, + 0x2EBF0, + 0x2EE5E, + 0x2F800, + 0x2F801, + 0x2F802, + 0x2F803, + 0x2F804, + 0x2F805, + 0x2F806, + 0x2F807, + 0x2F808, + 0x2F809, + 0x2F80A, + 0x2F80B, + 0x2F80C, + 0x2F80D, + 0x2F80E, + 0x2F80F, + 0x2F810, + 0x2F811, + 0x2F812, + 0x2F813, + 0x2F814, + 0x2F815, + 0x2F816, + 0x2F817, + 0x2F818, + 0x2F819, + 0x2F81A, + 0x2F81B, + 0x2F81C, + 0x2F81D, + 0x2F81E, + 0x2F81F, + 0x2F820, + 0x2F821, + 0x2F822, + 0x2F823, + 0x2F824, + 0x2F825, + 0x2F826, + 0x2F827, + 0x2F828, + 0x2F829, + 0x2F82A, + 0x2F82B, + 0x2F82C, + 0x2F82D, + 0x2F82E, + 0x2F82F, + 0x2F830, + 0x2F831, + 0x2F834, + 0x2F835, + 0x2F836, + 0x2F837, + 0x2F838, + 0x2F839, + 0x2F83A, + 0x2F83B, + 0x2F83C, + 0x2F83D, + 0x2F83E, + 0x2F83F, + 0x2F840, + 0x2F841, + 0x2F842, + 0x2F843, + 0x2F844, + 0x2F845, + 0x2F847, + 0x2F848, + 0x2F849, + 0x2F84A, + 0x2F84B, + 0x2F84C, + 0x2F84D, + 0x2F84E, + 0x2F84F, + 0x2F850, + 0x2F851, + 0x2F852, + 0x2F853, + 0x2F854, + 0x2F855, + 0x2F856, + 0x2F857, + 0x2F858, + 0x2F859, + 0x2F85A, + 0x2F85B, + 0x2F85C, + 0x2F85D, + 0x2F85E, + 0x2F85F, + 0x2F860, + 0x2F861, + 0x2F862, + 0x2F863, + 0x2F864, + 0x2F865, + 0x2F866, + 0x2F867, + 0x2F868, + 0x2F869, + 0x2F86A, + 0x2F86C, + 0x2F86D, + 0x2F86E, + 0x2F86F, + 0x2F870, + 0x2F871, + 0x2F872, + 0x2F873, + 0x2F874, + 0x2F875, + 0x2F876, + 0x2F877, + 0x2F878, + 0x2F879, + 0x2F87A, + 0x2F87B, + 0x2F87C, + 0x2F87D, + 0x2F87E, + 0x2F87F, + 0x2F880, + 0x2F881, + 0x2F882, + 0x2F883, + 0x2F884, + 0x2F885, + 0x2F886, + 0x2F887, + 0x2F888, + 0x2F889, + 0x2F88A, + 0x2F88B, + 0x2F88C, + 0x2F88D, + 0x2F88E, + 0x2F88F, + 0x2F890, + 0x2F891, + 0x2F893, + 0x2F894, + 0x2F896, + 0x2F897, + 0x2F898, + 0x2F899, + 0x2F89A, + 0x2F89B, + 0x2F89C, + 0x2F89D, + 0x2F89E, + 0x2F89F, + 0x2F8A0, + 0x2F8A1, + 0x2F8A2, + 0x2F8A3, + 0x2F8A4, + 0x2F8A5, + 0x2F8A6, + 0x2F8A7, + 0x2F8A8, + 0x2F8A9, + 0x2F8AA, + 0x2F8AB, + 0x2F8AC, + 0x2F8AD, + 0x2F8AE, + 0x2F8AF, + 0x2F8B0, + 0x2F8B1, + 0x2F8B2, + 0x2F8B3, + 0x2F8B4, + 0x2F8B5, + 0x2F8B6, + 0x2F8B7, + 0x2F8B8, + 0x2F8B9, + 0x2F8BA, + 0x2F8BB, + 0x2F8BC, + 0x2F8BD, + 0x2F8BE, + 0x2F8BF, + 0x2F8C0, + 0x2F8C1, + 0x2F8C2, + 0x2F8C3, + 0x2F8C4, + 0x2F8C5, + 0x2F8C6, + 0x2F8C7, + 0x2F8C8, + 0x2F8C9, + 0x2F8CA, + 0x2F8CB, + 0x2F8CC, + 0x2F8CD, + 0x2F8CE, + 0x2F8CF, + 0x2F8D0, + 0x2F8D1, + 0x2F8D2, + 0x2F8D3, + 0x2F8D4, + 0x2F8D5, + 0x2F8D6, + 0x2F8D7, + 0x2F8D8, + 0x2F8D9, + 0x2F8DA, + 0x2F8DB, + 0x2F8DC, + 0x2F8DD, + 0x2F8DE, + 0x2F8DF, + 0x2F8E0, + 0x2F8E1, + 0x2F8E2, + 0x2F8E3, + 0x2F8E4, + 0x2F8E5, + 0x2F8E6, + 0x2F8E7, + 0x2F8E8, + 0x2F8E9, + 0x2F8EA, + 0x2F8EB, + 0x2F8EC, + 0x2F8ED, + 0x2F8EE, + 0x2F8EF, + 0x2F8F0, + 0x2F8F1, + 0x2F8F2, + 0x2F8F3, + 0x2F8F4, + 0x2F8F5, + 0x2F8F6, + 0x2F8F7, + 0x2F8F8, + 0x2F8F9, + 0x2F8FA, + 0x2F8FB, + 0x2F8FC, + 0x2F8FD, + 0x2F8FE, + 0x2F8FF, + 0x2F900, + 0x2F901, + 0x2F902, + 0x2F903, + 0x2F904, + 0x2F905, + 0x2F906, + 0x2F907, + 0x2F908, + 0x2F909, + 0x2F90A, + 0x2F90B, + 0x2F90C, + 0x2F90D, + 0x2F90E, + 0x2F90F, + 0x2F910, + 0x2F911, + 0x2F912, + 0x2F913, + 0x2F914, + 0x2F915, + 0x2F916, + 0x2F917, + 0x2F918, + 0x2F919, + 0x2F91A, + 0x2F91B, + 0x2F91C, + 0x2F91D, + 0x2F91E, + 0x2F91F, + 0x2F920, + 0x2F921, + 0x2F922, + 0x2F923, + 0x2F924, + 0x2F925, + 0x2F926, + 0x2F927, + 0x2F928, + 0x2F929, + 0x2F92A, + 0x2F92B, + 0x2F92C, + 0x2F92E, + 0x2F92F, + 0x2F930, + 0x2F931, + 0x2F932, + 0x2F933, + 0x2F934, + 0x2F935, + 0x2F936, + 0x2F937, + 0x2F938, + 0x2F939, + 0x2F93A, + 0x2F93B, + 0x2F93C, + 0x2F93D, + 0x2F93E, + 0x2F93F, + 0x2F940, + 0x2F941, + 0x2F942, + 0x2F943, + 0x2F944, + 0x2F945, + 0x2F946, + 0x2F948, + 0x2F949, + 0x2F94A, + 0x2F94B, + 0x2F94C, + 0x2F94D, + 0x2F94E, + 0x2F94F, + 0x2F950, + 0x2F951, + 0x2F952, + 0x2F953, + 0x2F954, + 0x2F955, + 0x2F956, + 0x2F957, + 0x2F958, + 0x2F959, + 0x2F95A, + 0x2F95B, + 0x2F95C, + 0x2F95D, + 0x2F95F, + 0x2F960, + 0x2F961, + 0x2F962, + 0x2F963, + 0x2F964, + 0x2F965, + 0x2F966, + 0x2F967, + 0x2F968, + 0x2F969, + 0x2F96A, + 0x2F96B, + 0x2F96C, + 0x2F96D, + 0x2F96E, + 0x2F96F, + 0x2F970, + 0x2F971, + 0x2F972, + 0x2F973, + 0x2F974, + 0x2F975, + 0x2F976, + 0x2F977, + 0x2F978, + 0x2F979, + 0x2F97A, + 0x2F97B, + 0x2F97C, + 0x2F97D, + 0x2F97E, + 0x2F97F, + 0x2F980, + 0x2F981, + 0x2F982, + 0x2F983, + 0x2F984, + 0x2F985, + 0x2F986, + 0x2F987, + 0x2F988, + 0x2F989, + 0x2F98A, + 0x2F98B, + 0x2F98C, + 0x2F98D, + 0x2F98E, + 0x2F98F, + 0x2F990, + 0x2F991, + 0x2F992, + 0x2F993, + 0x2F994, + 0x2F995, + 0x2F996, + 0x2F997, + 0x2F998, + 0x2F999, + 0x2F99A, + 0x2F99B, + 0x2F99C, + 0x2F99D, + 0x2F99E, + 0x2F99F, + 0x2F9A0, + 0x2F9A1, + 0x2F9A2, + 0x2F9A3, + 0x2F9A4, + 0x2F9A5, + 0x2F9A6, + 0x2F9A7, + 0x2F9A8, + 0x2F9A9, + 0x2F9AA, + 0x2F9AB, + 0x2F9AC, + 0x2F9AD, + 0x2F9AE, + 0x2F9AF, + 0x2F9B0, + 0x2F9B1, + 0x2F9B2, + 0x2F9B3, + 0x2F9B4, + 0x2F9B5, + 0x2F9B6, + 0x2F9B7, + 0x2F9B8, + 0x2F9B9, + 0x2F9BA, + 0x2F9BB, + 0x2F9BC, + 0x2F9BD, + 0x2F9BE, + 0x2F9BF, + 0x2F9C0, + 0x2F9C1, + 0x2F9C2, + 0x2F9C3, + 0x2F9C4, + 0x2F9C5, + 0x2F9C6, + 0x2F9C7, + 0x2F9C8, + 0x2F9C9, + 0x2F9CA, + 0x2F9CB, + 0x2F9CC, + 0x2F9CD, + 0x2F9CE, + 0x2F9CF, + 0x2F9D0, + 0x2F9D1, + 0x2F9D2, + 0x2F9D3, + 0x2F9D4, + 0x2F9D5, + 0x2F9D6, + 0x2F9D7, + 0x2F9D8, + 0x2F9D9, + 0x2F9DA, + 0x2F9DB, + 0x2F9DC, + 0x2F9DD, + 0x2F9DE, + 0x2F9DF, + 0x2F9E0, + 0x2F9E1, + 0x2F9E2, + 0x2F9E3, + 0x2F9E4, + 0x2F9E5, + 0x2F9E6, + 0x2F9E7, + 0x2F9E8, + 0x2F9E9, + 0x2F9EA, + 0x2F9EB, + 0x2F9EC, + 0x2F9ED, + 0x2F9EE, + 0x2F9EF, + 0x2F9F0, + 0x2F9F1, + 0x2F9F2, + 0x2F9F3, + 0x2F9F4, + 0x2F9F5, + 0x2F9F6, + 0x2F9F7, + 0x2F9F8, + 0x2F9F9, + 0x2F9FA, + 0x2F9FB, + 0x2F9FC, + 0x2F9FD, + 0x2F9FE, + 0x2FA00, + 0x2FA01, + 0x2FA02, + 0x2FA03, + 0x2FA04, + 0x2FA05, + 0x2FA06, + 0x2FA07, + 0x2FA08, + 0x2FA09, + 0x2FA0A, + 0x2FA0B, + 0x2FA0C, + 0x2FA0D, + 0x2FA0E, + 0x2FA0F, + 0x2FA10, + 0x2FA11, + 0x2FA12, + 0x2FA13, + 0x2FA14, + 0x2FA15, + 0x2FA16, + 0x2FA17, + 0x2FA18, + 0x2FA19, + 0x2FA1A, + 0x2FA1B, + 0x2FA1C, + 0x2FA1D, + 0x2FA1E, + 0x30000, + 0x3134B, + 0x31350, + 0x3347A, + 0xE0100, + 0xE01F0, + ), +) + +uts46_statuses: bytes = ( + b"VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" + b"VMMMMMMMMMMMMMMMMMMMMMMMMMMVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" + b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXMVVVVVVVMVMVVIVMVVMMMMVVMMMVMMMV" + b"MMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMDVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVMVMVMVMMV" + b"MVMVMVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVMVMVMM" + b"VMVMMVMMMVMMMMVMMVMMMVMMVMMVMVMVMMVMVMVMMVMMMVMVMMVMVMMMMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMMVMMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMMVMVMMMMVMVMVMVMVMMMMMMMMMVMMMMMM" + b"VMMMMMVMMVMMMVIVMVMVMVMVXMVMMXMMMMMMMXMXMMVMMMMMMMMMMMMMMMMMXMMM" + b"MMMMMMVDVMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMVMVMMMVMMVMVMMVMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVXMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMXVMVXVXVXVXVXVXVMMMMVXVXVXVXVXVXVXVXVXVX" + b"VXVXVMMMMMMMMVXVXVXVXVXVXVXVXVXVXVXMMXMVXVXVXVXVXVXVXVMXVMXVXVXV" + b"XVXVXVXMMMVXMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" + b"XVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" + b"XVMVXVXVXVXVXVXVXVMVXVXVXVXVXMMVXVMVMVXVMVMVMVMVMVXVMVMMMMMVMVMV" + b"XVMVMVMVMVMVXVXVXVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMXMXVMV" + b"IVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMMMMMMXVXVXVXVXVXVXVXVXVX" + b"VIVXVXVXVIVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVMMM" + b"MMMMMMVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMVXVXVMMMV" + b"MMMMMMMMMMMVMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMMMMMMMVXMMMMMMXVMMMMMMMMVMMMMMM" + b"MMVXMMMMMMXVXMXMXMXMVMMMMMMMMVMVMVMVMVMVMVMXMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMXVMMMMMMMMMMMMMMXVMMMMMMMMMVMXVM" + b"MMMXMMMVMVMMMMMMMMXMMMXVMMMMMMMMXMIDXVMVMVXVXMVMMVMMVMVMVMMMVMVM" + b"IXIMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMXVMVXVXMMMMVMMMVM" + b"MMMMMVMMVMMMVMMMVMVMVMVMMMMVMMMMMMMMMMVMMMMMVMMMMVMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVXVMMVMMVMMVXVXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMMMVMVXVMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMMMVMVMVMVMMMMVMVMVMM" + b"MMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVXVXVXVXVXMVXVXVXVXVX" + b"VXVXVXVXVXVXVXVMVMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMVMVMVMMMVXVXVMMVMVMXV" + b"XMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMIMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVMMMMMMMMMMMMMMVXVMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMMMMMMMMM" + b"MMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVXV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMMMMVM" + b"MMMMVMVMVMVMVMVMVMVMMMMVMVMMVMVMVMVMVMVMVMVMXMMMMMVMMVXVXVXVXVXV" + b"XVXVXVXVXVXVXVXVXVXVXVXVXVXVXVMMMMVMVXMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVX" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMVMVMVMMMMMMMMMMVMVMVMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMXMMMMMMXMMMMMXMVMMMMMMMMMMMMMMMMMMMMMMMMXMMMMM" + b"XMXMMXMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXMMMMMMMMMMMMMVIMMXMMM" + b"MMMXVXMMMMMMMMMMMMMMMMMMMVMMMMMMXMMMMMMMMMMMMMMMMMMMXMMMMXMMMVMX" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXIXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMIMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMXMMMMM" + b"MXMMMMMMXMMMXMMMMMMMXMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" + b"XVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVMMMMMMMMMMMXMMMMMMMMMMMMMMMXMMM" + b"MMMMXMMXVXVXVXVXVXVXVXVXVMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMXMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMXVXVXVXVMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMVXMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVIXVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVX" + b"VXVXVXVXVXVXVMMMMMMMVIVMMMMMMVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMXMMXMXMMXMMMMXMMMMMMMMMMMMXMXMMMMMMMXMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMMMMXMMMMM" + b"MMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMXMXMMMMMMMXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVX" + b"VXMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMXMXMXMMMMMMMMMMXMMMMXMXMXMX" + b"MXMXMXMMMXMMXMXMXMXMXMXMXMMXMXMMMMXMMMMMMMXMMMMXMMMMXMXMMMMMMMMM" + b"MXMMMMMMMMMMMMMMMMMXMMMXMMMMMXMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXMM" + b"MMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMVMMMVMVXVMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMXMMMMMMMMMXMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVMMMMMMMMMMVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXIX" +) + +uts46_replacements: tuple[Optional[str], ...] = ( + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + " ", + None, + None, + None, + None, + None, + None, + None, + " ̈", + None, + "a", + None, + None, + None, + None, + " ̄", + None, + None, + "2", + "3", + " ́", + "μ", + None, + None, + " ̧", + "1", + "o", + None, + "1⁄4", + "1⁄2", + "3⁄4", + None, + "à", + "á", + "â", + "ã", + "ä", + "å", + "æ", + "ç", + "è", + "é", + "ê", + "ë", + "ì", + "í", + "î", + "ï", + "ð", + "ñ", + "ò", + "ó", + "ô", + "õ", + "ö", + None, + "ø", + "ù", + "ú", + "û", + "ü", + "ý", + "þ", + "ss", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ā", + None, + "ă", + None, + "ą", + None, + "ć", + None, + "ĉ", + None, + "ċ", + None, + "č", + None, + "ď", + None, + "đ", + None, + "ē", + None, + "ĕ", + None, + "ė", + None, + "ę", + None, + "ě", + None, + "ĝ", + None, + "ğ", + None, + "ġ", + None, + "ģ", + None, + "ĥ", + None, + "ħ", + None, + "ĩ", + None, + "ī", + None, + "ĭ", + None, + "į", + None, + "i̇", + None, + "ij", + "ĵ", + None, + "ķ", + None, + "ĺ", + None, + "ļ", + None, + "ľ", + None, + "l·", + "ł", + None, + "ń", + None, + "ņ", + None, + "ň", + None, + "ʼn", + "ŋ", + None, + "ō", + None, + "ŏ", + None, + "ő", + None, + "œ", + None, + "ŕ", + None, + "ŗ", + None, + "ř", + None, + "ś", + None, + "ŝ", + None, + "ş", + None, + "š", + None, + "ţ", + None, + "ť", + None, + "ŧ", + None, + "ũ", + None, + "ū", + None, + "ŭ", + None, + "ů", + None, + "ű", + None, + "ų", + None, + "ŵ", + None, + "ŷ", + None, + "ÿ", + "ź", + None, + "ż", + None, + "ž", + None, + "s", + None, + "ɓ", + "ƃ", + None, + "ƅ", + None, + "ɔ", + "ƈ", + None, + "ɖ", + "ɗ", + "ƌ", + None, + "ǝ", + "ə", + "ɛ", + "ƒ", + None, + "ɠ", + "ɣ", + None, + "ɩ", + "ɨ", + "ƙ", + None, + "ɯ", + "ɲ", + None, + "ɵ", + "ơ", + None, + "ƣ", + None, + "ƥ", + None, + "ʀ", + "ƨ", + None, + "ʃ", + None, + "ƭ", + None, + "ʈ", + "ư", + None, + "ʊ", + "ʋ", + "ƴ", + None, + "ƶ", + None, + "ʒ", + "ƹ", + None, + "ƽ", + None, + "dž", + "lj", + "nj", + "ǎ", + None, + "ǐ", + None, + "ǒ", + None, + "ǔ", + None, + "ǖ", + None, + "ǘ", + None, + "ǚ", + None, + "ǜ", + None, + "ǟ", + None, + "ǡ", + None, + "ǣ", + None, + "ǥ", + None, + "ǧ", + None, + "ǩ", + None, + "ǫ", + None, + "ǭ", + None, + "ǯ", + None, + "dz", + "ǵ", + None, + "ƕ", + "ƿ", + "ǹ", + None, + "ǻ", + None, + "ǽ", + None, + "ǿ", + None, + "ȁ", + None, + "ȃ", + None, + "ȅ", + None, + "ȇ", + None, + "ȉ", + None, + "ȋ", + None, + "ȍ", + None, + "ȏ", + None, + "ȑ", + None, + "ȓ", + None, + "ȕ", + None, + "ȗ", + None, + "ș", + None, + "ț", + None, + "ȝ", + None, + "ȟ", + None, + "ƞ", + None, + "ȣ", + None, + "ȥ", + None, + "ȧ", + None, + "ȩ", + None, + "ȫ", + None, + "ȭ", + None, + "ȯ", + None, + "ȱ", + None, + "ȳ", + None, + "ⱥ", + "ȼ", + None, + "ƚ", + "ⱦ", + None, + "ɂ", + None, + "ƀ", + "ʉ", + "ʌ", + "ɇ", + None, + "ɉ", + None, + "ɋ", + None, + "ɍ", + None, + "ɏ", + None, + "h", + "ɦ", + "j", + "r", + "ɹ", + "ɻ", + "ʁ", + "w", + "y", + None, + " ̆", + " ̇", + " ̊", + " ̨", + " ̃", + " ̋", + None, + "ɣ", + "l", + "s", + "x", + "ʕ", + None, + "̀", + "́", + None, + "̓", + "̈́", + "ι", + None, + None, + None, + "ͱ", + None, + "ͳ", + None, + "ʹ", + None, + "ͷ", + None, + None, + " ι", + None, + ";", + "ϳ", + None, + " ́", + " ̈́", + "ά", + "·", + "έ", + "ή", + "ί", + None, + "ό", + None, + "ύ", + "ώ", + None, + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + None, + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "ϊ", + "ϋ", + None, + "σ", + None, + "ϗ", + "β", + "θ", + "υ", + "ύ", + "ϋ", + "φ", + "π", + None, + "ϙ", + None, + "ϛ", + None, + "ϝ", + None, + "ϟ", + None, + "ϡ", + None, + "ϣ", + None, + "ϥ", + None, + "ϧ", + None, + "ϩ", + None, + "ϫ", + None, + "ϭ", + None, + "ϯ", + None, + "κ", + "ρ", + "σ", + None, + "θ", + "ε", + None, + "ϸ", + None, + "σ", + "ϻ", + None, + "ͻ", + "ͼ", + "ͽ", + "ѐ", + "ё", + "ђ", + "ѓ", + "є", + "ѕ", + "і", + "ї", + "ј", + "љ", + "њ", + "ћ", + "ќ", + "ѝ", + "ў", + "џ", + "а", + "б", + "в", + "г", + "д", + "е", + "ж", + "з", + "и", + "й", + "к", + "л", + "м", + "н", + "о", + "п", + "р", + "с", + "т", + "у", + "ф", + "х", + "ц", + "ч", + "ш", + "щ", + "ъ", + "ы", + "ь", + "э", + "ю", + "я", + None, + "ѡ", + None, + "ѣ", + None, + "ѥ", + None, + "ѧ", + None, + "ѩ", + None, + "ѫ", + None, + "ѭ", + None, + "ѯ", + None, + "ѱ", + None, + "ѳ", + None, + "ѵ", + None, + "ѷ", + None, + "ѹ", + None, + "ѻ", + None, + "ѽ", + None, + "ѿ", + None, + "ҁ", + None, + "ҋ", + None, + "ҍ", + None, + "ҏ", + None, + "ґ", + None, + "ғ", + None, + "ҕ", + None, + "җ", + None, + "ҙ", + None, + "қ", + None, + "ҝ", + None, + "ҟ", + None, + "ҡ", + None, + "ң", + None, + "ҥ", + None, + "ҧ", + None, + "ҩ", + None, + "ҫ", + None, + "ҭ", + None, + "ү", + None, + "ұ", + None, + "ҳ", + None, + "ҵ", + None, + "ҷ", + None, + "ҹ", + None, + "һ", + None, + "ҽ", + None, + "ҿ", + None, + "ӏ", + "ӂ", + None, + "ӄ", + None, + "ӆ", + None, + "ӈ", + None, + "ӊ", + None, + "ӌ", + None, + "ӎ", + None, + "ӑ", + None, + "ӓ", + None, + "ӕ", + None, + "ӗ", + None, + "ә", + None, + "ӛ", + None, + "ӝ", + None, + "ӟ", + None, + "ӡ", + None, + "ӣ", + None, + "ӥ", + None, + "ӧ", + None, + "ө", + None, + "ӫ", + None, + "ӭ", + None, + "ӯ", + None, + "ӱ", + None, + "ӳ", + None, + "ӵ", + None, + "ӷ", + None, + "ӹ", + None, + "ӻ", + None, + "ӽ", + None, + "ӿ", + None, + "ԁ", + None, + "ԃ", + None, + "ԅ", + None, + "ԇ", + None, + "ԉ", + None, + "ԋ", + None, + "ԍ", + None, + "ԏ", + None, + "ԑ", + None, + "ԓ", + None, + "ԕ", + None, + "ԗ", + None, + "ԙ", + None, + "ԛ", + None, + "ԝ", + None, + "ԟ", + None, + "ԡ", + None, + "ԣ", + None, + "ԥ", + None, + "ԧ", + None, + "ԩ", + None, + "ԫ", + None, + "ԭ", + None, + "ԯ", + None, + None, + "ա", + "բ", + "գ", + "դ", + "ե", + "զ", + "է", + "ը", + "թ", + "ժ", + "ի", + "լ", + "խ", + "ծ", + "կ", + "հ", + "ձ", + "ղ", + "ճ", + "մ", + "յ", + "ն", + "շ", + "ո", + "չ", + "պ", + "ջ", + "ռ", + "ս", + "վ", + "տ", + "ր", + "ց", + "ւ", + "փ", + "ք", + "օ", + "ֆ", + None, + None, + "եւ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "اٴ", + "وٴ", + "ۇٴ", + "يٴ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "क़", + "ख़", + "ग़", + "ज़", + "ड़", + "ढ़", + "फ़", + "य़", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ড়", + "ঢ়", + None, + "য়", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ਲ਼", + None, + None, + "ਸ਼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ਖ਼", + "ਗ਼", + "ਜ਼", + None, + None, + "ਫ਼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ଡ଼", + "ଢ଼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ํา", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ໍາ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ຫນ", + "ຫມ", + None, + None, + None, + "་", + None, + "གྷ", + None, + None, + None, + "ཌྷ", + None, + "དྷ", + None, + "བྷ", + None, + "ཛྷ", + None, + "ཀྵ", + None, + None, + None, + "ཱི", + None, + "ཱུ", + "ྲྀ", + "ྲཱྀ", + "ླྀ", + "ླཱྀ", + None, + "ཱྀ", + None, + "ྒྷ", + None, + None, + None, + "ྜྷ", + None, + "ྡྷ", + None, + "ྦྷ", + None, + "ྫྷ", + None, + "ྐྵ", + None, + None, + None, + None, + None, + None, + None, + "ⴀ", + "ⴁ", + "ⴂ", + "ⴃ", + "ⴄ", + "ⴅ", + "ⴆ", + "ⴇ", + "ⴈ", + "ⴉ", + "ⴊ", + "ⴋ", + "ⴌ", + "ⴍ", + "ⴎ", + "ⴏ", + "ⴐ", + "ⴑ", + "ⴒ", + "ⴓ", + "ⴔ", + "ⴕ", + "ⴖ", + "ⴗ", + "ⴘ", + "ⴙ", + "ⴚ", + "ⴛ", + "ⴜ", + "ⴝ", + "ⴞ", + "ⴟ", + "ⴠ", + "ⴡ", + "ⴢ", + "ⴣ", + "ⴤ", + "ⴥ", + None, + "ⴧ", + None, + "ⴭ", + None, + None, + "ნ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "Ᏸ", + "Ᏹ", + "Ᏺ", + "Ᏻ", + "Ᏼ", + "Ᏽ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "в", + "д", + "о", + "с", + "т", + "ъ", + "ѣ", + "ꙋ", + "\u1c8a", + None, + None, + "ა", + "ბ", + "გ", + "დ", + "ე", + "ვ", + "ზ", + "თ", + "ი", + "კ", + "ლ", + "მ", + "ნ", + "ო", + "პ", + "ჟ", + "რ", + "ს", + "ტ", + "უ", + "ფ", + "ქ", + "ღ", + "ყ", + "შ", + "ჩ", + "ც", + "ძ", + "წ", + "ჭ", + "ხ", + "ჯ", + "ჰ", + "ჱ", + "ჲ", + "ჳ", + "ჴ", + "ჵ", + "ჶ", + "ჷ", + "ჸ", + "ჹ", + "ჺ", + None, + "ჽ", + "ჾ", + "ჿ", + None, + None, + None, + None, + None, + "a", + "æ", + "b", + None, + "d", + "e", + "ǝ", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + None, + "o", + "ȣ", + "p", + "r", + "t", + "u", + "w", + "a", + "ɐ", + "ɑ", + "ᴂ", + "b", + "d", + "e", + "ə", + "ɛ", + "ɜ", + "g", + None, + "k", + "m", + "ŋ", + "o", + "ɔ", + "ᴖ", + "ᴗ", + "p", + "t", + "u", + "ᴝ", + "ɯ", + "v", + "ᴥ", + "β", + "γ", + "δ", + "φ", + "χ", + "i", + "r", + "u", + "v", + "β", + "γ", + "ρ", + "φ", + "χ", + None, + "н", + None, + "ɒ", + "c", + "ɕ", + "ð", + "ɜ", + "f", + "ɟ", + "ɡ", + "ɥ", + "ɨ", + "ɩ", + "ɪ", + "ᵻ", + "ʝ", + "ɭ", + "ᶅ", + "ʟ", + "ɱ", + "ɰ", + "ɲ", + "ɳ", + "ɴ", + "ɵ", + "ɸ", + "ʂ", + "ʃ", + "ƫ", + "ʉ", + "ʊ", + "ᴜ", + "ʋ", + "ʌ", + "z", + "ʐ", + "ʑ", + "ʒ", + "θ", + None, + "ḁ", + None, + "ḃ", + None, + "ḅ", + None, + "ḇ", + None, + "ḉ", + None, + "ḋ", + None, + "ḍ", + None, + "ḏ", + None, + "ḑ", + None, + "ḓ", + None, + "ḕ", + None, + "ḗ", + None, + "ḙ", + None, + "ḛ", + None, + "ḝ", + None, + "ḟ", + None, + "ḡ", + None, + "ḣ", + None, + "ḥ", + None, + "ḧ", + None, + "ḩ", + None, + "ḫ", + None, + "ḭ", + None, + "ḯ", + None, + "ḱ", + None, + "ḳ", + None, + "ḵ", + None, + "ḷ", + None, + "ḹ", + None, + "ḻ", + None, + "ḽ", + None, + "ḿ", + None, + "ṁ", + None, + "ṃ", + None, + "ṅ", + None, + "ṇ", + None, + "ṉ", + None, + "ṋ", + None, + "ṍ", + None, + "ṏ", + None, + "ṑ", + None, + "ṓ", + None, + "ṕ", + None, + "ṗ", + None, + "ṙ", + None, + "ṛ", + None, + "ṝ", + None, + "ṟ", + None, + "ṡ", + None, + "ṣ", + None, + "ṥ", + None, + "ṧ", + None, + "ṩ", + None, + "ṫ", + None, + "ṭ", + None, + "ṯ", + None, + "ṱ", + None, + "ṳ", + None, + "ṵ", + None, + "ṷ", + None, + "ṹ", + None, + "ṻ", + None, + "ṽ", + None, + "ṿ", + None, + "ẁ", + None, + "ẃ", + None, + "ẅ", + None, + "ẇ", + None, + "ẉ", + None, + "ẋ", + None, + "ẍ", + None, + "ẏ", + None, + "ẑ", + None, + "ẓ", + None, + "ẕ", + None, + "aʾ", + "ṡ", + None, + "ß", + None, + "ạ", + None, + "ả", + None, + "ấ", + None, + "ầ", + None, + "ẩ", + None, + "ẫ", + None, + "ậ", + None, + "ắ", + None, + "ằ", + None, + "ẳ", + None, + "ẵ", + None, + "ặ", + None, + "ẹ", + None, + "ẻ", + None, + "ẽ", + None, + "ế", + None, + "ề", + None, + "ể", + None, + "ễ", + None, + "ệ", + None, + "ỉ", + None, + "ị", + None, + "ọ", + None, + "ỏ", + None, + "ố", + None, + "ồ", + None, + "ổ", + None, + "ỗ", + None, + "ộ", + None, + "ớ", + None, + "ờ", + None, + "ở", + None, + "ỡ", + None, + "ợ", + None, + "ụ", + None, + "ủ", + None, + "ứ", + None, + "ừ", + None, + "ử", + None, + "ữ", + None, + "ự", + None, + "ỳ", + None, + "ỵ", + None, + "ỷ", + None, + "ỹ", + None, + "ỻ", + None, + "ỽ", + None, + "ỿ", + None, + "ἀ", + "ἁ", + "ἂ", + "ἃ", + "ἄ", + "ἅ", + "ἆ", + "ἇ", + None, + None, + "ἐ", + "ἑ", + "ἒ", + "ἓ", + "ἔ", + "ἕ", + None, + None, + "ἠ", + "ἡ", + "ἢ", + "ἣ", + "ἤ", + "ἥ", + "ἦ", + "ἧ", + None, + "ἰ", + "ἱ", + "ἲ", + "ἳ", + "ἴ", + "ἵ", + "ἶ", + "ἷ", + None, + None, + "ὀ", + "ὁ", + "ὂ", + "ὃ", + "ὄ", + "ὅ", + None, + None, + None, + "ὑ", + None, + "ὓ", + None, + "ὕ", + None, + "ὗ", + None, + "ὠ", + "ὡ", + "ὢ", + "ὣ", + "ὤ", + "ὥ", + "ὦ", + "ὧ", + None, + "ά", + None, + "έ", + None, + "ή", + None, + "ί", + None, + "ό", + None, + "ύ", + None, + "ώ", + None, + "ἀι", + "ἁι", + "ἂι", + "ἃι", + "ἄι", + "ἅι", + "ἆι", + "ἇι", + "ἀι", + "ἁι", + "ἂι", + "ἃι", + "ἄι", + "ἅι", + "ἆι", + "ἇι", + "ἠι", + "ἡι", + "ἢι", + "ἣι", + "ἤι", + "ἥι", + "ἦι", + "ἧι", + "ἠι", + "ἡι", + "ἢι", + "ἣι", + "ἤι", + "ἥι", + "ἦι", + "ἧι", + "ὠι", + "ὡι", + "ὢι", + "ὣι", + "ὤι", + "ὥι", + "ὦι", + "ὧι", + "ὠι", + "ὡι", + "ὢι", + "ὣι", + "ὤι", + "ὥι", + "ὦι", + "ὧι", + None, + "ὰι", + "αι", + "άι", + None, + None, + "ᾶι", + "ᾰ", + "ᾱ", + "ὰ", + "ά", + "αι", + " ̓", + "ι", + " ̓", + " ͂", + " ̈͂", + "ὴι", + "ηι", + "ήι", + None, + None, + "ῆι", + "ὲ", + "έ", + "ὴ", + "ή", + "ηι", + " ̓̀", + " ̓́", + " ̓͂", + None, + "ΐ", + None, + None, + "ῐ", + "ῑ", + "ὶ", + "ί", + None, + " ̔̀", + " ̔́", + " ̔͂", + None, + "ΰ", + None, + "ῠ", + "ῡ", + "ὺ", + "ύ", + "ῥ", + " ̈̀", + " ̈́", + "`", + None, + "ὼι", + "ωι", + "ώι", + None, + None, + "ῶι", + "ὸ", + "ό", + "ὼ", + "ώ", + "ωι", + " ́", + " ̔", + None, + " ", + None, + "", + None, + None, + "‐", + None, + " ̳", + None, + None, + None, + None, + " ", + None, + "′′", + "′′′", + None, + "‵‵", + "‵‵‵", + None, + "!!", + None, + " ̅", + None, + "??", + "?!", + "!?", + None, + "′′′′", + None, + " ", + None, + None, + None, + "0", + "i", + None, + "4", + "5", + "6", + "7", + "8", + "9", + "+", + "−", + "=", + "(", + ")", + "n", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "+", + "−", + "=", + "(", + ")", + None, + "a", + "e", + "o", + "x", + "ə", + "h", + "k", + "l", + "m", + "n", + "p", + "s", + "t", + None, + None, + "rs", + None, + None, + None, + None, + "a/c", + "a/s", + "c", + "°c", + None, + "c/o", + "c/u", + "ɛ", + None, + "°f", + "g", + "h", + "ħ", + "i", + "l", + None, + "n", + "no", + None, + "p", + "q", + "r", + None, + "sm", + "tel", + "tm", + None, + "z", + None, + "ω", + None, + "z", + None, + "k", + "å", + "b", + "c", + None, + "e", + "f", + "ⅎ", + "m", + "o", + "א", + "ב", + "ג", + "ד", + "i", + None, + "fax", + "π", + "γ", + "π", + "∑", + None, + "d", + "e", + "i", + "j", + None, + "1⁄7", + "1⁄9", + "1⁄10", + "1⁄3", + "2⁄3", + "1⁄5", + "2⁄5", + "3⁄5", + "4⁄5", + "1⁄6", + "5⁄6", + "1⁄8", + "3⁄8", + "5⁄8", + "7⁄8", + "1⁄", + "i", + "ii", + "iii", + "iv", + "v", + "vi", + "vii", + "viii", + "ix", + "x", + "xi", + "xii", + "l", + "c", + "d", + "m", + "i", + "ii", + "iii", + "iv", + "v", + "vi", + "vii", + "viii", + "ix", + "x", + "xi", + "xii", + "l", + "c", + "d", + "m", + None, + "ↄ", + None, + "0⁄3", + None, + None, + None, + "∫∫", + "∫∫∫", + None, + "∮∮", + "∮∮∮", + None, + "〈", + "〉", + None, + None, + None, + None, + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "(1)", + "(2)", + "(3)", + "(4)", + "(5)", + "(6)", + "(7)", + "(8)", + "(9)", + "(10)", + "(11)", + "(12)", + "(13)", + "(14)", + "(15)", + "(16)", + "(17)", + "(18)", + "(19)", + "(20)", + None, + "(a)", + "(b)", + "(c)", + "(d)", + "(e)", + "(f)", + "(g)", + "(h)", + "(i)", + "(j)", + "(k)", + "(l)", + "(m)", + "(n)", + "(o)", + "(p)", + "(q)", + "(r)", + "(s)", + "(t)", + "(u)", + "(v)", + "(w)", + "(x)", + "(y)", + "(z)", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "0", + None, + "∫∫∫∫", + None, + "::=", + "==", + "===", + None, + "⫝̸", + None, + None, + None, + "ⰰ", + "ⰱ", + "ⰲ", + "ⰳ", + "ⰴ", + "ⰵ", + "ⰶ", + "ⰷ", + "ⰸ", + "ⰹ", + "ⰺ", + "ⰻ", + "ⰼ", + "ⰽ", + "ⰾ", + "ⰿ", + "ⱀ", + "ⱁ", + "ⱂ", + "ⱃ", + "ⱄ", + "ⱅ", + "ⱆ", + "ⱇ", + "ⱈ", + "ⱉ", + "ⱊ", + "ⱋ", + "ⱌ", + "ⱍ", + "ⱎ", + "ⱏ", + "ⱐ", + "ⱑ", + "ⱒ", + "ⱓ", + "ⱔ", + "ⱕ", + "ⱖ", + "ⱗ", + "ⱘ", + "ⱙ", + "ⱚ", + "ⱛ", + "ⱜ", + "ⱝ", + "ⱞ", + "ⱟ", + None, + "ⱡ", + None, + "ɫ", + "ᵽ", + "ɽ", + None, + "ⱨ", + None, + "ⱪ", + None, + "ⱬ", + None, + "ɑ", + "ɱ", + "ɐ", + "ɒ", + None, + "ⱳ", + None, + "ⱶ", + None, + "j", + "v", + "ȿ", + "ɀ", + "ⲁ", + None, + "ⲃ", + None, + "ⲅ", + None, + "ⲇ", + None, + "ⲉ", + None, + "ⲋ", + None, + "ⲍ", + None, + "ⲏ", + None, + "ⲑ", + None, + "ⲓ", + None, + "ⲕ", + None, + "ⲗ", + None, + "ⲙ", + None, + "ⲛ", + None, + "ⲝ", + None, + "ⲟ", + None, + "ⲡ", + None, + "ⲣ", + None, + "ⲥ", + None, + "ⲧ", + None, + "ⲩ", + None, + "ⲫ", + None, + "ⲭ", + None, + "ⲯ", + None, + "ⲱ", + None, + "ⲳ", + None, + "ⲵ", + None, + "ⲷ", + None, + "ⲹ", + None, + "ⲻ", + None, + "ⲽ", + None, + "ⲿ", + None, + "ⳁ", + None, + "ⳃ", + None, + "ⳅ", + None, + "ⳇ", + None, + "ⳉ", + None, + "ⳋ", + None, + "ⳍ", + None, + "ⳏ", + None, + "ⳑ", + None, + "ⳓ", + None, + "ⳕ", + None, + "ⳗ", + None, + "ⳙ", + None, + "ⳛ", + None, + "ⳝ", + None, + "ⳟ", + None, + "ⳡ", + None, + "ⳣ", + None, + "ⳬ", + None, + "ⳮ", + None, + "ⳳ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ⵡ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "母", + None, + "龟", + None, + "一", + "丨", + "丶", + "丿", + "乙", + "亅", + "二", + "亠", + "人", + "儿", + "入", + "八", + "冂", + "冖", + "冫", + "几", + "凵", + "刀", + "力", + "勹", + "匕", + "匚", + "匸", + "十", + "卜", + "卩", + "厂", + "厶", + "又", + "口", + "囗", + "土", + "士", + "夂", + "夊", + "夕", + "大", + "女", + "子", + "宀", + "寸", + "小", + "尢", + "尸", + "屮", + "山", + "巛", + "工", + "己", + "巾", + "干", + "幺", + "广", + "廴", + "廾", + "弋", + "弓", + "彐", + "彡", + "彳", + "心", + "戈", + "戶", + "手", + "支", + "攴", + "文", + "斗", + "斤", + "方", + "无", + "日", + "曰", + "月", + "木", + "欠", + "止", + "歹", + "殳", + "毋", + "比", + "毛", + "氏", + "气", + "水", + "火", + "爪", + "父", + "爻", + "爿", + "片", + "牙", + "牛", + "犬", + "玄", + "玉", + "瓜", + "瓦", + "甘", + "生", + "用", + "田", + "疋", + "疒", + "癶", + "白", + "皮", + "皿", + "目", + "矛", + "矢", + "石", + "示", + "禸", + "禾", + "穴", + "立", + "竹", + "米", + "糸", + "缶", + "网", + "羊", + "羽", + "老", + "而", + "耒", + "耳", + "聿", + "肉", + "臣", + "自", + "至", + "臼", + "舌", + "舛", + "舟", + "艮", + "色", + "艸", + "虍", + "虫", + "血", + "行", + "衣", + "襾", + "見", + "角", + "言", + "谷", + "豆", + "豕", + "豸", + "貝", + "赤", + "走", + "足", + "身", + "車", + "辛", + "辰", + "辵", + "邑", + "酉", + "釆", + "里", + "金", + "長", + "門", + "阜", + "隶", + "隹", + "雨", + "靑", + "非", + "面", + "革", + "韋", + "韭", + "音", + "頁", + "風", + "飛", + "食", + "首", + "香", + "馬", + "骨", + "高", + "髟", + "鬥", + "鬯", + "鬲", + "鬼", + "魚", + "鳥", + "鹵", + "鹿", + "麥", + "麻", + "黃", + "黍", + "黑", + "黹", + "黽", + "鼎", + "鼓", + "鼠", + "鼻", + "齊", + "齒", + "龍", + "龜", + "龠", + None, + " ", + None, + ".", + None, + "〒", + None, + "十", + "卄", + "卅", + None, + None, + None, + None, + None, + " ゙", + " ゚", + None, + "より", + None, + "コト", + None, + None, + None, + "ᄀ", + "ᄁ", + "ᆪ", + "ᄂ", + "ᆬ", + "ᆭ", + "ᄃ", + "ᄄ", + "ᄅ", + "ᆰ", + "ᆱ", + "ᆲ", + "ᆳ", + "ᆴ", + "ᆵ", + "ᄚ", + "ᄆ", + "ᄇ", + "ᄈ", + "ᄡ", + "ᄉ", + "ᄊ", + "ᄋ", + "ᄌ", + "ᄍ", + "ᄎ", + "ᄏ", + "ᄐ", + "ᄑ", + "ᄒ", + "ᅡ", + "ᅢ", + "ᅣ", + "ᅤ", + "ᅥ", + "ᅦ", + "ᅧ", + "ᅨ", + "ᅩ", + "ᅪ", + "ᅫ", + "ᅬ", + "ᅭ", + "ᅮ", + "ᅯ", + "ᅰ", + "ᅱ", + "ᅲ", + "ᅳ", + "ᅴ", + "ᅵ", + None, + "ᄔ", + "ᄕ", + "ᇇ", + "ᇈ", + "ᇌ", + "ᇎ", + "ᇓ", + "ᇗ", + "ᇙ", + "ᄜ", + "ᇝ", + "ᇟ", + "ᄝ", + "ᄞ", + "ᄠ", + "ᄢ", + "ᄣ", + "ᄧ", + "ᄩ", + "ᄫ", + "ᄬ", + "ᄭ", + "ᄮ", + "ᄯ", + "ᄲ", + "ᄶ", + "ᅀ", + "ᅇ", + "ᅌ", + "ᇱ", + "ᇲ", + "ᅗ", + "ᅘ", + "ᅙ", + "ᆄ", + "ᆅ", + "ᆈ", + "ᆑ", + "ᆒ", + "ᆔ", + "ᆞ", + "ᆡ", + None, + None, + "一", + "二", + "三", + "四", + "上", + "中", + "下", + "甲", + "乙", + "丙", + "丁", + "天", + "地", + "人", + None, + None, + None, + "(ᄀ)", + "(ᄂ)", + "(ᄃ)", + "(ᄅ)", + "(ᄆ)", + "(ᄇ)", + "(ᄉ)", + "(ᄋ)", + "(ᄌ)", + "(ᄎ)", + "(ᄏ)", + "(ᄐ)", + "(ᄑ)", + "(ᄒ)", + "(가)", + "(나)", + "(다)", + "(라)", + "(마)", + "(바)", + "(사)", + "(아)", + "(자)", + "(차)", + "(카)", + "(타)", + "(파)", + "(하)", + "(주)", + "(오전)", + "(오후)", + None, + "(一)", + "(二)", + "(三)", + "(四)", + "(五)", + "(六)", + "(七)", + "(八)", + "(九)", + "(十)", + "(月)", + "(火)", + "(水)", + "(木)", + "(金)", + "(土)", + "(日)", + "(株)", + "(有)", + "(社)", + "(名)", + "(特)", + "(財)", + "(祝)", + "(労)", + "(代)", + "(呼)", + "(学)", + "(監)", + "(企)", + "(資)", + "(協)", + "(祭)", + "(休)", + "(自)", + "(至)", + "問", + "幼", + "文", + "箏", + None, + "pte", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "ᄀ", + "ᄂ", + "ᄃ", + "ᄅ", + "ᄆ", + "ᄇ", + "ᄉ", + "ᄋ", + "ᄌ", + "ᄎ", + "ᄏ", + "ᄐ", + "ᄑ", + "ᄒ", + "가", + "나", + "다", + "라", + "마", + "바", + "사", + "아", + "자", + "차", + "카", + "타", + "파", + "하", + "참고", + "주의", + "우", + None, + "一", + "二", + "三", + "四", + "五", + "六", + "七", + "八", + "九", + "十", + "月", + "火", + "水", + "木", + "金", + "土", + "日", + "株", + "有", + "社", + "名", + "特", + "財", + "祝", + "労", + "秘", + "男", + "女", + "適", + "優", + "印", + "注", + "項", + "休", + "写", + "正", + "上", + "中", + "下", + "左", + "右", + "医", + "宗", + "学", + "監", + "企", + "資", + "協", + "夜", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "43", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "1月", + "2月", + "3月", + "4月", + "5月", + "6月", + "7月", + "8月", + "9月", + "10月", + "11月", + "12月", + "hg", + "erg", + "ev", + "ltd", + "ア", + "イ", + "ウ", + "エ", + "オ", + "カ", + "キ", + "ク", + "ケ", + "コ", + "サ", + "シ", + "ス", + "セ", + "ソ", + "タ", + "チ", + "ツ", + "テ", + "ト", + "ナ", + "ニ", + "ヌ", + "ネ", + "ノ", + "ハ", + "ヒ", + "フ", + "ヘ", + "ホ", + "マ", + "ミ", + "ム", + "メ", + "モ", + "ヤ", + "ユ", + "ヨ", + "ラ", + "リ", + "ル", + "レ", + "ロ", + "ワ", + "ヰ", + "ヱ", + "ヲ", + "令和", + "アパート", + "アルファ", + "アンペア", + "アール", + "イニング", + "インチ", + "ウォン", + "エスクード", + "エーカー", + "オンス", + "オーム", + "カイリ", + "カラット", + "カロリー", + "ガロン", + "ガンマ", + "ギガ", + "ギニー", + "キュリー", + "ギルダー", + "キロ", + "キログラム", + "キロメートル", + "キロワット", + "グラム", + "グラムトン", + "クルゼイロ", + "クローネ", + "ケース", + "コルナ", + "コーポ", + "サイクル", + "サンチーム", + "シリング", + "センチ", + "セント", + "ダース", + "デシ", + "ドル", + "トン", + "ナノ", + "ノット", + "ハイツ", + "パーセント", + "パーツ", + "バーレル", + "ピアストル", + "ピクル", + "ピコ", + "ビル", + "ファラッド", + "フィート", + "ブッシェル", + "フラン", + "ヘクタール", + "ペソ", + "ペニヒ", + "ヘルツ", + "ペンス", + "ページ", + "ベータ", + "ポイント", + "ボルト", + "ホン", + "ポンド", + "ホール", + "ホーン", + "マイクロ", + "マイル", + "マッハ", + "マルク", + "マンション", + "ミクロン", + "ミリ", + "ミリバール", + "メガ", + "メガトン", + "メートル", + "ヤード", + "ヤール", + "ユアン", + "リットル", + "リラ", + "ルピー", + "ルーブル", + "レム", + "レントゲン", + "ワット", + "0点", + "1点", + "2点", + "3点", + "4点", + "5点", + "6点", + "7点", + "8点", + "9点", + "10点", + "11点", + "12点", + "13点", + "14点", + "15点", + "16点", + "17点", + "18点", + "19点", + "20点", + "21点", + "22点", + "23点", + "24点", + "hpa", + "da", + "au", + "bar", + "ov", + "pc", + "dm", + "dm2", + "dm3", + "iu", + "平成", + "昭和", + "大正", + "明治", + "株式会社", + "pa", + "na", + "μa", + "ma", + "ka", + "kb", + "mb", + "gb", + "cal", + "kcal", + "pf", + "nf", + "μf", + "μg", + "mg", + "kg", + "hz", + "khz", + "mhz", + "ghz", + "thz", + "μl", + "ml", + "dl", + "kl", + "fm", + "nm", + "μm", + "mm", + "cm", + "km", + "mm2", + "cm2", + "m2", + "km2", + "mm3", + "cm3", + "m3", + "km3", + "m∕s", + "m∕s2", + "pa", + "kpa", + "mpa", + "gpa", + "rad", + "rad∕s", + "rad∕s2", + "ps", + "ns", + "μs", + "ms", + "pv", + "nv", + "μv", + "mv", + "kv", + "mv", + "pw", + "nw", + "μw", + "mw", + "kw", + "mw", + "kω", + "mω", + None, + "bq", + "cc", + "cd", + "c∕kg", + None, + "db", + "gy", + "ha", + "hp", + "in", + "kk", + "km", + "kt", + "lm", + "ln", + "log", + "lx", + "mb", + "mil", + "mol", + "ph", + None, + "ppm", + "pr", + "sr", + "sv", + "wb", + "v∕m", + "a∕m", + "1日", + "2日", + "3日", + "4日", + "5日", + "6日", + "7日", + "8日", + "9日", + "10日", + "11日", + "12日", + "13日", + "14日", + "15日", + "16日", + "17日", + "18日", + "19日", + "20日", + "21日", + "22日", + "23日", + "24日", + "25日", + "26日", + "27日", + "28日", + "29日", + "30日", + "31日", + "gal", + None, + None, + None, + None, + None, + None, + "ꙁ", + None, + "ꙃ", + None, + "ꙅ", + None, + "ꙇ", + None, + "ꙉ", + None, + "ꙋ", + None, + "ꙍ", + None, + "ꙏ", + None, + "ꙑ", + None, + "ꙓ", + None, + "ꙕ", + None, + "ꙗ", + None, + "ꙙ", + None, + "ꙛ", + None, + "ꙝ", + None, + "ꙟ", + None, + "ꙡ", + None, + "ꙣ", + None, + "ꙥ", + None, + "ꙧ", + None, + "ꙩ", + None, + "ꙫ", + None, + "ꙭ", + None, + "ꚁ", + None, + "ꚃ", + None, + "ꚅ", + None, + "ꚇ", + None, + "ꚉ", + None, + "ꚋ", + None, + "ꚍ", + None, + "ꚏ", + None, + "ꚑ", + None, + "ꚓ", + None, + "ꚕ", + None, + "ꚗ", + None, + "ꚙ", + None, + "ꚛ", + None, + "ъ", + "ь", + None, + None, + None, + "ꜣ", + None, + "ꜥ", + None, + "ꜧ", + None, + "ꜩ", + None, + "ꜫ", + None, + "ꜭ", + None, + "ꜯ", + None, + "ꜳ", + None, + "ꜵ", + None, + "ꜷ", + None, + "ꜹ", + None, + "ꜻ", + None, + "ꜽ", + None, + "ꜿ", + None, + "ꝁ", + None, + "ꝃ", + None, + "ꝅ", + None, + "ꝇ", + None, + "ꝉ", + None, + "ꝋ", + None, + "ꝍ", + None, + "ꝏ", + None, + "ꝑ", + None, + "ꝓ", + None, + "ꝕ", + None, + "ꝗ", + None, + "ꝙ", + None, + "ꝛ", + None, + "ꝝ", + None, + "ꝟ", + None, + "ꝡ", + None, + "ꝣ", + None, + "ꝥ", + None, + "ꝧ", + None, + "ꝩ", + None, + "ꝫ", + None, + "ꝭ", + None, + "ꝯ", + None, + "ꝯ", + None, + "ꝺ", + None, + "ꝼ", + None, + "ᵹ", + "ꝿ", + None, + "ꞁ", + None, + "ꞃ", + None, + "ꞅ", + None, + "ꞇ", + None, + "ꞌ", + None, + "ɥ", + None, + "ꞑ", + None, + "ꞓ", + None, + "ꞗ", + None, + "ꞙ", + None, + "ꞛ", + None, + "ꞝ", + None, + "ꞟ", + None, + "ꞡ", + None, + "ꞣ", + None, + "ꞥ", + None, + "ꞧ", + None, + "ꞩ", + None, + "ɦ", + "ɜ", + "ɡ", + "ɬ", + "ɪ", + None, + "ʞ", + "ʇ", + "ʝ", + "ꭓ", + "ꞵ", + None, + "ꞷ", + None, + "ꞹ", + None, + "ꞻ", + None, + "ꞽ", + None, + "ꞿ", + None, + "ꟁ", + None, + "ꟃ", + None, + "ꞔ", + "ʂ", + "ᶎ", + "ꟈ", + None, + "ꟊ", + None, + "ɤ", + "\ua7cd", + None, + "\ua7cf", + None, + "ꟑ", + None, + "ꟓ", + None, + "ꟕ", + None, + "ꟗ", + None, + "ꟙ", + None, + "\ua7db", + None, + "ƛ", + None, + "s", + "c", + "f", + "q", + "ꟶ", + None, + "ħ", + "œ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ꜧ", + "ꬷ", + "ɫ", + "ꭒ", + None, + "ʍ", + None, + None, + "Ꭰ", + "Ꭱ", + "Ꭲ", + "Ꭳ", + "Ꭴ", + "Ꭵ", + "Ꭶ", + "Ꭷ", + "Ꭸ", + "Ꭹ", + "Ꭺ", + "Ꭻ", + "Ꭼ", + "Ꭽ", + "Ꭾ", + "Ꭿ", + "Ꮀ", + "Ꮁ", + "Ꮂ", + "Ꮃ", + "Ꮄ", + "Ꮅ", + "Ꮆ", + "Ꮇ", + "Ꮈ", + "Ꮉ", + "Ꮊ", + "Ꮋ", + "Ꮌ", + "Ꮍ", + "Ꮎ", + "Ꮏ", + "Ꮐ", + "Ꮑ", + "Ꮒ", + "Ꮓ", + "Ꮔ", + "Ꮕ", + "Ꮖ", + "Ꮗ", + "Ꮘ", + "Ꮙ", + "Ꮚ", + "Ꮛ", + "Ꮜ", + "Ꮝ", + "Ꮞ", + "Ꮟ", + "Ꮠ", + "Ꮡ", + "Ꮢ", + "Ꮣ", + "Ꮤ", + "Ꮥ", + "Ꮦ", + "Ꮧ", + "Ꮨ", + "Ꮩ", + "Ꮪ", + "Ꮫ", + "Ꮬ", + "Ꮭ", + "Ꮮ", + "Ꮯ", + "Ꮰ", + "Ꮱ", + "Ꮲ", + "Ꮳ", + "Ꮴ", + "Ꮵ", + "Ꮶ", + "Ꮷ", + "Ꮸ", + "Ꮹ", + "Ꮺ", + "Ꮻ", + "Ꮼ", + "Ꮽ", + "Ꮾ", + "Ꮿ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "豈", + "更", + "車", + "賈", + "滑", + "串", + "句", + "龜", + "契", + "金", + "喇", + "奈", + "懶", + "癩", + "羅", + "蘿", + "螺", + "裸", + "邏", + "樂", + "洛", + "烙", + "珞", + "落", + "酪", + "駱", + "亂", + "卵", + "欄", + "爛", + "蘭", + "鸞", + "嵐", + "濫", + "藍", + "襤", + "拉", + "臘", + "蠟", + "廊", + "朗", + "浪", + "狼", + "郎", + "來", + "冷", + "勞", + "擄", + "櫓", + "爐", + "盧", + "老", + "蘆", + "虜", + "路", + "露", + "魯", + "鷺", + "碌", + "祿", + "綠", + "菉", + "錄", + "鹿", + "論", + "壟", + "弄", + "籠", + "聾", + "牢", + "磊", + "賂", + "雷", + "壘", + "屢", + "樓", + "淚", + "漏", + "累", + "縷", + "陋", + "勒", + "肋", + "凜", + "凌", + "稜", + "綾", + "菱", + "陵", + "讀", + "拏", + "樂", + "諾", + "丹", + "寧", + "怒", + "率", + "異", + "北", + "磻", + "便", + "復", + "不", + "泌", + "數", + "索", + "參", + "塞", + "省", + "葉", + "說", + "殺", + "辰", + "沈", + "拾", + "若", + "掠", + "略", + "亮", + "兩", + "凉", + "梁", + "糧", + "良", + "諒", + "量", + "勵", + "呂", + "女", + "廬", + "旅", + "濾", + "礪", + "閭", + "驪", + "麗", + "黎", + "力", + "曆", + "歷", + "轢", + "年", + "憐", + "戀", + "撚", + "漣", + "煉", + "璉", + "秊", + "練", + "聯", + "輦", + "蓮", + "連", + "鍊", + "列", + "劣", + "咽", + "烈", + "裂", + "說", + "廉", + "念", + "捻", + "殮", + "簾", + "獵", + "令", + "囹", + "寧", + "嶺", + "怜", + "玲", + "瑩", + "羚", + "聆", + "鈴", + "零", + "靈", + "領", + "例", + "禮", + "醴", + "隸", + "惡", + "了", + "僚", + "寮", + "尿", + "料", + "樂", + "燎", + "療", + "蓼", + "遼", + "龍", + "暈", + "阮", + "劉", + "杻", + "柳", + "流", + "溜", + "琉", + "留", + "硫", + "紐", + "類", + "六", + "戮", + "陸", + "倫", + "崙", + "淪", + "輪", + "律", + "慄", + "栗", + "率", + "隆", + "利", + "吏", + "履", + "易", + "李", + "梨", + "泥", + "理", + "痢", + "罹", + "裏", + "裡", + "里", + "離", + "匿", + "溺", + "吝", + "燐", + "璘", + "藺", + "隣", + "鱗", + "麟", + "林", + "淋", + "臨", + "立", + "笠", + "粒", + "狀", + "炙", + "識", + "什", + "茶", + "刺", + "切", + "度", + "拓", + "糖", + "宅", + "洞", + "暴", + "輻", + "行", + "降", + "見", + "廓", + "兀", + "嗀", + None, + "塚", + None, + "晴", + None, + "凞", + "猪", + "益", + "礼", + "神", + "祥", + "福", + "靖", + "精", + "羽", + None, + "蘒", + None, + "諸", + None, + "逸", + "都", + None, + "飯", + "飼", + "館", + "鶴", + "郞", + "隷", + "侮", + "僧", + "免", + "勉", + "勤", + "卑", + "喝", + "嘆", + "器", + "塀", + "墨", + "層", + "屮", + "悔", + "慨", + "憎", + "懲", + "敏", + "既", + "暑", + "梅", + "海", + "渚", + "漢", + "煮", + "爫", + "琢", + "碑", + "社", + "祉", + "祈", + "祐", + "祖", + "祝", + "禍", + "禎", + "穀", + "突", + "節", + "練", + "縉", + "繁", + "署", + "者", + "臭", + "艹", + "著", + "褐", + "視", + "謁", + "謹", + "賓", + "贈", + "辶", + "逸", + "難", + "響", + "頻", + "恵", + "𤋮", + "舘", + None, + "並", + "况", + "全", + "侀", + "充", + "冀", + "勇", + "勺", + "喝", + "啕", + "喙", + "嗢", + "塚", + "墳", + "奄", + "奔", + "婢", + "嬨", + "廒", + "廙", + "彩", + "徭", + "惘", + "慎", + "愈", + "憎", + "慠", + "懲", + "戴", + "揄", + "搜", + "摒", + "敖", + "晴", + "朗", + "望", + "杖", + "歹", + "殺", + "流", + "滛", + "滋", + "漢", + "瀞", + "煮", + "瞧", + "爵", + "犯", + "猪", + "瑱", + "甆", + "画", + "瘝", + "瘟", + "益", + "盛", + "直", + "睊", + "着", + "磌", + "窱", + "節", + "类", + "絛", + "練", + "缾", + "者", + "荒", + "華", + "蝹", + "襁", + "覆", + "視", + "調", + "諸", + "請", + "謁", + "諾", + "諭", + "謹", + "變", + "贈", + "輸", + "遲", + "醙", + "鉶", + "陼", + "難", + "靖", + "韛", + "響", + "頋", + "頻", + "鬒", + "龜", + "𢡊", + "𢡄", + "𣏕", + "㮝", + "䀘", + "䀹", + "𥉉", + "𥳐", + "𧻓", + "齃", + "龎", + None, + "ff", + "fi", + "fl", + "ffi", + "ffl", + "st", + None, + "մն", + "մե", + "մի", + "վն", + "մխ", + None, + "יִ", + None, + "ײַ", + "ע", + "א", + "ד", + "ה", + "כ", + "ל", + "ם", + "ר", + "ת", + "+", + "שׁ", + "שׂ", + "שּׁ", + "שּׂ", + "אַ", + "אָ", + "אּ", + "בּ", + "גּ", + "דּ", + "הּ", + "וּ", + "זּ", + None, + "טּ", + "יּ", + "ךּ", + "כּ", + "לּ", + None, + "מּ", + None, + "נּ", + "סּ", + None, + "ףּ", + "פּ", + None, + "צּ", + "קּ", + "רּ", + "שּ", + "תּ", + "וֹ", + "בֿ", + "כֿ", + "פֿ", + "אל", + "ٱ", + "ٻ", + "پ", + "ڀ", + "ٺ", + "ٿ", + "ٹ", + "ڤ", + "ڦ", + "ڄ", + "ڃ", + "چ", + "ڇ", + "ڍ", + "ڌ", + "ڎ", + "ڈ", + "ژ", + "ڑ", + "ک", + "گ", + "ڳ", + "ڱ", + "ں", + "ڻ", + "ۀ", + "ہ", + "ھ", + "ے", + "ۓ", + None, + "ڭ", + "ۇ", + "ۆ", + "ۈ", + "ۇٴ", + "ۋ", + "ۅ", + "ۉ", + "ې", + "ى", + "ئا", + "ئە", + "ئو", + "ئۇ", + "ئۆ", + "ئۈ", + "ئې", + "ئى", + "ی", + "ئج", + "ئح", + "ئم", + "ئى", + "ئي", + "بج", + "بح", + "بخ", + "بم", + "بى", + "بي", + "تج", + "تح", + "تخ", + "تم", + "تى", + "تي", + "ثج", + "ثم", + "ثى", + "ثي", + "جح", + "جم", + "حج", + "حم", + "خج", + "خح", + "خم", + "سج", + "سح", + "سخ", + "سم", + "صح", + "صم", + "ضج", + "ضح", + "ضخ", + "ضم", + "طح", + "طم", + "ظم", + "عج", + "عم", + "غج", + "غم", + "فج", + "فح", + "فخ", + "فم", + "فى", + "في", + "قح", + "قم", + "قى", + "قي", + "كا", + "كج", + "كح", + "كخ", + "كل", + "كم", + "كى", + "كي", + "لج", + "لح", + "لخ", + "لم", + "لى", + "لي", + "مج", + "مح", + "مخ", + "مم", + "مى", + "مي", + "نج", + "نح", + "نخ", + "نم", + "نى", + "ني", + "هج", + "هم", + "هى", + "هي", + "يج", + "يح", + "يخ", + "يم", + "يى", + "يي", + "ذٰ", + "رٰ", + "ىٰ", + " ٌّ", + " ٍّ", + " َّ", + " ُّ", + " ِّ", + " ّٰ", + "ئر", + "ئز", + "ئم", + "ئن", + "ئى", + "ئي", + "بر", + "بز", + "بم", + "بن", + "بى", + "بي", + "تر", + "تز", + "تم", + "تن", + "تى", + "تي", + "ثر", + "ثز", + "ثم", + "ثن", + "ثى", + "ثي", + "فى", + "في", + "قى", + "قي", + "كا", + "كل", + "كم", + "كى", + "كي", + "لم", + "لى", + "لي", + "ما", + "مم", + "نر", + "نز", + "نم", + "نن", + "نى", + "ني", + "ىٰ", + "ير", + "يز", + "يم", + "ين", + "يى", + "يي", + "ئج", + "ئح", + "ئخ", + "ئم", + "ئه", + "بج", + "بح", + "بخ", + "بم", + "به", + "تج", + "تح", + "تخ", + "تم", + "ته", + "ثم", + "جح", + "جم", + "حج", + "حم", + "خج", + "خم", + "سج", + "سح", + "سخ", + "سم", + "صح", + "صخ", + "صم", + "ضج", + "ضح", + "ضخ", + "ضم", + "طح", + "ظم", + "عج", + "عم", + "غج", + "غم", + "فج", + "فح", + "فخ", + "فم", + "قح", + "قم", + "كج", + "كح", + "كخ", + "كل", + "كم", + "لج", + "لح", + "لخ", + "لم", + "له", + "مج", + "مح", + "مخ", + "مم", + "نج", + "نح", + "نخ", + "نم", + "نه", + "هج", + "هم", + "هٰ", + "يج", + "يح", + "يخ", + "يم", + "يه", + "ئم", + "ئه", + "بم", + "به", + "تم", + "ته", + "ثم", + "ثه", + "سم", + "سه", + "شم", + "شه", + "كل", + "كم", + "لم", + "نم", + "نه", + "يم", + "يه", + "ـَّ", + "ـُّ", + "ـِّ", + "طى", + "طي", + "عى", + "عي", + "غى", + "غي", + "سى", + "سي", + "شى", + "شي", + "حى", + "حي", + "جى", + "جي", + "خى", + "خي", + "صى", + "صي", + "ضى", + "ضي", + "شج", + "شح", + "شخ", + "شم", + "شر", + "سر", + "صر", + "ضر", + "طى", + "طي", + "عى", + "عي", + "غى", + "غي", + "سى", + "سي", + "شى", + "شي", + "حى", + "حي", + "جى", + "جي", + "خى", + "خي", + "صى", + "صي", + "ضى", + "ضي", + "شج", + "شح", + "شخ", + "شم", + "شر", + "سر", + "صر", + "ضر", + "شج", + "شح", + "شخ", + "شم", + "سه", + "شه", + "طم", + "سج", + "سح", + "سخ", + "شج", + "شح", + "شخ", + "طم", + "ظم", + "اً", + None, + "تجم", + "تحج", + "تحم", + "تخم", + "تمج", + "تمح", + "تمخ", + "جمح", + "حمي", + "حمى", + "سحج", + "سجح", + "سجى", + "سمح", + "سمج", + "سمم", + "صحح", + "صمم", + "شحم", + "شجي", + "شمخ", + "شمم", + "ضحى", + "ضخم", + "طمح", + "طمم", + "طمي", + "عجم", + "عمم", + "عمى", + "غمم", + "غمي", + "غمى", + "فخم", + "قمح", + "قمم", + "لحم", + "لحي", + "لحى", + "لجج", + "لخم", + "لمح", + "محج", + "محم", + "محي", + "مجح", + "مجم", + "مخج", + "مخم", + None, + "مجخ", + "همج", + "همم", + "نحم", + "نحى", + "نجم", + "نجى", + "نمي", + "نمى", + "يمم", + "بخي", + "تجي", + "تجى", + "تخي", + "تخى", + "تمي", + "تمى", + "جمي", + "جحى", + "جمى", + "سخى", + "صحي", + "شحي", + "ضحي", + "لجي", + "لمي", + "يحي", + "يجي", + "يمي", + "ممي", + "قمي", + "نحي", + "قمح", + "لحم", + "عمي", + "كمي", + "نجح", + "مخي", + "لجم", + "كمم", + "لجم", + "نجح", + "جحي", + "حجي", + "مجي", + "فمي", + "بحي", + "كمم", + "عجم", + "صمم", + "سخي", + "نجي", + None, + None, + "صلے", + "قلے", + "الله", + "اكبر", + "محمد", + "صلعم", + "رسول", + "عليه", + "وسلم", + "صلى", + "صلى الله عليه وسلم", + "جل جلاله", + "ریال", + None, + None, + ",", + "、", + None, + ":", + ";", + "!", + "?", + "〖", + "〗", + None, + None, + None, + "—", + "–", + "_", + "(", + ")", + "{", + "}", + "〔", + "〕", + "【", + "】", + "《", + "》", + "〈", + "〉", + "「", + "」", + "『", + "』", + None, + "[", + "]", + " ̅", + "_", + ",", + "、", + None, + ";", + ":", + "?", + "!", + "—", + "(", + ")", + "{", + "}", + "〔", + "〕", + "#", + "&", + "*", + "+", + "-", + "<", + ">", + "=", + None, + "\\", + "$", + "%", + "@", + None, + " ً", + "ـً", + " ٌ", + None, + " ٍ", + None, + " َ", + "ـَ", + " ُ", + "ـُ", + " ِ", + "ـِ", + " ّ", + "ـّ", + " ْ", + "ـْ", + "ء", + "آ", + "أ", + "ؤ", + "إ", + "ئ", + "ا", + "ب", + "ة", + "ت", + "ث", + "ج", + "ح", + "خ", + "د", + "ذ", + "ر", + "ز", + "س", + "ش", + "ص", + "ض", + "ط", + "ظ", + "ع", + "غ", + "ف", + "ق", + "ك", + "ل", + "م", + "ن", + "ه", + "و", + "ى", + "ي", + "لآ", + "لأ", + "لإ", + "لا", + None, + None, + None, + "!", + '"', + "#", + "$", + "%", + "&", + "'", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "?", + "@", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "[", + "\\", + "]", + "^", + "_", + "`", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~", + "⦅", + "⦆", + ".", + "「", + "」", + "、", + "・", + "ヲ", + "ァ", + "ィ", + "ゥ", + "ェ", + "ォ", + "ャ", + "ュ", + "ョ", + "ッ", + "ー", + "ア", + "イ", + "ウ", + "エ", + "オ", + "カ", + "キ", + "ク", + "ケ", + "コ", + "サ", + "シ", + "ス", + "セ", + "ソ", + "タ", + "チ", + "ツ", + "テ", + "ト", + "ナ", + "ニ", + "ヌ", + "ネ", + "ノ", + "ハ", + "ヒ", + "フ", + "ヘ", + "ホ", + "マ", + "ミ", + "ム", + "メ", + "モ", + "ヤ", + "ユ", + "ヨ", + "ラ", + "リ", + "ル", + "レ", + "ロ", + "ワ", + "ン", + "゙", + "゚", + None, + "ᄀ", + "ᄁ", + "ᆪ", + "ᄂ", + "ᆬ", + "ᆭ", + "ᄃ", + "ᄄ", + "ᄅ", + "ᆰ", + "ᆱ", + "ᆲ", + "ᆳ", + "ᆴ", + "ᆵ", + "ᄚ", + "ᄆ", + "ᄇ", + "ᄈ", + "ᄡ", + "ᄉ", + "ᄊ", + "ᄋ", + "ᄌ", + "ᄍ", + "ᄎ", + "ᄏ", + "ᄐ", + "ᄑ", + "ᄒ", + None, + "ᅡ", + "ᅢ", + "ᅣ", + "ᅤ", + "ᅥ", + "ᅦ", + None, + "ᅧ", + "ᅨ", + "ᅩ", + "ᅪ", + "ᅫ", + "ᅬ", + None, + "ᅭ", + "ᅮ", + "ᅯ", + "ᅰ", + "ᅱ", + "ᅲ", + None, + "ᅳ", + "ᅴ", + "ᅵ", + None, + "¢", + "£", + "¬", + " ̄", + "¦", + "¥", + "₩", + None, + "│", + "←", + "↑", + "→", + "↓", + "■", + "○", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𐐨", + "𐐩", + "𐐪", + "𐐫", + "𐐬", + "𐐭", + "𐐮", + "𐐯", + "𐐰", + "𐐱", + "𐐲", + "𐐳", + "𐐴", + "𐐵", + "𐐶", + "𐐷", + "𐐸", + "𐐹", + "𐐺", + "𐐻", + "𐐼", + "𐐽", + "𐐾", + "𐐿", + "𐑀", + "𐑁", + "𐑂", + "𐑃", + "𐑄", + "𐑅", + "𐑆", + "𐑇", + "𐑈", + "𐑉", + "𐑊", + "𐑋", + "𐑌", + "𐑍", + "𐑎", + "𐑏", + None, + None, + None, + None, + "𐓘", + "𐓙", + "𐓚", + "𐓛", + "𐓜", + "𐓝", + "𐓞", + "𐓟", + "𐓠", + "𐓡", + "𐓢", + "𐓣", + "𐓤", + "𐓥", + "𐓦", + "𐓧", + "𐓨", + "𐓩", + "𐓪", + "𐓫", + "𐓬", + "𐓭", + "𐓮", + "𐓯", + "𐓰", + "𐓱", + "𐓲", + "𐓳", + "𐓴", + "𐓵", + "𐓶", + "𐓷", + "𐓸", + "𐓹", + "𐓺", + "𐓻", + None, + None, + None, + None, + None, + None, + None, + None, + "𐖗", + "𐖘", + "𐖙", + "𐖚", + "𐖛", + "𐖜", + "𐖝", + "𐖞", + "𐖟", + "𐖠", + "𐖡", + None, + "𐖣", + "𐖤", + "𐖥", + "𐖦", + "𐖧", + "𐖨", + "𐖩", + "𐖪", + "𐖫", + "𐖬", + "𐖭", + "𐖮", + "𐖯", + "𐖰", + "𐖱", + None, + "𐖳", + "𐖴", + "𐖵", + "𐖶", + "𐖷", + "𐖸", + "𐖹", + None, + "𐖻", + "𐖼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ː", + "ˑ", + "æ", + "ʙ", + "ɓ", + None, + "ʣ", + "ꭦ", + "ʥ", + "ʤ", + "ɖ", + "ɗ", + "ᶑ", + "ɘ", + "ɞ", + "ʩ", + "ɤ", + "ɢ", + "ɠ", + "ʛ", + "ħ", + "ʜ", + "ɧ", + "ʄ", + "ʪ", + "ʫ", + "ɬ", + "𝼄", + "ꞎ", + "ɮ", + "𝼅", + "ʎ", + "𝼆", + "ø", + "ɶ", + "ɷ", + "q", + "ɺ", + "𝼈", + "ɽ", + "ɾ", + "ʀ", + "ʨ", + "ʦ", + "ꭧ", + "ʧ", + "ʈ", + "ⱱ", + None, + "ʏ", + "ʡ", + "ʢ", + "ʘ", + "ǀ", + "ǁ", + "ǂ", + "𝼊", + "𝼞", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𐳀", + "𐳁", + "𐳂", + "𐳃", + "𐳄", + "𐳅", + "𐳆", + "𐳇", + "𐳈", + "𐳉", + "𐳊", + "𐳋", + "𐳌", + "𐳍", + "𐳎", + "𐳏", + "𐳐", + "𐳑", + "𐳒", + "𐳓", + "𐳔", + "𐳕", + "𐳖", + "𐳗", + "𐳘", + "𐳙", + "𐳚", + "𐳛", + "𐳜", + "𐳝", + "𐳞", + "𐳟", + "𐳠", + "𐳡", + "𐳢", + "𐳣", + "𐳤", + "𐳥", + "𐳦", + "𐳧", + "𐳨", + "𐳩", + "𐳪", + "𐳫", + "𐳬", + "𐳭", + "𐳮", + "𐳯", + "𐳰", + "𐳱", + "𐳲", + None, + None, + None, + None, + None, + None, + None, + None, + "\U00010d70", + "\U00010d71", + "\U00010d72", + "\U00010d73", + "\U00010d74", + "\U00010d75", + "\U00010d76", + "\U00010d77", + "\U00010d78", + "\U00010d79", + "\U00010d7a", + "\U00010d7b", + "\U00010d7c", + "\U00010d7d", + "\U00010d7e", + "\U00010d7f", + "\U00010d80", + "\U00010d81", + "\U00010d82", + "\U00010d83", + "\U00010d84", + "\U00010d85", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𑣀", + "𑣁", + "𑣂", + "𑣃", + "𑣄", + "𑣅", + "𑣆", + "𑣇", + "𑣈", + "𑣉", + "𑣊", + "𑣋", + "𑣌", + "𑣍", + "𑣎", + "𑣏", + "𑣐", + "𑣑", + "𑣒", + "𑣓", + "𑣔", + "𑣕", + "𑣖", + "𑣗", + "𑣘", + "𑣙", + "𑣚", + "𑣛", + "𑣜", + "𑣝", + "𑣞", + "𑣟", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𖹠", + "𖹡", + "𖹢", + "𖹣", + "𖹤", + "𖹥", + "𖹦", + "𖹧", + "𖹨", + "𖹩", + "𖹪", + "𖹫", + "𖹬", + "𖹭", + "𖹮", + "𖹯", + "𖹰", + "𖹱", + "𖹲", + "𖹳", + "𖹴", + "𖹵", + "𖹶", + "𖹷", + "𖹸", + "𖹹", + "𖹺", + "𖹻", + "𖹼", + "𖹽", + "𖹾", + "𖹿", + None, + None, + "\U00016ebb", + "\U00016ebc", + "\U00016ebd", + "\U00016ebe", + "\U00016ebf", + "\U00016ec0", + "\U00016ec1", + "\U00016ec2", + "\U00016ec3", + "\U00016ec4", + "\U00016ec5", + "\U00016ec6", + "\U00016ec7", + "\U00016ec8", + "\U00016ec9", + "\U00016eca", + "\U00016ecb", + "\U00016ecc", + "\U00016ecd", + "\U00016ece", + "\U00016ecf", + "\U00016ed0", + "\U00016ed1", + "\U00016ed2", + "\U00016ed3", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𝅗𝅥", + "𝅘𝅥", + "𝅘𝅥𝅮", + "𝅘𝅥𝅯", + "𝅘𝅥𝅰", + "𝅘𝅥𝅱", + "𝅘𝅥𝅲", + None, + None, + None, + "𝆹𝅥", + "𝆺𝅥", + "𝆹𝅥𝅮", + "𝆺𝅥𝅮", + "𝆹𝅥𝅯", + "𝆺𝅥𝅯", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + None, + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + None, + "c", + "d", + None, + "g", + None, + "j", + "k", + None, + "n", + "o", + "p", + "q", + None, + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + None, + "f", + None, + "h", + "i", + "j", + "k", + "l", + "m", + "n", + None, + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + None, + "d", + "e", + "f", + "g", + None, + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + None, + "s", + "t", + "u", + "v", + "w", + "x", + "y", + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + None, + "d", + "e", + "f", + "g", + None, + "i", + "j", + "k", + "l", + "m", + None, + "o", + None, + "s", + "t", + "u", + "v", + "w", + "x", + "y", + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "ı", + "ȷ", + None, + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "ϝ", + None, + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "а", + "б", + "в", + "г", + "д", + "е", + "ж", + "з", + "и", + "к", + "л", + "м", + "о", + "п", + "р", + "с", + "т", + "у", + "ф", + "х", + "ц", + "ч", + "ш", + "ы", + "э", + "ю", + "ꚉ", + "ә", + "і", + "ј", + "ө", + "ү", + "ӏ", + "а", + "б", + "в", + "г", + "д", + "е", + "ж", + "з", + "и", + "к", + "л", + "о", + "п", + "с", + "у", + "ф", + "х", + "ц", + "ч", + "ш", + "ъ", + "ы", + "ґ", + "і", + "ѕ", + "џ", + "ҫ", + "ꙑ", + "ұ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𞤢", + "𞤣", + "𞤤", + "𞤥", + "𞤦", + "𞤧", + "𞤨", + "𞤩", + "𞤪", + "𞤫", + "𞤬", + "𞤭", + "𞤮", + "𞤯", + "𞤰", + "𞤱", + "𞤲", + "𞤳", + "𞤴", + "𞤵", + "𞤶", + "𞤷", + "𞤸", + "𞤹", + "𞤺", + "𞤻", + "𞤼", + "𞤽", + "𞤾", + "𞤿", + "𞥀", + "𞥁", + "𞥂", + "𞥃", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ا", + "ب", + "ج", + "د", + None, + "و", + "ز", + "ح", + "ط", + "ي", + "ك", + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + "ر", + "ش", + "ت", + "ث", + "خ", + "ذ", + "ض", + "ظ", + "غ", + "ٮ", + "ں", + "ڡ", + "ٯ", + None, + "ب", + "ج", + None, + "ه", + None, + "ح", + None, + "ي", + "ك", + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + None, + "ش", + "ت", + "ث", + "خ", + None, + "ض", + None, + "غ", + None, + "ج", + None, + "ح", + None, + "ي", + None, + "ل", + None, + "ن", + "س", + "ع", + None, + "ص", + "ق", + None, + "ش", + None, + "خ", + None, + "ض", + None, + "غ", + None, + "ں", + None, + "ٯ", + None, + "ب", + "ج", + None, + "ه", + None, + "ح", + "ط", + "ي", + "ك", + None, + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + None, + "ش", + "ت", + "ث", + "خ", + None, + "ض", + "ظ", + "غ", + "ٮ", + None, + "ڡ", + None, + "ا", + "ب", + "ج", + "د", + "ه", + "و", + "ز", + "ح", + "ط", + "ي", + None, + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + "ر", + "ش", + "ت", + "ث", + "خ", + "ذ", + "ض", + "ظ", + "غ", + None, + "ب", + "ج", + "د", + None, + "و", + "ز", + "ح", + "ط", + "ي", + None, + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + "ر", + "ش", + "ت", + "ث", + "خ", + "ذ", + "ض", + "ظ", + "غ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "0,", + "1,", + "2,", + "3,", + "4,", + "5,", + "6,", + "7,", + "8,", + "9,", + None, + "(a)", + "(b)", + "(c)", + "(d)", + "(e)", + "(f)", + "(g)", + "(h)", + "(i)", + "(j)", + "(k)", + "(l)", + "(m)", + "(n)", + "(o)", + "(p)", + "(q)", + "(r)", + "(s)", + "(t)", + "(u)", + "(v)", + "(w)", + "(x)", + "(y)", + "(z)", + "〔s〕", + "c", + "r", + "cd", + "wz", + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "hv", + "mv", + "sd", + "ss", + "ppv", + "wc", + None, + "mc", + "md", + "mr", + None, + "dj", + None, + None, + None, + "ほか", + "ココ", + "サ", + None, + "手", + "字", + "双", + "デ", + "二", + "多", + "解", + "天", + "交", + "映", + "無", + "料", + "前", + "後", + "再", + "新", + "初", + "終", + "生", + "販", + "声", + "吹", + "演", + "投", + "捕", + "一", + "三", + "遊", + "左", + "中", + "右", + "指", + "走", + "打", + "禁", + "空", + "合", + "満", + "有", + "月", + "申", + "割", + "営", + "配", + None, + "〔本〕", + "〔三〕", + "〔二〕", + "〔安〕", + "〔点〕", + "〔打〕", + "〔盗〕", + "〔勝〕", + "〔敗〕", + None, + "得", + "可", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "丽", + "丸", + "乁", + "𠄢", + "你", + "侮", + "侻", + "倂", + "偺", + "備", + "僧", + "像", + "㒞", + "𠘺", + "免", + "兔", + "兤", + "具", + "𠔜", + "㒹", + "內", + "再", + "𠕋", + "冗", + "冤", + "仌", + "冬", + "况", + "𩇟", + "凵", + "刃", + "㓟", + "刻", + "剆", + "割", + "剷", + "㔕", + "勇", + "勉", + "勤", + "勺", + "包", + "匆", + "北", + "卉", + "卑", + "博", + "即", + "卽", + "卿", + "𠨬", + "灰", + "及", + "叟", + "𠭣", + "叫", + "叱", + "吆", + "咞", + "吸", + "呈", + "周", + "咢", + "哶", + "唐", + "啓", + "啣", + "善", + "喙", + "喫", + "喳", + "嗂", + "圖", + "嘆", + "圗", + "噑", + "噴", + "切", + "壮", + "城", + "埴", + "堍", + "型", + "堲", + "報", + "墬", + "𡓤", + "売", + "壷", + "夆", + "多", + "夢", + "奢", + "𡚨", + "𡛪", + "姬", + "娛", + "娧", + "姘", + "婦", + "㛮", + "㛼", + "嬈", + "嬾", + "𡧈", + "寃", + "寘", + "寧", + "寳", + "𡬘", + "寿", + "将", + "当", + "尢", + "㞁", + "屠", + "屮", + "峀", + "岍", + "𡷤", + "嵃", + "𡷦", + "嵮", + "嵫", + "嵼", + "巡", + "巢", + "㠯", + "巽", + "帨", + "帽", + "幩", + "㡢", + "𢆃", + "㡼", + "庰", + "庳", + "庶", + "廊", + "𪎒", + "廾", + "𢌱", + "舁", + "弢", + "㣇", + "𣊸", + "𦇚", + "形", + "彫", + "㣣", + "徚", + "忍", + "志", + "忹", + "悁", + "㤺", + "㤜", + "悔", + "𢛔", + "惇", + "慈", + "慌", + "慎", + "慌", + "慺", + "憎", + "憲", + "憤", + "憯", + "懞", + "懲", + "懶", + "成", + "戛", + "扝", + "抱", + "拔", + "捐", + "𢬌", + "挽", + "拼", + "捨", + "掃", + "揤", + "𢯱", + "搢", + "揅", + "掩", + "㨮", + "摩", + "摾", + "撝", + "摷", + "㩬", + "敏", + "敬", + "𣀊", + "旣", + "書", + "晉", + "㬙", + "暑", + "㬈", + "㫤", + "冒", + "冕", + "最", + "暜", + "肭", + "䏙", + "朗", + "望", + "朡", + "杞", + "杓", + "𣏃", + "㭉", + "柺", + "枅", + "桒", + "梅", + "𣑭", + "梎", + "栟", + "椔", + "㮝", + "楂", + "榣", + "槪", + "檨", + "𣚣", + "櫛", + "㰘", + "次", + "𣢧", + "歔", + "㱎", + "歲", + "殟", + "殺", + "殻", + "𣪍", + "𡴋", + "𣫺", + "汎", + "𣲼", + "沿", + "泍", + "汧", + "洖", + "派", + "海", + "流", + "浩", + "浸", + "涅", + "𣴞", + "洴", + "港", + "湮", + "㴳", + "滋", + "滇", + "𣻑", + "淹", + "潮", + "𣽞", + "𣾎", + "濆", + "瀹", + "瀞", + "瀛", + "㶖", + "灊", + "災", + "灷", + "炭", + "𠔥", + "煅", + "𤉣", + "熜", + "𤎫", + "爨", + "爵", + "牐", + "𤘈", + "犀", + "犕", + "𤜵", + "𤠔", + "獺", + "王", + "㺬", + "玥", + "㺸", + "瑇", + "瑜", + "瑱", + "璅", + "瓊", + "㼛", + "甤", + "𤰶", + "甾", + "𤲒", + "異", + "𢆟", + "瘐", + "𤾡", + "𤾸", + "𥁄", + "㿼", + "䀈", + "直", + "𥃳", + "𥃲", + "𥄙", + "𥄳", + "眞", + "真", + "睊", + "䀹", + "瞋", + "䁆", + "䂖", + "𥐝", + "硎", + "碌", + "磌", + "䃣", + "𥘦", + "祖", + "𥚚", + "𥛅", + "福", + "秫", + "䄯", + "穀", + "穊", + "穏", + "𥥼", + "𥪧", + "竮", + "䈂", + "𥮫", + "篆", + "築", + "䈧", + "𥲀", + "糒", + "䊠", + "糨", + "糣", + "紀", + "𥾆", + "絣", + "䌁", + "緇", + "縂", + "繅", + "䌴", + "𦈨", + "𦉇", + "䍙", + "𦋙", + "罺", + "𦌾", + "羕", + "翺", + "者", + "𦓚", + "𦔣", + "聠", + "𦖨", + "聰", + "𣍟", + "䏕", + "育", + "脃", + "䐋", + "脾", + "媵", + "𦞧", + "𦞵", + "𣎓", + "𣎜", + "舁", + "舄", + "辞", + "䑫", + "芑", + "芋", + "芝", + "劳", + "花", + "芳", + "芽", + "苦", + "𦬼", + "若", + "茝", + "荣", + "莭", + "茣", + "莽", + "菧", + "著", + "荓", + "菊", + "菌", + "菜", + "𦰶", + "𦵫", + "𦳕", + "䔫", + "蓱", + "蓳", + "蔖", + "𧏊", + "蕤", + "𦼬", + "䕝", + "䕡", + "𦾱", + "𧃒", + "䕫", + "虐", + "虜", + "虧", + "虩", + "蚩", + "蚈", + "蜎", + "蛢", + "蝹", + "蜨", + "蝫", + "螆", + "䗗", + "蟡", + "蠁", + "䗹", + "衠", + "衣", + "𧙧", + "裗", + "裞", + "䘵", + "裺", + "㒻", + "𧢮", + "𧥦", + "䚾", + "䛇", + "誠", + "諭", + "變", + "豕", + "𧲨", + "貫", + "賁", + "贛", + "起", + "𧼯", + "𠠄", + "跋", + "趼", + "跰", + "𠣞", + "軔", + "輸", + "𨗒", + "𨗭", + "邔", + "郱", + "鄑", + "𨜮", + "鄛", + "鈸", + "鋗", + "鋘", + "鉼", + "鏹", + "鐕", + "𨯺", + "開", + "䦕", + "閷", + "𨵷", + "䧦", + "雃", + "嶲", + "霣", + "𩅅", + "𩈚", + "䩮", + "䩶", + "韠", + "𩐊", + "䪲", + "𩒖", + "頋", + "頩", + "𩖶", + "飢", + "䬳", + "餩", + "馧", + "駂", + "駾", + "䯎", + "𩬰", + "鬒", + "鱀", + "鳽", + "䳎", + "䳭", + "鵧", + "𪃎", + "䳸", + "𪄅", + "𪈎", + "𪊑", + "麻", + "䵖", + "黹", + "黾", + "鼅", + "鼏", + "鼖", + "鼻", + "𪘀", + None, + None, + None, + None, + None, + None, + None, +) diff --git a/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/INSTALLER b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/METADATA b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..d30675f566270080d2fc5dce8c7b1d38549b3e70 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/METADATA @@ -0,0 +1,92 @@ +Metadata-Version: 2.4 +Name: lxml +Version: 6.1.1 +Summary: Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API. +Home-page: https://lxml.de/ +Author: lxml dev team +Author-email: lxml@lxml.de +Maintainer: lxml dev team +Maintainer-email: lxml@lxml.de +License: BSD-3-Clause +Project-URL: Source, https://github.com/lxml/lxml +Project-URL: Bug Tracker, https://bugs.launchpad.net/lxml +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Information Technology +Classifier: Programming Language :: Cython +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: C +Classifier: Operating System :: OS Independent +Classifier: Topic :: Text Processing :: Markup :: HTML +Classifier: Topic :: Text Processing :: Markup :: XML +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Requires-Python: >=3.8 +License-File: LICENSE.txt +License-File: LICENSES.txt +Provides-Extra: source +Provides-Extra: cssselect +Requires-Dist: cssselect>=0.7; extra == "cssselect" +Provides-Extra: html5 +Requires-Dist: html5lib; extra == "html5" +Provides-Extra: htmlsoup +Requires-Dist: BeautifulSoup4; extra == "htmlsoup" +Provides-Extra: html-clean +Requires-Dist: lxml_html_clean; extra == "html-clean" +Dynamic: author +Dynamic: author-email +Dynamic: classifier +Dynamic: description +Dynamic: home-page +Dynamic: license +Dynamic: license-file +Dynamic: maintainer +Dynamic: maintainer-email +Dynamic: project-url +Dynamic: provides-extra +Dynamic: requires-python +Dynamic: summary + +lxml is a Pythonic, mature binding for the libxml2 and libxslt libraries. +It provides safe and convenient access to these libraries using the +ElementTree API. + +It extends the ElementTree API significantly to offer support for XPath, +RelaxNG, XML Schema, XSLT, C14N and much more. + +To contact the project, go to the `project home page `_ +or see our bug tracker at https://launchpad.net/lxml + +In case you want to use the current in-development version of lxml, +you can get it from the github repository at +https://github.com/lxml/lxml . Note that this requires Cython to +build the sources, see the build instructions on the project home page. + + +After an official release of a new stable series, bug fixes may become available at +https://github.com/lxml/lxml/tree/lxml-6.1 . +Running ``pip install https://github.com/lxml/lxml/archive/refs/heads/lxml-6.1.tar.gz`` +will install the unreleased branch state as soon as a maintenance branch has been established. +Note that this requires Cython to be installed at an appropriate version for the build. + +6.1.1 (2026-05-18) +================== + +Bugs fixed +---------- + +* The known link attributes in ``lxml.html.defs.link_attrs`` were missing ``xlink:href``, + which can be used for URL bypass attacks in embedded SVG/MathML/etc. content. + https://github.com/fedora-python/lxml_html_clean/security/advisories/GHSA-4jhm-jv67-739f + +* The Linux wheels use a patched libxslt 1.1.43, fixing CVE-2025-7424 and CVE-2025-11731. + +* The Windows wheels use libxslt 1.1.45, fixing CVE-2025-7424 and CVE-2025-11731. + + diff --git a/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/RECORD b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..4cde943594deade9fcce61d5df962e2645794560 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/RECORD @@ -0,0 +1,205 @@ +lxml-6.1.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +lxml-6.1.1.dist-info/METADATA,sha256=qbvABHCIHPEzcHkgBEmV_9YGNyPiA8po1l4vKiWIa8c,3518 +lxml-6.1.1.dist-info/RECORD,, +lxml-6.1.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +lxml-6.1.1.dist-info/WHEEL,sha256=SQkZSuBb7wXZ5yRnujKqNRqvyHEU3tCjYXOOP_GWtwc,152 +lxml-6.1.1.dist-info/licenses/LICENSE.txt,sha256=j8K1aBM1FuRoRdIUeRet7uFkjnCumrXtbFQXr-9M6FU,1507 +lxml-6.1.1.dist-info/licenses/LICENSES.txt,sha256=QdSd1AaqDhVIptXyGjDWv2OLPNlutyid00jYPtLkA5I,1514 +lxml-6.1.1.dist-info/top_level.txt,sha256=NjD988wqaKq512nshNdLt-uDxsjkp4Bh51m6N-dhUrk,5 +lxml/ElementInclude.py,sha256=PSLeZFvCa76WHJulPLxcZXJtCI2-4dK2CtqPRiYOAQg,8560 +lxml/__init__.py,sha256=OKLD7sP5BpJ5x3k2-I1dl5ozoGx3xVvO3ml2ISuEZQg,574 +lxml/__pycache__/ElementInclude.cpython-313.pyc,, +lxml/__pycache__/__init__.cpython-313.pyc,, +lxml/__pycache__/_elementpath.cpython-313.pyc,, +lxml/__pycache__/builder.cpython-313.pyc,, +lxml/__pycache__/cssselect.cpython-313.pyc,, +lxml/__pycache__/doctestcompare.cpython-313.pyc,, +lxml/__pycache__/pyclasslookup.cpython-313.pyc,, +lxml/__pycache__/sax.cpython-313.pyc,, +lxml/__pycache__/usedoctest.cpython-313.pyc,, +lxml/_elementpath.cpython-313-x86_64-linux-gnu.so,sha256=C_Ixzf4Bix-CXQDThRGoeJwHDO3lX5jFsRS2YVGzTik,209096 +lxml/_elementpath.py,sha256=b80hM3ndAkTtRX6v54za3LkkAqCcd0700BbMPZHnTBU,10959 +lxml/apihelpers.pxi,sha256=CugqfQgyn6aFtwp7eH144RUpV2v1ItxoalhJcFbORTQ,64604 +lxml/builder.cpython-313-x86_64-linux-gnu.so,sha256=_J7rIdOd5f-MYYNI1PQGMxfsiqiNOe4rprylpdlUSwI,116784 +lxml/builder.py,sha256=sL7iKT8NXVmpO_p2p2nHTjShjZOGMFLgFzgbhUFl68g,8487 +lxml/classlookup.pxi,sha256=Tax8Vhbm5C6UCjgmRFsYjW0pFHxIuTthH1MOgASDLgc,22435 +lxml/cleanup.pxi,sha256=ZNEpbv7qx_ICPzsxhCaMUHCOfiznOoZ_u3jlYXHAuh4,8454 +lxml/cssselect.py,sha256=_wZdX-B9p5MeIYABmENIYRWEkwXwX-7jO8Dkf-1rUZU,3306 +lxml/debug.pxi,sha256=KTcpR8-slUYvmIPbE35GoHDNTb-gjTEvD7bw6LltM4c,1125 +lxml/docloader.pxi,sha256=fcoVd_mwTFZhjItCv_TfwXpgVek7JlvFObSfRxmVqs0,5790 +lxml/doctestcompare.py,sha256=40EDnkwpcvW86qNa86990OXF42xdHaosSZoiBsEjkzU,17731 +lxml/dtd.pxi,sha256=IAKkmA4ZoC68sqAWcTqoS8jEGYcPQrVMCZgn4iLBYko,15281 +lxml/etree.cpython-313-x86_64-linux-gnu.so,sha256=ULxScNYxXkoou6JfkhGtuPXtKtp95iejbI6KFaSH9QE,5243760 +lxml/etree.h,sha256=hWgHumv7DeTpH0nHhbsCIe693cIcimtLabExxwBxR-c,9792 +lxml/etree.pyx,sha256=zgobCeGw6lw902OA0FWuwlOpqA_0eZeLM2baWi1pzio,138216 +lxml/etree_api.h,sha256=UG16a1ThLt259FSG3vUy6ka6OIat-6LLJyVBLz6d3OM,17710 +lxml/extensions.pxi,sha256=saxxc1mZyA8WdwIIL90B4lQb351j8-G7ttLPI5k-8Hw,32248 +lxml/html/ElementSoup.py,sha256=s_dLobLMuKn2DhexR-iDXdZrMFg1RjLy1feHsIeZMpw,320 +lxml/html/__init__.py,sha256=CC5WdsvSptZhr9MZya1qsL6JKVbviYdrHOhXrGhmORg,64425 +lxml/html/__pycache__/ElementSoup.cpython-313.pyc,, +lxml/html/__pycache__/__init__.cpython-313.pyc,, +lxml/html/__pycache__/_diffcommand.cpython-313.pyc,, +lxml/html/__pycache__/_difflib.cpython-313.pyc,, +lxml/html/__pycache__/_html5builder.cpython-313.pyc,, +lxml/html/__pycache__/_setmixin.cpython-313.pyc,, +lxml/html/__pycache__/builder.cpython-313.pyc,, +lxml/html/__pycache__/clean.cpython-313.pyc,, +lxml/html/__pycache__/defs.cpython-313.pyc,, +lxml/html/__pycache__/diff.cpython-313.pyc,, +lxml/html/__pycache__/formfill.cpython-313.pyc,, +lxml/html/__pycache__/html5parser.cpython-313.pyc,, +lxml/html/__pycache__/soupparser.cpython-313.pyc,, +lxml/html/__pycache__/usedoctest.cpython-313.pyc,, +lxml/html/_diffcommand.py,sha256=kz_7EP9PmYWuczlZcGiw74_rG0eTKvQ2lrO0rkiwlYE,2081 +lxml/html/_difflib.cpython-313-x86_64-linux-gnu.so,sha256=ZnVzi9sYhgb-l87GjnhgA8A4svt4yovh9wCVFMiis14,533400 +lxml/html/_difflib.py,sha256=GgH_jVrZQC8tI8WV_lFZQsXFJ3mOTAPup1zjBJNvkPo,84954 +lxml/html/_html5builder.py,sha256=NLaT-Ev-aBgJpeQl-6ZbJChLZK5GV-znDkHOJD5VQC4,3230 +lxml/html/_setmixin.py,sha256=8IFIOLmVz0G-XzsD2tCEkSFWO-dgPBHgvHufC8ni67s,1188 +lxml/html/builder.py,sha256=Uz3r5uiuCdoN0UPa7ngoLMwAadVIhslzGvlRPGigY_M,6187 +lxml/html/clean.py,sha256=WU0KfXAyjDFn2ojE7T1dfk4cl_FduBu5ZbFSTc9Up_o,503 +lxml/html/defs.py,sha256=6z6B1ws_2l9zgw3DL7ML62wwBYe6m5reJp6m3uTKptQ,5398 +lxml/html/diff.cpython-313-x86_64-linux-gnu.so,sha256=9dK-bJ6Frdbw79kK4lXdR_jKJ8UddeIjWN9FZe7TRFA,361432 +lxml/html/diff.py,sha256=Za0By-yeYlQEjUu7m7xKB288kKiy8VBS5gT0RPOaFY0,32989 +lxml/html/formfill.py,sha256=umgk0BbkAI1W6q9musFbL-cDnI_aap2NsLBJqk0UmVI,9681 +lxml/html/html5parser.py,sha256=dnyC4cqHxywjZSzk0mu2L7THTZjxhg4yF4pncjusa_w,8634 +lxml/html/soupparser.py,sha256=xo8VvNeOEb-SChuXLKCRECh8J7HBiJLE9sAbEskoUUQ,10197 +lxml/html/usedoctest.py,sha256=tPlmVz4KK1GRKV5DJLrdVECeqsT9PlDzSqqTodVi5s0,249 +lxml/includes/__init__.pxd,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +lxml/includes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +lxml/includes/__pycache__/__init__.cpython-313.pyc,, +lxml/includes/c14n.pxd,sha256=DBQcOJ0c_YS245ohMb8fmuEC1kFyv1LrNY_8Mf-syZg,1110 +lxml/includes/config.pxd,sha256=H6Mrl8It21hzRI2hzMId9W48QqkYYkoLT4dniLNmdTw,96 +lxml/includes/dtdvalid.pxd,sha256=Nv0OykjYehv2lO-Zj--q6jS3TAC_dvQVPSgPMuse1NM,689 +lxml/includes/etree_defs.h,sha256=h_UjJTmNUqPyKNNrWB9hxmt6v4CF7_83XVY8dOfxqW0,14524 +lxml/includes/etreepublic.pxd,sha256=Bn4d3JkWPqXputXqI-eJ0xmPrwNFPTfDCa7axgjB7FM,10184 +lxml/includes/extlibs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +lxml/includes/extlibs/__pycache__/__init__.cpython-313.pyc,, +lxml/includes/extlibs/libcharset.h,sha256=GA0FumrbNI4VDGlzq3lf5CLaCwXgn4unw2l0btGQFwI,1510 +lxml/includes/extlibs/localcharset.h,sha256=Z_AagaQeq0aDE7NPsVOqEf4nO4KcUp46ggo4d0ONIOQ,6338 +lxml/includes/extlibs/zconf.h,sha256=BxihG-sylbNF-ymmO0S2VLKC2CxM0FE_YiVYfyspuLs,16921 +lxml/includes/extlibs/zlib.h,sha256=gYZn1qtqN_50acsGp_DLLCyy8slIoD5azPGkp0vzAgo,103848 +lxml/includes/htmlparser.pxd,sha256=9uASkP5dU7OE2lCOLT-z2e01qSbFlp4ehgwdostF_qk,2802 +lxml/includes/libexslt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +lxml/includes/libexslt/__pycache__/__init__.cpython-313.pyc,, +lxml/includes/libexslt/exslt.h,sha256=eSW5tMJAewSUANLqk7AGEiU8b2BbCNRyauHnez7nKSU,3114 +lxml/includes/libexslt/exsltconfig.h,sha256=QHxzEbRlv_h0USBvpr0Zrl0Muzlc71VCrvgR6lqnLEY,1172 +lxml/includes/libexslt/exsltexports.h,sha256=1Jm9KTXm2FUUJIZ6V6-Uw55yG0BMULX3_goyxDd2LL8,1077 +lxml/includes/libxml/HTMLparser.h,sha256=sU4xGqj-vBtEvzlxA3hBPWJboifvkc4F1hynKXmsl3k,9569 +lxml/includes/libxml/HTMLtree.h,sha256=Q7UBKFbQ8fx4d_dMnmR6ay8JmfOhopFkDp2B63YkLDU,3517 +lxml/includes/libxml/SAX.h,sha256=SFnG27EFrYGUB9HDL_xSIGBwEns5pl07rApXWThFZFM,386 +lxml/includes/libxml/SAX2.h,sha256=RfFP5o3le-Rg8bnA2GW7L7L9_pfXCs3TieODcv1DTWY,4240 +lxml/includes/libxml/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +lxml/includes/libxml/__pycache__/__init__.cpython-313.pyc,, +lxml/includes/libxml/c14n.h,sha256=BSBXw6nIZutC8mWvbRrLLmoWjw3wRt-nM93vjXGMCm8,2742 +lxml/includes/libxml/catalog.h,sha256=H9ssTCaBjtDqc-AZqCk1R7h8F2iD9szqLjJyHpaczXg,4633 +lxml/includes/libxml/chvalid.h,sha256=TZcceNp6Cw0QlYwIqK9GxyYqL5UiAjpQyjt_yrZGTQE,5087 +lxml/includes/libxml/debugXML.h,sha256=XXRNI39gJW7bGRC4SzE4ad-SJ906BsUGz3AwOtkKuS4,1667 +lxml/includes/libxml/dict.h,sha256=SweaPGMtTTf4je6dNTIoEzcfEvpsAT9_PhR7FC0K-rQ,1770 +lxml/includes/libxml/encoding.h,sha256=haL7ratww2wkIERGmtwUqU2BbTVe52FZFU7MmrOpsPk,9623 +lxml/includes/libxml/entities.h,sha256=LEOCA826-0f8dhRJzC_2hvUVsSH7lKQjrea9hSTdBbo,4419 +lxml/includes/libxml/globals.h,sha256=NH8zyRI5cXJJGp5k2aLxOm-reJEGOFX6LYP82GBXRlY,583 +lxml/includes/libxml/hash.h,sha256=KIIpAYKBfGUU3ydWhGehUyfuauZz_Ps0gyambzQo_rc,7017 +lxml/includes/libxml/list.h,sha256=oh7iJNQajRA_cHsNk9CcFPYkaW2smf4J_MpedPPjC4k,3128 +lxml/includes/libxml/nanoftp.h,sha256=22PBtWhJueYLFvwukt4oFooRct_xJA83hbluHRBNXUM,302 +lxml/includes/libxml/nanohttp.h,sha256=bLbzYjAyAKmP3ComMOPH6XaUImu6bNAESF1HrVtRve0,2124 +lxml/includes/libxml/parser.h,sha256=Uq7-ce55UUAsvo4n6CiBlNQpmowewvWhOsQtgGM1UQ8,48498 +lxml/includes/libxml/parserInternals.h,sha256=8_Wr6UgRzm8BRn1RPLxyBkw6BagAdDvVqMA_e181_EI,14539 +lxml/includes/libxml/relaxng.h,sha256=VXZ74r5Yja06KqypdBHc8neDwPxQ2aMrsWHSdRt5oi4,5991 +lxml/includes/libxml/schemasInternals.h,sha256=V8M4In3zf24EX55Yt4dcfxwp7NpHGYViKnLKwtyrPJ4,26233 +lxml/includes/libxml/schematron.h,sha256=8EhPDhvtlMxl9e0C5rSbEruOvzJS5BC_OOFbq9RXZnY,4255 +lxml/includes/libxml/threads.h,sha256=mT3CgK4lXK7-NDnUOFXqYuCK6fyY70S3BsHF-TnT45k,1619 +lxml/includes/libxml/tree.h,sha256=zTRLt6h5x6ApyeXgs90CKQZSAl2hKm7b5NxtPKUQFAE,36106 +lxml/includes/libxml/uri.h,sha256=J9teJHme5z883c4twF5oImEYY-E3xSvhdSGpyRVtvIg,2855 +lxml/includes/libxml/valid.h,sha256=By61IbPvk_eLux7a8x0mOaly7oclFaSGaFE8b2xZcUE,13226 +lxml/includes/libxml/xinclude.h,sha256=K3I5jhw2zAMj26LuRNZc15Bwv2JE2hWxwVn4TCqv2b4,3258 +lxml/includes/libxml/xlink.h,sha256=TVLOkISrcKDelo9n_XIUyPiStDYa8NxuF2dz70aBFCI,5062 +lxml/includes/libxml/xmlIO.h,sha256=FvbuMYTy1-S5PScabE03wz0oWKf626pmXvOPZNuLm-w,11948 +lxml/includes/libxml/xmlautomata.h,sha256=7Sc3YgPz1ZIBKCHPSxs5oAwJEZWQ1RT2kyUw85pUtmU,4004 +lxml/includes/libxml/xmlerror.h,sha256=mMfltMxUza6kiSBfP2QfnY3UlMP_rEXKfX0wruBLl4A,37561 +lxml/includes/libxml/xmlexports.h,sha256=IyV3AMeQVbOl0wkjlnNX4B8WUZ-5GNKQmxZc6-maWUU,2025 +lxml/includes/libxml/xmlmemory.h,sha256=m7wGvVMxNzZiuOAo3vkjxaVWstc8aQLzb6obbjPsebE,4658 +lxml/includes/libxml/xmlmodule.h,sha256=ERUHUmDdZRmh6NjLYWUpse51rLWR8qNjPHOtdgmlLF0,1198 +lxml/includes/libxml/xmlreader.h,sha256=BAHinlSOTXX3DEax9BniaIIPAXJyLGfzym9R-27LCcU,12387 +lxml/includes/libxml/xmlregexp.h,sha256=_q6C1XRy8DS3kSmLbEKpvkKQciTgjTJgGc_zUQ6m22M,2632 +lxml/includes/libxml/xmlsave.h,sha256=zcEQr9sO5CsFrnoOLshhdsqMEr8k4AeFhbkYyNfO9Fs,2934 +lxml/includes/libxml/xmlschemas.h,sha256=5AfLnYUcfmxHRzg0dVpdHig--4ui1-XDwDgpKGDKCiU,7067 +lxml/includes/libxml/xmlschemastypes.h,sha256=MYwlGmoKAo3lHRaaKgnCXiLmPT9KRjdxyCJ7TEyZ6jM,4583 +lxml/includes/libxml/xmlstring.h,sha256=d5PpqxP1I1sfmCUHvVJtjoC9h7hLHcAAQ5ok_Rtf50I,5271 +lxml/includes/libxml/xmlunicode.h,sha256=8sq3wEW2AiyTCuc3ZceOEkce7lfrI7VnkRfwEQgc6pU,278 +lxml/includes/libxml/xmlversion.h,sha256=oVpaE_xbttaeZNFKSuSfcLOceWz7LQgKP71Z1msXZNo,5112 +lxml/includes/libxml/xmlwriter.h,sha256=BEUwYNKx3xymDE9vepksEK7yVq9SXYm1d2pQnzlPy90,20688 +lxml/includes/libxml/xpath.h,sha256=CQv6X_pRhuXoCVpqoDXYB7FfusLK7AuPxCNigwhNYAA,16156 +lxml/includes/libxml/xpathInternals.h,sha256=mc9B5tdpfssyz_NPUzww6dKuWCtBybBiBRJkTe4AE4U,18504 +lxml/includes/libxml/xpointer.h,sha256=DAxMsfPp2SSZgXFrPbxBA84RwTMRf35Qg_LBbUzPQhA,1026 +lxml/includes/libxslt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +lxml/includes/libxslt/__pycache__/__init__.cpython-313.pyc,, +lxml/includes/libxslt/attributes.h,sha256=qKwzfGf7r89esLC65s96iYJWRA-s-Ezss2_V6Mmo1hk,957 +lxml/includes/libxslt/documents.h,sha256=kBihgH5pqRvFalhm_fOFHtJTFhTpBcm681yT5dxgwfw,2704 +lxml/includes/libxslt/extensions.h,sha256=W5UMyJqUP_1zt6sXZ0mgc0gAIwDJrZ8gjByhyrWqvd8,6899 +lxml/includes/libxslt/extra.h,sha256=6X3Wu3NdPtrlqz-Koo7dB-rccnnszi6j3zg599gTByg,1640 +lxml/includes/libxslt/functions.h,sha256=fc4CZj-9KeBHzO9-WWU_bNqmaEZAz3n7NNwClIBXk14,1972 +lxml/includes/libxslt/imports.h,sha256=18kIjoGqdFXR63Ce3ZtzxsTiYV3XGKpchYakMUPDuUI,1840 +lxml/includes/libxslt/keys.h,sha256=16v25VEluS7jYhgg6gYFwVxgGMn-1ctnlhhWWT4RcBY,1155 +lxml/includes/libxslt/namespaces.h,sha256=VofSn2Kkn-a5JyRKCmY3jPp7amQy3n09vzy0KUQt4q0,1666 +lxml/includes/libxslt/numbersInternals.h,sha256=Eg5gYZ5p3h0_e5wyI61S-0E6_ArVJzv0yr63j6BU2fc,2019 +lxml/includes/libxslt/pattern.h,sha256=tJ-BPfs9UYgiZMMoQZbhij3g7xVppYq7TrrOu25eR7Q,2110 +lxml/includes/libxslt/preproc.h,sha256=D_LjEdHhsdyBnEAvflnwFgoR4hGUb72kgEhXkkmPRsw,896 +lxml/includes/libxslt/security.h,sha256=fUD1cy_WxFCTvTNAF0WOQIU4p5CNWn1LHFyZJd-Fx5U,2652 +lxml/includes/libxslt/templates.h,sha256=bnt6Jqui6KU5pNUdMNPbQZkZ5d-VTWqC0TMGkOlVoIo,2268 +lxml/includes/libxslt/transform.h,sha256=ICT7meUV0OTAx27WaKVrKj-aUmR9LSpTNaOAJd2UStg,6311 +lxml/includes/libxslt/variables.h,sha256=cQAgPe4QCcK2uKbWg7Iz-9peM9xWGm7m3M6jQm0sjIA,3143 +lxml/includes/libxslt/xslt.h,sha256=wmFx2Q31Pd8Iq2phAQpY9J3QQatb8lWg3gABtqKFgEw,1964 +lxml/includes/libxslt/xsltInternals.h,sha256=2EbEKYmnYZq0HjGnUMAlpqnqZJurRXzjlgk5Js1WYaY,57949 +lxml/includes/libxslt/xsltconfig.h,sha256=cV5scdRK6xmOHeOg3OCw6hBfcQ_nrtNs_tKefX67304,2910 +lxml/includes/libxslt/xsltexports.h,sha256=1-luH-0bCIgBAlKAXhV-dqHBfwOAQNDamiYbxIlTf0k,1124 +lxml/includes/libxslt/xsltlocale.h,sha256=ppxGEmJfZIJgwRQzCM0_77p9WNekEWq1NrdYZrQl4IE,942 +lxml/includes/libxslt/xsltutils.h,sha256=1eguYgR9-jeNOVlBUktHboaq-VLX6JXraO80TfbARKM,9085 +lxml/includes/lxml-version.h,sha256=QCE72dSV-W29efxzYz7IG_wH5leX_x0mZdFbQmH2C4w,71 +lxml/includes/relaxng.pxd,sha256=HzHlQ6mCcf_tj_JZ9NAVJTVAv8ScCkE8Ifq15y3bS0c,2615 +lxml/includes/schematron.pxd,sha256=Hob7xh-K-MKqp7WiG8thMagf5EkQzmgfi4ds0EF91JA,1604 +lxml/includes/tree.pxd,sha256=XApzMRy_LSqCtQ-OTS-vNSW7CT_OWstybfIT2H84LsA,20179 +lxml/includes/uri.pxd,sha256=3vOXw6AbSPxAM9uo71T1qnfx-wd9ezXLDQtWsb2zX0I,145 +lxml/includes/xinclude.pxd,sha256=CuO_XZNB6E2JK1qXXWn11APrjFQV5kA6SMyb77WZn0A,804 +lxml/includes/xmlerror.pxd,sha256=OQqayytkV0NigAPbsQCCcvmy7luRe0XhVzpTdzJjP3g,58837 +lxml/includes/xmlparser.pxd,sha256=eDGyU5kZyNVksK0dUhMIi7rnE-LSevXsqyl72v99Ess,13730 +lxml/includes/xmlschema.pxd,sha256=OLZPd2WDJyopiXJJyo-dAyyYHaeSYFiMAI4tqIiv-Ik,1702 +lxml/includes/xpath.pxd,sha256=e8-ZYUbRG7N1mHETAlknJ_QqAteOosrYLRgpH-OsTkg,5603 +lxml/includes/xslt.pxd,sha256=4yl3pOu7pAvsx5Tc-W4IWCoB8wgtSSR62HI1jqu6jko,8241 +lxml/isoschematron/__init__.py,sha256=uauerYeKTlWFCJSqieIHhF5l6rYV2myeEJ0Imd1LzRc,13274 +lxml/isoschematron/__pycache__/__init__.cpython-313.pyc,, +lxml/isoschematron/resources/rng/iso-schematron.rng,sha256=VsWxPyi3iViJDDbjJJw0wWkEHkLrz9zoCA8zJLor9N4,18337 +lxml/isoschematron/resources/xsl/RNG2Schtrn.xsl,sha256=ObebsB8Wt-d3uIA_U5NU85TpnQ3PxPX38TdOAqosMac,3172 +lxml/isoschematron/resources/xsl/XSD2Schtrn.xsl,sha256=QweRrIIM-zFcgg98GXA2CaWfIbgVE0XKEeYSfvv67A0,4563 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_abstract_expand.xsl,sha256=xSZ_Ekq_I-62ZpiE5AqYYHwFW_qh855zt9V4_s7rbkY,11703 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_dsdl_include.xsl,sha256=x42QJ-dxQ1waPzydsCoQnp2Xj15y53nW43O7BuoDRHk,39957 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_schematron_message.xsl,sha256=Tr9BnO6pzjVWwhqJfm10UlvAy95EgfSCz2iMlrVGT6Q,2015 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_schematron_skeleton_for_xslt1.xsl,sha256=ue8q_88X4e_jsJizo31GRNBxNhdxkEE9fY20oq0Iqwk,71764 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_svrl_for_xslt1.xsl,sha256=BBAdsVSi5zAzeGepuN6gS1saQINDqITXKplmmj4dTWg,20382 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/readme.txt,sha256=OGLiFswuLJEW5EPYKOeoauuCJFEtVa6jyzBE1OcJI98,3310 +lxml/iterparse.pxi,sha256=rERnJ9D0Cvn87YDMSq8V880DKV2WthXs5BXSSY2Baj4,16845 +lxml/lxml.etree.h,sha256=hWgHumv7DeTpH0nHhbsCIe693cIcimtLabExxwBxR-c,9792 +lxml/lxml.etree_api.h,sha256=O_s_RIicKl7_h5U0aKu9dKSzJCTYSDJI6jlX7b-2v0Q,17715 +lxml/nsclasses.pxi,sha256=5pzNBhBtlqObPdThL9QIGRs1Dxj1qnr0PyXuTCURqTg,9129 +lxml/objectify.cpython-313-x86_64-linux-gnu.so,sha256=W-NrFzOKF2UAsFooyolhmjmTlWAP6xMMS3-o2avGi84,2920888 +lxml/objectify.pyx,sha256=8lIjgklL3LJ3PYAcWZ5hPLuLFa7oLqEspco4bSz4EI4,75850 +lxml/objectpath.pxi,sha256=s5TNG2-EbaWWKLFAiX303B95zK_Ui8ausB__3QvFFGw,11450 +lxml/parser.pxi,sha256=wv8ODQRXZk0r0ZVHY4zsF2vLbL_SFGwYIFSKcWAVCdQ,85821 +lxml/parsertarget.pxi,sha256=v1PidxRaG5giwXcTDkpBI7PDFmsZuOcK0y9LdkQaY8M,6326 +lxml/proxy.pxi,sha256=LnqqSZVUsJ5MnuVrhYrLwMbAKWuv1eOn-XYslut6CLM,24015 +lxml/public-api.pxi,sha256=yVPuZV2LwN5Ki73mD85V9hnUXxOs5VMeFMLtQMNf0Qk,6756 +lxml/pyclasslookup.py,sha256=gLD1HM2HtITYYiGzjEOewSwbB7XkVx_NZv_quCt79Oc,92 +lxml/readonlytree.pxi,sha256=n2MRjz81EADWBXJx0miosN8PJ4i5y3guQJT_RZF6qKg,19008 +lxml/relaxng.pxi,sha256=3OQ-fZMzP-KF5vM6HTozT_9ee3J0DJnpj9RcHC8LoMw,6339 +lxml/sax.cpython-313-x86_64-linux-gnu.so,sha256=sld5taFW6xBO4HBfpkHKVbtLPwcPJJEc4nBq7C0KRmU,182128 +lxml/sax.py,sha256=5ryEnOMIMeLpLel-PFpszMHq99mWNmsnGrMvLWIm1UM,9708 +lxml/saxparser.pxi,sha256=yV67FWD60DfmFi7hbV_eLyJIMu7X2MOfV2g4X99iUR4,33538 +lxml/schematron.pxi,sha256=F2OHKZUl57-byBk_wWtPTnHZ1fwlj0FtwG3VuGtG-UY,6064 +lxml/serializer.pxi,sha256=iIXfechFHfvFs2sTk7wMIy3sDJxmaMPbNO33mkLLBUE,68063 +lxml/usedoctest.py,sha256=qRgZKQVcAZcl-zN0AIXVJnOsETUXz2nPXkxuzs1lGgk,230 +lxml/xinclude.pxi,sha256=7eBrI_OK47mmrHQ0ixbixRI8pKqQ1nwkMV-OmKUVlD4,2456 +lxml/xmlerror.pxi,sha256=MAQv92NXFoaMOzT8WeeM0L3YyksFoo48tpz3KZHTqcY,50234 +lxml/xmlid.pxi,sha256=5zf9oR6bsCtavGiOmilNyHqYwgG_bnrIabSd2SURtm0,6073 +lxml/xmlschema.pxi,sha256=mumNoHni5S3BQPtcmOHRd61KRaVWu4eOie2wQeB0e6E,8490 +lxml/xpath.pxi,sha256=aqW24V817dUxps4Gnc8h7Tm3QVlITKvxU5_9WgJUIFg,19132 +lxml/xslt.pxi,sha256=lLVwvlhMKL970rVqThP6Jnn79L3MK9UG0DGnXnNy7YI,36356 +lxml/xsltext.pxi,sha256=TImDiAPlAezC07P7RY1N9YChA7AuKFH-G53hXdel9yc,11088 diff --git a/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/REQUESTED b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/WHEEL b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..cae480d7e9313ac98aab68854862b78db147fd3f --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: setuptools (82.0.1) +Root-Is-Purelib: false +Tag: cp313-cp313-manylinux_2_26_x86_64 +Tag: cp313-cp313-manylinux_2_28_x86_64 + diff --git a/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/licenses/LICENSE.txt b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/licenses/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..0bdf03913ffdfd09be5a524c9303bd54a132d450 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/licenses/LICENSE.txt @@ -0,0 +1,31 @@ +BSD 3-Clause License + +Copyright (c) 2004 Infrae. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + 3. Neither the name of Infrae nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INFRAE OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/licenses/LICENSES.txt b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/licenses/LICENSES.txt new file mode 100644 index 0000000000000000000000000000000000000000..9f97c18aa0b162ffcc3e745d80efb9def1c82515 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/licenses/LICENSES.txt @@ -0,0 +1,29 @@ +lxml is copyright Infrae and distributed under the BSD license (see +doc/licenses/BSD.txt), with the following exceptions: + +Some code, such a selftest.py, selftest2.py and +src/lxml/_elementpath.py are derived from ElementTree and +cElementTree. See doc/licenses/elementtree.txt for the license text. + +lxml.cssselect and lxml.html are copyright Ian Bicking and distributed +under the BSD license (see doc/licenses/BSD.txt). + +test.py, the test-runner script, is GPL and copyright Shuttleworth +Foundation. See doc/licenses/GPL.txt. It is believed the unchanged +inclusion of test.py to run the unit test suite falls under the +"aggregation" clause of the GPL and thus does not affect the license +of the rest of the package. + +The isoschematron implementation uses several XSL and RelaxNG resources: + * The (XML syntax) RelaxNG schema for schematron, copyright International + Organization for Standardization (see + src/lxml/isoschematron/resources/rng/iso-schematron.rng for the license + text) + * The skeleton iso-schematron-xlt1 pure-xslt schematron implementation + xsl stylesheets, copyright Rick Jelliffe and Academia Sinica Computing + Center, Taiwan (see the xsl files here for the license text: + src/lxml/isoschematron/resources/xsl/iso-schematron-xslt1/) + * The xsd/rng schema schematron extraction xsl transformations are unlicensed + and copyright the respective authors as noted (see + src/lxml/isoschematron/resources/xsl/RNG2Schtrn.xsl and + src/lxml/isoschematron/resources/xsl/XSD2Schtrn.xsl) diff --git a/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/top_level.txt b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab90481d5d75200bdc779014d93d69ff85bf9742 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml-6.1.1.dist-info/top_level.txt @@ -0,0 +1 @@ +lxml diff --git a/.venv/lib/python3.13/site-packages/lxml/ElementInclude.py b/.venv/lib/python3.13/site-packages/lxml/ElementInclude.py new file mode 100644 index 0000000000000000000000000000000000000000..21884336f534cd2013165934111146684c9909cf --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/ElementInclude.py @@ -0,0 +1,244 @@ +# +# ElementTree +# $Id: ElementInclude.py 1862 2004-06-18 07:31:02Z Fredrik $ +# +# limited xinclude support for element trees +# +# history: +# 2003-08-15 fl created +# 2003-11-14 fl fixed default loader +# +# Copyright (c) 2003-2004 by Fredrik Lundh. All rights reserved. +# +# fredrik@pythonware.com +# http://www.pythonware.com +# +# -------------------------------------------------------------------- +# The ElementTree toolkit is +# +# Copyright (c) 1999-2004 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Secret Labs AB or the author not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD +# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- +# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR +# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# -------------------------------------------------------------------- + +""" +Limited XInclude support for the ElementTree package. + +While lxml.etree has full support for XInclude (see +`etree.ElementTree.xinclude()`), this module provides a simpler, pure +Python, ElementTree compatible implementation that supports a simple +form of custom URL resolvers. +""" + +from lxml import etree +try: + from urlparse import urljoin + from urllib2 import urlopen +except ImportError: + # Python 3 + from urllib.parse import urljoin + from urllib.request import urlopen + +XINCLUDE = "{http://www.w3.org/2001/XInclude}" + +XINCLUDE_INCLUDE = XINCLUDE + "include" +XINCLUDE_FALLBACK = XINCLUDE + "fallback" +XINCLUDE_ITER_TAG = XINCLUDE + "*" + +# For security reasons, the inclusion depth is limited to this read-only value by default. +DEFAULT_MAX_INCLUSION_DEPTH = 6 + + +## +# Fatal include error. + +class FatalIncludeError(etree.LxmlSyntaxError): + pass + + +class LimitedRecursiveIncludeError(FatalIncludeError): + pass + + +## +# ET compatible default loader. +# This loader reads an included resource from disk. +# +# @param href Resource reference. +# @param parse Parse mode. Either "xml" or "text". +# @param encoding Optional text encoding. +# @return The expanded resource. If the parse mode is "xml", this +# is an ElementTree instance. If the parse mode is "text", this +# is a Unicode string. If the loader fails, it can return None +# or raise an IOError exception. +# @throws IOError If the loader fails to load the resource. + +def default_loader(href, parse, encoding=None): + file = open(href, 'rb') + if parse == "xml": + data = etree.parse(file).getroot() + else: + data = file.read() + if not encoding: + encoding = 'utf-8' + data = data.decode(encoding) + file.close() + return data + + +## +# Default loader used by lxml.etree - handles custom resolvers properly +# + +def _lxml_default_loader(href, parse, encoding=None, parser=None): + if parse == "xml": + data = etree.parse(href, parser).getroot() + else: + if "://" in href: + f = urlopen(href) + else: + f = open(href, 'rb') + data = f.read() + f.close() + if not encoding: + encoding = 'utf-8' + data = data.decode(encoding) + return data + + +## +# Wrapper for ET compatibility - drops the parser + +def _wrap_et_loader(loader): + def load(href, parse, encoding=None, parser=None): + return loader(href, parse, encoding) + return load + + +## +# Expand XInclude directives. +# +# @param elem Root element. +# @param loader Optional resource loader. If omitted, it defaults +# to {@link default_loader}. If given, it should be a callable +# that implements the same interface as default_loader. +# @param base_url The base URL of the original file, to resolve +# relative include file references. +# @param max_depth The maximum number of recursive inclusions. +# Limited to reduce the risk of malicious content explosion. +# Pass None to disable the limitation. +# @throws LimitedRecursiveIncludeError If the {@link max_depth} was exceeded. +# @throws FatalIncludeError If the function fails to include a given +# resource, or if the tree contains malformed XInclude elements. +# @throws IOError If the function fails to load a given resource. +# @returns the node or its replacement if it was an XInclude node + +def include(elem, loader=None, base_url=None, + max_depth=DEFAULT_MAX_INCLUSION_DEPTH): + if max_depth is None: + max_depth = -1 + elif max_depth < 0: + raise ValueError("expected non-negative depth or None for 'max_depth', got %r" % max_depth) + + if base_url is None: + if hasattr(elem, 'getroot'): + tree = elem + elem = elem.getroot() + else: + tree = elem.getroottree() + if hasattr(tree, 'docinfo'): + base_url = tree.docinfo.URL + elif hasattr(elem, 'getroot'): + elem = elem.getroot() + _include(elem, loader, base_url, max_depth) + + +def _include(elem, loader=None, base_url=None, + max_depth=DEFAULT_MAX_INCLUSION_DEPTH, _parent_hrefs=None): + if loader is not None: + load_include = _wrap_et_loader(loader) + else: + load_include = _lxml_default_loader + + if _parent_hrefs is None: + _parent_hrefs = set() + + parser = elem.getroottree().parser + + include_elements = list( + elem.iter(XINCLUDE_ITER_TAG)) + + for e in include_elements: + if e.tag == XINCLUDE_INCLUDE: + # process xinclude directive + href = urljoin(base_url, e.get("href")) + parse = e.get("parse", "xml") + parent = e.getparent() + if parse == "xml": + if href in _parent_hrefs: + raise FatalIncludeError( + "recursive include of %r detected" % href + ) + if max_depth == 0: + raise LimitedRecursiveIncludeError( + "maximum xinclude depth reached when including file %s" % href) + node = load_include(href, parse, parser=parser) + if node is None: + raise FatalIncludeError( + "cannot load %r as %r" % (href, parse) + ) + node = _include(node, loader, href, max_depth - 1, {href} | _parent_hrefs) + if e.tail: + node.tail = (node.tail or "") + e.tail + if parent is None: + return node # replaced the root node! + parent.replace(e, node) + elif parse == "text": + text = load_include(href, parse, encoding=e.get("encoding")) + if text is None: + raise FatalIncludeError( + "cannot load %r as %r" % (href, parse) + ) + predecessor = e.getprevious() + if predecessor is not None: + predecessor.tail = (predecessor.tail or "") + text + elif parent is None: + return text # replaced the root node! + else: + parent.text = (parent.text or "") + text + (e.tail or "") + parent.remove(e) + else: + raise FatalIncludeError( + "unknown parse type in xi:include tag (%r)" % parse + ) + elif e.tag == XINCLUDE_FALLBACK: + parent = e.getparent() + if parent is not None and parent.tag != XINCLUDE_INCLUDE: + raise FatalIncludeError( + "xi:fallback tag must be child of xi:include (%r)" % e.tag + ) + else: + raise FatalIncludeError( + "Invalid element found in XInclude namespace (%r)" % e.tag + ) + return elem diff --git a/.venv/lib/python3.13/site-packages/lxml/__init__.py b/.venv/lib/python3.13/site-packages/lxml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6c821ac10297c8c23ed78b5408679d3ee05959ac --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/__init__.py @@ -0,0 +1,22 @@ +# this is a package + +__version__ = "6.1.1" + + +def get_include(): + """ + Returns a list of header include paths (for lxml itself, libxml2 + and libxslt) needed to compile C code against lxml if it was built + with statically linked libraries. + """ + import os + lxml_path = __path__[0] + include_path = os.path.join(lxml_path, 'includes') + includes = [include_path, lxml_path] + + for name in os.listdir(include_path): + path = os.path.join(include_path, name) + if os.path.isdir(path): + includes.append(path) + + return includes diff --git a/.venv/lib/python3.13/site-packages/lxml/_elementpath.cpython-313-x86_64-linux-gnu.so b/.venv/lib/python3.13/site-packages/lxml/_elementpath.cpython-313-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..b1241f350aa36fcb70c9a7529d20043f5958bb71 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/_elementpath.cpython-313-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bf231cdfe018b1f825d00d38511a8789c070cede55f98c5b114b66151b34e29 +size 209096 diff --git a/.venv/lib/python3.13/site-packages/lxml/_elementpath.py b/.venv/lib/python3.13/site-packages/lxml/_elementpath.py new file mode 100644 index 0000000000000000000000000000000000000000..760a1e00b8e1611e3915085482af1d9efc3a6ae1 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/_elementpath.py @@ -0,0 +1,343 @@ +# cython: language_level=3 + +# +# ElementTree +# $Id: ElementPath.py 3375 2008-02-13 08:05:08Z fredrik $ +# +# limited xpath support for element trees +# +# history: +# 2003-05-23 fl created +# 2003-05-28 fl added support for // etc +# 2003-08-27 fl fixed parsing of periods in element names +# 2007-09-10 fl new selection engine +# 2007-09-12 fl fixed parent selector +# 2007-09-13 fl added iterfind; changed findall to return a list +# 2007-11-30 fl added namespaces support +# 2009-10-30 fl added child element value filter +# +# Copyright (c) 2003-2009 by Fredrik Lundh. All rights reserved. +# +# fredrik@pythonware.com +# http://www.pythonware.com +# +# -------------------------------------------------------------------- +# The ElementTree toolkit is +# +# Copyright (c) 1999-2009 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Secret Labs AB or the author not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD +# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- +# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR +# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# -------------------------------------------------------------------- + +## +# Implementation module for XPath support. There's usually no reason +# to import this module directly; the ElementTree does this for +# you, if needed. +## + + +import re + +xpath_tokenizer_re = re.compile( + "(" + "'[^']*'|\"[^\"]*\"|" + "::|" + "//?|" + r"\.\.|" + r"\(\)|" + r"[/.*:\[\]\(\)@=])|" + r"((?:\{[^}]+\})?[^/\[\]\(\)@=\s]+)|" + r"\s+" + ) + +def xpath_tokenizer(pattern, namespaces=None, with_prefixes=True): + # ElementTree uses '', lxml used None originally. + default_namespace = (namespaces.get(None) or namespaces.get('')) if namespaces else None + parsing_attribute = False + for token in xpath_tokenizer_re.findall(pattern): + ttype, tag = token + if tag and tag[0] != "{": + if ":" in tag and with_prefixes: + prefix, uri = tag.split(":", 1) + try: + if not namespaces: + raise KeyError + yield ttype, "{%s}%s" % (namespaces[prefix], uri) + except KeyError: + raise SyntaxError("prefix %r not found in prefix map" % prefix) + elif tag.isdecimal(): + yield token # index + elif default_namespace and not parsing_attribute: + yield ttype, "{%s}%s" % (default_namespace, tag) + else: + yield token + parsing_attribute = False + else: + yield token + parsing_attribute = ttype == '@' + + +def prepare_child(next, token): + tag = token[1] + def select(result): + for elem in result: + yield from elem.iterchildren(tag) + return select + +def prepare_star(next, token): + def select(result): + for elem in result: + yield from elem.iterchildren('*') + return select + +def prepare_self(next, token): + def select(result): + return result + return select + +def prepare_descendant(next, token): + token = next() + if token[0] == "*": + tag = "*" + elif not token[0]: + tag = token[1] + else: + raise SyntaxError("invalid descendant") + def select(result): + for elem in result: + yield from elem.iterdescendants(tag) + return select + +def prepare_parent(next, token): + def select(result): + for elem in result: + parent = elem.getparent() + if parent is not None: + yield parent + return select + +def prepare_predicate(next, token): + # FIXME: replace with real parser!!! refs: + # http://effbot.org/zone/simple-iterator-parser.htm + # http://javascript.crockford.com/tdop/tdop.html + signature = '' + predicate = [] + while 1: + token = next() + if token[0] == "]": + break + if token == ('', ''): + # ignore whitespace + continue + if token[0] and token[0][:1] in "'\"": + token = "'", token[0][1:-1] + signature += token[0] or "-" + predicate.append(token[1]) + + # use signature to determine predicate type + if signature == "@-": + # [@attribute] predicate + key = predicate[1] + def select(result): + for elem in result: + if elem.get(key) is not None: + yield elem + return select + if signature == "@-='": + # [@attribute='value'] + key = predicate[1] + value = predicate[-1] + def select(result): + for elem in result: + if elem.get(key) == value: + yield elem + return select + if signature == "-" and not re.match(r"-?\d+$", predicate[0]): + # [tag] + tag = predicate[0] + def select(result): + for elem in result: + for _ in elem.iterchildren(tag): + yield elem + break + return select + if signature == ".='" or (signature == "-='" and not re.match(r"-?\d+$", predicate[0])): + # [.='value'] or [tag='value'] + tag = predicate[0] + value = predicate[-1] + if tag: + def select(result): + for elem in result: + for e in elem.iterchildren(tag): + if "".join(e.itertext()) == value: + yield elem + break + else: + def select(result): + for elem in result: + if "".join(elem.itertext()) == value: + yield elem + return select + if signature == "-" or signature == "-()" or signature == "-()-": + # [index] or [last()] or [last()-index] + if signature == "-": + # [index] + index = int(predicate[0]) - 1 + if index < 0: + if index == -1: + raise SyntaxError( + "indices in path predicates are 1-based, not 0-based") + else: + raise SyntaxError("path index >= 1 expected") + else: + if predicate[0] != "last": + raise SyntaxError("unsupported function") + if signature == "-()-": + try: + index = int(predicate[2]) - 1 + except ValueError: + raise SyntaxError("unsupported expression") + else: + index = -1 + def select(result): + for elem in result: + parent = elem.getparent() + if parent is None: + continue + try: + # FIXME: what if the selector is "*" ? + elems = list(parent.iterchildren(elem.tag)) + if elems[index] is elem: + yield elem + except IndexError: + pass + return select + raise SyntaxError("invalid predicate") + +ops = { + "": prepare_child, + "*": prepare_star, + ".": prepare_self, + "..": prepare_parent, + "//": prepare_descendant, + "[": prepare_predicate, +} + + +# -------------------------------------------------------------------- + +_cache = {} + + +def _build_path_iterator(path, namespaces, with_prefixes=True): + """compile selector pattern""" + if path[-1:] == "/": + path += "*" # implicit all (FIXME: keep this?) + + cache_key = (path,) + if namespaces: + # lxml originally used None for the default namespace but ElementTree uses the + # more convenient (all-strings-dict) empty string, so we support both here, + # preferring the more convenient '', as long as they aren't ambiguous. + if None in namespaces: + if '' in namespaces and namespaces[None] != namespaces['']: + raise ValueError("Ambiguous default namespace provided: %r versus %r" % ( + namespaces[None], namespaces[''])) + cache_key += (namespaces[None],) + tuple(sorted( + item for item in namespaces.items() if item[0] is not None)) + else: + cache_key += tuple(sorted(namespaces.items())) + + try: + return _cache[cache_key] + except KeyError: + pass + if len(_cache) > 100: + _cache.clear() + + if path[:1] == "/": + raise SyntaxError("cannot use absolute path on element") + stream = iter(xpath_tokenizer(path, namespaces, with_prefixes=with_prefixes)) + try: + _next = stream.next + except AttributeError: + # Python 3 + _next = stream.__next__ + try: + token = _next() + except StopIteration: + raise SyntaxError("empty path expression") + selector = [] + while 1: + try: + selector.append(ops[token[0]](_next, token)) + except StopIteration: + raise SyntaxError("invalid path") + try: + token = _next() + if token[0] == "/": + token = _next() + except StopIteration: + break + _cache[cache_key] = selector + return selector + + +## +# Iterate over the matching nodes + +def iterfind(elem, path, namespaces=None, with_prefixes=True): + selector = _build_path_iterator(path, namespaces, with_prefixes=with_prefixes) + result = iter((elem,)) + for select in selector: + result = select(result) + return result + + +## +# Find first matching object. + +def find(elem, path, namespaces=None, with_prefixes=True): + it = iterfind(elem, path, namespaces, with_prefixes=with_prefixes) + try: + return next(it) + except StopIteration: + return None + + +## +# Find all matching objects. + +def findall(elem, path, namespaces=None, with_prefixes=True): + return list(iterfind(elem, path, namespaces)) + + +## +# Find text for first matching object. + +def findtext(elem, path, default=None, namespaces=None, with_prefixes=True): + el = find(elem, path, namespaces, with_prefixes=with_prefixes) + if el is None: + return default + else: + return el.text or '' diff --git a/.venv/lib/python3.13/site-packages/lxml/apihelpers.pxi b/.venv/lib/python3.13/site-packages/lxml/apihelpers.pxi new file mode 100644 index 0000000000000000000000000000000000000000..87a27d9ffedfff2b575d71faf7bd828db72dda5f --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/apihelpers.pxi @@ -0,0 +1,1819 @@ +# Private/public helper functions for API functions + +from lxml.includes cimport uri + + +cdef void displayNode(xmlNode* c_node, indent) noexcept: + # to help with debugging + cdef xmlNode* c_child + try: + print(indent * ' ', c_node) + c_child = c_node.children + while c_child is not NULL: + displayNode(c_child, indent + 1) + c_child = c_child.next + finally: + return # swallow any exceptions + +cdef inline bint _isHtmlDocument(_Element element) except -1: + cdef xmlNode* c_node = element._c_node + return ( + c_node is not NULL and c_node.doc is not NULL and + c_node.doc.properties & tree.XML_DOC_HTML != 0 + ) + +cdef inline int _assertValidNode(_Element element) except -1: + assert element._c_node is not NULL, "invalid Element proxy at %s" % id(element) + +cdef inline int _assertValidDoc(_Document doc) except -1: + assert doc._c_doc is not NULL, "invalid Document proxy at %s" % id(doc) + +cdef _Document _documentOrRaise(object input): + """Call this to get the document of a _Document, _ElementTree or _Element + object, or to raise an exception if it can't be determined. + + Should be used in all API functions for consistency. + """ + cdef _Document doc + if isinstance(input, _ElementTree): + if (<_ElementTree>input)._context_node is not None: + doc = (<_ElementTree>input)._context_node._doc + else: + doc = None + elif isinstance(input, _Element): + doc = (<_Element>input)._doc + elif isinstance(input, _Document): + doc = <_Document>input + else: + raise TypeError, f"Invalid input object: {python._fqtypename(input).decode('utf8')}" + if doc is None: + raise ValueError, f"Input object has no document: {python._fqtypename(input).decode('utf8')}" + _assertValidDoc(doc) + return doc + +cdef _Element _rootNodeOrRaise(object input): + """Call this to get the root node of a _Document, _ElementTree or + _Element object, or to raise an exception if it can't be determined. + + Should be used in all API functions for consistency. + """ + cdef _Element node + if isinstance(input, _ElementTree): + node = (<_ElementTree>input)._context_node + elif isinstance(input, _Element): + node = <_Element>input + elif isinstance(input, _Document): + node = (<_Document>input).getroot() + else: + raise TypeError, f"Invalid input object: {python._fqtypename(input).decode('utf8')}" + if (node is None or not node._c_node or + node._c_node.type != tree.XML_ELEMENT_NODE): + raise ValueError, f"Input object is not an XML element: {python._fqtypename(input).decode('utf8')}" + _assertValidNode(node) + return node + +cdef bint _isAncestorOrSame(xmlNode* c_ancestor, xmlNode* c_node) noexcept: + while c_node: + if c_node is c_ancestor: + return True + c_node = c_node.parent + return False + +cdef _Element _makeElement(tag, xmlDoc* c_doc, _Document doc, + _BaseParser parser, text, tail, attrib, nsmap, + dict extra_attrs): + """Create a new element and initialize text content, namespaces and + attributes. + + This helper function will reuse as much of the existing document as + possible: + + If 'parser' is None, the parser will be inherited from 'doc' or the + default parser will be used. + + If 'doc' is None, 'c_doc' is used to create a new _Document and the new + element is made its root node. + + If 'c_doc' is also NULL, a new xmlDoc will be created. + """ + cdef xmlNode* c_node + if doc is not None: + c_doc = doc._c_doc + ns_utf, name_utf = _getNsTag(tag) + if parser is not None and parser._for_html: + _htmlTagValidOrRaise(name_utf) + if c_doc is NULL: + c_doc = _newHTMLDoc() + else: + _tagValidOrRaise(name_utf) + if c_doc is NULL: + c_doc = _newXMLDoc() + c_node = _createElement(c_doc, name_utf) + if c_node is NULL: + if doc is None and c_doc is not NULL: + tree.xmlFreeDoc(c_doc) + raise MemoryError() + try: + if doc is None: + tree.xmlDocSetRootElement(c_doc, c_node) + doc = _documentFactory(c_doc, parser) + if text is not None: + _setNodeText(c_node, text) + if tail is not None: + _setTailText(c_node, tail) + # add namespaces to node if necessary + _setNodeNamespaces(c_node, doc, ns_utf, nsmap) + _initNodeAttributes(c_node, doc, attrib, extra_attrs) + return _elementFactory(doc, c_node) + except: + # free allocated c_node/c_doc unless Python does it for us + if c_node.doc is not c_doc: + # node not yet in document => will not be freed by document + if tail is not None: + _removeText(c_node.next) # tail + tree.xmlFreeNode(c_node) + if doc is None: + # c_doc will not be freed by doc + tree.xmlFreeDoc(c_doc) + raise + +cdef int _initNewElement(_Element element, bint is_html, name_utf, ns_utf, + _BaseParser parser, attrib, nsmap, dict extra_attrs) except -1: + """Initialise a new Element object. + + This is used when users instantiate a Python Element subclass + directly, without it being mapped to an existing XML node. + """ + cdef xmlDoc* c_doc + cdef xmlNode* c_node + cdef _Document doc + if is_html: + _htmlTagValidOrRaise(name_utf) + c_doc = _newHTMLDoc() + else: + _tagValidOrRaise(name_utf) + c_doc = _newXMLDoc() + c_node = _createElement(c_doc, name_utf) + if c_node is NULL: + if c_doc is not NULL: + tree.xmlFreeDoc(c_doc) + raise MemoryError() + tree.xmlDocSetRootElement(c_doc, c_node) + doc = _documentFactory(c_doc, parser) + # add namespaces to node if necessary + _setNodeNamespaces(c_node, doc, ns_utf, nsmap) + _initNodeAttributes(c_node, doc, attrib, extra_attrs) + _registerProxy(element, doc, c_node) + element._init() + return 0 + +cdef _Element _makeSubElement(_Element parent, tag, text, tail, + attrib, nsmap, dict extra_attrs): + """Create a new child element and initialize text content, namespaces and + attributes. + """ + cdef xmlNode* c_node + cdef xmlDoc* c_doc + if parent is None or parent._doc is None: + return None + _assertValidNode(parent) + ns_utf, name_utf = _getNsTag(tag) + c_doc = parent._doc._c_doc + + if parent._doc._parser is not None and parent._doc._parser._for_html: + _htmlTagValidOrRaise(name_utf) + else: + _tagValidOrRaise(name_utf) + + c_node = _createElement(c_doc, name_utf) + if c_node is NULL: + raise MemoryError() + tree.xmlAddChild(parent._c_node, c_node) + + try: + if text is not None: + _setNodeText(c_node, text) + if tail is not None: + _setTailText(c_node, tail) + + # add namespaces to node if necessary + _setNodeNamespaces(c_node, parent._doc, ns_utf, nsmap) + _initNodeAttributes(c_node, parent._doc, attrib, extra_attrs) + return _elementFactory(parent._doc, c_node) + except: + # make sure we clean up in case of an error + _removeNode(parent._doc, c_node) + raise + + +cdef int _setNodeNamespaces(xmlNode* c_node, _Document doc, + object node_ns_utf, object nsmap) except -1: + """Lookup current namespace prefixes, then set namespace structure for + node (if 'node_ns_utf' was provided) and register new ns-prefix mappings. + + 'node_ns_utf' should only be passed for a newly created node. + """ + cdef xmlNs* c_ns + cdef list nsdefs + + if nsmap: + for prefix, href in _iter_nsmap(nsmap): + href_utf = _utf8(href) + _uriValidOrRaise(href_utf) + c_href = _xcstr(href_utf) + if prefix is not None: + prefix_utf = _utf8(prefix) + _prefixValidOrRaise(prefix_utf) + c_prefix = _xcstr(prefix_utf) + else: + c_prefix = NULL + # add namespace with prefix if it is not already known + c_ns = tree.xmlSearchNs(doc._c_doc, c_node, c_prefix) + if c_ns is NULL or \ + c_ns.href is NULL or \ + tree.xmlStrcmp(c_ns.href, c_href) != 0: + c_ns = tree.xmlNewNs(c_node, c_href, c_prefix) + if c_ns is NULL: + # libxml2 has two error conditions: "out of memory" and "prefix exists already". + # We ignore the latter for compatibility reasons. It currently only appears + # during namespace cleanup. + c_ns = c_node.nsDef + while c_ns is not NULL: + if c_prefix is NULL: + if c_ns.prefix is NULL: + break + elif tree.xmlStrcmp(c_ns.prefix, c_prefix) == 0: + break + c_ns = c_ns.next + else: + raise MemoryError() + if href_utf == node_ns_utf: + tree.xmlSetNs(c_node, c_ns) + node_ns_utf = None + + if node_ns_utf is not None: + _uriValidOrRaise(node_ns_utf) + doc._setNodeNs(c_node, _xcstr(node_ns_utf)) + return 0 + + +cdef dict _build_nsmap(xmlNode* c_node): + """ + Namespace prefix->URI mapping known in the context of this Element. + This includes all namespace declarations of the parents. + """ + cdef xmlNs* c_ns + nsmap = {} + while c_node is not NULL and c_node.type == tree.XML_ELEMENT_NODE: + c_ns = c_node.nsDef + while c_ns is not NULL: + if c_ns.prefix or c_ns.href: + prefix = funicodeOrNone(c_ns.prefix) + if prefix not in nsmap: + nsmap[prefix] = funicodeOrNone(c_ns.href) + c_ns = c_ns.next + c_node = c_node.parent + return nsmap + + +cdef _iter_nsmap(nsmap): + """ + Create a reproducibly ordered iterable from an nsmap mapping. + Tries to preserve an existing order and sorts if it assumes no order. + + The difference to _iter_attrib() is that None doesn't sort with strings + in Py3.x. + """ + if isinstance(nsmap, dict): + # dicts are insertion-ordered in Py3.6+ => keep the user provided order. + return nsmap.items() + if len(nsmap) <= 1: + return nsmap.items() + if isinstance(nsmap, OrderedDict): + return nsmap.items() # keep existing order + if None not in nsmap: + return sorted(nsmap.items()) + + # Move the default namespace to the end. This makes sure libxml2 + # prefers a prefix if the ns is defined redundantly on the same + # element. That way, users can work around a problem themselves + # where default namespace attributes on non-default namespaced + # elements serialise without prefix (i.e. into the non-default + # namespace). + default_ns = nsmap[None] + nsdefs = [(k, v) for k, v in nsmap.items() if k is not None] + nsdefs.sort() + nsdefs.append((None, default_ns)) + return nsdefs + + +cdef _iter_attrib(attrib): + """ + Create a reproducibly ordered iterable from an attrib mapping. + Tries to preserve an existing order and sorts if it assumes no order. + """ + # dicts are insertion-ordered in Py3.6+ => keep the user provided order. + if isinstance(attrib, (dict, _Attrib, OrderedDict)): + return attrib.items() + # assume it's an unordered mapping of some kind + return sorted(attrib.items()) + + +cdef _initNodeAttributes(xmlNode* c_node, _Document doc, attrib, dict extra): + """Initialise the attributes of an element node. + """ + cdef bint is_html + cdef xmlNs* c_ns + if attrib is not None and not hasattr(attrib, 'items'): + raise TypeError, f"Invalid attribute dictionary: {python._fqtypename(attrib).decode('utf8')}" + if not attrib and not extra: + return # nothing to do + is_html = doc._parser._for_html + seen = set() + if extra: + for name, value in extra.items(): + _addAttributeToNode(c_node, doc, is_html, name, value, seen) + if attrib: + for name, value in _iter_attrib(attrib): + _addAttributeToNode(c_node, doc, is_html, name, value, seen) + + +cdef int _addAttributeToNode(xmlNode* c_node, _Document doc, bint is_html, + name, value, set seen_tags) except -1: + ns_utf, name_utf = tag = _getNsTag(name) + if tag in seen_tags: + return 0 + seen_tags.add(tag) + if not is_html: + _attributeValidOrRaise(name_utf) + value_utf = _utf8(value) + if ns_utf is None: + new_attr = tree.xmlNewProp(c_node, _xcstr(name_utf), _xcstr(value_utf)) + else: + _uriValidOrRaise(ns_utf) + c_ns = doc._findOrBuildNodeNs(c_node, _xcstr(ns_utf), NULL, 1) + new_attr = tree.xmlNewNsProp(c_node, c_ns, _xcstr(name_utf), _xcstr(value_utf)) + if new_attr is NULL: + raise MemoryError() + return 0 + + +ctypedef struct _ns_node_ref: + xmlNs* ns + xmlNode* node + + +cdef int _collectNsDefs(xmlNode* c_element, _ns_node_ref **_c_ns_list, + size_t *_c_ns_list_len, size_t *_c_ns_list_size) except -1: + c_ns_list = _c_ns_list[0] + cdef size_t c_ns_list_len = _c_ns_list_len[0] + cdef size_t c_ns_list_size = _c_ns_list_size[0] + + c_nsdef = c_element.nsDef + while c_nsdef is not NULL: + if c_ns_list_len >= c_ns_list_size: + if c_ns_list is NULL: + c_ns_list_size = 20 + else: + c_ns_list_size *= 2 + c_nsref_ptr = <_ns_node_ref*> python.lxml_realloc( + c_ns_list, c_ns_list_size, sizeof(_ns_node_ref)) + if c_nsref_ptr is NULL: + if c_ns_list is not NULL: + python.lxml_free(c_ns_list) + _c_ns_list[0] = NULL + raise MemoryError() + c_ns_list = c_nsref_ptr + + c_ns_list[c_ns_list_len] = _ns_node_ref(c_nsdef, c_element) + c_ns_list_len += 1 + c_nsdef = c_nsdef.next + + _c_ns_list_size[0] = c_ns_list_size + _c_ns_list_len[0] = c_ns_list_len + _c_ns_list[0] = c_ns_list + + +cdef int _removeUnusedNamespaceDeclarations(xmlNode* c_element, set prefixes_to_keep) except -1: + """Remove any namespace declarations from a subtree that are not used by + any of its elements (or attributes). + + If a 'prefixes_to_keep' is provided, it must be a set of prefixes. + Any corresponding namespace mappings will not be removed as part of the cleanup. + """ + cdef xmlNode* c_node + cdef _ns_node_ref* c_ns_list = NULL + cdef size_t c_ns_list_size = 0 + cdef size_t c_ns_list_len = 0 + cdef size_t i + + if c_element.parent and c_element.parent.type == tree.XML_DOCUMENT_NODE: + # include declarations on the document node + _collectNsDefs(c_element.parent, &c_ns_list, &c_ns_list_len, &c_ns_list_size) + + tree.BEGIN_FOR_EACH_ELEMENT_FROM(c_element, c_element, 1) + # collect all new namespace declarations into the ns list + if c_element.nsDef: + _collectNsDefs(c_element, &c_ns_list, &c_ns_list_len, &c_ns_list_size) + + # remove all namespace declarations from the list that are referenced + if c_ns_list_len and c_element.type == tree.XML_ELEMENT_NODE: + c_node = c_element + while c_node and c_ns_list_len: + if c_node.ns: + for i in range(c_ns_list_len): + if c_node.ns is c_ns_list[i].ns: + c_ns_list_len -= 1 + c_ns_list[i] = c_ns_list[c_ns_list_len] + #c_ns_list[c_ns_list_len] = _ns_node_ref(NULL, NULL) + break + if c_node is c_element: + # continue with attributes + c_node = c_element.properties + else: + c_node = c_node.next + tree.END_FOR_EACH_ELEMENT_FROM(c_element) + + if c_ns_list is NULL: + return 0 + + # free all namespace declarations that remained in the list, + # except for those we should keep explicitly + cdef xmlNs* c_nsdef + for i in range(c_ns_list_len): + if prefixes_to_keep is not None: + if c_ns_list[i].ns.prefix and c_ns_list[i].ns.prefix in prefixes_to_keep: + continue + c_node = c_ns_list[i].node + c_nsdef = c_node.nsDef + if c_nsdef is c_ns_list[i].ns: + c_node.nsDef = c_node.nsDef.next + else: + while c_nsdef.next is not c_ns_list[i].ns: + c_nsdef = c_nsdef.next + c_nsdef.next = c_nsdef.next.next + tree.xmlFreeNs(c_ns_list[i].ns) + + if c_ns_list is not NULL: + python.lxml_free(c_ns_list) + return 0 + +cdef xmlNs* _searchNsByHref(xmlNode* c_node, const_xmlChar* c_href, bint is_attribute) noexcept: + """Search a namespace declaration that covers a node (element or + attribute). + + For attributes, try to find a prefixed namespace declaration + instead of the default namespaces. This helps in supporting + round-trips for attributes on elements with a different namespace. + """ + cdef xmlNs* c_ns + cdef xmlNs* c_default_ns = NULL + cdef xmlNode* c_element + if c_href is NULL or c_node is NULL or c_node.type == tree.XML_ENTITY_REF_NODE: + return NULL + if tree.xmlStrcmp(c_href, tree.XML_XML_NAMESPACE) == 0: + # no special cases here, let libxml2 handle this + return tree.xmlSearchNsByHref(c_node.doc, c_node, c_href) + if c_node.type == tree.XML_ATTRIBUTE_NODE: + is_attribute = 1 + while c_node is not NULL and c_node.type != tree.XML_ELEMENT_NODE: + c_node = c_node.parent + c_element = c_node + while c_node is not NULL: + if c_node.type == tree.XML_ELEMENT_NODE: + c_ns = c_node.nsDef + while c_ns is not NULL: + if c_ns.href is not NULL and tree.xmlStrcmp(c_href, c_ns.href) == 0: + if c_ns.prefix is NULL and is_attribute: + # for attributes, continue searching a named + # prefix, but keep the first default namespace + # declaration that we found + if c_default_ns is NULL: + c_default_ns = c_ns + elif tree.xmlSearchNs( + c_element.doc, c_element, c_ns.prefix) is c_ns: + # start node is in namespace scope => found! + return c_ns + c_ns = c_ns.next + if c_node is not c_element and c_node.ns is not NULL: + # optimise: the node may have the namespace itself + c_ns = c_node.ns + if c_ns.href is not NULL and tree.xmlStrcmp(c_href, c_ns.href) == 0: + if c_ns.prefix is NULL and is_attribute: + # for attributes, continue searching a named + # prefix, but keep the first default namespace + # declaration that we found + if c_default_ns is NULL: + c_default_ns = c_ns + elif tree.xmlSearchNs( + c_element.doc, c_element, c_ns.prefix) is c_ns: + # start node is in namespace scope => found! + return c_ns + c_node = c_node.parent + # nothing found => use a matching default namespace or fail + if c_default_ns is not NULL: + if tree.xmlSearchNs(c_element.doc, c_element, NULL) is c_default_ns: + return c_default_ns + return NULL + +cdef int _replaceNodeByChildren(_Document doc, xmlNode* c_node) except -1: + # NOTE: this does not deallocate the node, just unlink it! + cdef xmlNode* c_parent + cdef xmlNode* c_child + if c_node.children is NULL: + tree.xmlUnlinkNode(c_node) + return 0 + + c_parent = c_node.parent + # fix parent links of children + c_child = c_node.children + while c_child is not NULL: + c_child.parent = c_parent + c_child = c_child.next + + # fix namespace references of children if their parent's namespace + # declarations get lost + if c_node.nsDef is not NULL: + c_child = c_node.children + while c_child is not NULL: + moveNodeToDocument(doc, doc._c_doc, c_child) + c_child = c_child.next + + # fix sibling links to/from child slice + if c_node.prev is NULL: + c_parent.children = c_node.children + else: + c_node.prev.next = c_node.children + c_node.children.prev = c_node.prev + if c_node.next is NULL: + c_parent.last = c_node.last + else: + c_node.next.prev = c_node.last + c_node.last.next = c_node.next + + # unlink c_node + c_node.children = c_node.last = NULL + c_node.parent = c_node.next = c_node.prev = NULL + return 0 + +cdef unicode _attributeValue(xmlNode* c_element, xmlAttr* c_attrib_node): + c_href = _getNs(c_attrib_node) + value = tree.xmlGetNsProp(c_element, c_attrib_node.name, c_href) + try: + result = funicode(value) + finally: + tree.xmlFree(value) + return result + +cdef unicode _attributeValueFromNsName(xmlNode* c_element, + const_xmlChar* c_href, const_xmlChar* c_name): + c_result = tree.xmlGetNsProp(c_element, c_name, c_href) + if c_result is NULL: + return None + try: + result = funicode(c_result) + finally: + tree.xmlFree(c_result) + return result + +cdef object _getNodeAttributeValue(xmlNode* c_node, key, default): + ns, tag = _getNsTag(key) + c_href = NULL if ns is None else _xcstr(ns) + c_result = tree.xmlGetNsProp(c_node, _xcstr(tag), c_href) + if c_result is NULL: + # XXX free namespace that is not in use..? + return default + try: + result = funicode(c_result) + finally: + tree.xmlFree(c_result) + return result + +cdef inline object _getAttributeValue(_Element element, key, default): + return _getNodeAttributeValue(element._c_node, key, default) + +cdef int _setAttributeValue(_Element element, key, value) except -1: + cdef const_xmlChar* c_value + cdef xmlNs* c_ns + ns, tag = _getNsTag(key) + is_html = element._doc._parser._for_html + if not is_html: + _attributeValidOrRaise(tag) + c_tag = _xcstr(tag) + if value is None and is_html: + c_value = NULL + else: + if isinstance(value, QName): + value = _resolveQNameText(element, value) + else: + value = _utf8(value) + c_value = _xcstr(value) + if ns is None: + c_ns = NULL + else: + c_ns = element._doc._findOrBuildNodeNs(element._c_node, _xcstr(ns), NULL, 1) + tree.xmlSetNsProp(element._c_node, c_ns, c_tag, c_value) + return 0 + +cdef int _delAttribute(_Element element, key) except -1: + ns, tag = _getNsTag(key) + c_href = NULL if ns is None else _xcstr(ns) + if _delAttributeFromNsName(element._c_node, c_href, _xcstr(tag)): + raise KeyError, key + return 0 + +cdef int _delAttributeFromNsName(xmlNode* c_node, const_xmlChar* c_href, const_xmlChar* c_name) noexcept: + c_attr = tree.xmlHasNsProp(c_node, c_name, c_href) + if c_attr is NULL: + # XXX free namespace that is not in use..? + return -1 + tree.xmlRemoveProp(c_attr) + return 0 + +cdef list _collectAttributes(xmlNode* c_node, int collecttype): + """Collect all attributes of a node in a list. Depending on collecttype, + it collects either the name (1), the value (2) or the name-value tuples. + """ + cdef Py_ssize_t count + c_attr = c_node.properties + count = 0 + while c_attr is not NULL: + if c_attr.type == tree.XML_ATTRIBUTE_NODE: + count += 1 + c_attr = c_attr.next + + if not count: + return [] + + attributes = [None] * count + c_attr = c_node.properties + count = 0 + while c_attr is not NULL: + if c_attr.type == tree.XML_ATTRIBUTE_NODE: + if collecttype == 1: + item = _namespacedName(c_attr) + elif collecttype == 2: + item = _attributeValue(c_node, c_attr) + else: + item = (_namespacedName(c_attr), + _attributeValue(c_node, c_attr)) + attributes[count] = item + count += 1 + c_attr = c_attr.next + return attributes + +cdef object __RE_XML_ENCODING = re.compile( + r'^(<\?xml[^>]+)\s+encoding\s*=\s*["\'][^"\']*["\'](\s*\?>|)', re.U) + +cdef object __REPLACE_XML_ENCODING = __RE_XML_ENCODING.sub +cdef object __HAS_XML_ENCODING = __RE_XML_ENCODING.match + +cdef object _stripEncodingDeclaration(object xml_string): + # this is a hack to remove the XML encoding declaration from unicode + return __REPLACE_XML_ENCODING(r'\g<1>\g<2>', xml_string) + +cdef bint _hasEncodingDeclaration(object xml_string) except -1: + # check if a (unicode) string has an XML encoding declaration + return __HAS_XML_ENCODING(xml_string) is not None + +cdef inline bint _hasText(xmlNode* c_node) noexcept: + return c_node is not NULL and _textNodeOrSkip(c_node.children) is not NULL + +cdef inline bint _hasTail(xmlNode* c_node) noexcept: + return c_node is not NULL and _textNodeOrSkip(c_node.next) is not NULL + +cdef inline bint _hasNonWhitespaceTail(xmlNode* c_node) except -1: + return _hasNonWhitespaceText(c_node, tail=True) + +cdef bint _hasNonWhitespaceText(xmlNode* c_node, bint tail=False) except -1: + c_text_node = c_node and _textNodeOrSkip(c_node.next if tail else c_node.children) + if c_text_node is NULL: + return False + while c_text_node is not NULL: + if c_text_node.content[0] != c'\0' and not _collectText(c_text_node).isspace(): + return True + c_text_node = _textNodeOrSkip(c_text_node.next) + return False + +cdef unicode _collectText(xmlNode* c_node): + """Collect all text nodes and return them as a unicode string. + + Start collecting at c_node. + + If there was no text to collect, return None + """ + cdef Py_ssize_t scount + cdef xmlChar* c_text + cdef xmlNode* c_node_cur + # check for multiple text nodes + scount = 0 + c_text = NULL + c_node_cur = c_node = _textNodeOrSkip(c_node) + while c_node_cur is not NULL: + if c_node_cur.content[0] != c'\0': + c_text = c_node_cur.content + scount += 1 + c_node_cur = _textNodeOrSkip(c_node_cur.next) + + # handle two most common cases first + if c_text is NULL: + return '' if scount > 0 else None + if scount == 1: + return funicode(c_text) + + # the rest is not performance critical anymore + result = b'' + while c_node is not NULL: + result += c_node.content + c_node = _textNodeOrSkip(c_node.next) + return funicode(result) + +cdef void _removeText(xmlNode* c_node) noexcept: + """Remove all text nodes. + + Start removing at c_node. + """ + cdef xmlNode* c_next + c_node = _textNodeOrSkip(c_node) + while c_node is not NULL: + c_next = _textNodeOrSkip(c_node.next) + tree.xmlUnlinkNode(c_node) + tree.xmlFreeNode(c_node) + c_node = c_next + +cdef xmlNode* _createTextNode(xmlDoc* doc, value) except NULL: + cdef xmlNode* c_text_node + if isinstance(value, CDATA): + c_text_node = tree.xmlNewCDataBlock( + doc, _xcstr((value)._utf8_data), + python.PyBytes_GET_SIZE((value)._utf8_data)) + else: + text = _utf8(value) + c_text_node = tree.xmlNewDocText(doc, _xcstr(text)) + if not c_text_node: + raise MemoryError() + return c_text_node + +cdef int _setNodeText(xmlNode* c_node, value) except -1: + # remove all text nodes at the start first + _removeText(c_node.children) + if value is None: + return 0 + # now add new text node with value at start + c_text_node = _createTextNode(c_node.doc, value) + if c_node.children is NULL: + tree.xmlAddChild(c_node, c_text_node) + else: + tree.xmlAddPrevSibling(c_node.children, c_text_node) + return 0 + +cdef int _setTailText(xmlNode* c_node, value) except -1: + # remove all text nodes at the start first + _removeText(c_node.next) + if value is None: + return 0 + # now append new text node with value + c_text_node = _createTextNode(c_node.doc, value) + tree.xmlAddNextSibling(c_node, c_text_node) + return 0 + +cdef bytes _resolveQNameText(_Element element, value): + cdef xmlNs* c_ns + ns, tag = _getNsTag(value) + if ns is None: + return tag + else: + c_ns = element._doc._findOrBuildNodeNs( + element._c_node, _xcstr(ns), NULL, 0) + return python.PyBytes_FromFormat('%s:%s', c_ns.prefix, _cstr(tag)) + +cdef inline bint _hasChild(xmlNode* c_node) noexcept: + return c_node is not NULL and _findChildForwards(c_node, 0) is not NULL + +cdef inline Py_ssize_t _countElements(xmlNode* c_node) noexcept: + "Counts the elements within the following siblings and the node itself." + cdef Py_ssize_t count + count = 0 + while c_node is not NULL: + if _isElement(c_node): + count += 1 + c_node = c_node.next + return count + + +cdef int _findChildSlice( + slice sliceobject, xmlNode* c_parent, + xmlNode** c_start_node, Py_ssize_t* c_step, Py_ssize_t* c_length) except -1: + """Resolve a children slice. + + Returns the start node, step size and the slice length in the + pointer arguments. + """ + cdef Py_ssize_t start = 0, stop = 0, childcount + childcount = _countElements(c_parent.children) + if childcount == 0: + c_start_node[0] = NULL + c_length[0] = 0 + if sliceobject.step is None: + c_step[0] = 1 + else: + python._PyEval_SliceIndex(sliceobject.step, c_step) + return 0 + + python.PySlice_GetIndicesEx( + sliceobject, childcount, &start, &stop, c_step, c_length) + + if start > childcount // 2: + c_start_node[0] = _findChildBackwards(c_parent, childcount - start - 1) + else: + c_start_node[0] = _findChild(c_parent, start) + return 0 + + +cdef bint _isFullSlice(slice sliceobject) except -1: + """Conservative guess if this slice is a full slice as in ``s[:]``. + """ + cdef Py_ssize_t step = 0 + if sliceobject is None: + return 0 + if sliceobject.start is None and \ + sliceobject.stop is None: + if sliceobject.step is None: + return 1 + python._PyEval_SliceIndex(sliceobject.step, &step) + if step == 1: + return 1 + return 0 + return 0 + +cdef _collectChildren(_Element element): + cdef xmlNode* c_node + cdef list result = [] + c_node = element._c_node.children + if c_node is not NULL: + if not _isElement(c_node): + c_node = _nextElement(c_node) + while c_node is not NULL: + result.append(_elementFactory(element._doc, c_node)) + c_node = _nextElement(c_node) + return result + +cdef inline xmlNode* _findChild(xmlNode* c_node, Py_ssize_t index) noexcept: + if index < 0: + return _findChildBackwards(c_node, -index - 1) + else: + return _findChildForwards(c_node, index) + +cdef inline xmlNode* _findChildForwards(xmlNode* c_node, Py_ssize_t index) noexcept: + """Return child element of c_node with index, or return NULL if not found. + """ + cdef xmlNode* c_child + cdef Py_ssize_t c + c_child = c_node.children + c = 0 + while c_child is not NULL: + if _isElement(c_child): + if c == index: + return c_child + c += 1 + c_child = c_child.next + return NULL + +cdef inline xmlNode* _findChildBackwards(xmlNode* c_node, Py_ssize_t index) noexcept: + """Return child element of c_node with index, or return NULL if not found. + Search from the end. + """ + cdef xmlNode* c_child + cdef Py_ssize_t c + c_child = c_node.last + c = 0 + while c_child is not NULL: + if _isElement(c_child): + if c == index: + return c_child + c += 1 + c_child = c_child.prev + return NULL + +cdef inline xmlNode* _textNodeOrSkip(xmlNode* c_node) noexcept nogil: + """Return the node if it's a text node. Skip over ignorable nodes in a + series of text nodes. Return NULL if a non-ignorable node is found. + + This is used to skip over XInclude nodes when collecting adjacent text + nodes. + """ + while c_node is not NULL: + if c_node.type == tree.XML_TEXT_NODE or \ + c_node.type == tree.XML_CDATA_SECTION_NODE: + return c_node + elif c_node.type == tree.XML_XINCLUDE_START or \ + c_node.type == tree.XML_XINCLUDE_END: + c_node = c_node.next + else: + return NULL + return NULL + +cdef inline xmlNode* _nextElement(xmlNode* c_node) noexcept: + """Given a node, find the next sibling that is an element. + """ + if c_node is NULL: + return NULL + c_node = c_node.next + while c_node is not NULL: + if _isElement(c_node): + return c_node + c_node = c_node.next + return NULL + +cdef inline xmlNode* _previousElement(xmlNode* c_node) noexcept: + """Given a node, find the next sibling that is an element. + """ + if c_node is NULL: + return NULL + c_node = c_node.prev + while c_node is not NULL: + if _isElement(c_node): + return c_node + c_node = c_node.prev + return NULL + +cdef inline xmlNode* _parentElement(xmlNode* c_node) noexcept: + "Given a node, find the parent element." + if c_node is NULL or not _isElement(c_node): + return NULL + c_node = c_node.parent + if c_node is NULL or not _isElement(c_node): + return NULL + return c_node + +cdef inline bint _tagMatches(xmlNode* c_node, const_xmlChar* c_href, const_xmlChar* c_name) noexcept: + """Tests if the node matches namespace URI and tag name. + + A node matches if it matches both c_href and c_name. + + A node matches c_href if any of the following is true: + * c_href is NULL + * its namespace is NULL and c_href is the empty string + * its namespace string equals the c_href string + + A node matches c_name if any of the following is true: + * c_name is NULL + * its name string equals the c_name string + """ + if c_node is NULL: + return 0 + if c_node.type != tree.XML_ELEMENT_NODE: + # not an element, only succeed if we match everything + return c_name is NULL and c_href is NULL + if c_name is NULL: + if c_href is NULL: + # always match + return 1 + else: + c_node_href = _getNs(c_node) + if c_node_href is NULL: + return c_href[0] == c'\0' + else: + return tree.xmlStrcmp(c_node_href, c_href) == 0 + elif c_href is NULL: + if _getNs(c_node) is not NULL: + return 0 + return c_node.name == c_name or tree.xmlStrcmp(c_node.name, c_name) == 0 + elif c_node.name == c_name or tree.xmlStrcmp(c_node.name, c_name) == 0: + c_node_href = _getNs(c_node) + if c_node_href is NULL: + return c_href[0] == c'\0' + else: + return tree.xmlStrcmp(c_node_href, c_href) == 0 + else: + return 0 + +cdef inline bint _tagMatchesExactly(xmlNode* c_node, qname* c_qname) noexcept: + """Tests if the node matches namespace URI and tag name. + + This differs from _tagMatches() in that it does not consider a + NULL value in qname.href a wildcard, and that it expects the c_name + to be taken from the doc dict, i.e. it only compares the names by + address. + + A node matches if it matches both href and c_name of the qname. + + A node matches c_href if any of the following is true: + * its namespace is NULL and c_href is the empty string + * its namespace string equals the c_href string + + A node matches c_name if any of the following is true: + * c_name is NULL + * its name string points to the same address (!) as c_name + """ + return _nsTagMatchesExactly(_getNs(c_node), c_node.name, c_qname) + +cdef inline bint _nsTagMatchesExactly(const_xmlChar* c_node_href, + const_xmlChar* c_node_name, + qname* c_qname) noexcept: + """Tests if name and namespace URI match those of c_qname. + + This differs from _tagMatches() in that it does not consider a + NULL value in qname.href a wildcard, and that it expects the c_name + to be taken from the doc dict, i.e. it only compares the names by + address. + + A node matches if it matches both href and c_name of the qname. + + A node matches c_href if any of the following is true: + * its namespace is NULL and c_href is the empty string + * its namespace string equals the c_href string + + A node matches c_name if any of the following is true: + * c_name is NULL + * its name string points to the same address (!) as c_name + """ + cdef char* c_href + if c_qname.c_name is not NULL and c_qname.c_name is not c_node_name: + return 0 + if c_qname.href is NULL: + return 1 + c_href = python.__cstr(c_qname.href) + if c_href[0] == b'\0': + return c_node_href is NULL or c_node_href[0] == b'\0' + elif c_node_href is NULL: + return 0 + else: + return tree.xmlStrcmp(c_href, c_node_href) == 0 + +cdef Py_ssize_t _mapTagsToQnameMatchArray(xmlDoc* c_doc, list ns_tags, + qname* c_ns_tags, bint force_into_dict) except -1: + """Map a sequence of (name, namespace) pairs to a qname array for efficient + matching with _tagMatchesExactly() above. + + Note that each qname struct in the array owns its href byte string object + if it is not NULL. + """ + cdef Py_ssize_t count = 0, i, c_tag_len + cdef bytes ns, tag + cdef const_xmlChar* c_tag + + for ns, tag in ns_tags: + if tag is None: + c_tag = NULL + else: + c_tag_len = len(tag) + if c_tag_len > limits.INT_MAX: + # too long, not in the dict => not in the document + continue + elif force_into_dict: + c_tag = tree.xmlDictLookup(c_doc.dict, _xcstr(tag), c_tag_len) + if c_tag is NULL: + # clean up before raising the error + for i in xrange(count): + cpython.ref.Py_XDECREF(c_ns_tags[i].href) + raise MemoryError() + else: + c_tag = tree.xmlDictExists(c_doc.dict, _xcstr(tag), c_tag_len) + if c_tag is NULL: + # not in the dict => not in the document + continue + + c_ns_tags[count].c_name = c_tag + if ns is None: + c_ns_tags[count].href = NULL + else: + cpython.ref.Py_INCREF(ns) # keep an owned reference! + c_ns_tags[count].href = ns + count += 1 + return count + +cdef int _removeNode(_Document doc, xmlNode* c_node) except -1: + """Unlink and free a node and subnodes if possible. Otherwise, make sure + it's self-contained. + """ + cdef xmlNode* c_next + c_next = c_node.next + tree.xmlUnlinkNode(c_node) + _moveTail(c_next, c_node) + if not attemptDeallocation(c_node): + # make namespaces absolute + moveNodeToDocument(doc, c_node.doc, c_node) + return 0 + +cdef int _removeSiblings(xmlNode* c_element, tree.xmlElementType node_type, bint with_tail) except -1: + cdef xmlNode* c_node + cdef xmlNode* c_next + c_node = c_element.next + while c_node is not NULL: + c_next = _nextElement(c_node) + if c_node.type == node_type: + if with_tail: + _removeText(c_node.next) + tree.xmlUnlinkNode(c_node) + attemptDeallocation(c_node) + c_node = c_next + c_node = c_element.prev + while c_node is not NULL: + c_next = _previousElement(c_node) + if c_node.type == node_type: + if with_tail: + _removeText(c_node.next) + tree.xmlUnlinkNode(c_node) + attemptDeallocation(c_node) + c_node = c_next + return 0 + +cdef void _moveTail(xmlNode* c_tail, xmlNode* c_target) noexcept: + cdef xmlNode* c_next + # tail support: look for any text nodes trailing this node and + # move them too + c_tail = _textNodeOrSkip(c_tail) + while c_tail is not NULL: + c_next = _textNodeOrSkip(c_tail.next) + c_target = tree.xmlAddNextSibling(c_target, c_tail) + c_tail = c_next + +cdef int _copyTail(xmlNode* c_tail, xmlNode* c_target) except -1: + cdef xmlNode* c_new_tail + # tail copying support: look for any text nodes trailing this node and + # copy it to the target node + c_tail = _textNodeOrSkip(c_tail) + while c_tail is not NULL: + if c_target.doc is not c_tail.doc: + c_new_tail = tree.xmlDocCopyNode(c_tail, c_target.doc, 0) + else: + c_new_tail = tree.xmlCopyNode(c_tail, 0) + if c_new_tail is NULL: + raise MemoryError() + c_target = tree.xmlAddNextSibling(c_target, c_new_tail) + c_tail = _textNodeOrSkip(c_tail.next) + return 0 + +cdef int _copyNonElementSiblings(xmlNode* c_node, xmlNode* c_target) except -1: + cdef xmlNode* c_copy + cdef xmlNode* c_sibling = c_node + while c_sibling.prev != NULL and \ + (c_sibling.prev.type == tree.XML_PI_NODE or + c_sibling.prev.type == tree.XML_COMMENT_NODE or + c_sibling.prev.type == tree.XML_DTD_NODE): + c_sibling = c_sibling.prev + while c_sibling != c_node: + if c_sibling.type == tree.XML_DTD_NODE: + c_copy = _copyDtd(c_sibling) + if c_sibling == c_node.doc.intSubset: + c_target.doc.intSubset = c_copy + else: # c_sibling == c_node.doc.extSubset + c_target.doc.extSubset = c_copy + else: + c_copy = tree.xmlDocCopyNode(c_sibling, c_target.doc, 1) + if c_copy is NULL: + raise MemoryError() + tree.xmlAddPrevSibling(c_target, c_copy) + c_sibling = c_sibling.next + while c_sibling.next != NULL and \ + (c_sibling.next.type == tree.XML_PI_NODE or + c_sibling.next.type == tree.XML_COMMENT_NODE): + c_sibling = c_sibling.next + c_copy = tree.xmlDocCopyNode(c_sibling, c_target.doc, 1) + if c_copy is NULL: + raise MemoryError() + tree.xmlAddNextSibling(c_target, c_copy) + +cdef int _deleteSlice(_Document doc, xmlNode* c_node, + Py_ssize_t count, Py_ssize_t step) except -1: + """Delete slice, ``count`` items starting with ``c_node`` with a step + width of ``step``. + """ + cdef xmlNode* c_next + cdef Py_ssize_t c, i + cdef _node_to_node_function next_element + if c_node is NULL: + return 0 + if step > 0: + next_element = _nextElement + else: + step = -step if step != python.PY_SSIZE_T_MIN else python.PY_SSIZE_T_MAX + next_element = _previousElement + # now start deleting nodes + c = 0 + c_next = c_node + while c_node is not NULL and c < count: + for i in range(step): + c_next = next_element(c_next) + if c_next is NULL: + break + _removeNode(doc, c_node) + c += 1 + c_node = c_next + return 0 + +cdef int _replaceSlice(_Element parent, xmlNode* c_node, + Py_ssize_t slicelength, Py_ssize_t step, + bint left_to_right, elements) except -1: + """Replace the slice of ``count`` elements starting at ``c_node`` with + positive step width ``step`` by the Elements in ``elements``. The + direction is given by the boolean argument ``left_to_right``. + + ``c_node`` may be NULL to indicate the end of the children list. + """ + cdef xmlNode* c_orig_neighbour + cdef xmlNode* c_next + cdef xmlDoc* c_source_doc + cdef _Element element + cdef Py_ssize_t seqlength, i, c + cdef _node_to_node_function next_element + assert step > 0, step + if left_to_right: + next_element = _nextElement + else: + next_element = _previousElement + + if not isinstance(elements, (list, tuple)): + elements = list(elements) + + if step != 1 or not left_to_right: + # *replacing* children stepwise with list => check size! + seqlength = len(elements) + if seqlength != slicelength: + raise ValueError, f"attempt to assign sequence of size {seqlength} " \ + f"to extended slice of size {slicelength}" + + if c_node is NULL: + # no children yet => add all elements straight away + if left_to_right: + for element in elements: + assert element is not None, "Node must not be None" + _appendChild(parent, element) + else: + for element in elements: + assert element is not None, "Node must not be None" + _prependChild(parent, element) + return 0 + + # remove the elements first as some might be re-added + if left_to_right: + # L->R, remember left neighbour + c_orig_neighbour = _previousElement(c_node) + else: + # R->L, remember right neighbour + c_orig_neighbour = _nextElement(c_node) + + # We remove the original slice elements one by one. Since we hold + # a Python reference to all elements that we will insert, it is + # safe to let _removeNode() try (and fail) to free them even if + # the element itself or one of its descendents will be reinserted. + c = 0 + c_next = c_node + while c_node is not NULL and c < slicelength: + for i in range(step): + c_next = next_element(c_next) + if c_next is NULL: + break + _removeNode(parent._doc, c_node) + c += 1 + c_node = c_next + + # make sure each element is inserted only once + elements = iter(elements) + + # find the first node right of the new insertion point + if left_to_right: + if c_orig_neighbour is not NULL: + c_node = next_element(c_orig_neighbour) + else: + # before the first element + c_node = _findChildForwards(parent._c_node, 0) + elif c_orig_neighbour is NULL: + # at the end, but reversed stepping + # append one element and go to the next insertion point + for element in elements: + assert element is not None, "Node must not be None" + _appendChild(parent, element) + c_node = element._c_node + if slicelength > 0: + slicelength -= 1 + for i in range(1, step): + c_node = next_element(c_node) + if c_node is NULL: + break + break + else: + c_node = c_orig_neighbour + + if left_to_right: + # adjust step size after removing slice as we are not stepping + # over the newly inserted elements + step -= 1 + + # now insert elements where we removed them + if c_node is not NULL: + for element in elements: + assert element is not None, "Node must not be None" + _assertValidNode(element) + # move element and tail over + c_source_doc = element._c_node.doc + c_next = element._c_node.next + tree.xmlAddPrevSibling(c_node, element._c_node) + _moveTail(c_next, element._c_node) + + # integrate element into new document + moveNodeToDocument(parent._doc, c_source_doc, element._c_node) + + # stop at the end of the slice + if slicelength > 0: + slicelength -= 1 + for i in range(step): + c_node = next_element(c_node) + if c_node is NULL: + break + if c_node is NULL: + break + else: + # everything inserted + return 0 + + # append the remaining elements at the respective end + if left_to_right: + for element in elements: + assert element is not None, "Node must not be None" + _assertValidNode(element) + _appendChild(parent, element) + else: + for element in elements: + assert element is not None, "Node must not be None" + _assertValidNode(element) + _prependChild(parent, element) + + return 0 + + +cdef int _linkChild(xmlNode* c_parent, xmlNode* c_node) except -1: + """Adaptation of 'xmlAddChild()' that deep-fix the document links iteratively. + """ + assert _isElement(c_node) + c_node.parent = c_parent + if c_parent.children is NULL: + c_parent.children = c_parent.last = c_node + else: + c_node.prev = c_parent.last + c_parent.last.next = c_node + c_parent.last = c_node + + _setTreeDoc(c_node, c_parent.doc) + return 0 + + +cdef int _appendChild(_Element parent, _Element child) except -1: + """Append a new child to a parent element. + """ + c_node = child._c_node + c_source_doc = c_node.doc + # prevent cycles + if _isAncestorOrSame(c_node, parent._c_node): + raise ValueError("cannot append parent to itself") + # store possible text node + c_next = c_node.next + # move node itself + tree.xmlUnlinkNode(c_node) + # do not call xmlAddChild() here since it would deep-traverse the tree + _linkChild(parent._c_node, c_node) + _moveTail(c_next, c_node) + # uh oh, elements may be pointing to different doc when + # parent element has moved; change them too.. + moveNodeToDocument(parent._doc, c_source_doc, c_node) + return 0 + +cdef int _prependChild(_Element parent, _Element child) except -1: + """Prepend a new child to a parent element. + """ + c_node = child._c_node + c_source_doc = c_node.doc + # prevent cycles + if _isAncestorOrSame(c_node, parent._c_node): + raise ValueError("cannot append parent to itself") + # store possible text node + c_next = c_node.next + # move node itself + c_child = _findChildForwards(parent._c_node, 0) + if c_child is NULL: + tree.xmlUnlinkNode(c_node) + # do not call xmlAddChild() here since it would deep-traverse the tree + _linkChild(parent._c_node, c_node) + else: + tree.xmlAddPrevSibling(c_child, c_node) + _moveTail(c_next, c_node) + # uh oh, elements may be pointing to different doc when + # parent element has moved; change them too.. + moveNodeToDocument(parent._doc, c_source_doc, c_node) + return 0 + +cdef int _appendSibling(_Element element, _Element sibling) except -1: + """Add a new sibling behind an element. + """ + return _addSibling(element, sibling, as_next=True) + +cdef int _prependSibling(_Element element, _Element sibling) except -1: + """Add a new sibling before an element. + """ + return _addSibling(element, sibling, as_next=False) + +cdef int _addSibling(_Element element, _Element sibling, bint as_next) except -1: + c_node = sibling._c_node + c_source_doc = c_node.doc + # prevent cycles + if _isAncestorOrSame(c_node, element._c_node): + if element._c_node is c_node: + return 0 # nothing to do + raise ValueError("cannot add ancestor as sibling, please break cycle first") + # store possible text node + c_next = c_node.next + # move node itself + if as_next: + # must insert after any tail text + c_next_node = _nextElement(element._c_node) + if c_next_node is NULL: + c_next_node = element._c_node + while c_next_node.next: + c_next_node = c_next_node.next + tree.xmlAddNextSibling(c_next_node, c_node) + else: + tree.xmlAddPrevSibling(c_next_node, c_node) + else: + tree.xmlAddPrevSibling(element._c_node, c_node) + _moveTail(c_next, c_node) + # uh oh, elements may be pointing to different doc when + # parent element has moved; change them too.. + moveNodeToDocument(element._doc, c_source_doc, c_node) + return 0 + +cdef inline bint isutf8(const_xmlChar* s) noexcept: + cdef xmlChar c = s[0] + while c != c'\0': + if c & 0x80: + return True + s += 1 + c = s[0] + return False + +cdef bint isutf8l(const_xmlChar* s, size_t length) noexcept: + """ + Search for non-ASCII characters in the string, knowing its length in advance. + """ + cdef unsigned int i + cdef unsigned long non_ascii_mask + cdef const unsigned long *lptr = s + + cdef const unsigned long *end = lptr + length // sizeof(unsigned long) + if length >= sizeof(non_ascii_mask): + # Build constant 0x80808080... mask (and let the C compiler fold it). + non_ascii_mask = 0 + for i in range(sizeof(non_ascii_mask) // 2): + non_ascii_mask = (non_ascii_mask << 16) | 0x8080 + + # Advance to long-aligned character before we start reading longs. + while (s) % sizeof(unsigned long) and s < end: + if s[0] & 0x80: + return True + s += 1 + + # Read one long at a time + lptr = s + while lptr < end: + if lptr[0] & non_ascii_mask: + return True + lptr += 1 + s = lptr + + while s < (end + length % sizeof(unsigned long)): + if s[0] & 0x80: + return True + s += 1 + + return False + +cdef int _is_valid_xml_ascii(bytes pystring) except -1: + """Check if a string is XML ascii content.""" + cdef signed char ch + # When ch is a *signed* char, non-ascii characters are negative integers + # and xmlIsChar_ch does not accept them. + for ch in pystring: + if not tree.xmlIsChar_ch(ch): + return 0 + return 1 + +cdef bint _is_valid_xml_utf8(bytes pystring) except -1: + """Check if a string is like valid UTF-8 XML content.""" + cdef const_xmlChar* s = _xcstr(pystring) + cdef const_xmlChar* c_end = s + len(pystring) + cdef unsigned long next3 = 0 + if s < c_end - 2: + next3 = (s[0] << 8) | (s[1]) + + while s < c_end - 2: + next3 = 0x00ffffff & ((next3 << 8) | s[2]) + if s[0] & 0x80: + # 0xefbfbe and 0xefbfbf are utf-8 encodings of + # forbidden characters \ufffe and \uffff + if next3 == 0x00efbfbe or next3 == 0x00efbfbf: + return 0 + # 0xeda080 and 0xedbfbf are utf-8 encodings of + # \ud800 and \udfff. Anything between them (inclusive) + # is forbidden, because they are surrogate blocks in utf-16. + if 0x00eda080 <= next3 <= 0x00edbfbf: + return 0 + elif not tree.xmlIsChar_ch(s[0]): + return 0 # invalid ascii char + s += 1 + + while s < c_end: + if not s[0] & 0x80 and not tree.xmlIsChar_ch(s[0]): + return 0 # invalid ascii char + s += 1 + + return 1 + +cdef inline unicode funicodeOrNone(const_xmlChar* s): + return funicode(s) if s is not NULL else None + +cdef inline unicode funicodeOrEmpty(const_xmlChar* s): + return funicode(s) if s is not NULL else '' + +cdef unicode funicode(const_xmlChar* s): + return s.decode('UTF-8') + +cdef bytes _utf8(object s): + """Test if a string is valid user input and encode it to UTF-8. + Reject all bytes/unicode input that contains non-XML characters. + Reject all bytes input that contains non-ASCII characters. + """ + cdef int valid + cdef bytes utf8_string + if isinstance(s, unicode): + utf8_string = (s).encode('utf8') + valid = _is_valid_xml_utf8(utf8_string) + elif isinstance(s, (bytes, bytearray)): + utf8_string = s if type(s) is bytes else bytes(s) + valid = _is_valid_xml_ascii(utf8_string) + else: + raise TypeError("Argument must be bytes or unicode, got '%.200s'" % type(s).__name__) + if not valid: + raise ValueError( + "All strings must be XML compatible: Unicode or ASCII, no NULL bytes or control characters") + return utf8_string + + +cdef bytes _utf8orNone(object s): + return _utf8(s) if s is not None else None + + +cdef enum: + NO_FILE_PATH = 0 + ABS_UNIX_FILE_PATH = 1 + ABS_WIN_FILE_PATH = 2 + REL_FILE_PATH = 3 + + +cdef bint _isFilePath(const_xmlChar* c_path) noexcept: + "simple heuristic to see if a path is a filename" + cdef xmlChar c + # test if it looks like an absolute Unix path or a Windows network path + if c_path[0] == c'/': + return ABS_UNIX_FILE_PATH + + # test if it looks like an absolute Windows path or URL + if c'a' <= c_path[0] <= c'z' or c'A' <= c_path[0] <= c'Z': + c_path += 1 + if c_path[0] == c':' and c_path[1] in b'\0\\': + return ABS_WIN_FILE_PATH # C: or C:\... + + # test if it looks like a URL with scheme:// + while c'a' <= c_path[0] <= c'z' or c'A' <= c_path[0] <= c'Z': + c_path += 1 + if c_path[0] == c':' and c_path[1] == c'/' and c_path[2] == c'/': + return NO_FILE_PATH + + # assume it's a relative path + return REL_FILE_PATH + + +cdef object _getFSPathOrObject(object obj): + """ + Get the __fspath__ attribute of an object if it exists. + Otherwise, the original object is returned. + """ + if _isString(obj): + return obj + try: + return python.PyOS_FSPath(obj) + except TypeError: + return obj + + +cdef object _encodeFilename(object filename): + """Make sure a filename is 8-bit encoded (or None). + """ + if filename is None: + return None + elif isinstance(filename, bytes): + return filename + elif isinstance(filename, unicode): + filename8 = (filename).encode('utf8') + if _isFilePath(filename8): + try: + return python.PyUnicode_AsEncodedString( + filename, _C_FILENAME_ENCODING, NULL) + except UnicodeEncodeError: + pass + return filename8 + else: + raise TypeError("Argument must be string or unicode.") + +cdef object _decodeFilename(const_xmlChar* c_path): + """Make the filename a unicode string if we are in Py3. + """ + return _decodeFilenameWithLength(c_path, tree.xmlStrlen(c_path)) + +cdef object _decodeFilenameWithLength(const_xmlChar* c_path, size_t c_len): + """Make the filename a unicode string if we are in Py3. + """ + if _isFilePath(c_path): + try: + return python.PyUnicode_Decode( + c_path, c_len, _C_FILENAME_ENCODING, NULL) + except UnicodeDecodeError: + pass + try: + return (c_path)[:c_len].decode('UTF-8') + except UnicodeDecodeError: + # this is a stupid fallback, but it might still work... + return (c_path)[:c_len].decode('latin-1', 'replace') + +cdef object _encodeFilenameUTF8(object filename): + """Recode filename as UTF-8. Tries ASCII, local filesystem encoding and + UTF-8 as source encoding. + """ + cdef char* c_filename + if filename is None: + return None + elif isinstance(filename, bytes): + if not isutf8l(filename, len(filename)): + # plain ASCII! + return filename + c_filename = _cstr(filename) + try: + # try to decode with default encoding + filename = python.PyUnicode_Decode( + c_filename, len(filename), + _C_FILENAME_ENCODING, NULL) + except UnicodeDecodeError as decode_exc: + try: + # try if it's proper UTF-8 + (filename).decode('utf8') + return filename + except UnicodeDecodeError: + raise decode_exc # otherwise re-raise original exception + if isinstance(filename, unicode): + return (filename).encode('utf8') + else: + raise TypeError("Argument must be string or unicode.") + +cdef tuple _getNsTag(tag): + """Given a tag, find namespace URI and tag name. + Return None for NS uri if no namespace URI provided. + """ + return __getNsTag(tag, 0) + +cdef tuple _getNsTagWithEmptyNs(tag): + """Given a tag, find namespace URI and tag name. Return None for NS uri + if no namespace URI provided, or the empty string if namespace + part is '{}'. + """ + return __getNsTag(tag, 1) + +cdef tuple __getNsTag(tag, bint empty_ns): + cdef char* c_tag + cdef char* c_ns_end + cdef Py_ssize_t taglen + cdef Py_ssize_t nslen + cdef bytes ns = None + # _isString() is much faster than isinstance() + if not _isString(tag) and isinstance(tag, QName): + tag = (tag).text + tag = _utf8(tag) + c_tag = _cstr(tag) + if c_tag[0] == c'{': + c_tag += 1 + c_ns_end = cstring_h.strchr(c_tag, c'}') + if c_ns_end is NULL: + raise ValueError, "Invalid tag name" + nslen = c_ns_end - c_tag + taglen = python.PyBytes_GET_SIZE(tag) - nslen - 2 + if taglen == 0: + raise ValueError, "Empty tag name" + if nslen > 0: + ns = c_tag[:nslen] + elif empty_ns: + ns = b'' + tag = c_ns_end[1:taglen+1] + elif python.PyBytes_GET_SIZE(tag) == 0: + raise ValueError, "Empty tag name" + return ns, tag + +cdef inline int _pyXmlNameIsValid(name_utf8): + return _xmlNameIsValid(_xcstr(name_utf8)) and b':' not in name_utf8 + +cdef inline int _pyHtmlNameIsValid(name_utf8): + return _htmlNameIsValid(_xcstr(name_utf8)) + +cdef inline int _xmlNameIsValid(const_xmlChar* c_name) noexcept: + return tree.xmlValidateNameValue(c_name) + +cdef int _htmlNameIsValid(const_xmlChar* c_name) noexcept: + if c_name is NULL or c_name[0] == c'\0': + return 0 + while c_name[0] != c'\0': + if c_name[0] in b'&<>/"\'\t\n\x0B\x0C\r ': + return 0 + c_name += 1 + return 1 + +cdef bint _characterReferenceIsValid(const_xmlChar* c_name) noexcept: + cdef bint is_hex + if c_name[0] == c'x': + c_name += 1 + is_hex = 1 + else: + is_hex = 0 + if c_name[0] == c'\0': + return 0 + while c_name[0] != c'\0': + if c_name[0] < c'0' or c_name[0] > c'9': + if not is_hex: + return 0 + if not (c'a' <= c_name[0] <= c'f'): + if not (c'A' <= c_name[0] <= c'F'): + return 0 + c_name += 1 + return 1 + +cdef int _tagValidOrRaise(tag_utf) except -1: + if not _pyXmlNameIsValid(tag_utf): + raise ValueError(f"Invalid tag name {(tag_utf).decode('utf8')!r}") + return 0 + +cdef int _htmlTagValidOrRaise(tag_utf) except -1: + if not _pyHtmlNameIsValid(tag_utf): + raise ValueError(f"Invalid HTML tag name {(tag_utf).decode('utf8')!r}") + return 0 + +cdef int _attributeValidOrRaise(name_utf) except -1: + if not _pyXmlNameIsValid(name_utf): + raise ValueError(f"Invalid attribute name {(name_utf).decode('utf8')!r}") + return 0 + +cdef int _prefixValidOrRaise(tag_utf) except -1: + if not _pyXmlNameIsValid(tag_utf): + raise ValueError(f"Invalid namespace prefix {(tag_utf).decode('utf8')!r}") + return 0 + +cdef int _uriValidOrRaise(uri_utf) except -1: + cdef uri.xmlURI* c_uri = uri.xmlParseURI(_cstr(uri_utf)) + if c_uri is NULL: + raise ValueError(f"Invalid namespace URI {(uri_utf).decode('utf8')!r}") + uri.xmlFreeURI(c_uri) + return 0 + +cdef inline unicode _namespacedName(xmlNode* c_node): + return _namespacedNameFromNsName(_getNs(c_node), c_node.name) + + +cdef unicode _namespacedNameFromNsName(const_xmlChar* c_href, const_xmlChar* c_name): + name = funicode(c_name) + if c_href is NULL: + return name + href = funicode(c_href) + return f"{{{href}}}{name}" + + +cdef _getFilenameForFile(source): + """Given a Python File or Gzip object, give filename back. + + Returns None if not a file object. + """ + # urllib2 provides a geturl() method + try: + return source.geturl() + except: + pass + # file instances have a name attribute + try: + filename = source.name + if _isString(filename): + return os_path_abspath(filename) + except: + pass + # gzip file instances have a filename attribute (before Py3k) + try: + filename = source.filename + if _isString(filename): + return os_path_abspath(filename) + except: + pass + # can't determine filename + return None diff --git a/.venv/lib/python3.13/site-packages/lxml/builder.cpython-313-x86_64-linux-gnu.so b/.venv/lib/python3.13/site-packages/lxml/builder.cpython-313-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..2f086609376e46f001ab170629a8fb4a8625b246 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/builder.cpython-313-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc9eeb21d39de5ff8c618348d4f4063317ec8aa88d39ee2ba6bca5a5d9544b02 +size 116784 diff --git a/.venv/lib/python3.13/site-packages/lxml/builder.py b/.venv/lib/python3.13/site-packages/lxml/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..55d9e0d5ae3a353eeb80282d6eeb956cb845f31a --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/builder.py @@ -0,0 +1,243 @@ +# cython: language_level=2 + +# +# Element generator factory by Fredrik Lundh. +# +# Source: +# http://online.effbot.org/2006_11_01_archive.htm#et-builder +# http://effbot.python-hosting.com/file/stuff/sandbox/elementlib/builder.py +# +# -------------------------------------------------------------------- +# The ElementTree toolkit is +# +# Copyright (c) 1999-2004 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Secret Labs AB or the author not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD +# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- +# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR +# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# -------------------------------------------------------------------- + +""" +The ``E`` Element factory for generating XML documents. +""" + + +import lxml.etree as ET +_QName = ET.QName + +from functools import partial + +try: + from types import GenericAlias as _GenericAlias +except ImportError: + # Python 3.8 - we only need this as return value from "__class_getitem__" + def _GenericAlias(cls, item): + return f"{cls.__name__}[{item.__name__}]" + +try: + basestring +except NameError: + basestring = str + +try: + unicode +except NameError: + unicode = str + + +class ElementMaker: + """Element generator factory. + + Unlike the ordinary Element factory, the E factory allows you to pass in + more than just a tag and some optional attributes; you can also pass in + text and other elements. The text is added as either text or tail + attributes, and elements are inserted at the right spot. Some small + examples:: + + >>> from lxml import etree as ET + >>> from lxml.builder import E + + >>> ET.tostring(E("tag")) + '' + >>> ET.tostring(E("tag", "text")) + 'text' + >>> ET.tostring(E("tag", "text", key="value")) + 'text' + >>> ET.tostring(E("tag", E("subtag", "text"), "tail")) + 'texttail' + + For simple tags, the factory also allows you to write ``E.tag(...)`` instead + of ``E('tag', ...)``:: + + >>> ET.tostring(E.tag()) + '' + >>> ET.tostring(E.tag("text")) + 'text' + >>> ET.tostring(E.tag(E.subtag("text"), "tail")) + 'texttail' + + Here's a somewhat larger example; this shows how to generate HTML + documents, using a mix of prepared factory functions for inline elements, + nested ``E.tag`` calls, and embedded XHTML fragments:: + + # some common inline elements + A = E.a + I = E.i + B = E.b + + def CLASS(v): + # helper function, 'class' is a reserved word + return {'class': v} + + page = ( + E.html( + E.head( + E.title("This is a sample document") + ), + E.body( + E.h1("Hello!", CLASS("title")), + E.p("This is a paragraph with ", B("bold"), " text in it!"), + E.p("This is another paragraph, with a ", + A("link", href="http://www.python.org"), "."), + E.p("Here are some reserved characters: ."), + ET.XML("

And finally, here is an embedded XHTML fragment.

"), + ) + ) + ) + + print ET.tostring(page) + + Here's a prettyprinted version of the output from the above script:: + + + + This is a sample document + + +

Hello!

+

This is a paragraph with bold text in it!

+

This is another paragraph, with link.

+

Here are some reserved characters: <spam&egg>.

+

And finally, here is an embedded XHTML fragment.

+ + + + For namespace support, you can pass a namespace map (``nsmap``) + and/or a specific target ``namespace`` to the ElementMaker class:: + + >>> E = ElementMaker(namespace="http://my.ns/") + >>> print(ET.tostring( E.test )) + + + >>> E = ElementMaker(namespace="http://my.ns/", nsmap={'p':'http://my.ns/'}) + >>> print(ET.tostring( E.test )) + + """ + + def __init__(self, typemap=None, + namespace=None, nsmap=None, makeelement=None): + self._namespace = '{' + namespace + '}' if namespace is not None else None + self._nsmap = dict(nsmap) if nsmap else None + + assert makeelement is None or callable(makeelement) + self._makeelement = makeelement if makeelement is not None else ET.Element + + # initialize the default type map functions for this element factory + typemap = dict(typemap) if typemap else {} + + def add_text(elem, item): + try: + last_child = elem[-1] + except IndexError: + elem.text = (elem.text or "") + item + else: + last_child.tail = (last_child.tail or "") + item + + def add_cdata(elem, cdata): + if elem.text: + raise ValueError("Can't add a CDATA section. Element already has some text: %r" % elem.text) + elem.text = cdata + + if str not in typemap: + typemap[str] = add_text + if unicode not in typemap: + typemap[unicode] = add_text + if ET.CDATA not in typemap: + typemap[ET.CDATA] = add_cdata + + def add_dict(elem, item): + attrib = elem.attrib + for k, v in item.items(): + if isinstance(v, basestring): + attrib[k] = v + else: + attrib[k] = typemap[type(v)](None, v) + + if dict not in typemap: + typemap[dict] = add_dict + + self._typemap = typemap + + def __call__(self, tag, *children, **attrib): + typemap = self._typemap + + # We'll usually get a 'str', and the compiled type check is very fast. + if not isinstance(tag, str) and isinstance(tag, _QName): + # A QName is explicitly qualified, do not look at self._namespace. + tag = tag.text + elif self._namespace is not None and tag[0] != '{': + tag = self._namespace + tag + elem = self._makeelement(tag, nsmap=self._nsmap) + if attrib: + typemap[dict](elem, attrib) + + for item in children: + if callable(item): + item = item() + t = typemap.get(type(item)) + if t is None: + if ET.iselement(item): + elem.append(item) + continue + for basetype in type(item).__mro__: + # See if the typemap knows of any of this type's bases. + t = typemap.get(basetype) + if t is not None: + break + else: + raise TypeError("bad argument type: %s(%r)" % + (type(item).__name__, item)) + v = t(elem, item) + if v: + typemap.get(type(v))(elem, v) + + return elem + + def __getattr__(self, tag): + return partial(self, tag) + + # Allow subscripting ElementMaker in type annotations (PEP 560) + def __class_getitem__(cls, item): + return _GenericAlias(cls, item) + + +# create factory object +E = ElementMaker() diff --git a/.venv/lib/python3.13/site-packages/lxml/classlookup.pxi b/.venv/lib/python3.13/site-packages/lxml/classlookup.pxi new file mode 100644 index 0000000000000000000000000000000000000000..92d1d47a58657a7741d20f48cfe3525a66dbc722 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/classlookup.pxi @@ -0,0 +1,580 @@ +# Configurable Element class lookup + +################################################################################ +# Custom Element classes + +cdef public class ElementBase(_Element) [ type LxmlElementBaseType, + object LxmlElementBase ]: + """ElementBase(*children, attrib=None, nsmap=None, **_extra) + + The public Element class. All custom Element classes must inherit + from this one. To create an Element, use the `Element()` factory. + + BIG FAT WARNING: Subclasses *must not* override __init__ or + __new__ as it is absolutely undefined when these objects will be + created or destroyed. All persistent state of Elements must be + stored in the underlying XML. If you really need to initialize + the object after creation, you can implement an ``_init(self)`` + method that will be called directly after object creation. + + Subclasses of this class can be instantiated to create a new + Element. By default, the tag name will be the class name and the + namespace will be empty. You can modify this with the following + class attributes: + + * TAG - the tag name, possibly containing a namespace in Clark + notation + + * NAMESPACE - the default namespace URI, unless provided as part + of the TAG attribute. + + * HTML - flag if the class is an HTML tag, as opposed to an XML + tag. This only applies to un-namespaced tags and defaults to + false (i.e. XML). + + * PARSER - the parser that provides the configuration for the + newly created document. Providing an HTML parser here will + default to creating an HTML element. + + In user code, the latter three are commonly inherited in class + hierarchies that implement a common namespace. + """ + def __init__(self, *children, attrib=None, nsmap=None, **_extra): + """ElementBase(*children, attrib=None, nsmap=None, **_extra) + """ + cdef bint is_html = 0 + cdef _BaseParser parser + cdef _Element last_child + # don't use normal attribute access as it might be overridden + _getattr = object.__getattribute__ + try: + namespace = _utf8(_getattr(self, 'NAMESPACE')) + except AttributeError: + namespace = None + try: + ns, tag = _getNsTag(_getattr(self, 'TAG')) + if ns is not None: + namespace = ns + except AttributeError: + tag = _utf8(_getattr(_getattr(self, '__class__'), '__name__')) + if b'.' in tag: + tag = tag.split(b'.')[-1] + try: + parser = _getattr(self, 'PARSER') + except AttributeError: + parser = None + for child in children: + if isinstance(child, _Element): + parser = (<_Element>child)._doc._parser + break + if isinstance(parser, HTMLParser): + is_html = 1 + if namespace is None: + try: + is_html = _getattr(self, 'HTML') + except AttributeError: + pass + _initNewElement(self, is_html, tag, namespace, parser, + attrib, nsmap, _extra) + last_child = None + for child in children: + if _isString(child): + if last_child is None: + _setNodeText(self._c_node, + (_collectText(self._c_node.children) or '') + child) + else: + _setTailText(last_child._c_node, + (_collectText(last_child._c_node.next) or '') + child) + elif isinstance(child, _Element): + last_child = child + _appendChild(self, last_child) + elif isinstance(child, type) and issubclass(child, ElementBase): + last_child = child() + _appendChild(self, last_child) + else: + raise TypeError, f"Invalid child type: {type(child)!r}" + +cdef class CommentBase(_Comment): + """All custom Comment classes must inherit from this one. + + To create an XML Comment instance, use the ``Comment()`` factory. + + Subclasses *must not* override __init__ or __new__ as it is + absolutely undefined when these objects will be created or + destroyed. All persistent state of Comments must be stored in the + underlying XML. If you really need to initialize the object after + creation, you can implement an ``_init(self)`` method that will be + called after object creation. + """ + def __init__(self, text): + # copied from Comment() factory + cdef _Document doc + cdef xmlDoc* c_doc + if text is None: + text = b'' + else: + text = _utf8(text) + c_doc = _newXMLDoc() + doc = _documentFactory(c_doc, None) + self._c_node = _createComment(c_doc, _xcstr(text)) + if self._c_node is NULL: + raise MemoryError() + tree.xmlAddChild(c_doc, self._c_node) + _registerProxy(self, doc, self._c_node) + self._init() + +cdef class PIBase(_ProcessingInstruction): + """All custom Processing Instruction classes must inherit from this one. + + To create an XML ProcessingInstruction instance, use the ``PI()`` + factory. + + Subclasses *must not* override __init__ or __new__ as it is + absolutely undefined when these objects will be created or + destroyed. All persistent state of PIs must be stored in the + underlying XML. If you really need to initialize the object after + creation, you can implement an ``_init(self)`` method that will be + called after object creation. + """ + def __init__(self, target, text=None): + # copied from PI() factory + cdef _Document doc + cdef xmlDoc* c_doc + target = _utf8(target) + if text is None: + text = b'' + else: + text = _utf8(text) + c_doc = _newXMLDoc() + doc = _documentFactory(c_doc, None) + self._c_node = _createPI(c_doc, _xcstr(target), _xcstr(text)) + if self._c_node is NULL: + raise MemoryError() + tree.xmlAddChild(c_doc, self._c_node) + _registerProxy(self, doc, self._c_node) + self._init() + +cdef class EntityBase(_Entity): + """All custom Entity classes must inherit from this one. + + To create an XML Entity instance, use the ``Entity()`` factory. + + Subclasses *must not* override __init__ or __new__ as it is + absolutely undefined when these objects will be created or + destroyed. All persistent state of Entities must be stored in the + underlying XML. If you really need to initialize the object after + creation, you can implement an ``_init(self)`` method that will be + called after object creation. + """ + def __init__(self, name): + cdef _Document doc + cdef xmlDoc* c_doc + name_utf = _utf8(name) + c_name = _xcstr(name_utf) + if c_name[0] == c'#': + if not _characterReferenceIsValid(c_name + 1): + raise ValueError, f"Invalid character reference: '{name}'" + elif not _xmlNameIsValid(c_name): + raise ValueError, f"Invalid entity reference: '{name}'" + c_doc = _newXMLDoc() + doc = _documentFactory(c_doc, None) + self._c_node = _createEntity(c_doc, c_name) + if self._c_node is NULL: + raise MemoryError() + tree.xmlAddChild(c_doc, self._c_node) + _registerProxy(self, doc, self._c_node) + self._init() + + +cdef int _validateNodeClass(xmlNode* c_node, cls) except -1: + if c_node.type == tree.XML_ELEMENT_NODE: + expected = ElementBase + elif c_node.type == tree.XML_COMMENT_NODE: + expected = CommentBase + elif c_node.type == tree.XML_ENTITY_REF_NODE: + expected = EntityBase + elif c_node.type == tree.XML_PI_NODE: + expected = PIBase + else: + assert False, f"Unknown node type: {c_node.type}" + + if not (isinstance(cls, type) and issubclass(cls, expected)): + raise TypeError( + f"result of class lookup must be subclass of {type(expected)}, got {type(cls)}") + return 0 + + +################################################################################ +# Element class lookup + +ctypedef public object (*_element_class_lookup_function)(object, _Document, xmlNode*) + +# class to store element class lookup functions +cdef public class ElementClassLookup [ type LxmlElementClassLookupType, + object LxmlElementClassLookup ]: + """ElementClassLookup(self) + Superclass of Element class lookups. + """ + cdef _element_class_lookup_function _lookup_function + + +cdef public class FallbackElementClassLookup(ElementClassLookup) \ + [ type LxmlFallbackElementClassLookupType, + object LxmlFallbackElementClassLookup ]: + """FallbackElementClassLookup(self, fallback=None) + + Superclass of Element class lookups with additional fallback. + """ + cdef readonly ElementClassLookup fallback + cdef _element_class_lookup_function _fallback_function + def __cinit__(self): + # fall back to default lookup + self._fallback_function = _lookupDefaultElementClass + + def __init__(self, ElementClassLookup fallback=None): + if fallback is not None: + self._setFallback(fallback) + else: + self._fallback_function = _lookupDefaultElementClass + + cdef void _setFallback(self, ElementClassLookup lookup): + """Sets the fallback scheme for this lookup method. + """ + self.fallback = lookup + self._fallback_function = lookup._lookup_function + if self._fallback_function is NULL: + self._fallback_function = _lookupDefaultElementClass + + def set_fallback(self, ElementClassLookup lookup not None): + """set_fallback(self, lookup) + + Sets the fallback scheme for this lookup method. + """ + self._setFallback(lookup) + +cdef inline object _callLookupFallback(FallbackElementClassLookup lookup, + _Document doc, xmlNode* c_node): + return lookup._fallback_function(lookup.fallback, doc, c_node) + + +################################################################################ +# default lookup scheme + +cdef class ElementDefaultClassLookup(ElementClassLookup): + """ElementDefaultClassLookup(self, element=None, comment=None, pi=None, entity=None) + Element class lookup scheme that always returns the default Element + class. + + The keyword arguments ``element``, ``comment``, ``pi`` and ``entity`` + accept the respective Element classes. + """ + cdef readonly object element_class + cdef readonly object comment_class + cdef readonly object pi_class + cdef readonly object entity_class + def __cinit__(self): + self._lookup_function = _lookupDefaultElementClass + + def __init__(self, element=None, comment=None, pi=None, entity=None): + if element is None: + self.element_class = _Element + elif issubclass(element, ElementBase): + self.element_class = element + else: + raise TypeError, "element class must be subclass of ElementBase" + + if comment is None: + self.comment_class = _Comment + elif issubclass(comment, CommentBase): + self.comment_class = comment + else: + raise TypeError, "comment class must be subclass of CommentBase" + + if entity is None: + self.entity_class = _Entity + elif issubclass(entity, EntityBase): + self.entity_class = entity + else: + raise TypeError, "Entity class must be subclass of EntityBase" + + if pi is None: + self.pi_class = None # special case, see below + elif issubclass(pi, PIBase): + self.pi_class = pi + else: + raise TypeError, "PI class must be subclass of PIBase" + +cdef object _lookupDefaultElementClass(state, _Document _doc, xmlNode* c_node): + "Trivial class lookup function that always returns the default class." + if c_node.type == tree.XML_ELEMENT_NODE: + if state is not None: + return (state).element_class + else: + return _Element + elif c_node.type == tree.XML_COMMENT_NODE: + if state is not None: + return (state).comment_class + else: + return _Comment + elif c_node.type == tree.XML_ENTITY_REF_NODE: + if state is not None: + return (state).entity_class + else: + return _Entity + elif c_node.type == tree.XML_PI_NODE: + if state is None or (state).pi_class is None: + # special case XSLT-PI + if c_node.name is not NULL and c_node.content is not NULL: + if tree.xmlStrcmp(c_node.name, "xml-stylesheet") == 0: + if tree.xmlStrstr(c_node.content, "text/xsl") is not NULL or \ + tree.xmlStrstr(c_node.content, "text/xml") is not NULL: + return _XSLTProcessingInstruction + return _ProcessingInstruction + else: + return (state).pi_class + else: + assert False, f"Unknown node type: {c_node.type}" + + +################################################################################ +# attribute based lookup scheme + +cdef class AttributeBasedElementClassLookup(FallbackElementClassLookup): + """AttributeBasedElementClassLookup(self, attribute_name, class_mapping, fallback=None) + Checks an attribute of an Element and looks up the value in a + class dictionary. + + Arguments: + - attribute name - '{ns}name' style string + - class mapping - Python dict mapping attribute values to Element classes + - fallback - optional fallback lookup mechanism + + A None key in the class mapping will be checked if the attribute is + missing. + """ + cdef object _class_mapping + cdef tuple _pytag + cdef const_xmlChar* _c_ns + cdef const_xmlChar* _c_name + def __cinit__(self): + self._lookup_function = _attribute_class_lookup + + def __init__(self, attribute_name, class_mapping, + ElementClassLookup fallback=None): + self._pytag = _getNsTag(attribute_name) + ns, name = self._pytag + if ns is None: + self._c_ns = NULL + else: + self._c_ns = _xcstr(ns) + self._c_name = _xcstr(name) + self._class_mapping = dict(class_mapping) + + FallbackElementClassLookup.__init__(self, fallback) + +cdef object _attribute_class_lookup(state, _Document doc, xmlNode* c_node): + cdef AttributeBasedElementClassLookup lookup + cdef python.PyObject* dict_result + + lookup = state + if c_node.type == tree.XML_ELEMENT_NODE: + value = _attributeValueFromNsName( + c_node, lookup._c_ns, lookup._c_name) + dict_result = python.PyDict_GetItem(lookup._class_mapping, value) + if dict_result is not NULL: + cls = dict_result + _validateNodeClass(c_node, cls) + return cls + return _callLookupFallback(lookup, doc, c_node) + + +################################################################################ +# per-parser lookup scheme + +cdef class ParserBasedElementClassLookup(FallbackElementClassLookup): + """ParserBasedElementClassLookup(self, fallback=None) + Element class lookup based on the XML parser. + """ + def __cinit__(self): + self._lookup_function = _parser_class_lookup + +cdef object _parser_class_lookup(state, _Document doc, xmlNode* c_node): + if doc._parser._class_lookup is not None: + return doc._parser._class_lookup._lookup_function( + doc._parser._class_lookup, doc, c_node) + return _callLookupFallback(state, doc, c_node) + + +################################################################################ +# custom class lookup based on node type, namespace, name + +cdef class CustomElementClassLookup(FallbackElementClassLookup): + """CustomElementClassLookup(self, fallback=None) + Element class lookup based on a subclass method. + + You can inherit from this class and override the method:: + + lookup(self, type, doc, namespace, name) + + to lookup the element class for a node. Arguments of the method: + * type: one of 'element', 'comment', 'PI', 'entity' + * doc: document that the node is in + * namespace: namespace URI of the node (or None for comments/PIs/entities) + * name: name of the element/entity, None for comments, target for PIs + + If you return None from this method, the fallback will be called. + """ + def __cinit__(self): + self._lookup_function = _custom_class_lookup + + def lookup(self, type, doc, namespace, name): + "lookup(self, type, doc, namespace, name)" + return None + +cdef object _custom_class_lookup(state, _Document doc, xmlNode* c_node): + cdef CustomElementClassLookup lookup + + lookup = state + + if c_node.type == tree.XML_ELEMENT_NODE: + element_type = "element" + elif c_node.type == tree.XML_COMMENT_NODE: + element_type = "comment" + elif c_node.type == tree.XML_PI_NODE: + element_type = "PI" + elif c_node.type == tree.XML_ENTITY_REF_NODE: + element_type = "entity" + else: + element_type = "element" + if c_node.name is NULL: + name = None + else: + name = funicode(c_node.name) + c_str = tree._getNs(c_node) + ns = funicode(c_str) if c_str is not NULL else None + + cls = lookup.lookup(element_type, doc, ns, name) + if cls is not None: + _validateNodeClass(c_node, cls) + return cls + return _callLookupFallback(lookup, doc, c_node) + + +################################################################################ +# read-only tree based class lookup + +cdef class PythonElementClassLookup(FallbackElementClassLookup): + """PythonElementClassLookup(self, fallback=None) + Element class lookup based on a subclass method. + + This class lookup scheme allows access to the entire XML tree in + read-only mode. To use it, re-implement the ``lookup(self, doc, + root)`` method in a subclass:: + + from lxml import etree, pyclasslookup + + class MyElementClass(etree.ElementBase): + honkey = True + + class MyLookup(pyclasslookup.PythonElementClassLookup): + def lookup(self, doc, root): + if root.tag == "sometag": + return MyElementClass + else: + for child in root: + if child.tag == "someothertag": + return MyElementClass + # delegate to default + return None + + If you return None from this method, the fallback will be called. + + The first argument is the opaque document instance that contains + the Element. The second argument is a lightweight Element proxy + implementation that is only valid during the lookup. Do not try + to keep a reference to it. Once the lookup is done, the proxy + will be invalid. + + Also, you cannot wrap such a read-only Element in an ElementTree, + and you must take care not to keep a reference to them outside of + the `lookup()` method. + + Note that the API of the Element objects is not complete. It is + purely read-only and does not support all features of the normal + `lxml.etree` API (such as XPath, extended slicing or some + iteration methods). + + See https://lxml.de/element_classes.html + """ + def __cinit__(self): + self._lookup_function = _python_class_lookup + + def lookup(self, doc, element): + """lookup(self, doc, element) + + Override this method to implement your own lookup scheme. + """ + return None + +cdef object _python_class_lookup(state, _Document doc, tree.xmlNode* c_node): + cdef PythonElementClassLookup lookup + cdef _ReadOnlyProxy proxy + lookup = state + + proxy = _newReadOnlyProxy(None, c_node) + cls = lookup.lookup(doc, proxy) + _freeReadOnlyProxies(proxy) + + if cls is not None: + _validateNodeClass(c_node, cls) + return cls + return _callLookupFallback(lookup, doc, c_node) + +################################################################################ +# Global setup + +cdef _element_class_lookup_function LOOKUP_ELEMENT_CLASS +cdef object ELEMENT_CLASS_LOOKUP_STATE + +cdef void _setElementClassLookupFunction( + _element_class_lookup_function function, object state): + global LOOKUP_ELEMENT_CLASS, ELEMENT_CLASS_LOOKUP_STATE + if function is NULL: + state = DEFAULT_ELEMENT_CLASS_LOOKUP + function = DEFAULT_ELEMENT_CLASS_LOOKUP._lookup_function + + ELEMENT_CLASS_LOOKUP_STATE = state + LOOKUP_ELEMENT_CLASS = function + +def set_element_class_lookup(ElementClassLookup lookup = None): + """set_element_class_lookup(lookup = None) + + Set the global element class lookup method. + + This defines the main entry point for looking up element implementations. + The standard implementation uses the :class:`ParserBasedElementClassLookup` + to delegate to different lookup schemes for each parser. + + .. warning:: + + This should only be changed by applications, not by library packages. + In most cases, parser specific lookups should be preferred, + which can be configured via + :meth:`~lxml.etree.XMLParser.set_element_class_lookup` + (and the same for HTML parsers). + + Globally replacing the element class lookup by something other than a + :class:`ParserBasedElementClassLookup` will prevent parser specific lookup + schemes from working. Several tools rely on parser specific lookups, + including :mod:`lxml.html` and :mod:`lxml.objectify`. + """ + if lookup is None or lookup._lookup_function is NULL: + _setElementClassLookupFunction(NULL, None) + else: + _setElementClassLookupFunction(lookup._lookup_function, lookup) + +# default setup: parser delegation +cdef ParserBasedElementClassLookup DEFAULT_ELEMENT_CLASS_LOOKUP +DEFAULT_ELEMENT_CLASS_LOOKUP = ParserBasedElementClassLookup() + +set_element_class_lookup(DEFAULT_ELEMENT_CLASS_LOOKUP) diff --git a/.venv/lib/python3.13/site-packages/lxml/cleanup.pxi b/.venv/lib/python3.13/site-packages/lxml/cleanup.pxi new file mode 100644 index 0000000000000000000000000000000000000000..8e266b33f0f3aef34f3448276abfb2cb8b1e4772 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/cleanup.pxi @@ -0,0 +1,215 @@ +# functions for tree cleanup and removing elements from subtrees + +def cleanup_namespaces(tree_or_element, top_nsmap=None, keep_ns_prefixes=None): + """cleanup_namespaces(tree_or_element, top_nsmap=None, keep_ns_prefixes=None) + + Remove all namespace declarations from a subtree that are not used + by any of the elements or attributes in that tree. + + If a 'top_nsmap' is provided, it must be a mapping from prefixes + to namespace URIs. These namespaces will be declared on the top + element of the subtree before running the cleanup, which allows + moving namespace declarations to the top of the tree. + + If a 'keep_ns_prefixes' is provided, it must be a list of prefixes. + These prefixes will not be removed as part of the cleanup. + """ + element = _rootNodeOrRaise(tree_or_element) + c_element = element._c_node + + if top_nsmap: + doc = element._doc + # declare namespaces from nsmap, then apply them to the subtree + _setNodeNamespaces(c_element, doc, None, top_nsmap) + moveNodeToDocument(doc, c_element.doc, c_element) + + keep_ns_prefixes = ( + set([_utf8(prefix) for prefix in keep_ns_prefixes]) + if keep_ns_prefixes else None) + + _removeUnusedNamespaceDeclarations(c_element, keep_ns_prefixes) + + +def strip_attributes(tree_or_element, *attribute_names): + """strip_attributes(tree_or_element, *attribute_names) + + Delete all attributes with the provided attribute names from an + Element (or ElementTree) and its descendants. + + Attribute names can contain wildcards as in `_Element.iter`. + + Example usage:: + + strip_attributes(root_element, + 'simpleattr', + '{http://some/ns}attrname', + '{http://other/ns}*') + """ + cdef _MultiTagMatcher matcher + element = _rootNodeOrRaise(tree_or_element) + if not attribute_names: + return + + matcher = _MultiTagMatcher.__new__(_MultiTagMatcher, attribute_names) + matcher.cacheTags(element._doc) + if matcher.rejectsAllAttributes(): + return + _strip_attributes(element._c_node, matcher) + + +cdef _strip_attributes(xmlNode* c_node, _MultiTagMatcher matcher): + cdef xmlAttr* c_attr + cdef xmlAttr* c_next_attr + tree.BEGIN_FOR_EACH_ELEMENT_FROM(c_node, c_node, 1) + if c_node.type == tree.XML_ELEMENT_NODE: + c_attr = c_node.properties + while c_attr is not NULL: + c_next_attr = c_attr.next + if matcher.matchesAttribute(c_attr): + tree.xmlRemoveProp(c_attr) + c_attr = c_next_attr + tree.END_FOR_EACH_ELEMENT_FROM(c_node) + + +def strip_elements(tree_or_element, *tag_names, bint with_tail=True): + """strip_elements(tree_or_element, *tag_names, with_tail=True) + + Delete all elements with the provided tag names from a tree or + subtree. This will remove the elements and their entire subtree, + including all their attributes, text content and descendants. It + will also remove the tail text of the element unless you + explicitly set the ``with_tail`` keyword argument option to False. + + Tag names can contain wildcards as in `_Element.iter`. + + Note that this will not delete the element (or ElementTree root + element) that you passed even if it matches. It will only treat + its descendants. If you want to include the root element, check + its tag name directly before even calling this function. + + Example usage:: + + strip_elements(some_element, + 'simpletagname', # non-namespaced tag + '{http://some/ns}tagname', # namespaced tag + '{http://some/other/ns}*' # any tag from a namespace + lxml.etree.Comment # comments + ) + """ + cdef _MultiTagMatcher matcher + doc = _documentOrRaise(tree_or_element) + element = _rootNodeOrRaise(tree_or_element) + if not tag_names: + return + + matcher = _MultiTagMatcher.__new__(_MultiTagMatcher, tag_names) + matcher.cacheTags(doc) + if matcher.rejectsAll(): + return + + if isinstance(tree_or_element, _ElementTree): + # include PIs and comments next to the root node + if matcher.matchesType(tree.XML_COMMENT_NODE): + _removeSiblings(element._c_node, tree.XML_COMMENT_NODE, with_tail) + if matcher.matchesType(tree.XML_PI_NODE): + _removeSiblings(element._c_node, tree.XML_PI_NODE, with_tail) + _strip_elements(doc, element._c_node, matcher, with_tail) + +cdef _strip_elements(_Document doc, xmlNode* c_node, _MultiTagMatcher matcher, + bint with_tail): + cdef xmlNode* c_child + cdef xmlNode* c_next + + tree.BEGIN_FOR_EACH_ELEMENT_FROM(c_node, c_node, 1) + if c_node.type == tree.XML_ELEMENT_NODE: + # we run through the children here to prevent any problems + # with the tree iteration which would occur if we unlinked the + # c_node itself + c_child = _findChildForwards(c_node, 0) + while c_child is not NULL: + c_next = _nextElement(c_child) + if matcher.matches(c_child): + if c_child.type == tree.XML_ELEMENT_NODE: + if not with_tail: + tree.xmlUnlinkNode(c_child) + _removeNode(doc, c_child) + else: + if with_tail: + _removeText(c_child.next) + tree.xmlUnlinkNode(c_child) + attemptDeallocation(c_child) + c_child = c_next + tree.END_FOR_EACH_ELEMENT_FROM(c_node) + + +def strip_tags(tree_or_element, *tag_names): + """strip_tags(tree_or_element, *tag_names) + + Delete all elements with the provided tag names from a tree or + subtree. This will remove the elements and their attributes, but + *not* their text/tail content or descendants. Instead, it will + merge the text content and children of the element into its + parent. + + Tag names can contain wildcards as in `_Element.iter`. + + Note that this will not delete the element (or ElementTree root + element) that you passed even if it matches. It will only treat + its descendants. + + Example usage:: + + strip_tags(some_element, + 'simpletagname', # non-namespaced tag + '{http://some/ns}tagname', # namespaced tag + '{http://some/other/ns}*' # any tag from a namespace + Comment # comments (including their text!) + ) + """ + cdef _MultiTagMatcher matcher + doc = _documentOrRaise(tree_or_element) + element = _rootNodeOrRaise(tree_or_element) + if not tag_names: + return + + matcher = _MultiTagMatcher.__new__(_MultiTagMatcher, tag_names) + matcher.cacheTags(doc) + if matcher.rejectsAll(): + return + + if isinstance(tree_or_element, _ElementTree): + # include PIs and comments next to the root node + if matcher.matchesType(tree.XML_COMMENT_NODE): + _removeSiblings(element._c_node, tree.XML_COMMENT_NODE, 0) + if matcher.matchesType(tree.XML_PI_NODE): + _removeSiblings(element._c_node, tree.XML_PI_NODE, 0) + _strip_tags(doc, element._c_node, matcher) + +cdef _strip_tags(_Document doc, xmlNode* c_node, _MultiTagMatcher matcher): + cdef xmlNode* c_child + cdef xmlNode* c_next + + tree.BEGIN_FOR_EACH_ELEMENT_FROM(c_node, c_node, 1) + if c_node.type == tree.XML_ELEMENT_NODE: + # we run through the children here to prevent any problems + # with the tree iteration which would occur if we unlinked the + # c_node itself + c_child = _findChildForwards(c_node, 0) + while c_child is not NULL: + if not matcher.matches(c_child): + c_child = _nextElement(c_child) + continue + if c_child.type == tree.XML_ELEMENT_NODE: + c_next = _findChildForwards(c_child, 0) or _nextElement(c_child) + _replaceNodeByChildren(doc, c_child) + if not attemptDeallocation(c_child): + if c_child.nsDef is not NULL: + # make namespaces absolute + moveNodeToDocument(doc, doc._c_doc, c_child) + c_child = c_next + else: + c_next = _nextElement(c_child) + tree.xmlUnlinkNode(c_child) + attemptDeallocation(c_child) + c_child = c_next + tree.END_FOR_EACH_ELEMENT_FROM(c_node) diff --git a/.venv/lib/python3.13/site-packages/lxml/cssselect.py b/.venv/lib/python3.13/site-packages/lxml/cssselect.py new file mode 100644 index 0000000000000000000000000000000000000000..54cd75ac9bfecdec7ea81e91b0840c6edd401515 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/cssselect.py @@ -0,0 +1,101 @@ +"""CSS Selectors based on XPath. + +This module supports selecting XML/HTML tags based on CSS selectors. +See the `CSSSelector` class for details. + +This is a thin wrapper around cssselect 0.7 or later. +""" + + +from . import etree +try: + import cssselect as external_cssselect +except ImportError: + raise ImportError( + 'cssselect does not seem to be installed. ' + 'See https://pypi.org/project/cssselect/') + + +SelectorSyntaxError = external_cssselect.SelectorSyntaxError +ExpressionError = external_cssselect.ExpressionError +SelectorError = external_cssselect.SelectorError + + +__all__ = ['SelectorSyntaxError', 'ExpressionError', 'SelectorError', + 'CSSSelector'] + + +class LxmlTranslator(external_cssselect.GenericTranslator): + """ + A custom CSS selector to XPath translator with lxml-specific extensions. + """ + def xpath_contains_function(self, xpath, function): + # Defined there, removed in later drafts: + # http://www.w3.org/TR/2001/CR-css3-selectors-20011113/#content-selectors + if function.argument_types() not in (['STRING'], ['IDENT']): + raise ExpressionError( + "Expected a single string or ident for :contains(), got %r" + % function.arguments) + value = function.arguments[0].value + return xpath.add_condition( + 'contains(__lxml_internal_css:lower-case(string(.)), %s)' + % self.xpath_literal(value.lower())) + + +class LxmlHTMLTranslator(LxmlTranslator, external_cssselect.HTMLTranslator): + """ + lxml extensions + HTML support. + """ + + +def _make_lower_case(context, s): + return s.lower() + +ns = etree.FunctionNamespace('http://codespeak.net/lxml/css/') +ns.prefix = '__lxml_internal_css' +ns['lower-case'] = _make_lower_case + + +class CSSSelector(etree.XPath): + """A CSS selector. + + Usage:: + + >>> from lxml import etree, cssselect + >>> select = cssselect.CSSSelector("a tag > child") + + >>> root = etree.XML("TEXT") + >>> [ el.tag for el in select(root) ] + ['child'] + + To use CSS namespaces, you need to pass a prefix-to-namespace + mapping as ``namespaces`` keyword argument:: + + >>> rdfns = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#' + >>> select_ns = cssselect.CSSSelector('root > rdf|Description', + ... namespaces={'rdf': rdfns}) + + >>> rdf = etree.XML(( + ... '' + ... 'blah' + ... '') % rdfns) + >>> [(el.tag, el.text) for el in select_ns(rdf)] + [('{http://www.w3.org/1999/02/22-rdf-syntax-ns#}Description', 'blah')] + + """ + def __init__(self, css, namespaces=None, translator='xml'): + if translator == 'xml': + translator = LxmlTranslator() + elif translator == 'html': + translator = LxmlHTMLTranslator() + elif translator == 'xhtml': + translator = LxmlHTMLTranslator(xhtml=True) + path = translator.css_to_xpath(css) + super().__init__(path, namespaces=namespaces) + self.css = css + + def __repr__(self): + return '<%s %x for %r>' % ( + self.__class__.__name__, + abs(id(self)), + self.css) diff --git a/.venv/lib/python3.13/site-packages/lxml/debug.pxi b/.venv/lib/python3.13/site-packages/lxml/debug.pxi new file mode 100644 index 0000000000000000000000000000000000000000..d728e841937db9c1f81949a5d0a2e5f1d0e1e594 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/debug.pxi @@ -0,0 +1,36 @@ +@cython.final +@cython.internal +cdef class _MemDebug: + """Debugging support for the memory allocation in libxml2. + """ + def bytes_used(self): + """bytes_used(self) + + Returns the total amount of memory (in bytes) currently used by libxml2. + Note that libxml2 constrains this value to a C int, which limits + the accuracy on 64 bit systems. + """ + return tree.xmlMemUsed() + + def blocks_used(self): + """blocks_used(self) + + Returns the total number of memory blocks currently allocated by libxml2. + Note that libxml2 constrains this value to a C int, which limits + the accuracy on 64 bit systems. + """ + return tree.xmlMemBlocks() + + def dict_size(self): + """dict_size(self) + + Returns the current size of the global name dictionary used by libxml2 + for the current thread. Each thread has its own dictionary. + """ + c_dict = __GLOBAL_PARSER_CONTEXT._getThreadDict(NULL) + if c_dict is NULL: + raise MemoryError() + return tree.xmlDictSize(c_dict) + + +memory_debugger = _MemDebug() diff --git a/.venv/lib/python3.13/site-packages/lxml/docloader.pxi b/.venv/lib/python3.13/site-packages/lxml/docloader.pxi new file mode 100644 index 0000000000000000000000000000000000000000..07e0cd7e0ed52b14ec88b2876308fae66dfd651e --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/docloader.pxi @@ -0,0 +1,179 @@ +# Custom resolver API + +ctypedef enum _InputDocumentDataType: + PARSER_DATA_INVALID + PARSER_DATA_EMPTY + PARSER_DATA_STRING + PARSER_DATA_FILENAME + PARSER_DATA_FILE + +@cython.final +@cython.internal +cdef class _InputDocument: + cdef _InputDocumentDataType _type + cdef bytes _data_bytes + cdef object _filename + cdef object _file + cdef bint _close_file + + def __cinit__(self): + self._type = PARSER_DATA_INVALID + + +cdef class Resolver: + "This is the base class of all resolvers." + def resolve(self, system_url, public_id, context): + """resolve(self, system_url, public_id, context) + + Override this method to resolve an external source by + ``system_url`` and ``public_id``. The third argument is an + opaque context object. + + Return the result of one of the ``resolve_*()`` methods. + """ + return None + + def resolve_empty(self, context): + """resolve_empty(self, context) + + Return an empty input document. + + Pass context as parameter. + """ + cdef _InputDocument doc_ref + doc_ref = _InputDocument() + doc_ref._type = PARSER_DATA_EMPTY + return doc_ref + + def resolve_string(self, string, context, *, base_url=None): + """resolve_string(self, string, context, base_url=None) + + Return a parsable string as input document. + + Pass data string and context as parameters. You can pass the + source URL or filename through the ``base_url`` keyword + argument. + """ + cdef _InputDocument doc_ref + if isinstance(string, unicode): + string = (string).encode('utf8') + elif not isinstance(string, bytes): + raise TypeError, "argument must be a byte string or unicode string" + doc_ref = _InputDocument() + doc_ref._type = PARSER_DATA_STRING + doc_ref._data_bytes = string + if base_url is not None: + doc_ref._filename = _encodeFilename(base_url) + return doc_ref + + def resolve_filename(self, filename, context): + """resolve_filename(self, filename, context) + + Return the name of a parsable file as input document. + + Pass filename and context as parameters. You can also pass a + URL with an HTTP, FTP or file target. + """ + cdef _InputDocument doc_ref + doc_ref = _InputDocument() + doc_ref._type = PARSER_DATA_FILENAME + doc_ref._filename = _encodeFilename(filename) + return doc_ref + + def resolve_file(self, f, context, *, base_url=None, bint close=True): + """resolve_file(self, f, context, base_url=None, close=True) + + Return an open file-like object as input document. + + Pass open file and context as parameters. You can pass the + base URL or filename of the file through the ``base_url`` + keyword argument. If the ``close`` flag is True (the + default), the file will be closed after reading. + + Note that using ``.resolve_filename()`` is more efficient, + especially in threaded environments. + """ + cdef _InputDocument doc_ref + try: + f.read + except AttributeError: + raise TypeError, "Argument is not a file-like object" + doc_ref = _InputDocument() + doc_ref._type = PARSER_DATA_FILE + if base_url is not None: + doc_ref._filename = _encodeFilename(base_url) + else: + doc_ref._filename = _getFilenameForFile(f) + doc_ref._close_file = close + doc_ref._file = f + return doc_ref + +@cython.final +@cython.internal +cdef class _ResolverRegistry: + cdef object _resolvers + cdef Resolver _default_resolver + def __cinit__(self, Resolver default_resolver=None): + self._resolvers = set() + self._default_resolver = default_resolver + + def add(self, Resolver resolver not None): + """add(self, resolver) + + Register a resolver. + + For each requested entity, the 'resolve' method of the resolver will + be called and the result will be passed to the parser. If this method + returns None, the request will be delegated to other resolvers or the + default resolver. The resolvers will be tested in an arbitrary order + until the first match is found. + """ + self._resolvers.add(resolver) + + def remove(self, resolver): + "remove(self, resolver)" + self._resolvers.discard(resolver) + + cdef _ResolverRegistry _copy(self): + cdef _ResolverRegistry registry + registry = _ResolverRegistry(self._default_resolver) + registry._resolvers = self._resolvers.copy() + return registry + + def copy(self): + "copy(self)" + return self._copy() + + def resolve(self, system_url, public_id, context): + "resolve(self, system_url, public_id, context)" + for resolver in self._resolvers: + result = resolver.resolve(system_url, public_id, context) + if result is not None: + return result + if self._default_resolver is None: + return None + return self._default_resolver.resolve(system_url, public_id, context) + + def __repr__(self): + return repr(self._resolvers) + + +@cython.internal +cdef class _ResolverContext(_ExceptionContext): + cdef _ResolverRegistry _resolvers + cdef _TempStore _storage + + @cython.final + cdef int clear(self) except -1: + _ExceptionContext.clear(self) + self._storage.clear() + return 0 + + +cdef _initResolverContext(_ResolverContext context, + _ResolverRegistry resolvers): + if resolvers is None: + context._resolvers = _ResolverRegistry() + else: + context._resolvers = resolvers + context._storage = _TempStore() diff --git a/.venv/lib/python3.13/site-packages/lxml/doctestcompare.py b/.venv/lib/python3.13/site-packages/lxml/doctestcompare.py new file mode 100644 index 0000000000000000000000000000000000000000..8099771de906a37ed007c779f152fe96f182060d --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/doctestcompare.py @@ -0,0 +1,488 @@ +""" +lxml-based doctest output comparison. + +Note: normally, you should just import the `lxml.usedoctest` and +`lxml.html.usedoctest` modules from within a doctest, instead of this +one:: + + >>> import lxml.usedoctest # for XML output + + >>> import lxml.html.usedoctest # for HTML output + +To use this module directly, you must call ``lxmldoctest.install()``, +which will cause doctest to use this in all subsequent calls. + +This changes the way output is checked and comparisons are made for +XML or HTML-like content. + +XML or HTML content is noticed because the example starts with ``<`` +(it's HTML if it starts with ```` or include an ``any`` +attribute in the tag. An ``any`` tag matches any tag, while the +attribute matches any and all attributes. + +When a match fails, the reformatted example and gotten text is +displayed (indented), and a rough diff-like output is given. Anything +marked with ``+`` is in the output but wasn't supposed to be, and +similarly ``-`` means its in the example but wasn't in the output. + +You can disable parsing on one line with ``# doctest:+NOPARSE_MARKUP`` +""" + +from lxml import etree +import sys +import re +import doctest +try: + from html import escape as html_escape +except ImportError: + from cgi import escape as html_escape + +__all__ = ['PARSE_HTML', 'PARSE_XML', 'NOPARSE_MARKUP', 'LXMLOutputChecker', + 'LHTMLOutputChecker', 'install', 'temp_install'] + +PARSE_HTML = doctest.register_optionflag('PARSE_HTML') +PARSE_XML = doctest.register_optionflag('PARSE_XML') +NOPARSE_MARKUP = doctest.register_optionflag('NOPARSE_MARKUP') + +OutputChecker = doctest.OutputChecker + +def strip(v): + if v is None: + return None + else: + return v.strip() + +def norm_whitespace(v): + return _norm_whitespace_re.sub(' ', v) + +_html_parser = etree.HTMLParser(recover=False, remove_blank_text=True) + +def html_fromstring(html): + return etree.fromstring(html, _html_parser) + +# We use this to distinguish repr()s from elements: +_repr_re = re.compile(r'^<[^>]+ (at|object) ') +_norm_whitespace_re = re.compile(r'[ \t\n][ \t\n]+') + +class LXMLOutputChecker(OutputChecker): + + empty_tags = ( + 'param', 'img', 'area', 'br', 'basefont', 'input', + 'base', 'meta', 'link', 'col') + + def get_default_parser(self): + return etree.XML + + def check_output(self, want, got, optionflags): + alt_self = getattr(self, '_temp_override_self', None) + if alt_self is not None: + super_method = self._temp_call_super_check_output + self = alt_self + else: + super_method = OutputChecker.check_output + parser = self.get_parser(want, got, optionflags) + if not parser: + return super_method( + self, want, got, optionflags) + try: + want_doc = parser(want) + except etree.XMLSyntaxError: + return False + try: + got_doc = parser(got) + except etree.XMLSyntaxError: + return False + return self.compare_docs(want_doc, got_doc) + + def get_parser(self, want, got, optionflags): + parser = None + if NOPARSE_MARKUP & optionflags: + return None + if PARSE_HTML & optionflags: + parser = html_fromstring + elif PARSE_XML & optionflags: + parser = etree.XML + elif (want.strip().lower().startswith('' % el.tag + return '<%s %s>' % (el.tag, ' '.join(attrs)) + + def format_end_tag(self, el): + if isinstance(el, etree.CommentBase): + # FIXME: probably PIs should be handled specially too? + return '-->' + return '' % el.tag + + def collect_diff(self, want, got, html, indent): + parts = [] + if not len(want) and not len(got): + parts.append(' '*indent) + parts.append(self.collect_diff_tag(want, got)) + if not self.html_empty_tag(got, html): + parts.append(self.collect_diff_text(want.text, got.text)) + parts.append(self.collect_diff_end_tag(want, got)) + parts.append(self.collect_diff_text(want.tail, got.tail)) + parts.append('\n') + return ''.join(parts) + parts.append(' '*indent) + parts.append(self.collect_diff_tag(want, got)) + parts.append('\n') + if strip(want.text) or strip(got.text): + parts.append(' '*indent) + parts.append(self.collect_diff_text(want.text, got.text)) + parts.append('\n') + want_children = list(want) + got_children = list(got) + while want_children or got_children: + if not want_children: + parts.append(self.format_doc(got_children.pop(0), html, indent+2, '+')) + continue + if not got_children: + parts.append(self.format_doc(want_children.pop(0), html, indent+2, '-')) + continue + parts.append(self.collect_diff( + want_children.pop(0), got_children.pop(0), html, indent+2)) + parts.append(' '*indent) + parts.append(self.collect_diff_end_tag(want, got)) + parts.append('\n') + if strip(want.tail) or strip(got.tail): + parts.append(' '*indent) + parts.append(self.collect_diff_text(want.tail, got.tail)) + parts.append('\n') + return ''.join(parts) + + def collect_diff_tag(self, want, got): + if not self.tag_compare(want.tag, got.tag): + tag = '%s (got: %s)' % (want.tag, got.tag) + else: + tag = got.tag + attrs = [] + any = want.tag == 'any' or 'any' in want.attrib + for name, value in sorted(got.attrib.items()): + if name not in want.attrib and not any: + attrs.append('+%s="%s"' % (name, self.format_text(value, False))) + else: + if name in want.attrib: + text = self.collect_diff_text(want.attrib[name], value, False) + else: + text = self.format_text(value, False) + attrs.append('%s="%s"' % (name, text)) + if not any: + for name, value in sorted(want.attrib.items()): + if name in got.attrib: + continue + attrs.append('-%s="%s"' % (name, self.format_text(value, False))) + if attrs: + tag = '<%s %s>' % (tag, ' '.join(attrs)) + else: + tag = '<%s>' % tag + return tag + + def collect_diff_end_tag(self, want, got): + if want.tag != got.tag: + tag = '%s (got: %s)' % (want.tag, got.tag) + else: + tag = got.tag + return '' % tag + + def collect_diff_text(self, want, got, strip=True): + if self.text_compare(want, got, strip): + if not got: + return '' + return self.format_text(got, strip) + text = '%s (got: %s)' % (want, got) + return self.format_text(text, strip) + +class LHTMLOutputChecker(LXMLOutputChecker): + def get_default_parser(self): + return html_fromstring + +def install(html=False): + """ + Install doctestcompare for all future doctests. + + If html is true, then by default the HTML parser will be used; + otherwise the XML parser is used. + """ + if html: + doctest.OutputChecker = LHTMLOutputChecker + else: + doctest.OutputChecker = LXMLOutputChecker + +def temp_install(html=False, del_module=None): + """ + Use this *inside* a doctest to enable this checker for this + doctest only. + + If html is true, then by default the HTML parser will be used; + otherwise the XML parser is used. + """ + if html: + Checker = LHTMLOutputChecker + else: + Checker = LXMLOutputChecker + frame = _find_doctest_frame() + dt_self = frame.f_locals['self'] + checker = Checker() + old_checker = dt_self._checker + dt_self._checker = checker + # The unfortunate thing is that there is a local variable 'check' + # in the function that runs the doctests, that is a bound method + # into the output checker. We have to update that. We can't + # modify the frame, so we have to modify the object in place. The + # only way to do this is to actually change the func_code + # attribute of the method. We change it, and then wait for + # __record_outcome to be run, which signals the end of the __run + # method, at which point we restore the previous check_output + # implementation. + check_func = frame.f_locals['check'].__func__ + checker_check_func = checker.check_output.__func__ + # Because we can't patch up func_globals, this is the only global + # in check_output that we care about: + doctest.etree = etree + _RestoreChecker(dt_self, old_checker, checker, + check_func, checker_check_func, + del_module) + +class _RestoreChecker: + def __init__(self, dt_self, old_checker, new_checker, check_func, clone_func, + del_module): + self.dt_self = dt_self + self.checker = old_checker + self.checker._temp_call_super_check_output = self.call_super + self.checker._temp_override_self = new_checker + self.check_func = check_func + self.clone_func = clone_func + self.del_module = del_module + self.install_clone() + self.install_dt_self() + def install_clone(self): + self.func_code = self.check_func.__code__ + self.func_globals = self.check_func.__globals__ + self.check_func.__code__ = self.clone_func.__code__ + def uninstall_clone(self): + self.check_func.__code__ = self.func_code + def install_dt_self(self): + self.prev_func = self.dt_self._DocTestRunner__record_outcome + self.dt_self._DocTestRunner__record_outcome = self + def uninstall_dt_self(self): + self.dt_self._DocTestRunner__record_outcome = self.prev_func + def uninstall_module(self): + if self.del_module: + import sys + del sys.modules[self.del_module] + if '.' in self.del_module: + package, module = self.del_module.rsplit('.', 1) + package_mod = sys.modules[package] + delattr(package_mod, module) + def __call__(self, *args, **kw): + self.uninstall_clone() + self.uninstall_dt_self() + del self.checker._temp_override_self + del self.checker._temp_call_super_check_output + result = self.prev_func(*args, **kw) + self.uninstall_module() + return result + def call_super(self, *args, **kw): + self.uninstall_clone() + try: + return self.check_func(*args, **kw) + finally: + self.install_clone() + +def _find_doctest_frame(): + import sys + frame = sys._getframe(1) + while frame: + l = frame.f_locals + if 'BOOM' in l: + # Sign of doctest + return frame + frame = frame.f_back + raise LookupError( + "Could not find doctest (only use this function *inside* a doctest)") + +__test__ = { + 'basic': ''' + >>> temp_install() + >>> print """stuff""" + ... + >>> print """""" + + + + >>> print """blahblahblah""" # doctest: +NOPARSE_MARKUP, +ELLIPSIS + ...foo /> + '''} + +if __name__ == '__main__': + import doctest + doctest.testmod() + + diff --git a/.venv/lib/python3.13/site-packages/lxml/dtd.pxi b/.venv/lib/python3.13/site-packages/lxml/dtd.pxi new file mode 100644 index 0000000000000000000000000000000000000000..ee1b3d4750c7596e72bac0eaa905908b60dcb1e8 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/dtd.pxi @@ -0,0 +1,479 @@ +# support for DTD validation +from lxml.includes cimport dtdvalid + +cdef class DTDError(LxmlError): + """Base class for DTD errors. + """ + +cdef class DTDParseError(DTDError): + """Error while parsing a DTD. + """ + +cdef class DTDValidateError(DTDError): + """Error while validating an XML document with a DTD. + """ + + +cdef inline int _assertValidDTDNode(node, void *c_node) except -1: + assert c_node is not NULL, "invalid DTD proxy at %s" % id(node) + + +@cython.final +@cython.internal +@cython.freelist(8) +cdef class _DTDElementContentDecl: + cdef DTD _dtd + cdef tree.xmlElementContent* _c_node + + def __repr__(self): + return "<%s.%s object name=%r type=%r occur=%r at 0x%x>" % (self.__class__.__module__, self.__class__.__name__, self.name, self.type, self.occur, id(self)) + + @property + def name(self): + _assertValidDTDNode(self, self._c_node) + return funicodeOrNone(self._c_node.name) + + @property + def type(self): + _assertValidDTDNode(self, self._c_node) + cdef int type = self._c_node.type + if type == tree.XML_ELEMENT_CONTENT_PCDATA: + return "pcdata" + elif type == tree.XML_ELEMENT_CONTENT_ELEMENT: + return "element" + elif type == tree.XML_ELEMENT_CONTENT_SEQ: + return "seq" + elif type == tree.XML_ELEMENT_CONTENT_OR: + return "or" + else: + return None + + @property + def occur(self): + _assertValidDTDNode(self, self._c_node) + cdef int occur = self._c_node.ocur + if occur == tree.XML_ELEMENT_CONTENT_ONCE: + return "once" + elif occur == tree.XML_ELEMENT_CONTENT_OPT: + return "opt" + elif occur == tree.XML_ELEMENT_CONTENT_MULT: + return "mult" + elif occur == tree.XML_ELEMENT_CONTENT_PLUS: + return "plus" + else: + return None + + @property + def left(self): + _assertValidDTDNode(self, self._c_node) + c1 = self._c_node.c1 + if c1: + node = <_DTDElementContentDecl>_DTDElementContentDecl.__new__(_DTDElementContentDecl) + node._dtd = self._dtd + node._c_node = c1 + return node + else: + return None + + @property + def right(self): + _assertValidDTDNode(self, self._c_node) + c2 = self._c_node.c2 + if c2: + node = <_DTDElementContentDecl>_DTDElementContentDecl.__new__(_DTDElementContentDecl) + node._dtd = self._dtd + node._c_node = c2 + return node + else: + return None + + +@cython.final +@cython.internal +@cython.freelist(8) +cdef class _DTDAttributeDecl: + cdef DTD _dtd + cdef tree.xmlAttribute* _c_node + + def __repr__(self): + return "<%s.%s object name=%r elemname=%r prefix=%r type=%r default=%r default_value=%r at 0x%x>" % (self.__class__.__module__, self.__class__.__name__, self.name, self.elemname, self.prefix, self.type, self.default, self.default_value, id(self)) + + @property + def name(self): + _assertValidDTDNode(self, self._c_node) + return funicodeOrNone(self._c_node.name) + + @property + def elemname(self): + _assertValidDTDNode(self, self._c_node) + return funicodeOrNone(self._c_node.elem) + + @property + def prefix(self): + _assertValidDTDNode(self, self._c_node) + return funicodeOrNone(self._c_node.prefix) + + @property + def type(self): + _assertValidDTDNode(self, self._c_node) + cdef int type = self._c_node.atype + if type == tree.XML_ATTRIBUTE_CDATA: + return "cdata" + elif type == tree.XML_ATTRIBUTE_ID: + return "id" + elif type == tree.XML_ATTRIBUTE_IDREF: + return "idref" + elif type == tree.XML_ATTRIBUTE_IDREFS: + return "idrefs" + elif type == tree.XML_ATTRIBUTE_ENTITY: + return "entity" + elif type == tree.XML_ATTRIBUTE_ENTITIES: + return "entities" + elif type == tree.XML_ATTRIBUTE_NMTOKEN: + return "nmtoken" + elif type == tree.XML_ATTRIBUTE_NMTOKENS: + return "nmtokens" + elif type == tree.XML_ATTRIBUTE_ENUMERATION: + return "enumeration" + elif type == tree.XML_ATTRIBUTE_NOTATION: + return "notation" + else: + return None + + @property + def default(self): + _assertValidDTDNode(self, self._c_node) + cdef int default = self._c_node.def_ + if default == tree.XML_ATTRIBUTE_NONE: + return "none" + elif default == tree.XML_ATTRIBUTE_REQUIRED: + return "required" + elif default == tree.XML_ATTRIBUTE_IMPLIED: + return "implied" + elif default == tree.XML_ATTRIBUTE_FIXED: + return "fixed" + else: + return None + + @property + def default_value(self): + _assertValidDTDNode(self, self._c_node) + return funicodeOrNone(self._c_node.defaultValue) + + def itervalues(self): + _assertValidDTDNode(self, self._c_node) + cdef tree.xmlEnumeration *c_node = self._c_node.tree + while c_node is not NULL: + yield funicode(c_node.name) + c_node = c_node.next + + def values(self): + return list(self.itervalues()) + + +@cython.final +@cython.internal +@cython.freelist(8) +cdef class _DTDElementDecl: + cdef DTD _dtd + cdef tree.xmlElement* _c_node + + def __repr__(self): + return "<%s.%s object name=%r prefix=%r type=%r at 0x%x>" % (self.__class__.__module__, self.__class__.__name__, self.name, self.prefix, self.type, id(self)) + + @property + def name(self): + _assertValidDTDNode(self, self._c_node) + return funicodeOrNone(self._c_node.name) + + @property + def prefix(self): + _assertValidDTDNode(self, self._c_node) + return funicodeOrNone(self._c_node.prefix) + + @property + def type(self): + _assertValidDTDNode(self, self._c_node) + cdef int type = self._c_node.etype + if type == tree.XML_ELEMENT_TYPE_UNDEFINED: + return "undefined" + elif type == tree.XML_ELEMENT_TYPE_EMPTY: + return "empty" + elif type == tree.XML_ELEMENT_TYPE_ANY: + return "any" + elif type == tree.XML_ELEMENT_TYPE_MIXED: + return "mixed" + elif type == tree.XML_ELEMENT_TYPE_ELEMENT: + return "element" + else: + return None + + @property + def content(self): + _assertValidDTDNode(self, self._c_node) + cdef tree.xmlElementContent *content = self._c_node.content + if content: + node = <_DTDElementContentDecl>_DTDElementContentDecl.__new__(_DTDElementContentDecl) + node._dtd = self._dtd + node._c_node = content + return node + else: + return None + + def iterattributes(self): + _assertValidDTDNode(self, self._c_node) + cdef tree.xmlAttribute *c_node = self._c_node.attributes + while c_node: + node = <_DTDAttributeDecl>_DTDAttributeDecl.__new__(_DTDAttributeDecl) + node._dtd = self._dtd + node._c_node = c_node + yield node + c_node = c_node.nexth + + def attributes(self): + return list(self.iterattributes()) + + +@cython.final +@cython.internal +@cython.freelist(8) +cdef class _DTDEntityDecl: + cdef DTD _dtd + cdef tree.xmlEntity* _c_node + def __repr__(self): + return "<%s.%s object name=%r at 0x%x>" % (self.__class__.__module__, self.__class__.__name__, self.name, id(self)) + + @property + def name(self): + _assertValidDTDNode(self, self._c_node) + return funicodeOrNone(self._c_node.name) + + @property + def orig(self): + _assertValidDTDNode(self, self._c_node) + return funicodeOrNone(self._c_node.orig) + + @property + def content(self): + _assertValidDTDNode(self, self._c_node) + return funicodeOrNone(self._c_node.content) + + @property + def system_url(self): + _assertValidDTDNode(self, self._c_node) + return funicodeOrNone(self._c_node.SystemID) + + +################################################################################ +# DTD + +cdef class DTD(_Validator): + """DTD(self, file=None, external_id=None) + A DTD validator. + + Can load from filesystem directly given a filename or file-like object. + Alternatively, pass the keyword parameter ``external_id`` to load from a + catalog. + """ + cdef tree.xmlDtd* _c_dtd + def __init__(self, file=None, *, external_id=None): + _Validator.__init__(self) + if file is not None: + file = _getFSPathOrObject(file) + if _isString(file): + file = _encodeFilename(file) + with self._error_log: + orig_loader = _register_document_loader() + self._c_dtd = xmlparser.xmlParseDTD(NULL, _xcstr(file)) + _reset_document_loader(orig_loader) + elif hasattr(file, 'read'): + orig_loader = _register_document_loader() + self._c_dtd = _parseDtdFromFilelike(file) + _reset_document_loader(orig_loader) + else: + raise DTDParseError, "file must be a filename, file-like or path-like object" + elif external_id is not None: + external_id_utf = _utf8(external_id) + with self._error_log: + orig_loader = _register_document_loader() + self._c_dtd = xmlparser.xmlParseDTD(external_id_utf, NULL) + _reset_document_loader(orig_loader) + else: + raise DTDParseError, "either filename or external ID required" + + if self._c_dtd is NULL: + raise DTDParseError( + self._error_log._buildExceptionMessage("error parsing DTD"), + self._error_log) + + @property + def name(self): + if self._c_dtd is NULL: + return None + return funicodeOrNone(self._c_dtd.name) + + @property + def external_id(self): + if self._c_dtd is NULL: + return None + return funicodeOrNone(self._c_dtd.ExternalID) + + @property + def system_url(self): + if self._c_dtd is NULL: + return None + return funicodeOrNone(self._c_dtd.SystemID) + + def iterelements(self): + cdef tree.xmlNode *c_node = self._c_dtd.children if self._c_dtd is not NULL else NULL + while c_node is not NULL: + if c_node.type == tree.XML_ELEMENT_DECL: + node = _DTDElementDecl() + node._dtd = self + node._c_node = c_node + yield node + c_node = c_node.next + + def elements(self): + return list(self.iterelements()) + + def iterentities(self): + cdef tree.xmlNode *c_node = self._c_dtd.children if self._c_dtd is not NULL else NULL + while c_node is not NULL: + if c_node.type == tree.XML_ENTITY_DECL: + node = _DTDEntityDecl() + node._dtd = self + node._c_node = c_node + yield node + c_node = c_node.next + + def entities(self): + return list(self.iterentities()) + + def __dealloc__(self): + tree.xmlFreeDtd(self._c_dtd) + + def __call__(self, etree): + """__call__(self, etree) + + Validate doc using the DTD. + + Returns true if the document is valid, false if not. + """ + cdef _Document doc + cdef _Element root_node + cdef xmlDoc* c_doc + cdef dtdvalid.xmlValidCtxt* valid_ctxt + cdef int ret = -1 + + assert self._c_dtd is not NULL, "DTD not initialised" + doc = _documentOrRaise(etree) + root_node = _rootNodeOrRaise(etree) + + valid_ctxt = dtdvalid.xmlNewValidCtxt() + if valid_ctxt is NULL: + raise DTDError("Failed to create validation context") + + # work around error reporting bug in libxml2 <= 2.9.1 (and later?) + # https://bugzilla.gnome.org/show_bug.cgi?id=724903 + valid_ctxt.error = _nullGenericErrorFunc + valid_ctxt.userData = NULL + + try: + with self._error_log: + c_doc = _fakeRootDoc(doc._c_doc, root_node._c_node) + ret = dtdvalid.xmlValidateDtd(valid_ctxt, c_doc, self._c_dtd) + _destroyFakeDoc(doc._c_doc, c_doc) + finally: + dtdvalid.xmlFreeValidCtxt(valid_ctxt) + + if ret == -1: + raise DTDValidateError("Internal error in DTD validation", + self._error_log) + return ret == 1 + + +cdef tree.xmlDtd* _parseDtdFromFilelike(file) except NULL: + cdef _ExceptionContext exc_context + cdef _FileReaderContext dtd_parser + cdef _ErrorLog error_log + cdef tree.xmlDtd* c_dtd = NULL + exc_context = _ExceptionContext() + dtd_parser = _FileReaderContext(file, exc_context, None) + error_log = _ErrorLog() + + with error_log: + c_dtd = dtd_parser._readDtd() + + exc_context._raise_if_stored() + if c_dtd is NULL: + raise DTDParseError("error parsing DTD", error_log) + return c_dtd + +cdef DTD _dtdFactory(tree.xmlDtd* c_dtd): + # do not run through DTD.__init__()! + cdef DTD dtd + if c_dtd is NULL: + return None + dtd = DTD.__new__(DTD) + dtd._c_dtd = _copyDtd(c_dtd) + _Validator.__init__(dtd) + return dtd + + +cdef tree.xmlDtd* _copyDtd(tree.xmlDtd* c_orig_dtd) except NULL: + """ + Copy a DTD. libxml2 (currently) fails to set up the element->attributes + links when copying DTDs, so we have to rebuild them here. + """ + c_dtd = tree.xmlCopyDtd(c_orig_dtd) + if not c_dtd: + raise MemoryError + cdef tree.xmlNode* c_node = c_dtd.children + while c_node: + if c_node.type == tree.XML_ATTRIBUTE_DECL: + _linkDtdAttribute(c_dtd, c_node) + c_node = c_node.next + return c_dtd + + +cdef void _linkDtdAttribute(tree.xmlDtd* c_dtd, tree.xmlAttribute* c_attr) noexcept: + """ + Create the link to the DTD attribute declaration from the corresponding + element declaration. + """ + c_elem = dtdvalid.xmlGetDtdElementDesc(c_dtd, c_attr.elem) + if not c_elem: + # no such element? something is wrong with the DTD ... + return + c_pos = c_elem.attributes + if not c_pos: + c_elem.attributes = c_attr + c_attr.nexth = NULL + return + # libxml2 keeps namespace declarations first, and we need to make + # sure we don't re-insert attributes that are already there + if _isDtdNsDecl(c_attr): + if not _isDtdNsDecl(c_pos): + c_elem.attributes = c_attr + c_attr.nexth = c_pos + return + while c_pos != c_attr and c_pos.nexth and _isDtdNsDecl(c_pos.nexth): + c_pos = c_pos.nexth + else: + # append at end + while c_pos != c_attr and c_pos.nexth: + c_pos = c_pos.nexth + if c_pos == c_attr: + return + c_attr.nexth = c_pos.nexth + c_pos.nexth = c_attr + + +cdef bint _isDtdNsDecl(tree.xmlAttribute* c_attr) noexcept: + if cstring_h.strcmp(c_attr.name, "xmlns") == 0: + return True + if (c_attr.prefix is not NULL and + cstring_h.strcmp(c_attr.prefix, "xmlns") == 0): + return True + return False diff --git a/.venv/lib/python3.13/site-packages/lxml/etree.cpython-313-x86_64-linux-gnu.so b/.venv/lib/python3.13/site-packages/lxml/etree.cpython-313-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..ef9ba39ff618c6be840002a2cfae54c8025f4097 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/etree.cpython-313-x86_64-linux-gnu.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50bc5270d6315e4a28bba25f9211adb8f5ed2ada7de627a36c8e8a15a487f501 +size 5243760 diff --git a/.venv/lib/python3.13/site-packages/lxml/etree.h b/.venv/lib/python3.13/site-packages/lxml/etree.h new file mode 100644 index 0000000000000000000000000000000000000000..10a60398a015f0891a3ecd7e7296b14ce5615191 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/etree.h @@ -0,0 +1,244 @@ +/* Generated by Cython 3.2.4 */ + +#ifndef __PYX_HAVE__lxml__etree +#define __PYX_HAVE__lxml__etree + +#include "Python.h" +struct LxmlDocument; +struct LxmlElement; +struct LxmlElementTree; +struct LxmlElementTagMatcher; +struct LxmlElementIterator; +struct LxmlElementBase; +struct LxmlElementClassLookup; +struct LxmlFallbackElementClassLookup; + +/* "lxml/etree.pyx":456 + * + * # type of a function that steps from node to node + * ctypedef public xmlNode* (*_node_to_node_function)(xmlNode*) # <<<<<<<<<<<<<< + * + * +*/ +typedef xmlNode *(*_node_to_node_function)(xmlNode *); + +/* "lxml/etree.pyx":470 + * # Public Python API + * + * @cython.final # <<<<<<<<<<<<<< + * @cython.freelist(8) + * cdef public class _Document [ type LxmlDocumentType, object LxmlDocument ]: +*/ +struct LxmlDocument { + PyObject_HEAD + struct __pyx_vtabstruct_4lxml_5etree__Document *__pyx_vtab; + int _ns_counter; + PyObject *_prefix_tail; + xmlDoc *_c_doc; + struct __pyx_obj_4lxml_5etree__BaseParser *_parser; +}; + +/* "lxml/etree.pyx":822 + * + * + * @cython.no_gc_clear # <<<<<<<<<<<<<< + * cdef public class _Element [ type LxmlElementType, object LxmlElement ]: + * """Element class. +*/ +struct LxmlElement { + PyObject_HEAD + struct LxmlDocument *_doc; + xmlNode *_c_node; + PyObject *_tag; +}; + +/* "lxml/etree.pyx":1996 + * + * + * cdef public class _ElementTree [ type LxmlElementTreeType, # <<<<<<<<<<<<<< + * object LxmlElementTree ]: + * cdef _Document _doc +*/ +struct LxmlElementTree { + PyObject_HEAD + struct __pyx_vtabstruct_4lxml_5etree__ElementTree *__pyx_vtab; + struct LxmlDocument *_doc; + struct LxmlElement *_context_node; +}; + +/* "lxml/etree.pyx":2770 + * + * + * cdef public class _ElementTagMatcher [ object LxmlElementTagMatcher, # <<<<<<<<<<<<<< + * type LxmlElementTagMatcherType ]: + * """ +*/ +struct LxmlElementTagMatcher { + PyObject_HEAD + struct __pyx_vtabstruct_4lxml_5etree__ElementTagMatcher *__pyx_vtab; + PyObject *_pystrings; + int _node_type; + char *_href; + char *_name; +}; + +/* "lxml/etree.pyx":2801 + * self._name = NULL + * + * cdef public class _ElementIterator(_ElementTagMatcher) [ # <<<<<<<<<<<<<< + * object LxmlElementIterator, type LxmlElementIteratorType ]: + * """ +*/ +struct LxmlElementIterator { + struct LxmlElementTagMatcher __pyx_base; + struct LxmlElement *_node; + _node_to_node_function _next_element; +}; + +/* "src/lxml/classlookup.pxi":6 + * # Custom Element classes + * + * cdef public class ElementBase(_Element) [ type LxmlElementBaseType, # <<<<<<<<<<<<<< + * object LxmlElementBase ]: + * """ElementBase(*children, attrib=None, nsmap=None, **_extra) +*/ +struct LxmlElementBase { + struct LxmlElement __pyx_base; +}; + +/* "src/lxml/classlookup.pxi":210 + * # Element class lookup + * + * ctypedef public object (*_element_class_lookup_function)(object, _Document, xmlNode*) # <<<<<<<<<<<<<< + * + * # class to store element class lookup functions +*/ +typedef PyObject *(*_element_class_lookup_function)(PyObject *, struct LxmlDocument *, xmlNode *); + +/* "src/lxml/classlookup.pxi":213 + * + * # class to store element class lookup functions + * cdef public class ElementClassLookup [ type LxmlElementClassLookupType, # <<<<<<<<<<<<<< + * object LxmlElementClassLookup ]: + * """ElementClassLookup(self) +*/ +struct LxmlElementClassLookup { + PyObject_HEAD + _element_class_lookup_function _lookup_function; +}; + +/* "src/lxml/classlookup.pxi":221 + * + * + * cdef public class FallbackElementClassLookup(ElementClassLookup) \ # <<<<<<<<<<<<<< + * [ type LxmlFallbackElementClassLookupType, + * object LxmlFallbackElementClassLookup ]: +*/ +struct LxmlFallbackElementClassLookup { + struct LxmlElementClassLookup __pyx_base; + struct __pyx_vtabstruct_4lxml_5etree_FallbackElementClassLookup *__pyx_vtab; + struct LxmlElementClassLookup *fallback; + _element_class_lookup_function _fallback_function; +}; + +#ifndef __PYX_HAVE_API__lxml__etree + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#ifndef DL_IMPORT + #define DL_IMPORT(_T) _T +#endif + +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlDocumentType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementTreeType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementTagMatcherType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementIteratorType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementBaseType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementClassLookupType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlFallbackElementClassLookupType; + +__PYX_EXTERN_C struct LxmlElement *deepcopyNodeToDocument(struct LxmlDocument *, xmlNode *); +__PYX_EXTERN_C struct LxmlElementTree *elementTreeFactory(struct LxmlElement *); +__PYX_EXTERN_C struct LxmlElementTree *newElementTree(struct LxmlElement *, PyObject *); +__PYX_EXTERN_C struct LxmlElementTree *adoptExternalDocument(xmlDoc *, PyObject *, int); +__PYX_EXTERN_C struct LxmlElement *elementFactory(struct LxmlDocument *, xmlNode *); +__PYX_EXTERN_C struct LxmlElement *makeElement(PyObject *, struct LxmlDocument *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *); +__PYX_EXTERN_C struct LxmlElement *makeSubElement(struct LxmlElement *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *); +__PYX_EXTERN_C void setElementClassLookupFunction(_element_class_lookup_function, PyObject *); +__PYX_EXTERN_C PyObject *lookupDefaultElementClass(PyObject *, PyObject *, xmlNode *); +__PYX_EXTERN_C PyObject *lookupNamespaceElementClass(PyObject *, PyObject *, xmlNode *); +__PYX_EXTERN_C PyObject *callLookupFallback(struct LxmlFallbackElementClassLookup *, struct LxmlDocument *, xmlNode *); +__PYX_EXTERN_C int tagMatches(xmlNode *, const xmlChar *, const xmlChar *); +__PYX_EXTERN_C struct LxmlDocument *documentOrRaise(PyObject *); +__PYX_EXTERN_C struct LxmlElement *rootNodeOrRaise(PyObject *); +__PYX_EXTERN_C int hasText(xmlNode *); +__PYX_EXTERN_C int hasTail(xmlNode *); +__PYX_EXTERN_C PyObject *textOf(xmlNode *); +__PYX_EXTERN_C PyObject *tailOf(xmlNode *); +__PYX_EXTERN_C int setNodeText(xmlNode *, PyObject *); +__PYX_EXTERN_C int setTailText(xmlNode *, PyObject *); +__PYX_EXTERN_C PyObject *attributeValue(xmlNode *, xmlAttr *); +__PYX_EXTERN_C PyObject *attributeValueFromNsName(xmlNode *, const xmlChar *, const xmlChar *); +__PYX_EXTERN_C PyObject *getAttributeValue(struct LxmlElement *, PyObject *, PyObject *); +__PYX_EXTERN_C PyObject *iterattributes(struct LxmlElement *, int); +__PYX_EXTERN_C PyObject *collectAttributes(xmlNode *, int); +__PYX_EXTERN_C int setAttributeValue(struct LxmlElement *, PyObject *, PyObject *); +__PYX_EXTERN_C int delAttribute(struct LxmlElement *, PyObject *); +__PYX_EXTERN_C int delAttributeFromNsName(xmlNode *, const xmlChar *, const xmlChar *); +__PYX_EXTERN_C int hasChild(xmlNode *); +__PYX_EXTERN_C xmlNode *findChild(xmlNode *, Py_ssize_t); +__PYX_EXTERN_C xmlNode *findChildForwards(xmlNode *, Py_ssize_t); +__PYX_EXTERN_C xmlNode *findChildBackwards(xmlNode *, Py_ssize_t); +__PYX_EXTERN_C xmlNode *nextElement(xmlNode *); +__PYX_EXTERN_C xmlNode *previousElement(xmlNode *); +__PYX_EXTERN_C void appendChild(struct LxmlElement *, struct LxmlElement *); +__PYX_EXTERN_C int appendChildToElement(struct LxmlElement *, struct LxmlElement *); +__PYX_EXTERN_C PyObject *pyunicode(const xmlChar *); +__PYX_EXTERN_C PyObject *utf8(PyObject *); +__PYX_EXTERN_C PyObject *getNsTag(PyObject *); +__PYX_EXTERN_C PyObject *getNsTagWithEmptyNs(PyObject *); +__PYX_EXTERN_C PyObject *namespacedName(xmlNode *); +__PYX_EXTERN_C PyObject *namespacedNameFromNsName(const xmlChar *, const xmlChar *); +__PYX_EXTERN_C void iteratorStoreNext(struct LxmlElementIterator *, struct LxmlElement *); +__PYX_EXTERN_C void initTagMatch(struct LxmlElementTagMatcher *, PyObject *); +__PYX_EXTERN_C xmlNs *findOrBuildNodeNsPrefix(struct LxmlDocument *, xmlNode *, const xmlChar *, const xmlChar *); + +#endif /* !__PYX_HAVE_API__lxml__etree */ + +/* WARNING: the interface of the module init function changed in CPython 3.5. */ +/* It now returns a PyModuleDef instance instead of a PyModule instance. */ + +/* WARNING: Use PyImport_AppendInittab("etree", PyInit_etree) instead of calling PyInit_etree directly from Python 3.5 */ +PyMODINIT_FUNC PyInit_etree(void); + +#if PY_VERSION_HEX >= 0x03050000 && (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) || (defined(__cplusplus) && __cplusplus >= 201402L)) +#if defined(__cplusplus) && __cplusplus >= 201402L +[[deprecated("Use PyImport_AppendInittab(\"etree\", PyInit_etree) instead of calling PyInit_etree directly.")]] inline +#elif defined(__GNUC__) || defined(__clang__) +__attribute__ ((__deprecated__("Use PyImport_AppendInittab(\"etree\", PyInit_etree) instead of calling PyInit_etree directly."), __unused__)) __inline__ +#elif defined(_MSC_VER) +__declspec(deprecated("Use PyImport_AppendInittab(\"etree\", PyInit_etree) instead of calling PyInit_etree directly.")) __inline +#endif +static PyObject* __PYX_WARN_IF_PyInit_etree_INIT_CALLED(PyObject* res) { + return res; +} +#define PyInit_etree() __PYX_WARN_IF_PyInit_etree_INIT_CALLED(PyInit_etree()) +#endif + +#endif /* !__PYX_HAVE__lxml__etree */ diff --git a/.venv/lib/python3.13/site-packages/lxml/etree.pyx b/.venv/lib/python3.13/site-packages/lxml/etree.pyx new file mode 100644 index 0000000000000000000000000000000000000000..149a414df04e316ad4b08c67a7706bb1bb725e8f --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/etree.pyx @@ -0,0 +1,3858 @@ +# cython: binding=True +# cython: auto_pickle=False +# cython: language_level=3 + +""" +The ``lxml.etree`` module implements the extended ElementTree API for XML. +""" + +__docformat__ = "restructuredtext en" + +__all__ = [ + 'AttributeBasedElementClassLookup', 'C14NError', 'C14NWriterTarget', 'CDATA', + 'Comment', 'CommentBase', 'CustomElementClassLookup', 'DEBUG', + 'DTD', 'DTDError', 'DTDParseError', 'DTDValidateError', + 'DocumentInvalid', 'ETCompatXMLParser', 'ETXPath', 'Element', + 'ElementBase', 'ElementClassLookup', 'ElementDefaultClassLookup', + 'ElementNamespaceClassLookup', 'ElementTree', 'Entity', 'EntityBase', + 'Error', 'ErrorDomains', 'ErrorLevels', 'ErrorTypes', 'Extension', + 'FallbackElementClassLookup', 'FunctionNamespace', 'HTML', 'HTMLParser', + 'ICONV_COMPILED_VERSION', + 'LIBXML_COMPILED_VERSION', 'LIBXML_VERSION', + 'LIBXML_FEATURES', + 'LIBXSLT_COMPILED_VERSION', 'LIBXSLT_VERSION', + 'LXML_VERSION', + 'LxmlError', 'LxmlRegistryError', 'LxmlSyntaxError', + 'NamespaceRegistryError', 'PI', 'PIBase', 'ParseError', + 'ParserBasedElementClassLookup', 'ParserError', 'ProcessingInstruction', + 'PyErrorLog', 'PythonElementClassLookup', 'QName', 'RelaxNG', + 'RelaxNGError', 'RelaxNGErrorTypes', 'RelaxNGParseError', + 'RelaxNGValidateError', 'Resolver', 'Schematron', 'SchematronError', + 'SchematronParseError', 'SchematronValidateError', 'SerialisationError', + 'SubElement', 'TreeBuilder', 'XInclude', 'XIncludeError', 'XML', + 'XMLDTDID', 'XMLID', 'XMLParser', 'XMLSchema', 'XMLSchemaError', + 'XMLSchemaParseError', 'XMLSchemaValidateError', 'XMLSyntaxError', + 'XMLTreeBuilder', 'XPath', 'XPathDocumentEvaluator', 'XPathError', + 'XPathEvalError', 'XPathEvaluator', 'XPathFunctionError', 'XPathResultError', + 'XPathSyntaxError', 'XSLT', 'XSLTAccessControl', 'XSLTApplyError', + 'XSLTError', 'XSLTExtension', 'XSLTExtensionError', 'XSLTParseError', + 'XSLTSaveError', 'canonicalize', + 'cleanup_namespaces', 'clear_error_log', 'dump', + 'fromstring', 'fromstringlist', 'get_default_parser', 'iselement', + 'iterparse', 'iterwalk', 'parse', 'parseid', 'register_namespace', + 'set_default_parser', 'set_element_class_lookup', 'strip_attributes', + 'strip_elements', 'strip_tags', 'tostring', 'tostringlist', 'tounicode', + 'use_global_python_log' + ] + +cimport cython + +from lxml cimport python +from lxml.includes cimport tree, config +from lxml.includes.tree cimport xmlDoc, xmlNode, xmlAttr, xmlNs, _isElement, _getNs +from lxml.includes.tree cimport const_xmlChar, xmlChar, _xcstr +from lxml.python cimport _cstr, _isString +from lxml.includes cimport xpath +from lxml.includes cimport c14n + +# Cython's standard declarations +cimport cpython.mem +cimport cpython.ref +from libc cimport limits, stdio, stdlib +from libc cimport string as cstring_h # not to be confused with stdlib 'string' +from libc.string cimport const_char + +cdef object os_path_abspath +from os.path import abspath as os_path_abspath + +cdef object BytesIO, StringIO +from io import BytesIO, StringIO + +cdef object OrderedDict +from collections import OrderedDict + +cdef object _elementpath +from lxml import _elementpath + +cdef object sys +import sys + +cdef object re +import re + +cdef object partial +from functools import partial + +cdef object islice +from itertools import islice + +cdef object ITER_EMPTY = iter(()) + +cdef object MutableMapping +from collections.abc import MutableMapping + +class _ImmutableMapping(MutableMapping): + def __getitem__(self, key): + raise KeyError, key + + def __setitem__(self, key, value): + raise KeyError, key + + def __delitem__(self, key): + raise KeyError, key + + def __contains__(self, key): + return False + + def __len__(self): + return 0 + + def __iter__(self): + return ITER_EMPTY + iterkeys = itervalues = iteritems = __iter__ + +cdef object IMMUTABLE_EMPTY_MAPPING = _ImmutableMapping() +del _ImmutableMapping + + +# the rules +# --------- +# any libxml C argument/variable is prefixed with c_ +# any non-public function/class is prefixed with an underscore +# instance creation is always through factories + +# what to do with libxml2/libxslt error messages? +# 0 : drop +# 1 : use log +DEF __DEBUG = 1 + +# maximum number of lines in the libxml2/xslt log if __DEBUG == 1 +DEF __MAX_LOG_SIZE = 100 + +# make the compiled-in debug state publicly available +DEBUG = __DEBUG + +# A struct to store a cached qualified tag name+href pair. +# While we can borrow the c_name from the document dict, +# PyPy requires us to store a Python reference for the +# namespace in order to keep the byte buffer alive. +cdef struct qname: + const_xmlChar* c_name + python.PyObject* href + +# initialize parser (and threading) +xmlparser.xmlInitParser() + +# global per-thread setup +tree.xmlThrDefIndentTreeOutput(1) +tree.xmlThrDefLineNumbersDefaultValue(1) + +_initThreadLogging() + +# filename encoding +cdef bytes _FILENAME_ENCODING = (sys.getfilesystemencoding() or sys.getdefaultencoding() or 'ascii').encode("UTF-8") +cdef char* _C_FILENAME_ENCODING = _cstr(_FILENAME_ENCODING) + +# set up some default namespace prefixes +cdef dict _DEFAULT_NAMESPACE_PREFIXES = { + b"http://www.w3.org/XML/1998/namespace": b'xml', + b"http://www.w3.org/1999/xhtml": b"html", + b"http://www.w3.org/1999/XSL/Transform": b"xsl", + b"http://www.w3.org/1999/02/22-rdf-syntax-ns#": b"rdf", + b"http://schemas.xmlsoap.org/wsdl/": b"wsdl", + # xml schema + b"http://www.w3.org/2001/XMLSchema": b"xs", + b"http://www.w3.org/2001/XMLSchema-instance": b"xsi", + # dublin core + b"http://purl.org/dc/elements/1.1/": b"dc", + # objectify + b"http://codespeak.net/lxml/objectify/pytype" : b"py", +} + +# To avoid runtime encoding overhead, we keep a Unicode copy +# of the uri-prefix mapping as (str, str) items view. +cdef object _DEFAULT_NAMESPACE_PREFIXES_ITEMS = [] + +cdef _update_default_namespace_prefixes_items(): + cdef bytes ns, prefix + global _DEFAULT_NAMESPACE_PREFIXES_ITEMS + _DEFAULT_NAMESPACE_PREFIXES_ITEMS = { + ns.decode('utf-8') : prefix.decode('utf-8') + for ns, prefix in _DEFAULT_NAMESPACE_PREFIXES.items() + }.items() + +_update_default_namespace_prefixes_items() + +cdef object _check_internal_prefix = re.compile(br"ns\d+$").match + +def register_namespace(prefix, uri): + """Registers a namespace prefix that newly created Elements in that + namespace will use. The registry is global, and any existing + mapping for either the given prefix or the namespace URI will be + removed. + """ + prefix_utf, uri_utf = _utf8(prefix), _utf8(uri) + if _check_internal_prefix(prefix_utf): + raise ValueError("Prefix format reserved for internal use") + _tagValidOrRaise(prefix_utf) + _uriValidOrRaise(uri_utf) + if (uri_utf == b"http://www.w3.org/XML/1998/namespace" and prefix_utf != b'xml' + or prefix_utf == b'xml' and uri_utf != b"http://www.w3.org/XML/1998/namespace"): + raise ValueError("Cannot change the 'xml' prefix of the XML namespace") + for k, v in list(_DEFAULT_NAMESPACE_PREFIXES.items()): + if k == uri_utf or v == prefix_utf: + del _DEFAULT_NAMESPACE_PREFIXES[k] + _DEFAULT_NAMESPACE_PREFIXES[uri_utf] = prefix_utf + _update_default_namespace_prefixes_items() + + +# Error superclass for ElementTree compatibility +cdef class Error(Exception): + pass + +# module level superclass for all exceptions +cdef class LxmlError(Error): + """Main exception base class for lxml. All other exceptions inherit from + this one. + """ + def __init__(self, message, error_log=None): + super(_Error, self).__init__(message) + if error_log is None: + self.error_log = __copyGlobalErrorLog() + else: + self.error_log = error_log.copy() + +cdef object _Error = Error + + +# superclass for all syntax errors +class LxmlSyntaxError(LxmlError, SyntaxError): + """Base class for all syntax errors. + """ + +cdef class C14NError(LxmlError): + """Error during C14N serialisation. + """ + +# version information +cdef tuple __unpackDottedVersion(version): + version_list = [] + l = (version.decode("ascii").replace('-', '.').split('.') + [0]*4)[:4] + for item in l: + try: + item = int(item) + except ValueError: + if item.startswith('dev'): + count = item[3:] + item = -300 + elif item.startswith('alpha'): + count = item[5:] + item = -200 + elif item.startswith('beta'): + count = item[4:] + item = -100 + else: + count = 0 + if count: + item += int(count) + version_list.append(item) + return tuple(version_list) + +cdef tuple __unpackIntVersion(int c_version, int base=100): + return ( + ((c_version // (base*base)) % base), + ((c_version // base) % base), + (c_version % base) + ) + +cdef int _LIBXML_VERSION_INT +try: + _LIBXML_VERSION_INT = int( + re.match('[0-9]+', (tree.xmlParserVersion).decode("ascii")).group(0)) +except Exception: + print("Unknown libxml2 version: " + (tree.xmlParserVersion).decode("latin1")) + _LIBXML_VERSION_INT = 0 + +LIBXML_VERSION = __unpackIntVersion(_LIBXML_VERSION_INT) +LIBXML_COMPILED_VERSION = __unpackIntVersion(tree.LIBXML_VERSION) +LXML_VERSION = __unpackDottedVersion(tree.LXML_VERSION_STRING) + +__version__ = tree.LXML_VERSION_STRING.decode("ascii") + +cdef extern from *: + """ + #ifdef ZLIB_VERNUM + #define __lxml_zlib_version (ZLIB_VERNUM >> 4) + #else + #define __lxml_zlib_version 0 + #endif + #ifdef _LIBICONV_VERSION + #define __lxml_iconv_version (_LIBICONV_VERSION << 8) + #else + #define __lxml_iconv_version 0 + #endif + """ + # zlib isn't included automatically by libxml2's headers + #long ZLIB_HEX_VERSION "__lxml_zlib_version" + long LIBICONV_HEX_VERSION "__lxml_iconv_version" + +#ZLIB_COMPILED_VERSION = __unpackIntVersion(ZLIB_HEX_VERSION, base=0x10) +ICONV_COMPILED_VERSION = __unpackIntVersion(LIBICONV_HEX_VERSION, base=0x100)[:2] + + +cdef extern from "libxml/xmlversion.h": + """ + static const char* const _lxml_lib_features[] = { +#ifdef LIBXML_HTML_ENABLED + "html", +#endif +#ifdef LIBXML_FTP_ENABLED + "ftp", +#endif +#ifdef LIBXML_HTTP_ENABLED + "http", +#endif +#ifdef LIBXML_CATALOG_ENABLED + "catalog", +#endif +#ifdef LIBXML_XPATH_ENABLED + "xpath", +#endif +#ifdef LIBXML_ICONV_ENABLED + "iconv", +#endif +#ifdef LIBXML_ICU_ENABLED + "icu", +#endif +#ifdef LIBXML_REGEXP_ENABLED + "regexp", +#endif +#ifdef LIBXML_SCHEMAS_ENABLED + "xmlschema", +#endif +#ifdef LIBXML_SCHEMATRON_ENABLED + "schematron", +#endif +#ifdef LIBXML_ZLIB_ENABLED + "zlib", +#endif +#ifdef LIBXML_LZMA_ENABLED + "lzma", +#endif + 0 + }; + """ + const char* const* _LXML_LIB_FEATURES "_lxml_lib_features" + + +cdef set _copy_lib_features(): + features = set() + feature = _LXML_LIB_FEATURES + while feature[0]: + features.add(feature[0].decode('ASCII')) + feature += 1 + return features + +LIBXML_COMPILED_FEATURES = _copy_lib_features() +LIBXML_FEATURES = { + feature_name for feature_id, feature_name in [ + #XML_WITH_THREAD = 1 + #XML_WITH_TREE = 2 + #XML_WITH_OUTPUT = 3 + #XML_WITH_PUSH = 4 + #XML_WITH_READER = 5 + #XML_WITH_PATTERN = 6 + #XML_WITH_WRITER = 7 + #XML_WITH_SAX1 = 8 + (xmlparser.XML_WITH_FTP, "ftp"), # XML_WITH_FTP = 9 + (xmlparser.XML_WITH_HTTP, "http"), # XML_WITH_HTTP = 10 + #XML_WITH_VALID = 11 + (xmlparser.XML_WITH_HTML, "html"), # XML_WITH_HTML = 12 + #XML_WITH_LEGACY = 13 + #XML_WITH_C14N = 14 + (xmlparser.XML_WITH_CATALOG, "catalog"), # XML_WITH_CATALOG = 15 + (xmlparser.XML_WITH_XPATH, "xpath"), # XML_WITH_XPATH = 16 + #XML_WITH_XPTR = 17 + #XML_WITH_XINCLUDE = 18 + (xmlparser.XML_WITH_ICONV, "iconv"), # XML_WITH_ICONV = 19 + #XML_WITH_ISO8859X = 20 + #XML_WITH_UNICODE = 21 + (xmlparser.XML_WITH_REGEXP, "regexp"), # XML_WITH_REGEXP = 22 + #XML_WITH_AUTOMATA = 23 + #XML_WITH_EXPR = 24 + (xmlparser.XML_WITH_SCHEMAS, "xmlschema"), # XML_WITH_SCHEMAS = 25 + (xmlparser.XML_WITH_SCHEMATRON, "schematron"), # XML_WITH_SCHEMATRON = 26 + #XML_WITH_MODULES = 27 + #XML_WITH_DEBUG = 28 + #XML_WITH_DEBUG_MEM = 29 + #XML_WITH_DEBUG_RUN = 30 # unused + (xmlparser.XML_WITH_ZLIB, "zlib"), # XML_WITH_ZLIB = 31 + (xmlparser.XML_WITH_ICU, "icu"), # XML_WITH_ICU = 32 + (xmlparser.XML_WITH_LZMA, "lzma"), # XML_WITH_LZMA = 33 + ] if xmlparser.xmlHasFeature(feature_id) +} + +cdef bint HAS_ZLIB_COMPRESSION = xmlparser.xmlHasFeature(xmlparser.XML_WITH_ZLIB) + + +# class for temporary storage of Python references, +# used e.g. for XPath results +@cython.final +@cython.internal +cdef class _TempStore: + cdef list _storage + def __init__(self): + self._storage = [] + + cdef int add(self, obj) except -1: + self._storage.append(obj) + return 0 + + cdef int clear(self) except -1: + del self._storage[:] + return 0 + + +# class for temporarily storing exceptions raised in extensions +@cython.internal +cdef class _ExceptionContext: + cdef object _exc_info + + cdef int clear(self) except -1: + self._exc_info = None + return 0 + + @cython.final + cdef void _store_raised(self) noexcept: + try: + self._exc_info = sys.exc_info() + except BaseException as e: + self._store_exception(e) + finally: + return # and swallow any further exceptions + + @cython.final + cdef int _store_exception(self, exception) except -1: + self._exc_info = (exception, None, None) + return 0 + + @cython.final + cdef bint _has_raised(self) except -1: + return self._exc_info is not None + + @cython.final + cdef int _raise_if_stored(self) except -1: + if self._exc_info is None: + return 0 + type, value, traceback = self._exc_info + self._exc_info = None + if value is None and traceback is None: + raise type + else: + raise type, value, traceback + + +# type of a function that steps from node to node +ctypedef public xmlNode* (*_node_to_node_function)(xmlNode*) + + +################################################################################ +# Include submodules + +include "proxy.pxi" # Proxy handling (element backpointers/memory/etc.) +include "apihelpers.pxi" # Private helper functions +include "xmlerror.pxi" # Error and log handling + + +################################################################################ +# Public Python API + +@cython.final +@cython.freelist(8) +cdef public class _Document [ type LxmlDocumentType, object LxmlDocument ]: + """Internal base class to reference a libxml document. + + When instances of this class are garbage collected, the libxml + document is cleaned up. + """ + cdef int _ns_counter + cdef bytes _prefix_tail + cdef xmlDoc* _c_doc + cdef _BaseParser _parser + + def __dealloc__(self): + # if there are no more references to the document, it is safe + # to clean the whole thing up, as all nodes have a reference to + # the document + tree.xmlFreeDoc(self._c_doc) + + @cython.final + cdef getroot(self): + # return an element proxy for the document root + cdef xmlNode* c_node + c_node = tree.xmlDocGetRootElement(self._c_doc) + if c_node is NULL: + return None + return _elementFactory(self, c_node) + + @cython.final + cdef bint hasdoctype(self) noexcept: + # DOCTYPE gets parsed into internal subset (xmlDTD*) + return self._c_doc is not NULL and self._c_doc.intSubset is not NULL + + @cython.final + cdef getdoctype(self): + # get doctype info: root tag, public/system ID (or None if not known) + cdef tree.xmlDtd* c_dtd + cdef xmlNode* c_root_node + public_id = None + sys_url = None + c_dtd = self._c_doc.intSubset + if c_dtd is not NULL: + if c_dtd.ExternalID is not NULL: + public_id = funicode(c_dtd.ExternalID) + if c_dtd.SystemID is not NULL: + sys_url = funicode(c_dtd.SystemID) + c_dtd = self._c_doc.extSubset + if c_dtd is not NULL: + if not public_id and c_dtd.ExternalID is not NULL: + public_id = funicode(c_dtd.ExternalID) + if not sys_url and c_dtd.SystemID is not NULL: + sys_url = funicode(c_dtd.SystemID) + c_root_node = tree.xmlDocGetRootElement(self._c_doc) + if c_root_node is NULL: + root_name = None + else: + root_name = funicode(c_root_node.name) + return root_name, public_id, sys_url + + @cython.final + cdef getxmlinfo(self): + # return XML version and encoding (or None if not known) + cdef xmlDoc* c_doc = self._c_doc + if c_doc.version is NULL: + version = None + else: + version = funicode(c_doc.version) + if c_doc.encoding is NULL: + encoding = None + else: + encoding = funicode(c_doc.encoding) + return version, encoding + + @cython.final + cdef isstandalone(self): + # returns True for "standalone=true", + # False for "standalone=false", None if not provided + if self._c_doc.standalone == -1: + return None + else: + return (self._c_doc.standalone == 1) + + @cython.final + cdef bytes buildNewPrefix(self): + # get a new unique prefix ("nsX") for this document + cdef bytes ns + if self._ns_counter < len(_PREFIX_CACHE): + ns = _PREFIX_CACHE[self._ns_counter] + else: + ns = python.PyBytes_FromFormat("ns%d", self._ns_counter) + if self._prefix_tail is not None: + ns += self._prefix_tail + self._ns_counter += 1 + if self._ns_counter < 0: + # overflow! + self._ns_counter = 0 + if self._prefix_tail is None: + self._prefix_tail = b"A" + else: + self._prefix_tail += b"A" + return ns + + @cython.final + cdef xmlNs* _findOrBuildNodeNs(self, xmlNode* c_node, + const_xmlChar* c_href, const_xmlChar* c_prefix, + bint is_attribute) except NULL: + """Get or create namespace structure for a node. Reuses the prefix if + possible. + """ + cdef xmlNs* c_ns + cdef xmlNs* c_doc_ns + cdef python.PyObject* dict_result + if c_node.type != tree.XML_ELEMENT_NODE: + assert c_node.type == tree.XML_ELEMENT_NODE, \ + "invalid node type %d, expected %d" % ( + c_node.type, tree.XML_ELEMENT_NODE) + # look for existing ns declaration + c_ns = _searchNsByHref(c_node, c_href, is_attribute) + if c_ns is not NULL: + if is_attribute and c_ns.prefix is NULL: + # do not put namespaced attributes into the default + # namespace as this would break serialisation + pass + else: + return c_ns + + # none found => determine a suitable new prefix + if c_prefix is NULL: + dict_result = python.PyDict_GetItem( + _DEFAULT_NAMESPACE_PREFIXES, c_href) + if dict_result is not NULL: + prefix = dict_result + else: + prefix = self.buildNewPrefix() + c_prefix = _xcstr(prefix) + + # make sure the prefix is not in use already + while tree.xmlSearchNs(self._c_doc, c_node, c_prefix) is not NULL: + prefix = self.buildNewPrefix() + c_prefix = _xcstr(prefix) + + # declare the namespace and return it + c_ns = tree.xmlNewNs(c_node, c_href, c_prefix) + if c_ns is NULL: + raise MemoryError() + return c_ns + + @cython.final + cdef int _setNodeNs(self, xmlNode* c_node, const_xmlChar* c_href) except -1: + "Lookup namespace structure and set it for the node." + c_ns = self._findOrBuildNodeNs(c_node, c_href, NULL, 0) + tree.xmlSetNs(c_node, c_ns) + + +cdef tuple __initPrefixCache(): + cdef int i + return tuple([ python.PyBytes_FromFormat("ns%d", i) + for i in range(26) ]) + +cdef tuple _PREFIX_CACHE = __initPrefixCache() + + +cdef _Document _documentFactory(xmlDoc* c_doc, _BaseParser parser): + cdef _Document result + result = _Document.__new__(_Document) + result._c_doc = c_doc + result._ns_counter = 0 + result._prefix_tail = None + if parser is None: + parser = __GLOBAL_PARSER_CONTEXT.getDefaultParser() + result._parser = parser + return result + + +cdef object _find_invalid_public_id_characters = re.compile( + ur"[^\x20\x0D\x0Aa-zA-Z0-9'()+,./:=?;!*#@$_%-]+").search + + +cdef class DocInfo: + "Document information provided by parser and DTD." + cdef _Document _doc + def __cinit__(self, tree): + "Create a DocInfo object for an ElementTree object or root Element." + self._doc = _documentOrRaise(tree) + root_name, public_id, system_url = self._doc.getdoctype() + if not root_name and (public_id or system_url): + raise ValueError, "Could not find root node" + + @property + def root_name(self): + """Returns the name of the root node as defined by the DOCTYPE.""" + root_name, public_id, system_url = self._doc.getdoctype() + return root_name + + @cython.final + cdef tree.xmlDtd* _get_c_dtd(self) noexcept: + """"Return the DTD. Create it if it does not yet exist.""" + cdef xmlDoc* c_doc = self._doc._c_doc + cdef xmlNode* c_root_node + cdef const_xmlChar* c_name + + if c_doc.intSubset: + return c_doc.intSubset + + c_root_node = tree.xmlDocGetRootElement(c_doc) + c_name = c_root_node.name if c_root_node else NULL + return tree.xmlCreateIntSubset(c_doc, c_name, NULL, NULL) + + def clear(self): + """Removes DOCTYPE and internal subset from the document.""" + cdef xmlDoc* c_doc = self._doc._c_doc + cdef tree.xmlNode* c_dtd = c_doc.intSubset + if c_dtd is NULL: + return + tree.xmlUnlinkNode(c_dtd) + tree.xmlFreeNode(c_dtd) + + property public_id: + """Public ID of the DOCTYPE. + + Mutable. May be set to a valid string or None. If a DTD does not + exist, setting this variable (even to None) will create one. + """ + def __get__(self): + root_name, public_id, system_url = self._doc.getdoctype() + return public_id + + def __set__(self, value): + cdef xmlChar* c_value = NULL + if value is not None: + match = _find_invalid_public_id_characters(value) + if match: + raise ValueError, f'Invalid character(s) {match.group(0)!r} in public_id.' + value = _utf8(value) + c_value = tree.xmlStrdup(_xcstr(value)) + if not c_value: + raise MemoryError() + + c_dtd = self._get_c_dtd() + if not c_dtd: + tree.xmlFree(c_value) + raise MemoryError() + if c_dtd.ExternalID: + tree.xmlFree(c_dtd.ExternalID) + c_dtd.ExternalID = c_value + + property system_url: + """System ID of the DOCTYPE. + + Mutable. May be set to a valid string or None. If a DTD does not + exist, setting this variable (even to None) will create one. + """ + def __get__(self): + root_name, public_id, system_url = self._doc.getdoctype() + return system_url + + def __set__(self, value): + cdef xmlChar* c_value = NULL + if value is not None: + bvalue = _utf8(value) + # sys_url may be any valid unicode string that can be + # enclosed in single quotes or quotes. + if b"'" in bvalue and b'"' in bvalue: + raise ValueError( + 'System URL may not contain both single (\') and double quotes (").') + c_value = tree.xmlStrdup(_xcstr(bvalue)) + if not c_value: + raise MemoryError() + + c_dtd = self._get_c_dtd() + if not c_dtd: + tree.xmlFree(c_value) + raise MemoryError() + if c_dtd.SystemID: + tree.xmlFree(c_dtd.SystemID) + c_dtd.SystemID = c_value + + @property + def xml_version(self): + """Returns the XML version as declared by the document.""" + xml_version, encoding = self._doc.getxmlinfo() + return xml_version + + @property + def encoding(self): + """Returns the encoding name as declared by the document.""" + xml_version, encoding = self._doc.getxmlinfo() + return encoding + + @property + def standalone(self): + """Returns the standalone flag as declared by the document. The possible + values are True (``standalone='yes'``), False + (``standalone='no'`` or flag not provided in the declaration), + and None (unknown or no declaration found). Note that a + normal truth test on this value will always tell if the + ``standalone`` flag was set to ``'yes'`` or not. + """ + return self._doc.isstandalone() + + property URL: + "The source URL of the document (or None if unknown)." + def __get__(self): + if self._doc._c_doc.URL is NULL: + return None + return _decodeFilename(self._doc._c_doc.URL) + def __set__(self, url): + url = _encodeFilename(url) + c_oldurl = self._doc._c_doc.URL + if url is None: + self._doc._c_doc.URL = NULL + else: + self._doc._c_doc.URL = tree.xmlStrdup(_xcstr(url)) + if c_oldurl is not NULL: + tree.xmlFree(c_oldurl) + + @property + def doctype(self): + """Returns a DOCTYPE declaration string for the document.""" + root_name, public_id, system_url = self._doc.getdoctype() + if system_url: + # If '"' in system_url, we must escape it with single + # quotes, otherwise escape with double quotes. If url + # contains both a single quote and a double quote, XML + # standard is being violated. + if '"' in system_url: + quoted_system_url = f"'{system_url}'" + else: + quoted_system_url = f'"{system_url}"' + if public_id: + if system_url: + return f'' + else: + return f'' + elif system_url: + return f'' + elif self._doc.hasdoctype(): + return f'' + else: + return '' + + @property + def internalDTD(self): + """Returns a DTD validator based on the internal subset of the document.""" + return _dtdFactory(self._doc._c_doc.intSubset) + + @property + def externalDTD(self): + """Returns a DTD validator based on the external subset of the document.""" + return _dtdFactory(self._doc._c_doc.extSubset) + + +@cython.no_gc_clear +cdef public class _Element [ type LxmlElementType, object LxmlElement ]: + """Element class. + + References a document object and a libxml node. + + By pointing to a Document instance, a reference is kept to + _Document as long as there is some pointer to a node in it. + """ + cdef _Document _doc + cdef xmlNode* _c_node + cdef object _tag + + def _init(self): + """_init(self) + + Called after object initialisation. Custom subclasses may override + this if they recursively call _init() in the superclasses. + """ + + @cython.linetrace(False) + @cython.profile(False) + def __dealloc__(self): + #print("trying to free node:", self._c_node) + #displayNode(self._c_node, 0) + if self._c_node is not NULL: + _unregisterProxy(self) + attemptDeallocation(self._c_node) + + # MANIPULATORS + + def __setitem__(self, x, value): + """__setitem__(self, x, value) + + Replaces the given subelement index or slice. + """ + cdef xmlNode* c_node = NULL + cdef xmlNode* c_next + cdef xmlDoc* c_source_doc + cdef _Element element + cdef bint left_to_right + cdef Py_ssize_t slicelength = 0, step = 0 + _assertValidNode(self) + if value is None: + raise ValueError, "cannot assign None" + if isinstance(x, slice): + # slice assignment + _findChildSlice(x, self._c_node, &c_node, &step, &slicelength) + if step > 0: + left_to_right = 1 + else: + left_to_right = 0 + step = -step if step != python.PY_SSIZE_T_MIN else python.PY_SSIZE_T_MAX + _replaceSlice(self, c_node, slicelength, step, left_to_right, value) + return + else: + # otherwise: normal item assignment + element = value + _assertValidNode(element) + c_node = _findChild(self._c_node, x) + if c_node is NULL: + raise IndexError, "list index out of range" + c_source_doc = element._c_node.doc + c_next = element._c_node.next + _removeText(c_node.next) + tree.xmlReplaceNode(c_node, element._c_node) + _moveTail(c_next, element._c_node) + moveNodeToDocument(self._doc, c_source_doc, element._c_node) + if not attemptDeallocation(c_node): + moveNodeToDocument(self._doc, c_node.doc, c_node) + + def __delitem__(self, x): + """__delitem__(self, x) + + Deletes the given subelement or a slice. + """ + cdef xmlNode* c_node = NULL + cdef xmlNode* c_next + cdef Py_ssize_t step = 0, slicelength = 0 + _assertValidNode(self) + if isinstance(x, slice): + # slice deletion + if _isFullSlice(x): + c_node = self._c_node.children + if c_node is not NULL: + if not _isElement(c_node): + c_node = _nextElement(c_node) + while c_node is not NULL: + c_next = _nextElement(c_node) + _removeNode(self._doc, c_node) + c_node = c_next + else: + _findChildSlice(x, self._c_node, &c_node, &step, &slicelength) + _deleteSlice(self._doc, c_node, slicelength, step) + else: + # item deletion + c_node = _findChild(self._c_node, x) + if c_node is NULL: + raise IndexError, f"index out of range: {x}" + _removeNode(self._doc, c_node) + + def __deepcopy__(self, memo): + "__deepcopy__(self, memo)" + return self.__copy__() + + def __copy__(self): + "__copy__(self)" + cdef xmlDoc* c_doc + cdef xmlNode* c_node + cdef _Document new_doc + _assertValidNode(self) + c_doc = _copyDocRoot(self._doc._c_doc, self._c_node) # recursive + new_doc = _documentFactory(c_doc, self._doc._parser) + root = new_doc.getroot() + if root is not None: + return root + # Comment/PI + c_node = c_doc.children + while c_node is not NULL and c_node.type != self._c_node.type: + c_node = c_node.next + if c_node is NULL: + return None + return _elementFactory(new_doc, c_node) + + def set(self, key, value): + """set(self, key, value) + + Sets an element attribute. + In HTML documents (not XML or XHTML), the value None is allowed and creates + an attribute without value (just the attribute name). + """ + _assertValidNode(self) + _setAttributeValue(self, key, value) + + def append(self, _Element element not None): + """append(self, element) + + Adds a subelement to the end of this element. + """ + _assertValidNode(self) + _assertValidNode(element) + _appendChild(self, element) + + def addnext(self, _Element element not None): + """addnext(self, element) + + Adds the element as a following sibling directly after this + element. + + This is normally used to set a processing instruction or comment after + the root node of a document. Note that tail text is automatically + discarded when adding at the root level. + """ + _assertValidNode(self) + _assertValidNode(element) + if self._c_node.parent != NULL and not _isElement(self._c_node.parent): + if element._c_node.type not in (tree.XML_PI_NODE, tree.XML_COMMENT_NODE): + raise TypeError, "Only processing instructions and comments can be siblings of the root element" + element.tail = None + _appendSibling(self, element) + + def addprevious(self, _Element element not None): + """addprevious(self, element) + + Adds the element as a preceding sibling directly before this + element. + + This is normally used to set a processing instruction or comment + before the root node of a document. Note that tail text is + automatically discarded when adding at the root level. + """ + _assertValidNode(self) + _assertValidNode(element) + if self._c_node.parent != NULL and not _isElement(self._c_node.parent): + if element._c_node.type != tree.XML_PI_NODE: + if element._c_node.type != tree.XML_COMMENT_NODE: + raise TypeError, "Only processing instructions and comments can be siblings of the root element" + element.tail = None + _prependSibling(self, element) + + def extend(self, elements): + """extend(self, elements) + + Extends the current children by the elements in the iterable. + """ + cdef _Element element + _assertValidNode(self) + for element in elements: + if element is None: + raise TypeError, "Node must not be None" + _assertValidNode(element) + _appendChild(self, element) + + def clear(self, bint keep_tail=False): + """clear(self, keep_tail=False) + + Resets an element. This function removes all subelements, clears + all attributes and sets the text and tail properties to None. + + Pass ``keep_tail=True`` to leave the tail text untouched. + """ + cdef xmlAttr* c_attr + cdef xmlAttr* c_attr_next + cdef xmlNode* c_node + cdef xmlNode* c_node_next + _assertValidNode(self) + c_node = self._c_node + # remove self.text and self.tail + _removeText(c_node.children) + if not keep_tail: + _removeText(c_node.next) + # remove all attributes + c_attr = c_node.properties + if c_attr: + c_node.properties = NULL + tree.xmlFreePropList(c_attr) + # remove all subelements + c_node = c_node.children + if c_node and not _isElement(c_node): + c_node = _nextElement(c_node) + while c_node is not NULL: + c_node_next = _nextElement(c_node) + _removeNode(self._doc, c_node) + c_node = c_node_next + + def insert(self, index: int, _Element element not None): + """insert(self, index, element) + + Inserts a subelement at the given position in this element + """ + cdef xmlNode* c_node + cdef xmlNode* c_next + cdef xmlDoc* c_source_doc + _assertValidNode(self) + _assertValidNode(element) + c_node = _findChild(self._c_node, index) + if c_node is NULL: + _appendChild(self, element) + return + # prevent cycles + if _isAncestorOrSame(element._c_node, self._c_node): + raise ValueError("cannot append parent to itself") + c_source_doc = element._c_node.doc + c_next = element._c_node.next + tree.xmlAddPrevSibling(c_node, element._c_node) + _moveTail(c_next, element._c_node) + moveNodeToDocument(self._doc, c_source_doc, element._c_node) + + def remove(self, _Element element not None): + """remove(self, element) + + Removes a matching subelement. Unlike the find methods, this + method compares elements based on identity, not on tag value + or contents. + """ + cdef xmlNode* c_node + cdef xmlNode* c_next + _assertValidNode(self) + _assertValidNode(element) + c_node = element._c_node + if c_node.parent is not self._c_node: + raise ValueError, "Element is not a child of this node." + c_next = element._c_node.next + tree.xmlUnlinkNode(c_node) + _moveTail(c_next, c_node) + # fix namespace declarations + moveNodeToDocument(self._doc, c_node.doc, c_node) + + def replace(self, _Element old_element not None, + _Element new_element not None): + """replace(self, old_element, new_element) + + Replaces a subelement with the element passed as second argument. + """ + cdef xmlNode* c_old_node + cdef xmlNode* c_old_next + cdef xmlNode* c_new_node + cdef xmlNode* c_new_next + cdef xmlDoc* c_source_doc + _assertValidNode(self) + _assertValidNode(old_element) + _assertValidNode(new_element) + c_old_node = old_element._c_node + if c_old_node.parent is not self._c_node: + raise ValueError, "Element is not a child of this node." + c_new_node = new_element._c_node + # prevent cycles + if _isAncestorOrSame(c_new_node, self._c_node): + raise ValueError("cannot append parent to itself") + # replace node + c_old_next = c_old_node.next + c_new_next = c_new_node.next + c_source_doc = c_new_node.doc + tree.xmlReplaceNode(c_old_node, c_new_node) + _moveTail(c_new_next, c_new_node) + _moveTail(c_old_next, c_old_node) + moveNodeToDocument(self._doc, c_source_doc, c_new_node) + # fix namespace declarations + moveNodeToDocument(self._doc, c_old_node.doc, c_old_node) + + # PROPERTIES + property tag: + """Element tag + """ + def __get__(self): + if self._tag is not None: + return self._tag + _assertValidNode(self) + self._tag = _namespacedName(self._c_node) + return self._tag + + def __set__(self, value): + cdef _BaseParser parser + _assertValidNode(self) + ns, name = _getNsTag(value) + parser = self._doc._parser + if parser is not None and parser._for_html: + _htmlTagValidOrRaise(name) + else: + _tagValidOrRaise(name) + self._tag = value + tree.xmlNodeSetName(self._c_node, _xcstr(name)) + if ns is None: + self._c_node.ns = NULL + else: + self._doc._setNodeNs(self._c_node, _xcstr(ns)) + + @property + def attrib(self): + """Element attribute dictionary. Where possible, use get(), set(), + keys(), values() and items() to access element attributes. + """ + return _Attrib.__new__(_Attrib, self) + + property text: + """Text before the first subelement. This is either a string or + the value None, if there was no text. + """ + def __get__(self): + _assertValidNode(self) + return _collectText(self._c_node.children) + + def __set__(self, value): + _assertValidNode(self) + if isinstance(value, QName): + value = _resolveQNameText(self, value).decode('utf8') + _setNodeText(self._c_node, value) + + # using 'del el.text' is the wrong thing to do + #def __del__(self): + # _setNodeText(self._c_node, None) + + property tail: + """Text after this element's end tag, but before the next sibling + element's start tag. This is either a string or the value None, if + there was no text. + """ + def __get__(self): + _assertValidNode(self) + return _collectText(self._c_node.next) + + def __set__(self, value): + _assertValidNode(self) + _setTailText(self._c_node, value) + + # using 'del el.tail' is the wrong thing to do + #def __del__(self): + # _setTailText(self._c_node, None) + + # not in ElementTree, read-only + @property + def prefix(self): + """Namespace prefix or None. + """ + if self._c_node.ns is not NULL: + if self._c_node.ns.prefix is not NULL: + return funicode(self._c_node.ns.prefix) + return None + + # not in ElementTree, read-only + property sourceline: + """Original line number as found by the parser or None if unknown. + """ + def __get__(self): + cdef long line + _assertValidNode(self) + line = tree.xmlGetLineNo(self._c_node) + return line if line > 0 else None + + def __set__(self, line): + _assertValidNode(self) + if line <= 0: + self._c_node.line = 0 + else: + self._c_node.line = line + + # not in ElementTree, read-only + @property + def nsmap(self): + """Namespace prefix->URI mapping known in the context of this + Element. This includes all namespace declarations of the + parents. + + Note that changing the returned dict has no effect on the Element. + """ + _assertValidNode(self) + return _build_nsmap(self._c_node) + + # not in ElementTree, read-only + property base: + """The base URI of the Element (xml:base or HTML base URL). + None if the base URI is unknown. + + Note that the value depends on the URL of the document that + holds the Element if there is no xml:base attribute on the + Element or its ancestors. + + Setting this property will set an xml:base attribute on the + Element, regardless of the document type (XML or HTML). + """ + def __get__(self): + _assertValidNode(self) + c_base = tree.xmlNodeGetBase(self._doc._c_doc, self._c_node) + if c_base is NULL: + if self._doc._c_doc.URL is NULL: + return None + return _decodeFilename(self._doc._c_doc.URL) + try: + base = _decodeFilename(c_base) + finally: + tree.xmlFree(c_base) + return base + + def __set__(self, url): + _assertValidNode(self) + if url is None: + c_base = NULL + else: + url = _encodeFilename(url) + c_base = _xcstr(url) + tree.xmlNodeSetBase(self._c_node, c_base) + + # ACCESSORS + def __repr__(self): + "__repr__(self)" + return "" % (self.tag, id(self)) + + def __getitem__(self, x): + """Returns the subelement at the given position or the requested + slice. + """ + cdef xmlNode* c_node = NULL + cdef Py_ssize_t step = 0, slicelength = 0 + cdef Py_ssize_t c, i + cdef _node_to_node_function next_element + cdef list result + _assertValidNode(self) + if isinstance(x, slice): + # slicing + if _isFullSlice(x): + return _collectChildren(self) + _findChildSlice(x, self._c_node, &c_node, &step, &slicelength) + if c_node is NULL: + return [] + if step > 0: + next_element = _nextElement + else: + step = -step if step != python.PY_SSIZE_T_MIN else python.PY_SSIZE_T_MAX + next_element = _previousElement + result = [] + c = 0 + while c_node is not NULL and c < slicelength: + result.append(_elementFactory(self._doc, c_node)) + c += 1 + for i in range(step): + c_node = next_element(c_node) + if c_node is NULL: + break + return result + else: + # indexing + c_node = _findChild(self._c_node, x) + if c_node is NULL: + raise IndexError, "list index out of range" + return _elementFactory(self._doc, c_node) + + def __len__(self): + """__len__(self) + + Returns the number of subelements. + """ + _assertValidNode(self) + return _countElements(self._c_node.children) + + def __bool__(self): + """__bool__(self)""" + import warnings + warnings.warn( + "Truth-testing of elements was a source of confusion and will always " + "return True in future versions. " + "Use specific 'len(elem)' or 'elem is not None' test instead.", + FutureWarning + ) + # emulate old behaviour + _assertValidNode(self) + return _hasChild(self._c_node) + + def __contains__(self, element): + "__contains__(self, element)" + cdef xmlNode* c_node + _assertValidNode(self) + if not isinstance(element, _Element): + return 0 + c_node = (<_Element>element)._c_node + return c_node is not NULL and c_node.parent is self._c_node + + def __iter__(self): + "__iter__(self)" + return ElementChildIterator(self) + + def __reversed__(self): + "__reversed__(self)" + return ElementChildIterator(self, reversed=True) + + def index(self, child: _Element, start: int = None, stop: int = None): + """index(self, child, start=None, stop=None) + + Find the position of the child within the parent. + + This method is not part of the original ElementTree API. + """ + cdef Py_ssize_t k, l + cdef Py_ssize_t c_start, c_stop + cdef xmlNode* c_child + cdef xmlNode* c_start_node + _assertValidNode(self) + _assertValidNode(child) + c_child = child._c_node + if c_child.parent is not self._c_node: + raise ValueError, "Element is not a child of this node." + + # handle the unbounded search straight away (normal case) + if stop is None and (start is None or start == 0): + k = 0 + c_child = c_child.prev + while c_child is not NULL: + if _isElement(c_child): + k += 1 + c_child = c_child.prev + return k + + # check indices + if start is None: + c_start = 0 + else: + c_start = start + if stop is None: + c_stop = 0 + else: + c_stop = stop + if c_stop == 0 or \ + c_start >= c_stop and (c_stop > 0 or c_start < 0): + raise ValueError, "list.index(x): x not in slice" + + # for negative slice indices, check slice before searching index + if c_start < 0 or c_stop < 0: + # start from right, at most up to leftmost(c_start, c_stop) + if c_start < c_stop: + k = -c_start + else: + k = -c_stop + c_start_node = self._c_node.last + l = 1 + while c_start_node != c_child and l < k: + if _isElement(c_start_node): + l += 1 + c_start_node = c_start_node.prev + if c_start_node == c_child: + # found! before slice end? + if c_stop < 0 and l <= -c_stop: + raise ValueError, "list.index(x): x not in slice" + elif c_start < 0: + raise ValueError, "list.index(x): x not in slice" + + # now determine the index backwards from child + c_child = c_child.prev + k = 0 + if c_stop > 0: + # we can optimize: stop after c_stop elements if not found + while c_child != NULL and k < c_stop: + if _isElement(c_child): + k += 1 + c_child = c_child.prev + if k < c_stop: + return k + else: + # traverse all + while c_child != NULL: + if _isElement(c_child): + k = k + 1 + c_child = c_child.prev + if c_start > 0: + if k >= c_start: + return k + else: + return k + if c_start != 0 or c_stop != 0: + raise ValueError, "list.index(x): x not in slice" + else: + raise ValueError, "list.index(x): x not in list" + + def get(self, key, default=None): + """get(self, key, default=None) + + Gets an element attribute. + """ + _assertValidNode(self) + return _getAttributeValue(self, key, default) + + def keys(self): + """keys(self) + + Gets a list of attribute names. The names are returned in an + arbitrary order (just like for an ordinary Python dictionary). + """ + _assertValidNode(self) + return _collectAttributes(self._c_node, 1) + + def values(self): + """values(self) + + Gets element attribute values as a sequence of strings. The + attributes are returned in an arbitrary order. + """ + _assertValidNode(self) + return _collectAttributes(self._c_node, 2) + + def items(self): + """items(self) + + Gets element attributes, as a sequence. The attributes are returned in + an arbitrary order. + """ + _assertValidNode(self) + return _collectAttributes(self._c_node, 3) + + def getchildren(self): + """getchildren(self) + + Returns all direct children. The elements are returned in document + order. + + :deprecated: Note that this method has been deprecated as of + ElementTree 1.3 and lxml 2.0. New code should use + ``list(element)`` or simply iterate over elements. + """ + _assertValidNode(self) + return _collectChildren(self) + + def getparent(self): + """getparent(self) + + Returns the parent of this element or None for the root element. + """ + cdef xmlNode* c_node + #_assertValidNode(self) # not needed + c_node = _parentElement(self._c_node) + if c_node is NULL: + return None + return _elementFactory(self._doc, c_node) + + def getnext(self): + """getnext(self) + + Returns the following sibling of this element or None. + """ + cdef xmlNode* c_node + #_assertValidNode(self) # not needed + c_node = _nextElement(self._c_node) + if c_node is NULL: + return None + return _elementFactory(self._doc, c_node) + + def getprevious(self): + """getprevious(self) + + Returns the preceding sibling of this element or None. + """ + cdef xmlNode* c_node + #_assertValidNode(self) # not needed + c_node = _previousElement(self._c_node) + if c_node is NULL: + return None + return _elementFactory(self._doc, c_node) + + def itersiblings(self, tag=None, *tags, preceding=False): + """itersiblings(self, tag=None, *tags, preceding=False) + + Iterate over the following or preceding siblings of this element. + + The direction is determined by the 'preceding' keyword which + defaults to False, i.e. forward iteration over the following + siblings. When True, the iterator yields the preceding + siblings in reverse document order, i.e. starting right before + the current element and going backwards. + + Can be restricted to find only elements with specific tags, + see `iter`. + """ + if preceding: + if self._c_node and not self._c_node.prev: + return ITER_EMPTY + elif self._c_node and not self._c_node.next: + return ITER_EMPTY + if tag is not None: + tags += (tag,) + return SiblingsIterator(self, tags, preceding=preceding) + + def iterancestors(self, tag=None, *tags): + """iterancestors(self, tag=None, *tags) + + Iterate over the ancestors of this element (from parent to parent). + + Can be restricted to find only elements with specific tags, + see `iter`. + """ + if self._c_node and not self._c_node.parent: + return ITER_EMPTY + if tag is not None: + tags += (tag,) + return AncestorsIterator(self, tags) + + def iterdescendants(self, tag=None, *tags): + """iterdescendants(self, tag=None, *tags) + + Iterate over the descendants of this element in document order. + + As opposed to ``el.iter()``, this iterator does not yield the element + itself. The returned elements can be restricted to find only elements + with specific tags, see `iter`. + """ + if self._c_node and not self._c_node.children: + return ITER_EMPTY + if tag is not None: + tags += (tag,) + return ElementDepthFirstIterator(self, tags, inclusive=False) + + def iterchildren(self, tag=None, *tags, reversed=False): + """iterchildren(self, tag=None, *tags, reversed=False) + + Iterate over the children of this element. + + As opposed to using normal iteration on this element, the returned + elements can be reversed with the 'reversed' keyword and restricted + to find only elements with specific tags, see `iter`. + """ + if self._c_node and not self._c_node.children: + return ITER_EMPTY + if tag is not None: + tags += (tag,) + return ElementChildIterator(self, tags, reversed=reversed) + + def getroottree(self): + """getroottree(self) + + Return an ElementTree for the root node of the document that + contains this element. + + This is the same as following element.getparent() up the tree until it + returns None (for the root element) and then build an ElementTree for + the last parent that was returned.""" + _assertValidDoc(self._doc) + return _elementTreeFactory(self._doc, None) + + def getiterator(self, tag=None, *tags): + """getiterator(self, tag=None, *tags) + + Returns a sequence or iterator of all elements in the subtree in + document order (depth first pre-order), starting with this + element. + + Can be restricted to find only elements with specific tags, + see `iter`. + + :deprecated: Note that this method is deprecated as of + ElementTree 1.3 and lxml 2.0. It returns an iterator in + lxml, which diverges from the original ElementTree + behaviour. If you want an efficient iterator, use the + ``element.iter()`` method instead. You should only use this + method in new code if you require backwards compatibility + with older versions of lxml or ElementTree. + """ + if tag is not None: + tags += (tag,) + return ElementDepthFirstIterator(self, tags) + + def iter(self, tag=None, *tags): + """iter(self, tag=None, *tags) + + Iterate over all elements in the subtree in document order (depth + first pre-order), starting with this element. + + Can be restricted to find only elements with specific tags: + pass ``"{ns}localname"`` as tag. Either or both of ``ns`` and + ``localname`` can be ``*`` for a wildcard; ``ns`` can be empty + for no namespace. ``"localname"`` is equivalent to ``"{}localname"`` + (i.e. no namespace) but ``"*"`` is ``"{*}*"`` (any or no namespace), + not ``"{}*"``. + + You can also pass the Element, Comment, ProcessingInstruction and + Entity factory functions to look only for the specific element type. + + Passing multiple tags (or a sequence of tags) instead of a single tag + will let the iterator return all elements matching any of these tags, + in document order. + """ + if tag is not None: + tags += (tag,) + return ElementDepthFirstIterator(self, tags) + + def itertext(self, tag=None, *tags, with_tail=True): + """itertext(self, tag=None, *tags, with_tail=True) + + Iterates over the text content of a subtree. + + You can pass tag names to restrict text content to specific elements, + see `iter`. + + You can set the ``with_tail`` keyword argument to ``False`` to skip + over tail text. + """ + if tag is not None: + tags += (tag,) + return ElementTextIterator(self, tags, with_tail=with_tail) + + def makeelement(self, _tag, attrib=None, nsmap=None, **_extra): + """makeelement(self, _tag, attrib=None, nsmap=None, **_extra) + + Creates a new element associated with the same document. + """ + _assertValidDoc(self._doc) + return _makeElement(_tag, NULL, self._doc, None, None, None, + attrib, nsmap, _extra) + + def find(self, path, namespaces=None): + """find(self, path, namespaces=None) + + Finds the first matching subelement, by tag name or path. + + The optional ``namespaces`` argument accepts a + prefix-to-namespace mapping that allows the usage of XPath + prefixes in the path expression. + """ + if isinstance(path, QName): + path = (path).text + return _elementpath.find(self, path, namespaces, with_prefixes=not _isHtmlDocument(self)) + + def findtext(self, path, default=None, namespaces=None): + """findtext(self, path, default=None, namespaces=None) + + Finds text for the first matching subelement, by tag name or path. + + The optional ``namespaces`` argument accepts a + prefix-to-namespace mapping that allows the usage of XPath + prefixes in the path expression. + """ + if isinstance(path, QName): + path = (path).text + return _elementpath.findtext(self, path, default, namespaces, with_prefixes=not _isHtmlDocument(self)) + + def findall(self, path, namespaces=None): + """findall(self, path, namespaces=None) + + Finds all matching subelements, by tag name or path. + + The optional ``namespaces`` argument accepts a + prefix-to-namespace mapping that allows the usage of XPath + prefixes in the path expression. + """ + if isinstance(path, QName): + path = (path).text + return _elementpath.findall(self, path, namespaces, with_prefixes=not _isHtmlDocument(self)) + + def iterfind(self, path, namespaces=None): + """iterfind(self, path, namespaces=None) + + Iterates over all matching subelements, by tag name or path. + + The optional ``namespaces`` argument accepts a + prefix-to-namespace mapping that allows the usage of XPath + prefixes in the path expression. + """ + if isinstance(path, QName): + path = (path).text + return _elementpath.iterfind(self, path, namespaces, with_prefixes=not _isHtmlDocument(self)) + + def xpath(self, _path, *, namespaces=None, extensions=None, + smart_strings=True, **_variables): + """xpath(self, _path, namespaces=None, extensions=None, smart_strings=True, **_variables) + + Evaluate an xpath expression using the element as context node. + """ + evaluator = XPathElementEvaluator(self, namespaces=namespaces, + extensions=extensions, + smart_strings=smart_strings) + return evaluator(_path, **_variables) + + def cssselect(self, expr, *, translator='xml'): + """ + Run the CSS expression on this element and its children, + returning a list of the results. + + Equivalent to lxml.cssselect.CSSSelect(expr)(self) -- note + that pre-compiling the expression can provide a substantial + speedup. + """ + # Do the import here to make the dependency optional. + from lxml.cssselect import CSSSelector + return CSSSelector(expr, translator=translator)(self) + + +@cython.linetrace(False) +cdef _Element _elementFactory(_Document doc, xmlNode* c_node): + cdef _Element result + result = getProxy(c_node) + if result is not None: + return result + if c_node is NULL: + return None + + element_class = LOOKUP_ELEMENT_CLASS( + ELEMENT_CLASS_LOOKUP_STATE, doc, c_node) + if type(element_class) is not type: + if not isinstance(element_class, type): + raise TypeError(f"Element class is not a type, got {type(element_class)}") + if hasProxy(c_node): + # prevent re-entry race condition - we just called into Python + return getProxy(c_node) + result = element_class.__new__(element_class) + if hasProxy(c_node): + # prevent re-entry race condition - we just called into Python + result._c_node = NULL + return getProxy(c_node) + + _registerProxy(result, doc, c_node) + if element_class is not _Element: + result._init() + return result + + +@cython.internal +cdef class __ContentOnlyElement(_Element): + cdef int _raiseImmutable(self) except -1: + raise TypeError, "this element does not have children or attributes" + + def set(self, key, value): + "set(self, key, value)" + self._raiseImmutable() + + def append(self, value): + "append(self, value)" + self._raiseImmutable() + + def insert(self, index, value): + "insert(self, index, value)" + self._raiseImmutable() + + def __setitem__(self, index, value): + "__setitem__(self, index, value)" + self._raiseImmutable() + + @property + def attrib(self): + return IMMUTABLE_EMPTY_MAPPING + + property text: + def __get__(self): + _assertValidNode(self) + return funicodeOrEmpty(self._c_node.content) + + def __set__(self, value): + cdef tree.xmlDict* c_dict + _assertValidNode(self) + if value is None: + c_text = NULL + else: + value = _utf8(value) + c_text = _xcstr(value) + tree.xmlNodeSetContent(self._c_node, c_text) + + # ACCESSORS + def __getitem__(self, x): + "__getitem__(self, x)" + if isinstance(x, slice): + return [] + else: + raise IndexError, "list index out of range" + + def __len__(self): + "__len__(self)" + return 0 + + def get(self, key, default=None): + "get(self, key, default=None)" + return None + + def keys(self): + "keys(self)" + return [] + + def items(self): + "items(self)" + return [] + + def values(self): + "values(self)" + return [] + +cdef class _Comment(__ContentOnlyElement): + @property + def tag(self): + return Comment + + def __repr__(self): + return "" % self.text + +cdef class _ProcessingInstruction(__ContentOnlyElement): + @property + def tag(self): + return ProcessingInstruction + + property target: + # not in ElementTree + def __get__(self): + _assertValidNode(self) + return funicode(self._c_node.name) + + def __set__(self, value): + _assertValidNode(self) + value = _utf8(value) + c_text = _xcstr(value) + tree.xmlNodeSetName(self._c_node, c_text) + + def __repr__(self): + text = self.text + if text: + return "" % (self.target, text) + else: + return "" % self.target + + def get(self, key, default=None): + """get(self, key, default=None) + + Try to parse pseudo-attributes from the text content of the + processing instruction, search for one with the given key as + name and return its associated value. + + Note that this is only a convenience method for the most + common case that all text content is structured in + attribute-like name-value pairs with properly quoted values. + It is not guaranteed to work for all possible text content. + """ + return self.attrib.get(key, default) + + @property + def attrib(self): + """Returns a dict containing all pseudo-attributes that can be + parsed from the text content of this processing instruction. + Note that modifying the dict currently has no effect on the + XML node, although this is not guaranteed to stay this way. + """ + return { attr : (value1 or value2) + for attr, value1, value2 in _FIND_PI_ATTRIBUTES(' ' + self.text) } + +cdef object _FIND_PI_ATTRIBUTES = re.compile(r'\s+(\w+)\s*=\s*(?:\'([^\']*)\'|"([^"]*)")', re.U).findall + +cdef class _Entity(__ContentOnlyElement): + @property + def tag(self): + return Entity + + property name: + # not in ElementTree + def __get__(self): + _assertValidNode(self) + return funicode(self._c_node.name) + + def __set__(self, value): + _assertValidNode(self) + value_utf = _utf8(value) + if b'&' in value_utf or b';' in value_utf: + raise ValueError, f"Invalid entity name '{value}'" + tree.xmlNodeSetName(self._c_node, _xcstr(value_utf)) + + @property + def text(self): + # FIXME: should this be None or '&[VALUE];' or the resolved + # entity value ? + _assertValidNode(self) + return f'&{funicode(self._c_node.name)};' + + def __repr__(self): + return "&%s;" % self.name + + +cdef class QName: + """QName(text_or_uri_or_element, tag=None) + + QName wrapper for qualified XML names. + + Pass a tag name by itself or a namespace URI and a tag name to + create a qualified name. Alternatively, pass an Element to + extract its tag name. ``None`` as first argument is ignored in + order to allow for generic 2-argument usage. + + The ``text`` property holds the qualified name in + ``{namespace}tagname`` notation. The ``namespace`` and + ``localname`` properties hold the respective parts of the tag + name. + + You can pass QName objects wherever a tag name is expected. Also, + setting Element text from a QName will resolve the namespace prefix + on assignment and set a qualified text value. This is helpful in XML + languages like SOAP or XML-Schema that use prefixed tag names in + their text content. + """ + cdef readonly unicode text + cdef readonly unicode localname + cdef readonly unicode namespace + def __init__(self, text_or_uri_or_element, tag=None): + if text_or_uri_or_element is None: + # Allow None as no namespace. + text_or_uri_or_element, tag = tag, None + if not _isString(text_or_uri_or_element): + if isinstance(text_or_uri_or_element, _Element): + text_or_uri_or_element = (<_Element>text_or_uri_or_element).tag + if not _isString(text_or_uri_or_element): + raise ValueError, f"Invalid input tag of type {type(text_or_uri_or_element)!r}" + elif isinstance(text_or_uri_or_element, QName): + text_or_uri_or_element = (text_or_uri_or_element).text + elif text_or_uri_or_element is not None: + text_or_uri_or_element = unicode(text_or_uri_or_element) + else: + raise ValueError, f"Invalid input tag of type {type(text_or_uri_or_element)!r}" + + ns_utf, tag_utf = _getNsTag(text_or_uri_or_element) + if tag is not None: + # either ('ns', 'tag') or ('{ns}oldtag', 'newtag') + if ns_utf is None: + ns_utf = tag_utf # case 1: namespace ended up as tag name + tag_utf = _utf8(tag) + _tagValidOrRaise(tag_utf) + self.localname = (tag_utf).decode('utf8') + if ns_utf is None: + self.namespace = None + self.text = self.localname + else: + self.namespace = (ns_utf).decode('utf8') + self.text = "{%s}%s" % (self.namespace, self.localname) + def __str__(self): + return self.text + def __hash__(self): + return hash(self.text) + def __richcmp__(self, other, int op): + try: + if type(other) is QName: + other = (other).text + elif not isinstance(other, unicode): + other = unicode(other) + except (ValueError, UnicodeDecodeError): + return NotImplemented + return python.PyObject_RichCompare(self.text, other, op) + + +cdef public class _ElementTree [ type LxmlElementTreeType, + object LxmlElementTree ]: + cdef _Document _doc + cdef _Element _context_node + + # Note that _doc is only used to store the original document if we do not + # have a _context_node. All methods should prefer self._context_node._doc + # to honour tree restructuring. _doc can happily be None! + + @cython.final + cdef int _assertHasRoot(self) except -1: + """We have to take care here: the document may not have a root node! + This can happen if ElementTree() is called without any argument and + the caller 'forgets' to call parse() afterwards, so this is a bug in + the caller program. + """ + assert self._context_node is not None, \ + "ElementTree not initialized, missing root" + return 0 + + def parse(self, source, _BaseParser parser=None, *, base_url=None): + """parse(self, source, parser=None, base_url=None) + + Updates self with the content of source and returns its root. + """ + cdef _Document doc = None + try: + doc = _parseDocument(source, parser, base_url) + except _TargetParserResult as result_container: + # raises a TypeError if we don't get an _Element + self._context_node = result_container.result + else: + self._context_node = doc.getroot() + self._doc = None if self._context_node is not None else doc + return self._context_node + + def _setroot(self, _Element root not None): + """_setroot(self, root) + + Relocate the ElementTree to a new root node. + """ + _assertValidNode(root) + if root._c_node.type != tree.XML_ELEMENT_NODE: + raise TypeError, "Only elements can be the root of an ElementTree" + self._context_node = root + self._doc = None + + def getroot(self): + """getroot(self) + + Gets the root element for this tree. + """ + return self._context_node + + def __copy__(self): + return _elementTreeFactory(self._doc, self._context_node) + + def __deepcopy__(self, memo): + cdef _Element root + cdef _Document doc + cdef xmlDoc* c_doc + if self._context_node is not None: + root = self._context_node.__copy__() + assert root is not None + _assertValidNode(root) + _copyNonElementSiblings(self._context_node._c_node, root._c_node) + return _elementTreeFactory(None, root) + elif self._doc is not None: + _assertValidDoc(self._doc) + c_doc = tree.xmlCopyDoc(self._doc._c_doc, 1) + if c_doc is NULL: + raise MemoryError() + doc = _documentFactory(c_doc, self._doc._parser) + return _elementTreeFactory(doc, None) + else: + # so what ... + return self + + # not in ElementTree + @property + def docinfo(self) -> DocInfo: + """Information about the document provided by parser and DTD.""" + self._assertHasRoot() + return DocInfo(self._context_node._doc) + + # not in ElementTree, read-only + @property + def parser(self): + """The parser that was used to parse the document in this ElementTree. + """ + if self._context_node is not None and \ + self._context_node._doc is not None: + return self._context_node._doc._parser + if self._doc is not None: + return self._doc._parser + return None + + def write(self, file, *, encoding=None, method="xml", + bint pretty_print=False, xml_declaration=None, bint with_tail=True, + standalone=None, doctype=None, compression=0, + bint exclusive=False, inclusive_ns_prefixes=None, + bint with_comments=True, bint strip_text=False, + docstring=None): + """write(self, file, encoding=None, method="xml", + pretty_print=False, xml_declaration=None, with_tail=True, + standalone=None, doctype=None, compression=0, + exclusive=False, inclusive_ns_prefixes=None, + with_comments=True, strip_text=False) + + Write the tree to a filename, file or file-like object. + + Defaults to ASCII encoding and writing a declaration as needed. + + The keyword argument 'method' selects the output method: + 'xml', 'html', 'text', 'c14n' or 'c14n2'. Default is 'xml'. + + With ``method="c14n"`` (C14N version 1), the options ``exclusive``, + ``with_comments`` and ``inclusive_ns_prefixes`` request exclusive + C14N, include comments, and list the inclusive prefixes respectively. + + With ``method="c14n2"`` (C14N version 2), the ``with_comments`` and + ``strip_text`` options control the output of comments and text space + according to C14N 2.0. + + Passing a boolean value to the ``standalone`` option will + output an XML declaration with the corresponding + ``standalone`` flag. + + The ``doctype`` option allows passing in a plain string that will + be serialised before the XML tree. Note that passing in non + well-formed content here will make the XML output non well-formed. + Also, an existing doctype in the document tree will not be removed + when serialising an ElementTree instance. + + The ``compression`` option enables GZip compression level 1-9. + + The ``inclusive_ns_prefixes`` should be a list of namespace strings + (i.e. ['xs', 'xsi']) that will be promoted to the top-level element + during exclusive C14N serialisation. This parameter is ignored if + exclusive mode=False. + + If exclusive=True and no list is provided, a namespace will only be + rendered if it is used by the immediate parent or one of its attributes + and its prefix and values have not already been rendered by an ancestor + of the namespace node's parent element. + """ + cdef bint write_declaration + cdef int is_standalone + + self._assertHasRoot() + _assertValidNode(self._context_node) + if compression is None or compression < 0: + compression = 0 + + # C14N serialisation + if method in ('c14n', 'c14n2'): + if encoding is not None: + raise ValueError("Cannot specify encoding with C14N") + if xml_declaration: + raise ValueError("Cannot enable XML declaration in C14N") + + if method == 'c14n': + _tofilelikeC14N(file, self._context_node, exclusive, with_comments, + compression, inclusive_ns_prefixes) + else: # c14n2 + with _open_utf8_file(file, compression=compression) as f: + target = C14NWriterTarget( + f.write, with_comments=with_comments, strip_text=strip_text) + _tree_to_target(self, target) + return + + if not with_comments: + raise ValueError("Can only discard comments in C14N serialisation") + # suppress decl. in default case (purely for ElementTree compatibility) + if xml_declaration is not None: + write_declaration = xml_declaration + if encoding is None: + encoding = 'ASCII' + else: + encoding = encoding.upper() + elif encoding is None: + encoding = 'ASCII' + write_declaration = 0 + else: + encoding = encoding.upper() + write_declaration = encoding not in ( + 'US-ASCII', 'ASCII', 'UTF8', 'UTF-8') + if standalone is None: + is_standalone = -1 + elif standalone: + write_declaration = 1 + is_standalone = 1 + else: + write_declaration = 1 + is_standalone = 0 + + if docstring is not None and doctype is None: + import warnings + warnings.warn( + "The 'docstring' option is deprecated. Use 'doctype' instead.", + DeprecationWarning) + doctype = docstring + + _tofilelike(file, self._context_node, encoding, doctype, method, + write_declaration, 1, pretty_print, with_tail, + is_standalone, compression) + + def getpath(self, _Element element not None): + """getpath(self, element) + + Returns a structural, absolute XPath expression to find the element. + + For namespaced elements, the expression uses prefixes from the + document, which therefore need to be provided in order to make any + use of the expression in XPath. + + Also see the method getelementpath(self, element), which returns a + self-contained ElementPath expression. + """ + cdef _Document doc + cdef _Element root + cdef xmlDoc* c_doc + _assertValidNode(element) + if self._context_node is not None: + root = self._context_node + doc = root._doc + elif self._doc is not None: + doc = self._doc + root = doc.getroot() + else: + raise ValueError, "Element is not in this tree." + _assertValidDoc(doc) + _assertValidNode(root) + if element._doc is not doc: + raise ValueError, "Element is not in this tree." + + c_doc = _fakeRootDoc(doc._c_doc, root._c_node) + c_path = tree.xmlGetNodePath(element._c_node) + _destroyFakeDoc(doc._c_doc, c_doc) + if c_path is NULL: + raise MemoryError() + path = funicode(c_path) + tree.xmlFree(c_path) + return path + + def getelementpath(self, _Element element not None): + """getelementpath(self, element) + + Returns a structural, absolute ElementPath expression to find the + element. This path can be used in the .find() method to look up + the element, provided that the elements along the path and their + list of immediate children were not modified in between. + + ElementPath has the advantage over an XPath expression (as returned + by the .getpath() method) that it does not require additional prefix + declarations. It is always self-contained. + """ + cdef _Element root + cdef Py_ssize_t count + _assertValidNode(element) + if element._c_node.type != tree.XML_ELEMENT_NODE: + raise ValueError, "input is not an Element" + if self._context_node is not None: + root = self._context_node + elif self._doc is not None: + root = self._doc.getroot() + else: + raise ValueError, "Element is not in this tree" + _assertValidNode(root) + if element._doc is not root._doc: + raise ValueError, "Element is not in this tree" + + path = [] + c_element = element._c_node + while c_element is not root._c_node: + c_name = c_element.name + c_href = _getNs(c_element) + tag = _namespacedNameFromNsName(c_href, c_name) + if c_href is NULL: + c_href = b'' # no namespace (NULL is wildcard) + # use tag[N] if there are preceding siblings with the same tag + count = 0 + c_node = c_element.prev + while c_node is not NULL: + if c_node.type == tree.XML_ELEMENT_NODE: + if _tagMatches(c_node, c_href, c_name): + count += 1 + c_node = c_node.prev + if count: + tag = f'{tag}[{count+1}]' + else: + # use tag[1] if there are following siblings with the same tag + c_node = c_element.next + while c_node is not NULL: + if c_node.type == tree.XML_ELEMENT_NODE: + if _tagMatches(c_node, c_href, c_name): + tag += '[1]' + break + c_node = c_node.next + + path.append(tag) + c_element = c_element.parent + if c_element is NULL or c_element.type != tree.XML_ELEMENT_NODE: + raise ValueError, "Element is not in this tree." + if not path: + return '.' + path.reverse() + return '/'.join(path) + + def getiterator(self, tag=None, *tags): + """getiterator(self, *tags, tag=None) + + Returns a sequence or iterator of all elements in document order + (depth first pre-order), starting with the root element. + + Can be restricted to find only elements with specific tags, + see `_Element.iter`. + + :deprecated: Note that this method is deprecated as of + ElementTree 1.3 and lxml 2.0. It returns an iterator in + lxml, which diverges from the original ElementTree + behaviour. If you want an efficient iterator, use the + ``tree.iter()`` method instead. You should only use this + method in new code if you require backwards compatibility + with older versions of lxml or ElementTree. + """ + root = self.getroot() + if root is None: + return ITER_EMPTY + if tag is not None: + tags += (tag,) + return root.getiterator(*tags) + + def iter(self, tag=None, *tags): + """iter(self, tag=None, *tags) + + Creates an iterator for the root element. The iterator loops over + all elements in this tree, in document order. Note that siblings + of the root element (comments or processing instructions) are not + returned by the iterator. + + Can be restricted to find only elements with specific tags, + see `_Element.iter`. + """ + root = self.getroot() + if root is None: + return ITER_EMPTY + if tag is not None: + tags += (tag,) + return root.iter(*tags) + + def find(self, path, namespaces=None): + """find(self, path, namespaces=None) + + Finds the first toplevel element with given tag. Same as + ``tree.getroot().find(path)``. + + The optional ``namespaces`` argument accepts a + prefix-to-namespace mapping that allows the usage of XPath + prefixes in the path expression. + """ + self._assertHasRoot() + root = self.getroot() + if _isString(path): + if path[:1] == "/": + path = "." + path + from warnings import warn + warn( + "This search incorrectly ignores the root element, and will be " + "fixed in a future version. If you rely on the current " + f"behaviour, change it to {path!r}", + FutureWarning, stacklevel=1 + ) + return root.find(path, namespaces) + + def findtext(self, path, default=None, namespaces=None): + """findtext(self, path, default=None, namespaces=None) + + Finds the text for the first element matching the ElementPath + expression. Same as getroot().findtext(path) + + The optional ``namespaces`` argument accepts a + prefix-to-namespace mapping that allows the usage of XPath + prefixes in the path expression. + """ + self._assertHasRoot() + root = self.getroot() + if _isString(path): + if path[:1] == "/": + path = "." + path + from warnings import warn + warn( + "This search incorrectly ignores the root element, and will be " + "fixed in a future version. If you rely on the current " + f"behaviour, change it to {path!r}", + FutureWarning, stacklevel=1 + ) + return root.findtext(path, default, namespaces) + + def findall(self, path, namespaces=None): + """findall(self, path, namespaces=None) + + Finds all elements matching the ElementPath expression. Same as + getroot().findall(path). + + The optional ``namespaces`` argument accepts a + prefix-to-namespace mapping that allows the usage of XPath + prefixes in the path expression. + """ + self._assertHasRoot() + root = self.getroot() + if _isString(path): + if path[:1] == "/": + path = "." + path + from warnings import warn + warn( + "This search incorrectly ignores the root element, and will be " + "fixed in a future version. If you rely on the current " + f"behaviour, change it to {path!r}", + FutureWarning, stacklevel=1 + ) + return root.findall(path, namespaces) + + def iterfind(self, path, namespaces=None): + """iterfind(self, path, namespaces=None) + + Iterates over all elements matching the ElementPath expression. + Same as getroot().iterfind(path). + + The optional ``namespaces`` argument accepts a + prefix-to-namespace mapping that allows the usage of XPath + prefixes in the path expression. + """ + self._assertHasRoot() + root = self.getroot() + if _isString(path): + if path[:1] == "/": + path = "." + path + from warnings import warn + warn( + "This search incorrectly ignores the root element, and will be " + "fixed in a future version. If you rely on the current " + f"behaviour, change it to {path!r}", + FutureWarning, stacklevel=1 + ) + return root.iterfind(path, namespaces) + + def xpath(self, _path, *, namespaces=None, extensions=None, + smart_strings=True, **_variables): + """xpath(self, _path, namespaces=None, extensions=None, smart_strings=True, **_variables) + + XPath evaluate in context of document. + + ``namespaces`` is an optional dictionary with prefix to namespace URI + mappings, used by XPath. ``extensions`` defines additional extension + functions. + + Returns a list (nodeset), or bool, float or string. + + In case of a list result, return Element for element nodes, + string for text and attribute values. + + Note: if you are going to apply multiple XPath expressions + against the same document, it is more efficient to use + XPathEvaluator directly. + """ + self._assertHasRoot() + evaluator = XPathDocumentEvaluator(self, namespaces=namespaces, + extensions=extensions, + smart_strings=smart_strings) + return evaluator(_path, **_variables) + + def xslt(self, _xslt, extensions=None, access_control=None, **_kw): + """xslt(self, _xslt, extensions=None, access_control=None, **_kw) + + Transform this document using other document. + + xslt is a tree that should be XSLT + keyword parameters are XSLT transformation parameters. + + Returns the transformed tree. + + Note: if you are going to apply the same XSLT stylesheet against + multiple documents, it is more efficient to use the XSLT + class directly. + """ + self._assertHasRoot() + style = XSLT(_xslt, extensions=extensions, + access_control=access_control) + return style(self, **_kw) + + def relaxng(self, relaxng): + """relaxng(self, relaxng) + + Validate this document using other document. + + The relaxng argument is a tree that should contain a Relax NG schema. + + Returns True or False, depending on whether validation + succeeded. + + Note: if you are going to apply the same Relax NG schema against + multiple documents, it is more efficient to use the RelaxNG + class directly. + """ + self._assertHasRoot() + schema = RelaxNG(relaxng) + return schema.validate(self) + + def xmlschema(self, xmlschema): + """xmlschema(self, xmlschema) + + Validate this document using other document. + + The xmlschema argument is a tree that should contain an XML Schema. + + Returns True or False, depending on whether validation + succeeded. + + Note: If you are going to apply the same XML Schema against + multiple documents, it is more efficient to use the XMLSchema + class directly. + """ + self._assertHasRoot() + schema = XMLSchema(xmlschema) + return schema.validate(self) + + def xinclude(self): + """xinclude(self) + + Process the XInclude nodes in this document and include the + referenced XML fragments. + + There is support for loading files through the file system, HTTP and + FTP. + + Note that XInclude does not support custom resolvers in Python space + due to restrictions of libxml2 <= 2.6.29. + """ + self._assertHasRoot() + XInclude()(self._context_node) + + def write_c14n(self, file, *, bint exclusive=False, bint with_comments=True, + compression=0, inclusive_ns_prefixes=None): + """write_c14n(self, file, exclusive=False, with_comments=True, + compression=0, inclusive_ns_prefixes=None) + + C14N write of document. Always writes UTF-8. + + The ``compression`` option enables GZip compression level 1-9. + + The ``inclusive_ns_prefixes`` should be a list of namespace strings + (i.e. ['xs', 'xsi']) that will be promoted to the top-level element + during exclusive C14N serialisation. This parameter is ignored if + exclusive mode=False. + + If exclusive=True and no list is provided, a namespace will only be + rendered if it is used by the immediate parent or one of its attributes + and its prefix and values have not already been rendered by an ancestor + of the namespace node's parent element. + + NOTE: This method is deprecated as of lxml 4.4 and will be removed in a + future release. Use ``.write(f, method="c14n")`` instead. + """ + self._assertHasRoot() + _assertValidNode(self._context_node) + if compression is None or compression < 0: + compression = 0 + + _tofilelikeC14N(file, self._context_node, exclusive, with_comments, + compression, inclusive_ns_prefixes) + +cdef _ElementTree _elementTreeFactory(_Document doc, _Element context_node): + return _newElementTree(doc, context_node, _ElementTree) + +cdef _ElementTree _newElementTree(_Document doc, _Element context_node, + object baseclass): + cdef _ElementTree result + result = baseclass() + if context_node is None and doc is not None: + context_node = doc.getroot() + if context_node is None: + _assertValidDoc(doc) + result._doc = doc + else: + _assertValidNode(context_node) + result._context_node = context_node + return result + + +@cython.final +@cython.freelist(16) +cdef class _Attrib: + """A dict-like proxy for the ``Element.attrib`` property. + """ + cdef _Element _element + def __cinit__(self, _Element element not None): + _assertValidNode(element) + self._element = element + + # MANIPULATORS + def __setitem__(self, key, value): + _assertValidNode(self._element) + _setAttributeValue(self._element, key, value) + + def __delitem__(self, key): + _assertValidNode(self._element) + _delAttribute(self._element, key) + + def update(self, sequence_or_dict): + _assertValidNode(self._element) + if isinstance(sequence_or_dict, (dict, _Attrib)): + sequence_or_dict = sequence_or_dict.items() + for key, value in sequence_or_dict: + _setAttributeValue(self._element, key, value) + + def pop(self, key, *default): + if len(default) > 1: + raise TypeError, f"pop expected at most 2 arguments, got {len(default)+1}" + _assertValidNode(self._element) + result = _getAttributeValue(self._element, key, None) + if result is None: + if not default: + raise KeyError, key + result = default[0] + else: + _delAttribute(self._element, key) + return result + + def clear(self): + _assertValidNode(self._element) + c_attrs = self._element._c_node.properties + if c_attrs: + self._element._c_node.properties = NULL + tree.xmlFreePropList(c_attrs) + + # ACCESSORS + def __repr__(self): + _assertValidNode(self._element) + return repr(dict( _collectAttributes(self._element._c_node, 3) )) + + def __copy__(self): + _assertValidNode(self._element) + return dict(_collectAttributes(self._element._c_node, 3)) + + def __deepcopy__(self, memo): + _assertValidNode(self._element) + return dict(_collectAttributes(self._element._c_node, 3)) + + def __getitem__(self, key): + _assertValidNode(self._element) + result = _getAttributeValue(self._element, key, None) + if result is None: + raise KeyError, key + return result + + def __bool__(self): + _assertValidNode(self._element) + cdef xmlAttr* c_attr = self._element._c_node.properties + while c_attr is not NULL: + if c_attr.type == tree.XML_ATTRIBUTE_NODE: + return 1 + c_attr = c_attr.next + return 0 + + def __len__(self): + _assertValidNode(self._element) + cdef xmlAttr* c_attr = self._element._c_node.properties + cdef Py_ssize_t c = 0 + while c_attr is not NULL: + if c_attr.type == tree.XML_ATTRIBUTE_NODE: + c += 1 + c_attr = c_attr.next + return c + + def get(self, key, default=None): + _assertValidNode(self._element) + return _getAttributeValue(self._element, key, default) + + def keys(self): + _assertValidNode(self._element) + return _collectAttributes(self._element._c_node, 1) + + def __iter__(self): + _assertValidNode(self._element) + return iter(_collectAttributes(self._element._c_node, 1)) + + def iterkeys(self): + _assertValidNode(self._element) + return iter(_collectAttributes(self._element._c_node, 1)) + + def values(self): + _assertValidNode(self._element) + return _collectAttributes(self._element._c_node, 2) + + def itervalues(self): + _assertValidNode(self._element) + return iter(_collectAttributes(self._element._c_node, 2)) + + def items(self): + _assertValidNode(self._element) + return _collectAttributes(self._element._c_node, 3) + + def iteritems(self): + _assertValidNode(self._element) + return iter(_collectAttributes(self._element._c_node, 3)) + + def has_key(self, key): + _assertValidNode(self._element) + return key in self + + def __contains__(self, key): + _assertValidNode(self._element) + cdef xmlNode* c_node + ns, tag = _getNsTag(key) + c_node = self._element._c_node + c_href = NULL if ns is None else _xcstr(ns) + return 1 if tree.xmlHasNsProp(c_node, _xcstr(tag), c_href) else 0 + + def __richcmp__(self, other, int op): + try: + one = dict(self.items()) + if not isinstance(other, dict): + other = dict(other) + except (TypeError, ValueError): + return NotImplemented + return python.PyObject_RichCompare(one, other, op) + +MutableMapping.register(_Attrib) + + +@cython.final +@cython.internal +cdef class _AttribIterator: + """Attribute iterator - for internal use only! + """ + # XML attributes must not be removed while running! + cdef _Element _node + cdef xmlAttr* _c_attr + cdef int _keysvalues # 1 - keys, 2 - values, 3 - items (key, value) + def __iter__(self): + return self + + def __next__(self): + cdef xmlAttr* c_attr + if self._node is None: + raise StopIteration + c_attr = self._c_attr + while c_attr is not NULL and c_attr.type != tree.XML_ATTRIBUTE_NODE: + c_attr = c_attr.next + if c_attr is NULL: + self._node = None + raise StopIteration + + self._c_attr = c_attr.next + if self._keysvalues == 1: + return _namespacedName(c_attr) + elif self._keysvalues == 2: + return _attributeValue(self._node._c_node, c_attr) + else: + return (_namespacedName(c_attr), + _attributeValue(self._node._c_node, c_attr)) + +cdef object _attributeIteratorFactory(_Element element, int keysvalues): + cdef _AttribIterator attribs + if element._c_node.properties is NULL: + return ITER_EMPTY + attribs = _AttribIterator() + attribs._node = element + attribs._c_attr = element._c_node.properties + attribs._keysvalues = keysvalues + return attribs + + +cdef public class _ElementTagMatcher [ object LxmlElementTagMatcher, + type LxmlElementTagMatcherType ]: + """ + Dead but public. :) + """ + cdef object _pystrings + cdef int _node_type + cdef char* _href + cdef char* _name + cdef _initTagMatch(self, tag): + self._href = NULL + self._name = NULL + if tag is None: + self._node_type = 0 + elif tag is Comment: + self._node_type = tree.XML_COMMENT_NODE + elif tag is ProcessingInstruction: + self._node_type = tree.XML_PI_NODE + elif tag is Entity: + self._node_type = tree.XML_ENTITY_REF_NODE + elif tag is Element: + self._node_type = tree.XML_ELEMENT_NODE + else: + self._node_type = tree.XML_ELEMENT_NODE + self._pystrings = _getNsTag(tag) + if self._pystrings[0] is not None: + self._href = _cstr(self._pystrings[0]) + self._name = _cstr(self._pystrings[1]) + if self._name[0] == c'*' and self._name[1] == c'\0': + self._name = NULL + +cdef public class _ElementIterator(_ElementTagMatcher) [ + object LxmlElementIterator, type LxmlElementIteratorType ]: + """ + Dead but public. :) + """ + # we keep Python references here to control GC + cdef _Element _node + cdef _node_to_node_function _next_element + def __iter__(self): + return self + + cdef void _storeNext(self, _Element node): + cdef xmlNode* c_node + c_node = self._next_element(node._c_node) + while c_node is not NULL and \ + self._node_type != 0 and \ + (self._node_type != c_node.type or + not _tagMatches(c_node, self._href, self._name)): + c_node = self._next_element(c_node) + if c_node is NULL: + self._node = None + else: + # Python ref: + self._node = _elementFactory(node._doc, c_node) + + def __next__(self): + cdef xmlNode* c_node + cdef _Element current_node + if self._node is None: + raise StopIteration + # Python ref: + current_node = self._node + self._storeNext(current_node) + return current_node + +@cython.final +@cython.internal +cdef class _MultiTagMatcher: + """ + Match an xmlNode against a list of tags. + """ + cdef list _py_tags + cdef qname* _cached_tags + cdef size_t _tag_count + cdef size_t _cached_size + cdef _Document _cached_doc + cdef int _node_types + + def __cinit__(self, tags): + self._py_tags = [] + self.initTagMatch(tags) + + def __dealloc__(self): + self._clear() + + cdef bint rejectsAll(self) noexcept: + return not self._tag_count and not self._node_types + + cdef bint rejectsAllAttributes(self) noexcept: + return not self._tag_count + + cdef bint matchesType(self, int node_type) noexcept: + if node_type == tree.XML_ELEMENT_NODE and self._tag_count: + return True + return self._node_types & (1 << node_type) + + cdef void _clear(self) noexcept: + cdef size_t i, count + count = self._tag_count + self._tag_count = 0 + if self._cached_tags: + for i in range(count): + cpython.ref.Py_XDECREF(self._cached_tags[i].href) + python.lxml_free(self._cached_tags) + self._cached_tags = NULL + + cdef initTagMatch(self, tags): + self._cached_doc = None + del self._py_tags[:] + self._clear() + if tags is None or tags == (): + # no selection in tags argument => match anything + self._node_types = ( + 1 << tree.XML_COMMENT_NODE | + 1 << tree.XML_PI_NODE | + 1 << tree.XML_ENTITY_REF_NODE | + 1 << tree.XML_ELEMENT_NODE) + else: + self._node_types = 0 + self._storeTags(tags, set()) + + cdef _storeTags(self, tag, set seen): + if tag is Comment: + self._node_types |= 1 << tree.XML_COMMENT_NODE + elif tag is ProcessingInstruction: + self._node_types |= 1 << tree.XML_PI_NODE + elif tag is Entity: + self._node_types |= 1 << tree.XML_ENTITY_REF_NODE + elif tag is Element: + self._node_types |= 1 << tree.XML_ELEMENT_NODE + elif python._isString(tag): + if tag in seen: + return + seen.add(tag) + if tag in ('*', '{*}*'): + self._node_types |= 1 << tree.XML_ELEMENT_NODE + else: + href, name = _getNsTag(tag) + if name == b'*': + name = None + if href is None: + href = b'' # no namespace + elif href == b'*': + href = None # wildcard: any namespace, including none + self._py_tags.append((href, name)) + elif isinstance(tag, QName): + self._storeTags(tag.text, seen) + else: + # support a sequence of tags + for item in tag: + self._storeTags(item, seen) + + cdef inline int cacheTags(self, _Document doc, bint force_into_dict=False) except -1: + """ + Look up the tag names in the doc dict to enable string pointer comparisons. + """ + cdef size_t dict_size = tree.xmlDictSize(doc._c_doc.dict) + if doc is self._cached_doc and dict_size == self._cached_size: + # doc and dict didn't change => names already cached + return 0 + self._tag_count = 0 + if not self._py_tags: + self._cached_doc = doc + self._cached_size = dict_size + return 0 + if not self._cached_tags: + self._cached_tags = python.lxml_malloc(len(self._py_tags), sizeof(qname)) + if not self._cached_tags: + self._cached_doc = None + raise MemoryError() + self._tag_count = _mapTagsToQnameMatchArray( + doc._c_doc, self._py_tags, self._cached_tags, force_into_dict) + self._cached_doc = doc + self._cached_size = dict_size + return 0 + + cdef inline bint matches(self, xmlNode* c_node) noexcept: + cdef qname* c_qname + if self._node_types & (1 << c_node.type): + return True + elif c_node.type == tree.XML_ELEMENT_NODE: + for c_qname in self._cached_tags[:self._tag_count]: + if _tagMatchesExactly(c_node, c_qname): + return True + return False + + cdef inline bint matchesNsTag(self, const_xmlChar* c_href, + const_xmlChar* c_name) noexcept: + cdef qname* c_qname + if self._node_types & (1 << tree.XML_ELEMENT_NODE): + return True + for c_qname in self._cached_tags[:self._tag_count]: + if _nsTagMatchesExactly(c_href, c_name, c_qname): + return True + return False + + cdef inline bint matchesAttribute(self, xmlAttr* c_attr) noexcept: + """Attribute matches differ from Element matches in that they do + not care about node types. + """ + cdef qname* c_qname + for c_qname in self._cached_tags[:self._tag_count]: + if _tagMatchesExactly(c_attr, c_qname): + return True + return False + +cdef class _ElementMatchIterator: + cdef _Element _node + cdef _node_to_node_function _next_element + cdef _MultiTagMatcher _matcher + + @cython.final + cdef _initTagMatcher(self, tags): + self._matcher = _MultiTagMatcher.__new__(_MultiTagMatcher, tags) + + def __iter__(self): + return self + + @cython.final + cdef int _storeNext(self, _Element node) except -1: + self._matcher.cacheTags(node._doc) + c_node = self._next_element(node._c_node) + while c_node is not NULL and not self._matcher.matches(c_node): + c_node = self._next_element(c_node) + # store Python ref to next node to make sure it's kept alive + self._node = _elementFactory(node._doc, c_node) if c_node is not NULL else None + return 0 + + def __next__(self): + cdef _Element current_node = self._node + if current_node is None: + raise StopIteration + self._storeNext(current_node) + return current_node + +cdef class ElementChildIterator(_ElementMatchIterator): + """ElementChildIterator(self, node, tag=None, reversed=False) + Iterates over the children of an element. + """ + def __cinit__(self, _Element node not None, tag=None, *, bint reversed=False): + cdef xmlNode* c_node + _assertValidNode(node) + self._initTagMatcher(tag) + if reversed: + c_node = _findChildBackwards(node._c_node, 0) + self._next_element = _previousElement + else: + c_node = _findChildForwards(node._c_node, 0) + self._next_element = _nextElement + self._matcher.cacheTags(node._doc) + while c_node is not NULL and not self._matcher.matches(c_node): + c_node = self._next_element(c_node) + # store Python ref to next node to make sure it's kept alive + self._node = _elementFactory(node._doc, c_node) if c_node is not NULL else None + +cdef class SiblingsIterator(_ElementMatchIterator): + """SiblingsIterator(self, node, tag=None, preceding=False) + Iterates over the siblings of an element. + + You can pass the boolean keyword ``preceding`` to specify the direction. + """ + def __cinit__(self, _Element node not None, tag=None, *, bint preceding=False): + _assertValidNode(node) + self._initTagMatcher(tag) + if preceding: + self._next_element = _previousElement + else: + self._next_element = _nextElement + self._storeNext(node) + +cdef class AncestorsIterator(_ElementMatchIterator): + """AncestorsIterator(self, node, tag=None) + Iterates over the ancestors of an element (from parent to parent). + """ + def __cinit__(self, _Element node not None, tag=None): + _assertValidNode(node) + self._initTagMatcher(tag) + self._next_element = _parentElement + self._storeNext(node) + +cdef class ElementDepthFirstIterator: + """ElementDepthFirstIterator(self, node, tag=None, inclusive=True) + Iterates over an element and its sub-elements in document order (depth + first pre-order). + + Note that this also includes comments, entities and processing + instructions. To filter them out, check if the ``tag`` property + of the returned element is a string (i.e. not None and not a + factory function), or pass the ``Element`` factory for the ``tag`` + argument to receive only Elements. + + If the optional ``tag`` argument is not None, the iterator returns only + the elements that match the respective name and namespace. + + The optional boolean argument 'inclusive' defaults to True and can be set + to False to exclude the start element itself. + + Note that the behaviour of this iterator is completely undefined if the + tree it traverses is modified during iteration. + """ + # we keep Python references here to control GC + # keep the next Element after the one we return, and the (s)top node + cdef _Element _next_node + cdef _Element _top_node + cdef _MultiTagMatcher _matcher + def __cinit__(self, _Element node not None, tag=None, *, bint inclusive=True): + _assertValidNode(node) + self._top_node = node + self._next_node = node + self._matcher = _MultiTagMatcher.__new__(_MultiTagMatcher, tag) + self._matcher.cacheTags(node._doc) + if not inclusive or not self._matcher.matches(node._c_node): + # find start node (this cannot raise StopIteration, self._next_node != None) + next(self) + + def __iter__(self): + return self + + def __next__(self): + cdef xmlNode* c_node + cdef _Element current_node = self._next_node + if current_node is None: + raise StopIteration + c_node = current_node._c_node + self._matcher.cacheTags(current_node._doc) + if not self._matcher._tag_count: + # no tag name was found in the dict => not in document either + # try to match by node type + c_node = self._nextNodeAnyTag(c_node) + else: + c_node = self._nextNodeMatchTag(c_node) + if c_node is NULL: + self._next_node = None + else: + self._next_node = _elementFactory(current_node._doc, c_node) + return current_node + + @cython.final + cdef xmlNode* _nextNodeAnyTag(self, xmlNode* c_node) noexcept: + cdef int node_types = self._matcher._node_types + if not node_types: + return NULL + tree.BEGIN_FOR_EACH_ELEMENT_FROM(self._top_node._c_node, c_node, 0) + if node_types & (1 << c_node.type): + return c_node + tree.END_FOR_EACH_ELEMENT_FROM(c_node) + return NULL + + @cython.final + cdef xmlNode* _nextNodeMatchTag(self, xmlNode* c_node) noexcept: + tree.BEGIN_FOR_EACH_ELEMENT_FROM(self._top_node._c_node, c_node, 0) + if self._matcher.matches(c_node): + return c_node + tree.END_FOR_EACH_ELEMENT_FROM(c_node) + return NULL + + +cdef class ElementTextIterator: + """ElementTextIterator(self, element, tag=None, with_tail=True) + Iterates over the text content of a subtree. + + You can pass the ``tag`` keyword argument to restrict text content to a + specific tag name. + + You can set the ``with_tail`` keyword argument to ``False`` to skip over + tail text (e.g. if you know that it's only whitespace from pretty-printing). + """ + cdef object _events + cdef _Element _start_element + def __cinit__(self, _Element element not None, tag=None, *, bint with_tail=True): + _assertValidNode(element) + if with_tail: + events = ("start", "comment", "pi", "end") + else: + events = ("start",) + self._start_element = element + self._events = iterwalk(element, events=events, tag=tag) + + def __iter__(self): + return self + + def __next__(self): + cdef _Element element + result = None + while result is None: + event, element = next(self._events) # raises StopIteration + if event == "start": + result = element.text + elif element is not self._start_element: + result = element.tail + return result + + +cdef xmlNode* _createElement(xmlDoc* c_doc, object name_utf) except NULL: + cdef xmlNode* c_node + c_node = tree.xmlNewDocNode(c_doc, NULL, _xcstr(name_utf), NULL) + return c_node + +cdef xmlNode* _createComment(xmlDoc* c_doc, const_xmlChar* text) noexcept: + cdef xmlNode* c_node + c_node = tree.xmlNewDocComment(c_doc, text) + return c_node + +cdef xmlNode* _createPI(xmlDoc* c_doc, const_xmlChar* target, const_xmlChar* text) noexcept: + cdef xmlNode* c_node + c_node = tree.xmlNewDocPI(c_doc, target, text) + return c_node + +cdef xmlNode* _createEntity(xmlDoc* c_doc, const_xmlChar* name) noexcept: + cdef xmlNode* c_node + c_node = tree.xmlNewReference(c_doc, name) + return c_node + +# module-level API for ElementTree + +from abc import ABC + +class Element(ABC): + """Element(_tag, attrib=None, nsmap=None, **_extra) + + Element factory, as a class. + + An instance of this class is an object implementing the + Element interface. + + >>> element = Element("test") + >>> type(element) + + >>> isinstance(element, Element) + True + >>> issubclass(_Element, Element) + True + + Also look at the `_Element.makeelement()` and + `_BaseParser.makeelement()` methods, which provide a faster way to + create an Element within a specific document or parser context. + """ + def __new__(cls, _tag, attrib=None, nsmap=None, **_extra): + return _makeElement(_tag, NULL, None, None, None, None, + attrib, nsmap, _extra) + +# Register _Element as a virtual subclass of Element +Element.register(_Element) + + +def Comment(text=None): + """Comment(text=None) + + Comment element factory. This factory function creates a special element that will + be serialized as an XML comment. + """ + cdef _Document doc + cdef xmlNode* c_node + cdef xmlDoc* c_doc + + if text is None: + text = b'' + else: + text = _utf8(text) + if b'--' in text or text.endswith(b'-'): + raise ValueError("Comment may not contain '--' or end with '-'") + + c_doc = _newXMLDoc() + doc = _documentFactory(c_doc, None) + c_node = _createComment(c_doc, _xcstr(text)) + tree.xmlAddChild(c_doc, c_node) + return _elementFactory(doc, c_node) + + +def ProcessingInstruction(target, text=None): + """ProcessingInstruction(target, text=None) + + ProcessingInstruction element factory. This factory function creates a + special element that will be serialized as an XML processing instruction. + """ + cdef _Document doc + cdef xmlNode* c_node + cdef xmlDoc* c_doc + + target = _utf8(target) + _tagValidOrRaise(target) + if target.lower() == b'xml': + raise ValueError, f"Invalid PI name '{target}'" + + if text is None: + text = b'' + else: + text = _utf8(text) + if b'?>' in text: + raise ValueError, "PI text must not contain '?>'" + + c_doc = _newXMLDoc() + doc = _documentFactory(c_doc, None) + c_node = _createPI(c_doc, _xcstr(target), _xcstr(text)) + tree.xmlAddChild(c_doc, c_node) + return _elementFactory(doc, c_node) + +PI = ProcessingInstruction + + +cdef class CDATA: + """CDATA(data) + + CDATA factory. This factory creates an opaque data object that + can be used to set Element text. The usual way to use it is:: + + >>> el = Element('content') + >>> el.text = CDATA('a string') + + >>> print(el.text) + a string + >>> print(tostring(el, encoding="unicode")) + + """ + cdef bytes _utf8_data + def __cinit__(self, data): + self._utf8_data = _utf8(data) + + +def Entity(name): + """Entity(name) + + Entity factory. This factory function creates a special element + that will be serialized as an XML entity reference or character + reference. Note, however, that entities will not be automatically + declared in the document. A document that uses entity references + requires a DTD to define the entities. + """ + cdef _Document doc + cdef xmlNode* c_node + cdef xmlDoc* c_doc + name_utf = _utf8(name) + c_name = _xcstr(name_utf) + if c_name[0] == c'#': + if not _characterReferenceIsValid(c_name + 1): + raise ValueError, f"Invalid character reference: '{name}'" + elif not _xmlNameIsValid(c_name): + raise ValueError, f"Invalid entity reference: '{name}'" + c_doc = _newXMLDoc() + doc = _documentFactory(c_doc, None) + c_node = _createEntity(c_doc, c_name) + tree.xmlAddChild(c_doc, c_node) + return _elementFactory(doc, c_node) + + +def SubElement(_Element _parent not None, _tag, + attrib=None, nsmap=None, **_extra): + """SubElement(_parent, _tag, attrib=None, nsmap=None, **_extra) + + Subelement factory. This function creates an element instance, and + appends it to an existing element. + """ + return _makeSubElement(_parent, _tag, None, None, attrib, nsmap, _extra) + +from typing import Generic, TypeVar + +T = TypeVar("T") + +class ElementTree(ABC, Generic[T]): + def __new__(cls, _Element element=None, *, file=None, _BaseParser parser=None): + """ElementTree(element=None, file=None, parser=None) + + ElementTree wrapper class. + """ + cdef xmlNode* c_next + cdef xmlNode* c_node + cdef xmlNode* c_node_copy + cdef xmlDoc* c_doc + cdef _ElementTree etree + cdef _Document doc + + if element is not None: + doc = element._doc + elif file is not None: + try: + doc = _parseDocument(file, parser, None) + except _TargetParserResult as result_container: + return result_container.result + else: + c_doc = _newXMLDoc() + doc = _documentFactory(c_doc, parser) + + return _elementTreeFactory(doc, element) + +# Register _ElementTree as a virtual subclass of ElementTree +ElementTree.register(_ElementTree) + +# Remove "ABC" and typing helpers from module dict +del ABC, Generic, TypeVar, T + +def HTML(text, _BaseParser parser=None, *, base_url=None): + """HTML(text, parser=None, base_url=None) + + Parses an HTML document from a string constant. Returns the root + node (or the result returned by a parser target). This function + can be used to embed "HTML literals" in Python code. + + To override the parser with a different ``HTMLParser`` you can pass it to + the ``parser`` keyword argument. + + The ``base_url`` keyword argument allows to set the original base URL of + the document to support relative Paths when looking up external entities + (DTD, XInclude, ...). + """ + cdef _Document doc + if parser is None: + parser = __GLOBAL_PARSER_CONTEXT.getDefaultParser() + if not isinstance(parser, HTMLParser): + parser = __DEFAULT_HTML_PARSER + try: + doc = _parseMemoryDocument(text, base_url, parser) + return doc.getroot() + except _TargetParserResult as result_container: + return result_container.result + + +def XML(text, _BaseParser parser=None, *, base_url=None): + """XML(text, parser=None, base_url=None) + + Parses an XML document or fragment from a string constant. + Returns the root node (or the result returned by a parser target). + This function can be used to embed "XML literals" in Python code, + like in + + >>> root = XML("") + >>> print(root.tag) + root + + To override the parser with a different ``XMLParser`` you can pass it to + the ``parser`` keyword argument. + + The ``base_url`` keyword argument allows to set the original base URL of + the document to support relative Paths when looking up external entities + (DTD, XInclude, ...). + """ + cdef _Document doc + if parser is None: + parser = __GLOBAL_PARSER_CONTEXT.getDefaultParser() + if not isinstance(parser, XMLParser): + parser = __DEFAULT_XML_PARSER + try: + doc = _parseMemoryDocument(text, base_url, parser) + return doc.getroot() + except _TargetParserResult as result_container: + return result_container.result + + +def fromstring(text, _BaseParser parser=None, *, base_url=None): + """fromstring(text, parser=None, base_url=None) + + Parses an XML document or fragment from a string. Returns the + root node (or the result returned by a parser target). + + To override the default parser with a different parser you can pass it to + the ``parser`` keyword argument. + + The ``base_url`` keyword argument allows to set the original base URL of + the document to support relative Paths when looking up external entities + (DTD, XInclude, ...). + """ + cdef _Document doc + try: + doc = _parseMemoryDocument(text, base_url, parser) + return doc.getroot() + except _TargetParserResult as result_container: + return result_container.result + + +def fromstringlist(strings, _BaseParser parser=None): + """fromstringlist(strings, parser=None) + + Parses an XML document from a sequence of strings. Returns the + root node (or the result returned by a parser target). + + To override the default parser with a different parser you can pass it to + the ``parser`` keyword argument. + """ + cdef _Document doc + if isinstance(strings, (bytes, unicode)): + raise ValueError("passing a single string into fromstringlist() is not" + " efficient, use fromstring() instead") + if parser is None: + parser = __GLOBAL_PARSER_CONTEXT.getDefaultParser() + feed = parser.feed + for data in strings: + feed(data) + return parser.close() + + +def iselement(element): + """iselement(element) + + Checks if an object appears to be a valid element object. + """ + return isinstance(element, _Element) and (<_Element>element)._c_node is not NULL + + +def indent(tree, space=" ", *, Py_ssize_t level=0): + """indent(tree, space=" ", level=0) + + Indent an XML document by inserting newlines and indentation space + after elements. + + *tree* is the ElementTree or Element to modify. The (root) element + itself will not be changed, but the tail text of all elements in its + subtree will be adapted. + + *space* is the whitespace to insert for each indentation level, two + space characters by default. + + *level* is the initial indentation level. Setting this to a higher + value than 0 can be used for indenting subtrees that are more deeply + nested inside of a document. + """ + root = _rootNodeOrRaise(tree) + if level < 0: + raise ValueError(f"Initial indentation level must be >= 0, got {level}") + if _hasChild(root._c_node): + space = _utf8(space) + indent = b"\n" + level * space + _indent_children(root._c_node, 1, space, [indent, indent + space]) + + +cdef int _indent_children(xmlNode* c_node, Py_ssize_t level, bytes one_space, list indentations) except -1: + # Reuse indentation strings for speed. + if len(indentations) <= level: + indentations.append(indentations[-1] + one_space) + + # Start a new indentation level for the first child. + child_indentation = indentations[level] + if not _hasNonWhitespaceText(c_node): + _setNodeText(c_node, child_indentation) + + # Recursively indent all children. + cdef xmlNode* c_child = _findChildForwards(c_node, 0) + while c_child is not NULL: + if _hasChild(c_child): + _indent_children(c_child, level+1, one_space, indentations) + c_next_child = _nextElement(c_child) + if not _hasNonWhitespaceTail(c_child): + if c_next_child is NULL: + # Dedent after the last child. + child_indentation = indentations[level-1] + _setTailText(c_child, child_indentation) + c_child = c_next_child + return 0 + + +def dump(_Element elem not None, *, bint pretty_print=True, bint with_tail=True): + """dump(elem, pretty_print=True, with_tail=True) + + Writes an element tree or element structure to sys.stdout. This function + should be used for debugging only. + """ + xml = tostring(elem, pretty_print=pretty_print, with_tail=with_tail, encoding='unicode') + if not pretty_print: + xml += '\n' + sys.stdout.write(xml) + + +def tostring(element_or_tree, *, encoding=None, method="xml", + xml_declaration=None, bint pretty_print=False, bint with_tail=True, + standalone=None, doctype=None, + # method='c14n' + bint exclusive=False, inclusive_ns_prefixes=None, + # method='c14n2' + bint with_comments=True, bint strip_text=False, + ): + """tostring(element_or_tree, encoding=None, method="xml", + xml_declaration=None, pretty_print=False, with_tail=True, + standalone=None, doctype=None, + exclusive=False, inclusive_ns_prefixes=None, + with_comments=True, strip_text=False, + ) + + Serialize an element to an encoded string representation of its XML + tree. + + Defaults to ASCII encoding without XML declaration. This + behaviour can be configured with the keyword arguments 'encoding' + (string) and 'xml_declaration' (bool). Note that changing the + encoding to a non UTF-8 compatible encoding will enable a + declaration by default. + + You can also serialise to a Unicode string without declaration by + passing the name ``'unicode'`` as encoding (or the ``str`` function + in Py3 or ``unicode`` in Py2). This changes the return value from + a byte string to an unencoded unicode string. + + The keyword argument 'pretty_print' (bool) enables formatted XML. + + The keyword argument 'method' selects the output method: 'xml', + 'html', plain 'text' (text content without tags), 'c14n' or 'c14n2'. + Default is 'xml'. + + With ``method="c14n"`` (C14N version 1), the options ``exclusive``, + ``with_comments`` and ``inclusive_ns_prefixes`` request exclusive + C14N, include comments, and list the inclusive prefixes respectively. + + With ``method="c14n2"`` (C14N version 2), the ``with_comments`` and + ``strip_text`` options control the output of comments and text space + according to C14N 2.0. + + Passing a boolean value to the ``standalone`` option will output + an XML declaration with the corresponding ``standalone`` flag. + + The ``doctype`` option allows passing in a plain string that will + be serialised before the XML tree. Note that passing in non + well-formed content here will make the XML output non well-formed. + Also, an existing doctype in the document tree will not be removed + when serialising an ElementTree instance. + + You can prevent the tail text of the element from being serialised + by passing the boolean ``with_tail`` option. This has no impact + on the tail text of children, which will always be serialised. + """ + cdef bint write_declaration + cdef int is_standalone + # C14N serialisation + if method in ('c14n', 'c14n2'): + if encoding is not None: + raise ValueError("Cannot specify encoding with C14N") + if xml_declaration: + raise ValueError("Cannot enable XML declaration in C14N") + if method == 'c14n': + return _tostringC14N(element_or_tree, exclusive, with_comments, inclusive_ns_prefixes) + else: + out = BytesIO() + target = C14NWriterTarget( + utf8_writer(out).write, + with_comments=with_comments, strip_text=strip_text) + _tree_to_target(element_or_tree, target) + return out.getvalue() + if not with_comments: + raise ValueError("Can only discard comments in C14N serialisation") + if strip_text: + raise ValueError("Can only strip text in C14N 2.0 serialisation") + if encoding is unicode or (encoding is not None and encoding.lower() == 'unicode'): + if xml_declaration: + raise ValueError, \ + "Serialisation to unicode must not request an XML declaration" + write_declaration = 0 + encoding = unicode + elif xml_declaration is None: + # by default, write an XML declaration only for non-standard encodings + write_declaration = encoding is not None and encoding.upper() not in \ + ('ASCII', 'UTF-8', 'UTF8', 'US-ASCII') + else: + write_declaration = xml_declaration + if encoding is None: + encoding = 'ASCII' + if standalone is None: + is_standalone = -1 + elif standalone: + write_declaration = 1 + is_standalone = 1 + else: + write_declaration = 1 + is_standalone = 0 + + if isinstance(element_or_tree, _Element): + return _tostring(<_Element>element_or_tree, encoding, doctype, method, + write_declaration, 0, pretty_print, with_tail, + is_standalone) + elif isinstance(element_or_tree, _ElementTree): + return _tostring((<_ElementTree>element_or_tree)._context_node, + encoding, doctype, method, write_declaration, 1, + pretty_print, with_tail, is_standalone) + else: + raise TypeError, f"Type '{python._fqtypename(element_or_tree).decode('utf8')}' cannot be serialized." + + + +def tostringlist(element_or_tree, *args, **kwargs): + """tostringlist(element_or_tree, *args, **kwargs) + + Serialize an element to an encoded string representation of its XML + tree, stored in a list of partial strings. + + This is purely for ElementTree 1.3 compatibility. The result is a + single string wrapped in a list. + """ + return [tostring(element_or_tree, *args, **kwargs)] + + +def tounicode(element_or_tree, *, method="xml", bint pretty_print=False, + bint with_tail=True, doctype=None): + """tounicode(element_or_tree, method="xml", pretty_print=False, + with_tail=True, doctype=None) + + Serialize an element to the Python unicode representation of its XML + tree. + + :deprecated: use ``tostring(el, encoding='unicode')`` instead. + + Note that the result does not carry an XML encoding declaration and is + therefore not necessarily suited for serialization to byte streams without + further treatment. + + The boolean keyword argument 'pretty_print' enables formatted XML. + + The keyword argument 'method' selects the output method: 'xml', + 'html' or plain 'text'. + + You can prevent the tail text of the element from being serialised + by passing the boolean ``with_tail`` option. This has no impact + on the tail text of children, which will always be serialised. + """ + if isinstance(element_or_tree, _Element): + return _tostring(<_Element>element_or_tree, unicode, doctype, method, + 0, 0, pretty_print, with_tail, -1) + elif isinstance(element_or_tree, _ElementTree): + return _tostring((<_ElementTree>element_or_tree)._context_node, + unicode, doctype, method, 0, 1, pretty_print, + with_tail, -1) + else: + raise TypeError, f"Type '{type(element_or_tree)}' cannot be serialized." + + +def parse(source, _BaseParser parser=None, *, base_url=None): + """parse(source, parser=None, base_url=None) + + Return an ElementTree object loaded with source elements. If no parser + is provided as second argument, the default parser is used. + + The ``source`` can be any of the following: + + - a file name/path + - a file object + - a file-like object + - a URL using the HTTP or FTP protocol + + To parse from a string, use the ``fromstring()`` function instead. + + Note that it is generally faster to parse from a file path or URL + than from an open file object or file-like object. Transparent + decompression from gzip compressed sources is supported (unless + explicitly disabled in libxml2). + + The ``base_url`` keyword allows setting a URL for the document + when parsing from a file-like object. This is needed when looking + up external entities (DTD, XInclude, ...) with relative paths. + """ + cdef _Document doc + try: + doc = _parseDocument(source, parser, base_url) + return _elementTreeFactory(doc, None) + except _TargetParserResult as result_container: + return result_container.result + + +def adopt_external_document(capsule, _BaseParser parser=None): + """adopt_external_document(capsule, parser=None) + + Unpack a libxml2 document pointer from a PyCapsule and wrap it in an + lxml ElementTree object. + + This allows external libraries to build XML/HTML trees using libxml2 + and then pass them efficiently into lxml for further processing. + + If a ``parser`` is provided, it will be used for configuring the + lxml document. No parsing will be done. + + The capsule must have the name ``"libxml2:xmlDoc"`` and its pointer + value must reference a correct libxml2 document of type ``xmlDoc*``. + The creator of the capsule must take care to correctly clean up the + document using an appropriate capsule destructor. By default, the + libxml2 document will be copied to let lxml safely own the memory + of the internal tree that it uses. + + If the capsule context is non-NULL, it must point to a C string that + can be compared using ``strcmp()``. If the context string equals + ``"destructor:xmlFreeDoc"``, the libxml2 document will not be copied + but the capsule invalidated instead by clearing its destructor and + name. That way, lxml takes ownership of the libxml2 document in memory + without creating a copy first, and the capsule destructor will not be + called. The document will then eventually be cleaned up by lxml using + the libxml2 API function ``xmlFreeDoc()`` once it is no longer used. + + If no copy is made, later modifications of the tree outside of lxml + should not be attempted after transferring the ownership. + """ + cdef xmlDoc* c_doc + cdef bint is_owned = False + c_doc = python.lxml_unpack_xmldoc_capsule(capsule, &is_owned) + doc = _adoptForeignDoc(c_doc, parser, is_owned) + return _elementTreeFactory(doc, None) + + +################################################################################ +# Include submodules + +include "readonlytree.pxi" # Read-only implementation of Element proxies +include "classlookup.pxi" # Element class lookup mechanisms +include "nsclasses.pxi" # Namespace implementation and registry +include "docloader.pxi" # Support for custom document loaders +include "parser.pxi" # XML and HTML parsers +include "saxparser.pxi" # SAX-like Parser interface and tree builder +include "parsertarget.pxi" # ET Parser target +include "serializer.pxi" # XML output functions +include "iterparse.pxi" # incremental XML parsing +include "xmlid.pxi" # XMLID and IDDict +include "xinclude.pxi" # XInclude +include "cleanup.pxi" # Cleanup and recursive element removal functions + + +################################################################################ +# Include submodules for XPath and XSLT + +include "extensions.pxi" # XPath/XSLT extension functions +include "xpath.pxi" # XPath evaluation +include "xslt.pxi" # XSL transformations +include "xsltext.pxi" # XSL extension elements + + +################################################################################ +# Validation + +cdef class DocumentInvalid(LxmlError): + """Validation error. + + Raised by all document validators when their ``assertValid(tree)`` + method fails. + """ + + +cdef class _Validator: + "Base class for XML validators." + cdef _ErrorLog _error_log + def __cinit__(self): + self._error_log = _ErrorLog() + + def validate(self, etree): + """validate(self, etree) + + Validate the document using this schema. + + Returns true if document is valid, false if not. + """ + return self(etree) + + def assertValid(self, etree): + """assertValid(self, etree) + + Raises `DocumentInvalid` if the document does not comply with the schema. + """ + if not self(etree): + raise DocumentInvalid(self._error_log._buildExceptionMessage( + "Document does not comply with schema"), + self._error_log) + + def assert_(self, etree): + """assert_(self, etree) + + Raises `AssertionError` if the document does not comply with the schema. + """ + if not self(etree): + raise AssertionError, self._error_log._buildExceptionMessage( + "Document does not comply with schema") + + cpdef _append_log_message(self, int domain, int type, int level, int line, + message, filename): + self._error_log._receiveGeneric(domain, type, level, line, message, + filename) + + cpdef _clear_error_log(self): + self._error_log.clear() + + @property + def error_log(self): + """The log of validation errors and warnings.""" + assert self._error_log is not None, "XPath evaluator not initialised" + return self._error_log.copy() + +include "dtd.pxi" # DTD +include "relaxng.pxi" # RelaxNG +include "xmlschema.pxi" # XMLSchema +include "schematron.pxi" # Schematron (requires libxml2 2.6.21+) + +################################################################################ +# Public C API + +include "public-api.pxi" + +################################################################################ +# Other stuff + +include "debug.pxi" diff --git a/.venv/lib/python3.13/site-packages/lxml/etree_api.h b/.venv/lib/python3.13/site-packages/lxml/etree_api.h new file mode 100644 index 0000000000000000000000000000000000000000..702fef2ac4c9f0acb9adaeca88c71b8c4268fe0f --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/etree_api.h @@ -0,0 +1,214 @@ +/* Generated by Cython 3.2.4 */ + +#ifndef __PYX_HAVE_API__lxml__etree +#define __PYX_HAVE_API__lxml__etree +#ifdef __MINGW64__ +#define MS_WIN64 +#endif +#include "Python.h" +#include "etree.h" + +static struct LxmlElement *(*__pyx_api_f_4lxml_5etree_deepcopyNodeToDocument)(struct LxmlDocument *, xmlNode *) = 0; +#define deepcopyNodeToDocument __pyx_api_f_4lxml_5etree_deepcopyNodeToDocument +static struct LxmlElementTree *(*__pyx_api_f_4lxml_5etree_elementTreeFactory)(struct LxmlElement *) = 0; +#define elementTreeFactory __pyx_api_f_4lxml_5etree_elementTreeFactory +static struct LxmlElementTree *(*__pyx_api_f_4lxml_5etree_newElementTree)(struct LxmlElement *, PyObject *) = 0; +#define newElementTree __pyx_api_f_4lxml_5etree_newElementTree +static struct LxmlElementTree *(*__pyx_api_f_4lxml_5etree_adoptExternalDocument)(xmlDoc *, PyObject *, int) = 0; +#define adoptExternalDocument __pyx_api_f_4lxml_5etree_adoptExternalDocument +static struct LxmlElement *(*__pyx_api_f_4lxml_5etree_elementFactory)(struct LxmlDocument *, xmlNode *) = 0; +#define elementFactory __pyx_api_f_4lxml_5etree_elementFactory +static struct LxmlElement *(*__pyx_api_f_4lxml_5etree_makeElement)(PyObject *, struct LxmlDocument *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *) = 0; +#define makeElement __pyx_api_f_4lxml_5etree_makeElement +static struct LxmlElement *(*__pyx_api_f_4lxml_5etree_makeSubElement)(struct LxmlElement *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *) = 0; +#define makeSubElement __pyx_api_f_4lxml_5etree_makeSubElement +static void (*__pyx_api_f_4lxml_5etree_setElementClassLookupFunction)(_element_class_lookup_function, PyObject *) = 0; +#define setElementClassLookupFunction __pyx_api_f_4lxml_5etree_setElementClassLookupFunction +static PyObject *(*__pyx_api_f_4lxml_5etree_lookupDefaultElementClass)(PyObject *, PyObject *, xmlNode *) = 0; +#define lookupDefaultElementClass __pyx_api_f_4lxml_5etree_lookupDefaultElementClass +static PyObject *(*__pyx_api_f_4lxml_5etree_lookupNamespaceElementClass)(PyObject *, PyObject *, xmlNode *) = 0; +#define lookupNamespaceElementClass __pyx_api_f_4lxml_5etree_lookupNamespaceElementClass +static PyObject *(*__pyx_api_f_4lxml_5etree_callLookupFallback)(struct LxmlFallbackElementClassLookup *, struct LxmlDocument *, xmlNode *) = 0; +#define callLookupFallback __pyx_api_f_4lxml_5etree_callLookupFallback +static int (*__pyx_api_f_4lxml_5etree_tagMatches)(xmlNode *, const xmlChar *, const xmlChar *) = 0; +#define tagMatches __pyx_api_f_4lxml_5etree_tagMatches +static struct LxmlDocument *(*__pyx_api_f_4lxml_5etree_documentOrRaise)(PyObject *) = 0; +#define documentOrRaise __pyx_api_f_4lxml_5etree_documentOrRaise +static struct LxmlElement *(*__pyx_api_f_4lxml_5etree_rootNodeOrRaise)(PyObject *) = 0; +#define rootNodeOrRaise __pyx_api_f_4lxml_5etree_rootNodeOrRaise +static int (*__pyx_api_f_4lxml_5etree_hasText)(xmlNode *) = 0; +#define hasText __pyx_api_f_4lxml_5etree_hasText +static int (*__pyx_api_f_4lxml_5etree_hasTail)(xmlNode *) = 0; +#define hasTail __pyx_api_f_4lxml_5etree_hasTail +static PyObject *(*__pyx_api_f_4lxml_5etree_textOf)(xmlNode *) = 0; +#define textOf __pyx_api_f_4lxml_5etree_textOf +static PyObject *(*__pyx_api_f_4lxml_5etree_tailOf)(xmlNode *) = 0; +#define tailOf __pyx_api_f_4lxml_5etree_tailOf +static int (*__pyx_api_f_4lxml_5etree_setNodeText)(xmlNode *, PyObject *) = 0; +#define setNodeText __pyx_api_f_4lxml_5etree_setNodeText +static int (*__pyx_api_f_4lxml_5etree_setTailText)(xmlNode *, PyObject *) = 0; +#define setTailText __pyx_api_f_4lxml_5etree_setTailText +static PyObject *(*__pyx_api_f_4lxml_5etree_attributeValue)(xmlNode *, xmlAttr *) = 0; +#define attributeValue __pyx_api_f_4lxml_5etree_attributeValue +static PyObject *(*__pyx_api_f_4lxml_5etree_attributeValueFromNsName)(xmlNode *, const xmlChar *, const xmlChar *) = 0; +#define attributeValueFromNsName __pyx_api_f_4lxml_5etree_attributeValueFromNsName +static PyObject *(*__pyx_api_f_4lxml_5etree_getAttributeValue)(struct LxmlElement *, PyObject *, PyObject *) = 0; +#define getAttributeValue __pyx_api_f_4lxml_5etree_getAttributeValue +static PyObject *(*__pyx_api_f_4lxml_5etree_iterattributes)(struct LxmlElement *, int) = 0; +#define iterattributes __pyx_api_f_4lxml_5etree_iterattributes +static PyObject *(*__pyx_api_f_4lxml_5etree_collectAttributes)(xmlNode *, int) = 0; +#define collectAttributes __pyx_api_f_4lxml_5etree_collectAttributes +static int (*__pyx_api_f_4lxml_5etree_setAttributeValue)(struct LxmlElement *, PyObject *, PyObject *) = 0; +#define setAttributeValue __pyx_api_f_4lxml_5etree_setAttributeValue +static int (*__pyx_api_f_4lxml_5etree_delAttribute)(struct LxmlElement *, PyObject *) = 0; +#define delAttribute __pyx_api_f_4lxml_5etree_delAttribute +static int (*__pyx_api_f_4lxml_5etree_delAttributeFromNsName)(xmlNode *, const xmlChar *, const xmlChar *) = 0; +#define delAttributeFromNsName __pyx_api_f_4lxml_5etree_delAttributeFromNsName +static int (*__pyx_api_f_4lxml_5etree_hasChild)(xmlNode *) = 0; +#define hasChild __pyx_api_f_4lxml_5etree_hasChild +static xmlNode *(*__pyx_api_f_4lxml_5etree_findChild)(xmlNode *, Py_ssize_t) = 0; +#define findChild __pyx_api_f_4lxml_5etree_findChild +static xmlNode *(*__pyx_api_f_4lxml_5etree_findChildForwards)(xmlNode *, Py_ssize_t) = 0; +#define findChildForwards __pyx_api_f_4lxml_5etree_findChildForwards +static xmlNode *(*__pyx_api_f_4lxml_5etree_findChildBackwards)(xmlNode *, Py_ssize_t) = 0; +#define findChildBackwards __pyx_api_f_4lxml_5etree_findChildBackwards +static xmlNode *(*__pyx_api_f_4lxml_5etree_nextElement)(xmlNode *) = 0; +#define nextElement __pyx_api_f_4lxml_5etree_nextElement +static xmlNode *(*__pyx_api_f_4lxml_5etree_previousElement)(xmlNode *) = 0; +#define previousElement __pyx_api_f_4lxml_5etree_previousElement +static void (*__pyx_api_f_4lxml_5etree_appendChild)(struct LxmlElement *, struct LxmlElement *) = 0; +#define appendChild __pyx_api_f_4lxml_5etree_appendChild +static int (*__pyx_api_f_4lxml_5etree_appendChildToElement)(struct LxmlElement *, struct LxmlElement *) = 0; +#define appendChildToElement __pyx_api_f_4lxml_5etree_appendChildToElement +static PyObject *(*__pyx_api_f_4lxml_5etree_pyunicode)(const xmlChar *) = 0; +#define pyunicode __pyx_api_f_4lxml_5etree_pyunicode +static PyObject *(*__pyx_api_f_4lxml_5etree_utf8)(PyObject *) = 0; +#define utf8 __pyx_api_f_4lxml_5etree_utf8 +static PyObject *(*__pyx_api_f_4lxml_5etree_getNsTag)(PyObject *) = 0; +#define getNsTag __pyx_api_f_4lxml_5etree_getNsTag +static PyObject *(*__pyx_api_f_4lxml_5etree_getNsTagWithEmptyNs)(PyObject *) = 0; +#define getNsTagWithEmptyNs __pyx_api_f_4lxml_5etree_getNsTagWithEmptyNs +static PyObject *(*__pyx_api_f_4lxml_5etree_namespacedName)(xmlNode *) = 0; +#define namespacedName __pyx_api_f_4lxml_5etree_namespacedName +static PyObject *(*__pyx_api_f_4lxml_5etree_namespacedNameFromNsName)(const xmlChar *, const xmlChar *) = 0; +#define namespacedNameFromNsName __pyx_api_f_4lxml_5etree_namespacedNameFromNsName +static void (*__pyx_api_f_4lxml_5etree_iteratorStoreNext)(struct LxmlElementIterator *, struct LxmlElement *) = 0; +#define iteratorStoreNext __pyx_api_f_4lxml_5etree_iteratorStoreNext +static void (*__pyx_api_f_4lxml_5etree_initTagMatch)(struct LxmlElementTagMatcher *, PyObject *) = 0; +#define initTagMatch __pyx_api_f_4lxml_5etree_initTagMatch +static xmlNs *(*__pyx_api_f_4lxml_5etree_findOrBuildNodeNsPrefix)(struct LxmlDocument *, xmlNode *, const xmlChar *, const xmlChar *) = 0; +#define findOrBuildNodeNsPrefix __pyx_api_f_4lxml_5etree_findOrBuildNodeNsPrefix +static int __Pyx_ImportFunction_3_2_4(PyObject *module, const char *funcname, void (**f)(void), const char *sig); + +#ifndef __PYX_HAVE_RT_ImportFromPxd_3_2_4 +#define __PYX_HAVE_RT_ImportFromPxd_3_2_4 +static int __Pyx_ImportFromPxd_3_2_4(PyObject *module, const char *name, void **p, const char *sig, const char *what) { + PyObject *d = 0; + PyObject *cobj = 0; + d = PyObject_GetAttrString(module, "__pyx_capi__"); + if (!d) + goto bad; +#if (defined(Py_LIMITED_API) && Py_LIMITED_API >= 0x030d0000) || (!defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030d0000) + PyDict_GetItemStringRef(d, name, &cobj); +#else + cobj = PyDict_GetItemString(d, name); + Py_XINCREF(cobj); +#endif + if (!cobj) { + PyErr_Format(PyExc_ImportError, + "%.200s does not export expected C %.8s %.200s", + PyModule_GetName(module), what, name); + goto bad; + } + if (!PyCapsule_IsValid(cobj, sig)) { + PyErr_Format(PyExc_TypeError, + "C %.8s %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", + what, PyModule_GetName(module), name, sig, PyCapsule_GetName(cobj)); + goto bad; + } + *p = PyCapsule_GetPointer(cobj, sig); + if (!(*p)) + goto bad; + Py_DECREF(d); + Py_DECREF(cobj); + return 0; +bad: + Py_XDECREF(d); + Py_XDECREF(cobj); + return -1; +} +#endif + +#ifndef __PYX_HAVE_RT_ImportFunction_3_2_4 +#define __PYX_HAVE_RT_ImportFunction_3_2_4 +static int __Pyx_ImportFunction_3_2_4(PyObject *module, const char *funcname, void (**f)(void), const char *sig) { + union { + void (*fp)(void); + void *p; + } tmp; + int result = __Pyx_ImportFromPxd_3_2_4(module, funcname, &tmp.p, sig, "function"); + if (result == 0) { + *f = tmp.fp; + } + return result; +} +#endif + + +static int import_lxml__etree(void) { + PyObject *module = 0; + module = PyImport_ImportModule("lxml.etree"); + if (!module) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "deepcopyNodeToDocument", (void (**)(void))&__pyx_api_f_4lxml_5etree_deepcopyNodeToDocument, "struct LxmlElement *(struct LxmlDocument *, xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "elementTreeFactory", (void (**)(void))&__pyx_api_f_4lxml_5etree_elementTreeFactory, "struct LxmlElementTree *(struct LxmlElement *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "newElementTree", (void (**)(void))&__pyx_api_f_4lxml_5etree_newElementTree, "struct LxmlElementTree *(struct LxmlElement *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "adoptExternalDocument", (void (**)(void))&__pyx_api_f_4lxml_5etree_adoptExternalDocument, "struct LxmlElementTree *(xmlDoc *, PyObject *, int)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "elementFactory", (void (**)(void))&__pyx_api_f_4lxml_5etree_elementFactory, "struct LxmlElement *(struct LxmlDocument *, xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "makeElement", (void (**)(void))&__pyx_api_f_4lxml_5etree_makeElement, "struct LxmlElement *(PyObject *, struct LxmlDocument *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "makeSubElement", (void (**)(void))&__pyx_api_f_4lxml_5etree_makeSubElement, "struct LxmlElement *(struct LxmlElement *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "setElementClassLookupFunction", (void (**)(void))&__pyx_api_f_4lxml_5etree_setElementClassLookupFunction, "void (_element_class_lookup_function, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "lookupDefaultElementClass", (void (**)(void))&__pyx_api_f_4lxml_5etree_lookupDefaultElementClass, "PyObject *(PyObject *, PyObject *, xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "lookupNamespaceElementClass", (void (**)(void))&__pyx_api_f_4lxml_5etree_lookupNamespaceElementClass, "PyObject *(PyObject *, PyObject *, xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "callLookupFallback", (void (**)(void))&__pyx_api_f_4lxml_5etree_callLookupFallback, "PyObject *(struct LxmlFallbackElementClassLookup *, struct LxmlDocument *, xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "tagMatches", (void (**)(void))&__pyx_api_f_4lxml_5etree_tagMatches, "int (xmlNode *, const xmlChar *, const xmlChar *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "documentOrRaise", (void (**)(void))&__pyx_api_f_4lxml_5etree_documentOrRaise, "struct LxmlDocument *(PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "rootNodeOrRaise", (void (**)(void))&__pyx_api_f_4lxml_5etree_rootNodeOrRaise, "struct LxmlElement *(PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "hasText", (void (**)(void))&__pyx_api_f_4lxml_5etree_hasText, "int (xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "hasTail", (void (**)(void))&__pyx_api_f_4lxml_5etree_hasTail, "int (xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "textOf", (void (**)(void))&__pyx_api_f_4lxml_5etree_textOf, "PyObject *(xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "tailOf", (void (**)(void))&__pyx_api_f_4lxml_5etree_tailOf, "PyObject *(xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "setNodeText", (void (**)(void))&__pyx_api_f_4lxml_5etree_setNodeText, "int (xmlNode *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "setTailText", (void (**)(void))&__pyx_api_f_4lxml_5etree_setTailText, "int (xmlNode *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "attributeValue", (void (**)(void))&__pyx_api_f_4lxml_5etree_attributeValue, "PyObject *(xmlNode *, xmlAttr *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "attributeValueFromNsName", (void (**)(void))&__pyx_api_f_4lxml_5etree_attributeValueFromNsName, "PyObject *(xmlNode *, const xmlChar *, const xmlChar *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "getAttributeValue", (void (**)(void))&__pyx_api_f_4lxml_5etree_getAttributeValue, "PyObject *(struct LxmlElement *, PyObject *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "iterattributes", (void (**)(void))&__pyx_api_f_4lxml_5etree_iterattributes, "PyObject *(struct LxmlElement *, int)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "collectAttributes", (void (**)(void))&__pyx_api_f_4lxml_5etree_collectAttributes, "PyObject *(xmlNode *, int)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "setAttributeValue", (void (**)(void))&__pyx_api_f_4lxml_5etree_setAttributeValue, "int (struct LxmlElement *, PyObject *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "delAttribute", (void (**)(void))&__pyx_api_f_4lxml_5etree_delAttribute, "int (struct LxmlElement *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "delAttributeFromNsName", (void (**)(void))&__pyx_api_f_4lxml_5etree_delAttributeFromNsName, "int (xmlNode *, const xmlChar *, const xmlChar *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "hasChild", (void (**)(void))&__pyx_api_f_4lxml_5etree_hasChild, "int (xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "findChild", (void (**)(void))&__pyx_api_f_4lxml_5etree_findChild, "xmlNode *(xmlNode *, Py_ssize_t)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "findChildForwards", (void (**)(void))&__pyx_api_f_4lxml_5etree_findChildForwards, "xmlNode *(xmlNode *, Py_ssize_t)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "findChildBackwards", (void (**)(void))&__pyx_api_f_4lxml_5etree_findChildBackwards, "xmlNode *(xmlNode *, Py_ssize_t)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "nextElement", (void (**)(void))&__pyx_api_f_4lxml_5etree_nextElement, "xmlNode *(xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "previousElement", (void (**)(void))&__pyx_api_f_4lxml_5etree_previousElement, "xmlNode *(xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "appendChild", (void (**)(void))&__pyx_api_f_4lxml_5etree_appendChild, "void (struct LxmlElement *, struct LxmlElement *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "appendChildToElement", (void (**)(void))&__pyx_api_f_4lxml_5etree_appendChildToElement, "int (struct LxmlElement *, struct LxmlElement *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "pyunicode", (void (**)(void))&__pyx_api_f_4lxml_5etree_pyunicode, "PyObject *(const xmlChar *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "utf8", (void (**)(void))&__pyx_api_f_4lxml_5etree_utf8, "PyObject *(PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "getNsTag", (void (**)(void))&__pyx_api_f_4lxml_5etree_getNsTag, "PyObject *(PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "getNsTagWithEmptyNs", (void (**)(void))&__pyx_api_f_4lxml_5etree_getNsTagWithEmptyNs, "PyObject *(PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "namespacedName", (void (**)(void))&__pyx_api_f_4lxml_5etree_namespacedName, "PyObject *(xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "namespacedNameFromNsName", (void (**)(void))&__pyx_api_f_4lxml_5etree_namespacedNameFromNsName, "PyObject *(const xmlChar *, const xmlChar *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "iteratorStoreNext", (void (**)(void))&__pyx_api_f_4lxml_5etree_iteratorStoreNext, "void (struct LxmlElementIterator *, struct LxmlElement *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "initTagMatch", (void (**)(void))&__pyx_api_f_4lxml_5etree_initTagMatch, "void (struct LxmlElementTagMatcher *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction_3_2_4(module, "findOrBuildNodeNsPrefix", (void (**)(void))&__pyx_api_f_4lxml_5etree_findOrBuildNodeNsPrefix, "xmlNs *(struct LxmlDocument *, xmlNode *, const xmlChar *, const xmlChar *)") < 0) goto bad; + Py_DECREF(module); module = 0; + return 0; + bad: + Py_XDECREF(module); + return -1; +} + +#endif /* !__PYX_HAVE_API__lxml__etree */ diff --git a/.venv/lib/python3.13/site-packages/lxml/extensions.pxi b/.venv/lib/python3.13/site-packages/lxml/extensions.pxi new file mode 100644 index 0000000000000000000000000000000000000000..5945080c923a80f573ecf02a74b2706637efb09e --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/extensions.pxi @@ -0,0 +1,838 @@ +# support for extension functions in XPath and XSLT + +cdef class XPathError(LxmlError): + """Base class of all XPath errors. + """ + +cdef class XPathEvalError(XPathError): + """Error during XPath evaluation. + """ + +cdef class XPathFunctionError(XPathEvalError): + """Internal error looking up an XPath extension function. + """ + +cdef class XPathResultError(XPathEvalError): + """Error handling an XPath result. + """ + + +# forward declarations + +ctypedef int (*_register_function)(void* ctxt, name_utf, ns_uri_utf) +cdef class _ExsltRegExp + +################################################################################ +# Base class for XSLT and XPath evaluation contexts: functions, namespaces, ... + +@cython.internal +cdef class _BaseContext: + cdef xpath.xmlXPathContext* _xpathCtxt + cdef _Document _doc + cdef dict _extensions + cdef list _namespaces + cdef list _global_namespaces + cdef dict _utf_refs + cdef dict _function_cache + cdef dict _eval_context_dict + cdef bint _build_smart_strings + # for exception handling and temporary reference keeping: + cdef _TempStore _temp_refs + cdef set _temp_documents + cdef _ExceptionContext _exc + cdef _ErrorLog _error_log + + def __init__(self, namespaces, extensions, error_log, enable_regexp, + build_smart_strings): + cdef _ExsltRegExp _regexp + cdef dict new_extensions + cdef list ns + self._utf_refs = {} + self._global_namespaces = [] + self._function_cache = {} + self._eval_context_dict = None + self._error_log = error_log + + if extensions is not None: + # convert extensions to UTF-8 + if isinstance(extensions, dict): + extensions = (extensions,) + # format: [ {(ns, name):function} ] -> {(ns_utf, name_utf):function} + new_extensions = {} + for extension in extensions: + for (ns_uri, name), function in extension.items(): + if name is None: + raise ValueError, "extensions must have non empty names" + ns_utf = self._to_utf(ns_uri) + name_utf = self._to_utf(name) + new_extensions[(ns_utf, name_utf)] = function + extensions = new_extensions or None + + if namespaces is not None: + if isinstance(namespaces, dict): + namespaces = namespaces.items() + if namespaces: + ns = [] + for prefix, ns_uri in namespaces: + if prefix is None or not prefix: + raise TypeError, \ + "empty namespace prefix is not supported in XPath" + if ns_uri is None or not ns_uri: + raise TypeError, \ + "setting default namespace is not supported in XPath" + prefix_utf = self._to_utf(prefix) + ns_uri_utf = self._to_utf(ns_uri) + ns.append( (prefix_utf, ns_uri_utf) ) + namespaces = ns + else: + namespaces = None + + self._doc = None + self._exc = _ExceptionContext() + self._extensions = extensions + self._namespaces = namespaces + self._temp_refs = _TempStore() + self._temp_documents = set() + self._build_smart_strings = build_smart_strings + + if enable_regexp: + _regexp = _ExsltRegExp() + _regexp._register_in_context(self) + + cdef _BaseContext _copy(self): + cdef _BaseContext context + if self._namespaces is not None: + namespaces = self._namespaces[:] + else: + namespaces = None + context = self.__class__(namespaces, None, self._error_log, False, + self._build_smart_strings) + if self._extensions is not None: + context._extensions = self._extensions.copy() + return context + + cdef bytes _to_utf(self, s): + "Convert to UTF-8 and keep a reference to the encoded string" + cdef python.PyObject* dict_result + if s is None: + return None + dict_result = python.PyDict_GetItem(self._utf_refs, s) + if dict_result is not NULL: + return dict_result + utf = _utf8(s) + self._utf_refs[s] = utf + if python.IS_PYPY: + # use C level refs, PyPy refs are not enough! + python.Py_INCREF(utf) + return utf + + cdef void _set_xpath_context(self, xpath.xmlXPathContext* xpathCtxt) noexcept: + self._xpathCtxt = xpathCtxt + xpathCtxt.userData = self + # Need a cast here because older libxml2 releases do not use 'const' in the functype. + xpathCtxt.error = _receiveXPathError + + @cython.final + cdef _register_context(self, _Document doc): + self._doc = doc + self._exc.clear() + + @cython.final + cdef _cleanup_context(self): + #xpath.xmlXPathRegisteredNsCleanup(self._xpathCtxt) + #self.unregisterGlobalNamespaces() + if python.IS_PYPY: + # clean up double refs in PyPy (see "_to_utf()" method) + for ref in self._utf_refs.itervalues(): + python.Py_DECREF(ref) + self._utf_refs.clear() + self._eval_context_dict = None + self._doc = None + + @cython.final + cdef _release_context(self): + if self._xpathCtxt is not NULL: + self._xpathCtxt.userData = NULL + self._xpathCtxt = NULL + + # namespaces (internal UTF-8 methods with leading '_') + + cdef addNamespace(self, prefix, ns_uri): + cdef list namespaces + if prefix is None: + raise TypeError, "empty prefix is not supported in XPath" + prefix_utf = self._to_utf(prefix) + ns_uri_utf = self._to_utf(ns_uri) + new_item = (prefix_utf, ns_uri_utf) + if self._namespaces is None: + self._namespaces = [new_item] + else: + namespaces = [] + for item in self._namespaces: + if item[0] == prefix_utf: + item = new_item + new_item = None + namespaces.append(item) + if new_item is not None: + namespaces.append(new_item) + self._namespaces = namespaces + if self._xpathCtxt is not NULL: + xpath.xmlXPathRegisterNs( + self._xpathCtxt, _xcstr(prefix_utf), _xcstr(ns_uri_utf)) + + cdef registerNamespace(self, prefix, ns_uri): + if prefix is None: + raise TypeError, "empty prefix is not supported in XPath" + prefix_utf = self._to_utf(prefix) + ns_uri_utf = self._to_utf(ns_uri) + self._global_namespaces.append(prefix_utf) + xpath.xmlXPathRegisterNs(self._xpathCtxt, + _xcstr(prefix_utf), _xcstr(ns_uri_utf)) + + cdef registerLocalNamespaces(self): + if self._namespaces is None: + return + for prefix_utf, ns_uri_utf in self._namespaces: + xpath.xmlXPathRegisterNs( + self._xpathCtxt, _xcstr(prefix_utf), _xcstr(ns_uri_utf)) + + cdef registerGlobalNamespaces(self): + cdef list ns_prefixes = _find_all_extension_prefixes() + if python.PyList_GET_SIZE(ns_prefixes) > 0: + for prefix_utf, ns_uri_utf in ns_prefixes: + self._global_namespaces.append(prefix_utf) + xpath.xmlXPathRegisterNs( + self._xpathCtxt, _xcstr(prefix_utf), _xcstr(ns_uri_utf)) + + cdef unregisterGlobalNamespaces(self): + if python.PyList_GET_SIZE(self._global_namespaces) > 0: + for prefix_utf in self._global_namespaces: + xpath.xmlXPathRegisterNs(self._xpathCtxt, + _xcstr(prefix_utf), NULL) + del self._global_namespaces[:] + + cdef void _unregisterNamespace(self, prefix_utf) noexcept: + xpath.xmlXPathRegisterNs(self._xpathCtxt, + _xcstr(prefix_utf), NULL) + + # extension functions + + cdef int _addLocalExtensionFunction(self, ns_utf, name_utf, function) except -1: + if self._extensions is None: + self._extensions = {} + self._extensions[(ns_utf, name_utf)] = function + return 0 + + cdef registerGlobalFunctions(self, void* ctxt, + _register_function reg_func): + cdef python.PyObject* dict_result + cdef dict d + for ns_utf, ns_functions in __FUNCTION_NAMESPACE_REGISTRIES.iteritems(): + dict_result = python.PyDict_GetItem( + self._function_cache, ns_utf) + if dict_result is not NULL: + d = dict_result + else: + d = {} + self._function_cache[ns_utf] = d + for name_utf, function in ns_functions.iteritems(): + d[name_utf] = function + reg_func(ctxt, name_utf, ns_utf) + + cdef registerLocalFunctions(self, void* ctxt, + _register_function reg_func): + cdef python.PyObject* dict_result + cdef dict d + if self._extensions is None: + return # done + last_ns = None + d = None + for (ns_utf, name_utf), function in self._extensions.iteritems(): + if ns_utf is not last_ns or d is None: + last_ns = ns_utf + dict_result = python.PyDict_GetItem( + self._function_cache, ns_utf) + if dict_result is not NULL: + d = dict_result + else: + d = {} + self._function_cache[ns_utf] = d + d[name_utf] = function + reg_func(ctxt, name_utf, ns_utf) + + cdef unregisterAllFunctions(self, void* ctxt, + _register_function unreg_func): + for ns_utf, functions in self._function_cache.iteritems(): + for name_utf in functions: + unreg_func(ctxt, name_utf, ns_utf) + + cdef unregisterGlobalFunctions(self, void* ctxt, + _register_function unreg_func): + for ns_utf, functions in self._function_cache.items(): + for name_utf in functions: + if self._extensions is None or \ + (ns_utf, name_utf) not in self._extensions: + unreg_func(ctxt, name_utf, ns_utf) + + @cython.final + cdef _find_cached_function(self, const_xmlChar* c_ns_uri, const_xmlChar* c_name): + """Lookup an extension function in the cache and return it. + + Parameters: c_ns_uri may be NULL, c_name must not be NULL + """ + cdef python.PyObject* c_dict + cdef python.PyObject* dict_result + c_dict = python.PyDict_GetItem( + self._function_cache, None if c_ns_uri is NULL else c_ns_uri) + if c_dict is not NULL: + dict_result = python.PyDict_GetItem( + c_dict, c_name) + if dict_result is not NULL: + return dict_result + return None + + # Python access to the XPath context for extension functions + + @property + def context_node(self): + cdef xmlNode* c_node + if self._xpathCtxt is NULL: + raise XPathError, \ + "XPath context is only usable during the evaluation" + c_node = self._xpathCtxt.node + if c_node is NULL: + raise XPathError, "no context node" + if c_node.doc != self._xpathCtxt.doc: + raise XPathError, \ + "document-external context nodes are not supported" + if self._doc is None: + raise XPathError, "document context is missing" + return _elementFactory(self._doc, c_node) + + @property + def eval_context(self): + if self._eval_context_dict is None: + self._eval_context_dict = {} + return self._eval_context_dict + + # Python reference keeping during XPath function evaluation + + @cython.final + cdef _release_temp_refs(self): + "Free temporarily referenced objects from this context." + self._temp_refs.clear() + self._temp_documents.clear() + + @cython.final + cdef _hold(self, obj): + """A way to temporarily hold references to nodes in the evaluator. + + This is needed because otherwise nodes created in XPath extension + functions would be reference counted too soon, during the XPath + evaluation. This is most important in the case of exceptions. + """ + cdef _Element element + if isinstance(obj, _Element): + self._temp_refs.add(obj) + self._temp_documents.add((<_Element>obj)._doc) + return + elif _isString(obj) or not python.PySequence_Check(obj): + return + for o in obj: + if isinstance(o, _Element): + #print "Holding element:", element._c_node + self._temp_refs.add(o) + #print "Holding document:", element._doc._c_doc + self._temp_documents.add((<_Element>o)._doc) + + @cython.final + cdef _Document _findDocumentForNode(self, xmlNode* c_node): + """If an XPath expression returns an element from a different + document than the current context document, we call this to + see if it was possibly created by an extension and is a known + document instance. + """ + cdef _Document doc + for doc in self._temp_documents: + if doc is not None and doc._c_doc is c_node.doc: + return doc + return None + + +# libxml2 keeps these error messages in a static array in its code +# and doesn't give us access to them ... + +cdef tuple LIBXML2_XPATH_ERROR_MESSAGES = ( + b"Ok", + b"Number encoding", + b"Unfinished literal", + b"Start of literal", + b"Expected $ for variable reference", + b"Undefined variable", + b"Invalid predicate", + b"Invalid expression", + b"Missing closing curly brace", + b"Unregistered function", + b"Invalid operand", + b"Invalid type", + b"Invalid number of arguments", + b"Invalid context size", + b"Invalid context position", + b"Memory allocation error", + b"Syntax error", + b"Resource error", + b"Sub resource error", + b"Undefined namespace prefix", + b"Encoding error", + b"Char out of XML range", + b"Invalid or incomplete context", + b"Stack usage error", + b"Forbidden variable\n", + b"?? Unknown error ??\n", +) + +cdef void _forwardXPathError(void* c_ctxt, const xmlerror.xmlError* c_error) noexcept with gil: + cdef xmlerror.xmlError error + cdef int xpath_code + if c_error.message is not NULL: + error.message = c_error.message + else: + xpath_code = c_error.code - xmlerror.XML_XPATH_EXPRESSION_OK + if 0 <= xpath_code < len(LIBXML2_XPATH_ERROR_MESSAGES): + error.message = _cstr(LIBXML2_XPATH_ERROR_MESSAGES[xpath_code]) + else: + error.message = b"unknown error" + error.domain = c_error.domain + error.code = c_error.code + error.level = c_error.level + error.line = c_error.line + error.int2 = c_error.int1 # column + error.file = c_error.file + error.node = NULL + + (<_BaseContext>c_ctxt)._error_log._receive(&error) + +cdef void _receiveXPathError(void* c_context, const xmlerror.xmlError* error) noexcept nogil: + if not __DEBUG: + return + if c_context is NULL: + _forwardError(NULL, error) + else: + _forwardXPathError(c_context, error) + + +def Extension(module, function_mapping=None, *, ns=None): + """Extension(module, function_mapping=None, ns=None) + + Build a dictionary of extension functions from the functions + defined in a module or the methods of an object. + + As second argument, you can pass an additional mapping of + attribute names to XPath function names, or a list of function + names that should be taken. + + The ``ns`` keyword argument accepts a namespace URI for the XPath + functions. + """ + cdef dict functions = {} + if isinstance(function_mapping, dict): + for function_name, xpath_name in function_mapping.items(): + functions[(ns, xpath_name)] = getattr(module, function_name) + else: + if function_mapping is None: + function_mapping = [ name for name in dir(module) + if not name.startswith('_') ] + for function_name in function_mapping: + functions[(ns, function_name)] = getattr(module, function_name) + return functions + +################################################################################ +# EXSLT regexp implementation + +@cython.final +@cython.internal +cdef class _ExsltRegExp: + cdef dict _compile_map + def __cinit__(self): + self._compile_map = {} + + cdef _make_string(self, value): + if _isString(value): + return value + elif isinstance(value, list): + # node set: take recursive text concatenation of first element + if python.PyList_GET_SIZE(value) == 0: + return '' + firstnode = value[0] + if _isString(firstnode): + return firstnode + elif isinstance(firstnode, _Element): + c_text = tree.xmlNodeGetContent((<_Element>firstnode)._c_node) + if c_text is NULL: + raise MemoryError() + try: + return funicode(c_text) + finally: + tree.xmlFree(c_text) + else: + return unicode(firstnode) + else: + return unicode(value) + + cdef _compile(self, rexp, ignore_case): + cdef python.PyObject* c_result + rexp = self._make_string(rexp) + key = (rexp, ignore_case) + c_result = python.PyDict_GetItem(self._compile_map, key) + if c_result is not NULL: + return c_result + py_flags = re.UNICODE + if ignore_case: + py_flags = py_flags | re.IGNORECASE + rexp_compiled = re.compile(rexp, py_flags) + self._compile_map[key] = rexp_compiled + return rexp_compiled + + def test(self, ctxt, s, rexp, flags=''): + flags = self._make_string(flags) + s = self._make_string(s) + rexpc = self._compile(rexp, 'i' in flags) + if rexpc.search(s) is None: + return False + else: + return True + + def match(self, ctxt, s, rexp, flags=''): + cdef list result_list + flags = self._make_string(flags) + s = self._make_string(s) + rexpc = self._compile(rexp, 'i' in flags) + if 'g' in flags: + results = rexpc.findall(s) + if not results: + return () + else: + result = rexpc.search(s) + if not result: + return () + results = [ result.group() ] + results.extend( result.groups('') ) + result_list = [] + root = Element('matches') + for s_match in results: + if python.PyTuple_CheckExact(s_match): + s_match = ''.join(s_match) + elem = SubElement(root, 'match') + elem.text = s_match + result_list.append(elem) + return result_list + + def replace(self, ctxt, s, rexp, flags, replacement): + replacement = self._make_string(replacement) + flags = self._make_string(flags) + s = self._make_string(s) + rexpc = self._compile(rexp, 'i' in flags) + count: object = 0 if 'g' in flags else 1 + return rexpc.sub(replacement, s, count) + + cdef _register_in_context(self, _BaseContext context): + ns = b"http://exslt.org/regular-expressions" + context._addLocalExtensionFunction(ns, b"test", self.test) + context._addLocalExtensionFunction(ns, b"match", self.match) + context._addLocalExtensionFunction(ns, b"replace", self.replace) + + +################################################################################ +# helper functions + +cdef xpath.xmlXPathObject* _wrapXPathObject(object obj, _Document doc, + _BaseContext context) except NULL: + cdef xpath.xmlNodeSet* resultSet + cdef _Element fake_node = None + cdef xmlNode* c_node + + if isinstance(obj, unicode): + obj = _utf8(obj) + if isinstance(obj, bytes): + # libxml2 copies the string value + return xpath.xmlXPathNewCString(_cstr(obj)) + if isinstance(obj, bool): + return xpath.xmlXPathNewBoolean(obj) + if python.PyNumber_Check(obj): + return xpath.xmlXPathNewFloat(obj) + if obj is None: + resultSet = xpath.xmlXPathNodeSetCreate(NULL) + elif isinstance(obj, _Element): + resultSet = xpath.xmlXPathNodeSetCreate((<_Element>obj)._c_node) + elif python.PySequence_Check(obj): + resultSet = xpath.xmlXPathNodeSetCreate(NULL) + try: + for value in obj: + if isinstance(value, _Element): + if context is not None: + context._hold(value) + xpath.xmlXPathNodeSetAdd(resultSet, (<_Element>value)._c_node) + else: + if context is None or doc is None: + raise XPathResultError, \ + f"Non-Element values not supported at this point - got {value!r}" + # support strings by appending text nodes to an Element + if isinstance(value, unicode): + value = _utf8(value) + if isinstance(value, bytes): + if fake_node is None: + fake_node = _makeElement("text-root", NULL, doc, None, + None, None, None, None, None) + context._hold(fake_node) + else: + # append a comment node to keep the text nodes separate + c_node = tree.xmlNewDocComment(doc._c_doc, "") + if c_node is NULL: + raise MemoryError() + tree.xmlAddChild(fake_node._c_node, c_node) + context._hold(value) + c_node = tree.xmlNewDocText(doc._c_doc, _xcstr(value)) + if c_node is NULL: + raise MemoryError() + tree.xmlAddChild(fake_node._c_node, c_node) + xpath.xmlXPathNodeSetAdd(resultSet, c_node) + else: + raise XPathResultError, \ + f"This is not a supported node-set result: {value!r}" + except: + xpath.xmlXPathFreeNodeSet(resultSet) + raise + else: + raise XPathResultError, f"Unknown return type: {python._fqtypename(obj).decode('utf8')}" + return xpath.xmlXPathWrapNodeSet(resultSet) + +cdef object _unwrapXPathObject(xpath.xmlXPathObject* xpathObj, + _Document doc, _BaseContext context): + if xpathObj.type == xpath.XPATH_UNDEFINED: + raise XPathResultError, "Undefined xpath result" + elif xpathObj.type == xpath.XPATH_NODESET: + return _createNodeSetResult(xpathObj, doc, context) + elif xpathObj.type == xpath.XPATH_BOOLEAN: + return xpathObj.boolval + elif xpathObj.type == xpath.XPATH_NUMBER: + return xpathObj.floatval + elif xpathObj.type == xpath.XPATH_STRING: + stringval = funicode(xpathObj.stringval) + if context._build_smart_strings: + stringval = _elementStringResultFactory( + stringval, None, None, False) + return stringval + elif xpathObj.type == xpath.XPATH_POINT: + raise NotImplementedError, "XPATH_POINT" + elif xpathObj.type == xpath.XPATH_RANGE: + raise NotImplementedError, "XPATH_RANGE" + elif xpathObj.type == xpath.XPATH_LOCATIONSET: + raise NotImplementedError, "XPATH_LOCATIONSET" + elif xpathObj.type == xpath.XPATH_USERS: + raise NotImplementedError, "XPATH_USERS" + elif xpathObj.type == xpath.XPATH_XSLT_TREE: + return _createNodeSetResult(xpathObj, doc, context) + else: + raise XPathResultError, f"Unknown xpath result {xpathObj.type}" + +cdef object _createNodeSetResult(xpath.xmlXPathObject* xpathObj, _Document doc, + _BaseContext context): + cdef xmlNode* c_node + cdef int i + cdef list result + result = [] + if xpathObj.nodesetval is NULL: + return result + for i in range(xpathObj.nodesetval.nodeNr): + c_node = xpathObj.nodesetval.nodeTab[i] + _unpackNodeSetEntry(result, c_node, doc, context, + xpathObj.type == xpath.XPATH_XSLT_TREE) + return result + +cdef _unpackNodeSetEntry(list results, xmlNode* c_node, _Document doc, + _BaseContext context, bint is_fragment): + cdef xmlNode* c_child + if _isElement(c_node): + if c_node.doc != doc._c_doc and c_node.doc._private is NULL: + # XXX: works, but maybe not always the right thing to do? + # XPath: only runs when extensions create or copy trees + # -> we store Python refs to these, so that is OK + # XSLT: can it leak when merging trees from multiple sources? + c_node = tree.xmlDocCopyNode(c_node, doc._c_doc, 1) + if not c_node: + raise MemoryError() + # FIXME: call _instantiateElementFromXPath() instead? + results.append( + _fakeDocElementFactory(doc, c_node)) + elif c_node.type == tree.XML_TEXT_NODE or \ + c_node.type == tree.XML_CDATA_SECTION_NODE or \ + c_node.type == tree.XML_ATTRIBUTE_NODE: + results.append( + _buildElementStringResult(doc, c_node, context)) + elif c_node.type == tree.XML_NAMESPACE_DECL: + results.append( (funicodeOrNone((c_node).prefix), + funicodeOrNone((c_node).href)) ) + elif c_node.type == tree.XML_DOCUMENT_NODE or \ + c_node.type == tree.XML_HTML_DOCUMENT_NODE: + # ignored for everything but result tree fragments + if is_fragment: + c_child = c_node.children + while c_child is not NULL: + _unpackNodeSetEntry(results, c_child, doc, context, 0) + c_child = c_child.next + elif c_node.type == tree.XML_XINCLUDE_START or \ + c_node.type == tree.XML_XINCLUDE_END: + pass + else: + raise NotImplementedError, \ + f"Not yet implemented result node type: {c_node.type}" + +cdef void _freeXPathObject(xpath.xmlXPathObject* xpathObj) noexcept: + """Free the XPath object, but *never* free the *content* of node sets. + Python dealloc will do that for us. + """ + if xpathObj.nodesetval is not NULL: + xpath.xmlXPathFreeNodeSet(xpathObj.nodesetval) + xpathObj.nodesetval = NULL + xpath.xmlXPathFreeObject(xpathObj) + +cdef _Element _instantiateElementFromXPath(xmlNode* c_node, _Document doc, + _BaseContext context): + # NOTE: this may copy the element - only call this when it can't leak + if c_node.doc != doc._c_doc and c_node.doc._private is NULL: + # not from the context document and not from a fake document + # either => may still be from a known document, e.g. one + # created by an extension function + node_doc = context._findDocumentForNode(c_node) + if node_doc is None: + # not from a known document at all! => can only make a + # safety copy here + c_node = tree.xmlDocCopyNode(c_node, doc._c_doc, 1) + if not c_node: + raise MemoryError() + else: + doc = node_doc + return _fakeDocElementFactory(doc, c_node) + +################################################################################ +# special str/unicode subclasses + +@cython.final +cdef class _ElementUnicodeResult(unicode): + cdef _Element _parent + cdef readonly object attrname + cdef readonly bint is_tail + + def getparent(self): + return self._parent + + @property + def is_text(self): + return self._parent is not None and not (self.is_tail or self.attrname is not None) + + @property + def is_attribute(self): + return self.attrname is not None + +cdef object _elementStringResultFactory(string_value, _Element parent, + attrname, bint is_tail): + result = _ElementUnicodeResult(string_value) + result._parent = parent + result.is_tail = is_tail + result.attrname = attrname + return result + +cdef object _buildElementStringResult(_Document doc, xmlNode* c_node, + _BaseContext context): + cdef _Element parent = None + cdef object attrname = None + cdef xmlNode* c_element + cdef bint is_tail + + if c_node.type == tree.XML_ATTRIBUTE_NODE: + attrname = _namespacedName(c_node) + is_tail = 0 + s = tree.xmlNodeGetContent(c_node) + if s is NULL: + raise MemoryError() + try: + value = funicode(s) + finally: + tree.xmlFree(s) + c_element = NULL + else: + #assert c_node.type == tree.XML_TEXT_NODE or c_node.type == tree.XML_CDATA_SECTION_NODE, "invalid node type" + # may be tail text or normal text + value = funicode(c_node.content) + c_element = _previousElement(c_node) + is_tail = c_element is not NULL + + if not context._build_smart_strings: + return value + + if c_element is NULL: + # non-tail text or attribute text + c_element = c_node.parent + while c_element is not NULL and not _isElement(c_element): + c_element = c_element.parent + + if c_element is not NULL: + parent = _instantiateElementFromXPath(c_element, doc, context) + + return _elementStringResultFactory( + value, parent, attrname, is_tail) + +################################################################################ +# callbacks for XPath/XSLT extension functions + +cdef void _extension_function_call(_BaseContext context, function, + xpath.xmlXPathParserContext* ctxt, int nargs) noexcept: + cdef _Document doc + cdef xpath.xmlXPathObject* obj + cdef list args + cdef int i + doc = context._doc + try: + args = [] + for i in range(nargs): + obj = xpath.valuePop(ctxt) + try: + o = _unwrapXPathObject(obj, doc, context) + finally: + _freeXPathObject(obj) + args.append(o) + args.reverse() + + res = function(context, *args) + # wrap result for XPath consumption + obj = _wrapXPathObject(res, doc, context) + # prevent Python from deallocating elements handed to libxml2 + context._hold(res) + xpath.valuePush(ctxt, obj) + except: + xpath.xmlXPathErr(ctxt, xpath.XPATH_EXPR_ERROR) + context._exc._store_raised() + finally: + return # swallow any further exceptions + +# lookup the function by name and call it + +cdef void _xpath_function_call(xpath.xmlXPathParserContext* ctxt, + int nargs) noexcept with gil: + cdef _BaseContext context + cdef xpath.xmlXPathContext* rctxt = ctxt.context + context = <_BaseContext> rctxt.userData + try: + function = context._find_cached_function(rctxt.functionURI, rctxt.function) + if function is not None: + _extension_function_call(context, function, ctxt, nargs) + else: + xpath.xmlXPathErr(ctxt, xpath.XPATH_UNKNOWN_FUNC_ERROR) + context._exc._store_exception(XPathFunctionError( + f"XPath function '{_namespacedNameFromNsName(rctxt.functionURI, rctxt.function)}' not found")) + except: + # may not be the right error, but we need to tell libxml2 *something* + xpath.xmlXPathErr(ctxt, xpath.XPATH_UNKNOWN_FUNC_ERROR) + context._exc._store_raised() + finally: + return # swallow any further exceptions diff --git a/.venv/lib/python3.13/site-packages/lxml/html/ElementSoup.py b/.venv/lib/python3.13/site-packages/lxml/html/ElementSoup.py new file mode 100644 index 0000000000000000000000000000000000000000..c35365d0510605c08588775d4e5598586c8f8311 --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/html/ElementSoup.py @@ -0,0 +1,10 @@ +__doc__ = """Legacy interface to the BeautifulSoup HTML parser. +""" + +__all__ = ["parse", "convert_tree"] + +from .soupparser import convert_tree, parse as _parse + +def parse(file, beautifulsoup=None, makeelement=None): + root = _parse(file, beautifulsoup=beautifulsoup, makeelement=makeelement) + return root.getroot() diff --git a/.venv/lib/python3.13/site-packages/lxml/html/__init__.py b/.venv/lib/python3.13/site-packages/lxml/html/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2cee9f441d896b088bd873aa1ceef12abbb31dda --- /dev/null +++ b/.venv/lib/python3.13/site-packages/lxml/html/__init__.py @@ -0,0 +1,1927 @@ +# Copyright (c) 2004 Ian Bicking. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. Neither the name of Ian Bicking nor the names of its contributors may +# be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IAN BICKING OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""The ``lxml.html`` tool set for HTML handling. +""" + + +__all__ = [ + 'document_fromstring', 'fragment_fromstring', 'fragments_fromstring', 'fromstring', + 'tostring', 'Element', 'defs', 'open_in_browser', 'submit_form', + 'find_rel_links', 'find_class', 'make_links_absolute', + 'resolve_base_href', 'iterlinks', 'rewrite_links', 'parse'] + + +import copy +import re + +from collections.abc import MutableMapping, MutableSet +from functools import partial +from urllib.parse import urljoin + +from .. import etree +from . import defs +from ._setmixin import SetMixin + + +def __fix_docstring(s): + # TODO: remove and clean up doctests + if not s: + return s + sub = re.compile(r"^(\s*)u'", re.M).sub + return sub(r"\1'", s) + + +XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" + +_rel_links_xpath = etree.XPath("descendant-or-self::a[@rel]|descendant-or-self::x:a[@rel]", + namespaces={'x':XHTML_NAMESPACE}) +_options_xpath = etree.XPath("descendant-or-self::option|descendant-or-self::x:option", + namespaces={'x':XHTML_NAMESPACE}) +_forms_xpath = etree.XPath("descendant-or-self::form|descendant-or-self::x:form", + namespaces={'x':XHTML_NAMESPACE}) +#_class_xpath = etree.XPath(r"descendant-or-self::*[regexp:match(@class, concat('\b', $class_name, '\b'))]", {'regexp': 'http://exslt.org/regular-expressions'}) +_class_xpath = etree.XPath("descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), concat(' ', $class_name, ' '))]") +_id_xpath = etree.XPath("descendant-or-self::*[@id=$id]") +_collect_string_content = etree.XPath("string()", smart_strings=False) +_iter_css_urls = re.compile(r'url\(('+'["][^"]*["]|'+"['][^']*[']|"+r'[^)]*)\)', re.I).finditer +_iter_css_imports = re.compile(r'@import "(.*?)"').finditer +_label_xpath = etree.XPath("//label[@for=$id]|//x:label[@for=$id]", + namespaces={'x':XHTML_NAMESPACE}) +_archive_re = re.compile(r'[^ ]+') +_parse_meta_refresh_url = re.compile( + r'[^;=]*;\s*(?:url\s*=\s*)?(?P.*)$', re.I).search + + +def _unquote_match(s, pos): + if s[:1] == '"' and s[-1:] == '"' or s[:1] == "'" and s[-1:] == "'": + return s[1:-1], pos+1 + else: + return s,pos + + +def _transform_result(typ, result): + """Convert the result back into the input type. + """ + if issubclass(typ, bytes): + return tostring(result, encoding='utf-8') + elif issubclass(typ, str): + return tostring(result, encoding='unicode') + else: + return result + + +def _nons(tag): + if isinstance(tag, str): + if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE: + return tag.split('}')[-1] + return tag + + +class Classes(MutableSet): + """Provides access to an element's class attribute as a set-like collection. + Usage:: + + >>> el = fromstring('') + >>> classes = el.classes # or: classes = Classes(el.attrib) + >>> classes |= ['block', 'paragraph'] + >>> el.get('class') + 'hidden large block paragraph' + >>> classes.toggle('hidden') + False + >>> el.get('class') + 'large block paragraph' + >>> classes -= ('some', 'classes', 'block') + >>> el.get('class') + 'large paragraph' + """ + def __init__(self, attributes): + self._attributes = attributes + self._get_class_value = partial(attributes.get, 'class', '') + + def add(self, value): + """ + Add a class. + + This has no effect if the class is already present. + """ + if not value or re.search(r'\s', value): + raise ValueError("Invalid class name: %r" % value) + classes = self._get_class_value().split() + if value in classes: + return + classes.append(value) + self._attributes['class'] = ' '.join(classes) + + def discard(self, value): + """ + Remove a class if it is currently present. + + If the class is not present, do nothing. + """ + if not value or re.search(r'\s', value): + raise ValueError("Invalid class name: %r" % value) + classes = [name for name in self._get_class_value().split() + if name != value] + if classes: + self._attributes['class'] = ' '.join(classes) + elif 'class' in self._attributes: + del self._attributes['class'] + + def remove(self, value): + """ + Remove a class; it must currently be present. + + If the class is not present, raise a KeyError. + """ + if not value or re.search(r'\s', value): + raise ValueError("Invalid class name: %r" % value) + super().remove(value) + + def __contains__(self, name): + classes = self._get_class_value() + return name in classes and name in classes.split() + + def __iter__(self): + return iter(self._get_class_value().split()) + + def __len__(self): + return len(self._get_class_value().split()) + + # non-standard methods + + def update(self, values): + """ + Add all names from 'values'. + """ + classes = self._get_class_value().split() + extended = False + for value in values: + if value not in classes: + classes.append(value) + extended = True + if extended: + self._attributes['class'] = ' '.join(classes) + + def toggle(self, value): + """ + Add a class name if it isn't there yet, or remove it if it exists. + + Returns true if the class was added (and is now enabled) and + false if it was removed (and is now disabled). + """ + if not value or re.search(r'\s', value): + raise ValueError("Invalid class name: %r" % value) + classes = self._get_class_value().split() + try: + classes.remove(value) + enabled = False + except ValueError: + classes.append(value) + enabled = True + if classes: + self._attributes['class'] = ' '.join(classes) + else: + del self._attributes['class'] + return enabled + + +class HtmlMixin: + + def set(self, key, value=None): + """set(self, key, value=None) + + Sets an element attribute. If no value is provided, or if the value is None, + creates a 'boolean' attribute without value, e.g. "
" + for ``form.set('novalidate')``. + """ + super().set(key, value) + + @property + def classes(self): + """ + A set-like wrapper around the 'class' attribute. + """ + return Classes(self.attrib) + + @classes.setter + def classes(self, classes): + assert isinstance(classes, Classes) # only allow "el.classes |= ..." etc. + value = classes._get_class_value() + if value: + self.set('class', value) + elif self.get('class') is not None: + del self.attrib['class'] + + @property + def base_url(self): + """ + Returns the base URL, given when the page was parsed. + + Use with ``urlparse.urljoin(el.base_url, href)`` to get + absolute URLs. + """ + return self.getroottree().docinfo.URL + + @property + def forms(self): + """ + Return a list of all the forms + """ + return _forms_xpath(self) + + @property + def body(self): + """ + Return the element. Can be called from a child element + to get the document's head. + """ + for element in self.getroottree().iter("body", f"{{{XHTML_NAMESPACE}}}body"): + return element + return None + + @property + def head(self): + """ + Returns the element. Can be called from a child + element to get the document's head. + """ + for element in self.getroottree().iter("head", f"{{{XHTML_NAMESPACE}}}head"): + return element + return None + + @property + def label(self): + """ + Get or set any