Spaces:
Configuration error
Configuration error
Upload folder using huggingface_hub (part 8)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +10 -0
- .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/INSTALLER +1 -0
- .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/METADATA +373 -0
- .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/RECORD +380 -0
- .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/REQUESTED +0 -0
- .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/WHEEL +5 -0
- .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/entry_points.txt +7 -0
- .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/licenses/LICENSE +201 -0
- .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/top_level.txt +1 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/novita.py +69 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/nscale.py +44 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/openai.py +23 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/ovhcloud.py +10 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/publicai.py +6 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/replicate.py +160 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/scaleway.py +28 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/together.py +291 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/wavespeed.py +138 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/zai_org.py +127 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/lfs.py +442 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/py.typed +0 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/repocard.py +835 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/repocard_data.py +776 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/serialization/__init__.py +26 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/serialization/_base.py +212 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/serialization/_dduf.py +386 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/serialization/_torch.py +1046 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/templates/datasetcard_template.md +143 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/templates/modelcard_template.md +200 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/__init__.py +134 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_auth.py +456 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_assets.py +133 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_manager.py +901 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_chunk_utils.py +64 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_datetime.py +66 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_deprecation.py +136 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_detect_agent.py +204 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_dotenv.py +68 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_experimental.py +67 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_fixes.py +123 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_git_credential.py +119 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_headers.py +208 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_hf_uris.py +642 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_http.py +1163 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_lfs.py +109 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_oauth_device.py +197 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_pagination.py +52 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_parsing.py +116 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_paths.py +155 -0
- .venv/lib/python3.13/site-packages/huggingface_hub/utils/_runtime.py +445 -0
.gitattributes
CHANGED
|
@@ -52,3 +52,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 52 |
.venv/lib/python3.13/site-packages/ada92cb5d92a588d1b93__mypyc.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
| 53 |
.venv/lib/python3.13/site-packages/cryptography/hazmat/bindings/_rust.abi3.so filter=lfs diff=lfs merge=lfs -text
|
| 54 |
.venv/lib/python3.13/site-packages/hf_xet/hf_xet.abi3.so filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
.venv/lib/python3.13/site-packages/ada92cb5d92a588d1b93__mypyc.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
| 53 |
.venv/lib/python3.13/site-packages/cryptography/hazmat/bindings/_rust.abi3.so filter=lfs diff=lfs merge=lfs -text
|
| 54 |
.venv/lib/python3.13/site-packages/hf_xet/hf_xet.abi3.so filter=lfs diff=lfs merge=lfs -text
|
| 55 |
+
.venv/lib/python3.13/site-packages/lxml/_elementpath.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
| 56 |
+
.venv/lib/python3.13/site-packages/lxml/builder.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
| 57 |
+
.venv/lib/python3.13/site-packages/lxml/etree.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
| 58 |
+
.venv/lib/python3.13/site-packages/lxml/html/_difflib.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
| 59 |
+
.venv/lib/python3.13/site-packages/lxml/html/diff.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
| 60 |
+
.venv/lib/python3.13/site-packages/lxml/objectify.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
| 61 |
+
.venv/lib/python3.13/site-packages/lxml/sax.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
| 62 |
+
.venv/lib/python3.13/site-packages/numpy/_core/_multiarray_tests.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
| 63 |
+
.venv/lib/python3.13/site-packages/numpy/_core/_multiarray_umath.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
| 64 |
+
.venv/lib/python3.13/site-packages/numpy/_core/_simd.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
|
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/INSTALLER
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
pip
|
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/METADATA
ADDED
|
@@ -0,0 +1,373 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Metadata-Version: 2.4
|
| 2 |
+
Name: huggingface_hub
|
| 3 |
+
Version: 1.27.0
|
| 4 |
+
Summary: Client library to download and publish models, datasets and other repos on the huggingface.co hub
|
| 5 |
+
Home-page: https://github.com/huggingface/huggingface_hub
|
| 6 |
+
Author: Hugging Face, Inc.
|
| 7 |
+
Author-email: julien@huggingface.co
|
| 8 |
+
License: Apache-2.0
|
| 9 |
+
Keywords: model-hub machine-learning models natural-language-processing deep-learning pytorch pretrained-models
|
| 10 |
+
Classifier: Intended Audience :: Developers
|
| 11 |
+
Classifier: Intended Audience :: Education
|
| 12 |
+
Classifier: Intended Audience :: Science/Research
|
| 13 |
+
Classifier: License :: OSI Approved :: Apache Software License
|
| 14 |
+
Classifier: Operating System :: OS Independent
|
| 15 |
+
Classifier: Programming Language :: Python :: 3
|
| 16 |
+
Classifier: Programming Language :: Python :: 3 :: Only
|
| 17 |
+
Classifier: Programming Language :: Python :: 3.10
|
| 18 |
+
Classifier: Programming Language :: Python :: 3.11
|
| 19 |
+
Classifier: Programming Language :: Python :: 3.12
|
| 20 |
+
Classifier: Programming Language :: Python :: 3.13
|
| 21 |
+
Classifier: Programming Language :: Python :: 3.14
|
| 22 |
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
| 23 |
+
Requires-Python: >=3.10.0
|
| 24 |
+
Description-Content-Type: text/markdown
|
| 25 |
+
License-File: LICENSE
|
| 26 |
+
Requires-Dist: click<9.0.0,>=8.4.2
|
| 27 |
+
Requires-Dist: filelock>=3.10.0
|
| 28 |
+
Requires-Dist: fsspec>=2023.5.0
|
| 29 |
+
Requires-Dist: hf-xet<2.0.0,>=1.5.2; platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "arm64" or platform_machine == "aarch64"
|
| 30 |
+
Requires-Dist: httpx<1,>=0.23.0
|
| 31 |
+
Requires-Dist: packaging>=20.9
|
| 32 |
+
Requires-Dist: pyyaml>=5.1
|
| 33 |
+
Requires-Dist: tqdm>=4.42.1
|
| 34 |
+
Requires-Dist: typing-extensions>=4.1.0
|
| 35 |
+
Provides-Extra: oauth
|
| 36 |
+
Requires-Dist: authlib>=1.3.2; extra == "oauth"
|
| 37 |
+
Requires-Dist: fastapi; extra == "oauth"
|
| 38 |
+
Requires-Dist: httpx; extra == "oauth"
|
| 39 |
+
Requires-Dist: itsdangerous; extra == "oauth"
|
| 40 |
+
Provides-Extra: torch
|
| 41 |
+
Requires-Dist: torch; extra == "torch"
|
| 42 |
+
Requires-Dist: safetensors[torch]; extra == "torch"
|
| 43 |
+
Provides-Extra: fastai
|
| 44 |
+
Requires-Dist: toml; extra == "fastai"
|
| 45 |
+
Requires-Dist: fastai>=2.4; extra == "fastai"
|
| 46 |
+
Requires-Dist: fastcore>=1.3.27; extra == "fastai"
|
| 47 |
+
Provides-Extra: hf-xet
|
| 48 |
+
Requires-Dist: hf-xet<2.0.0,>=1.5.2; extra == "hf-xet"
|
| 49 |
+
Provides-Extra: mcp
|
| 50 |
+
Requires-Dist: mcp>=1.8.0; extra == "mcp"
|
| 51 |
+
Provides-Extra: testing
|
| 52 |
+
Requires-Dist: authlib>=1.3.2; extra == "testing"
|
| 53 |
+
Requires-Dist: fastapi; extra == "testing"
|
| 54 |
+
Requires-Dist: httpx; extra == "testing"
|
| 55 |
+
Requires-Dist: itsdangerous; extra == "testing"
|
| 56 |
+
Requires-Dist: jedi; extra == "testing"
|
| 57 |
+
Requires-Dist: Jinja2; extra == "testing"
|
| 58 |
+
Requires-Dist: pytest>=8.4.2; extra == "testing"
|
| 59 |
+
Requires-Dist: pytest-cov; extra == "testing"
|
| 60 |
+
Requires-Dist: pytest-env; extra == "testing"
|
| 61 |
+
Requires-Dist: pytest-xdist; extra == "testing"
|
| 62 |
+
Requires-Dist: pytest-vcr; extra == "testing"
|
| 63 |
+
Requires-Dist: pytest-asyncio; extra == "testing"
|
| 64 |
+
Requires-Dist: pytest-rerunfailures>=16.2; extra == "testing"
|
| 65 |
+
Requires-Dist: pytest-mock; extra == "testing"
|
| 66 |
+
Requires-Dist: urllib3<2.0; extra == "testing"
|
| 67 |
+
Requires-Dist: soundfile; extra == "testing"
|
| 68 |
+
Requires-Dist: Pillow; extra == "testing"
|
| 69 |
+
Requires-Dist: numpy; extra == "testing"
|
| 70 |
+
Requires-Dist: duckdb; extra == "testing"
|
| 71 |
+
Requires-Dist: fastapi; extra == "testing"
|
| 72 |
+
Provides-Extra: gradio
|
| 73 |
+
Requires-Dist: gradio>=5.0.0; extra == "gradio"
|
| 74 |
+
Requires-Dist: requests; extra == "gradio"
|
| 75 |
+
Provides-Extra: typing
|
| 76 |
+
Requires-Dist: typing-extensions>=4.8.0; extra == "typing"
|
| 77 |
+
Requires-Dist: types-PyYAML; extra == "typing"
|
| 78 |
+
Requires-Dist: types-simplejson; extra == "typing"
|
| 79 |
+
Requires-Dist: types-toml; extra == "typing"
|
| 80 |
+
Requires-Dist: types-tqdm; extra == "typing"
|
| 81 |
+
Requires-Dist: types-urllib3; extra == "typing"
|
| 82 |
+
Provides-Extra: quality
|
| 83 |
+
Requires-Dist: ruff>=0.9.0; extra == "quality"
|
| 84 |
+
Requires-Dist: mypy==1.15.0; extra == "quality"
|
| 85 |
+
Requires-Dist: libcst>=1.4.0; extra == "quality"
|
| 86 |
+
Requires-Dist: ty; extra == "quality"
|
| 87 |
+
Provides-Extra: all
|
| 88 |
+
Requires-Dist: authlib>=1.3.2; extra == "all"
|
| 89 |
+
Requires-Dist: fastapi; extra == "all"
|
| 90 |
+
Requires-Dist: httpx; extra == "all"
|
| 91 |
+
Requires-Dist: itsdangerous; extra == "all"
|
| 92 |
+
Requires-Dist: jedi; extra == "all"
|
| 93 |
+
Requires-Dist: Jinja2; extra == "all"
|
| 94 |
+
Requires-Dist: pytest>=8.4.2; extra == "all"
|
| 95 |
+
Requires-Dist: pytest-cov; extra == "all"
|
| 96 |
+
Requires-Dist: pytest-env; extra == "all"
|
| 97 |
+
Requires-Dist: pytest-xdist; extra == "all"
|
| 98 |
+
Requires-Dist: pytest-vcr; extra == "all"
|
| 99 |
+
Requires-Dist: pytest-asyncio; extra == "all"
|
| 100 |
+
Requires-Dist: pytest-rerunfailures>=16.2; extra == "all"
|
| 101 |
+
Requires-Dist: pytest-mock; extra == "all"
|
| 102 |
+
Requires-Dist: urllib3<2.0; extra == "all"
|
| 103 |
+
Requires-Dist: soundfile; extra == "all"
|
| 104 |
+
Requires-Dist: Pillow; extra == "all"
|
| 105 |
+
Requires-Dist: numpy; extra == "all"
|
| 106 |
+
Requires-Dist: duckdb; extra == "all"
|
| 107 |
+
Requires-Dist: fastapi; extra == "all"
|
| 108 |
+
Requires-Dist: ruff>=0.9.0; extra == "all"
|
| 109 |
+
Requires-Dist: mypy==1.15.0; extra == "all"
|
| 110 |
+
Requires-Dist: libcst>=1.4.0; extra == "all"
|
| 111 |
+
Requires-Dist: ty; extra == "all"
|
| 112 |
+
Requires-Dist: typing-extensions>=4.8.0; extra == "all"
|
| 113 |
+
Requires-Dist: types-PyYAML; extra == "all"
|
| 114 |
+
Requires-Dist: types-simplejson; extra == "all"
|
| 115 |
+
Requires-Dist: types-toml; extra == "all"
|
| 116 |
+
Requires-Dist: types-tqdm; extra == "all"
|
| 117 |
+
Requires-Dist: types-urllib3; extra == "all"
|
| 118 |
+
Provides-Extra: dev
|
| 119 |
+
Requires-Dist: authlib>=1.3.2; extra == "dev"
|
| 120 |
+
Requires-Dist: fastapi; extra == "dev"
|
| 121 |
+
Requires-Dist: httpx; extra == "dev"
|
| 122 |
+
Requires-Dist: itsdangerous; extra == "dev"
|
| 123 |
+
Requires-Dist: jedi; extra == "dev"
|
| 124 |
+
Requires-Dist: Jinja2; extra == "dev"
|
| 125 |
+
Requires-Dist: pytest>=8.4.2; extra == "dev"
|
| 126 |
+
Requires-Dist: pytest-cov; extra == "dev"
|
| 127 |
+
Requires-Dist: pytest-env; extra == "dev"
|
| 128 |
+
Requires-Dist: pytest-xdist; extra == "dev"
|
| 129 |
+
Requires-Dist: pytest-vcr; extra == "dev"
|
| 130 |
+
Requires-Dist: pytest-asyncio; extra == "dev"
|
| 131 |
+
Requires-Dist: pytest-rerunfailures>=16.2; extra == "dev"
|
| 132 |
+
Requires-Dist: pytest-mock; extra == "dev"
|
| 133 |
+
Requires-Dist: urllib3<2.0; extra == "dev"
|
| 134 |
+
Requires-Dist: soundfile; extra == "dev"
|
| 135 |
+
Requires-Dist: Pillow; extra == "dev"
|
| 136 |
+
Requires-Dist: numpy; extra == "dev"
|
| 137 |
+
Requires-Dist: duckdb; extra == "dev"
|
| 138 |
+
Requires-Dist: fastapi; extra == "dev"
|
| 139 |
+
Requires-Dist: ruff>=0.9.0; extra == "dev"
|
| 140 |
+
Requires-Dist: mypy==1.15.0; extra == "dev"
|
| 141 |
+
Requires-Dist: libcst>=1.4.0; extra == "dev"
|
| 142 |
+
Requires-Dist: ty; extra == "dev"
|
| 143 |
+
Requires-Dist: typing-extensions>=4.8.0; extra == "dev"
|
| 144 |
+
Requires-Dist: types-PyYAML; extra == "dev"
|
| 145 |
+
Requires-Dist: types-simplejson; extra == "dev"
|
| 146 |
+
Requires-Dist: types-toml; extra == "dev"
|
| 147 |
+
Requires-Dist: types-tqdm; extra == "dev"
|
| 148 |
+
Requires-Dist: types-urllib3; extra == "dev"
|
| 149 |
+
Dynamic: author
|
| 150 |
+
Dynamic: author-email
|
| 151 |
+
Dynamic: classifier
|
| 152 |
+
Dynamic: description
|
| 153 |
+
Dynamic: description-content-type
|
| 154 |
+
Dynamic: home-page
|
| 155 |
+
Dynamic: keywords
|
| 156 |
+
Dynamic: license
|
| 157 |
+
Dynamic: license-file
|
| 158 |
+
Dynamic: provides-extra
|
| 159 |
+
Dynamic: requires-dist
|
| 160 |
+
Dynamic: requires-python
|
| 161 |
+
Dynamic: summary
|
| 162 |
+
|
| 163 |
+
<p align="center">
|
| 164 |
+
<picture>
|
| 165 |
+
<source media="(prefers-color-scheme: dark)" srcset="https://huggingface.co/datasets/huggingface/documentation-images/raw/main/huggingface_hub-dark.svg">
|
| 166 |
+
<source media="(prefers-color-scheme: light)" srcset="https://huggingface.co/datasets/huggingface/documentation-images/raw/main/huggingface_hub.svg">
|
| 167 |
+
<img alt="huggingface_hub library logo" src="https://huggingface.co/datasets/huggingface/documentation-images/raw/main/huggingface_hub.svg" width="352" height="59" style="max-width: 100%">
|
| 168 |
+
</picture>
|
| 169 |
+
<br/>
|
| 170 |
+
<br/>
|
| 171 |
+
</p>
|
| 172 |
+
|
| 173 |
+
<p align="center">
|
| 174 |
+
<i>The official CLI and Python client for the Hugging Face Hub.</i>
|
| 175 |
+
<br/>
|
| 176 |
+
<a href="#what-is-huggingface_hub">About</a>
|
| 177 |
+
·
|
| 178 |
+
<a href="https://huggingface.co/docs/huggingface_hub">Documentation</a>
|
| 179 |
+
·
|
| 180 |
+
<a href="https://huggingface.co/docs/huggingface_hub/en/installation">Install</a>
|
| 181 |
+
·
|
| 182 |
+
<a href="https://huggingface.co/docs/huggingface_hub/en/guides/cli">CLI Guide</a>
|
| 183 |
+
·
|
| 184 |
+
<a href="https://github.com/huggingface/huggingface_hub/blob/main/CONTRIBUTING.md">Contributing</a>
|
| 185 |
+
</p>
|
| 186 |
+
|
| 187 |
+
<p align="center">
|
| 188 |
+
<a href="https://huggingface.co/docs/huggingface_hub/en/index"><img alt="Documentation" src="https://img.shields.io/website/http/huggingface.co/docs/huggingface_hub/index.svg?down_color=red&down_message=offline&up_message=online&label=doc"></a>
|
| 189 |
+
<a href="https://github.com/huggingface/huggingface_hub/releases"><img alt="GitHub release" src="https://img.shields.io/github/release/huggingface/huggingface_hub.svg"></a>
|
| 190 |
+
<a href="https://github.com/huggingface/huggingface_hub"><img alt="PyPi version" src="https://img.shields.io/pypi/pyversions/huggingface_hub.svg"></a>
|
| 191 |
+
<a href="https://pypi.org/project/huggingface-hub"><img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/huggingface_hub"></a>
|
| 192 |
+
<a href="https://codecov.io/gh/huggingface/huggingface_hub"><img alt="Code coverage" src="https://codecov.io/gh/huggingface/huggingface_hub/branch/main/graph/badge.svg?token=RXP95LE2XL"></a>
|
| 193 |
+
</p>
|
| 194 |
+
|
| 195 |
+
<h4 align="center">
|
| 196 |
+
<p>
|
| 197 |
+
<b>English</b> |
|
| 198 |
+
<a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_de.md">Deutsch</a> |
|
| 199 |
+
<a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_fr.md">Français</a> |
|
| 200 |
+
<a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_hi.md">हिंदी</a> |
|
| 201 |
+
<a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_ko.md">한국어</a> |
|
| 202 |
+
<a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_cn.md">中文 (简体)</a> |
|
| 203 |
+
<a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_kn.md">ಕನ್ನಡ</a>
|
| 204 |
+
</p>
|
| 205 |
+
</h4>
|
| 206 |
+
|
| 207 |
+
## Quick start
|
| 208 |
+
|
| 209 |
+
Install the [`hf` CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli) with the standalone installer:
|
| 210 |
+
|
| 211 |
+
```bash
|
| 212 |
+
# On macOS and Linux.
|
| 213 |
+
curl -LsSf https://hf.co/cli/install.sh | bash
|
| 214 |
+
```
|
| 215 |
+
|
| 216 |
+
```powershell
|
| 217 |
+
# On Windows.
|
| 218 |
+
powershell -ExecutionPolicy ByPass -c "irm https://hf.co/cli/install.ps1 | iex"
|
| 219 |
+
```
|
| 220 |
+
|
| 221 |
+
Log in, then start working with the Hub:
|
| 222 |
+
|
| 223 |
+
```bash
|
| 224 |
+
# Log in (use --token $HF_TOKEN in non-interactive environments)
|
| 225 |
+
hf auth login
|
| 226 |
+
|
| 227 |
+
# Find models served by Inference Providers
|
| 228 |
+
hf models ls --warm
|
| 229 |
+
|
| 230 |
+
# Download a model
|
| 231 |
+
hf download Qwen/Qwen3-0.6B
|
| 232 |
+
|
| 233 |
+
# Upload files to your own repo
|
| 234 |
+
hf upload username/my-cool-model ./model.safetensors
|
| 235 |
+
|
| 236 |
+
# Sync a local folder to a storage bucket
|
| 237 |
+
hf buckets sync ./checkpoints hf://buckets/username/my-bucket
|
| 238 |
+
|
| 239 |
+
# Run a job on Hugging Face infrastructure
|
| 240 |
+
hf jobs run python:3.12 python -c "print('Hello from the cloud!')"
|
| 241 |
+
|
| 242 |
+
# Discover everything else
|
| 243 |
+
hf --help
|
| 244 |
+
```
|
| 245 |
+
|
| 246 |
+
The Hub uses tokens to authenticate applications (see [docs](https://huggingface.co/docs/hub/security-tokens)). Check out the [CLI guide](https://huggingface.co/docs/huggingface_hub/en/guides/cli) for a tour of the main features.
|
| 247 |
+
|
| 248 |
+
## What is `huggingface_hub`?
|
| 249 |
+
|
| 250 |
+
The `huggingface_hub` library allows you to interact with the [Hugging Face Hub](https://huggingface.co/), a platform democratizing open-source Machine Learning for creators and collaborators. Discover pre-trained models and datasets for your projects, play with the thousands of machine learning apps hosted on the Hub, or create and share your own models, datasets and demos with the community. Everything ships in one package with two interfaces: the [`hf` CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli) for your terminal and the `huggingface_hub` library for Python — both designed to work well for humans and AI agents. Use them to:
|
| 251 |
+
|
| 252 |
+
- [Download files](https://huggingface.co/docs/huggingface_hub/en/guides/download) from the Hub.
|
| 253 |
+
- [Upload files](https://huggingface.co/docs/huggingface_hub/en/guides/upload) to the Hub.
|
| 254 |
+
- [Manage your repositories](https://huggingface.co/docs/huggingface_hub/en/guides/repository).
|
| 255 |
+
- [Run Inference](https://huggingface.co/docs/huggingface_hub/en/guides/inference) on deployed models.
|
| 256 |
+
- [Run Jobs](https://huggingface.co/docs/huggingface_hub/en/guides/jobs) on Hugging Face infrastructure.
|
| 257 |
+
- [Search](https://huggingface.co/docs/huggingface_hub/en/guides/search) for models, datasets and Spaces.
|
| 258 |
+
- [Share Model Cards](https://huggingface.co/docs/huggingface_hub/en/guides/model-cards) to document your models.
|
| 259 |
+
- [Engage with the community](https://huggingface.co/docs/huggingface_hub/en/guides/community) through PRs and comments.
|
| 260 |
+
- Do all of the above from the terminal with the [`hf` CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli).
|
| 261 |
+
|
| 262 |
+
## Built for humans and AI agents
|
| 263 |
+
|
| 264 |
+
The `hf` CLI is designed for people and coding agents alike: the same commands adapt their output when run by an agent. If you use Claude Code, Codex, Cursor, or another coding agent, install the `hf` CLI Skill — a command reference generated from your installed CLI:
|
| 265 |
+
|
| 266 |
+
```bash
|
| 267 |
+
# for Codex, Cursor, OpenCode, Pi and other agents that load skills from `.agents/skills`
|
| 268 |
+
hf skills add
|
| 269 |
+
# includes the above + Claude Code
|
| 270 |
+
hf skills add --claude
|
| 271 |
+
```
|
| 272 |
+
|
| 273 |
+
Learn more in the [Hugging Face CLI for AI agents guide](https://huggingface.co/docs/hub/agents-cli) and the [announcement blog post](https://huggingface.co/blog/hf-cli-for-agents).
|
| 274 |
+
|
| 275 |
+
## Use the Python library
|
| 276 |
+
|
| 277 |
+
Install the `huggingface_hub` package with [pip](https://pypi.org/project/huggingface-hub/) (this also installs the `hf` CLI):
|
| 278 |
+
|
| 279 |
+
```bash
|
| 280 |
+
pip install huggingface_hub
|
| 281 |
+
```
|
| 282 |
+
|
| 283 |
+
We recommend using [`uv`](https://docs.astral.sh/uv/) for a fast and reliable install:
|
| 284 |
+
|
| 285 |
+
```bash
|
| 286 |
+
uv pip install huggingface_hub
|
| 287 |
+
```
|
| 288 |
+
|
| 289 |
+
In order to keep the package minimal by default, `huggingface_hub` comes with optional dependencies useful for some use cases. For example, if you want to use the MCP module, run:
|
| 290 |
+
|
| 291 |
+
```bash
|
| 292 |
+
pip install "huggingface_hub[mcp]"
|
| 293 |
+
```
|
| 294 |
+
|
| 295 |
+
To learn more about installation and optional dependencies, check out the [installation guide](https://huggingface.co/docs/huggingface_hub/en/installation).
|
| 296 |
+
|
| 297 |
+
### Download files
|
| 298 |
+
|
| 299 |
+
Download a single file
|
| 300 |
+
|
| 301 |
+
```py
|
| 302 |
+
from huggingface_hub import hf_hub_download
|
| 303 |
+
|
| 304 |
+
hf_hub_download(repo_id="zai-org/GLM-5.2", filename="config.json")
|
| 305 |
+
```
|
| 306 |
+
|
| 307 |
+
Or an entire repository
|
| 308 |
+
|
| 309 |
+
```py
|
| 310 |
+
from huggingface_hub import snapshot_download
|
| 311 |
+
|
| 312 |
+
snapshot_download("sentence-transformers/all-MiniLM-L6-v2")
|
| 313 |
+
```
|
| 314 |
+
|
| 315 |
+
Files will be downloaded in a local cache folder. More details in [this guide](https://huggingface.co/docs/huggingface_hub/en/guides/manage-cache).
|
| 316 |
+
|
| 317 |
+
### Create a repository
|
| 318 |
+
|
| 319 |
+
```py
|
| 320 |
+
from huggingface_hub import create_repo
|
| 321 |
+
|
| 322 |
+
create_repo(repo_id="super-cool-model")
|
| 323 |
+
```
|
| 324 |
+
|
| 325 |
+
### Upload files
|
| 326 |
+
|
| 327 |
+
Upload a single file
|
| 328 |
+
|
| 329 |
+
```py
|
| 330 |
+
from huggingface_hub import upload_file
|
| 331 |
+
|
| 332 |
+
upload_file(
|
| 333 |
+
path_or_fileobj="/home/lysandre/dummy-test/README.md",
|
| 334 |
+
path_in_repo="README.md",
|
| 335 |
+
repo_id="lysandre/test-model",
|
| 336 |
+
)
|
| 337 |
+
```
|
| 338 |
+
|
| 339 |
+
Or an entire folder
|
| 340 |
+
|
| 341 |
+
```py
|
| 342 |
+
from huggingface_hub import upload_folder
|
| 343 |
+
|
| 344 |
+
upload_folder(
|
| 345 |
+
folder_path="/path/to/local/space",
|
| 346 |
+
repo_id="username/my-cool-space",
|
| 347 |
+
repo_type="space",
|
| 348 |
+
)
|
| 349 |
+
```
|
| 350 |
+
|
| 351 |
+
More details in the [upload guide](https://huggingface.co/docs/huggingface_hub/en/guides/upload).
|
| 352 |
+
|
| 353 |
+
## Integrating with the Hub.
|
| 354 |
+
|
| 355 |
+
We're partnering with cool open source ML libraries to provide free model hosting and versioning. You can find the existing integrations [here](https://huggingface.co/docs/hub/libraries).
|
| 356 |
+
|
| 357 |
+
The advantages are:
|
| 358 |
+
|
| 359 |
+
- Free model or dataset hosting for libraries and their users.
|
| 360 |
+
- Built-in file versioning, even with very large files, made possible by [Xet](https://huggingface.co/docs/hub/xet/index), the Hub's chunk-deduplicated storage backend.
|
| 361 |
+
- In-browser widgets to play with the uploaded models.
|
| 362 |
+
- Anyone can upload a new model for your library, they just need to add the corresponding tag for the model to be discoverable.
|
| 363 |
+
- Fast downloads! We use Cloudfront (a CDN) to geo-replicate downloads so they're blazing fast from anywhere on the globe.
|
| 364 |
+
- Usage stats and more features to come.
|
| 365 |
+
|
| 366 |
+
If you would like to integrate your library, feel free to open an issue to begin the discussion. We wrote a [step-by-step guide](https://huggingface.co/docs/hub/adding-a-library) with ❤️ showing how to do this integration.
|
| 367 |
+
|
| 368 |
+
## Contributions (feature requests, bugs, etc.) are super welcome 💙💚💛💜🧡❤️
|
| 369 |
+
|
| 370 |
+
Everyone is welcome to contribute, and we value everybody's contribution. Code is not the only way to help the community.
|
| 371 |
+
Answering questions, helping others, reaching out and improving the documentations are immensely valuable to the community.
|
| 372 |
+
We wrote a [contribution guide](https://github.com/huggingface/huggingface_hub/blob/main/CONTRIBUTING.md) to summarize
|
| 373 |
+
how to get started to contribute to this repository.
|
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/RECORD
ADDED
|
@@ -0,0 +1,380 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
../../../bin/hf,sha256=JQijLHciXBpOWGr6WRb8N0z29GfP0tPh2NA2HNavJvo,273
|
| 2 |
+
../../../bin/huggingface-cli,sha256=nNpWnVatSJHRXWFCuXQb_WZFtQ_HulgkcuNiCSf64xo,285
|
| 3 |
+
../../../bin/tiny-agents,sha256=0CQNfsvoFKWfo0ngJRPM94Fb2eOQaPFL-J1-Oblf3iA,283
|
| 4 |
+
huggingface_hub-1.27.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
|
| 5 |
+
huggingface_hub-1.27.0.dist-info/METADATA,sha256=kdWT-K9bIzIoJpph_rdX1f5gcwxRWXkZ5PTijS2m2co,16314
|
| 6 |
+
huggingface_hub-1.27.0.dist-info/RECORD,,
|
| 7 |
+
huggingface_hub-1.27.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
| 8 |
+
huggingface_hub-1.27.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
| 9 |
+
huggingface_hub-1.27.0.dist-info/entry_points.txt,sha256=zP7F_bBSdircPQFysHQZ9F3Lcn5_dCSOEZxVlGCsG0w,212
|
| 10 |
+
huggingface_hub-1.27.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
| 11 |
+
huggingface_hub-1.27.0.dist-info/top_level.txt,sha256=8KzlQJAY4miUvjAssOAJodqKOw3harNzuiwGQ9qLSSk,16
|
| 12 |
+
huggingface_hub/__init__.py,sha256=eZLm_W9SknOwodC709pz-SVyWudPEfO0-yrtIlL1FMc,60623
|
| 13 |
+
huggingface_hub/__pycache__/__init__.cpython-313.pyc,,
|
| 14 |
+
huggingface_hub/__pycache__/_buckets.cpython-313.pyc,,
|
| 15 |
+
huggingface_hub/__pycache__/_commit_api.cpython-313.pyc,,
|
| 16 |
+
huggingface_hub/__pycache__/_commit_scheduler.cpython-313.pyc,,
|
| 17 |
+
huggingface_hub/__pycache__/_dataset_viewer.cpython-313.pyc,,
|
| 18 |
+
huggingface_hub/__pycache__/_eval_results.cpython-313.pyc,,
|
| 19 |
+
huggingface_hub/__pycache__/_inference_endpoints.cpython-313.pyc,,
|
| 20 |
+
huggingface_hub/__pycache__/_jobs_api.cpython-313.pyc,,
|
| 21 |
+
huggingface_hub/__pycache__/_local_folder.cpython-313.pyc,,
|
| 22 |
+
huggingface_hub/__pycache__/_login.cpython-313.pyc,,
|
| 23 |
+
huggingface_hub/__pycache__/_oauth.cpython-313.pyc,,
|
| 24 |
+
huggingface_hub/__pycache__/_oidc.cpython-313.pyc,,
|
| 25 |
+
huggingface_hub/__pycache__/_revision.cpython-313.pyc,,
|
| 26 |
+
huggingface_hub/__pycache__/_sandbox.cpython-313.pyc,,
|
| 27 |
+
huggingface_hub/__pycache__/_sandbox_cache.cpython-313.pyc,,
|
| 28 |
+
huggingface_hub/__pycache__/_snapshot_download.cpython-313.pyc,,
|
| 29 |
+
huggingface_hub/__pycache__/_space_api.cpython-313.pyc,,
|
| 30 |
+
huggingface_hub/__pycache__/_tensorboard_logger.cpython-313.pyc,,
|
| 31 |
+
huggingface_hub/__pycache__/_tree_cache.cpython-313.pyc,,
|
| 32 |
+
huggingface_hub/__pycache__/_upload_large_folder.cpython-313.pyc,,
|
| 33 |
+
huggingface_hub/__pycache__/_upload_pipeline.cpython-313.pyc,,
|
| 34 |
+
huggingface_hub/__pycache__/_webhooks_payload.cpython-313.pyc,,
|
| 35 |
+
huggingface_hub/__pycache__/_webhooks_server.cpython-313.pyc,,
|
| 36 |
+
huggingface_hub/__pycache__/community.cpython-313.pyc,,
|
| 37 |
+
huggingface_hub/__pycache__/constants.cpython-313.pyc,,
|
| 38 |
+
huggingface_hub/__pycache__/dataclasses.cpython-313.pyc,,
|
| 39 |
+
huggingface_hub/__pycache__/errors.cpython-313.pyc,,
|
| 40 |
+
huggingface_hub/__pycache__/fastai_utils.cpython-313.pyc,,
|
| 41 |
+
huggingface_hub/__pycache__/file_download.cpython-313.pyc,,
|
| 42 |
+
huggingface_hub/__pycache__/hf_api.cpython-313.pyc,,
|
| 43 |
+
huggingface_hub/__pycache__/hf_file_system.cpython-313.pyc,,
|
| 44 |
+
huggingface_hub/__pycache__/hub_mixin.cpython-313.pyc,,
|
| 45 |
+
huggingface_hub/__pycache__/lfs.cpython-313.pyc,,
|
| 46 |
+
huggingface_hub/__pycache__/repocard.cpython-313.pyc,,
|
| 47 |
+
huggingface_hub/__pycache__/repocard_data.cpython-313.pyc,,
|
| 48 |
+
huggingface_hub/_buckets.py,sha256=XyLbsLzS2fpQz0aAIjKzV-1YbQ4XYZ4-HXQTVq0TXoU,45311
|
| 49 |
+
huggingface_hub/_commit_api.py,sha256=lalqhJKl8_RWVHgAdnzZKG_fy4E7fbyHIev6nDLrj4w,45378
|
| 50 |
+
huggingface_hub/_commit_scheduler.py,sha256=VG3zjb7xb66NrFMq_pB9dWIsoqSPoF6CntovV7vUiWg,14684
|
| 51 |
+
huggingface_hub/_dataset_viewer.py,sha256=LE9k4s0WJ0JNdriJ8L6qV_-xfbUVC1VEh7CBd0jOjLI,5675
|
| 52 |
+
huggingface_hub/_eval_results.py,sha256=Og7p7bCl2HQG6ZWlSDyir-CRTXjNWPu3S3q65cTGffs,8126
|
| 53 |
+
huggingface_hub/_hot_reload/__init__.py,sha256=nQEuyVZLAS_463Dff1JH5MorTpkYsBVFCyPhYWY3UyI,606
|
| 54 |
+
huggingface_hub/_hot_reload/__pycache__/__init__.cpython-313.pyc,,
|
| 55 |
+
huggingface_hub/_hot_reload/__pycache__/client.cpython-313.pyc,,
|
| 56 |
+
huggingface_hub/_hot_reload/__pycache__/sse_client.cpython-313.pyc,,
|
| 57 |
+
huggingface_hub/_hot_reload/__pycache__/types.cpython-313.pyc,,
|
| 58 |
+
huggingface_hub/_hot_reload/client.py,sha256=Ax7qT4CUi5wrg3mDzqkAG8ZzDsCwdcIIS1FIXK0SqD8,4269
|
| 59 |
+
huggingface_hub/_hot_reload/sse_client.py,sha256=49kmsB2P9sxgbZ16ed6F4aLDs3rsHrNpbp6iKlvF90c,4948
|
| 60 |
+
huggingface_hub/_hot_reload/types.py,sha256=d3AOJCLUxT41Fhz4jUrFevm9bfpooe_3PQhpBdgNYSs,2725
|
| 61 |
+
huggingface_hub/_inference_endpoints.py,sha256=FWSixNvBQlqLU4Z0yUC7Ch7a8V8Ci6eJBAD4pbTMKhk,18424
|
| 62 |
+
huggingface_hub/_jobs_api.py,sha256=f7NhziUMH6jWpWClnLPFx4XIZ0xVGAdTGfBxIn2IVy0,23679
|
| 63 |
+
huggingface_hub/_local_folder.py,sha256=KsetQw7pW0ZwFI2RLQPqVq0ZnORogPkBwRYcbRqNd5o,20654
|
| 64 |
+
huggingface_hub/_login.py,sha256=wAMdhqMSSh3GBJDozodmk4IIsAvb9J6LaLY9KnXFoyY,21818
|
| 65 |
+
huggingface_hub/_oauth.py,sha256=S8evEd8JLJHz-JiSjGH4WiIi_A7oDox_xH7E9rN1Jic,18628
|
| 66 |
+
huggingface_hub/_oidc.py,sha256=3kcCLavWQ5FQOwizUPzLaBb1ft2FIJo0bSXL24S3r4U,7098
|
| 67 |
+
huggingface_hub/_revision.py,sha256=NCmIVCR-_HjmxeYiDntcsTU-bLSfJLq8PLvvG8UhZtI,1696
|
| 68 |
+
huggingface_hub/_sandbox.py,sha256=XEXZi5S3rYyw_o3td3rB0xOVdmhpAlnyUykBsAw9KP8,80148
|
| 69 |
+
huggingface_hub/_sandbox_cache.py,sha256=UYpvHOWZj7CRhacYs5Ot-5N1wFcAapiAyniYdoX-jX8,5742
|
| 70 |
+
huggingface_hub/_snapshot_download.py,sha256=fOte9xmpaok0LGbubJZhSRd7t0nn-p00mWMebCx56qc,31029
|
| 71 |
+
huggingface_hub/_space_api.py,sha256=MF3ezhgl0Vpt3RP1wuwZ7VI9_rSyIlIJ5SmDzIwQlcQ,13424
|
| 72 |
+
huggingface_hub/_tensorboard_logger.py,sha256=GA-LhoG4Z-0Zq7w5B9vTIDJbWUkAhiG2d8a-ktlwGDY,8377
|
| 73 |
+
huggingface_hub/_tree_cache.py,sha256=M7CHMKhwY1cuhe9fb3WT1X8jmd9BXa2Sx_HS4TIXFgI,5843
|
| 74 |
+
huggingface_hub/_upload_large_folder.py,sha256=BWA1zyOK9W_3YUerX9YWQa3wAYtHRe_ypgsO79JEkqo,31235
|
| 75 |
+
huggingface_hub/_upload_pipeline.py,sha256=6YaQNtubAEaPyRoRBjPQrmsdSnCExg2KzfBuhjNqI5I,28994
|
| 76 |
+
huggingface_hub/_webhooks_payload.py,sha256=B9zqljcm20DGeVxJKba570i6RP-cR9CEupMGBuNDHHA,3544
|
| 77 |
+
huggingface_hub/_webhooks_server.py,sha256=VXOk3FHkbIc28phxknJfaFy5B_EoKOKihoCKyOaWZQk,15767
|
| 78 |
+
huggingface_hub/cli/__init__.py,sha256=tWu3Is-m2tlJwXhM_vT2mPRgpnt9KhU9qvw5aRsnjU0,664
|
| 79 |
+
huggingface_hub/cli/__pycache__/__init__.cpython-313.pyc,,
|
| 80 |
+
huggingface_hub/cli/__pycache__/_city_game.cpython-313.pyc,,
|
| 81 |
+
huggingface_hub/cli/__pycache__/_cli_utils.cpython-313.pyc,,
|
| 82 |
+
huggingface_hub/cli/__pycache__/_completion.cpython-313.pyc,,
|
| 83 |
+
huggingface_hub/cli/__pycache__/_cp.cpython-313.pyc,,
|
| 84 |
+
huggingface_hub/cli/__pycache__/_errors.cpython-313.pyc,,
|
| 85 |
+
huggingface_hub/cli/__pycache__/_file_listing.cpython-313.pyc,,
|
| 86 |
+
huggingface_hub/cli/__pycache__/_framework.cpython-313.pyc,,
|
| 87 |
+
huggingface_hub/cli/__pycache__/_help_formatter.cpython-313.pyc,,
|
| 88 |
+
huggingface_hub/cli/__pycache__/_output.cpython-313.pyc,,
|
| 89 |
+
huggingface_hub/cli/__pycache__/_skills.cpython-313.pyc,,
|
| 90 |
+
huggingface_hub/cli/__pycache__/auth.cpython-313.pyc,,
|
| 91 |
+
huggingface_hub/cli/__pycache__/buckets.cpython-313.pyc,,
|
| 92 |
+
huggingface_hub/cli/__pycache__/cache.cpython-313.pyc,,
|
| 93 |
+
huggingface_hub/cli/__pycache__/collections.cpython-313.pyc,,
|
| 94 |
+
huggingface_hub/cli/__pycache__/datasets.cpython-313.pyc,,
|
| 95 |
+
huggingface_hub/cli/__pycache__/deprecated_cli.cpython-313.pyc,,
|
| 96 |
+
huggingface_hub/cli/__pycache__/discussions.cpython-313.pyc,,
|
| 97 |
+
huggingface_hub/cli/__pycache__/download.cpython-313.pyc,,
|
| 98 |
+
huggingface_hub/cli/__pycache__/extensions.cpython-313.pyc,,
|
| 99 |
+
huggingface_hub/cli/__pycache__/hf.cpython-313.pyc,,
|
| 100 |
+
huggingface_hub/cli/__pycache__/inference_endpoints.cpython-313.pyc,,
|
| 101 |
+
huggingface_hub/cli/__pycache__/jobs.cpython-313.pyc,,
|
| 102 |
+
huggingface_hub/cli/__pycache__/lfs.cpython-313.pyc,,
|
| 103 |
+
huggingface_hub/cli/__pycache__/models.cpython-313.pyc,,
|
| 104 |
+
huggingface_hub/cli/__pycache__/papers.cpython-313.pyc,,
|
| 105 |
+
huggingface_hub/cli/__pycache__/repo_files.cpython-313.pyc,,
|
| 106 |
+
huggingface_hub/cli/__pycache__/repos.cpython-313.pyc,,
|
| 107 |
+
huggingface_hub/cli/__pycache__/sandbox.cpython-313.pyc,,
|
| 108 |
+
huggingface_hub/cli/__pycache__/skills.cpython-313.pyc,,
|
| 109 |
+
huggingface_hub/cli/__pycache__/spaces.cpython-313.pyc,,
|
| 110 |
+
huggingface_hub/cli/__pycache__/system.cpython-313.pyc,,
|
| 111 |
+
huggingface_hub/cli/__pycache__/upload.cpython-313.pyc,,
|
| 112 |
+
huggingface_hub/cli/__pycache__/upload_large_folder.cpython-313.pyc,,
|
| 113 |
+
huggingface_hub/cli/__pycache__/webhooks.cpython-313.pyc,,
|
| 114 |
+
huggingface_hub/cli/_city_game.py,sha256=2qkC9dCtPwf4-aqrnbN-vApf0YqIcGGjn2v-etrTfqg,20585
|
| 115 |
+
huggingface_hub/cli/_cli_utils.py,sha256=uFCEHDWmxTri9ei1kmN4t4ybdm4wO5Cj0e6DvLWWFao,41824
|
| 116 |
+
huggingface_hub/cli/_completion.py,sha256=BDP6Vob9TLFp6K3lr0XOY98FyD-rm22KrjwsO_YGNbE,3759
|
| 117 |
+
huggingface_hub/cli/_cp.py,sha256=Z-2Z_tBs1e4XBqRO5R9QnabU4qTi9HDlbA52jTBaCkY,10436
|
| 118 |
+
huggingface_hub/cli/_errors.py,sha256=X0rUTO-3eCK5v_MLGEza1BxcuXGccPY-h7ShzmuQc34,5907
|
| 119 |
+
huggingface_hub/cli/_file_listing.py,sha256=h_dTEphSqtvknzqJ_9cIK3Cjlz5M6cycYD3cqUW9Fn0,7986
|
| 120 |
+
huggingface_hub/cli/_framework.py,sha256=HqfAEYGWRSjKABAZi2zVNgryyU9QKW8u-tRXfR_4nQ8,27364
|
| 121 |
+
huggingface_hub/cli/_help_formatter.py,sha256=-vsa3HfHQFkKWms77EGyp9i22t44ETmVt2L-e8qP-KM,1288
|
| 122 |
+
huggingface_hub/cli/_output.py,sha256=OxFrvAcDU1TRDV0AbNh4EI1P6tqifSeVT-yqfF92U_0,14754
|
| 123 |
+
huggingface_hub/cli/_skills.py,sha256=s9Q4o4AIerDyLNuGzUB1ISXANQPgSiAs2CRcAY4NckA,15135
|
| 124 |
+
huggingface_hub/cli/auth.py,sha256=Ew79M60pTMCnS1ewKVK3fd8ZyfHrNjREOmNWuFXy37M,6056
|
| 125 |
+
huggingface_hub/cli/buckets.py,sha256=J8xMP88fQ6ri3f65Z8XEImI2GNA4KKWloLJJQCBezZw,21062
|
| 126 |
+
huggingface_hub/cli/cache.py,sha256=2T0oTYXsrtcJxhQENSpXnYYNUkagsvvdrf5gbCACBjA,28574
|
| 127 |
+
huggingface_hub/cli/collections.py,sha256=Ed41zJqrvK-AWkHdkeHWm_aJKR_ZeNIovCGvWG04itk,10120
|
| 128 |
+
huggingface_hub/cli/datasets.py,sha256=X1xMJTFhHH5NJtmjTS6NN1g4ffai6Abh_obDEJM5mvo,10497
|
| 129 |
+
huggingface_hub/cli/deprecated_cli.py,sha256=vAK_CR4NE9uzNJ0I59SyBnIln5nlypG4CPoohMoPMVg,1090
|
| 130 |
+
huggingface_hub/cli/discussions.py,sha256=obvvN4mtsFnfMs67YTZaMvHkVaM6A-FYr5n-wGkF_jE,14029
|
| 131 |
+
huggingface_hub/cli/download.py,sha256=o6fMKJoNQMxCm1RQUnX3blihJ17mantxkf0F68rZIxs,9669
|
| 132 |
+
huggingface_hub/cli/extensions.py,sha256=qNM1DmzcQXt8kPWQfISc8Pvyku8BxAs-BOVP6sKRBOM,25572
|
| 133 |
+
huggingface_hub/cli/hf.py,sha256=84sV7zWF5SB7Q48FMXpA9XwKsMFGaCAzNklrkV7l-vQ,5549
|
| 134 |
+
huggingface_hub/cli/inference_endpoints.py,sha256=F-td5VCPrzgBZbkRwc6HhasJDrJDbLmreHoe4nTgktA,18820
|
| 135 |
+
huggingface_hub/cli/jobs.py,sha256=EVGuxn2GZqkQolHib98PztJeJ64Opq-MjgIub2WTYb4,52840
|
| 136 |
+
huggingface_hub/cli/lfs.py,sha256=MAccazVPX0n-ySRaPocaQUHP7oBnVrvWZt0RhmNIavs,5936
|
| 137 |
+
huggingface_hub/cli/models.py,sha256=eG7H5yDjCVrdL7W_-wpcl70qXlv8bC-roSEu8AroPws,10148
|
| 138 |
+
huggingface_hub/cli/papers.py,sha256=HvT-8zj2fi14ua6awWpHphlbJ362FEJIwSCBJRQZ_cM,4982
|
| 139 |
+
huggingface_hub/cli/repo_files.py,sha256=sGbD5wND3miF71ThfOBJm4xzPRUpJBeLFdvyj229NAM,2443
|
| 140 |
+
huggingface_hub/cli/repos.py,sha256=QS3xYKvspLhCfO5VvpORrSiF6XhSVFclmBxsacVDP2c,20192
|
| 141 |
+
huggingface_hub/cli/sandbox.py,sha256=rTCZf6WhoDa7UiNhsTU_4Lh5uKqKYOs73xXGAAqJGdA,18379
|
| 142 |
+
huggingface_hub/cli/skills.py,sha256=PcZsDiB8XC_-98c9Pw5upmof4FAAlMQPZUmRT0T8kCQ,19926
|
| 143 |
+
huggingface_hub/cli/spaces.py,sha256=AU655GGrQmF1s1yO8yO6ml2Z8veE7zsh4UMuR5b13RI,40800
|
| 144 |
+
huggingface_hub/cli/system.py,sha256=8_xaa-GLGp5tC-slv7cuFnpUQT3zozevsftYGlvFTgs,3071
|
| 145 |
+
huggingface_hub/cli/upload.py,sha256=P6sq8Dj81a2wazUgZEqbliRleVAAmJrf8ajFDDzHrBE,11318
|
| 146 |
+
huggingface_hub/cli/upload_large_folder.py,sha256=A-2qxAdLrYd8IogfZS2nDhSR1cpA_aaxhXst638WkAk,4081
|
| 147 |
+
huggingface_hub/cli/webhooks.py,sha256=HdpPp01c6Ece_EuNNJr46rnRki3vF4j4o9-SqYRUHnM,8550
|
| 148 |
+
huggingface_hub/community.py,sha256=N5OE5Rbc2BMwpNXCFOPQAwn0otqQunsXdEnSE_0J2Xo,12325
|
| 149 |
+
huggingface_hub/constants.py,sha256=leJzcvH7fkoIpb8LLLd21gB_j71AWSonum0MRV18a6c,13792
|
| 150 |
+
huggingface_hub/dataclasses.py,sha256=dkPHOKq1YVlto9TWMkeX8BnErmbZk6tVtqD1S_n6Cw8,25896
|
| 151 |
+
huggingface_hub/errors.py,sha256=MsfOFq6Sxo2bZ8PTPWbQb250v7aqZJO5-mmWSnX_kDk,18695
|
| 152 |
+
huggingface_hub/fastai_utils.py,sha256=jwKeVXH_VP0zxnOKVNQCVCkF-stH70l-bExbbrYkBKQ,16583
|
| 153 |
+
huggingface_hub/file_download.py,sha256=3IPbCapgC-tvlh9ErLCzA-RerYXR_1XIhDKvUOjHvKA,89165
|
| 154 |
+
huggingface_hub/hf_api.py,sha256=RNpf-DpBdBQQ5-lYjB8APlRaCOIcSK4lWUTMkvQiPcA,673285
|
| 155 |
+
huggingface_hub/hf_file_system.py,sha256=3_tdhCz6IaLVbcfk_tpbQUd7VfndHyOmbI_6q4Lq36o,60869
|
| 156 |
+
huggingface_hub/hub_mixin.py,sha256=ljKAtm3qQsY-ARtxY-PCFx0koFXLI37JytMrzpqfnQo,36912
|
| 157 |
+
huggingface_hub/inference/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
| 158 |
+
huggingface_hub/inference/__pycache__/__init__.cpython-313.pyc,,
|
| 159 |
+
huggingface_hub/inference/__pycache__/_client.cpython-313.pyc,,
|
| 160 |
+
huggingface_hub/inference/__pycache__/_common.cpython-313.pyc,,
|
| 161 |
+
huggingface_hub/inference/_client.py,sha256=mENW3_gqgJR_9PuC7dwFZYjkQETPvrIrw8rU74PeusQ,157692
|
| 162 |
+
huggingface_hub/inference/_common.py,sha256=X-fBP8s6xuNTAuQsGVIethE8dNQaa-mK7Ds54qVVDz8,14999
|
| 163 |
+
huggingface_hub/inference/_generated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
| 164 |
+
huggingface_hub/inference/_generated/__pycache__/__init__.cpython-313.pyc,,
|
| 165 |
+
huggingface_hub/inference/_generated/__pycache__/_async_client.cpython-313.pyc,,
|
| 166 |
+
huggingface_hub/inference/_generated/_async_client.py,sha256=MQNKNIqcgqpGvL-HskuX1yJV6QDDuUEjM0mtZAZv8ss,160903
|
| 167 |
+
huggingface_hub/inference/_generated/types/__init__.py,sha256=e625ENOmTUM4uUL1eAlsZOK7CovNoGBDH-TxSNQhaC4,6869
|
| 168 |
+
huggingface_hub/inference/_generated/types/__pycache__/__init__.cpython-313.pyc,,
|
| 169 |
+
huggingface_hub/inference/_generated/types/__pycache__/audio_classification.cpython-313.pyc,,
|
| 170 |
+
huggingface_hub/inference/_generated/types/__pycache__/audio_to_audio.cpython-313.pyc,,
|
| 171 |
+
huggingface_hub/inference/_generated/types/__pycache__/automatic_speech_recognition.cpython-313.pyc,,
|
| 172 |
+
huggingface_hub/inference/_generated/types/__pycache__/base.cpython-313.pyc,,
|
| 173 |
+
huggingface_hub/inference/_generated/types/__pycache__/chat_completion.cpython-313.pyc,,
|
| 174 |
+
huggingface_hub/inference/_generated/types/__pycache__/depth_estimation.cpython-313.pyc,,
|
| 175 |
+
huggingface_hub/inference/_generated/types/__pycache__/document_question_answering.cpython-313.pyc,,
|
| 176 |
+
huggingface_hub/inference/_generated/types/__pycache__/feature_extraction.cpython-313.pyc,,
|
| 177 |
+
huggingface_hub/inference/_generated/types/__pycache__/fill_mask.cpython-313.pyc,,
|
| 178 |
+
huggingface_hub/inference/_generated/types/__pycache__/image_classification.cpython-313.pyc,,
|
| 179 |
+
huggingface_hub/inference/_generated/types/__pycache__/image_segmentation.cpython-313.pyc,,
|
| 180 |
+
huggingface_hub/inference/_generated/types/__pycache__/image_text_to_image.cpython-313.pyc,,
|
| 181 |
+
huggingface_hub/inference/_generated/types/__pycache__/image_text_to_video.cpython-313.pyc,,
|
| 182 |
+
huggingface_hub/inference/_generated/types/__pycache__/image_to_image.cpython-313.pyc,,
|
| 183 |
+
huggingface_hub/inference/_generated/types/__pycache__/image_to_text.cpython-313.pyc,,
|
| 184 |
+
huggingface_hub/inference/_generated/types/__pycache__/image_to_video.cpython-313.pyc,,
|
| 185 |
+
huggingface_hub/inference/_generated/types/__pycache__/object_detection.cpython-313.pyc,,
|
| 186 |
+
huggingface_hub/inference/_generated/types/__pycache__/question_answering.cpython-313.pyc,,
|
| 187 |
+
huggingface_hub/inference/_generated/types/__pycache__/sentence_similarity.cpython-313.pyc,,
|
| 188 |
+
huggingface_hub/inference/_generated/types/__pycache__/summarization.cpython-313.pyc,,
|
| 189 |
+
huggingface_hub/inference/_generated/types/__pycache__/table_question_answering.cpython-313.pyc,,
|
| 190 |
+
huggingface_hub/inference/_generated/types/__pycache__/text2text_generation.cpython-313.pyc,,
|
| 191 |
+
huggingface_hub/inference/_generated/types/__pycache__/text_classification.cpython-313.pyc,,
|
| 192 |
+
huggingface_hub/inference/_generated/types/__pycache__/text_generation.cpython-313.pyc,,
|
| 193 |
+
huggingface_hub/inference/_generated/types/__pycache__/text_to_audio.cpython-313.pyc,,
|
| 194 |
+
huggingface_hub/inference/_generated/types/__pycache__/text_to_image.cpython-313.pyc,,
|
| 195 |
+
huggingface_hub/inference/_generated/types/__pycache__/text_to_speech.cpython-313.pyc,,
|
| 196 |
+
huggingface_hub/inference/_generated/types/__pycache__/text_to_video.cpython-313.pyc,,
|
| 197 |
+
huggingface_hub/inference/_generated/types/__pycache__/token_classification.cpython-313.pyc,,
|
| 198 |
+
huggingface_hub/inference/_generated/types/__pycache__/translation.cpython-313.pyc,,
|
| 199 |
+
huggingface_hub/inference/_generated/types/__pycache__/video_classification.cpython-313.pyc,,
|
| 200 |
+
huggingface_hub/inference/_generated/types/__pycache__/visual_question_answering.cpython-313.pyc,,
|
| 201 |
+
huggingface_hub/inference/_generated/types/__pycache__/zero_shot_classification.cpython-313.pyc,,
|
| 202 |
+
huggingface_hub/inference/_generated/types/__pycache__/zero_shot_image_classification.cpython-313.pyc,,
|
| 203 |
+
huggingface_hub/inference/_generated/types/__pycache__/zero_shot_object_detection.cpython-313.pyc,,
|
| 204 |
+
huggingface_hub/inference/_generated/types/audio_classification.py,sha256=DGWEvrRPX7xpXlbYY4LMojGcULdEZFTshhNR81rTNcQ,1567
|
| 205 |
+
huggingface_hub/inference/_generated/types/audio_to_audio.py,sha256=2Ep4WkePL7oJwcp5nRJqApwviumGHbft9HhXE9XLHj4,891
|
| 206 |
+
huggingface_hub/inference/_generated/types/automatic_speech_recognition.py,sha256=X0uCj9qwAcOmlKtV3vWH5cAgLGY0PuYIy2eSH7GE1qs,5439
|
| 207 |
+
huggingface_hub/inference/_generated/types/base.py,sha256=EPWp1qudjeEimEAyQ-ptdxpAACoRM8tW342-Ds7pP3I,6938
|
| 208 |
+
huggingface_hub/inference/_generated/types/chat_completion.py,sha256=3yc06ZI0NCZzsaeoxSMWyIHqGc8Vn_6aExncuXa1llE,11100
|
| 209 |
+
huggingface_hub/inference/_generated/types/depth_estimation.py,sha256=5Kn4xNGJg235Cofr1lzO59eRGCuLTZun8z5t6a2do8Q,910
|
| 210 |
+
huggingface_hub/inference/_generated/types/document_question_answering.py,sha256=aUZcrDGexjw8Fnm2rACm_3my77V15NnQHZxj-sa7bjo,3146
|
| 211 |
+
huggingface_hub/inference/_generated/types/feature_extraction.py,sha256=DiO47PqwhdNta_KPDBsXfCPIL6BWNhyCLymxORlqUA0,1509
|
| 212 |
+
huggingface_hub/inference/_generated/types/fill_mask.py,sha256=NhEskgoiMst7U_0cV_S-wryga4_2T5LgpbyQ23vHMp0,1680
|
| 213 |
+
huggingface_hub/inference/_generated/types/image_classification.py,sha256=xWr4OY65EcIIhd7MI610a7xlaAVHQeydi8Gc7tlaPTU,1579
|
| 214 |
+
huggingface_hub/inference/_generated/types/image_segmentation.py,sha256=lZKzbn_gkSi1sYGLb9ryeRr0aDAVrfYh0H5Iiv5JNsY,1935
|
| 215 |
+
huggingface_hub/inference/_generated/types/image_text_to_image.py,sha256=pRQgYKXJbIpEjRBxuIE6NRDtk_km9DSTQ2s2qxkYJRM,2585
|
| 216 |
+
huggingface_hub/inference/_generated/types/image_text_to_video.py,sha256=MVhP8D4eYjpT3hWYkkpuiME-hWY09o7C4rIn_wmnAmY,2450
|
| 217 |
+
huggingface_hub/inference/_generated/types/image_to_image.py,sha256=P9TeMFSnZI_pa6DyinzDPWL0PNm_INJwsqFUEzpwIcU,2262
|
| 218 |
+
huggingface_hub/inference/_generated/types/image_to_text.py,sha256=DCDS0Nqi-4AuATitvZm4EgrzYQuQEQ5lzBcaHuCmHnY,4740
|
| 219 |
+
huggingface_hub/inference/_generated/types/image_to_video.py,sha256=iMCjiFTUza9AW_MyJ1videhYgvMY9QSzQ59kNdTE9QY,2206
|
| 220 |
+
huggingface_hub/inference/_generated/types/object_detection.py,sha256=zNILlyZJh59rNNfqXM5x9uprDCPfEU-bvBtabeKSMzI,1965
|
| 221 |
+
huggingface_hub/inference/_generated/types/question_answering.py,sha256=lIRf8DjRiAsjzWUBEsGt354SB4bvejLokf-8luHw6_o,2845
|
| 222 |
+
huggingface_hub/inference/_generated/types/sentence_similarity.py,sha256=eNGiO4dR-Lft9yOLE0E97jfVyOT_ExDWZW5GeJGdatU,1027
|
| 223 |
+
huggingface_hub/inference/_generated/types/summarization.py,sha256=5Dpt6LRteT-hO9esa3Ty_-xSL5U-zTtVZzy3iKwZRng,1472
|
| 224 |
+
huggingface_hub/inference/_generated/types/table_question_answering.py,sha256=bwn616XJ-eLa5JKzn3m9rZZ3z63INGEUwRSOCWlEh-4,2269
|
| 225 |
+
huggingface_hub/inference/_generated/types/text2text_generation.py,sha256=fotSlPWtIlp_LG26AQvNRXxU-jCUfda5PWsxCkbFeKM,1591
|
| 226 |
+
huggingface_hub/inference/_generated/types/text_classification.py,sha256=uamBzz4YzWQ9mb6yuriykX8r47ZHf0K3eJpOLzB2qJw,1439
|
| 227 |
+
huggingface_hub/inference/_generated/types/text_generation.py,sha256=FRUljypvCSQ2RaiylhlHeLJfHNy8qZvuty6SCB_zcaA,5813
|
| 228 |
+
huggingface_hub/inference/_generated/types/text_to_audio.py,sha256=CiaTUSBdFtOs3gGTosqyQrjCwrN8mVJLRxDDWUtwZpQ,4677
|
| 229 |
+
huggingface_hub/inference/_generated/types/text_to_image.py,sha256=Ju25WIDHE-1n8jlZ_udG4rpAaPYpmghR7-cDgVfuf48,1869
|
| 230 |
+
huggingface_hub/inference/_generated/types/text_to_speech.py,sha256=WqDzWkhZsDcjapkoFrkyjF903SKs7aHhjypsYrn8oBw,4693
|
| 231 |
+
huggingface_hub/inference/_generated/types/text_to_video.py,sha256=KjcDbtChwrIxIdTH6vcesUaMcjL2xjPf2nWEnHazj4Y,1756
|
| 232 |
+
huggingface_hub/inference/_generated/types/token_classification.py,sha256=8KRBlqgFDszaUnPMvdRkZNQaZBtDfBhKZ6SggQldQMM,1894
|
| 233 |
+
huggingface_hub/inference/_generated/types/translation.py,sha256=-j2PC1-qfoxvGMaKesqf5jXelqj0BDlEpfvZ7dsIEJM,1742
|
| 234 |
+
huggingface_hub/inference/_generated/types/video_classification.py,sha256=vfaCqp8MuRMMtGvYCX0x4-AcwEd71kPHgSYJTNIOxLQ,1668
|
| 235 |
+
huggingface_hub/inference/_generated/types/visual_question_answering.py,sha256=sWIB-SuEpW21oU6O5qQF3tt5PavUFsFOa8Xg4upTtek,1654
|
| 236 |
+
huggingface_hub/inference/_generated/types/zero_shot_classification.py,sha256=gx-7WHad7xsNa89iSKSteKHjaHrKbIyzN_nSdJN3HCY,1697
|
| 237 |
+
huggingface_hub/inference/_generated/types/zero_shot_image_classification.py,sha256=JZTQvcbSGrOQlWcPuJvAMT8YziDpBRn1O2RQSw507uY,1449
|
| 238 |
+
huggingface_hub/inference/_generated/types/zero_shot_object_detection.py,sha256=sjdpVUN5zW9aYBymLVUs6i5HVk2qkUBO9ysEjHmsXVM,1605
|
| 239 |
+
huggingface_hub/inference/_mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
| 240 |
+
huggingface_hub/inference/_mcp/__pycache__/__init__.cpython-313.pyc,,
|
| 241 |
+
huggingface_hub/inference/_mcp/__pycache__/_cli_hacks.cpython-313.pyc,,
|
| 242 |
+
huggingface_hub/inference/_mcp/__pycache__/agent.cpython-313.pyc,,
|
| 243 |
+
huggingface_hub/inference/_mcp/__pycache__/cli.cpython-313.pyc,,
|
| 244 |
+
huggingface_hub/inference/_mcp/__pycache__/constants.cpython-313.pyc,,
|
| 245 |
+
huggingface_hub/inference/_mcp/__pycache__/mcp_client.cpython-313.pyc,,
|
| 246 |
+
huggingface_hub/inference/_mcp/__pycache__/types.cpython-313.pyc,,
|
| 247 |
+
huggingface_hub/inference/_mcp/__pycache__/utils.cpython-313.pyc,,
|
| 248 |
+
huggingface_hub/inference/_mcp/_cli_hacks.py,sha256=kIA8FfRfOWS3SrZrMTbuVOjE6yxfKfMK75Co5egTWHA,3284
|
| 249 |
+
huggingface_hub/inference/_mcp/agent.py,sha256=ufIzMGHore5n252hV5GZPM0ouDXIl6tv5Jl_5gHXnbg,4250
|
| 250 |
+
huggingface_hub/inference/_mcp/cli.py,sha256=RKdVQ_VdEgf1AL_QBCvwrsUQ0Ux4R1yerrE46DrV5C0,9642
|
| 251 |
+
huggingface_hub/inference/_mcp/constants.py,sha256=0Ih6ivEvG-pfF8NwErFngzP4Rof4lQoz6ms_eKSAkUI,2463
|
| 252 |
+
huggingface_hub/inference/_mcp/mcp_client.py,sha256=dGp8PhN6aVw4bDnuSySFSiguHUiz-nzhgv89CVdO7pI,17243
|
| 253 |
+
huggingface_hub/inference/_mcp/types.py,sha256=yHNfPsM9MhD06oeKdkbmrBsW-3WhUeqA26fyfRfx_bk,929
|
| 254 |
+
huggingface_hub/inference/_mcp/utils.py,sha256=gxSB_rBjQ6VrkApKFsxk6-UzhijxkDVNuZrsjW5pL8k,4318
|
| 255 |
+
huggingface_hub/inference/_providers/__init__.py,sha256=RSm8iJhLmkIvzUdAR9FGwOYuuWauK2RKVnZH_YHnK7s,9979
|
| 256 |
+
huggingface_hub/inference/_providers/__pycache__/__init__.cpython-313.pyc,,
|
| 257 |
+
huggingface_hub/inference/_providers/__pycache__/_common.cpython-313.pyc,,
|
| 258 |
+
huggingface_hub/inference/_providers/__pycache__/baseten.cpython-313.pyc,,
|
| 259 |
+
huggingface_hub/inference/_providers/__pycache__/cerebras.cpython-313.pyc,,
|
| 260 |
+
huggingface_hub/inference/_providers/__pycache__/cohere.cpython-313.pyc,,
|
| 261 |
+
huggingface_hub/inference/_providers/__pycache__/deepinfra.cpython-313.pyc,,
|
| 262 |
+
huggingface_hub/inference/_providers/__pycache__/fal_ai.cpython-313.pyc,,
|
| 263 |
+
huggingface_hub/inference/_providers/__pycache__/featherless_ai.cpython-313.pyc,,
|
| 264 |
+
huggingface_hub/inference/_providers/__pycache__/fireworks_ai.cpython-313.pyc,,
|
| 265 |
+
huggingface_hub/inference/_providers/__pycache__/groq.cpython-313.pyc,,
|
| 266 |
+
huggingface_hub/inference/_providers/__pycache__/hf_inference.cpython-313.pyc,,
|
| 267 |
+
huggingface_hub/inference/_providers/__pycache__/novita.cpython-313.pyc,,
|
| 268 |
+
huggingface_hub/inference/_providers/__pycache__/nscale.cpython-313.pyc,,
|
| 269 |
+
huggingface_hub/inference/_providers/__pycache__/openai.cpython-313.pyc,,
|
| 270 |
+
huggingface_hub/inference/_providers/__pycache__/ovhcloud.cpython-313.pyc,,
|
| 271 |
+
huggingface_hub/inference/_providers/__pycache__/publicai.cpython-313.pyc,,
|
| 272 |
+
huggingface_hub/inference/_providers/__pycache__/replicate.cpython-313.pyc,,
|
| 273 |
+
huggingface_hub/inference/_providers/__pycache__/scaleway.cpython-313.pyc,,
|
| 274 |
+
huggingface_hub/inference/_providers/__pycache__/together.cpython-313.pyc,,
|
| 275 |
+
huggingface_hub/inference/_providers/__pycache__/wavespeed.cpython-313.pyc,,
|
| 276 |
+
huggingface_hub/inference/_providers/__pycache__/zai_org.cpython-313.pyc,,
|
| 277 |
+
huggingface_hub/inference/_providers/_common.py,sha256=T3iKT6McNs7GeVfK3q-l7CBbuih2mTFcqkRluQn7GnQ,14308
|
| 278 |
+
huggingface_hub/inference/_providers/baseten.py,sha256=1uucLZp6HiF8OjfeFOkiuJP6WqunP_xzwk3yWPF8vGE,213
|
| 279 |
+
huggingface_hub/inference/_providers/cerebras.py,sha256=QOJ-1U-os7uE7p6eUnn_P_APq-yQhx28be7c3Tq2EuA,210
|
| 280 |
+
huggingface_hub/inference/_providers/cohere.py,sha256=P9kbIuvQ2rXI1yNmgbw5VKFFTE0huLq2k-BCyDkDico,1226
|
| 281 |
+
huggingface_hub/inference/_providers/deepinfra.py,sha256=4Behgf5V7FTv7CbN6vPEIYwv1z4gztdRRU-KVoOgxVE,4703
|
| 282 |
+
huggingface_hub/inference/_providers/fal_ai.py,sha256=Y2vo5Dl3e8EoehplwNshu8LbDT1e7V4ZyhAMqs0wXws,11705
|
| 283 |
+
huggingface_hub/inference/_providers/featherless_ai.py,sha256=C8OHdFpoFyA--pawLTekUmUSRq4sw_r0D-cSPekD4kE,1347
|
| 284 |
+
huggingface_hub/inference/_providers/fireworks_ai.py,sha256=go6XPum8u0-g768HJ_r0S4Gb445gZJxnuY_acktz-9c,1188
|
| 285 |
+
huggingface_hub/inference/_providers/groq.py,sha256=JTk2JV4ZOlaohho7zLAFQtk92kGVsPmLJ1hmzcwsqvQ,315
|
| 286 |
+
huggingface_hub/inference/_providers/hf_inference.py,sha256=z-NoYyjppc2iBPh6uVGMDikIrhnW8-dD1CJHo665ko0,9803
|
| 287 |
+
huggingface_hub/inference/_providers/novita.py,sha256=ATEoSdPAPLMfx3JpBc0sOyLh4upJYP6xHBuo4YEDYvg,2470
|
| 288 |
+
huggingface_hub/inference/_providers/nscale.py,sha256=T1L2JLI9LqYT9_YEdW76YEtxfDNtdGkoFhH-p2LmSxg,1767
|
| 289 |
+
huggingface_hub/inference/_providers/openai.py,sha256=wwjaaQ55xLmDsDnHpZk52xbuLoGZfWzJkFsE8AdFVaI,1054
|
| 290 |
+
huggingface_hub/inference/_providers/ovhcloud.py,sha256=tdmymlkbddMJKV7NRZ-tH2wymbLPFDTqUSXpWJUXyDQ,314
|
| 291 |
+
huggingface_hub/inference/_providers/publicai.py,sha256=1I2W6rORloB5QHSvky4njZO2XKLTwA-kPdNoauoT5rg,210
|
| 292 |
+
huggingface_hub/inference/_providers/replicate.py,sha256=hte0ZB2RtGFwpAuLrFp2Gbgsn3EOlccCnJkWTDTW__A,6027
|
| 293 |
+
huggingface_hub/inference/_providers/scaleway.py,sha256=MfIc7ZND1sPr__rOmNHZVg0VpECQo0bVyksgIm_32xQ,1174
|
| 294 |
+
huggingface_hub/inference/_providers/together.py,sha256=lrymDZ86n5yoR_Zz7o3AzzSePyo0s7BOBU5digLxb5o,12220
|
| 295 |
+
huggingface_hub/inference/_providers/wavespeed.py,sha256=MGM7Y7r2nQiH_EH0t5UE1o1fT-o3sS7RaQlicS9WHsg,5028
|
| 296 |
+
huggingface_hub/inference/_providers/zai_org.py,sha256=gLJZOEmCPmUZvdM7VDn2nxm4ac3veHdoMgQNR63UeWE,4739
|
| 297 |
+
huggingface_hub/lfs.py,sha256=8ymYNHRXgvgUUM9w0EAnC-vkSOOq_1EFJEJNoevoeE8,16588
|
| 298 |
+
huggingface_hub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
| 299 |
+
huggingface_hub/repocard.py,sha256=7u7hiKjNuPYOwa3XPOPUnYPpuv4buya4WM47kVexklY,35602
|
| 300 |
+
huggingface_hub/repocard_data.py,sha256=f834MUSXGqrVcC02668r3ACrPzKKOAg75y3aB_hXm1U,34308
|
| 301 |
+
huggingface_hub/serialization/__init__.py,sha256=jCiw_vVQYW52gwVfWiqgocf2Q19kGTQlRGVpf-4SLP8,963
|
| 302 |
+
huggingface_hub/serialization/__pycache__/__init__.cpython-313.pyc,,
|
| 303 |
+
huggingface_hub/serialization/__pycache__/_base.cpython-313.pyc,,
|
| 304 |
+
huggingface_hub/serialization/__pycache__/_dduf.cpython-313.pyc,,
|
| 305 |
+
huggingface_hub/serialization/__pycache__/_torch.cpython-313.pyc,,
|
| 306 |
+
huggingface_hub/serialization/_base.py,sha256=8vYeDIsqgyOO3I_uii44Bkb50LsMz3kkD4gwAPzhfWU,8436
|
| 307 |
+
huggingface_hub/serialization/_dduf.py,sha256=FmGRg5wkXI5sNCgZgCNJFmqb_nVrrQYOHf4Lqq0d64I,15385
|
| 308 |
+
huggingface_hub/serialization/_torch.py,sha256=BtaCP-G3oYRP6Pj9MhI-ygBsk3JbBPnVDW9huSOgtkk,46897
|
| 309 |
+
huggingface_hub/templates/datasetcard_template.md,sha256=W-EMqR6wndbrnZorkVv56URWPG49l7MATGeI015kTvs,5503
|
| 310 |
+
huggingface_hub/templates/modelcard_template.md,sha256=4AqArS3cqdtbit5Bo-DhjcnDFR-pza5hErLLTPM4Yuc,6870
|
| 311 |
+
huggingface_hub/utils/__init__.py,sha256=pdVhPWWZkv0QM7kAQw7Bf259RZdPtYBLQqQwLQN-VC4,4045
|
| 312 |
+
huggingface_hub/utils/__pycache__/__init__.cpython-313.pyc,,
|
| 313 |
+
huggingface_hub/utils/__pycache__/_auth.cpython-313.pyc,,
|
| 314 |
+
huggingface_hub/utils/__pycache__/_cache_assets.cpython-313.pyc,,
|
| 315 |
+
huggingface_hub/utils/__pycache__/_cache_manager.cpython-313.pyc,,
|
| 316 |
+
huggingface_hub/utils/__pycache__/_chunk_utils.cpython-313.pyc,,
|
| 317 |
+
huggingface_hub/utils/__pycache__/_datetime.cpython-313.pyc,,
|
| 318 |
+
huggingface_hub/utils/__pycache__/_deprecation.cpython-313.pyc,,
|
| 319 |
+
huggingface_hub/utils/__pycache__/_detect_agent.cpython-313.pyc,,
|
| 320 |
+
huggingface_hub/utils/__pycache__/_dotenv.cpython-313.pyc,,
|
| 321 |
+
huggingface_hub/utils/__pycache__/_experimental.cpython-313.pyc,,
|
| 322 |
+
huggingface_hub/utils/__pycache__/_fixes.cpython-313.pyc,,
|
| 323 |
+
huggingface_hub/utils/__pycache__/_git_credential.cpython-313.pyc,,
|
| 324 |
+
huggingface_hub/utils/__pycache__/_headers.cpython-313.pyc,,
|
| 325 |
+
huggingface_hub/utils/__pycache__/_hf_uris.cpython-313.pyc,,
|
| 326 |
+
huggingface_hub/utils/__pycache__/_http.cpython-313.pyc,,
|
| 327 |
+
huggingface_hub/utils/__pycache__/_lfs.cpython-313.pyc,,
|
| 328 |
+
huggingface_hub/utils/__pycache__/_oauth_device.cpython-313.pyc,,
|
| 329 |
+
huggingface_hub/utils/__pycache__/_pagination.cpython-313.pyc,,
|
| 330 |
+
huggingface_hub/utils/__pycache__/_parsing.cpython-313.pyc,,
|
| 331 |
+
huggingface_hub/utils/__pycache__/_paths.cpython-313.pyc,,
|
| 332 |
+
huggingface_hub/utils/__pycache__/_runtime.cpython-313.pyc,,
|
| 333 |
+
huggingface_hub/utils/__pycache__/_safetensors.cpython-313.pyc,,
|
| 334 |
+
huggingface_hub/utils/__pycache__/_subprocess.cpython-313.pyc,,
|
| 335 |
+
huggingface_hub/utils/__pycache__/_telemetry.cpython-313.pyc,,
|
| 336 |
+
huggingface_hub/utils/__pycache__/_terminal.cpython-313.pyc,,
|
| 337 |
+
huggingface_hub/utils/__pycache__/_typing.cpython-313.pyc,,
|
| 338 |
+
huggingface_hub/utils/__pycache__/_validators.cpython-313.pyc,,
|
| 339 |
+
huggingface_hub/utils/__pycache__/_verification.cpython-313.pyc,,
|
| 340 |
+
huggingface_hub/utils/__pycache__/_xet.cpython-313.pyc,,
|
| 341 |
+
huggingface_hub/utils/__pycache__/_xet_progress_reporting.cpython-313.pyc,,
|
| 342 |
+
huggingface_hub/utils/__pycache__/endpoint_helpers.cpython-313.pyc,,
|
| 343 |
+
huggingface_hub/utils/__pycache__/insecure_hashlib.cpython-313.pyc,,
|
| 344 |
+
huggingface_hub/utils/__pycache__/logging.cpython-313.pyc,,
|
| 345 |
+
huggingface_hub/utils/__pycache__/sha.cpython-313.pyc,,
|
| 346 |
+
huggingface_hub/utils/__pycache__/tqdm.cpython-313.pyc,,
|
| 347 |
+
huggingface_hub/utils/_auth.py,sha256=Ykuq3yi67jxHmFdyLSc43xvGuCAIR7-h-dajX8IxpGQ,19958
|
| 348 |
+
huggingface_hub/utils/_cache_assets.py,sha256=nnzHRtQAR50dQeIK6qKddsmjjTW9v9HZ7b9bq7PJqss,5691
|
| 349 |
+
huggingface_hub/utils/_cache_manager.py,sha256=BFixsojrtG1K8rsD89EsGtXSktXbt0-VlJJkPRz5tso,35733
|
| 350 |
+
huggingface_hub/utils/_chunk_utils.py,sha256=pTjy8Z-KLU4W_6D3OUh3E8lCodWDCd6aJwjNDU0C5U8,2121
|
| 351 |
+
huggingface_hub/utils/_datetime.py,sha256=tbNyI0Dkh27oScPUtLIT_8apqIIkXZYbigjOn9S3aMw,2755
|
| 352 |
+
huggingface_hub/utils/_deprecation.py,sha256=n4kNHbGipquSObJ-gxodcfd6lqoe_8s-VIsTuo3Oruk,4865
|
| 353 |
+
huggingface_hub/utils/_detect_agent.py,sha256=Iw2KW9BUZWfIypn9CtiTmEibHwKK9xx5Ar48z1cMU-Q,7337
|
| 354 |
+
huggingface_hub/utils/_dotenv.py,sha256=NdlEM8OXtzwlwmJCmdc6oGubFObrpgr408y8gnSs7ls,2604
|
| 355 |
+
huggingface_hub/utils/_experimental.py,sha256=q9vUvc1JybVFRQ0GRREG-BLouZEJC_40MxcwbAlOud0,2464
|
| 356 |
+
huggingface_hub/utils/_fixes.py,sha256=jTK1VLmc0ZC9ROSXjKoL5F6kOZFByzirRWIdXxhBfWU,4124
|
| 357 |
+
huggingface_hub/utils/_git_credential.py,sha256=1BhjvIScCOAToDORLOKrR3Szs-m0E5AYHPmC0SD7Nrs,4548
|
| 358 |
+
huggingface_hub/utils/_headers.py,sha256=Q9bu9mvOpm-9rfv-YRlr_Ywy_aMZkmRcN3MbVEcJAJA,8234
|
| 359 |
+
huggingface_hub/utils/_hf_uris.py,sha256=RTHrVKLT5vVU-ca8roXHrKwbN3njowanvxHjx5B4sUg,27558
|
| 360 |
+
huggingface_hub/utils/_http.py,sha256=zx1TItko5i0_CzL95zHyt36XXrtVLnn5F9fl6ODq8aw,46197
|
| 361 |
+
huggingface_hub/utils/_lfs.py,sha256=xMU-ROgNAUpDkzbH6yRZsE-eVYUYTNMGrgVb_QdTv-k,3942
|
| 362 |
+
huggingface_hub/utils/_oauth_device.py,sha256=o7ICmI01yDt1oAFNMUJvN1ThSfaCDpC8AHYSQgcnFGw,7983
|
| 363 |
+
huggingface_hub/utils/_pagination.py,sha256=h-TJjFbX3FqkA1T0tGZ0EhxMvuRSCg-UfL_a-BMs54I,1943
|
| 364 |
+
huggingface_hub/utils/_parsing.py,sha256=DVWuO6s_XMiGfXyc0jyX5hi_yGgBHpF9KLoeQM-VVK0,3586
|
| 365 |
+
huggingface_hub/utils/_paths.py,sha256=kWpUMIK9rmKb2LHgHCkk2DnroDzucrwbj-urCJCgfGw,5962
|
| 366 |
+
huggingface_hub/utils/_runtime.py,sha256=GVJ_Dt6y_48TZWl0ui9yPdW8M7rqGblRGyUqKaXxkCo,13493
|
| 367 |
+
huggingface_hub/utils/_safetensors.py,sha256=qFE7OA-vjU8X0zBBtOeNRzLu147KF-PdFoaxkYfWB4M,4426
|
| 368 |
+
huggingface_hub/utils/_subprocess.py,sha256=tFVBBNot_HLVqQ79y873TGb12C4PUTMwub-GhzDTemE,4542
|
| 369 |
+
huggingface_hub/utils/_telemetry.py,sha256=Tpa3YmOLhK_MEnL2i2fRLQy8aqAEZWbj6MKC18Hs8BA,4824
|
| 370 |
+
huggingface_hub/utils/_terminal.py,sha256=whHgs0Co-1LatoCmJE_ojqE_cWP6b5dWPWpU-uSFWUU,8969
|
| 371 |
+
huggingface_hub/utils/_typing.py,sha256=1LeE785YedppXSR9a1fQDu5rhTxX5v9kBYBaHrP65rA,3542
|
| 372 |
+
huggingface_hub/utils/_validators.py,sha256=tuC1U4yxB-4XKIajZMnKZUSKvZ6qPpydY8LItfa0g40,8419
|
| 373 |
+
huggingface_hub/utils/_verification.py,sha256=ZSilnolSkYmHB3vmBlWuuVr53xz59P5uGnxC3H-Hjc0,5434
|
| 374 |
+
huggingface_hub/utils/_xet.py,sha256=lVCUmCxzmBFYRSkJxFLtfhe7_HJWdLzJdENL-ln5Grc,6556
|
| 375 |
+
huggingface_hub/utils/_xet_progress_reporting.py,sha256=MnNCxB9ieflArNNyz3R4kpkjOdt6mQxi8qp8xqzXQTM,14162
|
| 376 |
+
huggingface_hub/utils/endpoint_helpers.py,sha256=9VtIAlxQ5H_4y30sjCAgbu7XCqAtNLC7aRYxaNn0hLI,2366
|
| 377 |
+
huggingface_hub/utils/insecure_hashlib.py,sha256=z3dVUFvdBZ8kQI_8Vzvvlr3ims-EBiY-SYPdnzIKOkw,1008
|
| 378 |
+
huggingface_hub/utils/logging.py,sha256=WaXk5gRa8Ml_LUIH34QCr8suYj8i_8Wot4IDnYJyWyM,4870
|
| 379 |
+
huggingface_hub/utils/sha.py,sha256=h8wxheZpcv671RhtiIFcFmQTDSrogN3kwQx3ZaNEUHg,2121
|
| 380 |
+
huggingface_hub/utils/tqdm.py,sha256=xXjKGy2y2gMqpKeMUqXcQWc66buW39NwkTr3hlWH6MY,16729
|
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/REQUESTED
ADDED
|
File without changes
|
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/WHEEL
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Wheel-Version: 1.0
|
| 2 |
+
Generator: setuptools (83.0.0)
|
| 3 |
+
Root-Is-Purelib: true
|
| 4 |
+
Tag: py3-none-any
|
| 5 |
+
|
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/entry_points.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[console_scripts]
|
| 2 |
+
hf = huggingface_hub.cli.hf:main
|
| 3 |
+
huggingface-cli = huggingface_hub.cli.deprecated_cli:main
|
| 4 |
+
tiny-agents = huggingface_hub.inference._mcp.cli:app
|
| 5 |
+
|
| 6 |
+
[fsspec.specs]
|
| 7 |
+
hf = huggingface_hub.HfFileSystem
|
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/licenses/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 10 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 13 |
+
the copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 16 |
+
other entities that control, are controlled by, or are under common
|
| 17 |
+
control with that entity. For the purposes of this definition,
|
| 18 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 19 |
+
direction or management of such entity, whether by contract or
|
| 20 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 21 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 24 |
+
exercising permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation
|
| 28 |
+
source, and configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical
|
| 31 |
+
transformation or translation of a Source form, including but
|
| 32 |
+
not limited to compiled object code, generated documentation,
|
| 33 |
+
and conversions to other media types.
|
| 34 |
+
|
| 35 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 36 |
+
Object form, made available under the License, as indicated by a
|
| 37 |
+
copyright notice that is included in or attached to the work
|
| 38 |
+
(an example is provided in the Appendix below).
|
| 39 |
+
|
| 40 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 41 |
+
form, that is based on (or derived from) the Work and for which the
|
| 42 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 43 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 44 |
+
of this License, Derivative Works shall not include works that remain
|
| 45 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 46 |
+
the Work and Derivative Works thereof.
|
| 47 |
+
|
| 48 |
+
"Contribution" shall mean any work of authorship, including
|
| 49 |
+
the original version of the Work and any modifications or additions
|
| 50 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 51 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 52 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 53 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 54 |
+
means any form of electronic, verbal, or written communication sent
|
| 55 |
+
to the Licensor or its representatives, including but not limited to
|
| 56 |
+
communication on electronic mailing lists, source code control systems,
|
| 57 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 58 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 59 |
+
excluding communication that is conspicuously marked or otherwise
|
| 60 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 61 |
+
|
| 62 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 63 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 64 |
+
subsequently incorporated within the Work.
|
| 65 |
+
|
| 66 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 67 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 68 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 69 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 70 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 71 |
+
Work and such Derivative Works in Source or Object form.
|
| 72 |
+
|
| 73 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 74 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 75 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 76 |
+
(except as stated in this section) patent license to make, have made,
|
| 77 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 78 |
+
where such license applies only to those patent claims licensable
|
| 79 |
+
by such Contributor that are necessarily infringed by their
|
| 80 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 81 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 82 |
+
institute patent litigation against any entity (including a
|
| 83 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 84 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 85 |
+
or contributory patent infringement, then any patent licenses
|
| 86 |
+
granted to You under this License for that Work shall terminate
|
| 87 |
+
as of the date such litigation is filed.
|
| 88 |
+
|
| 89 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 90 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 91 |
+
modifications, and in Source or Object form, provided that You
|
| 92 |
+
meet the following conditions:
|
| 93 |
+
|
| 94 |
+
(a) You must give any other recipients of the Work or
|
| 95 |
+
Derivative Works a copy of this License; and
|
| 96 |
+
|
| 97 |
+
(b) You must cause any modified files to carry prominent notices
|
| 98 |
+
stating that You changed the files; and
|
| 99 |
+
|
| 100 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 101 |
+
that You distribute, all copyright, patent, trademark, and
|
| 102 |
+
attribution notices from the Source form of the Work,
|
| 103 |
+
excluding those notices that do not pertain to any part of
|
| 104 |
+
the Derivative Works; and
|
| 105 |
+
|
| 106 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 107 |
+
distribution, then any Derivative Works that You distribute must
|
| 108 |
+
include a readable copy of the attribution notices contained
|
| 109 |
+
within such NOTICE file, excluding those notices that do not
|
| 110 |
+
pertain to any part of the Derivative Works, in at least one
|
| 111 |
+
of the following places: within a NOTICE text file distributed
|
| 112 |
+
as part of the Derivative Works; within the Source form or
|
| 113 |
+
documentation, if provided along with the Derivative Works; or,
|
| 114 |
+
within a display generated by the Derivative Works, if and
|
| 115 |
+
wherever such third-party notices normally appear. The contents
|
| 116 |
+
of the NOTICE file are for informational purposes only and
|
| 117 |
+
do not modify the License. You may add Your own attribution
|
| 118 |
+
notices within Derivative Works that You distribute, alongside
|
| 119 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 120 |
+
that such additional attribution notices cannot be construed
|
| 121 |
+
as modifying the License.
|
| 122 |
+
|
| 123 |
+
You may add Your own copyright statement to Your modifications and
|
| 124 |
+
may provide additional or different license terms and conditions
|
| 125 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 126 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 127 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 128 |
+
the conditions stated in this License.
|
| 129 |
+
|
| 130 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 131 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 132 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 133 |
+
this License, without any additional terms or conditions.
|
| 134 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 135 |
+
the terms of any separate license agreement you may have executed
|
| 136 |
+
with Licensor regarding such Contributions.
|
| 137 |
+
|
| 138 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 139 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 140 |
+
except as required for reasonable and customary use in describing the
|
| 141 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 142 |
+
|
| 143 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 144 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 145 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 146 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 147 |
+
implied, including, without limitation, any warranties or conditions
|
| 148 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 149 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 150 |
+
appropriateness of using or redistributing the Work and assume any
|
| 151 |
+
risks associated with Your exercise of permissions under this License.
|
| 152 |
+
|
| 153 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 154 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 155 |
+
unless required by applicable law (such as deliberate and grossly
|
| 156 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 157 |
+
liable to You for damages, including any direct, indirect, special,
|
| 158 |
+
incidental, or consequential damages of any character arising as a
|
| 159 |
+
result of this License or out of the use or inability to use the
|
| 160 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 161 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 162 |
+
other commercial damages or losses), even if such Contributor
|
| 163 |
+
has been advised of the possibility of such damages.
|
| 164 |
+
|
| 165 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 166 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 167 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 168 |
+
or other liability obligations and/or rights consistent with this
|
| 169 |
+
License. However, in accepting such obligations, You may act only
|
| 170 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 171 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 172 |
+
defend, and hold each Contributor harmless for any liability
|
| 173 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 174 |
+
of your accepting any such warranty or additional liability.
|
| 175 |
+
|
| 176 |
+
END OF TERMS AND CONDITIONS
|
| 177 |
+
|
| 178 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 179 |
+
|
| 180 |
+
To apply the Apache License to your work, attach the following
|
| 181 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 182 |
+
replaced with your own identifying information. (Don't include
|
| 183 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 184 |
+
comment syntax for the file format. We also recommend that a
|
| 185 |
+
file or class name and description of purpose be included on the
|
| 186 |
+
same "printed page" as the copyright notice for easier
|
| 187 |
+
identification within third-party archives.
|
| 188 |
+
|
| 189 |
+
Copyright [yyyy] [name of copyright owner]
|
| 190 |
+
|
| 191 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 192 |
+
you may not use this file except in compliance with the License.
|
| 193 |
+
You may obtain a copy of the License at
|
| 194 |
+
|
| 195 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 196 |
+
|
| 197 |
+
Unless required by applicable law or agreed to in writing, software
|
| 198 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 199 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 200 |
+
See the License for the specific language governing permissions and
|
| 201 |
+
limitations under the License.
|
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/top_level.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
huggingface_hub
|
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/novita.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any
|
| 2 |
+
|
| 3 |
+
from huggingface_hub.hf_api import InferenceProviderMapping
|
| 4 |
+
from huggingface_hub.inference._common import RequestParameters, _as_dict
|
| 5 |
+
from huggingface_hub.inference._providers._common import (
|
| 6 |
+
BaseConversationalTask,
|
| 7 |
+
BaseTextGenerationTask,
|
| 8 |
+
TaskProviderHelper,
|
| 9 |
+
filter_none,
|
| 10 |
+
)
|
| 11 |
+
from huggingface_hub.utils import get_session
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
_PROVIDER = "novita"
|
| 15 |
+
_BASE_URL = "https://api.novita.ai"
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class NovitaTextGenerationTask(BaseTextGenerationTask):
|
| 19 |
+
def __init__(self):
|
| 20 |
+
super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
|
| 21 |
+
|
| 22 |
+
def _prepare_route(self, mapped_model: str, api_key: str) -> str:
|
| 23 |
+
# there is no v1/ route for novita
|
| 24 |
+
return "/v3/openai/completions"
|
| 25 |
+
|
| 26 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 27 |
+
output = _as_dict(response)["choices"][0]
|
| 28 |
+
return {
|
| 29 |
+
"generated_text": output["text"],
|
| 30 |
+
"details": {
|
| 31 |
+
"finish_reason": output.get("finish_reason"),
|
| 32 |
+
"seed": output.get("seed"),
|
| 33 |
+
},
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class NovitaConversationalTask(BaseConversationalTask):
|
| 38 |
+
def __init__(self):
|
| 39 |
+
super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
|
| 40 |
+
|
| 41 |
+
def _prepare_route(self, mapped_model: str, api_key: str) -> str:
|
| 42 |
+
# there is no v1/ route for novita
|
| 43 |
+
return "/v3/openai/chat/completions"
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class NovitaTextToVideoTask(TaskProviderHelper):
|
| 47 |
+
def __init__(self):
|
| 48 |
+
super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task="text-to-video")
|
| 49 |
+
|
| 50 |
+
def _prepare_route(self, mapped_model: str, api_key: str) -> str:
|
| 51 |
+
return f"/v3/hf/{mapped_model}"
|
| 52 |
+
|
| 53 |
+
def _prepare_payload_as_dict(
|
| 54 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 55 |
+
) -> dict | None:
|
| 56 |
+
return {"prompt": inputs, **filter_none(parameters)}
|
| 57 |
+
|
| 58 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 59 |
+
response_dict = _as_dict(response)
|
| 60 |
+
if not (
|
| 61 |
+
isinstance(response_dict, dict)
|
| 62 |
+
and "video" in response_dict
|
| 63 |
+
and isinstance(response_dict["video"], dict)
|
| 64 |
+
and "video_url" in response_dict["video"]
|
| 65 |
+
):
|
| 66 |
+
raise ValueError("Expected response format: { 'video': { 'video_url': string } }")
|
| 67 |
+
|
| 68 |
+
video_url = response_dict["video"]["video_url"]
|
| 69 |
+
return get_session().get(video_url).content
|
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/nscale.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
from typing import Any
|
| 3 |
+
|
| 4 |
+
from huggingface_hub.hf_api import InferenceProviderMapping
|
| 5 |
+
from huggingface_hub.inference._common import RequestParameters, _as_dict
|
| 6 |
+
|
| 7 |
+
from ._common import BaseConversationalTask, TaskProviderHelper, filter_none
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class NscaleConversationalTask(BaseConversationalTask):
|
| 11 |
+
def __init__(self):
|
| 12 |
+
super().__init__(provider="nscale", base_url="https://inference.api.nscale.com")
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class NscaleTextToImageTask(TaskProviderHelper):
|
| 16 |
+
def __init__(self):
|
| 17 |
+
super().__init__(provider="nscale", base_url="https://inference.api.nscale.com", task="text-to-image")
|
| 18 |
+
|
| 19 |
+
def _prepare_route(self, mapped_model: str, api_key: str) -> str:
|
| 20 |
+
return "/v1/images/generations"
|
| 21 |
+
|
| 22 |
+
def _prepare_payload_as_dict(
|
| 23 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 24 |
+
) -> dict | None:
|
| 25 |
+
mapped_model = provider_mapping_info.provider_id
|
| 26 |
+
# Combine all parameters except inputs and parameters
|
| 27 |
+
parameters = filter_none(parameters)
|
| 28 |
+
if "width" in parameters and "height" in parameters:
|
| 29 |
+
parameters["size"] = f"{parameters.pop('width')}x{parameters.pop('height')}"
|
| 30 |
+
if "num_inference_steps" in parameters:
|
| 31 |
+
parameters.pop("num_inference_steps")
|
| 32 |
+
if "cfg_scale" in parameters:
|
| 33 |
+
parameters.pop("cfg_scale")
|
| 34 |
+
payload = {
|
| 35 |
+
"response_format": "b64_json",
|
| 36 |
+
"prompt": inputs,
|
| 37 |
+
"model": mapped_model,
|
| 38 |
+
**parameters,
|
| 39 |
+
}
|
| 40 |
+
return payload
|
| 41 |
+
|
| 42 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 43 |
+
response_dict = _as_dict(response)
|
| 44 |
+
return base64.b64decode(response_dict["data"][0]["b64_json"])
|
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/openai.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from huggingface_hub.hf_api import InferenceProviderMapping
|
| 2 |
+
from huggingface_hub.inference._providers._common import BaseConversationalTask
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class OpenAIConversationalTask(BaseConversationalTask):
|
| 6 |
+
def __init__(self):
|
| 7 |
+
super().__init__(provider="openai", base_url="https://api.openai.com")
|
| 8 |
+
|
| 9 |
+
def _prepare_api_key(self, api_key: str | None) -> str:
|
| 10 |
+
if api_key is None:
|
| 11 |
+
raise ValueError("You must provide an api_key to work with OpenAI API.")
|
| 12 |
+
if api_key.startswith("hf_"):
|
| 13 |
+
raise ValueError(
|
| 14 |
+
"OpenAI provider is not available through Hugging Face routing, please use your own OpenAI API key."
|
| 15 |
+
)
|
| 16 |
+
return api_key
|
| 17 |
+
|
| 18 |
+
def _prepare_mapping_info(self, model: str | None) -> InferenceProviderMapping:
|
| 19 |
+
if model is None:
|
| 20 |
+
raise ValueError("Please provide an OpenAI model ID, e.g. `gpt-4o` or `o1`.")
|
| 21 |
+
return InferenceProviderMapping(
|
| 22 |
+
provider="openai", providerId=model, task="conversational", status="live", hf_model_id=model
|
| 23 |
+
)
|
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/ovhcloud.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from huggingface_hub.inference._providers._common import BaseConversationalTask
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
_PROVIDER = "ovhcloud"
|
| 5 |
+
_BASE_URL = "https://oai.endpoints.kepler.ai.cloud.ovh.net"
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class OVHcloudConversationalTask(BaseConversationalTask):
|
| 9 |
+
def __init__(self):
|
| 10 |
+
super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
|
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/publicai.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from ._common import BaseConversationalTask
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class PublicAIConversationalTask(BaseConversationalTask):
|
| 5 |
+
def __init__(self):
|
| 6 |
+
super().__init__(provider="publicai", base_url="https://api.publicai.co")
|
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/replicate.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any
|
| 2 |
+
|
| 3 |
+
from huggingface_hub.hf_api import InferenceProviderMapping
|
| 4 |
+
from huggingface_hub.inference._common import RequestParameters, _as_dict, _as_url
|
| 5 |
+
from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none
|
| 6 |
+
from huggingface_hub.utils import get_session
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
_PROVIDER = "replicate"
|
| 10 |
+
_BASE_URL = "https://api.replicate.com"
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class ReplicateTask(TaskProviderHelper):
|
| 14 |
+
def __init__(self, task: str):
|
| 15 |
+
super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task)
|
| 16 |
+
|
| 17 |
+
def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]:
|
| 18 |
+
headers = super()._prepare_headers(headers, api_key)
|
| 19 |
+
headers["Prefer"] = "wait"
|
| 20 |
+
return headers
|
| 21 |
+
|
| 22 |
+
def _prepare_route(self, mapped_model: str, api_key: str) -> str:
|
| 23 |
+
if ":" in mapped_model:
|
| 24 |
+
return "/v1/predictions"
|
| 25 |
+
return f"/v1/models/{mapped_model}/predictions"
|
| 26 |
+
|
| 27 |
+
def _prepare_payload_as_dict(
|
| 28 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 29 |
+
) -> dict | None:
|
| 30 |
+
mapped_model = provider_mapping_info.provider_id
|
| 31 |
+
payload: dict[str, Any] = {"input": {"prompt": inputs, **filter_none(parameters)}}
|
| 32 |
+
if ":" in mapped_model:
|
| 33 |
+
version = mapped_model.split(":", 1)[1]
|
| 34 |
+
payload["version"] = version
|
| 35 |
+
return payload
|
| 36 |
+
|
| 37 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 38 |
+
response_dict = _as_dict(response)
|
| 39 |
+
if response_dict.get("output") is None:
|
| 40 |
+
raise TimeoutError(
|
| 41 |
+
f"Inference request timed out after 60 seconds. No output generated for model {response_dict.get('model')}"
|
| 42 |
+
"The model might be in cold state or starting up. Please try again later."
|
| 43 |
+
)
|
| 44 |
+
output_url = (
|
| 45 |
+
response_dict["output"] if isinstance(response_dict["output"], str) else response_dict["output"][0]
|
| 46 |
+
)
|
| 47 |
+
return get_session().get(output_url).content
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class ReplicateTextToImageTask(ReplicateTask):
|
| 51 |
+
def __init__(self):
|
| 52 |
+
super().__init__("text-to-image")
|
| 53 |
+
|
| 54 |
+
def _prepare_payload_as_dict(
|
| 55 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 56 |
+
) -> dict | None:
|
| 57 |
+
payload: dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore
|
| 58 |
+
if provider_mapping_info.adapter_weights_path is not None:
|
| 59 |
+
payload["input"]["lora_weights"] = f"https://huggingface.co/{provider_mapping_info.hf_model_id}"
|
| 60 |
+
return payload
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class ReplicateTextToSpeechTask(ReplicateTask):
|
| 64 |
+
def __init__(self):
|
| 65 |
+
super().__init__("text-to-speech")
|
| 66 |
+
|
| 67 |
+
def _prepare_payload_as_dict(
|
| 68 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 69 |
+
) -> dict | None:
|
| 70 |
+
payload: dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore
|
| 71 |
+
payload["input"]["text"] = payload["input"].pop("prompt") # rename "prompt" to "text" for TTS
|
| 72 |
+
return payload
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class ReplicateAutomaticSpeechRecognitionTask(ReplicateTask):
|
| 76 |
+
def __init__(self) -> None:
|
| 77 |
+
super().__init__("automatic-speech-recognition")
|
| 78 |
+
|
| 79 |
+
def _prepare_payload_as_dict(
|
| 80 |
+
self,
|
| 81 |
+
inputs: Any,
|
| 82 |
+
parameters: dict,
|
| 83 |
+
provider_mapping_info: InferenceProviderMapping,
|
| 84 |
+
) -> dict | None:
|
| 85 |
+
mapped_model = provider_mapping_info.provider_id
|
| 86 |
+
audio_url = _as_url(inputs, default_mime_type="audio/wav")
|
| 87 |
+
|
| 88 |
+
payload: dict[str, Any] = {
|
| 89 |
+
"input": {
|
| 90 |
+
**{"audio": audio_url},
|
| 91 |
+
**filter_none(parameters),
|
| 92 |
+
}
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
if ":" in mapped_model:
|
| 96 |
+
payload["version"] = mapped_model.split(":", 1)[1]
|
| 97 |
+
|
| 98 |
+
return payload
|
| 99 |
+
|
| 100 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 101 |
+
response_dict = _as_dict(response)
|
| 102 |
+
output = response_dict.get("output")
|
| 103 |
+
|
| 104 |
+
if isinstance(output, str):
|
| 105 |
+
return {"text": output}
|
| 106 |
+
|
| 107 |
+
if isinstance(output, list) and output:
|
| 108 |
+
first_item = output[0]
|
| 109 |
+
if isinstance(first_item, str):
|
| 110 |
+
return {"text": first_item}
|
| 111 |
+
if isinstance(first_item, dict):
|
| 112 |
+
output = first_item
|
| 113 |
+
|
| 114 |
+
text: str | None = None
|
| 115 |
+
if isinstance(output, dict):
|
| 116 |
+
transcription = output.get("transcription")
|
| 117 |
+
if isinstance(transcription, str):
|
| 118 |
+
text = transcription
|
| 119 |
+
|
| 120 |
+
translation = output.get("translation")
|
| 121 |
+
if isinstance(translation, str):
|
| 122 |
+
text = translation
|
| 123 |
+
|
| 124 |
+
txt_file = output.get("txt_file")
|
| 125 |
+
if isinstance(txt_file, str):
|
| 126 |
+
text_response = get_session().get(txt_file)
|
| 127 |
+
text_response.raise_for_status()
|
| 128 |
+
text = text_response.text
|
| 129 |
+
|
| 130 |
+
if text is not None:
|
| 131 |
+
return {"text": text}
|
| 132 |
+
|
| 133 |
+
raise ValueError("Received malformed response from Replicate automatic-speech-recognition API")
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
class ReplicateImageToImageTask(ReplicateTask):
|
| 137 |
+
def __init__(self):
|
| 138 |
+
super().__init__("image-to-image")
|
| 139 |
+
|
| 140 |
+
def _prepare_payload_as_dict(
|
| 141 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 142 |
+
) -> dict | None:
|
| 143 |
+
image_url = _as_url(inputs, default_mime_type="image/jpeg")
|
| 144 |
+
|
| 145 |
+
# Different Replicate models expect the image in different keys
|
| 146 |
+
payload: dict[str, Any] = {
|
| 147 |
+
"input": {
|
| 148 |
+
"image": image_url,
|
| 149 |
+
"images": [image_url],
|
| 150 |
+
"input_image": image_url,
|
| 151 |
+
"input_images": [image_url],
|
| 152 |
+
**filter_none(parameters),
|
| 153 |
+
}
|
| 154 |
+
}
|
| 155 |
+
|
| 156 |
+
mapped_model = provider_mapping_info.provider_id
|
| 157 |
+
if ":" in mapped_model:
|
| 158 |
+
version = mapped_model.split(":", 1)[1]
|
| 159 |
+
payload["version"] = version
|
| 160 |
+
return payload
|
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/scaleway.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any
|
| 2 |
+
|
| 3 |
+
from huggingface_hub.inference._common import RequestParameters, _as_dict
|
| 4 |
+
|
| 5 |
+
from ._common import BaseConversationalTask, InferenceProviderMapping, TaskProviderHelper, filter_none
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class ScalewayConversationalTask(BaseConversationalTask):
|
| 9 |
+
def __init__(self):
|
| 10 |
+
super().__init__(provider="scaleway", base_url="https://api.scaleway.ai")
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class ScalewayFeatureExtractionTask(TaskProviderHelper):
|
| 14 |
+
def __init__(self):
|
| 15 |
+
super().__init__(provider="scaleway", base_url="https://api.scaleway.ai", task="feature-extraction")
|
| 16 |
+
|
| 17 |
+
def _prepare_route(self, mapped_model: str, api_key: str) -> str:
|
| 18 |
+
return "/v1/embeddings"
|
| 19 |
+
|
| 20 |
+
def _prepare_payload_as_dict(
|
| 21 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 22 |
+
) -> dict | None:
|
| 23 |
+
parameters = filter_none(parameters)
|
| 24 |
+
return {"input": inputs, "model": provider_mapping_info.provider_id, **parameters}
|
| 25 |
+
|
| 26 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 27 |
+
embeddings = _as_dict(response)["data"]
|
| 28 |
+
return [embedding["embedding"] for embedding in embeddings]
|
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/together.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
import time
|
| 3 |
+
from abc import ABC
|
| 4 |
+
from typing import Any
|
| 5 |
+
|
| 6 |
+
from huggingface_hub.hf_api import InferenceProviderMapping
|
| 7 |
+
from huggingface_hub.inference._common import (
|
| 8 |
+
RequestParameters,
|
| 9 |
+
_as_dict,
|
| 10 |
+
_as_url,
|
| 11 |
+
)
|
| 12 |
+
from huggingface_hub.inference._providers._common import (
|
| 13 |
+
BaseConversationalTask,
|
| 14 |
+
BaseTextGenerationTask,
|
| 15 |
+
TaskProviderHelper,
|
| 16 |
+
filter_none,
|
| 17 |
+
)
|
| 18 |
+
from huggingface_hub.utils import get_session, hf_raise_for_status, logging
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
logger = logging.get_logger(__name__)
|
| 22 |
+
|
| 23 |
+
_PROVIDER = "together"
|
| 24 |
+
_BASE_URL = "https://api.together.xyz"
|
| 25 |
+
|
| 26 |
+
# Polling interval for async video generation (in seconds).
|
| 27 |
+
_VIDEO_POLLING_INTERVAL = 2.0
|
| 28 |
+
|
| 29 |
+
# Upper bound on status polls (initial response may already be terminal; each further poll is one attempt).
|
| 30 |
+
_VIDEO_MAX_POLL_ATTEMPTS = 150 # ~5 minutes at _VIDEO_POLLING_INTERVAL
|
| 31 |
+
|
| 32 |
+
# Job statuses that mean "keep polling". Together returns "queued" before transitioning to
|
| 33 |
+
# "in_progress", so we must treat both as pending.
|
| 34 |
+
_VIDEO_PENDING_STATUSES = {"queued", "in_progress"}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class TogetherTask(TaskProviderHelper, ABC):
|
| 38 |
+
"""Base class for Together API tasks."""
|
| 39 |
+
|
| 40 |
+
def __init__(self, task: str):
|
| 41 |
+
super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task)
|
| 42 |
+
|
| 43 |
+
def _prepare_route(self, mapped_model: str, api_key: str) -> str:
|
| 44 |
+
match self.task:
|
| 45 |
+
case "text-to-image" | "image-to-image":
|
| 46 |
+
return "/v1/images/generations"
|
| 47 |
+
case "text-to-speech":
|
| 48 |
+
return "/v1/audio/speech"
|
| 49 |
+
case "feature-extraction":
|
| 50 |
+
return "/v1/embeddings"
|
| 51 |
+
case "text-to-video" | "image-to-video":
|
| 52 |
+
# Video creation lives under /v2 (see https://docs.together.ai/reference/create-videos).
|
| 53 |
+
return "/v2/videos"
|
| 54 |
+
raise ValueError(f"Unsupported task '{self.task}' for Together API.")
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class TogetherTextGenerationTask(BaseTextGenerationTask):
|
| 58 |
+
def __init__(self):
|
| 59 |
+
super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
|
| 60 |
+
|
| 61 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 62 |
+
output = _as_dict(response)["choices"][0]
|
| 63 |
+
return {
|
| 64 |
+
"generated_text": output["text"],
|
| 65 |
+
"details": {
|
| 66 |
+
"finish_reason": output.get("finish_reason"),
|
| 67 |
+
"seed": output.get("seed"),
|
| 68 |
+
},
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
class TogetherConversationalTask(BaseConversationalTask):
|
| 73 |
+
def __init__(self):
|
| 74 |
+
super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
|
| 75 |
+
|
| 76 |
+
def _prepare_payload_as_dict(
|
| 77 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 78 |
+
) -> dict | None:
|
| 79 |
+
payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info)
|
| 80 |
+
if payload is None:
|
| 81 |
+
return None
|
| 82 |
+
# Together accepts response_format `{type: "json_schema", schema: <schema>}` (flattened),
|
| 83 |
+
# so unwrap the OpenAI-style `{type: "json_schema", json_schema: {schema}}` envelope.
|
| 84 |
+
response_format = payload.get("response_format")
|
| 85 |
+
if (
|
| 86 |
+
isinstance(response_format, dict)
|
| 87 |
+
and response_format.get("type") == "json_schema"
|
| 88 |
+
and isinstance(response_format.get("json_schema"), dict)
|
| 89 |
+
and "schema" in response_format["json_schema"]
|
| 90 |
+
):
|
| 91 |
+
payload["response_format"] = {
|
| 92 |
+
"type": "json_schema",
|
| 93 |
+
"schema": response_format["json_schema"]["schema"],
|
| 94 |
+
}
|
| 95 |
+
return payload
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
class TogetherTextToImageTask(TogetherTask):
|
| 99 |
+
def __init__(self):
|
| 100 |
+
super().__init__("text-to-image")
|
| 101 |
+
|
| 102 |
+
def _prepare_payload_as_dict(
|
| 103 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 104 |
+
) -> dict | None:
|
| 105 |
+
mapped_model = provider_mapping_info.provider_id
|
| 106 |
+
parameters = filter_none(parameters)
|
| 107 |
+
if "num_inference_steps" in parameters:
|
| 108 |
+
parameters["steps"] = parameters.pop("num_inference_steps")
|
| 109 |
+
|
| 110 |
+
return {"prompt": inputs, "response_format": "base64", **parameters, "model": mapped_model}
|
| 111 |
+
|
| 112 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 113 |
+
response_dict = _as_dict(response)
|
| 114 |
+
return base64.b64decode(response_dict["data"][0]["b64_json"])
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
class TogetherImageToImageTask(TogetherTask):
|
| 118 |
+
def __init__(self):
|
| 119 |
+
super().__init__("image-to-image")
|
| 120 |
+
|
| 121 |
+
def _prepare_payload_as_dict(
|
| 122 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 123 |
+
) -> dict | None:
|
| 124 |
+
mapped_model = provider_mapping_info.provider_id
|
| 125 |
+
image_url = _as_url(inputs, default_mime_type="image/jpeg")
|
| 126 |
+
|
| 127 |
+
# Filter `None` values first: the client always passes `"prompt": None` when the user
|
| 128 |
+
# omits the argument, so popping before filtering would yield `None` instead of the
|
| 129 |
+
# `""` default and send `"prompt": null` to Together (rejected by Flux Kontext).
|
| 130 |
+
parameters = filter_none(parameters)
|
| 131 |
+
prompt = parameters.pop("prompt", "")
|
| 132 |
+
if "num_inference_steps" in parameters:
|
| 133 |
+
parameters["steps"] = parameters.pop("num_inference_steps")
|
| 134 |
+
|
| 135 |
+
# Together exposes two mutually-exclusive image inputs (see
|
| 136 |
+
# https://docs.together.ai/docs/image-to-image): FLUX.1 Kontext only accepts
|
| 137 |
+
# `image_url`; FLUX.2 [dev] and Google models (Gemini 3 Pro Image, Flash Image
|
| 138 |
+
# 2.5) only accept `reference_images`. FLUX.2 [pro]/[flex] accept either but
|
| 139 |
+
# `reference_images` is the documented default. Use `image_url` only for
|
| 140 |
+
# FLUX.1 Kontext models and `reference_images` for everything else.
|
| 141 |
+
lowered = mapped_model.lower()
|
| 142 |
+
use_image_url = "kontext" in lowered and "flux.1" in lowered
|
| 143 |
+
image_field: dict[str, Any] = {"image_url": image_url} if use_image_url else {"reference_images": [image_url]}
|
| 144 |
+
return {
|
| 145 |
+
"prompt": prompt,
|
| 146 |
+
**image_field,
|
| 147 |
+
"response_format": "base64",
|
| 148 |
+
**parameters,
|
| 149 |
+
"model": mapped_model,
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 153 |
+
response_dict = _as_dict(response)
|
| 154 |
+
return base64.b64decode(response_dict["data"][0]["b64_json"])
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class TogetherFeatureExtractionTask(TogetherTask):
|
| 158 |
+
def __init__(self):
|
| 159 |
+
super().__init__("feature-extraction")
|
| 160 |
+
|
| 161 |
+
def _prepare_payload_as_dict(
|
| 162 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 163 |
+
) -> dict | None:
|
| 164 |
+
return {
|
| 165 |
+
"input": inputs,
|
| 166 |
+
"model": provider_mapping_info.provider_id,
|
| 167 |
+
**filter_none(parameters),
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 171 |
+
return [item["embedding"] for item in _as_dict(response)["data"]]
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
class TogetherTextToSpeechTask(TogetherTask):
|
| 175 |
+
def __init__(self):
|
| 176 |
+
super().__init__("text-to-speech")
|
| 177 |
+
|
| 178 |
+
def _prepare_payload_as_dict(
|
| 179 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 180 |
+
) -> dict | None:
|
| 181 |
+
# `voice` is required by the Together API and is model-specific
|
| 182 |
+
# (see https://docs.together.ai/docs/text-to-speech#supported-voices),
|
| 183 |
+
# so we don't set a default and let the API surface a clear error if missing.
|
| 184 |
+
return {
|
| 185 |
+
"input": inputs,
|
| 186 |
+
"model": provider_mapping_info.provider_id,
|
| 187 |
+
**filter_none(parameters),
|
| 188 |
+
}
|
| 189 |
+
|
| 190 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 191 |
+
if isinstance(response, bytes):
|
| 192 |
+
return response
|
| 193 |
+
raise ValueError(f"Expected raw audio bytes for text-to-speech, got {type(response).__name__}.")
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def _normalize_video_parameters(parameters: dict) -> dict:
|
| 197 |
+
"""Map HF inference-client conventions onto Together's video API parameter names."""
|
| 198 |
+
parameters = filter_none(parameters)
|
| 199 |
+
if "num_inference_steps" in parameters:
|
| 200 |
+
parameters["steps"] = parameters.pop("num_inference_steps")
|
| 201 |
+
if "target_size" in parameters:
|
| 202 |
+
target_size = parameters.pop("target_size")
|
| 203 |
+
if "width" in target_size:
|
| 204 |
+
parameters["width"] = target_size["width"]
|
| 205 |
+
if "height" in target_size:
|
| 206 |
+
parameters["height"] = target_size["height"]
|
| 207 |
+
return parameters
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
class TogetherVideoTask(TogetherTask, ABC):
|
| 211 |
+
"""Base class for Together's asynchronous video generation tasks."""
|
| 212 |
+
|
| 213 |
+
def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
|
| 214 |
+
if request_params is None:
|
| 215 |
+
raise ValueError("A `RequestParameters` object is required to poll Together video jobs.")
|
| 216 |
+
|
| 217 |
+
job = _as_dict(response)
|
| 218 |
+
job_id = job.get("id")
|
| 219 |
+
if not job_id:
|
| 220 |
+
raise ValueError("No job ID found in Together video generation response.")
|
| 221 |
+
|
| 222 |
+
# Status polling lives at the same /v2/videos URL with the job ID appended.
|
| 223 |
+
status_url = f"{request_params.url}/{job_id}"
|
| 224 |
+
|
| 225 |
+
logger.info("Generating video, polling for completion...")
|
| 226 |
+
# Together usually returns `status: "queued"` on the initial POST, but the field is
|
| 227 |
+
# optional per the spec — treat a missing status as "still pending" and poll, rather
|
| 228 |
+
# than falling through to the "unexpected status" error below.
|
| 229 |
+
status = job.get("status")
|
| 230 |
+
for _ in range(_VIDEO_MAX_POLL_ATTEMPTS):
|
| 231 |
+
if status is not None and status not in _VIDEO_PENDING_STATUSES:
|
| 232 |
+
break
|
| 233 |
+
time.sleep(_VIDEO_POLLING_INTERVAL)
|
| 234 |
+
status_response = get_session().get(status_url, headers=request_params.headers)
|
| 235 |
+
hf_raise_for_status(status_response)
|
| 236 |
+
job = status_response.json()
|
| 237 |
+
status = job.get("status")
|
| 238 |
+
if status is not None and status not in _VIDEO_PENDING_STATUSES:
|
| 239 |
+
break
|
| 240 |
+
else:
|
| 241 |
+
raise ValueError(
|
| 242 |
+
"Timed out while waiting for Together video generation "
|
| 243 |
+
f"— aborting after {_VIDEO_MAX_POLL_ATTEMPTS} status polls"
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
+
if status == "failed":
|
| 247 |
+
error = job.get("error") or {}
|
| 248 |
+
raise RuntimeError(f"Together video generation failed: {error.get('message') or 'Unknown error'}")
|
| 249 |
+
if status != "completed":
|
| 250 |
+
raise RuntimeError(f"Unexpected Together video job status: {status!r}")
|
| 251 |
+
|
| 252 |
+
video_url = (job.get("outputs") or {}).get("video_url")
|
| 253 |
+
if not video_url:
|
| 254 |
+
raise ValueError("No video URL found in completed Together video job.")
|
| 255 |
+
|
| 256 |
+
video_response = get_session().get(video_url)
|
| 257 |
+
hf_raise_for_status(video_response)
|
| 258 |
+
return video_response.content
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
class TogetherTextToVideoTask(TogetherVideoTask):
|
| 262 |
+
def __init__(self):
|
| 263 |
+
super().__init__("text-to-video")
|
| 264 |
+
|
| 265 |
+
def _prepare_payload_as_dict(
|
| 266 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 267 |
+
) -> dict | None:
|
| 268 |
+
return {
|
| 269 |
+
"prompt": inputs,
|
| 270 |
+
"model": provider_mapping_info.provider_id,
|
| 271 |
+
**_normalize_video_parameters(parameters),
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
class TogetherImageToVideoTask(TogetherVideoTask):
|
| 276 |
+
def __init__(self):
|
| 277 |
+
super().__init__("image-to-video")
|
| 278 |
+
|
| 279 |
+
def _prepare_payload_as_dict(
|
| 280 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 281 |
+
) -> dict | None:
|
| 282 |
+
# Together expects each keyframe as `{input_image, frame: "first" | "last"}`
|
| 283 |
+
# for i2v models. See https://docs.together.ai/docs/inference/videos/reference-and-keyframes.
|
| 284 |
+
# Note: `input_image` accepts a data URL or an HTTP(S) URL but the field is capped
|
| 285 |
+
# at ~60KB — users with larger inputs should host the image and pass `frame_images`
|
| 286 |
+
# directly via `extra_body`.
|
| 287 |
+
return {
|
| 288 |
+
"model": provider_mapping_info.provider_id,
|
| 289 |
+
"frame_images": [{"input_image": _as_url(inputs, default_mime_type="image/png"), "frame": "first"}],
|
| 290 |
+
**_normalize_video_parameters(parameters),
|
| 291 |
+
}
|
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/wavespeed.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
import time
|
| 3 |
+
from abc import ABC
|
| 4 |
+
from typing import Any
|
| 5 |
+
from urllib.parse import urlparse
|
| 6 |
+
|
| 7 |
+
from huggingface_hub.hf_api import InferenceProviderMapping
|
| 8 |
+
from huggingface_hub.inference._common import RequestParameters, _as_dict
|
| 9 |
+
from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none
|
| 10 |
+
from huggingface_hub.utils import get_session, hf_raise_for_status
|
| 11 |
+
from huggingface_hub.utils.logging import get_logger
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
logger = get_logger(__name__)
|
| 15 |
+
|
| 16 |
+
# Polling interval (in seconds)
|
| 17 |
+
_POLLING_INTERVAL = 0.5
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class WavespeedAITask(TaskProviderHelper, ABC):
|
| 21 |
+
def __init__(self, task: str):
|
| 22 |
+
super().__init__(provider="wavespeed", base_url="https://api.wavespeed.ai", task=task)
|
| 23 |
+
|
| 24 |
+
def _prepare_route(self, mapped_model: str, api_key: str) -> str:
|
| 25 |
+
return f"/api/v3/{mapped_model}"
|
| 26 |
+
|
| 27 |
+
def get_response(
|
| 28 |
+
self,
|
| 29 |
+
response: bytes | dict,
|
| 30 |
+
request_params: RequestParameters | None = None,
|
| 31 |
+
) -> Any:
|
| 32 |
+
response_dict = _as_dict(response)
|
| 33 |
+
data = response_dict.get("data", {})
|
| 34 |
+
result_path = data.get("urls", {}).get("get")
|
| 35 |
+
|
| 36 |
+
if not result_path:
|
| 37 |
+
raise ValueError("No result URL found in the response")
|
| 38 |
+
if request_params is None:
|
| 39 |
+
raise ValueError("A `RequestParameters` object should be provided to get responses with WaveSpeed AI.")
|
| 40 |
+
|
| 41 |
+
# Parse the request URL to determine base URL
|
| 42 |
+
parsed_url = urlparse(request_params.url)
|
| 43 |
+
# Add /wavespeed to base URL if going through HF router
|
| 44 |
+
if parsed_url.netloc == "router.huggingface.co":
|
| 45 |
+
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/wavespeed"
|
| 46 |
+
else:
|
| 47 |
+
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
|
| 48 |
+
|
| 49 |
+
# Extract path from result_path URL
|
| 50 |
+
if isinstance(result_path, str):
|
| 51 |
+
result_url_path = urlparse(result_path).path
|
| 52 |
+
else:
|
| 53 |
+
result_url_path = result_path
|
| 54 |
+
|
| 55 |
+
result_url = f"{base_url}{result_url_path}"
|
| 56 |
+
|
| 57 |
+
logger.info("Processing request, polling for results...")
|
| 58 |
+
|
| 59 |
+
# Poll until task is completed
|
| 60 |
+
while True:
|
| 61 |
+
time.sleep(_POLLING_INTERVAL)
|
| 62 |
+
result_response = get_session().get(result_url, headers=request_params.headers)
|
| 63 |
+
hf_raise_for_status(result_response)
|
| 64 |
+
|
| 65 |
+
result = result_response.json()
|
| 66 |
+
task_result = result.get("data", {})
|
| 67 |
+
status = task_result.get("status")
|
| 68 |
+
|
| 69 |
+
if status == "completed":
|
| 70 |
+
# Get content from the first output URL
|
| 71 |
+
if not task_result.get("outputs") or len(task_result["outputs"]) == 0:
|
| 72 |
+
raise ValueError("No output URL in completed response")
|
| 73 |
+
|
| 74 |
+
output_url = task_result["outputs"][0]
|
| 75 |
+
return get_session().get(output_url).content
|
| 76 |
+
elif status == "failed":
|
| 77 |
+
error_msg = task_result.get("error", "Task failed with no specific error message")
|
| 78 |
+
raise ValueError(f"WaveSpeed AI task failed: {error_msg}")
|
| 79 |
+
elif status in ["processing", "created"]:
|
| 80 |
+
continue
|
| 81 |
+
else:
|
| 82 |
+
raise ValueError(f"Unknown status: {status}")
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class WavespeedAITextToImageTask(WavespeedAITask):
|
| 86 |
+
def __init__(self):
|
| 87 |
+
super().__init__("text-to-image")
|
| 88 |
+
|
| 89 |
+
def _prepare_payload_as_dict(
|
| 90 |
+
self,
|
| 91 |
+
inputs: Any,
|
| 92 |
+
parameters: dict,
|
| 93 |
+
provider_mapping_info: InferenceProviderMapping,
|
| 94 |
+
) -> dict | None:
|
| 95 |
+
return {"prompt": inputs, **filter_none(parameters)}
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
class WavespeedAITextToVideoTask(WavespeedAITextToImageTask):
|
| 99 |
+
def __init__(self):
|
| 100 |
+
WavespeedAITask.__init__(self, "text-to-video")
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
class WavespeedAIImageToImageTask(WavespeedAITask):
|
| 104 |
+
def __init__(self):
|
| 105 |
+
super().__init__("image-to-image")
|
| 106 |
+
|
| 107 |
+
def _prepare_payload_as_dict(
|
| 108 |
+
self,
|
| 109 |
+
inputs: Any,
|
| 110 |
+
parameters: dict,
|
| 111 |
+
provider_mapping_info: InferenceProviderMapping,
|
| 112 |
+
) -> dict | None:
|
| 113 |
+
# Convert inputs to image (URL or base64)
|
| 114 |
+
if isinstance(inputs, str) and inputs.startswith(("http://", "https://")):
|
| 115 |
+
image = inputs
|
| 116 |
+
elif isinstance(inputs, str):
|
| 117 |
+
# If input is a file path, read it first
|
| 118 |
+
with open(inputs, "rb") as f:
|
| 119 |
+
file_content = f.read()
|
| 120 |
+
image_b64 = base64.b64encode(file_content).decode("utf-8")
|
| 121 |
+
image = f"data:image/jpeg;base64,{image_b64}"
|
| 122 |
+
else:
|
| 123 |
+
# If input is binary data
|
| 124 |
+
image_b64 = base64.b64encode(inputs).decode("utf-8")
|
| 125 |
+
image = f"data:image/jpeg;base64,{image_b64}"
|
| 126 |
+
|
| 127 |
+
# Extract prompt from parameters if present
|
| 128 |
+
prompt = parameters.pop("prompt", None)
|
| 129 |
+
payload = {"image": image, **filter_none(parameters)}
|
| 130 |
+
if prompt is not None:
|
| 131 |
+
payload["prompt"] = prompt
|
| 132 |
+
|
| 133 |
+
return payload
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
class WavespeedAIImageToVideoTask(WavespeedAIImageToImageTask):
|
| 137 |
+
def __init__(self):
|
| 138 |
+
WavespeedAITask.__init__(self, "image-to-video")
|
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/zai_org.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import time
|
| 2 |
+
from abc import ABC
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
from huggingface_hub.hf_api import InferenceProviderMapping
|
| 6 |
+
from huggingface_hub.inference._common import RequestParameters, _as_dict
|
| 7 |
+
from huggingface_hub.inference._providers._common import BaseConversationalTask, TaskProviderHelper, filter_none
|
| 8 |
+
from huggingface_hub.utils import get_session
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
_PROVIDER = "zai-org"
|
| 12 |
+
_BASE_URL = "https://api.z.ai"
|
| 13 |
+
_POLLING_INTERVAL = 5 # seconds
|
| 14 |
+
_MAX_POLL_ATTEMPTS = 60
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class ZaiTask(TaskProviderHelper, ABC):
|
| 18 |
+
def __init__(self, task: str):
|
| 19 |
+
super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task)
|
| 20 |
+
|
| 21 |
+
def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]:
|
| 22 |
+
headers = super()._prepare_headers(headers, api_key)
|
| 23 |
+
headers["Accept-Language"] = "en-US,en"
|
| 24 |
+
headers["x-source-channel"] = "hugging_face"
|
| 25 |
+
return headers
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class ZaiConversationalTask(BaseConversationalTask):
|
| 29 |
+
def __init__(self):
|
| 30 |
+
super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
|
| 31 |
+
|
| 32 |
+
def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]:
|
| 33 |
+
headers = super()._prepare_headers(headers, api_key)
|
| 34 |
+
headers["Accept-Language"] = "en-US,en"
|
| 35 |
+
headers["x-source-channel"] = "hugging_face"
|
| 36 |
+
return headers
|
| 37 |
+
|
| 38 |
+
def _prepare_route(self, mapped_model: str, api_key: str) -> str:
|
| 39 |
+
return "/api/paas/v4/chat/completions"
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class ZaiTextToImageTask(ZaiTask):
|
| 43 |
+
"""Text-to-image task for ZAI provider using async API."""
|
| 44 |
+
|
| 45 |
+
def __init__(self):
|
| 46 |
+
super().__init__("text-to-image")
|
| 47 |
+
|
| 48 |
+
def _prepare_route(self, mapped_model: str, api_key: str) -> str:
|
| 49 |
+
return "/api/paas/v4/async/images/generations"
|
| 50 |
+
|
| 51 |
+
def _prepare_payload_as_dict(
|
| 52 |
+
self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
|
| 53 |
+
) -> dict | None:
|
| 54 |
+
width = parameters.pop("width", None)
|
| 55 |
+
height = parameters.pop("height", None)
|
| 56 |
+
size = None
|
| 57 |
+
if width is not None and height is not None:
|
| 58 |
+
size = f"{width}x{height}"
|
| 59 |
+
|
| 60 |
+
payload: dict[str, Any] = {
|
| 61 |
+
"model": provider_mapping_info.provider_id,
|
| 62 |
+
"prompt": inputs,
|
| 63 |
+
}
|
| 64 |
+
if size is not None:
|
| 65 |
+
payload["size"] = size
|
| 66 |
+
|
| 67 |
+
payload.update(filter_none(parameters))
|
| 68 |
+
return payload
|
| 69 |
+
|
| 70 |
+
def get_response(
|
| 71 |
+
self,
|
| 72 |
+
response: bytes | dict,
|
| 73 |
+
request_params: RequestParameters | None = None,
|
| 74 |
+
) -> Any:
|
| 75 |
+
"""Handle async response by polling for results."""
|
| 76 |
+
response_dict = _as_dict(response)
|
| 77 |
+
|
| 78 |
+
task_id = response_dict.get("id")
|
| 79 |
+
if task_id is None:
|
| 80 |
+
raise ValueError("No task_id in response from ZAI API")
|
| 81 |
+
|
| 82 |
+
task_status = response_dict.get("task_status")
|
| 83 |
+
if task_status == "FAIL":
|
| 84 |
+
raise ValueError(f"ZAI image generation failed for request {task_id}")
|
| 85 |
+
|
| 86 |
+
if task_status == "PROCESSING" and request_params is not None:
|
| 87 |
+
return self._poll_for_result(task_id, request_params)
|
| 88 |
+
|
| 89 |
+
return self._extract_image(response_dict)
|
| 90 |
+
|
| 91 |
+
def _poll_for_result(self, task_id: str, request_params: RequestParameters) -> bytes:
|
| 92 |
+
"""Poll the async-result endpoint until completion."""
|
| 93 |
+
session = get_session()
|
| 94 |
+
base_url = request_params.url.rsplit("/api/paas/v4/async/images/generations", 1)[0]
|
| 95 |
+
poll_url = f"{base_url}/api/paas/v4/async-result/{task_id}"
|
| 96 |
+
|
| 97 |
+
for _ in range(_MAX_POLL_ATTEMPTS):
|
| 98 |
+
poll_response = session.get(poll_url, headers=request_params.headers)
|
| 99 |
+
poll_response.raise_for_status()
|
| 100 |
+
result = poll_response.json()
|
| 101 |
+
|
| 102 |
+
task_status = result.get("task_status")
|
| 103 |
+
if task_status == "SUCCESS":
|
| 104 |
+
return self._extract_image(result)
|
| 105 |
+
elif task_status == "FAIL":
|
| 106 |
+
raise ValueError(f"Zai text-to-image generation failed for request {task_id}")
|
| 107 |
+
|
| 108 |
+
time.sleep(_POLLING_INTERVAL)
|
| 109 |
+
|
| 110 |
+
raise ValueError(
|
| 111 |
+
f"Timed out while waiting for the result from Zai API - aborting after {_MAX_POLL_ATTEMPTS} attempts"
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
def _extract_image(self, result: dict) -> bytes:
|
| 115 |
+
"""Extract and download the image from the result."""
|
| 116 |
+
image_result = result.get("image_result")
|
| 117 |
+
if not image_result or not isinstance(image_result, list) or len(image_result) == 0:
|
| 118 |
+
raise ValueError("No image_result in response from ZAI API")
|
| 119 |
+
|
| 120 |
+
image_url = image_result[0].get("url")
|
| 121 |
+
if not image_url:
|
| 122 |
+
raise ValueError("No image URL in response from ZAI API")
|
| 123 |
+
|
| 124 |
+
session = get_session()
|
| 125 |
+
image_response = session.get(image_url)
|
| 126 |
+
image_response.raise_for_status()
|
| 127 |
+
return image_response.content
|
.venv/lib/python3.13/site-packages/huggingface_hub/lfs.py
ADDED
|
@@ -0,0 +1,442 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2019-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Git LFS related type definitions and utilities"""
|
| 15 |
+
|
| 16 |
+
import io
|
| 17 |
+
import re
|
| 18 |
+
from collections.abc import Iterable
|
| 19 |
+
from math import ceil
|
| 20 |
+
from os.path import getsize
|
| 21 |
+
from typing import TYPE_CHECKING, BinaryIO, TypedDict
|
| 22 |
+
from urllib.parse import unquote
|
| 23 |
+
|
| 24 |
+
from huggingface_hub import constants
|
| 25 |
+
|
| 26 |
+
from .utils import (
|
| 27 |
+
build_hf_headers,
|
| 28 |
+
fix_hf_endpoint_in_url,
|
| 29 |
+
hf_raise_for_status,
|
| 30 |
+
http_backoff,
|
| 31 |
+
logging,
|
| 32 |
+
validate_hf_hub_args,
|
| 33 |
+
)
|
| 34 |
+
from .utils._lfs import SliceFileObj
|
| 35 |
+
from .utils.sha import sha256, sha_fileobj
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
if TYPE_CHECKING:
|
| 39 |
+
from ._commit_api import CommitOperationAdd
|
| 40 |
+
|
| 41 |
+
logger = logging.get_logger(__name__)
|
| 42 |
+
|
| 43 |
+
OID_REGEX = re.compile(r"^[0-9a-f]{40}$")
|
| 44 |
+
|
| 45 |
+
LFS_MULTIPART_UPLOAD_COMMAND = "lfs-multipart-upload"
|
| 46 |
+
|
| 47 |
+
LFS_HEADERS = {
|
| 48 |
+
"Accept": "application/vnd.git-lfs+json",
|
| 49 |
+
"Content-Type": "application/vnd.git-lfs+json",
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class UploadInfo:
|
| 54 |
+
"""
|
| 55 |
+
Data structure holding required information to determine whether a blob
|
| 56 |
+
should be uploaded to the hub using the LFS protocol or the regular protocol.
|
| 57 |
+
|
| 58 |
+
The SHA256 of the blob is computed lazily: creating an `UploadInfo` from a local path only reads
|
| 59 |
+
the first 512 bytes of the file. The full file is read (and hashed) only if `sha256` is accessed
|
| 60 |
+
before it has been set. When a file is uploaded through the Xet protocol, the SHA256 is computed
|
| 61 |
+
during upload (single read pass) and set afterwards.
|
| 62 |
+
|
| 63 |
+
Args:
|
| 64 |
+
size (`int`):
|
| 65 |
+
Size in bytes of the blob
|
| 66 |
+
sample (`bytes`):
|
| 67 |
+
First 512 bytes of the blob
|
| 68 |
+
sha256 (`bytes`, *optional*):
|
| 69 |
+
SHA256 hash of the blob, if already known. Otherwise computed lazily from `source_path`.
|
| 70 |
+
source_path (`str`, *optional*):
|
| 71 |
+
Path to the local file the blob comes from. Required to lazily compute `sha256` if not provided.
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
def __init__(
|
| 75 |
+
self,
|
| 76 |
+
size: int,
|
| 77 |
+
sample: bytes,
|
| 78 |
+
sha256: bytes | None = None,
|
| 79 |
+
source_path: str | None = None,
|
| 80 |
+
):
|
| 81 |
+
if sha256 is None and source_path is None:
|
| 82 |
+
raise ValueError("Either `sha256` or `source_path` must be provided.")
|
| 83 |
+
self.size = size
|
| 84 |
+
self.sample = sample
|
| 85 |
+
self._sha256 = sha256
|
| 86 |
+
self._source_path = source_path
|
| 87 |
+
|
| 88 |
+
@property
|
| 89 |
+
def sha256(self) -> bytes:
|
| 90 |
+
"""SHA256 of the blob. If not set yet, reads the whole file from `source_path` to compute it."""
|
| 91 |
+
if self._sha256 is None:
|
| 92 |
+
assert self._source_path is not None # guaranteed by __init__
|
| 93 |
+
with open(self._source_path, "rb") as file:
|
| 94 |
+
self._sha256 = sha_fileobj(file)
|
| 95 |
+
return self._sha256
|
| 96 |
+
|
| 97 |
+
@sha256.setter
|
| 98 |
+
def sha256(self, value: bytes) -> None:
|
| 99 |
+
self._sha256 = value
|
| 100 |
+
|
| 101 |
+
@property
|
| 102 |
+
def is_hashed(self) -> bool:
|
| 103 |
+
"""Whether the SHA256 is already known (accessing `sha256` will not trigger a file read)."""
|
| 104 |
+
return self._sha256 is not None
|
| 105 |
+
|
| 106 |
+
def __repr__(self) -> str:
|
| 107 |
+
sha = self._sha256.hex() if self._sha256 is not None else "<not computed>"
|
| 108 |
+
return f"UploadInfo(size={self.size}, sha256={sha})"
|
| 109 |
+
|
| 110 |
+
@classmethod
|
| 111 |
+
def from_path(cls, path: str):
|
| 112 |
+
size = getsize(path)
|
| 113 |
+
with open(path, "rb") as file:
|
| 114 |
+
sample = file.peek(512)[:512]
|
| 115 |
+
return cls(size=size, sample=sample, source_path=path)
|
| 116 |
+
|
| 117 |
+
@classmethod
|
| 118 |
+
def from_bytes(cls, data: bytes):
|
| 119 |
+
sha = sha256(data).digest()
|
| 120 |
+
return cls(size=len(data), sample=data[:512], sha256=sha)
|
| 121 |
+
|
| 122 |
+
@classmethod
|
| 123 |
+
def from_fileobj(cls, fileobj: BinaryIO):
|
| 124 |
+
sample = fileobj.read(512)
|
| 125 |
+
fileobj.seek(0, io.SEEK_SET)
|
| 126 |
+
sha = sha_fileobj(fileobj)
|
| 127 |
+
size = fileobj.tell()
|
| 128 |
+
fileobj.seek(0, io.SEEK_SET)
|
| 129 |
+
return cls(size=size, sha256=sha, sample=sample)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
@validate_hf_hub_args
|
| 133 |
+
def post_lfs_batch_info(
|
| 134 |
+
upload_infos: Iterable[UploadInfo],
|
| 135 |
+
token: str | None,
|
| 136 |
+
repo_type: str,
|
| 137 |
+
repo_id: str,
|
| 138 |
+
revision: str | None = None,
|
| 139 |
+
endpoint: str | None = None,
|
| 140 |
+
headers: dict[str, str] | None = None,
|
| 141 |
+
transfers: list[str] | None = None,
|
| 142 |
+
) -> tuple[list[dict], list[dict], str | None]:
|
| 143 |
+
"""
|
| 144 |
+
Requests the LFS batch endpoint to retrieve upload instructions
|
| 145 |
+
|
| 146 |
+
Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
|
| 147 |
+
|
| 148 |
+
Args:
|
| 149 |
+
upload_infos (`Iterable` of `UploadInfo`):
|
| 150 |
+
`UploadInfo` for the files that are being uploaded, typically obtained
|
| 151 |
+
from `CommitOperationAdd.upload_info`
|
| 152 |
+
token (`str` or `None`):
|
| 153 |
+
An authentication token (see https://huggingface.co/settings/token).
|
| 154 |
+
Pass `None` to fall back to the local cached token (or no token if unauthenticated).
|
| 155 |
+
repo_type (`str`):
|
| 156 |
+
Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
|
| 157 |
+
repo_id (`str`):
|
| 158 |
+
A namespace (user or an organization) and a repo name separated
|
| 159 |
+
by a `/`.
|
| 160 |
+
revision (`str`, *optional*):
|
| 161 |
+
The git revision to upload to.
|
| 162 |
+
endpoint (`str`, *optional*):
|
| 163 |
+
The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`.
|
| 164 |
+
headers (`dict`, *optional*):
|
| 165 |
+
Additional headers to include in the request
|
| 166 |
+
transfers (`list`, *optional*):
|
| 167 |
+
List of transfer methods to use. Defaults to ["basic", "multipart"].
|
| 168 |
+
|
| 169 |
+
Returns:
|
| 170 |
+
`LfsBatchInfo`: 3-tuple:
|
| 171 |
+
- First element is the list of upload instructions from the server
|
| 172 |
+
- Second element is a list of errors, if any
|
| 173 |
+
- Third element is the chosen transfer adapter if provided by the server (e.g. "basic", "multipart", "xet")
|
| 174 |
+
|
| 175 |
+
Raises:
|
| 176 |
+
[`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
| 177 |
+
If an argument is invalid or the server response is malformed.
|
| 178 |
+
[`HfHubHTTPError`]
|
| 179 |
+
If the server returned an error.
|
| 180 |
+
"""
|
| 181 |
+
endpoint = endpoint if endpoint is not None else constants.ENDPOINT
|
| 182 |
+
url_prefix = ""
|
| 183 |
+
if repo_type in constants.REPO_TYPES_URL_PREFIXES:
|
| 184 |
+
url_prefix = constants.REPO_TYPES_URL_PREFIXES[repo_type]
|
| 185 |
+
batch_url = f"{endpoint}/{url_prefix}{repo_id}.git/info/lfs/objects/batch"
|
| 186 |
+
payload: dict = {
|
| 187 |
+
"operation": "upload",
|
| 188 |
+
"transfers": transfers if transfers is not None else ["basic", "multipart"],
|
| 189 |
+
"objects": [
|
| 190 |
+
{
|
| 191 |
+
"oid": upload.sha256.hex(),
|
| 192 |
+
"size": upload.size,
|
| 193 |
+
}
|
| 194 |
+
for upload in upload_infos
|
| 195 |
+
],
|
| 196 |
+
"hash_algo": "sha256",
|
| 197 |
+
}
|
| 198 |
+
if revision is not None:
|
| 199 |
+
payload["ref"] = {"name": unquote(revision)} # revision has been previously 'quoted'
|
| 200 |
+
|
| 201 |
+
headers = {
|
| 202 |
+
**LFS_HEADERS,
|
| 203 |
+
**build_hf_headers(token=token),
|
| 204 |
+
**(headers or {}),
|
| 205 |
+
}
|
| 206 |
+
resp = http_backoff("POST", batch_url, headers=headers, json=payload)
|
| 207 |
+
hf_raise_for_status(resp)
|
| 208 |
+
batch_info = resp.json()
|
| 209 |
+
|
| 210 |
+
objects = batch_info.get("objects", None)
|
| 211 |
+
if not isinstance(objects, list):
|
| 212 |
+
raise ValueError("Malformed response from server")
|
| 213 |
+
|
| 214 |
+
chosen_transfer = batch_info.get("transfer")
|
| 215 |
+
chosen_transfer = chosen_transfer if isinstance(chosen_transfer, str) else None
|
| 216 |
+
|
| 217 |
+
return (
|
| 218 |
+
[_validate_batch_actions(obj) for obj in objects if "error" not in obj],
|
| 219 |
+
[_validate_batch_error(obj) for obj in objects if "error" in obj],
|
| 220 |
+
chosen_transfer,
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
class PayloadPartT(TypedDict):
|
| 225 |
+
partNumber: int
|
| 226 |
+
etag: str
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
class CompletionPayloadT(TypedDict):
|
| 230 |
+
"""Payload that will be sent to the Hub when uploading multi-part."""
|
| 231 |
+
|
| 232 |
+
oid: str
|
| 233 |
+
parts: list[PayloadPartT]
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def lfs_upload(
|
| 237 |
+
operation: "CommitOperationAdd",
|
| 238 |
+
lfs_batch_action: dict,
|
| 239 |
+
token: str | None = None,
|
| 240 |
+
headers: dict[str, str] | None = None,
|
| 241 |
+
endpoint: str | None = None,
|
| 242 |
+
) -> None:
|
| 243 |
+
"""
|
| 244 |
+
Handles uploading a given object to the Hub with the LFS protocol.
|
| 245 |
+
|
| 246 |
+
Can be a No-op if the content of the file is already present on the hub large file storage.
|
| 247 |
+
|
| 248 |
+
Args:
|
| 249 |
+
operation (`CommitOperationAdd`):
|
| 250 |
+
The add operation triggering this upload.
|
| 251 |
+
lfs_batch_action (`dict`):
|
| 252 |
+
Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for
|
| 253 |
+
more details.
|
| 254 |
+
token (`str`, *optional*):
|
| 255 |
+
An authentication token (see https://huggingface.co/settings/token). Used to call the
|
| 256 |
+
optional LFS verify step at the end of the upload. If `None`, falls back to the local
|
| 257 |
+
cached token.
|
| 258 |
+
headers (`dict`, *optional*):
|
| 259 |
+
Headers to include in the request, including authentication and user agent headers.
|
| 260 |
+
endpoint (`str`, *optional*):
|
| 261 |
+
The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`.
|
| 262 |
+
|
| 263 |
+
Raises:
|
| 264 |
+
[`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
| 265 |
+
If `lfs_batch_action` is improperly formatted
|
| 266 |
+
[`HfHubHTTPError`]
|
| 267 |
+
If the upload resulted in an error
|
| 268 |
+
"""
|
| 269 |
+
# 0. If LFS file is already present, skip upload
|
| 270 |
+
_validate_batch_actions(lfs_batch_action)
|
| 271 |
+
actions = lfs_batch_action.get("actions")
|
| 272 |
+
if actions is None:
|
| 273 |
+
# The file was already uploaded
|
| 274 |
+
logger.debug(f"Content of file {operation.path_in_repo} is already present upstream - skipping upload")
|
| 275 |
+
return
|
| 276 |
+
|
| 277 |
+
# 1. Validate server response (check required keys in dict)
|
| 278 |
+
upload_action = lfs_batch_action["actions"]["upload"]
|
| 279 |
+
_validate_lfs_action(upload_action)
|
| 280 |
+
verify_action = lfs_batch_action["actions"].get("verify")
|
| 281 |
+
if verify_action is not None:
|
| 282 |
+
_validate_lfs_action(verify_action)
|
| 283 |
+
|
| 284 |
+
# 2. Upload file (either single part or multi-part)
|
| 285 |
+
header = upload_action.get("header", {})
|
| 286 |
+
chunk_size = header.get("chunk_size")
|
| 287 |
+
upload_url = fix_hf_endpoint_in_url(upload_action["href"], endpoint=endpoint)
|
| 288 |
+
if chunk_size is not None:
|
| 289 |
+
try:
|
| 290 |
+
chunk_size = int(chunk_size)
|
| 291 |
+
except (ValueError, TypeError):
|
| 292 |
+
raise ValueError(
|
| 293 |
+
f"Malformed response from LFS batch endpoint: `chunk_size` should be an integer. Got '{chunk_size}'."
|
| 294 |
+
)
|
| 295 |
+
_upload_multi_part(operation=operation, header=header, chunk_size=chunk_size, upload_url=upload_url)
|
| 296 |
+
else:
|
| 297 |
+
_upload_single_part(operation=operation, upload_url=upload_url)
|
| 298 |
+
|
| 299 |
+
# 3. Verify upload went well
|
| 300 |
+
if verify_action is not None:
|
| 301 |
+
_validate_lfs_action(verify_action)
|
| 302 |
+
verify_url = fix_hf_endpoint_in_url(verify_action["href"], endpoint)
|
| 303 |
+
verify_resp = http_backoff(
|
| 304 |
+
"POST",
|
| 305 |
+
verify_url,
|
| 306 |
+
headers=build_hf_headers(token=token, headers=headers),
|
| 307 |
+
json={"oid": operation.upload_info.sha256.hex(), "size": operation.upload_info.size},
|
| 308 |
+
)
|
| 309 |
+
hf_raise_for_status(verify_resp)
|
| 310 |
+
logger.debug(f"{operation.path_in_repo}: Upload successful")
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def _validate_lfs_action(lfs_action: dict):
|
| 314 |
+
"""validates response from the LFS batch endpoint"""
|
| 315 |
+
if not (
|
| 316 |
+
isinstance(lfs_action.get("href"), str)
|
| 317 |
+
and (lfs_action.get("header") is None or isinstance(lfs_action.get("header"), dict))
|
| 318 |
+
):
|
| 319 |
+
raise ValueError("lfs_action is improperly formatted")
|
| 320 |
+
return lfs_action
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
def _validate_batch_actions(lfs_batch_actions: dict):
|
| 324 |
+
"""validates response from the LFS batch endpoint"""
|
| 325 |
+
if not (isinstance(lfs_batch_actions.get("oid"), str) and isinstance(lfs_batch_actions.get("size"), int)):
|
| 326 |
+
raise ValueError("lfs_batch_actions is improperly formatted")
|
| 327 |
+
|
| 328 |
+
upload_action = lfs_batch_actions.get("actions", {}).get("upload")
|
| 329 |
+
verify_action = lfs_batch_actions.get("actions", {}).get("verify")
|
| 330 |
+
if upload_action is not None:
|
| 331 |
+
_validate_lfs_action(upload_action)
|
| 332 |
+
if verify_action is not None:
|
| 333 |
+
_validate_lfs_action(verify_action)
|
| 334 |
+
return lfs_batch_actions
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
def _validate_batch_error(lfs_batch_error: dict):
|
| 338 |
+
"""validates response from the LFS batch endpoint"""
|
| 339 |
+
if not (isinstance(lfs_batch_error.get("oid"), str) and isinstance(lfs_batch_error.get("size"), int)):
|
| 340 |
+
raise ValueError("lfs_batch_error is improperly formatted")
|
| 341 |
+
error_info = lfs_batch_error.get("error")
|
| 342 |
+
if not (
|
| 343 |
+
isinstance(error_info, dict)
|
| 344 |
+
and isinstance(error_info.get("message"), str)
|
| 345 |
+
and isinstance(error_info.get("code"), int)
|
| 346 |
+
):
|
| 347 |
+
raise ValueError("lfs_batch_error is improperly formatted")
|
| 348 |
+
return lfs_batch_error
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def _upload_single_part(operation: "CommitOperationAdd", upload_url: str) -> None:
|
| 352 |
+
"""
|
| 353 |
+
Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol)
|
| 354 |
+
|
| 355 |
+
Args:
|
| 356 |
+
upload_url (`str`):
|
| 357 |
+
The URL to PUT the file to.
|
| 358 |
+
fileobj:
|
| 359 |
+
The file-like object holding the data to upload.
|
| 360 |
+
|
| 361 |
+
Raises:
|
| 362 |
+
[`HfHubHTTPError`]
|
| 363 |
+
If the upload resulted in an error.
|
| 364 |
+
"""
|
| 365 |
+
with operation.as_file(with_tqdm=True) as fileobj:
|
| 366 |
+
# S3 might raise a transient 500 error -> let's retry if that happens
|
| 367 |
+
response = http_backoff("PUT", upload_url, data=fileobj)
|
| 368 |
+
hf_raise_for_status(response)
|
| 369 |
+
|
| 370 |
+
|
| 371 |
+
def _upload_multi_part(operation: "CommitOperationAdd", header: dict, chunk_size: int, upload_url: str) -> None:
|
| 372 |
+
"""
|
| 373 |
+
Uploads file using HF multipart LFS transfer protocol.
|
| 374 |
+
"""
|
| 375 |
+
# 1. Get upload URLs for each part
|
| 376 |
+
sorted_parts_urls = _get_sorted_parts_urls(header=header, upload_info=operation.upload_info, chunk_size=chunk_size)
|
| 377 |
+
|
| 378 |
+
# 2. Upload parts (pure Python)
|
| 379 |
+
response_headers = _upload_parts_iteratively(
|
| 380 |
+
operation=operation, sorted_parts_urls=sorted_parts_urls, chunk_size=chunk_size
|
| 381 |
+
)
|
| 382 |
+
|
| 383 |
+
# 3. Send completion request
|
| 384 |
+
# NOTE: `upload_url` is the Hub completion endpoint (not the S3 upload URLs).
|
| 385 |
+
completion_res = http_backoff(
|
| 386 |
+
"POST",
|
| 387 |
+
upload_url,
|
| 388 |
+
json=_get_completion_payload(response_headers, operation.upload_info.sha256.hex()),
|
| 389 |
+
headers=LFS_HEADERS,
|
| 390 |
+
)
|
| 391 |
+
hf_raise_for_status(completion_res)
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def _get_sorted_parts_urls(header: dict, upload_info: UploadInfo, chunk_size: int) -> list[str]:
|
| 395 |
+
sorted_part_upload_urls = [
|
| 396 |
+
upload_url
|
| 397 |
+
for _, upload_url in sorted(
|
| 398 |
+
[
|
| 399 |
+
(int(part_num, 10), upload_url)
|
| 400 |
+
for part_num, upload_url in header.items()
|
| 401 |
+
if part_num.isdigit() and len(part_num) > 0
|
| 402 |
+
],
|
| 403 |
+
key=lambda t: t[0],
|
| 404 |
+
)
|
| 405 |
+
]
|
| 406 |
+
num_parts = len(sorted_part_upload_urls)
|
| 407 |
+
if num_parts != ceil(upload_info.size / chunk_size):
|
| 408 |
+
raise ValueError("Invalid server response to upload large LFS file")
|
| 409 |
+
return sorted_part_upload_urls
|
| 410 |
+
|
| 411 |
+
|
| 412 |
+
def _get_completion_payload(response_headers: list[dict], oid: str) -> CompletionPayloadT:
|
| 413 |
+
parts: list[PayloadPartT] = []
|
| 414 |
+
for part_number, header in enumerate(response_headers):
|
| 415 |
+
etag = header.get("etag")
|
| 416 |
+
if etag is None or etag == "":
|
| 417 |
+
raise ValueError(f"Invalid etag (`{etag}`) returned for part {part_number + 1}")
|
| 418 |
+
parts.append(
|
| 419 |
+
{
|
| 420 |
+
"partNumber": part_number + 1,
|
| 421 |
+
"etag": etag,
|
| 422 |
+
}
|
| 423 |
+
)
|
| 424 |
+
return {"oid": oid, "parts": parts}
|
| 425 |
+
|
| 426 |
+
|
| 427 |
+
def _upload_parts_iteratively(
|
| 428 |
+
operation: "CommitOperationAdd", sorted_parts_urls: list[str], chunk_size: int
|
| 429 |
+
) -> list[dict]:
|
| 430 |
+
headers = []
|
| 431 |
+
with operation.as_file(with_tqdm=True) as fileobj:
|
| 432 |
+
for part_idx, part_upload_url in enumerate(sorted_parts_urls):
|
| 433 |
+
with SliceFileObj(
|
| 434 |
+
fileobj,
|
| 435 |
+
seek_from=chunk_size * part_idx,
|
| 436 |
+
read_limit=chunk_size,
|
| 437 |
+
) as fileobj_slice:
|
| 438 |
+
# S3 might raise a transient 500 error -> let's retry if that happens
|
| 439 |
+
part_upload_res = http_backoff("PUT", part_upload_url, data=fileobj_slice)
|
| 440 |
+
hf_raise_for_status(part_upload_res)
|
| 441 |
+
headers.append(part_upload_res.headers)
|
| 442 |
+
return headers # type: ignore
|
.venv/lib/python3.13/site-packages/huggingface_hub/py.typed
ADDED
|
File without changes
|
.venv/lib/python3.13/site-packages/huggingface_hub/repocard.py
ADDED
|
@@ -0,0 +1,835 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import re
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from typing import Any, Literal
|
| 5 |
+
|
| 6 |
+
import yaml
|
| 7 |
+
|
| 8 |
+
from huggingface_hub.file_download import hf_hub_download
|
| 9 |
+
from huggingface_hub.hf_api import upload_file
|
| 10 |
+
from huggingface_hub.repocard_data import (
|
| 11 |
+
CardData,
|
| 12 |
+
DatasetCardData,
|
| 13 |
+
EvalResult,
|
| 14 |
+
ModelCardData,
|
| 15 |
+
SpaceCardData,
|
| 16 |
+
eval_results_to_model_index,
|
| 17 |
+
model_index_to_eval_results,
|
| 18 |
+
)
|
| 19 |
+
from huggingface_hub.utils import HfHubHTTPError, get_session, hf_raise_for_status, is_jinja_available, yaml_dump
|
| 20 |
+
|
| 21 |
+
from . import constants
|
| 22 |
+
from .errors import EntryNotFoundError
|
| 23 |
+
from .utils import SoftTemporaryDirectory, logging, validate_hf_hub_args
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
logger = logging.get_logger(__name__)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
TEMPLATE_MODELCARD_PATH = Path(__file__).parent / "templates" / "modelcard_template.md"
|
| 30 |
+
TEMPLATE_DATASETCARD_PATH = Path(__file__).parent / "templates" / "datasetcard_template.md"
|
| 31 |
+
|
| 32 |
+
# exact same regex as in the Hub server. Please keep in sync.
|
| 33 |
+
# See https://github.com/huggingface/moon-landing/blob/main/server/lib/ViewMarkdown.ts#L18
|
| 34 |
+
REGEX_YAML_BLOCK = re.compile(r"^(\s*---(?:\r\n|\r|\n))([\S\s]*?)((?:\r\n|\r|\n)---[ \t]*(\r\n|\n|$))")
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class RepoCard:
|
| 38 |
+
card_data_class = CardData
|
| 39 |
+
default_template_path = TEMPLATE_MODELCARD_PATH
|
| 40 |
+
repo_type = "model"
|
| 41 |
+
|
| 42 |
+
def __init__(self, content: str, ignore_metadata_errors: bool = False):
|
| 43 |
+
"""Initialize a RepoCard from string content. The content should be a
|
| 44 |
+
Markdown file with a YAML block at the beginning and a Markdown body.
|
| 45 |
+
|
| 46 |
+
Args:
|
| 47 |
+
content (`str`): The content of the Markdown file.
|
| 48 |
+
|
| 49 |
+
Example:
|
| 50 |
+
```python
|
| 51 |
+
>>> from huggingface_hub.repocard import RepoCard
|
| 52 |
+
>>> text = '''
|
| 53 |
+
... ---
|
| 54 |
+
... language: en
|
| 55 |
+
... license: mit
|
| 56 |
+
... ---
|
| 57 |
+
...
|
| 58 |
+
... # My repo
|
| 59 |
+
... '''
|
| 60 |
+
>>> card = RepoCard(text)
|
| 61 |
+
>>> card.data.to_dict()
|
| 62 |
+
{'language': 'en', 'license': 'mit'}
|
| 63 |
+
>>> card.text
|
| 64 |
+
'\\n# My repo\\n'
|
| 65 |
+
|
| 66 |
+
```
|
| 67 |
+
> [!TIP]
|
| 68 |
+
> Raises the following error:
|
| 69 |
+
>
|
| 70 |
+
> - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
| 71 |
+
> when the content of the repo card metadata is not a dictionary.
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
# Set the content of the RepoCard, as well as underlying .data and .text attributes.
|
| 75 |
+
# See the `content` property setter for more details.
|
| 76 |
+
self.ignore_metadata_errors = ignore_metadata_errors
|
| 77 |
+
self.content = content
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def content(self):
|
| 81 |
+
"""The content of the RepoCard, including the YAML block and the Markdown body."""
|
| 82 |
+
line_break = _detect_line_ending(self._content) or "\n"
|
| 83 |
+
return f"---{line_break}{self.data.to_yaml(line_break=line_break, original_order=self._original_order)}{line_break}---{line_break}{self.text}"
|
| 84 |
+
|
| 85 |
+
@content.setter
|
| 86 |
+
def content(self, content: str):
|
| 87 |
+
"""Set the content of the RepoCard."""
|
| 88 |
+
self._content = content
|
| 89 |
+
|
| 90 |
+
match = REGEX_YAML_BLOCK.search(content)
|
| 91 |
+
if match:
|
| 92 |
+
# Metadata found in the YAML block
|
| 93 |
+
yaml_block = match.group(2)
|
| 94 |
+
self.text = content[match.end() :]
|
| 95 |
+
data_dict = yaml.safe_load(yaml_block)
|
| 96 |
+
|
| 97 |
+
if data_dict is None:
|
| 98 |
+
data_dict = {}
|
| 99 |
+
|
| 100 |
+
# The YAML block's data should be a dictionary
|
| 101 |
+
if not isinstance(data_dict, dict):
|
| 102 |
+
raise ValueError("repo card metadata block should be a dict")
|
| 103 |
+
else:
|
| 104 |
+
# Model card without metadata... create empty metadata
|
| 105 |
+
logger.warning("Repo card metadata block was not found. Setting CardData to empty.")
|
| 106 |
+
data_dict = {}
|
| 107 |
+
self.text = content
|
| 108 |
+
|
| 109 |
+
self.data = self.card_data_class(**data_dict, ignore_metadata_errors=self.ignore_metadata_errors)
|
| 110 |
+
self._original_order = list(data_dict.keys())
|
| 111 |
+
|
| 112 |
+
def __str__(self):
|
| 113 |
+
return self.content
|
| 114 |
+
|
| 115 |
+
def save(self, filepath: Path | str):
|
| 116 |
+
r"""Save a RepoCard to a file.
|
| 117 |
+
|
| 118 |
+
Args:
|
| 119 |
+
filepath (`Union[Path, str]`): Filepath to the markdown file to save.
|
| 120 |
+
|
| 121 |
+
Example:
|
| 122 |
+
```python
|
| 123 |
+
>>> from huggingface_hub.repocard import RepoCard
|
| 124 |
+
>>> card = RepoCard("---\nlanguage: en\n---\n# This is a test repo card")
|
| 125 |
+
>>> card.save("/tmp/test.md")
|
| 126 |
+
|
| 127 |
+
```
|
| 128 |
+
"""
|
| 129 |
+
filepath = Path(filepath)
|
| 130 |
+
filepath.parent.mkdir(parents=True, exist_ok=True)
|
| 131 |
+
# Preserve newlines as in the existing file.
|
| 132 |
+
with open(filepath, mode="w", newline="", encoding="utf-8") as f:
|
| 133 |
+
f.write(str(self))
|
| 134 |
+
|
| 135 |
+
@classmethod
|
| 136 |
+
def load(
|
| 137 |
+
cls,
|
| 138 |
+
repo_id_or_path: str | Path,
|
| 139 |
+
repo_type: str | None = None,
|
| 140 |
+
token: str | None = None,
|
| 141 |
+
ignore_metadata_errors: bool = False,
|
| 142 |
+
):
|
| 143 |
+
"""Initialize a RepoCard from a Hugging Face Hub repo's README.md or a local filepath.
|
| 144 |
+
|
| 145 |
+
Args:
|
| 146 |
+
repo_id_or_path (`Union[str, Path]`):
|
| 147 |
+
The repo ID associated with a Hugging Face Hub repo or a local filepath.
|
| 148 |
+
repo_type (`str`, *optional*):
|
| 149 |
+
The type of Hugging Face repo to push to. Defaults to None, which will use "model". Other options
|
| 150 |
+
are "dataset" and "space". Not used when loading from a local filepath. If this is called from a child
|
| 151 |
+
class, the default value will be the child class's `repo_type`.
|
| 152 |
+
token (`str`, *optional*):
|
| 153 |
+
Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to the stored token.
|
| 154 |
+
ignore_metadata_errors (`str`):
|
| 155 |
+
If True, errors while parsing the metadata section will be ignored. Some information might be lost during
|
| 156 |
+
the process. Use it at your own risk.
|
| 157 |
+
|
| 158 |
+
Returns:
|
| 159 |
+
[`huggingface_hub.repocard.RepoCard`]: The RepoCard (or subclass) initialized from the repo's
|
| 160 |
+
README.md file or filepath.
|
| 161 |
+
|
| 162 |
+
Example:
|
| 163 |
+
```python
|
| 164 |
+
>>> from huggingface_hub.repocard import RepoCard
|
| 165 |
+
>>> card = RepoCard.load("nateraw/food")
|
| 166 |
+
>>> assert card.data.tags == ["generated_from_trainer", "image-classification", "pytorch"]
|
| 167 |
+
|
| 168 |
+
```
|
| 169 |
+
"""
|
| 170 |
+
|
| 171 |
+
if Path(repo_id_or_path).is_file():
|
| 172 |
+
card_path = Path(repo_id_or_path)
|
| 173 |
+
elif isinstance(repo_id_or_path, str):
|
| 174 |
+
card_path = Path(
|
| 175 |
+
hf_hub_download(
|
| 176 |
+
repo_id_or_path,
|
| 177 |
+
constants.REPOCARD_NAME,
|
| 178 |
+
repo_type=repo_type or cls.repo_type,
|
| 179 |
+
token=token,
|
| 180 |
+
)
|
| 181 |
+
)
|
| 182 |
+
else:
|
| 183 |
+
raise ValueError(f"Cannot load RepoCard: path not found on disk ({repo_id_or_path}).")
|
| 184 |
+
|
| 185 |
+
# Preserve newlines in the existing file.
|
| 186 |
+
with card_path.open(mode="r", newline="", encoding="utf-8") as f:
|
| 187 |
+
return cls(f.read(), ignore_metadata_errors=ignore_metadata_errors)
|
| 188 |
+
|
| 189 |
+
def validate(self, repo_type: str | None = None):
|
| 190 |
+
"""Validates card against Hugging Face Hub's card validation logic.
|
| 191 |
+
Using this function requires access to the internet, so it is only called
|
| 192 |
+
internally by [`huggingface_hub.repocard.RepoCard.push_to_hub`].
|
| 193 |
+
|
| 194 |
+
Args:
|
| 195 |
+
repo_type (`str`, *optional*, defaults to "model"):
|
| 196 |
+
The type of Hugging Face repo to push to. Options are "model", "dataset", and "space".
|
| 197 |
+
If this function is called from a child class, the default will be the child class's `repo_type`.
|
| 198 |
+
|
| 199 |
+
> [!TIP]
|
| 200 |
+
> Raises the following errors:
|
| 201 |
+
>
|
| 202 |
+
> - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
| 203 |
+
> if the card fails validation checks.
|
| 204 |
+
> - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
|
| 205 |
+
> if the request to the Hub API fails for any other reason.
|
| 206 |
+
"""
|
| 207 |
+
|
| 208 |
+
# If repo type is provided, otherwise, use the repo type of the card.
|
| 209 |
+
repo_type = repo_type or self.repo_type
|
| 210 |
+
|
| 211 |
+
body = {
|
| 212 |
+
"repoType": repo_type,
|
| 213 |
+
"content": str(self),
|
| 214 |
+
}
|
| 215 |
+
headers = {"Accept": "text/plain"}
|
| 216 |
+
|
| 217 |
+
try:
|
| 218 |
+
response = get_session().post("https://huggingface.co/api/validate-yaml", json=body, headers=headers)
|
| 219 |
+
hf_raise_for_status(response)
|
| 220 |
+
except HfHubHTTPError as exc:
|
| 221 |
+
if response.status_code == 400:
|
| 222 |
+
raise ValueError(response.text)
|
| 223 |
+
else:
|
| 224 |
+
raise exc
|
| 225 |
+
|
| 226 |
+
def push_to_hub(
|
| 227 |
+
self,
|
| 228 |
+
repo_id: str,
|
| 229 |
+
token: str | None = None,
|
| 230 |
+
repo_type: str | None = None,
|
| 231 |
+
commit_message: str | None = None,
|
| 232 |
+
commit_description: str | None = None,
|
| 233 |
+
revision: str | None = None,
|
| 234 |
+
create_pr: bool | None = None,
|
| 235 |
+
parent_commit: str | None = None,
|
| 236 |
+
):
|
| 237 |
+
"""Push a RepoCard to a Hugging Face Hub repo.
|
| 238 |
+
|
| 239 |
+
Args:
|
| 240 |
+
repo_id (`str`):
|
| 241 |
+
The repo ID of the Hugging Face Hub repo to push to. Example: "nateraw/food".
|
| 242 |
+
token (`str`, *optional*):
|
| 243 |
+
Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to
|
| 244 |
+
the stored token.
|
| 245 |
+
repo_type (`str`, *optional*, defaults to "model"):
|
| 246 |
+
The type of Hugging Face repo to push to. Options are "model", "dataset", and "space". If this
|
| 247 |
+
function is called by a child class, it will default to the child class's `repo_type`.
|
| 248 |
+
commit_message (`str`, *optional*):
|
| 249 |
+
The summary / title / first line of the generated commit.
|
| 250 |
+
commit_description (`str`, *optional*)
|
| 251 |
+
The description of the generated commit.
|
| 252 |
+
revision (`str`, *optional*):
|
| 253 |
+
The git revision to commit from. Defaults to the head of the `"main"` branch.
|
| 254 |
+
create_pr (`bool`, *optional*):
|
| 255 |
+
Whether or not to create a Pull Request with this commit. Defaults to `False`.
|
| 256 |
+
parent_commit (`str`, *optional*):
|
| 257 |
+
The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported.
|
| 258 |
+
If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
|
| 259 |
+
If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
|
| 260 |
+
Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
|
| 261 |
+
especially useful if the repo is updated / committed too concurrently.
|
| 262 |
+
Returns:
|
| 263 |
+
`str`: URL of the commit which updated the card metadata.
|
| 264 |
+
"""
|
| 265 |
+
|
| 266 |
+
# If repo type is provided, otherwise, use the repo type of the card.
|
| 267 |
+
repo_type = repo_type or self.repo_type
|
| 268 |
+
|
| 269 |
+
# Validate card before pushing to hub
|
| 270 |
+
self.validate(repo_type=repo_type)
|
| 271 |
+
|
| 272 |
+
with SoftTemporaryDirectory() as tmpdir:
|
| 273 |
+
tmp_path = Path(tmpdir) / constants.REPOCARD_NAME
|
| 274 |
+
tmp_path.write_text(str(self), encoding="utf-8")
|
| 275 |
+
url = upload_file(
|
| 276 |
+
path_or_fileobj=str(tmp_path),
|
| 277 |
+
path_in_repo=constants.REPOCARD_NAME,
|
| 278 |
+
repo_id=repo_id,
|
| 279 |
+
token=token,
|
| 280 |
+
repo_type=repo_type,
|
| 281 |
+
commit_message=commit_message,
|
| 282 |
+
commit_description=commit_description,
|
| 283 |
+
create_pr=create_pr,
|
| 284 |
+
revision=revision,
|
| 285 |
+
parent_commit=parent_commit,
|
| 286 |
+
)
|
| 287 |
+
return url
|
| 288 |
+
|
| 289 |
+
@classmethod
|
| 290 |
+
def from_template(
|
| 291 |
+
cls,
|
| 292 |
+
card_data: CardData,
|
| 293 |
+
template_path: str | None = None,
|
| 294 |
+
template_str: str | None = None,
|
| 295 |
+
**template_kwargs,
|
| 296 |
+
):
|
| 297 |
+
"""Initialize a RepoCard from a template. By default, it uses the default template.
|
| 298 |
+
|
| 299 |
+
Templates are Jinja2 templates that can be customized by passing keyword arguments.
|
| 300 |
+
|
| 301 |
+
Args:
|
| 302 |
+
card_data (`huggingface_hub.CardData`):
|
| 303 |
+
A huggingface_hub.CardData instance containing the metadata you want to include in the YAML
|
| 304 |
+
header of the repo card on the Hugging Face Hub.
|
| 305 |
+
template_path (`str`, *optional*):
|
| 306 |
+
A path to a markdown file with optional Jinja template variables that can be filled
|
| 307 |
+
in with `template_kwargs`. Defaults to the default template.
|
| 308 |
+
template_str (`str`, *optional*):
|
| 309 |
+
A raw Jinja template string with optional variables. Used when neither `template_path`
|
| 310 |
+
nor the default template is appropriate. Ignored if `template_path` is also provided.
|
| 311 |
+
|
| 312 |
+
Returns:
|
| 313 |
+
[`huggingface_hub.repocard.RepoCard`]: A RepoCard instance with the specified card data and content from the
|
| 314 |
+
template.
|
| 315 |
+
"""
|
| 316 |
+
if is_jinja_available():
|
| 317 |
+
import jinja2
|
| 318 |
+
else:
|
| 319 |
+
raise ImportError(
|
| 320 |
+
"Using RepoCard.from_template requires Jinja2 to be installed. Please"
|
| 321 |
+
" install it with `pip install Jinja2`."
|
| 322 |
+
)
|
| 323 |
+
|
| 324 |
+
kwargs = card_data.to_dict().copy()
|
| 325 |
+
kwargs.update(template_kwargs) # Template_kwargs have priority
|
| 326 |
+
|
| 327 |
+
if template_path is not None:
|
| 328 |
+
template_str = Path(template_path).read_text()
|
| 329 |
+
if template_str is None:
|
| 330 |
+
template_str = Path(cls.default_template_path).read_text()
|
| 331 |
+
template = jinja2.Template(template_str)
|
| 332 |
+
content = template.render(card_data=card_data.to_yaml(), **kwargs)
|
| 333 |
+
return cls(content)
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
class ModelCard(RepoCard):
|
| 337 |
+
card_data_class = ModelCardData # type: ignore[assignment]
|
| 338 |
+
default_template_path = TEMPLATE_MODELCARD_PATH
|
| 339 |
+
repo_type = "model"
|
| 340 |
+
|
| 341 |
+
@classmethod
|
| 342 |
+
def from_template( # type: ignore # violates Liskov property but easier to use
|
| 343 |
+
cls,
|
| 344 |
+
card_data: ModelCardData,
|
| 345 |
+
template_path: str | None = None,
|
| 346 |
+
template_str: str | None = None,
|
| 347 |
+
**template_kwargs,
|
| 348 |
+
):
|
| 349 |
+
"""Initialize a ModelCard from a template. By default, it uses the default template, which can be found here:
|
| 350 |
+
https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/modelcard_template.md
|
| 351 |
+
|
| 352 |
+
Templates are Jinja2 templates that can be customized by passing keyword arguments.
|
| 353 |
+
|
| 354 |
+
Args:
|
| 355 |
+
card_data (`huggingface_hub.ModelCardData`):
|
| 356 |
+
A huggingface_hub.ModelCardData instance containing the metadata you want to include in the YAML
|
| 357 |
+
header of the model card on the Hugging Face Hub.
|
| 358 |
+
template_path (`str`, *optional*):
|
| 359 |
+
A path to a markdown file with optional Jinja template variables that can be filled
|
| 360 |
+
in with `template_kwargs`. Defaults to the default template.
|
| 361 |
+
template_str (`str`, *optional*):
|
| 362 |
+
A raw Jinja template string with optional variables. Used when neither `template_path`
|
| 363 |
+
nor the default template is appropriate. Ignored if `template_path` is also provided.
|
| 364 |
+
|
| 365 |
+
Returns:
|
| 366 |
+
[`huggingface_hub.ModelCard`]: A ModelCard instance with the specified card data and content from the
|
| 367 |
+
template.
|
| 368 |
+
|
| 369 |
+
Example:
|
| 370 |
+
```python
|
| 371 |
+
>>> from huggingface_hub import ModelCard, ModelCardData, EvalResult
|
| 372 |
+
|
| 373 |
+
>>> # Using the Default Template
|
| 374 |
+
>>> card_data = ModelCardData(
|
| 375 |
+
... language='en',
|
| 376 |
+
... license='mit',
|
| 377 |
+
... library_name='timm',
|
| 378 |
+
... tags=['image-classification', 'resnet'],
|
| 379 |
+
... datasets=['beans'],
|
| 380 |
+
... metrics=['accuracy'],
|
| 381 |
+
... )
|
| 382 |
+
>>> card = ModelCard.from_template(
|
| 383 |
+
... card_data,
|
| 384 |
+
... model_description='This model does x + y...'
|
| 385 |
+
... )
|
| 386 |
+
|
| 387 |
+
>>> # Including Evaluation Results
|
| 388 |
+
>>> card_data = ModelCardData(
|
| 389 |
+
... language='en',
|
| 390 |
+
... tags=['image-classification', 'resnet'],
|
| 391 |
+
... eval_results=[
|
| 392 |
+
... EvalResult(
|
| 393 |
+
... task_type='image-classification',
|
| 394 |
+
... dataset_type='beans',
|
| 395 |
+
... dataset_name='Beans',
|
| 396 |
+
... metric_type='accuracy',
|
| 397 |
+
... metric_value=0.9,
|
| 398 |
+
... ),
|
| 399 |
+
... ],
|
| 400 |
+
... model_name='my-cool-model',
|
| 401 |
+
... )
|
| 402 |
+
>>> card = ModelCard.from_template(card_data)
|
| 403 |
+
|
| 404 |
+
>>> # Using a Custom Template
|
| 405 |
+
>>> card_data = ModelCardData(
|
| 406 |
+
... language='en',
|
| 407 |
+
... tags=['image-classification', 'resnet']
|
| 408 |
+
... )
|
| 409 |
+
>>> card = ModelCard.from_template(
|
| 410 |
+
... card_data=card_data,
|
| 411 |
+
... template_path='./src/huggingface_hub/templates/modelcard_template.md',
|
| 412 |
+
... custom_template_var='custom value', # will be replaced in template if it exists
|
| 413 |
+
... )
|
| 414 |
+
|
| 415 |
+
```
|
| 416 |
+
"""
|
| 417 |
+
return super().from_template(card_data, template_path, template_str, **template_kwargs)
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
class DatasetCard(RepoCard):
|
| 421 |
+
card_data_class = DatasetCardData # type: ignore[assignment]
|
| 422 |
+
default_template_path = TEMPLATE_DATASETCARD_PATH
|
| 423 |
+
repo_type = "dataset"
|
| 424 |
+
|
| 425 |
+
@classmethod
|
| 426 |
+
def from_template( # type: ignore # violates Liskov property but easier to use
|
| 427 |
+
cls,
|
| 428 |
+
card_data: DatasetCardData,
|
| 429 |
+
template_path: str | None = None,
|
| 430 |
+
template_str: str | None = None,
|
| 431 |
+
**template_kwargs,
|
| 432 |
+
):
|
| 433 |
+
"""Initialize a DatasetCard from a template. By default, it uses the default template, which can be found here:
|
| 434 |
+
https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/datasetcard_template.md
|
| 435 |
+
|
| 436 |
+
Templates are Jinja2 templates that can be customized by passing keyword arguments.
|
| 437 |
+
|
| 438 |
+
Args:
|
| 439 |
+
card_data (`huggingface_hub.DatasetCardData`):
|
| 440 |
+
A huggingface_hub.DatasetCardData instance containing the metadata you want to include in the YAML
|
| 441 |
+
header of the dataset card on the Hugging Face Hub.
|
| 442 |
+
template_path (`str`, *optional*):
|
| 443 |
+
A path to a markdown file with optional Jinja template variables that can be filled
|
| 444 |
+
in with `template_kwargs`. Defaults to the default template.
|
| 445 |
+
template_str (`str`, *optional*):
|
| 446 |
+
A raw Jinja template string with optional variables. Used when neither `template_path`
|
| 447 |
+
nor the default template is appropriate. Ignored if `template_path` is also provided.
|
| 448 |
+
|
| 449 |
+
Returns:
|
| 450 |
+
[`huggingface_hub.DatasetCard`]: A DatasetCard instance with the specified card data and content from the
|
| 451 |
+
template.
|
| 452 |
+
|
| 453 |
+
Example:
|
| 454 |
+
```python
|
| 455 |
+
>>> from huggingface_hub import DatasetCard, DatasetCardData
|
| 456 |
+
|
| 457 |
+
>>> # Using the Default Template
|
| 458 |
+
>>> card_data = DatasetCardData(
|
| 459 |
+
... language='en',
|
| 460 |
+
... license='mit',
|
| 461 |
+
... annotations_creators='crowdsourced',
|
| 462 |
+
... task_categories=['text-classification'],
|
| 463 |
+
... task_ids=['sentiment-classification', 'text-scoring'],
|
| 464 |
+
... multilinguality='monolingual',
|
| 465 |
+
... pretty_name='My Text Classification Dataset',
|
| 466 |
+
... )
|
| 467 |
+
>>> card = DatasetCard.from_template(
|
| 468 |
+
... card_data,
|
| 469 |
+
... pretty_name=card_data.pretty_name,
|
| 470 |
+
... )
|
| 471 |
+
|
| 472 |
+
>>> # Using a Custom Template
|
| 473 |
+
>>> card_data = DatasetCardData(
|
| 474 |
+
... language='en',
|
| 475 |
+
... license='mit',
|
| 476 |
+
... )
|
| 477 |
+
>>> card = DatasetCard.from_template(
|
| 478 |
+
... card_data=card_data,
|
| 479 |
+
... template_path='./src/huggingface_hub/templates/datasetcard_template.md',
|
| 480 |
+
... custom_template_var='custom value', # will be replaced in template if it exists
|
| 481 |
+
... )
|
| 482 |
+
|
| 483 |
+
```
|
| 484 |
+
"""
|
| 485 |
+
return super().from_template(card_data, template_path, template_str, **template_kwargs)
|
| 486 |
+
|
| 487 |
+
|
| 488 |
+
class SpaceCard(RepoCard):
|
| 489 |
+
card_data_class = SpaceCardData # type: ignore[assignment]
|
| 490 |
+
default_template_path = TEMPLATE_MODELCARD_PATH
|
| 491 |
+
repo_type = "space"
|
| 492 |
+
|
| 493 |
+
|
| 494 |
+
def _detect_line_ending(content: str) -> Literal["\r", "\n", "\r\n", None]: # noqa: F722
|
| 495 |
+
"""Detect the line ending of a string. Used by RepoCard to avoid making huge diff on newlines.
|
| 496 |
+
|
| 497 |
+
Uses same implementation as in Hub server, keep it in sync.
|
| 498 |
+
|
| 499 |
+
Returns:
|
| 500 |
+
str: The detected line ending of the string.
|
| 501 |
+
"""
|
| 502 |
+
cr = content.count("\r")
|
| 503 |
+
lf = content.count("\n")
|
| 504 |
+
crlf = content.count("\r\n")
|
| 505 |
+
if cr + lf == 0:
|
| 506 |
+
return None
|
| 507 |
+
if crlf == cr and crlf == lf:
|
| 508 |
+
return "\r\n"
|
| 509 |
+
if cr > lf:
|
| 510 |
+
return "\r"
|
| 511 |
+
else:
|
| 512 |
+
return "\n"
|
| 513 |
+
|
| 514 |
+
|
| 515 |
+
def metadata_load(local_path: str | Path) -> dict | None:
|
| 516 |
+
content = Path(local_path).read_text()
|
| 517 |
+
match = REGEX_YAML_BLOCK.search(content)
|
| 518 |
+
if match:
|
| 519 |
+
yaml_block = match.group(2)
|
| 520 |
+
data = yaml.safe_load(yaml_block)
|
| 521 |
+
if data is None or isinstance(data, dict):
|
| 522 |
+
return data
|
| 523 |
+
raise ValueError("repo card metadata block should be a dict")
|
| 524 |
+
else:
|
| 525 |
+
return None
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def metadata_save(local_path: str | Path, data: dict) -> None:
|
| 529 |
+
"""
|
| 530 |
+
Save the metadata dict in the upper YAML part Trying to preserve newlines as
|
| 531 |
+
in the existing file. Docs about open() with newline="" parameter:
|
| 532 |
+
https://docs.python.org/3/library/functions.html?highlight=open#open Does
|
| 533 |
+
not work with "^M" linebreaks, which are replaced by \n
|
| 534 |
+
"""
|
| 535 |
+
line_break = "\n"
|
| 536 |
+
content = ""
|
| 537 |
+
# try to detect existing newline character
|
| 538 |
+
if os.path.exists(local_path):
|
| 539 |
+
with open(local_path, newline="", encoding="utf8") as readme:
|
| 540 |
+
content = readme.read()
|
| 541 |
+
if isinstance(readme.newlines, tuple):
|
| 542 |
+
line_break = readme.newlines[0]
|
| 543 |
+
elif isinstance(readme.newlines, str):
|
| 544 |
+
line_break = readme.newlines
|
| 545 |
+
|
| 546 |
+
# creates a new file if it not
|
| 547 |
+
with open(local_path, "w", newline="", encoding="utf8") as readme:
|
| 548 |
+
data_yaml = yaml_dump(data, sort_keys=False, line_break=line_break)
|
| 549 |
+
# sort_keys: keep dict order
|
| 550 |
+
match = REGEX_YAML_BLOCK.search(content)
|
| 551 |
+
if match:
|
| 552 |
+
output = content[: match.start()] + f"---{line_break}{data_yaml}---{line_break}" + content[match.end() :]
|
| 553 |
+
else:
|
| 554 |
+
output = f"---{line_break}{data_yaml}---{line_break}{content}"
|
| 555 |
+
|
| 556 |
+
readme.write(output)
|
| 557 |
+
readme.close()
|
| 558 |
+
|
| 559 |
+
|
| 560 |
+
def metadata_eval_result(
|
| 561 |
+
*,
|
| 562 |
+
model_pretty_name: str,
|
| 563 |
+
task_pretty_name: str,
|
| 564 |
+
task_id: str,
|
| 565 |
+
metrics_pretty_name: str,
|
| 566 |
+
metrics_id: str,
|
| 567 |
+
metrics_value: Any,
|
| 568 |
+
dataset_pretty_name: str,
|
| 569 |
+
dataset_id: str,
|
| 570 |
+
metrics_config: str | None = None,
|
| 571 |
+
metrics_verified: bool = False,
|
| 572 |
+
dataset_config: str | None = None,
|
| 573 |
+
dataset_split: str | None = None,
|
| 574 |
+
dataset_revision: str | None = None,
|
| 575 |
+
metrics_verification_token: str | None = None,
|
| 576 |
+
) -> dict:
|
| 577 |
+
"""
|
| 578 |
+
Creates a metadata dict with the result from a model evaluated on a dataset.
|
| 579 |
+
|
| 580 |
+
Args:
|
| 581 |
+
model_pretty_name (`str`):
|
| 582 |
+
The name of the model in natural language.
|
| 583 |
+
task_pretty_name (`str`):
|
| 584 |
+
The name of a task in natural language.
|
| 585 |
+
task_id (`str`):
|
| 586 |
+
Example: automatic-speech-recognition. A task id.
|
| 587 |
+
metrics_pretty_name (`str`):
|
| 588 |
+
A name for the metric in natural language. Example: Test WER.
|
| 589 |
+
metrics_id (`str`):
|
| 590 |
+
Example: wer. A metric id from https://hf.co/metrics.
|
| 591 |
+
metrics_value (`Any`):
|
| 592 |
+
The value from the metric. Example: 20.0 or "20.0 ± 1.2".
|
| 593 |
+
dataset_pretty_name (`str`):
|
| 594 |
+
The name of the dataset in natural language.
|
| 595 |
+
dataset_id (`str`):
|
| 596 |
+
Example: common_voice. A dataset id from https://hf.co/datasets.
|
| 597 |
+
metrics_config (`str`, *optional*):
|
| 598 |
+
The name of the metric configuration used in `load_metric()`.
|
| 599 |
+
Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.
|
| 600 |
+
metrics_verified (`bool`, *optional*, defaults to `False`):
|
| 601 |
+
Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.
|
| 602 |
+
dataset_config (`str`, *optional*):
|
| 603 |
+
Example: fr. The name of the dataset configuration used in `load_dataset()`.
|
| 604 |
+
dataset_split (`str`, *optional*):
|
| 605 |
+
Example: test. The name of the dataset split used in `load_dataset()`.
|
| 606 |
+
dataset_revision (`str`, *optional*):
|
| 607 |
+
Example: 5503434ddd753f426f4b38109466949a1217c2bb. The name of the dataset dataset revision
|
| 608 |
+
used in `load_dataset()`.
|
| 609 |
+
metrics_verification_token (`bool`, *optional*):
|
| 610 |
+
A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.
|
| 611 |
+
|
| 612 |
+
Returns:
|
| 613 |
+
`dict`: a metadata dict with the result from a model evaluated on a dataset.
|
| 614 |
+
|
| 615 |
+
Example:
|
| 616 |
+
```python
|
| 617 |
+
>>> from huggingface_hub import metadata_eval_result
|
| 618 |
+
>>> results = metadata_eval_result(
|
| 619 |
+
... model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
|
| 620 |
+
... task_pretty_name="Text Classification",
|
| 621 |
+
... task_id="text-classification",
|
| 622 |
+
... metrics_pretty_name="Accuracy",
|
| 623 |
+
... metrics_id="accuracy",
|
| 624 |
+
... metrics_value=0.2662102282047272,
|
| 625 |
+
... dataset_pretty_name="ReactionJPEG",
|
| 626 |
+
... dataset_id="julien-c/reactionjpeg",
|
| 627 |
+
... dataset_config="default",
|
| 628 |
+
... dataset_split="test",
|
| 629 |
+
... )
|
| 630 |
+
>>> results == {
|
| 631 |
+
... 'model-index': [
|
| 632 |
+
... {
|
| 633 |
+
... 'name': 'RoBERTa fine-tuned on ReactionGIF',
|
| 634 |
+
... 'results': [
|
| 635 |
+
... {
|
| 636 |
+
... 'task': {
|
| 637 |
+
... 'type': 'text-classification',
|
| 638 |
+
... 'name': 'Text Classification'
|
| 639 |
+
... },
|
| 640 |
+
... 'dataset': {
|
| 641 |
+
... 'name': 'ReactionJPEG',
|
| 642 |
+
... 'type': 'julien-c/reactionjpeg',
|
| 643 |
+
... 'config': 'default',
|
| 644 |
+
... 'split': 'test'
|
| 645 |
+
... },
|
| 646 |
+
... 'metrics': [
|
| 647 |
+
... {
|
| 648 |
+
... 'type': 'accuracy',
|
| 649 |
+
... 'value': 0.2662102282047272,
|
| 650 |
+
... 'name': 'Accuracy',
|
| 651 |
+
... 'verified': False
|
| 652 |
+
... }
|
| 653 |
+
... ]
|
| 654 |
+
... }
|
| 655 |
+
... ]
|
| 656 |
+
... }
|
| 657 |
+
... ]
|
| 658 |
+
... }
|
| 659 |
+
True
|
| 660 |
+
|
| 661 |
+
```
|
| 662 |
+
"""
|
| 663 |
+
|
| 664 |
+
return {
|
| 665 |
+
"model-index": eval_results_to_model_index(
|
| 666 |
+
model_name=model_pretty_name,
|
| 667 |
+
eval_results=[
|
| 668 |
+
EvalResult(
|
| 669 |
+
task_name=task_pretty_name,
|
| 670 |
+
task_type=task_id,
|
| 671 |
+
metric_name=metrics_pretty_name,
|
| 672 |
+
metric_type=metrics_id,
|
| 673 |
+
metric_value=metrics_value,
|
| 674 |
+
dataset_name=dataset_pretty_name,
|
| 675 |
+
dataset_type=dataset_id,
|
| 676 |
+
metric_config=metrics_config,
|
| 677 |
+
verified=metrics_verified,
|
| 678 |
+
verify_token=metrics_verification_token,
|
| 679 |
+
dataset_config=dataset_config,
|
| 680 |
+
dataset_split=dataset_split,
|
| 681 |
+
dataset_revision=dataset_revision,
|
| 682 |
+
)
|
| 683 |
+
],
|
| 684 |
+
)
|
| 685 |
+
}
|
| 686 |
+
|
| 687 |
+
|
| 688 |
+
@validate_hf_hub_args
|
| 689 |
+
def metadata_update(
|
| 690 |
+
repo_id: str,
|
| 691 |
+
metadata: dict,
|
| 692 |
+
*,
|
| 693 |
+
repo_type: str | None = None,
|
| 694 |
+
overwrite: bool = False,
|
| 695 |
+
token: str | None = None,
|
| 696 |
+
commit_message: str | None = None,
|
| 697 |
+
commit_description: str | None = None,
|
| 698 |
+
revision: str | None = None,
|
| 699 |
+
create_pr: bool = False,
|
| 700 |
+
parent_commit: str | None = None,
|
| 701 |
+
) -> str:
|
| 702 |
+
"""
|
| 703 |
+
Updates the metadata in the README.md of a repository on the Hugging Face Hub.
|
| 704 |
+
If the README.md file doesn't exist yet, a new one is created with metadata and
|
| 705 |
+
the default ModelCard or DatasetCard template. For `space` repo, an error is thrown
|
| 706 |
+
as a Space cannot exist without a `README.md` file.
|
| 707 |
+
|
| 708 |
+
Args:
|
| 709 |
+
repo_id (`str`):
|
| 710 |
+
The name of the repository.
|
| 711 |
+
metadata (`dict`):
|
| 712 |
+
A dictionary containing the metadata to be updated.
|
| 713 |
+
repo_type (`str`, *optional*):
|
| 714 |
+
Set to `"dataset"` or `"space"` if updating to a dataset or space,
|
| 715 |
+
`None` or `"model"` if updating to a model. Default is `None`.
|
| 716 |
+
overwrite (`bool`, *optional*, defaults to `False`):
|
| 717 |
+
If set to `True` an existing field can be overwritten, otherwise
|
| 718 |
+
attempting to overwrite an existing field will cause an error.
|
| 719 |
+
token (`str`, *optional*):
|
| 720 |
+
The Hugging Face authentication token.
|
| 721 |
+
commit_message (`str`, *optional*):
|
| 722 |
+
The summary / title / first line of the generated commit. Defaults to
|
| 723 |
+
`f"Update metadata with huggingface_hub"`
|
| 724 |
+
commit_description (`str` *optional*)
|
| 725 |
+
The description of the generated commit
|
| 726 |
+
revision (`str`, *optional*):
|
| 727 |
+
The git revision to commit from. Defaults to the head of the
|
| 728 |
+
`"main"` branch.
|
| 729 |
+
create_pr (`boolean`, *optional*):
|
| 730 |
+
Whether or not to create a Pull Request from `revision` with that commit.
|
| 731 |
+
Defaults to `False`.
|
| 732 |
+
parent_commit (`str`, *optional*):
|
| 733 |
+
The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported.
|
| 734 |
+
If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
|
| 735 |
+
If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
|
| 736 |
+
Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
|
| 737 |
+
especially useful if the repo is updated / committed too concurrently.
|
| 738 |
+
Returns:
|
| 739 |
+
`str`: URL of the commit which updated the card metadata.
|
| 740 |
+
|
| 741 |
+
Example:
|
| 742 |
+
```python
|
| 743 |
+
>>> from huggingface_hub import metadata_update
|
| 744 |
+
>>> metadata = {'model-index': [{'name': 'RoBERTa fine-tuned on ReactionGIF',
|
| 745 |
+
... 'results': [{'dataset': {'name': 'ReactionGIF',
|
| 746 |
+
... 'type': 'julien-c/reactiongif'},
|
| 747 |
+
... 'metrics': [{'name': 'Recall',
|
| 748 |
+
... 'type': 'recall',
|
| 749 |
+
... 'value': 0.7762102282047272}],
|
| 750 |
+
... 'task': {'name': 'Text Classification',
|
| 751 |
+
... 'type': 'text-classification'}}]}]}
|
| 752 |
+
>>> url = metadata_update("hf-internal-testing/reactiongif-roberta-card", metadata)
|
| 753 |
+
|
| 754 |
+
```
|
| 755 |
+
"""
|
| 756 |
+
commit_message = commit_message if commit_message is not None else "Update metadata with huggingface_hub"
|
| 757 |
+
|
| 758 |
+
# Card class given repo_type
|
| 759 |
+
card_class: type[RepoCard]
|
| 760 |
+
if repo_type is None or repo_type == "model":
|
| 761 |
+
card_class = ModelCard
|
| 762 |
+
elif repo_type == "dataset":
|
| 763 |
+
card_class = DatasetCard
|
| 764 |
+
elif repo_type == "space":
|
| 765 |
+
card_class = RepoCard
|
| 766 |
+
else:
|
| 767 |
+
raise ValueError(f"Unknown repo_type: {repo_type}")
|
| 768 |
+
|
| 769 |
+
# Either load repo_card from the Hub or create an empty one.
|
| 770 |
+
# NOTE: Will not create the repo if it doesn't exist.
|
| 771 |
+
try:
|
| 772 |
+
card = card_class.load(repo_id, token=token, repo_type=repo_type)
|
| 773 |
+
except EntryNotFoundError:
|
| 774 |
+
if repo_type == "space":
|
| 775 |
+
raise ValueError("Cannot update metadata on a Space that doesn't contain a `README.md` file.")
|
| 776 |
+
|
| 777 |
+
# Initialize a ModelCard or DatasetCard from default template and no data.
|
| 778 |
+
# Cast to the concrete expected card type to satisfy type checkers.
|
| 779 |
+
card = card_class.from_template(CardData()) # type: ignore
|
| 780 |
+
|
| 781 |
+
for key, value in metadata.items():
|
| 782 |
+
if key == "model-index":
|
| 783 |
+
# if the new metadata doesn't include a name, either use existing one or repo name
|
| 784 |
+
if "name" not in value[0]:
|
| 785 |
+
value[0]["name"] = getattr(card, "model_name", repo_id)
|
| 786 |
+
model_name, new_results = model_index_to_eval_results(value)
|
| 787 |
+
if card.data.eval_results is None:
|
| 788 |
+
card.data.eval_results = new_results
|
| 789 |
+
card.data.model_name = model_name
|
| 790 |
+
else:
|
| 791 |
+
existing_results = card.data.eval_results
|
| 792 |
+
|
| 793 |
+
# Iterate over new results
|
| 794 |
+
# Iterate over existing results
|
| 795 |
+
# If both results describe the same metric but value is different:
|
| 796 |
+
# If overwrite=True: overwrite the metric value
|
| 797 |
+
# Else: raise ValueError
|
| 798 |
+
# Else: append new result to existing ones.
|
| 799 |
+
for new_result in new_results:
|
| 800 |
+
result_found = False
|
| 801 |
+
for existing_result in existing_results:
|
| 802 |
+
if new_result.is_equal_except_value(existing_result):
|
| 803 |
+
if new_result != existing_result and not overwrite:
|
| 804 |
+
raise ValueError(
|
| 805 |
+
"You passed a new value for the existing metric"
|
| 806 |
+
f" 'name: {new_result.metric_name}, type: "
|
| 807 |
+
f"{new_result.metric_type}'. Set `overwrite=True`"
|
| 808 |
+
" to overwrite existing metrics."
|
| 809 |
+
)
|
| 810 |
+
result_found = True
|
| 811 |
+
existing_result.metric_value = new_result.metric_value
|
| 812 |
+
if existing_result.verified is True:
|
| 813 |
+
existing_result.verify_token = new_result.verify_token
|
| 814 |
+
if not result_found:
|
| 815 |
+
card.data.eval_results.append(new_result)
|
| 816 |
+
else:
|
| 817 |
+
# Any metadata that is not a result metric
|
| 818 |
+
if card.data.get(key) is not None and not overwrite and card.data.get(key) != value:
|
| 819 |
+
raise ValueError(
|
| 820 |
+
f"You passed a new value for the existing meta data field '{key}'."
|
| 821 |
+
" Set `overwrite=True` to overwrite existing metadata."
|
| 822 |
+
)
|
| 823 |
+
else:
|
| 824 |
+
card.data[key] = value
|
| 825 |
+
|
| 826 |
+
return card.push_to_hub(
|
| 827 |
+
repo_id,
|
| 828 |
+
token=token,
|
| 829 |
+
repo_type=repo_type,
|
| 830 |
+
commit_message=commit_message,
|
| 831 |
+
commit_description=commit_description,
|
| 832 |
+
create_pr=create_pr,
|
| 833 |
+
revision=revision,
|
| 834 |
+
parent_commit=parent_commit,
|
| 835 |
+
)
|
.venv/lib/python3.13/site-packages/huggingface_hub/repocard_data.py
ADDED
|
@@ -0,0 +1,776 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
from collections import defaultdict
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import Any
|
| 5 |
+
|
| 6 |
+
from huggingface_hub.utils import logging, yaml_dump
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
logger = logging.get_logger(__name__)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@dataclass
|
| 13 |
+
class EvalResult:
|
| 14 |
+
"""
|
| 15 |
+
Flattened representation of individual evaluation results found in model-index of Model Cards.
|
| 16 |
+
|
| 17 |
+
For more information on the model-index spec, see https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1.
|
| 18 |
+
|
| 19 |
+
Args:
|
| 20 |
+
task_type (`str`):
|
| 21 |
+
The task identifier. Example: "image-classification".
|
| 22 |
+
dataset_type (`str`):
|
| 23 |
+
The dataset identifier. Example: "common_voice". Use dataset id from https://hf.co/datasets.
|
| 24 |
+
dataset_name (`str`):
|
| 25 |
+
A pretty name for the dataset. Example: "Common Voice (French)".
|
| 26 |
+
metric_type (`str`):
|
| 27 |
+
The metric identifier. Example: "wer". Use metric id from https://hf.co/metrics.
|
| 28 |
+
metric_value (`Any`):
|
| 29 |
+
The metric value. Example: 0.9 or "20.0 ± 1.2".
|
| 30 |
+
task_name (`str`, *optional*):
|
| 31 |
+
A pretty name for the task. Example: "Speech Recognition".
|
| 32 |
+
dataset_config (`str`, *optional*):
|
| 33 |
+
The name of the dataset configuration used in `load_dataset()`.
|
| 34 |
+
Example: fr in `load_dataset("common_voice", "fr")`. See the `datasets` docs for more info:
|
| 35 |
+
https://hf.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name
|
| 36 |
+
dataset_split (`str`, *optional*):
|
| 37 |
+
The split used in `load_dataset()`. Example: "test".
|
| 38 |
+
dataset_revision (`str`, *optional*):
|
| 39 |
+
The revision (AKA Git Sha) of the dataset used in `load_dataset()`.
|
| 40 |
+
Example: 5503434ddd753f426f4b38109466949a1217c2bb
|
| 41 |
+
dataset_args (`dict[str, Any]`, *optional*):
|
| 42 |
+
The arguments passed during `Metric.compute()`. Example for `bleu`: `{"max_order": 4}`
|
| 43 |
+
metric_name (`str`, *optional*):
|
| 44 |
+
A pretty name for the metric. Example: "Test WER".
|
| 45 |
+
metric_config (`str`, *optional*):
|
| 46 |
+
The name of the metric configuration used in `load_metric()`.
|
| 47 |
+
Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.
|
| 48 |
+
See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations
|
| 49 |
+
metric_args (`dict[str, Any]`, *optional*):
|
| 50 |
+
The arguments passed during `Metric.compute()`. Example for `bleu`: max_order: 4
|
| 51 |
+
verified (`bool`, *optional*):
|
| 52 |
+
Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.
|
| 53 |
+
verify_token (`str`, *optional*):
|
| 54 |
+
A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.
|
| 55 |
+
source_name (`str`, *optional*):
|
| 56 |
+
The name of the source of the evaluation result. Example: "Open LLM Leaderboard".
|
| 57 |
+
source_url (`str`, *optional*):
|
| 58 |
+
The URL of the source of the evaluation result. Example: "https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard".
|
| 59 |
+
"""
|
| 60 |
+
|
| 61 |
+
# Required
|
| 62 |
+
|
| 63 |
+
# The task identifier
|
| 64 |
+
# Example: automatic-speech-recognition
|
| 65 |
+
task_type: str
|
| 66 |
+
|
| 67 |
+
# The dataset identifier
|
| 68 |
+
# Example: common_voice. Use dataset id from https://hf.co/datasets
|
| 69 |
+
dataset_type: str
|
| 70 |
+
|
| 71 |
+
# A pretty name for the dataset.
|
| 72 |
+
# Example: Common Voice (French)
|
| 73 |
+
dataset_name: str
|
| 74 |
+
|
| 75 |
+
# The metric identifier
|
| 76 |
+
# Example: wer. Use metric id from https://hf.co/metrics
|
| 77 |
+
metric_type: str
|
| 78 |
+
|
| 79 |
+
# Value of the metric.
|
| 80 |
+
# Example: 20.0 or "20.0 ± 1.2"
|
| 81 |
+
metric_value: Any
|
| 82 |
+
|
| 83 |
+
# Optional
|
| 84 |
+
|
| 85 |
+
# A pretty name for the task.
|
| 86 |
+
# Example: Speech Recognition
|
| 87 |
+
task_name: str | None = None
|
| 88 |
+
|
| 89 |
+
# The name of the dataset configuration used in `load_dataset()`.
|
| 90 |
+
# Example: fr in `load_dataset("common_voice", "fr")`.
|
| 91 |
+
# See the `datasets` docs for more info:
|
| 92 |
+
# https://huggingface.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name
|
| 93 |
+
dataset_config: str | None = None
|
| 94 |
+
|
| 95 |
+
# The split used in `load_dataset()`.
|
| 96 |
+
# Example: test
|
| 97 |
+
dataset_split: str | None = None
|
| 98 |
+
|
| 99 |
+
# The revision (AKA Git Sha) of the dataset used in `load_dataset()`.
|
| 100 |
+
# Example: 5503434ddd753f426f4b38109466949a1217c2bb
|
| 101 |
+
dataset_revision: str | None = None
|
| 102 |
+
|
| 103 |
+
# The arguments passed during `Metric.compute()`.
|
| 104 |
+
# Example for `bleu`: max_order: 4
|
| 105 |
+
dataset_args: dict[str, Any] | None = None
|
| 106 |
+
|
| 107 |
+
# A pretty name for the metric.
|
| 108 |
+
# Example: Test WER
|
| 109 |
+
metric_name: str | None = None
|
| 110 |
+
|
| 111 |
+
# The name of the metric configuration used in `load_metric()`.
|
| 112 |
+
# Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.
|
| 113 |
+
# See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations
|
| 114 |
+
metric_config: str | None = None
|
| 115 |
+
|
| 116 |
+
# The arguments passed during `Metric.compute()`.
|
| 117 |
+
# Example for `bleu`: max_order: 4
|
| 118 |
+
metric_args: dict[str, Any] | None = None
|
| 119 |
+
|
| 120 |
+
# Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.
|
| 121 |
+
verified: bool | None = None
|
| 122 |
+
|
| 123 |
+
# A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.
|
| 124 |
+
verify_token: str | None = None
|
| 125 |
+
|
| 126 |
+
# The name of the source of the evaluation result.
|
| 127 |
+
# Example: Open LLM Leaderboard
|
| 128 |
+
source_name: str | None = None
|
| 129 |
+
|
| 130 |
+
# The URL of the source of the evaluation result.
|
| 131 |
+
# Example: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard
|
| 132 |
+
source_url: str | None = None
|
| 133 |
+
|
| 134 |
+
@property
|
| 135 |
+
def unique_identifier(self) -> tuple:
|
| 136 |
+
"""Returns a tuple that uniquely identifies this evaluation."""
|
| 137 |
+
return (
|
| 138 |
+
self.task_type,
|
| 139 |
+
self.dataset_type,
|
| 140 |
+
self.dataset_config,
|
| 141 |
+
self.dataset_split,
|
| 142 |
+
self.dataset_revision,
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
def is_equal_except_value(self, other: "EvalResult") -> bool:
|
| 146 |
+
"""
|
| 147 |
+
Return True if `self` and `other` describe exactly the same metric but with a
|
| 148 |
+
different value.
|
| 149 |
+
"""
|
| 150 |
+
for key, _ in self.__dict__.items():
|
| 151 |
+
if key == "metric_value":
|
| 152 |
+
continue
|
| 153 |
+
# For metrics computed by Hugging Face's evaluation service, `verify_token` is derived from `metric_value`,
|
| 154 |
+
# so we exclude it here in the comparison.
|
| 155 |
+
if key != "verify_token" and getattr(self, key) != getattr(other, key):
|
| 156 |
+
return False
|
| 157 |
+
return True
|
| 158 |
+
|
| 159 |
+
def __post_init__(self) -> None:
|
| 160 |
+
if self.source_name is not None and self.source_url is None:
|
| 161 |
+
raise ValueError("If `source_name` is provided, `source_url` must also be provided.")
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
@dataclass
|
| 165 |
+
class CardData:
|
| 166 |
+
"""Structure containing metadata from a RepoCard.
|
| 167 |
+
|
| 168 |
+
[`CardData`] is the parent class of [`ModelCardData`] and [`DatasetCardData`].
|
| 169 |
+
|
| 170 |
+
Metadata can be exported as a dictionary or YAML. Export can be customized to alter the representation of the data
|
| 171 |
+
(example: flatten evaluation results). `CardData` behaves as a dictionary (can get, pop, set values) but do not
|
| 172 |
+
inherit from `dict` to allow this export step.
|
| 173 |
+
"""
|
| 174 |
+
|
| 175 |
+
def __init__(self, ignore_metadata_errors: bool = False, **kwargs):
|
| 176 |
+
self.__dict__.update(kwargs)
|
| 177 |
+
|
| 178 |
+
def to_dict(self):
|
| 179 |
+
"""Converts CardData to a dict.
|
| 180 |
+
|
| 181 |
+
Returns:
|
| 182 |
+
`dict`: CardData represented as a dictionary ready to be dumped to a YAML
|
| 183 |
+
block for inclusion in a README.md file.
|
| 184 |
+
"""
|
| 185 |
+
|
| 186 |
+
data_dict = copy.deepcopy(self.__dict__)
|
| 187 |
+
self._to_dict(data_dict)
|
| 188 |
+
return {key: value for key, value in data_dict.items() if value is not None}
|
| 189 |
+
|
| 190 |
+
def _to_dict(self, data_dict):
|
| 191 |
+
"""Use this method in child classes to alter the dict representation of the data. Alter the dict in-place.
|
| 192 |
+
|
| 193 |
+
Args:
|
| 194 |
+
data_dict (`dict`): The raw dict representation of the card data.
|
| 195 |
+
"""
|
| 196 |
+
pass
|
| 197 |
+
|
| 198 |
+
def to_yaml(self, line_break=None, original_order: list[str] | None = None) -> str:
|
| 199 |
+
"""Dumps CardData to a YAML block for inclusion in a README.md file.
|
| 200 |
+
|
| 201 |
+
Args:
|
| 202 |
+
line_break (str, *optional*):
|
| 203 |
+
The line break to use when dumping to yaml.
|
| 204 |
+
original_order (`list[str]`, *optional*):
|
| 205 |
+
If provided, reorder the metadata fields to match this list before dumping.
|
| 206 |
+
Any keys not in `original_order` are appended after the listed keys, preserving
|
| 207 |
+
their existing relative order. Useful for round-tripping a YAML block without
|
| 208 |
+
shuffling its keys.
|
| 209 |
+
|
| 210 |
+
Returns:
|
| 211 |
+
`str`: CardData represented as a YAML block.
|
| 212 |
+
"""
|
| 213 |
+
if original_order:
|
| 214 |
+
original_order_set = set(original_order)
|
| 215 |
+
self.__dict__ = {
|
| 216 |
+
k: self.__dict__[k]
|
| 217 |
+
for k in original_order + [k for k in self.__dict__ if k not in original_order_set]
|
| 218 |
+
if k in self.__dict__
|
| 219 |
+
}
|
| 220 |
+
return yaml_dump(self.to_dict(), sort_keys=False, line_break=line_break).strip()
|
| 221 |
+
|
| 222 |
+
def __repr__(self):
|
| 223 |
+
return repr(self.__dict__)
|
| 224 |
+
|
| 225 |
+
def __str__(self):
|
| 226 |
+
return self.to_yaml()
|
| 227 |
+
|
| 228 |
+
def get(self, key: str, default: Any = None) -> Any:
|
| 229 |
+
"""Get value for a given metadata key."""
|
| 230 |
+
value = self.__dict__.get(key)
|
| 231 |
+
return default if value is None else value
|
| 232 |
+
|
| 233 |
+
def pop(self, key: str, default: Any = None) -> Any:
|
| 234 |
+
"""Pop value for a given metadata key."""
|
| 235 |
+
return self.__dict__.pop(key, default)
|
| 236 |
+
|
| 237 |
+
def __getitem__(self, key: str) -> Any:
|
| 238 |
+
"""Get value for a given metadata key."""
|
| 239 |
+
return self.__dict__[key]
|
| 240 |
+
|
| 241 |
+
def __setitem__(self, key: str, value: Any) -> None:
|
| 242 |
+
"""Set value for a given metadata key."""
|
| 243 |
+
self.__dict__[key] = value
|
| 244 |
+
|
| 245 |
+
def __contains__(self, key: str) -> bool:
|
| 246 |
+
"""Check if a given metadata key is set."""
|
| 247 |
+
return key in self.__dict__
|
| 248 |
+
|
| 249 |
+
def __len__(self) -> int:
|
| 250 |
+
"""Return the number of metadata keys set."""
|
| 251 |
+
return len(self.__dict__)
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def _validate_eval_results(
|
| 255 |
+
eval_results: EvalResult | list[EvalResult] | None,
|
| 256 |
+
model_name: str | None,
|
| 257 |
+
) -> list[EvalResult]:
|
| 258 |
+
if eval_results is None:
|
| 259 |
+
return []
|
| 260 |
+
if isinstance(eval_results, EvalResult):
|
| 261 |
+
eval_results = [eval_results]
|
| 262 |
+
if not isinstance(eval_results, list) or not all(isinstance(r, EvalResult) for r in eval_results):
|
| 263 |
+
raise ValueError(
|
| 264 |
+
f"`eval_results` should be of type `EvalResult` or a list of `EvalResult`, got {type(eval_results)}."
|
| 265 |
+
)
|
| 266 |
+
if model_name is None:
|
| 267 |
+
raise ValueError("Passing `eval_results` requires `model_name` to be set.")
|
| 268 |
+
return eval_results
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
class ModelCardData(CardData):
|
| 272 |
+
"""Model Card Metadata that is used by Hugging Face Hub when included at the top of your README.md
|
| 273 |
+
|
| 274 |
+
Args:
|
| 275 |
+
base_model (`str` or `list[str]`, *optional*):
|
| 276 |
+
The identifier of the base model from which the model derives. This is applicable for example if your model is a
|
| 277 |
+
fine-tune or adapter of an existing model. The value must be the ID of a model on the Hub (or a list of IDs
|
| 278 |
+
if your model derives from multiple models). Defaults to None.
|
| 279 |
+
datasets (`Union[str, list[str]]`, *optional*):
|
| 280 |
+
Dataset or list of datasets that were used to train this model. Should be a dataset ID
|
| 281 |
+
found on https://hf.co/datasets. Defaults to None.
|
| 282 |
+
eval_results (`Union[list[EvalResult], EvalResult]`, *optional*):
|
| 283 |
+
List of `huggingface_hub.EvalResult` that define evaluation results of the model. If provided,
|
| 284 |
+
`model_name` is used to as a name on PapersWithCode's leaderboards. Defaults to `None`.
|
| 285 |
+
language (`Union[str, list[str]]`, *optional*):
|
| 286 |
+
Language of model's training data or metadata. It must be an ISO 639-1, 639-2 or
|
| 287 |
+
639-3 code (two/three letters), or a special value like "code", "multilingual". Defaults to `None`.
|
| 288 |
+
library_name (`str`, *optional*):
|
| 289 |
+
Name of library used by this model. Example: keras or any library from
|
| 290 |
+
https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/model-libraries.ts.
|
| 291 |
+
Defaults to None.
|
| 292 |
+
license (`str`, *optional*):
|
| 293 |
+
License of this model. Example: apache-2.0 or any license from
|
| 294 |
+
https://huggingface.co/docs/hub/repositories-licenses. Defaults to None.
|
| 295 |
+
license_name (`str`, *optional*):
|
| 296 |
+
Name of the license of this model. Defaults to None. To be used in conjunction with `license_link`.
|
| 297 |
+
Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a name. In that case, use `license` instead.
|
| 298 |
+
license_link (`str`, *optional*):
|
| 299 |
+
Link to the license of this model. Defaults to None. To be used in conjunction with `license_name`.
|
| 300 |
+
Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a link. In that case, use `license` instead.
|
| 301 |
+
metrics (`list[str]`, *optional*):
|
| 302 |
+
List of metrics used to evaluate this model. Should be a metric name that can be found
|
| 303 |
+
at https://hf.co/metrics. Example: 'accuracy'. Defaults to None.
|
| 304 |
+
model_name (`str`, *optional*):
|
| 305 |
+
A name for this model. It is used along with
|
| 306 |
+
`eval_results` to construct the `model-index` within the card's metadata. The name
|
| 307 |
+
you supply here is what will be used on PapersWithCode's leaderboards. If None is provided
|
| 308 |
+
then the repo name is used as a default. Defaults to None.
|
| 309 |
+
pipeline_tag (`str`, *optional*):
|
| 310 |
+
The pipeline tag associated with the model. Example: "text-classification".
|
| 311 |
+
tags (`list[str]`, *optional*):
|
| 312 |
+
List of tags to add to your model that can be used when filtering on the Hugging
|
| 313 |
+
Face Hub. Defaults to None.
|
| 314 |
+
ignore_metadata_errors (`str`):
|
| 315 |
+
If True, errors while parsing the metadata section will be ignored. Some information might be lost during
|
| 316 |
+
the process. Use it at your own risk.
|
| 317 |
+
kwargs (`dict`, *optional*):
|
| 318 |
+
Additional metadata that will be added to the model card. Defaults to None.
|
| 319 |
+
|
| 320 |
+
Example:
|
| 321 |
+
```python
|
| 322 |
+
>>> from huggingface_hub import ModelCardData
|
| 323 |
+
>>> card_data = ModelCardData(
|
| 324 |
+
... language="en",
|
| 325 |
+
... license="mit",
|
| 326 |
+
... library_name="timm",
|
| 327 |
+
... tags=['image-classification', 'resnet'],
|
| 328 |
+
... )
|
| 329 |
+
>>> card_data.to_dict()
|
| 330 |
+
{'language': 'en', 'license': 'mit', 'library_name': 'timm', 'tags': ['image-classification', 'resnet']}
|
| 331 |
+
|
| 332 |
+
```
|
| 333 |
+
"""
|
| 334 |
+
|
| 335 |
+
def __init__(
|
| 336 |
+
self,
|
| 337 |
+
*,
|
| 338 |
+
base_model: str | list[str] | None = None,
|
| 339 |
+
datasets: str | list[str] | None = None,
|
| 340 |
+
eval_results: list[EvalResult] | None = None,
|
| 341 |
+
language: str | list[str] | None = None,
|
| 342 |
+
library_name: str | None = None,
|
| 343 |
+
license: str | None = None,
|
| 344 |
+
license_name: str | None = None,
|
| 345 |
+
license_link: str | None = None,
|
| 346 |
+
metrics: list[str] | None = None,
|
| 347 |
+
model_name: str | None = None,
|
| 348 |
+
pipeline_tag: str | None = None,
|
| 349 |
+
tags: list[str] | None = None,
|
| 350 |
+
ignore_metadata_errors: bool = False,
|
| 351 |
+
**kwargs,
|
| 352 |
+
):
|
| 353 |
+
self.base_model = base_model
|
| 354 |
+
self.datasets = datasets
|
| 355 |
+
self.eval_results = eval_results
|
| 356 |
+
self.language = language
|
| 357 |
+
self.library_name = library_name
|
| 358 |
+
self.license = license
|
| 359 |
+
self.license_name = license_name
|
| 360 |
+
self.license_link = license_link
|
| 361 |
+
self.metrics = metrics
|
| 362 |
+
self.model_name = model_name
|
| 363 |
+
self.pipeline_tag = pipeline_tag
|
| 364 |
+
self.tags = _to_unique_list(tags)
|
| 365 |
+
|
| 366 |
+
model_index = kwargs.pop("model-index", None)
|
| 367 |
+
if model_index:
|
| 368 |
+
try:
|
| 369 |
+
model_name, eval_results = model_index_to_eval_results(model_index)
|
| 370 |
+
self.model_name = model_name
|
| 371 |
+
self.eval_results = eval_results
|
| 372 |
+
except (KeyError, TypeError) as error:
|
| 373 |
+
if ignore_metadata_errors:
|
| 374 |
+
logger.warning("Invalid model-index. Not loading eval results into CardData.")
|
| 375 |
+
else:
|
| 376 |
+
raise ValueError(
|
| 377 |
+
f"Invalid `model_index` in metadata cannot be parsed: {error.__class__} {error}. Pass"
|
| 378 |
+
" `ignore_metadata_errors=True` to ignore this error while loading a Model Card. Warning:"
|
| 379 |
+
" some information will be lost. Use it at your own risk."
|
| 380 |
+
)
|
| 381 |
+
|
| 382 |
+
super().__init__(**kwargs)
|
| 383 |
+
|
| 384 |
+
if self.eval_results:
|
| 385 |
+
try:
|
| 386 |
+
self.eval_results = _validate_eval_results(self.eval_results, self.model_name)
|
| 387 |
+
except Exception as e:
|
| 388 |
+
if ignore_metadata_errors:
|
| 389 |
+
logger.warning(f"Failed to validate eval_results: {e}. Not loading eval results into CardData.")
|
| 390 |
+
else:
|
| 391 |
+
raise ValueError(f"Failed to validate eval_results: {e}") from e
|
| 392 |
+
|
| 393 |
+
def _to_dict(self, data_dict):
|
| 394 |
+
"""Format the internal data dict. In this case, we convert eval results to a valid model index"""
|
| 395 |
+
if self.eval_results is not None:
|
| 396 |
+
data_dict["model-index"] = eval_results_to_model_index(self.model_name, self.eval_results) # type: ignore
|
| 397 |
+
del data_dict["eval_results"], data_dict["model_name"]
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
class DatasetCardData(CardData):
|
| 401 |
+
"""Dataset Card Metadata that is used by Hugging Face Hub when included at the top of your README.md
|
| 402 |
+
|
| 403 |
+
Args:
|
| 404 |
+
language (`list[str]`, *optional*):
|
| 405 |
+
Language of dataset's data or metadata. It must be an ISO 639-1, 639-2 or
|
| 406 |
+
639-3 code (two/three letters), or a special value like "code", "multilingual".
|
| 407 |
+
license (`Union[str, list[str]]`, *optional*):
|
| 408 |
+
License(s) of this dataset. Example: apache-2.0 or any license from
|
| 409 |
+
https://huggingface.co/docs/hub/repositories-licenses.
|
| 410 |
+
annotations_creators (`Union[str, list[str]]`, *optional*):
|
| 411 |
+
How the annotations for the dataset were created.
|
| 412 |
+
Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'no-annotation', 'other'.
|
| 413 |
+
language_creators (`Union[str, list[str]]`, *optional*):
|
| 414 |
+
How the text-based data in the dataset was created.
|
| 415 |
+
Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'other'
|
| 416 |
+
multilinguality (`Union[str, list[str]]`, *optional*):
|
| 417 |
+
Whether the dataset is multilingual.
|
| 418 |
+
Options are: 'monolingual', 'multilingual', 'translation', 'other'.
|
| 419 |
+
size_categories (`Union[str, list[str]]`, *optional*):
|
| 420 |
+
The number of examples in the dataset. Options are: 'n<1K', '1K<n<10K', '10K<n<100K',
|
| 421 |
+
'100K<n<1M', '1M<n<10M', '10M<n<100M', '100M<n<1B', '1B<n<10B', '10B<n<100B', '100B<n<1T', 'n>1T', and 'other'.
|
| 422 |
+
source_datasets (`list[str]]`, *optional*):
|
| 423 |
+
Indicates whether the dataset is an original dataset or extended from another existing dataset.
|
| 424 |
+
Options are: 'original' and 'extended'.
|
| 425 |
+
task_categories (`Union[str, list[str]]`, *optional*):
|
| 426 |
+
What categories of task does the dataset support?
|
| 427 |
+
task_ids (`Union[str, list[str]]`, *optional*):
|
| 428 |
+
What specific tasks does the dataset support?
|
| 429 |
+
paperswithcode_id (`str`, *optional*):
|
| 430 |
+
ID of the dataset on PapersWithCode.
|
| 431 |
+
pretty_name (`str`, *optional*):
|
| 432 |
+
A more human-readable name for the dataset. (ex. "Cats vs. Dogs")
|
| 433 |
+
train_eval_index (`dict`, *optional*):
|
| 434 |
+
A dictionary that describes the necessary spec for doing evaluation on the Hub.
|
| 435 |
+
If not provided, it will be gathered from the 'train-eval-index' key of the kwargs.
|
| 436 |
+
config_names (`Union[str, list[str]]`, *optional*):
|
| 437 |
+
A list of the available dataset configs for the dataset.
|
| 438 |
+
"""
|
| 439 |
+
|
| 440 |
+
def __init__(
|
| 441 |
+
self,
|
| 442 |
+
*,
|
| 443 |
+
language: str | list[str] | None = None,
|
| 444 |
+
license: str | list[str] | None = None,
|
| 445 |
+
annotations_creators: str | list[str] | None = None,
|
| 446 |
+
language_creators: str | list[str] | None = None,
|
| 447 |
+
multilinguality: str | list[str] | None = None,
|
| 448 |
+
size_categories: str | list[str] | None = None,
|
| 449 |
+
source_datasets: list[str] | None = None,
|
| 450 |
+
task_categories: str | list[str] | None = None,
|
| 451 |
+
task_ids: str | list[str] | None = None,
|
| 452 |
+
paperswithcode_id: str | None = None,
|
| 453 |
+
pretty_name: str | None = None,
|
| 454 |
+
train_eval_index: dict | None = None,
|
| 455 |
+
config_names: str | list[str] | None = None,
|
| 456 |
+
ignore_metadata_errors: bool = False,
|
| 457 |
+
**kwargs,
|
| 458 |
+
):
|
| 459 |
+
self.annotations_creators = annotations_creators
|
| 460 |
+
self.language_creators = language_creators
|
| 461 |
+
self.language = language
|
| 462 |
+
self.license = license
|
| 463 |
+
self.multilinguality = multilinguality
|
| 464 |
+
self.size_categories = size_categories
|
| 465 |
+
self.source_datasets = source_datasets
|
| 466 |
+
self.task_categories = task_categories
|
| 467 |
+
self.task_ids = task_ids
|
| 468 |
+
self.paperswithcode_id = paperswithcode_id
|
| 469 |
+
self.pretty_name = pretty_name
|
| 470 |
+
self.config_names = config_names
|
| 471 |
+
|
| 472 |
+
# TODO - maybe handle this similarly to EvalResult?
|
| 473 |
+
self.train_eval_index = train_eval_index or kwargs.pop("train-eval-index", None)
|
| 474 |
+
super().__init__(**kwargs)
|
| 475 |
+
|
| 476 |
+
def _to_dict(self, data_dict):
|
| 477 |
+
data_dict["train-eval-index"] = data_dict.pop("train_eval_index")
|
| 478 |
+
|
| 479 |
+
|
| 480 |
+
class SpaceCardData(CardData):
|
| 481 |
+
"""Space Card Metadata that is used by Hugging Face Hub when included at the top of your README.md
|
| 482 |
+
|
| 483 |
+
To get an exhaustive reference of Spaces configuration, please visit https://huggingface.co/docs/hub/spaces-config-reference#spaces-configuration-reference.
|
| 484 |
+
|
| 485 |
+
Args:
|
| 486 |
+
title (`str`, *optional*)
|
| 487 |
+
Title of the Space.
|
| 488 |
+
sdk (`str`, *optional*)
|
| 489 |
+
SDK of the Space (one of `gradio`, `streamlit`, `docker`, or `static`).
|
| 490 |
+
sdk_version (`str`, *optional*)
|
| 491 |
+
Version of the used SDK (if Gradio/Streamlit sdk).
|
| 492 |
+
python_version (`str`, *optional*)
|
| 493 |
+
Python version used in the Space (if Gradio/Streamlit sdk).
|
| 494 |
+
app_file (`str`, *optional*)
|
| 495 |
+
Path to your main application file (which contains either gradio or streamlit Python code, or static html code).
|
| 496 |
+
Path is relative to the root of the repository.
|
| 497 |
+
app_port (`str`, *optional*)
|
| 498 |
+
Port on which your application is running. Used only if sdk is `docker`.
|
| 499 |
+
license (`str`, *optional*)
|
| 500 |
+
License of this model. Example: apache-2.0 or any license from
|
| 501 |
+
https://huggingface.co/docs/hub/repositories-licenses.
|
| 502 |
+
duplicated_from (`str`, *optional*)
|
| 503 |
+
ID of the original Space if this is a duplicated Space.
|
| 504 |
+
models (list[`str`], *optional*)
|
| 505 |
+
List of models related to this Space. Should be a dataset ID found on https://hf.co/models.
|
| 506 |
+
datasets (`list[str]`, *optional*)
|
| 507 |
+
List of datasets related to this Space. Should be a dataset ID found on https://hf.co/datasets.
|
| 508 |
+
tags (`list[str]`, *optional*)
|
| 509 |
+
List of tags to add to your Space that can be used when filtering on the Hub.
|
| 510 |
+
ignore_metadata_errors (`str`):
|
| 511 |
+
If True, errors while parsing the metadata section will be ignored. Some information might be lost during
|
| 512 |
+
the process. Use it at your own risk.
|
| 513 |
+
kwargs (`dict`, *optional*):
|
| 514 |
+
Additional metadata that will be added to the space card.
|
| 515 |
+
|
| 516 |
+
Example:
|
| 517 |
+
```python
|
| 518 |
+
>>> from huggingface_hub import SpaceCardData
|
| 519 |
+
>>> card_data = SpaceCardData(
|
| 520 |
+
... title="Dreambooth Training",
|
| 521 |
+
... license="mit",
|
| 522 |
+
... sdk="gradio",
|
| 523 |
+
... duplicated_from="multimodalart/dreambooth-training"
|
| 524 |
+
... )
|
| 525 |
+
>>> card_data.to_dict()
|
| 526 |
+
{'title': 'Dreambooth Training', 'sdk': 'gradio', 'license': 'mit', 'duplicated_from': 'multimodalart/dreambooth-training'}
|
| 527 |
+
```
|
| 528 |
+
"""
|
| 529 |
+
|
| 530 |
+
def __init__(
|
| 531 |
+
self,
|
| 532 |
+
*,
|
| 533 |
+
title: str | None = None,
|
| 534 |
+
sdk: str | None = None,
|
| 535 |
+
sdk_version: str | None = None,
|
| 536 |
+
python_version: str | None = None,
|
| 537 |
+
app_file: str | None = None,
|
| 538 |
+
app_port: int | None = None,
|
| 539 |
+
license: str | None = None,
|
| 540 |
+
duplicated_from: str | None = None,
|
| 541 |
+
models: list[str] | None = None,
|
| 542 |
+
datasets: list[str] | None = None,
|
| 543 |
+
tags: list[str] | None = None,
|
| 544 |
+
ignore_metadata_errors: bool = False,
|
| 545 |
+
**kwargs,
|
| 546 |
+
):
|
| 547 |
+
self.title = title
|
| 548 |
+
self.sdk = sdk
|
| 549 |
+
self.sdk_version = sdk_version
|
| 550 |
+
self.python_version = python_version
|
| 551 |
+
self.app_file = app_file
|
| 552 |
+
self.app_port = app_port
|
| 553 |
+
self.license = license
|
| 554 |
+
self.duplicated_from = duplicated_from
|
| 555 |
+
self.models = models
|
| 556 |
+
self.datasets = datasets
|
| 557 |
+
self.tags = _to_unique_list(tags)
|
| 558 |
+
super().__init__(**kwargs)
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
def model_index_to_eval_results(model_index: list[dict[str, Any]]) -> tuple[str, list[EvalResult]]:
|
| 562 |
+
"""Takes in a model index and returns the model name and a list of `huggingface_hub.EvalResult` objects.
|
| 563 |
+
|
| 564 |
+
A detailed spec of the model index can be found here:
|
| 565 |
+
https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1
|
| 566 |
+
|
| 567 |
+
Args:
|
| 568 |
+
model_index (`list[dict[str, Any]]`):
|
| 569 |
+
A model index data structure, likely coming from a README.md file on the
|
| 570 |
+
Hugging Face Hub.
|
| 571 |
+
|
| 572 |
+
Returns:
|
| 573 |
+
model_name (`str`):
|
| 574 |
+
The name of the model as found in the model index. This is used as the
|
| 575 |
+
identifier for the model on leaderboards like PapersWithCode.
|
| 576 |
+
eval_results (`list[EvalResult]`):
|
| 577 |
+
A list of `huggingface_hub.EvalResult` objects containing the metrics
|
| 578 |
+
reported in the provided model_index.
|
| 579 |
+
|
| 580 |
+
Example:
|
| 581 |
+
```python
|
| 582 |
+
>>> from huggingface_hub.repocard_data import model_index_to_eval_results
|
| 583 |
+
>>> # Define a minimal model index
|
| 584 |
+
>>> model_index = [
|
| 585 |
+
... {
|
| 586 |
+
... "name": "my-cool-model",
|
| 587 |
+
... "results": [
|
| 588 |
+
... {
|
| 589 |
+
... "task": {
|
| 590 |
+
... "type": "image-classification"
|
| 591 |
+
... },
|
| 592 |
+
... "dataset": {
|
| 593 |
+
... "type": "beans",
|
| 594 |
+
... "name": "Beans"
|
| 595 |
+
... },
|
| 596 |
+
... "metrics": [
|
| 597 |
+
... {
|
| 598 |
+
... "type": "accuracy",
|
| 599 |
+
... "value": 0.9
|
| 600 |
+
... }
|
| 601 |
+
... ]
|
| 602 |
+
... }
|
| 603 |
+
... ]
|
| 604 |
+
... }
|
| 605 |
+
... ]
|
| 606 |
+
>>> model_name, eval_results = model_index_to_eval_results(model_index)
|
| 607 |
+
>>> model_name
|
| 608 |
+
'my-cool-model'
|
| 609 |
+
>>> eval_results[0].task_type
|
| 610 |
+
'image-classification'
|
| 611 |
+
>>> eval_results[0].metric_type
|
| 612 |
+
'accuracy'
|
| 613 |
+
|
| 614 |
+
```
|
| 615 |
+
"""
|
| 616 |
+
|
| 617 |
+
eval_results = []
|
| 618 |
+
for elem in model_index:
|
| 619 |
+
name = elem["name"]
|
| 620 |
+
results = elem["results"]
|
| 621 |
+
for result in results:
|
| 622 |
+
task_type = result["task"]["type"]
|
| 623 |
+
task_name = result["task"].get("name")
|
| 624 |
+
dataset_type = result["dataset"]["type"]
|
| 625 |
+
dataset_name = result["dataset"]["name"]
|
| 626 |
+
dataset_config = result["dataset"].get("config")
|
| 627 |
+
dataset_split = result["dataset"].get("split")
|
| 628 |
+
dataset_revision = result["dataset"].get("revision")
|
| 629 |
+
dataset_args = result["dataset"].get("args")
|
| 630 |
+
source_name = result.get("source", {}).get("name")
|
| 631 |
+
source_url = result.get("source", {}).get("url")
|
| 632 |
+
|
| 633 |
+
for metric in result["metrics"]:
|
| 634 |
+
metric_type = metric["type"]
|
| 635 |
+
metric_value = metric["value"]
|
| 636 |
+
metric_name = metric.get("name")
|
| 637 |
+
metric_args = metric.get("args")
|
| 638 |
+
metric_config = metric.get("config")
|
| 639 |
+
verified = metric.get("verified")
|
| 640 |
+
verify_token = metric.get("verifyToken")
|
| 641 |
+
|
| 642 |
+
eval_result = EvalResult(
|
| 643 |
+
task_type=task_type, # Required
|
| 644 |
+
dataset_type=dataset_type, # Required
|
| 645 |
+
dataset_name=dataset_name, # Required
|
| 646 |
+
metric_type=metric_type, # Required
|
| 647 |
+
metric_value=metric_value, # Required
|
| 648 |
+
task_name=task_name,
|
| 649 |
+
dataset_config=dataset_config,
|
| 650 |
+
dataset_split=dataset_split,
|
| 651 |
+
dataset_revision=dataset_revision,
|
| 652 |
+
dataset_args=dataset_args,
|
| 653 |
+
metric_name=metric_name,
|
| 654 |
+
metric_args=metric_args,
|
| 655 |
+
metric_config=metric_config,
|
| 656 |
+
verified=verified,
|
| 657 |
+
verify_token=verify_token,
|
| 658 |
+
source_name=source_name,
|
| 659 |
+
source_url=source_url,
|
| 660 |
+
)
|
| 661 |
+
eval_results.append(eval_result)
|
| 662 |
+
return name, eval_results
|
| 663 |
+
|
| 664 |
+
|
| 665 |
+
def _remove_none(obj):
|
| 666 |
+
"""
|
| 667 |
+
Recursively remove `None` values from a dict. Borrowed from: https://stackoverflow.com/a/20558778
|
| 668 |
+
"""
|
| 669 |
+
if isinstance(obj, (list, tuple, set)):
|
| 670 |
+
return type(obj)(_remove_none(x) for x in obj if x is not None)
|
| 671 |
+
elif isinstance(obj, dict):
|
| 672 |
+
return type(obj)((_remove_none(k), _remove_none(v)) for k, v in obj.items() if k is not None and v is not None)
|
| 673 |
+
else:
|
| 674 |
+
return obj
|
| 675 |
+
|
| 676 |
+
|
| 677 |
+
def eval_results_to_model_index(model_name: str, eval_results: list[EvalResult]) -> list[dict[str, Any]]:
|
| 678 |
+
"""Takes in given model name and list of `huggingface_hub.EvalResult` and returns a
|
| 679 |
+
valid model-index that will be compatible with the format expected by the
|
| 680 |
+
Hugging Face Hub.
|
| 681 |
+
|
| 682 |
+
Args:
|
| 683 |
+
model_name (`str`):
|
| 684 |
+
Name of the model (ex. "my-cool-model"). This is used as the identifier
|
| 685 |
+
for the model on leaderboards like PapersWithCode.
|
| 686 |
+
eval_results (`list[EvalResult]`):
|
| 687 |
+
List of `huggingface_hub.EvalResult` objects containing the metrics to be
|
| 688 |
+
reported in the model-index.
|
| 689 |
+
|
| 690 |
+
Returns:
|
| 691 |
+
model_index (`list[dict[str, Any]]`): The eval_results converted to a model-index.
|
| 692 |
+
|
| 693 |
+
Example:
|
| 694 |
+
```python
|
| 695 |
+
>>> from huggingface_hub.repocard_data import eval_results_to_model_index, EvalResult
|
| 696 |
+
>>> # Define minimal eval_results
|
| 697 |
+
>>> eval_results = [
|
| 698 |
+
... EvalResult(
|
| 699 |
+
... task_type="image-classification", # Required
|
| 700 |
+
... dataset_type="beans", # Required
|
| 701 |
+
... dataset_name="Beans", # Required
|
| 702 |
+
... metric_type="accuracy", # Required
|
| 703 |
+
... metric_value=0.9, # Required
|
| 704 |
+
... )
|
| 705 |
+
... ]
|
| 706 |
+
>>> eval_results_to_model_index("my-cool-model", eval_results)
|
| 707 |
+
[{'name': 'my-cool-model', 'results': [{'task': {'type': 'image-classification'}, 'dataset': {'name': 'Beans', 'type': 'beans'}, 'metrics': [{'type': 'accuracy', 'value': 0.9}]}]}]
|
| 708 |
+
|
| 709 |
+
```
|
| 710 |
+
"""
|
| 711 |
+
|
| 712 |
+
# Metrics are reported on a unique task-and-dataset basis.
|
| 713 |
+
# Here, we make a map of those pairs and the associated EvalResults.
|
| 714 |
+
task_and_ds_types_map: dict[Any, list[EvalResult]] = defaultdict(list)
|
| 715 |
+
for eval_result in eval_results:
|
| 716 |
+
task_and_ds_types_map[eval_result.unique_identifier].append(eval_result)
|
| 717 |
+
|
| 718 |
+
# Use the map from above to generate the model index data.
|
| 719 |
+
model_index_data: list[dict[str, Any]] = []
|
| 720 |
+
for results in task_and_ds_types_map.values():
|
| 721 |
+
# All items from `results` share same metadata
|
| 722 |
+
sample_result = results[0]
|
| 723 |
+
data: dict[str, Any] = {
|
| 724 |
+
"task": {
|
| 725 |
+
"type": sample_result.task_type,
|
| 726 |
+
"name": sample_result.task_name,
|
| 727 |
+
},
|
| 728 |
+
"dataset": {
|
| 729 |
+
"name": sample_result.dataset_name,
|
| 730 |
+
"type": sample_result.dataset_type,
|
| 731 |
+
"config": sample_result.dataset_config,
|
| 732 |
+
"split": sample_result.dataset_split,
|
| 733 |
+
"revision": sample_result.dataset_revision,
|
| 734 |
+
"args": sample_result.dataset_args,
|
| 735 |
+
},
|
| 736 |
+
"metrics": [
|
| 737 |
+
{
|
| 738 |
+
"type": result.metric_type,
|
| 739 |
+
"value": result.metric_value,
|
| 740 |
+
"name": result.metric_name,
|
| 741 |
+
"config": result.metric_config,
|
| 742 |
+
"args": result.metric_args,
|
| 743 |
+
"verified": result.verified,
|
| 744 |
+
"verifyToken": result.verify_token,
|
| 745 |
+
}
|
| 746 |
+
for result in results
|
| 747 |
+
],
|
| 748 |
+
}
|
| 749 |
+
if sample_result.source_url is not None:
|
| 750 |
+
source: dict[str, str] = {
|
| 751 |
+
"url": sample_result.source_url,
|
| 752 |
+
}
|
| 753 |
+
if sample_result.source_name is not None:
|
| 754 |
+
source["name"] = sample_result.source_name
|
| 755 |
+
data["source"] = source
|
| 756 |
+
model_index_data.append(data)
|
| 757 |
+
|
| 758 |
+
# TODO - Check if there cases where this list is longer than one?
|
| 759 |
+
# Finally, the model index itself is list of dicts.
|
| 760 |
+
model_index = [
|
| 761 |
+
{
|
| 762 |
+
"name": model_name,
|
| 763 |
+
"results": model_index_data,
|
| 764 |
+
}
|
| 765 |
+
]
|
| 766 |
+
return _remove_none(model_index)
|
| 767 |
+
|
| 768 |
+
|
| 769 |
+
def _to_unique_list(tags: list[str] | None) -> list[str] | None:
|
| 770 |
+
if tags is None:
|
| 771 |
+
return tags
|
| 772 |
+
unique_tags = [] # make tags unique + keep order explicitly
|
| 773 |
+
for tag in tags:
|
| 774 |
+
if tag not in unique_tags:
|
| 775 |
+
unique_tags.append(tag)
|
| 776 |
+
return unique_tags
|
.venv/lib/python3.13/site-packages/huggingface_hub/serialization/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
# ruff: noqa: F401
|
| 15 |
+
"""Contains helpers to serialize tensors."""
|
| 16 |
+
|
| 17 |
+
from ._base import StateDictSplit, split_state_dict_into_shards_factory
|
| 18 |
+
from ._torch import (
|
| 19 |
+
get_torch_storage_id,
|
| 20 |
+
get_torch_storage_size,
|
| 21 |
+
load_state_dict_from_file,
|
| 22 |
+
load_torch_model,
|
| 23 |
+
save_torch_model,
|
| 24 |
+
save_torch_state_dict,
|
| 25 |
+
split_torch_state_dict_into_shards,
|
| 26 |
+
)
|
.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_base.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains helpers to split tensors into shards."""
|
| 15 |
+
|
| 16 |
+
from collections.abc import Callable
|
| 17 |
+
from dataclasses import dataclass, field
|
| 18 |
+
from typing import Any, TypeVar
|
| 19 |
+
|
| 20 |
+
from .. import logging
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
TensorT = TypeVar("TensorT")
|
| 24 |
+
TensorSizeFn_T = Callable[[TensorT], int]
|
| 25 |
+
StorageIDFn_T = Callable[[TensorT], Any | None]
|
| 26 |
+
|
| 27 |
+
MAX_SHARD_SIZE = "5GB"
|
| 28 |
+
SIZE_UNITS = {
|
| 29 |
+
"TB": 10**12,
|
| 30 |
+
"GB": 10**9,
|
| 31 |
+
"MB": 10**6,
|
| 32 |
+
"KB": 10**3,
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
logger = logging.get_logger(__file__)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@dataclass
|
| 40 |
+
class StateDictSplit:
|
| 41 |
+
is_sharded: bool = field(init=False)
|
| 42 |
+
metadata: dict[str, Any]
|
| 43 |
+
filename_to_tensors: dict[str, list[str]]
|
| 44 |
+
tensor_to_filename: dict[str, str]
|
| 45 |
+
|
| 46 |
+
def __post_init__(self):
|
| 47 |
+
self.is_sharded = len(self.filename_to_tensors) > 1
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def split_state_dict_into_shards_factory(
|
| 51 |
+
state_dict: dict[str, TensorT],
|
| 52 |
+
*,
|
| 53 |
+
get_storage_size: TensorSizeFn_T,
|
| 54 |
+
filename_pattern: str,
|
| 55 |
+
get_storage_id: StorageIDFn_T = lambda tensor: None,
|
| 56 |
+
max_shard_size: int | str = MAX_SHARD_SIZE,
|
| 57 |
+
) -> StateDictSplit:
|
| 58 |
+
"""
|
| 59 |
+
Split a model state dictionary in shards so that each shard is smaller than a given size.
|
| 60 |
+
|
| 61 |
+
The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization
|
| 62 |
+
made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we
|
| 63 |
+
have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not
|
| 64 |
+
[6+2+2GB], [6+2GB], [6GB].
|
| 65 |
+
|
| 66 |
+
> [!WARNING]
|
| 67 |
+
> If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a
|
| 68 |
+
> size greater than `max_shard_size`.
|
| 69 |
+
|
| 70 |
+
Args:
|
| 71 |
+
state_dict (`dict[str, Tensor]`):
|
| 72 |
+
The state dictionary to save.
|
| 73 |
+
get_storage_size (`Callable[[Tensor], int]`):
|
| 74 |
+
A function that returns the size of a tensor when saved on disk in bytes.
|
| 75 |
+
get_storage_id (`Callable[[Tensor], Optional[Any]]`, *optional*):
|
| 76 |
+
A function that returns a unique identifier to a tensor storage. Multiple different tensors can share the
|
| 77 |
+
same underlying storage. This identifier is guaranteed to be unique and constant for this tensor's storage
|
| 78 |
+
during its lifetime. Two tensor storages with non-overlapping lifetimes may have the same id.
|
| 79 |
+
filename_pattern (`str`, *optional*):
|
| 80 |
+
The pattern to generate the files names in which the model will be saved. Pattern must be a string that
|
| 81 |
+
can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
|
| 82 |
+
max_shard_size (`int` or `str`, *optional*):
|
| 83 |
+
The maximum size of each shard, in bytes. Defaults to 5GB.
|
| 84 |
+
|
| 85 |
+
Returns:
|
| 86 |
+
[`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them.
|
| 87 |
+
"""
|
| 88 |
+
storage_id_to_tensors: dict[Any, list[str]] = {}
|
| 89 |
+
|
| 90 |
+
shard_list: list[dict[str, TensorT]] = []
|
| 91 |
+
current_shard: dict[str, TensorT] = {}
|
| 92 |
+
current_shard_size = 0
|
| 93 |
+
total_size = 0
|
| 94 |
+
|
| 95 |
+
if isinstance(max_shard_size, str):
|
| 96 |
+
max_shard_size = parse_size_to_int(max_shard_size)
|
| 97 |
+
|
| 98 |
+
for key, tensor in state_dict.items():
|
| 99 |
+
# when bnb serialization is used the weights in the state dict can be strings
|
| 100 |
+
# check: https://github.com/huggingface/transformers/pull/24416 for more details
|
| 101 |
+
if isinstance(tensor, str):
|
| 102 |
+
logger.info("Skipping tensor %s as it is a string (bnb serialization)", key)
|
| 103 |
+
continue
|
| 104 |
+
|
| 105 |
+
# If a `tensor` shares the same underlying storage as another tensor, we put `tensor` in the same `block`
|
| 106 |
+
storage_id = get_storage_id(tensor) # type: ignore[invalid-argument-type]
|
| 107 |
+
if storage_id is not None:
|
| 108 |
+
if storage_id in storage_id_to_tensors:
|
| 109 |
+
# We skip this tensor for now and will reassign to correct shard later
|
| 110 |
+
storage_id_to_tensors[storage_id].append(key)
|
| 111 |
+
continue
|
| 112 |
+
else:
|
| 113 |
+
# This is the first tensor with this storage_id, we create a new entry
|
| 114 |
+
# in the storage_id_to_tensors dict => we will assign the shard id later
|
| 115 |
+
storage_id_to_tensors[storage_id] = [key]
|
| 116 |
+
|
| 117 |
+
# Compute tensor size
|
| 118 |
+
tensor_size = get_storage_size(tensor) # type: ignore[invalid-argument-type]
|
| 119 |
+
|
| 120 |
+
# If this tensor is bigger than the maximal size, we put it in its own shard
|
| 121 |
+
if tensor_size > max_shard_size:
|
| 122 |
+
total_size += tensor_size
|
| 123 |
+
shard_list.append({key: tensor})
|
| 124 |
+
continue
|
| 125 |
+
|
| 126 |
+
# If this tensor is going to tip up over the maximal size, we split.
|
| 127 |
+
# Current shard already has some tensors, we add it to the list of shards and create a new one.
|
| 128 |
+
if current_shard_size + tensor_size > max_shard_size:
|
| 129 |
+
shard_list.append(current_shard)
|
| 130 |
+
current_shard = {}
|
| 131 |
+
current_shard_size = 0
|
| 132 |
+
|
| 133 |
+
# Add the tensor to the current shard
|
| 134 |
+
current_shard[key] = tensor
|
| 135 |
+
current_shard_size += tensor_size
|
| 136 |
+
total_size += tensor_size
|
| 137 |
+
|
| 138 |
+
# Add the last shard
|
| 139 |
+
if len(current_shard) > 0:
|
| 140 |
+
shard_list.append(current_shard)
|
| 141 |
+
nb_shards = len(shard_list)
|
| 142 |
+
|
| 143 |
+
# Loop over the tensors that share the same storage and assign them together
|
| 144 |
+
for storage_id, keys in storage_id_to_tensors.items():
|
| 145 |
+
# Let's try to find the shard where the first tensor of this storage is and put all tensors in the same shard
|
| 146 |
+
for shard in shard_list:
|
| 147 |
+
if keys[0] in shard:
|
| 148 |
+
for key in keys:
|
| 149 |
+
shard[key] = state_dict[key]
|
| 150 |
+
break
|
| 151 |
+
|
| 152 |
+
# If we only have one shard, we return it => no need to build the index
|
| 153 |
+
if nb_shards == 1:
|
| 154 |
+
filename = filename_pattern.format(suffix="")
|
| 155 |
+
# Use the keys from the shard itself rather than `state_dict` directly, so that tensors
|
| 156 |
+
# skipped above (e.g. string tensors from bnb serialization) are excluded from the index,
|
| 157 |
+
# consistently with the multi-shard path below.
|
| 158 |
+
keys = list(shard_list[0].keys())
|
| 159 |
+
return StateDictSplit(
|
| 160 |
+
metadata={"total_size": total_size},
|
| 161 |
+
filename_to_tensors={filename: keys},
|
| 162 |
+
tensor_to_filename={key: filename for key in keys},
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
# Now that each tensor is assigned to a shard, let's assign a filename to each shard
|
| 166 |
+
tensor_name_to_filename = {}
|
| 167 |
+
filename_to_tensors = {}
|
| 168 |
+
for idx, shard in enumerate(shard_list):
|
| 169 |
+
filename = filename_pattern.format(suffix=f"-{idx + 1:05d}-of-{nb_shards:05d}")
|
| 170 |
+
for key in shard:
|
| 171 |
+
tensor_name_to_filename[key] = filename
|
| 172 |
+
filename_to_tensors[filename] = list(shard.keys())
|
| 173 |
+
|
| 174 |
+
# Build the index and return
|
| 175 |
+
return StateDictSplit(
|
| 176 |
+
metadata={"total_size": total_size},
|
| 177 |
+
filename_to_tensors=filename_to_tensors,
|
| 178 |
+
tensor_to_filename=tensor_name_to_filename,
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def parse_size_to_int(size_as_str: str) -> int:
|
| 183 |
+
"""
|
| 184 |
+
Parse a size expressed as a string with digits and unit (like `"5MB"`) to an integer (in bytes).
|
| 185 |
+
|
| 186 |
+
Supported units are "TB", "GB", "MB", "KB".
|
| 187 |
+
|
| 188 |
+
Args:
|
| 189 |
+
size_as_str (`str`): The size to convert. Will be directly returned if an `int`.
|
| 190 |
+
|
| 191 |
+
Example:
|
| 192 |
+
|
| 193 |
+
```py
|
| 194 |
+
>>> parse_size_to_int("5MB")
|
| 195 |
+
5000000
|
| 196 |
+
```
|
| 197 |
+
"""
|
| 198 |
+
size_as_str = size_as_str.strip()
|
| 199 |
+
|
| 200 |
+
# Parse unit
|
| 201 |
+
unit = size_as_str[-2:].upper()
|
| 202 |
+
if unit not in SIZE_UNITS:
|
| 203 |
+
raise ValueError(f"Unit '{unit}' not supported. Supported units are TB, GB, MB, KB. Got '{size_as_str}'.")
|
| 204 |
+
multiplier = SIZE_UNITS[unit]
|
| 205 |
+
|
| 206 |
+
# Parse value
|
| 207 |
+
try:
|
| 208 |
+
value = float(size_as_str[:-2].strip())
|
| 209 |
+
except ValueError as e:
|
| 210 |
+
raise ValueError(f"Could not parse the size value from '{size_as_str}': {e}") from e
|
| 211 |
+
|
| 212 |
+
return int(value * multiplier)
|
.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_dduf.py
ADDED
|
@@ -0,0 +1,386 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import logging
|
| 3 |
+
import mmap
|
| 4 |
+
import os
|
| 5 |
+
import shutil
|
| 6 |
+
import zipfile
|
| 7 |
+
from collections.abc import Generator, Iterable
|
| 8 |
+
from contextlib import contextmanager
|
| 9 |
+
from dataclasses import dataclass, field
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
from ..errors import DDUFCorruptedFileError, DDUFExportError, DDUFInvalidEntryNameError
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
logger = logging.getLogger(__name__)
|
| 17 |
+
|
| 18 |
+
DDUF_ALLOWED_ENTRIES = {
|
| 19 |
+
# Allowed file extensions in a DDUF file
|
| 20 |
+
".json",
|
| 21 |
+
".model",
|
| 22 |
+
".safetensors",
|
| 23 |
+
".txt",
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
DDUF_FOLDER_REQUIRED_ENTRIES = {
|
| 27 |
+
# Each folder must contain at least one of these entries
|
| 28 |
+
"config.json",
|
| 29 |
+
"tokenizer_config.json",
|
| 30 |
+
"preprocessor_config.json",
|
| 31 |
+
"scheduler_config.json",
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@dataclass
|
| 36 |
+
class DDUFEntry:
|
| 37 |
+
"""Object representing a file entry in a DDUF file.
|
| 38 |
+
|
| 39 |
+
See [`read_dduf_file`] for how to read a DDUF file.
|
| 40 |
+
|
| 41 |
+
Attributes:
|
| 42 |
+
filename (str):
|
| 43 |
+
The name of the file in the DDUF archive.
|
| 44 |
+
offset (int):
|
| 45 |
+
The offset of the file in the DDUF archive.
|
| 46 |
+
length (int):
|
| 47 |
+
The length of the file in the DDUF archive.
|
| 48 |
+
dduf_path (str):
|
| 49 |
+
The path to the DDUF archive (for internal use).
|
| 50 |
+
"""
|
| 51 |
+
|
| 52 |
+
filename: str
|
| 53 |
+
length: int
|
| 54 |
+
offset: int
|
| 55 |
+
|
| 56 |
+
dduf_path: Path = field(repr=False)
|
| 57 |
+
|
| 58 |
+
@contextmanager
|
| 59 |
+
def as_mmap(self) -> Generator[bytes, None, None]:
|
| 60 |
+
"""Open the file as a memory-mapped file.
|
| 61 |
+
|
| 62 |
+
Useful to load safetensors directly from the file.
|
| 63 |
+
|
| 64 |
+
Example:
|
| 65 |
+
```py
|
| 66 |
+
>>> import safetensors.torch
|
| 67 |
+
>>> with entry.as_mmap() as mm:
|
| 68 |
+
... tensors = safetensors.torch.load(mm)
|
| 69 |
+
```
|
| 70 |
+
"""
|
| 71 |
+
with self.dduf_path.open("rb") as f:
|
| 72 |
+
with mmap.mmap(f.fileno(), length=0, access=mmap.ACCESS_READ) as mm:
|
| 73 |
+
yield mm[self.offset : self.offset + self.length]
|
| 74 |
+
|
| 75 |
+
def read_text(self, encoding: str = "utf-8") -> str:
|
| 76 |
+
"""Read the file as text.
|
| 77 |
+
|
| 78 |
+
Useful for '.txt' and '.json' entries.
|
| 79 |
+
|
| 80 |
+
Example:
|
| 81 |
+
```py
|
| 82 |
+
>>> import json
|
| 83 |
+
>>> index = json.loads(entry.read_text())
|
| 84 |
+
```
|
| 85 |
+
"""
|
| 86 |
+
with self.dduf_path.open("rb") as f:
|
| 87 |
+
f.seek(self.offset)
|
| 88 |
+
return f.read(self.length).decode(encoding=encoding)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def read_dduf_file(dduf_path: os.PathLike | str) -> dict[str, DDUFEntry]:
|
| 92 |
+
"""
|
| 93 |
+
Read a DDUF file and return a dictionary of entries.
|
| 94 |
+
|
| 95 |
+
Only the metadata is read, the data is not loaded in memory.
|
| 96 |
+
|
| 97 |
+
Args:
|
| 98 |
+
dduf_path (`str` or `os.PathLike`):
|
| 99 |
+
The path to the DDUF file to read.
|
| 100 |
+
|
| 101 |
+
Returns:
|
| 102 |
+
`dict[str, DDUFEntry]`:
|
| 103 |
+
A dictionary of [`DDUFEntry`] indexed by filename.
|
| 104 |
+
|
| 105 |
+
Raises:
|
| 106 |
+
- [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format).
|
| 107 |
+
|
| 108 |
+
Example:
|
| 109 |
+
```python
|
| 110 |
+
>>> import json
|
| 111 |
+
>>> import safetensors.torch
|
| 112 |
+
>>> from huggingface_hub import read_dduf_file
|
| 113 |
+
|
| 114 |
+
# Read DDUF metadata
|
| 115 |
+
>>> dduf_entries = read_dduf_file("FLUX.1-dev.dduf")
|
| 116 |
+
|
| 117 |
+
# Returns a mapping filename <> DDUFEntry
|
| 118 |
+
>>> dduf_entries["model_index.json"]
|
| 119 |
+
DDUFEntry(filename='model_index.json', offset=66, length=587)
|
| 120 |
+
|
| 121 |
+
# Load model index as JSON
|
| 122 |
+
>>> json.loads(dduf_entries["model_index.json"].read_text())
|
| 123 |
+
{'_class_name': 'FluxPipeline', '_diffusers_version': '0.32.0.dev0', '_name_or_path': 'black-forest-labs/FLUX.1-dev', ...
|
| 124 |
+
|
| 125 |
+
# Load VAE weights using safetensors
|
| 126 |
+
>>> with dduf_entries["vae/diffusion_pytorch_model.safetensors"].as_mmap() as mm:
|
| 127 |
+
... state_dict = safetensors.torch.load(mm)
|
| 128 |
+
```
|
| 129 |
+
"""
|
| 130 |
+
entries = {}
|
| 131 |
+
dduf_path = Path(dduf_path)
|
| 132 |
+
logger.info(f"Reading DDUF file {dduf_path}")
|
| 133 |
+
with zipfile.ZipFile(str(dduf_path), "r") as zf:
|
| 134 |
+
for info in zf.infolist():
|
| 135 |
+
logger.debug(f"Reading entry {info.filename}")
|
| 136 |
+
if info.compress_type != zipfile.ZIP_STORED:
|
| 137 |
+
raise DDUFCorruptedFileError("Data must not be compressed in DDUF file.")
|
| 138 |
+
|
| 139 |
+
try:
|
| 140 |
+
_validate_dduf_entry_name(info.filename)
|
| 141 |
+
except DDUFInvalidEntryNameError as e:
|
| 142 |
+
raise DDUFCorruptedFileError(f"Invalid entry name in DDUF file: {info.filename}") from e
|
| 143 |
+
|
| 144 |
+
offset = _get_data_offset(zf, info)
|
| 145 |
+
|
| 146 |
+
entries[info.filename] = DDUFEntry(
|
| 147 |
+
filename=info.filename, offset=offset, length=info.file_size, dduf_path=dduf_path
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
# Consistency checks on the DDUF file
|
| 151 |
+
if "model_index.json" not in entries:
|
| 152 |
+
raise DDUFCorruptedFileError("Missing required 'model_index.json' entry in DDUF file.")
|
| 153 |
+
index = json.loads(entries["model_index.json"].read_text())
|
| 154 |
+
_validate_dduf_structure(index, entries.keys())
|
| 155 |
+
|
| 156 |
+
logger.info(f"Done reading DDUF file {dduf_path}. Found {len(entries)} entries")
|
| 157 |
+
return entries
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def export_entries_as_dduf(dduf_path: str | os.PathLike, entries: Iterable[tuple[str, str | Path | bytes]]) -> None:
|
| 161 |
+
"""Write a DDUF file from an iterable of entries.
|
| 162 |
+
|
| 163 |
+
This is a lower-level helper than [`export_folder_as_dduf`] that allows more flexibility when serializing data.
|
| 164 |
+
In particular, you don't need to save the data on disk before exporting it in the DDUF file.
|
| 165 |
+
|
| 166 |
+
Args:
|
| 167 |
+
dduf_path (`str` or `os.PathLike`):
|
| 168 |
+
The path to the DDUF file to write.
|
| 169 |
+
entries (`Iterable[tuple[str, Union[str, Path, bytes]]]`):
|
| 170 |
+
An iterable of entries to write in the DDUF file. Each entry is a tuple with the filename and the content.
|
| 171 |
+
The filename should be the path to the file in the DDUF archive.
|
| 172 |
+
The content can be a string or a pathlib.Path representing a path to a file on the local disk or directly the content as bytes.
|
| 173 |
+
|
| 174 |
+
Raises:
|
| 175 |
+
- [`DDUFExportError`]: If anything goes wrong during the export (e.g. invalid entry name, missing 'model_index.json', etc.).
|
| 176 |
+
|
| 177 |
+
Example:
|
| 178 |
+
```python
|
| 179 |
+
# Export specific files from the local disk.
|
| 180 |
+
>>> from huggingface_hub import export_entries_as_dduf
|
| 181 |
+
>>> export_entries_as_dduf(
|
| 182 |
+
... dduf_path="stable-diffusion-v1-4-FP16.dduf",
|
| 183 |
+
... entries=[ # List entries to add to the DDUF file (here, only FP16 weights)
|
| 184 |
+
... ("model_index.json", "path/to/model_index.json"),
|
| 185 |
+
... ("vae/config.json", "path/to/vae/config.json"),
|
| 186 |
+
... ("vae/diffusion_pytorch_model.fp16.safetensors", "path/to/vae/diffusion_pytorch_model.fp16.safetensors"),
|
| 187 |
+
... ("text_encoder/config.json", "path/to/text_encoder/config.json"),
|
| 188 |
+
... ("text_encoder/model.fp16.safetensors", "path/to/text_encoder/model.fp16.safetensors"),
|
| 189 |
+
... # ... add more entries here
|
| 190 |
+
... ]
|
| 191 |
+
... )
|
| 192 |
+
```
|
| 193 |
+
|
| 194 |
+
```python
|
| 195 |
+
# Export state_dicts one by one from a loaded pipeline
|
| 196 |
+
>>> from diffusers import DiffusionPipeline
|
| 197 |
+
>>> from typing import Generator, Tuple
|
| 198 |
+
>>> import safetensors.torch
|
| 199 |
+
>>> from huggingface_hub import export_entries_as_dduf
|
| 200 |
+
>>> pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4")
|
| 201 |
+
... # ... do some work with the pipeline
|
| 202 |
+
|
| 203 |
+
>>> def as_entries(pipe: DiffusionPipeline) -> Generator[tuple[str, bytes], None, None]:
|
| 204 |
+
... # Build a generator that yields the entries to add to the DDUF file.
|
| 205 |
+
... # The first element of the tuple is the filename in the DDUF archive (must use UNIX separator!). The second element is the content of the file.
|
| 206 |
+
... # Entries will be evaluated lazily when the DDUF file is created (only 1 entry is loaded in memory at a time)
|
| 207 |
+
... yield "vae/config.json", pipe.vae.to_json_string().encode()
|
| 208 |
+
... yield "vae/diffusion_pytorch_model.safetensors", safetensors.torch.save(pipe.vae.state_dict())
|
| 209 |
+
... yield "text_encoder/config.json", pipe.text_encoder.config.to_json_string().encode()
|
| 210 |
+
... yield "text_encoder/model.safetensors", safetensors.torch.save(pipe.text_encoder.state_dict())
|
| 211 |
+
... # ... add more entries here
|
| 212 |
+
|
| 213 |
+
>>> export_entries_as_dduf(dduf_path="stable-diffusion-v1-4.dduf", entries=as_entries(pipe))
|
| 214 |
+
```
|
| 215 |
+
"""
|
| 216 |
+
logger.info(f"Exporting DDUF file '{dduf_path}'")
|
| 217 |
+
filenames = set()
|
| 218 |
+
index = None
|
| 219 |
+
with zipfile.ZipFile(str(dduf_path), "w", zipfile.ZIP_STORED) as archive:
|
| 220 |
+
for filename, content in entries:
|
| 221 |
+
if filename in filenames:
|
| 222 |
+
raise DDUFExportError(f"Can't add duplicate entry: {filename}")
|
| 223 |
+
filenames.add(filename)
|
| 224 |
+
|
| 225 |
+
if filename == "model_index.json":
|
| 226 |
+
try:
|
| 227 |
+
index = json.loads(_load_content(content).decode())
|
| 228 |
+
except json.JSONDecodeError as e:
|
| 229 |
+
raise DDUFExportError("Failed to parse 'model_index.json'.") from e
|
| 230 |
+
|
| 231 |
+
try:
|
| 232 |
+
filename = _validate_dduf_entry_name(filename)
|
| 233 |
+
except DDUFInvalidEntryNameError as e:
|
| 234 |
+
raise DDUFExportError(f"Invalid entry name: {filename}") from e
|
| 235 |
+
logger.debug(f"Adding entry '{filename}' to DDUF file")
|
| 236 |
+
_dump_content_in_archive(archive, filename, content)
|
| 237 |
+
|
| 238 |
+
# Consistency checks on the DDUF file
|
| 239 |
+
if index is None:
|
| 240 |
+
raise DDUFExportError("Missing required 'model_index.json' entry in DDUF file.")
|
| 241 |
+
try:
|
| 242 |
+
_validate_dduf_structure(index, filenames)
|
| 243 |
+
except DDUFCorruptedFileError as e:
|
| 244 |
+
raise DDUFExportError("Invalid DDUF file structure.") from e
|
| 245 |
+
|
| 246 |
+
logger.info(f"Done writing DDUF file {dduf_path}")
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def export_folder_as_dduf(dduf_path: str | os.PathLike, folder_path: str | os.PathLike) -> None:
|
| 250 |
+
"""
|
| 251 |
+
Export a folder as a DDUF file.
|
| 252 |
+
|
| 253 |
+
AUses [`export_entries_as_dduf`] under the hood.
|
| 254 |
+
|
| 255 |
+
Args:
|
| 256 |
+
dduf_path (`str` or `os.PathLike`):
|
| 257 |
+
The path to the DDUF file to write.
|
| 258 |
+
folder_path (`str` or `os.PathLike`):
|
| 259 |
+
The path to the folder containing the diffusion model.
|
| 260 |
+
|
| 261 |
+
Example:
|
| 262 |
+
```python
|
| 263 |
+
>>> from huggingface_hub import export_folder_as_dduf
|
| 264 |
+
>>> export_folder_as_dduf(dduf_path="FLUX.1-dev.dduf", folder_path="path/to/FLUX.1-dev")
|
| 265 |
+
```
|
| 266 |
+
"""
|
| 267 |
+
folder_path = Path(folder_path)
|
| 268 |
+
|
| 269 |
+
def _iterate_over_folder() -> Iterable[tuple[str, Path]]:
|
| 270 |
+
for path in Path(folder_path).glob("**/*"):
|
| 271 |
+
if not path.is_file():
|
| 272 |
+
continue
|
| 273 |
+
if path.suffix not in DDUF_ALLOWED_ENTRIES:
|
| 274 |
+
logger.debug(f"Skipping file '{path}' (file type not allowed)")
|
| 275 |
+
continue
|
| 276 |
+
path_in_archive = path.relative_to(folder_path)
|
| 277 |
+
if len(path_in_archive.parts) >= 3:
|
| 278 |
+
logger.debug(f"Skipping file '{path}' (nested directories not allowed)")
|
| 279 |
+
continue
|
| 280 |
+
yield path_in_archive.as_posix(), path
|
| 281 |
+
|
| 282 |
+
export_entries_as_dduf(dduf_path, _iterate_over_folder())
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
def _dump_content_in_archive(archive: zipfile.ZipFile, filename: str, content: str | os.PathLike | bytes) -> None:
|
| 286 |
+
with archive.open(filename, "w", force_zip64=True) as archive_fh:
|
| 287 |
+
if isinstance(content, (str, Path)):
|
| 288 |
+
content_path = Path(content)
|
| 289 |
+
with content_path.open("rb") as content_fh:
|
| 290 |
+
shutil.copyfileobj(content_fh, archive_fh, 1024 * 1024 * 8) # type: ignore[misc]
|
| 291 |
+
elif isinstance(content, bytes):
|
| 292 |
+
archive_fh.write(content)
|
| 293 |
+
else:
|
| 294 |
+
raise DDUFExportError(f"Invalid content type for {filename}. Must be str, Path or bytes.")
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def _load_content(content: str | Path | bytes) -> bytes:
|
| 298 |
+
"""Load the content of an entry as bytes.
|
| 299 |
+
|
| 300 |
+
Used only for small checks (not to dump content into archive).
|
| 301 |
+
"""
|
| 302 |
+
if isinstance(content, (str, Path)):
|
| 303 |
+
return Path(content).read_bytes()
|
| 304 |
+
elif isinstance(content, bytes):
|
| 305 |
+
return content
|
| 306 |
+
else:
|
| 307 |
+
raise DDUFExportError(f"Invalid content type. Must be str, Path or bytes. Got {type(content)}.")
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def _validate_dduf_entry_name(entry_name: str) -> str:
|
| 311 |
+
if "." + entry_name.split(".")[-1] not in DDUF_ALLOWED_ENTRIES:
|
| 312 |
+
raise DDUFInvalidEntryNameError(f"File type not allowed: {entry_name}")
|
| 313 |
+
if "\\" in entry_name:
|
| 314 |
+
raise DDUFInvalidEntryNameError(f"Entry names must use UNIX separators ('/'). Got {entry_name}.")
|
| 315 |
+
entry_name = entry_name.strip("/")
|
| 316 |
+
if entry_name.count("/") > 1:
|
| 317 |
+
raise DDUFInvalidEntryNameError(f"DDUF only supports 1 level of directory. Got {entry_name}.")
|
| 318 |
+
return entry_name
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def _validate_dduf_structure(index: Any, entry_names: Iterable[str]) -> None:
|
| 322 |
+
"""
|
| 323 |
+
Consistency checks on the DDUF file structure.
|
| 324 |
+
|
| 325 |
+
Rules:
|
| 326 |
+
- The 'model_index.json' entry is required and must contain a dictionary.
|
| 327 |
+
- Each folder name must correspond to an entry in 'model_index.json'.
|
| 328 |
+
- Each folder must contain at least a config file ('config.json', 'tokenizer_config.json', 'preprocessor_config.json', 'scheduler_config.json').
|
| 329 |
+
|
| 330 |
+
Args:
|
| 331 |
+
index (Any):
|
| 332 |
+
The content of the 'model_index.json' entry.
|
| 333 |
+
entry_names (Iterable[str]):
|
| 334 |
+
The list of entry names in the DDUF file.
|
| 335 |
+
|
| 336 |
+
Raises:
|
| 337 |
+
- [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format).
|
| 338 |
+
"""
|
| 339 |
+
if not isinstance(index, dict):
|
| 340 |
+
raise DDUFCorruptedFileError(f"Invalid 'model_index.json' content. Must be a dictionary. Got {type(index)}.")
|
| 341 |
+
|
| 342 |
+
dduf_folders = {entry.split("/")[0] for entry in entry_names if "/" in entry}
|
| 343 |
+
for folder in dduf_folders:
|
| 344 |
+
if folder not in index:
|
| 345 |
+
raise DDUFCorruptedFileError(f"Missing required entry '{folder}' in 'model_index.json'.")
|
| 346 |
+
if not any(f"{folder}/{required_entry}" in entry_names for required_entry in DDUF_FOLDER_REQUIRED_ENTRIES):
|
| 347 |
+
raise DDUFCorruptedFileError(
|
| 348 |
+
f"Missing required file in folder '{folder}'. Must contains at least one of {DDUF_FOLDER_REQUIRED_ENTRIES}."
|
| 349 |
+
)
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def _get_data_offset(zf: zipfile.ZipFile, info: zipfile.ZipInfo) -> int:
|
| 353 |
+
"""
|
| 354 |
+
Calculate the data offset for a file in a ZIP archive.
|
| 355 |
+
|
| 356 |
+
Args:
|
| 357 |
+
zf (`zipfile.ZipFile`):
|
| 358 |
+
The opened ZIP file. Must be opened in read mode.
|
| 359 |
+
info (`zipfile.ZipInfo`):
|
| 360 |
+
The file info.
|
| 361 |
+
|
| 362 |
+
Returns:
|
| 363 |
+
int: The offset of the file data in the ZIP archive.
|
| 364 |
+
"""
|
| 365 |
+
if zf.fp is None:
|
| 366 |
+
raise DDUFCorruptedFileError("ZipFile object must be opened in read mode.")
|
| 367 |
+
|
| 368 |
+
# Step 1: Get the local file header offset
|
| 369 |
+
header_offset = info.header_offset
|
| 370 |
+
|
| 371 |
+
# Step 2: Read the local file header
|
| 372 |
+
zf.fp.seek(header_offset)
|
| 373 |
+
local_file_header = zf.fp.read(30) # Fixed-size part of the local header
|
| 374 |
+
|
| 375 |
+
if len(local_file_header) < 30:
|
| 376 |
+
raise DDUFCorruptedFileError("Incomplete local file header.")
|
| 377 |
+
|
| 378 |
+
# Step 3: Parse the header fields to calculate the start of file data
|
| 379 |
+
# Local file header: https://en.wikipedia.org/wiki/ZIP_(file_format)#File_headers
|
| 380 |
+
filename_len = int.from_bytes(local_file_header[26:28], "little")
|
| 381 |
+
extra_field_len = int.from_bytes(local_file_header[28:30], "little")
|
| 382 |
+
|
| 383 |
+
# Data offset is after the fixed header, filename, and extra fields
|
| 384 |
+
data_offset = header_offset + 30 + filename_len + extra_field_len
|
| 385 |
+
|
| 386 |
+
return data_offset
|
.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_torch.py
ADDED
|
@@ -0,0 +1,1046 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains pytorch-specific helpers."""
|
| 15 |
+
|
| 16 |
+
import importlib
|
| 17 |
+
import importlib.util
|
| 18 |
+
import json
|
| 19 |
+
import os
|
| 20 |
+
import re
|
| 21 |
+
from collections import defaultdict, namedtuple
|
| 22 |
+
from collections.abc import Iterable
|
| 23 |
+
from functools import lru_cache
|
| 24 |
+
from pathlib import Path, PureWindowsPath
|
| 25 |
+
from typing import TYPE_CHECKING, Any, NamedTuple, Union
|
| 26 |
+
|
| 27 |
+
from packaging import version
|
| 28 |
+
|
| 29 |
+
from .. import constants, logging
|
| 30 |
+
from ._base import MAX_SHARD_SIZE, StateDictSplit, split_state_dict_into_shards_factory
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
logger = logging.get_logger(__file__)
|
| 34 |
+
|
| 35 |
+
if TYPE_CHECKING:
|
| 36 |
+
import torch
|
| 37 |
+
|
| 38 |
+
# SAVING
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def save_torch_model(
|
| 42 |
+
model: "torch.nn.Module",
|
| 43 |
+
save_directory: str | Path,
|
| 44 |
+
*,
|
| 45 |
+
filename_pattern: str | None = None,
|
| 46 |
+
force_contiguous: bool = True,
|
| 47 |
+
max_shard_size: int | str = MAX_SHARD_SIZE,
|
| 48 |
+
metadata: dict[str, str] | None = None,
|
| 49 |
+
safe_serialization: bool = True,
|
| 50 |
+
is_main_process: bool = True,
|
| 51 |
+
shared_tensors_to_discard: list[str] | None = None,
|
| 52 |
+
):
|
| 53 |
+
"""
|
| 54 |
+
Saves a given torch model to disk, handling sharding and shared tensors issues.
|
| 55 |
+
|
| 56 |
+
See also [`save_torch_state_dict`] to save a state dict with more flexibility.
|
| 57 |
+
|
| 58 |
+
For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors).
|
| 59 |
+
|
| 60 |
+
The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are
|
| 61 |
+
saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard,
|
| 62 |
+
an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses
|
| 63 |
+
[`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as
|
| 64 |
+
safetensors (the default). Otherwise, the shards are saved as pickle.
|
| 65 |
+
|
| 66 |
+
Before saving the model, the `save_directory` is cleaned from any previous shard files.
|
| 67 |
+
|
| 68 |
+
> [!WARNING]
|
| 69 |
+
> If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a
|
| 70 |
+
> size greater than `max_shard_size`.
|
| 71 |
+
|
| 72 |
+
> [!WARNING]
|
| 73 |
+
> If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving.
|
| 74 |
+
|
| 75 |
+
Args:
|
| 76 |
+
model (`torch.nn.Module`):
|
| 77 |
+
The model to save on disk.
|
| 78 |
+
save_directory (`str` or `Path`):
|
| 79 |
+
The directory in which the model will be saved.
|
| 80 |
+
filename_pattern (`str`, *optional*):
|
| 81 |
+
The pattern to generate the files names in which the model will be saved. Pattern must be a string that
|
| 82 |
+
can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
|
| 83 |
+
Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization`
|
| 84 |
+
parameter.
|
| 85 |
+
force_contiguous (`boolean`, *optional*):
|
| 86 |
+
Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the
|
| 87 |
+
model, but it could potentially change performance if the layout of the tensor was chosen specifically for
|
| 88 |
+
that reason. Defaults to `True`.
|
| 89 |
+
max_shard_size (`int` or `str`, *optional*):
|
| 90 |
+
The maximum size of each shard, in bytes. Defaults to 5GB.
|
| 91 |
+
metadata (`dict[str, str]`, *optional*):
|
| 92 |
+
Extra information to save along with the model. Some metadata will be added for each dropped tensors.
|
| 93 |
+
This information will not be enough to recover the entire shared structure but might help understanding
|
| 94 |
+
things.
|
| 95 |
+
safe_serialization (`bool`, *optional*):
|
| 96 |
+
Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle.
|
| 97 |
+
Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed
|
| 98 |
+
in a future version.
|
| 99 |
+
is_main_process (`bool`, *optional*):
|
| 100 |
+
Whether the process calling this is the main process or not. Useful when in distributed training like
|
| 101 |
+
TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on
|
| 102 |
+
the main process to avoid race conditions. Defaults to True.
|
| 103 |
+
shared_tensors_to_discard (`list[str]`, *optional*):
|
| 104 |
+
List of tensor names to drop when saving shared tensors. If not provided and shared tensors are
|
| 105 |
+
detected, it will drop the first name alphabetically.
|
| 106 |
+
|
| 107 |
+
Example:
|
| 108 |
+
|
| 109 |
+
```py
|
| 110 |
+
>>> from huggingface_hub import save_torch_model
|
| 111 |
+
>>> model = ... # A PyTorch model
|
| 112 |
+
|
| 113 |
+
# Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors.
|
| 114 |
+
>>> save_torch_model(model, "path/to/folder")
|
| 115 |
+
|
| 116 |
+
# Load model back
|
| 117 |
+
>>> from huggingface_hub import load_torch_model # TODO
|
| 118 |
+
>>> load_torch_model(model, "path/to/folder")
|
| 119 |
+
>>>
|
| 120 |
+
```
|
| 121 |
+
"""
|
| 122 |
+
save_torch_state_dict(
|
| 123 |
+
state_dict=model.state_dict(),
|
| 124 |
+
filename_pattern=filename_pattern,
|
| 125 |
+
force_contiguous=force_contiguous,
|
| 126 |
+
max_shard_size=max_shard_size,
|
| 127 |
+
metadata=metadata,
|
| 128 |
+
safe_serialization=safe_serialization,
|
| 129 |
+
save_directory=save_directory,
|
| 130 |
+
is_main_process=is_main_process,
|
| 131 |
+
shared_tensors_to_discard=shared_tensors_to_discard,
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def save_torch_state_dict(
|
| 136 |
+
state_dict: dict[str, "torch.Tensor"],
|
| 137 |
+
save_directory: str | Path,
|
| 138 |
+
*,
|
| 139 |
+
filename_pattern: str | None = None,
|
| 140 |
+
force_contiguous: bool = True,
|
| 141 |
+
max_shard_size: int | str = MAX_SHARD_SIZE,
|
| 142 |
+
metadata: dict[str, str] | None = None,
|
| 143 |
+
safe_serialization: bool = True,
|
| 144 |
+
is_main_process: bool = True,
|
| 145 |
+
shared_tensors_to_discard: list[str] | None = None,
|
| 146 |
+
) -> None:
|
| 147 |
+
"""
|
| 148 |
+
Save a model state dictionary to the disk, handling sharding and shared tensors issues.
|
| 149 |
+
|
| 150 |
+
See also [`save_torch_model`] to directly save a PyTorch model.
|
| 151 |
+
|
| 152 |
+
For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors).
|
| 153 |
+
|
| 154 |
+
The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are
|
| 155 |
+
saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard,
|
| 156 |
+
an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses
|
| 157 |
+
[`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as
|
| 158 |
+
safetensors (the default). Otherwise, the shards are saved as pickle.
|
| 159 |
+
|
| 160 |
+
Before saving the model, the `save_directory` is cleaned from any previous shard files.
|
| 161 |
+
|
| 162 |
+
> [!WARNING]
|
| 163 |
+
> If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a
|
| 164 |
+
> size greater than `max_shard_size`.
|
| 165 |
+
|
| 166 |
+
> [!WARNING]
|
| 167 |
+
> If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving.
|
| 168 |
+
|
| 169 |
+
Args:
|
| 170 |
+
state_dict (`dict[str, torch.Tensor]`):
|
| 171 |
+
The state dictionary to save.
|
| 172 |
+
save_directory (`str` or `Path`):
|
| 173 |
+
The directory in which the model will be saved.
|
| 174 |
+
filename_pattern (`str`, *optional*):
|
| 175 |
+
The pattern to generate the files names in which the model will be saved. Pattern must be a string that
|
| 176 |
+
can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
|
| 177 |
+
Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization`
|
| 178 |
+
parameter.
|
| 179 |
+
force_contiguous (`boolean`, *optional*):
|
| 180 |
+
Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the
|
| 181 |
+
model, but it could potentially change performance if the layout of the tensor was chosen specifically for
|
| 182 |
+
that reason. Defaults to `True`.
|
| 183 |
+
max_shard_size (`int` or `str`, *optional*):
|
| 184 |
+
The maximum size of each shard, in bytes. Defaults to 5GB.
|
| 185 |
+
metadata (`dict[str, str]`, *optional*):
|
| 186 |
+
Extra information to save along with the model. Some metadata will be added for each dropped tensors.
|
| 187 |
+
This information will not be enough to recover the entire shared structure but might help understanding
|
| 188 |
+
things.
|
| 189 |
+
safe_serialization (`bool`, *optional*):
|
| 190 |
+
Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle.
|
| 191 |
+
Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed
|
| 192 |
+
in a future version.
|
| 193 |
+
is_main_process (`bool`, *optional*):
|
| 194 |
+
Whether the process calling this is the main process or not. Useful when in distributed training like
|
| 195 |
+
TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on
|
| 196 |
+
the main process to avoid race conditions. Defaults to True.
|
| 197 |
+
shared_tensors_to_discard (`list[str]`, *optional*):
|
| 198 |
+
List of tensor names to drop when saving shared tensors. If not provided and shared tensors are
|
| 199 |
+
detected, it will drop the first name alphabetically.
|
| 200 |
+
|
| 201 |
+
Example:
|
| 202 |
+
|
| 203 |
+
```py
|
| 204 |
+
>>> from huggingface_hub import save_torch_state_dict
|
| 205 |
+
>>> model = ... # A PyTorch model
|
| 206 |
+
|
| 207 |
+
# Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors.
|
| 208 |
+
>>> state_dict = model_to_save.state_dict()
|
| 209 |
+
>>> save_torch_state_dict(state_dict, "path/to/folder")
|
| 210 |
+
```
|
| 211 |
+
"""
|
| 212 |
+
save_directory = str(save_directory)
|
| 213 |
+
|
| 214 |
+
if filename_pattern is None:
|
| 215 |
+
filename_pattern = (
|
| 216 |
+
constants.SAFETENSORS_WEIGHTS_FILE_PATTERN
|
| 217 |
+
if safe_serialization
|
| 218 |
+
else constants.PYTORCH_WEIGHTS_FILE_PATTERN
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
if metadata is None:
|
| 222 |
+
metadata = {}
|
| 223 |
+
if safe_serialization:
|
| 224 |
+
try:
|
| 225 |
+
from safetensors.torch import save_file as save_file_fn
|
| 226 |
+
except ImportError as e:
|
| 227 |
+
raise ImportError(
|
| 228 |
+
"Please install `safetensors` to use safe serialization. "
|
| 229 |
+
"You can install it with `pip install safetensors`."
|
| 230 |
+
) from e
|
| 231 |
+
# Clean state dict for safetensors
|
| 232 |
+
state_dict = _clean_state_dict_for_safetensors(
|
| 233 |
+
state_dict,
|
| 234 |
+
metadata,
|
| 235 |
+
force_contiguous=force_contiguous,
|
| 236 |
+
shared_tensors_to_discard=shared_tensors_to_discard,
|
| 237 |
+
)
|
| 238 |
+
else:
|
| 239 |
+
from torch import save as save_file_fn # type: ignore[assignment, no-redef]
|
| 240 |
+
|
| 241 |
+
logger.warning(
|
| 242 |
+
"You are using unsafe serialization. Due to security reasons, it is recommended not to load "
|
| 243 |
+
"pickled models from untrusted sources. If you intend to share your model, we strongly recommend "
|
| 244 |
+
"using safe serialization by installing `safetensors` with `pip install safetensors`."
|
| 245 |
+
)
|
| 246 |
+
# Split dict
|
| 247 |
+
state_dict_split = split_torch_state_dict_into_shards(
|
| 248 |
+
state_dict, filename_pattern=filename_pattern, max_shard_size=max_shard_size
|
| 249 |
+
)
|
| 250 |
+
|
| 251 |
+
# Only main process should clean up existing files to avoid race conditions in distributed environment
|
| 252 |
+
if is_main_process:
|
| 253 |
+
existing_files_regex = re.compile(filename_pattern.format(suffix=r"(-\d{5}-of-\d{5})?") + r"(\.index\.json)?")
|
| 254 |
+
for filename in os.listdir(save_directory):
|
| 255 |
+
if existing_files_regex.match(filename):
|
| 256 |
+
try:
|
| 257 |
+
logger.debug(f"Removing existing file '{filename}' from folder.")
|
| 258 |
+
os.remove(os.path.join(save_directory, filename))
|
| 259 |
+
except Exception as e:
|
| 260 |
+
logger.warning(
|
| 261 |
+
f"Error when trying to remove existing '{filename}' from folder: {e}. Continuing..."
|
| 262 |
+
)
|
| 263 |
+
|
| 264 |
+
# Save each shard
|
| 265 |
+
per_file_metadata = {"format": "pt"}
|
| 266 |
+
if not state_dict_split.is_sharded:
|
| 267 |
+
per_file_metadata.update(metadata)
|
| 268 |
+
safe_file_kwargs = {"metadata": per_file_metadata} if safe_serialization else {}
|
| 269 |
+
for filename, tensors in state_dict_split.filename_to_tensors.items():
|
| 270 |
+
shard = {tensor: state_dict[tensor] for tensor in tensors}
|
| 271 |
+
save_file_fn(shard, os.path.join(save_directory, filename), **safe_file_kwargs) # ty: ignore[invalid-argument-type]
|
| 272 |
+
logger.debug(f"Shard saved to {filename}")
|
| 273 |
+
|
| 274 |
+
# Save the index (if any)
|
| 275 |
+
if state_dict_split.is_sharded:
|
| 276 |
+
index_path = filename_pattern.format(suffix="") + ".index.json"
|
| 277 |
+
index = {
|
| 278 |
+
"metadata": {**state_dict_split.metadata, **metadata},
|
| 279 |
+
"weight_map": state_dict_split.tensor_to_filename,
|
| 280 |
+
}
|
| 281 |
+
with open(os.path.join(save_directory, index_path), "w") as f:
|
| 282 |
+
json.dump(index, f, indent=2)
|
| 283 |
+
logger.info(
|
| 284 |
+
f"The model is bigger than the maximum size per checkpoint ({max_shard_size}). "
|
| 285 |
+
f"Model weighs have been saved in {len(state_dict_split.filename_to_tensors)} checkpoint shards. "
|
| 286 |
+
f"You can find where each parameters has been saved in the index located at {index_path}."
|
| 287 |
+
)
|
| 288 |
+
|
| 289 |
+
logger.info(f"Model weights successfully saved to {save_directory}!")
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
def split_torch_state_dict_into_shards(
|
| 293 |
+
state_dict: dict[str, "torch.Tensor"],
|
| 294 |
+
*,
|
| 295 |
+
filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN,
|
| 296 |
+
max_shard_size: int | str = MAX_SHARD_SIZE,
|
| 297 |
+
) -> StateDictSplit:
|
| 298 |
+
"""
|
| 299 |
+
Split a model state dictionary in shards so that each shard is smaller than a given size.
|
| 300 |
+
|
| 301 |
+
The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization
|
| 302 |
+
made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we
|
| 303 |
+
have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not
|
| 304 |
+
[6+2+2GB], [6+2GB], [6GB].
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
> [!TIP]
|
| 308 |
+
> To save a model state dictionary to the disk, see [`save_torch_state_dict`]. This helper uses
|
| 309 |
+
> `split_torch_state_dict_into_shards` under the hood.
|
| 310 |
+
|
| 311 |
+
> [!WARNING]
|
| 312 |
+
> If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a
|
| 313 |
+
> size greater than `max_shard_size`.
|
| 314 |
+
|
| 315 |
+
Args:
|
| 316 |
+
state_dict (`dict[str, torch.Tensor]`):
|
| 317 |
+
The state dictionary to save.
|
| 318 |
+
filename_pattern (`str`, *optional*):
|
| 319 |
+
The pattern to generate the files names in which the model will be saved. Pattern must be a string that
|
| 320 |
+
can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
|
| 321 |
+
Defaults to `"model{suffix}.safetensors"`.
|
| 322 |
+
max_shard_size (`int` or `str`, *optional*):
|
| 323 |
+
The maximum size of each shard, in bytes. Defaults to 5GB.
|
| 324 |
+
|
| 325 |
+
Returns:
|
| 326 |
+
[`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them.
|
| 327 |
+
|
| 328 |
+
Example:
|
| 329 |
+
```py
|
| 330 |
+
>>> import json
|
| 331 |
+
>>> import os
|
| 332 |
+
>>> from safetensors.torch import save_file as safe_save_file
|
| 333 |
+
>>> from huggingface_hub import split_torch_state_dict_into_shards
|
| 334 |
+
|
| 335 |
+
>>> def save_state_dict(state_dict: dict[str, torch.Tensor], save_directory: str):
|
| 336 |
+
... state_dict_split = split_torch_state_dict_into_shards(state_dict)
|
| 337 |
+
... for filename, tensors in state_dict_split.filename_to_tensors.items():
|
| 338 |
+
... shard = {tensor: state_dict[tensor] for tensor in tensors}
|
| 339 |
+
... safe_save_file(
|
| 340 |
+
... shard,
|
| 341 |
+
... os.path.join(save_directory, filename),
|
| 342 |
+
... metadata={"format": "pt"},
|
| 343 |
+
... )
|
| 344 |
+
... if state_dict_split.is_sharded:
|
| 345 |
+
... index = {
|
| 346 |
+
... "metadata": state_dict_split.metadata,
|
| 347 |
+
... "weight_map": state_dict_split.tensor_to_filename,
|
| 348 |
+
... }
|
| 349 |
+
... with open(os.path.join(save_directory, "model.safetensors.index.json"), "w") as f:
|
| 350 |
+
... f.write(json.dumps(index, indent=2))
|
| 351 |
+
```
|
| 352 |
+
"""
|
| 353 |
+
return split_state_dict_into_shards_factory(
|
| 354 |
+
state_dict,
|
| 355 |
+
max_shard_size=max_shard_size,
|
| 356 |
+
filename_pattern=filename_pattern,
|
| 357 |
+
get_storage_size=get_torch_storage_size,
|
| 358 |
+
get_storage_id=get_torch_storage_id,
|
| 359 |
+
)
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
# LOADING
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
def load_torch_model(
|
| 366 |
+
model: "torch.nn.Module",
|
| 367 |
+
checkpoint_path: str | os.PathLike,
|
| 368 |
+
*,
|
| 369 |
+
strict: bool = False,
|
| 370 |
+
safe: bool = True,
|
| 371 |
+
weights_only: bool = False,
|
| 372 |
+
map_location: Union[str, "torch.device"] | None = None,
|
| 373 |
+
mmap: bool = False,
|
| 374 |
+
filename_pattern: str | None = None,
|
| 375 |
+
) -> NamedTuple:
|
| 376 |
+
"""
|
| 377 |
+
Load a checkpoint into a model, handling both sharded and non-sharded checkpoints.
|
| 378 |
+
|
| 379 |
+
Args:
|
| 380 |
+
model (`torch.nn.Module`):
|
| 381 |
+
The model in which to load the checkpoint.
|
| 382 |
+
checkpoint_path (`str` or `os.PathLike`):
|
| 383 |
+
Path to either the checkpoint file or directory containing the checkpoint(s).
|
| 384 |
+
strict (`bool`, *optional*, defaults to `False`):
|
| 385 |
+
Whether to strictly enforce that the keys in the model state dict match the keys in the checkpoint.
|
| 386 |
+
safe (`bool`, *optional*, defaults to `True`):
|
| 387 |
+
If `safe` is True, the safetensors files will be loaded. If `safe` is False, the function
|
| 388 |
+
will first attempt to load safetensors files if they are available, otherwise it will fall back to loading
|
| 389 |
+
pickle files. `filename_pattern` parameter takes precedence over `safe` parameter.
|
| 390 |
+
weights_only (`bool`, *optional*, defaults to `False`):
|
| 391 |
+
If True, only loads the model weights without optimizer states and other metadata.
|
| 392 |
+
Only supported in PyTorch >= 1.13.
|
| 393 |
+
map_location (`str` or `torch.device`, *optional*):
|
| 394 |
+
A `torch.device` object, string or a dict specifying how to remap storage locations. It
|
| 395 |
+
indicates the location where all tensors should be loaded.
|
| 396 |
+
mmap (`bool`, *optional*, defaults to `False`):
|
| 397 |
+
Whether to use memory-mapped file loading. Memory mapping can improve loading performance
|
| 398 |
+
for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints.
|
| 399 |
+
filename_pattern (`str`, *optional*):
|
| 400 |
+
The pattern to look for the index file. Pattern must be a string that
|
| 401 |
+
can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
|
| 402 |
+
Defaults to `"model{suffix}.safetensors"`.
|
| 403 |
+
Returns:
|
| 404 |
+
`NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields.
|
| 405 |
+
- `missing_keys` is a list of str containing the missing keys, i.e. keys that are in the model but not in the checkpoint.
|
| 406 |
+
- `unexpected_keys` is a list of str containing the unexpected keys, i.e. keys that are in the checkpoint but not in the model.
|
| 407 |
+
|
| 408 |
+
Raises:
|
| 409 |
+
[`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError)
|
| 410 |
+
If the checkpoint file or directory does not exist.
|
| 411 |
+
[`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
|
| 412 |
+
If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively.
|
| 413 |
+
[`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
| 414 |
+
If the checkpoint path is invalid or if the checkpoint format cannot be determined.
|
| 415 |
+
|
| 416 |
+
Example:
|
| 417 |
+
```python
|
| 418 |
+
>>> from huggingface_hub import load_torch_model
|
| 419 |
+
>>> model = ... # A PyTorch model
|
| 420 |
+
>>> load_torch_model(model, "path/to/checkpoint")
|
| 421 |
+
```
|
| 422 |
+
"""
|
| 423 |
+
checkpoint_path = Path(checkpoint_path)
|
| 424 |
+
|
| 425 |
+
if not checkpoint_path.exists():
|
| 426 |
+
raise ValueError(f"Checkpoint path {checkpoint_path} does not exist")
|
| 427 |
+
# 1. Check if checkpoint is a single file
|
| 428 |
+
if checkpoint_path.is_file():
|
| 429 |
+
state_dict = load_state_dict_from_file(
|
| 430 |
+
checkpoint_file=checkpoint_path,
|
| 431 |
+
map_location=map_location,
|
| 432 |
+
weights_only=weights_only,
|
| 433 |
+
)
|
| 434 |
+
return model.load_state_dict(state_dict, strict=strict)
|
| 435 |
+
|
| 436 |
+
# 2. If not, checkpoint_path is a directory
|
| 437 |
+
if filename_pattern is None:
|
| 438 |
+
filename_pattern = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN
|
| 439 |
+
index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json")
|
| 440 |
+
# Only fallback to pickle format if safetensors index is not found and safe is False.
|
| 441 |
+
if not index_path.is_file() and not safe:
|
| 442 |
+
filename_pattern = constants.PYTORCH_WEIGHTS_FILE_PATTERN
|
| 443 |
+
|
| 444 |
+
index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json")
|
| 445 |
+
|
| 446 |
+
if index_path.is_file():
|
| 447 |
+
return _load_sharded_checkpoint(
|
| 448 |
+
model=model,
|
| 449 |
+
save_directory=checkpoint_path,
|
| 450 |
+
strict=strict,
|
| 451 |
+
weights_only=weights_only,
|
| 452 |
+
filename_pattern=filename_pattern,
|
| 453 |
+
)
|
| 454 |
+
|
| 455 |
+
# Look for single model file
|
| 456 |
+
model_files = list(checkpoint_path.glob("*.safetensors" if safe else "*.bin"))
|
| 457 |
+
if len(model_files) == 1:
|
| 458 |
+
state_dict = load_state_dict_from_file(
|
| 459 |
+
checkpoint_file=model_files[0],
|
| 460 |
+
map_location=map_location,
|
| 461 |
+
weights_only=weights_only,
|
| 462 |
+
mmap=mmap,
|
| 463 |
+
)
|
| 464 |
+
return model.load_state_dict(state_dict, strict=strict)
|
| 465 |
+
|
| 466 |
+
raise ValueError(
|
| 467 |
+
f"Directory '{checkpoint_path}' does not contain a valid checkpoint. "
|
| 468 |
+
"Expected either a sharded checkpoint with an index file, or a single model file."
|
| 469 |
+
)
|
| 470 |
+
|
| 471 |
+
|
| 472 |
+
def _load_sharded_checkpoint(
|
| 473 |
+
model: "torch.nn.Module",
|
| 474 |
+
save_directory: os.PathLike,
|
| 475 |
+
*,
|
| 476 |
+
strict: bool = False,
|
| 477 |
+
weights_only: bool = False,
|
| 478 |
+
filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN,
|
| 479 |
+
) -> NamedTuple:
|
| 480 |
+
"""
|
| 481 |
+
Loads a sharded checkpoint into a model. This is the same as
|
| 482 |
+
[`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict)
|
| 483 |
+
but for a sharded checkpoint. Each shard is loaded one by one and removed from memory after being loaded into the model.
|
| 484 |
+
|
| 485 |
+
Args:
|
| 486 |
+
model (`torch.nn.Module`):
|
| 487 |
+
The model in which to load the checkpoint.
|
| 488 |
+
save_directory (`str` or `os.PathLike`):
|
| 489 |
+
A path to a folder containing the sharded checkpoint.
|
| 490 |
+
strict (`bool`, *optional*, defaults to `False`):
|
| 491 |
+
Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint.
|
| 492 |
+
weights_only (`bool`, *optional*, defaults to `False`):
|
| 493 |
+
If True, only loads the model weights without optimizer states and other metadata.
|
| 494 |
+
Only supported in PyTorch >= 1.13.
|
| 495 |
+
filename_pattern (`str`, *optional*, defaults to `"model{suffix}.safetensors"`):
|
| 496 |
+
The pattern to look for the index file. Pattern must be a string that
|
| 497 |
+
can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
|
| 498 |
+
Defaults to `"model{suffix}.safetensors"`.
|
| 499 |
+
|
| 500 |
+
Returns:
|
| 501 |
+
`NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields,
|
| 502 |
+
- `missing_keys` is a list of str containing the missing keys
|
| 503 |
+
- `unexpected_keys` is a list of str containing the unexpected keys
|
| 504 |
+
"""
|
| 505 |
+
|
| 506 |
+
# 1. Load and validate index file
|
| 507 |
+
# The index file contains mapping of parameter names to shard files
|
| 508 |
+
index_path = filename_pattern.format(suffix="") + ".index.json"
|
| 509 |
+
index_file = os.path.join(save_directory, index_path)
|
| 510 |
+
with open(index_file, encoding="utf-8") as f:
|
| 511 |
+
index = json.load(f)
|
| 512 |
+
|
| 513 |
+
# 2. Validate shard filenames from the index
|
| 514 |
+
# This prevents path traversal attacks and extension confusion attacks
|
| 515 |
+
# (e.g. a safetensors index referencing .bin pickle files)
|
| 516 |
+
expected_extension = Path(filename_pattern.format(suffix="")).suffix # e.g. ".safetensors"
|
| 517 |
+
shard_files = list(set(index["weight_map"].values()))
|
| 518 |
+
for shard_file in shard_files:
|
| 519 |
+
# Reject anything that could escape `save_directory` on any host OS:
|
| 520 |
+
# POSIX absolute ("/tmp/x"), Windows drive ("C:x", "C:\\x"), UNC
|
| 521 |
+
# ("\\\\server\\share\\x"), rooted-without-drive ("\\x", "/x"), or
|
| 522 |
+
# ".." traversal — including "..\\x" which `os.path.isabs` never caught on POSIX.
|
| 523 |
+
#
|
| 524 |
+
# We parse with `PureWindowsPath` *regardless of host OS*: it treats both "/" and
|
| 525 |
+
# "\\" as separators and exposes `drive` / `root`, so a single check rejects a
|
| 526 |
+
# malicious index file on Linux too (e.g. if it's later opened on Windows). The
|
| 527 |
+
# only over-strict case is a POSIX filename like "a:foo" which would be parsed as
|
| 528 |
+
# drive "a:" — such names are never produced for safetensors shards and would
|
| 529 |
+
# break on Windows anyway, so rejecting them is fine.
|
| 530 |
+
win_path = PureWindowsPath(shard_file)
|
| 531 |
+
if win_path.drive or win_path.root or ".." in win_path.parts:
|
| 532 |
+
raise ValueError(
|
| 533 |
+
f"Invalid shard filename '{shard_file}' in index file '{index_file}'. "
|
| 534 |
+
"Shard filenames must be relative paths without '..' components."
|
| 535 |
+
)
|
| 536 |
+
# Reject extension mismatch (e.g. .bin shard in a .safetensors index)
|
| 537 |
+
if not shard_file.endswith(expected_extension):
|
| 538 |
+
raise ValueError(
|
| 539 |
+
f"Invalid shard filename '{shard_file}' in index file '{index_file}'. "
|
| 540 |
+
f"Expected '{expected_extension}' extension to match the index format."
|
| 541 |
+
)
|
| 542 |
+
|
| 543 |
+
# 3. Validate keys if in strict mode
|
| 544 |
+
# This is done before loading any shards to fail fast
|
| 545 |
+
if strict:
|
| 546 |
+
_validate_keys_for_strict_loading(model, index["weight_map"].keys())
|
| 547 |
+
|
| 548 |
+
# 4. Load each shard using `load_state_dict`
|
| 549 |
+
# Get unique shard files (multiple parameters can be in same shard)
|
| 550 |
+
for shard_file in shard_files:
|
| 551 |
+
# Load shard into memory
|
| 552 |
+
shard_path = os.path.join(save_directory, shard_file)
|
| 553 |
+
state_dict = load_state_dict_from_file(
|
| 554 |
+
shard_path,
|
| 555 |
+
map_location="cpu",
|
| 556 |
+
weights_only=weights_only,
|
| 557 |
+
)
|
| 558 |
+
# Update model with parameters from this shard
|
| 559 |
+
model.load_state_dict(state_dict, strict=strict)
|
| 560 |
+
# Explicitly remove the state dict from memory
|
| 561 |
+
del state_dict
|
| 562 |
+
|
| 563 |
+
# 5. Return compatibility info
|
| 564 |
+
loaded_keys = set(index["weight_map"].keys())
|
| 565 |
+
model_keys = set(model.state_dict().keys())
|
| 566 |
+
return _IncompatibleKeys(
|
| 567 |
+
missing_keys=list(model_keys - loaded_keys), unexpected_keys=list(loaded_keys - model_keys)
|
| 568 |
+
)
|
| 569 |
+
|
| 570 |
+
|
| 571 |
+
def load_state_dict_from_file(
|
| 572 |
+
checkpoint_file: str | os.PathLike,
|
| 573 |
+
map_location: Union[str, "torch.device"] | None = None,
|
| 574 |
+
weights_only: bool = False,
|
| 575 |
+
mmap: bool = False,
|
| 576 |
+
) -> dict[str, "torch.Tensor"] | Any:
|
| 577 |
+
"""
|
| 578 |
+
Loads a checkpoint file, handling both safetensors and pickle checkpoint formats.
|
| 579 |
+
|
| 580 |
+
Args:
|
| 581 |
+
checkpoint_file (`str` or `os.PathLike`):
|
| 582 |
+
Path to the checkpoint file to load. Can be either a safetensors or pickle (`.bin`) checkpoint.
|
| 583 |
+
map_location (`str` or `torch.device`, *optional*):
|
| 584 |
+
A `torch.device` object, string or a dict specifying how to remap storage locations. It
|
| 585 |
+
indicates the location where all tensors should be loaded.
|
| 586 |
+
weights_only (`bool`, *optional*, defaults to `False`):
|
| 587 |
+
If True, only loads the model weights without optimizer states and other metadata.
|
| 588 |
+
Only supported for pickle (`.bin`) checkpoints with PyTorch >= 1.13. Has no effect when
|
| 589 |
+
loading safetensors files.
|
| 590 |
+
mmap (`bool`, *optional*, defaults to `False`):
|
| 591 |
+
Whether to use memory-mapped file loading. Memory mapping can improve loading performance
|
| 592 |
+
for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints. Has no effect when
|
| 593 |
+
loading safetensors files, as the `safetensors` library uses memory mapping by default.
|
| 594 |
+
|
| 595 |
+
Returns:
|
| 596 |
+
`Union[dict[str, "torch.Tensor"], Any]`: The loaded checkpoint.
|
| 597 |
+
- For safetensors files: always returns a dictionary mapping parameter names to tensors.
|
| 598 |
+
- For pickle files: returns any Python object that was pickled (commonly a state dict, but could be
|
| 599 |
+
an entire model, optimizer state, or any other Python object).
|
| 600 |
+
|
| 601 |
+
Raises:
|
| 602 |
+
[`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError)
|
| 603 |
+
If the checkpoint file does not exist.
|
| 604 |
+
[`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
|
| 605 |
+
If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively.
|
| 606 |
+
[`OSError`](https://docs.python.org/3/library/exceptions.html#OSError)
|
| 607 |
+
If the checkpoint file format is invalid or if git-lfs files are not properly downloaded.
|
| 608 |
+
[`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
| 609 |
+
If the checkpoint file path is empty or invalid.
|
| 610 |
+
|
| 611 |
+
Example:
|
| 612 |
+
```python
|
| 613 |
+
>>> from huggingface_hub import load_state_dict_from_file
|
| 614 |
+
|
| 615 |
+
# Load a PyTorch checkpoint
|
| 616 |
+
>>> state_dict = load_state_dict_from_file("path/to/model.bin", map_location="cpu")
|
| 617 |
+
>>> model.load_state_dict(state_dict)
|
| 618 |
+
|
| 619 |
+
# Load a safetensors checkpoint
|
| 620 |
+
>>> state_dict = load_state_dict_from_file("path/to/model.safetensors")
|
| 621 |
+
>>> model.load_state_dict(state_dict)
|
| 622 |
+
```
|
| 623 |
+
"""
|
| 624 |
+
checkpoint_path = Path(checkpoint_file)
|
| 625 |
+
|
| 626 |
+
# Check if file exists and is a regular file (not a directory)
|
| 627 |
+
if not checkpoint_path.is_file():
|
| 628 |
+
raise FileNotFoundError(
|
| 629 |
+
f"No checkpoint file found at '{checkpoint_path}'. Please verify the path is correct and "
|
| 630 |
+
"the file has been properly downloaded."
|
| 631 |
+
)
|
| 632 |
+
|
| 633 |
+
# Load safetensors checkpoint
|
| 634 |
+
if checkpoint_path.suffix == ".safetensors":
|
| 635 |
+
try:
|
| 636 |
+
from safetensors import safe_open
|
| 637 |
+
from safetensors.torch import load_file
|
| 638 |
+
except ImportError as e:
|
| 639 |
+
raise ImportError(
|
| 640 |
+
"Please install `safetensors` to load safetensors checkpoint. "
|
| 641 |
+
"You can install it with `pip install safetensors`."
|
| 642 |
+
) from e
|
| 643 |
+
|
| 644 |
+
# Check format of the archive
|
| 645 |
+
with safe_open(checkpoint_file, framework="pt") as f: # type: ignore[attr-defined]
|
| 646 |
+
metadata = f.metadata()
|
| 647 |
+
# see comment: https://github.com/huggingface/transformers/blob/3d213b57fe74302e5902d68ed9478c3ad1aaa713/src/transformers/modeling_utils.py#L3966
|
| 648 |
+
if metadata is not None and metadata.get("format") not in ["pt", "mlx"]:
|
| 649 |
+
raise OSError(
|
| 650 |
+
f"The safetensors archive passed at {checkpoint_file} does not contain the valid metadata. Make sure "
|
| 651 |
+
"you save your model with the `save_torch_model` method."
|
| 652 |
+
)
|
| 653 |
+
device = str(map_location.type) if map_location is not None and hasattr(map_location, "type") else map_location
|
| 654 |
+
# meta device is not supported with safetensors, falling back to CPU
|
| 655 |
+
if device == "meta":
|
| 656 |
+
logger.warning("Meta device is not supported with safetensors. Falling back to CPU device.")
|
| 657 |
+
device = "cpu"
|
| 658 |
+
return load_file(checkpoint_file, device=device) # type: ignore[arg-type]
|
| 659 |
+
# Otherwise, load from pickle
|
| 660 |
+
try:
|
| 661 |
+
import torch
|
| 662 |
+
from torch import load
|
| 663 |
+
except ImportError as e:
|
| 664 |
+
raise ImportError(
|
| 665 |
+
"Please install `torch` to load torch tensors. You can install it with `pip install torch`."
|
| 666 |
+
) from e
|
| 667 |
+
# Add additional kwargs, mmap is only supported in torch >= 2.1.0
|
| 668 |
+
additional_kwargs = {}
|
| 669 |
+
if version.parse(torch.__version__) >= version.parse("2.1.0"):
|
| 670 |
+
additional_kwargs["mmap"] = mmap
|
| 671 |
+
|
| 672 |
+
# weights_only is only supported in torch >= 1.13.0
|
| 673 |
+
if version.parse(torch.__version__) >= version.parse("1.13.0"):
|
| 674 |
+
additional_kwargs["weights_only"] = weights_only
|
| 675 |
+
|
| 676 |
+
return load(
|
| 677 |
+
checkpoint_file,
|
| 678 |
+
map_location=map_location,
|
| 679 |
+
**additional_kwargs,
|
| 680 |
+
)
|
| 681 |
+
|
| 682 |
+
|
| 683 |
+
# HELPERS
|
| 684 |
+
|
| 685 |
+
|
| 686 |
+
def _validate_keys_for_strict_loading(
|
| 687 |
+
model: "torch.nn.Module",
|
| 688 |
+
loaded_keys: Iterable[str],
|
| 689 |
+
) -> None:
|
| 690 |
+
"""
|
| 691 |
+
Validate that model keys match loaded keys when strict loading is enabled.
|
| 692 |
+
|
| 693 |
+
Args:
|
| 694 |
+
model: The PyTorch model being loaded
|
| 695 |
+
loaded_keys: The keys present in the checkpoint
|
| 696 |
+
|
| 697 |
+
Raises:
|
| 698 |
+
RuntimeError: If there are missing or unexpected keys in strict mode
|
| 699 |
+
"""
|
| 700 |
+
loaded_keys_set = set(loaded_keys)
|
| 701 |
+
model_keys = set(model.state_dict().keys())
|
| 702 |
+
missing_keys = model_keys - loaded_keys_set # Keys in model but not in checkpoint
|
| 703 |
+
unexpected_keys = loaded_keys_set - model_keys # Keys in checkpoint but not in model
|
| 704 |
+
|
| 705 |
+
if missing_keys or unexpected_keys:
|
| 706 |
+
error_message = f"Error(s) in loading state_dict for {model.__class__.__name__}"
|
| 707 |
+
if missing_keys:
|
| 708 |
+
str_missing_keys = ",".join([f'"{k}"' for k in sorted(missing_keys)])
|
| 709 |
+
error_message += f"\nMissing key(s): {str_missing_keys}."
|
| 710 |
+
if unexpected_keys:
|
| 711 |
+
str_unexpected_keys = ",".join([f'"{k}"' for k in sorted(unexpected_keys)])
|
| 712 |
+
error_message += f"\nUnexpected key(s): {str_unexpected_keys}."
|
| 713 |
+
raise RuntimeError(error_message)
|
| 714 |
+
|
| 715 |
+
|
| 716 |
+
def _get_unique_id(tensor: "torch.Tensor") -> int | tuple[Any, ...]:
|
| 717 |
+
"""Returns a unique id for plain tensor
|
| 718 |
+
or a (potentially nested) Tuple of unique id for the flattened Tensor
|
| 719 |
+
if the input is a wrapper tensor subclass Tensor
|
| 720 |
+
"""
|
| 721 |
+
|
| 722 |
+
try:
|
| 723 |
+
from torch.distributed.tensor import DTensor
|
| 724 |
+
|
| 725 |
+
if isinstance(tensor, DTensor):
|
| 726 |
+
local_tensor = tensor.to_local()
|
| 727 |
+
return local_tensor.storage().data_ptr()
|
| 728 |
+
except ImportError:
|
| 729 |
+
pass
|
| 730 |
+
|
| 731 |
+
try:
|
| 732 |
+
# for torch 2.1 and above we can also handle tensor subclasses
|
| 733 |
+
from torch.utils._python_dispatch import is_traceable_wrapper_subclass
|
| 734 |
+
|
| 735 |
+
if is_traceable_wrapper_subclass(tensor):
|
| 736 |
+
attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined]
|
| 737 |
+
return tuple(_get_unique_id(getattr(tensor, attr)) for attr in attrs)
|
| 738 |
+
|
| 739 |
+
except ImportError:
|
| 740 |
+
# for torch version less than 2.1, we can fall back to original implementation
|
| 741 |
+
pass
|
| 742 |
+
|
| 743 |
+
if tensor.device.type == "xla" and is_torch_tpu_available():
|
| 744 |
+
# NOTE: xla tensors don't have storage
|
| 745 |
+
# use some other unique id to distinguish.
|
| 746 |
+
# this is a XLA tensor, it must be created using torch_xla's
|
| 747 |
+
# device. So the following import is safe:
|
| 748 |
+
import torch_xla # type: ignore[import]
|
| 749 |
+
|
| 750 |
+
unique_id = torch_xla._XLAC._xla_get_tensor_id(tensor)
|
| 751 |
+
else:
|
| 752 |
+
unique_id = storage_ptr(tensor)
|
| 753 |
+
|
| 754 |
+
return unique_id
|
| 755 |
+
|
| 756 |
+
|
| 757 |
+
def get_torch_storage_id(tensor: "torch.Tensor") -> tuple["torch.device", int | tuple[Any, ...], int] | None:
|
| 758 |
+
"""
|
| 759 |
+
Return unique identifier to a tensor storage.
|
| 760 |
+
|
| 761 |
+
Multiple different tensors can share the same underlying storage. This identifier is
|
| 762 |
+
guaranteed to be unique and constant for this tensor's storage during its lifetime. Two tensor storages with
|
| 763 |
+
non-overlapping lifetimes may have the same id.
|
| 764 |
+
In the case of meta tensors, we return None since we can't tell if they share the same storage.
|
| 765 |
+
|
| 766 |
+
Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/pytorch_utils.py#L278.
|
| 767 |
+
"""
|
| 768 |
+
if tensor.device.type == "meta":
|
| 769 |
+
return None
|
| 770 |
+
else:
|
| 771 |
+
return tensor.device, _get_unique_id(tensor), get_torch_storage_size(tensor)
|
| 772 |
+
|
| 773 |
+
|
| 774 |
+
def get_torch_storage_size(tensor: "torch.Tensor") -> int:
|
| 775 |
+
"""
|
| 776 |
+
Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L31C1-L41C59
|
| 777 |
+
"""
|
| 778 |
+
try:
|
| 779 |
+
from torch.distributed.tensor import DTensor
|
| 780 |
+
|
| 781 |
+
if isinstance(tensor, DTensor):
|
| 782 |
+
# this returns the size of the FULL tensor in bytes
|
| 783 |
+
return tensor.nbytes
|
| 784 |
+
except ImportError:
|
| 785 |
+
pass
|
| 786 |
+
|
| 787 |
+
try:
|
| 788 |
+
# for torch 2.1 and above we can also handle tensor subclasses
|
| 789 |
+
from torch.utils._python_dispatch import is_traceable_wrapper_subclass
|
| 790 |
+
|
| 791 |
+
if is_traceable_wrapper_subclass(tensor):
|
| 792 |
+
attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined]
|
| 793 |
+
return sum(get_torch_storage_size(getattr(tensor, attr)) for attr in attrs)
|
| 794 |
+
except ImportError:
|
| 795 |
+
# for torch version less than 2.1, we can fall back to original implementation
|
| 796 |
+
pass
|
| 797 |
+
|
| 798 |
+
try:
|
| 799 |
+
return tensor.untyped_storage().nbytes()
|
| 800 |
+
except AttributeError:
|
| 801 |
+
# Fallback for torch==1.10
|
| 802 |
+
try:
|
| 803 |
+
return tensor.storage().size() * _get_dtype_size(tensor.dtype)
|
| 804 |
+
except NotImplementedError:
|
| 805 |
+
# Fallback for meta storage
|
| 806 |
+
# On torch >=2.0 this is the tensor size
|
| 807 |
+
return tensor.nelement() * _get_dtype_size(tensor.dtype)
|
| 808 |
+
|
| 809 |
+
|
| 810 |
+
@lru_cache
|
| 811 |
+
def is_torch_tpu_available(check_device=True):
|
| 812 |
+
"""
|
| 813 |
+
Checks if `torch_xla` is installed and potentially if a TPU is in the environment
|
| 814 |
+
|
| 815 |
+
Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/utils/import_utils.py#L463.
|
| 816 |
+
"""
|
| 817 |
+
if importlib.util.find_spec("torch_xla") is not None:
|
| 818 |
+
if check_device:
|
| 819 |
+
# We need to check if `xla_device` can be found, will raise a RuntimeError if not
|
| 820 |
+
try:
|
| 821 |
+
import torch_xla.core.xla_model as xm # type: ignore[import]
|
| 822 |
+
|
| 823 |
+
_ = xm.xla_device()
|
| 824 |
+
return True
|
| 825 |
+
except RuntimeError:
|
| 826 |
+
return False
|
| 827 |
+
return True
|
| 828 |
+
return False
|
| 829 |
+
|
| 830 |
+
|
| 831 |
+
def storage_ptr(tensor: "torch.Tensor") -> int | tuple[Any, ...]:
|
| 832 |
+
"""
|
| 833 |
+
Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L11.
|
| 834 |
+
"""
|
| 835 |
+
try:
|
| 836 |
+
# for torch 2.1 and above we can also handle tensor subclasses
|
| 837 |
+
from torch.utils._python_dispatch import is_traceable_wrapper_subclass
|
| 838 |
+
|
| 839 |
+
if is_traceable_wrapper_subclass(tensor):
|
| 840 |
+
return _get_unique_id(tensor) # type: ignore
|
| 841 |
+
except ImportError:
|
| 842 |
+
# for torch version less than 2.1, we can fall back to original implementation
|
| 843 |
+
pass
|
| 844 |
+
|
| 845 |
+
try:
|
| 846 |
+
return tensor.untyped_storage().data_ptr()
|
| 847 |
+
except Exception:
|
| 848 |
+
# Fallback for torch==1.10
|
| 849 |
+
try:
|
| 850 |
+
return tensor.storage().data_ptr()
|
| 851 |
+
except NotImplementedError:
|
| 852 |
+
# Fallback for meta storage
|
| 853 |
+
return 0
|
| 854 |
+
|
| 855 |
+
|
| 856 |
+
def _clean_state_dict_for_safetensors(
|
| 857 |
+
state_dict: dict[str, "torch.Tensor"],
|
| 858 |
+
metadata: dict[str, str],
|
| 859 |
+
force_contiguous: bool = True,
|
| 860 |
+
shared_tensors_to_discard: list[str] | None = None,
|
| 861 |
+
):
|
| 862 |
+
"""Remove shared tensors from state_dict and update metadata accordingly (for reloading).
|
| 863 |
+
|
| 864 |
+
Warning: `state_dict` and `metadata` are mutated in-place!
|
| 865 |
+
|
| 866 |
+
Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L155.
|
| 867 |
+
"""
|
| 868 |
+
to_removes = _remove_duplicate_names(state_dict, discard_names=shared_tensors_to_discard)
|
| 869 |
+
for kept_name, to_remove_group in to_removes.items():
|
| 870 |
+
for to_remove in to_remove_group:
|
| 871 |
+
if metadata is None:
|
| 872 |
+
metadata = {}
|
| 873 |
+
|
| 874 |
+
if to_remove not in metadata:
|
| 875 |
+
# Do not override user data
|
| 876 |
+
metadata[to_remove] = kept_name
|
| 877 |
+
del state_dict[to_remove]
|
| 878 |
+
if force_contiguous:
|
| 879 |
+
state_dict = {k: v.contiguous() for k, v in state_dict.items()}
|
| 880 |
+
return state_dict
|
| 881 |
+
|
| 882 |
+
|
| 883 |
+
def _end_ptr(tensor: "torch.Tensor") -> int:
|
| 884 |
+
"""
|
| 885 |
+
Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L23.
|
| 886 |
+
"""
|
| 887 |
+
if tensor.nelement():
|
| 888 |
+
stop = tensor.view(-1)[-1].data_ptr() + _get_dtype_size(tensor.dtype)
|
| 889 |
+
else:
|
| 890 |
+
stop = tensor.data_ptr()
|
| 891 |
+
return stop
|
| 892 |
+
|
| 893 |
+
|
| 894 |
+
def _filter_shared_not_shared(tensors: list[set[str]], state_dict: dict[str, "torch.Tensor"]) -> list[set[str]]:
|
| 895 |
+
"""
|
| 896 |
+
Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L44
|
| 897 |
+
"""
|
| 898 |
+
filtered_tensors = []
|
| 899 |
+
for shared in tensors:
|
| 900 |
+
if len(shared) < 2:
|
| 901 |
+
filtered_tensors.append(shared)
|
| 902 |
+
continue
|
| 903 |
+
|
| 904 |
+
areas = []
|
| 905 |
+
for name in shared:
|
| 906 |
+
tensor = state_dict[name]
|
| 907 |
+
areas.append((tensor.data_ptr(), _end_ptr(tensor), name))
|
| 908 |
+
areas.sort()
|
| 909 |
+
|
| 910 |
+
_, last_stop, last_name = areas[0]
|
| 911 |
+
filtered_tensors.append({last_name})
|
| 912 |
+
for start, stop, name in areas[1:]:
|
| 913 |
+
if start >= last_stop:
|
| 914 |
+
filtered_tensors.append({name})
|
| 915 |
+
else:
|
| 916 |
+
filtered_tensors[-1].add(name)
|
| 917 |
+
last_stop = stop
|
| 918 |
+
|
| 919 |
+
return filtered_tensors
|
| 920 |
+
|
| 921 |
+
|
| 922 |
+
def _find_shared_tensors(state_dict: dict[str, "torch.Tensor"]) -> list[set[str]]:
|
| 923 |
+
"""
|
| 924 |
+
Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L69.
|
| 925 |
+
"""
|
| 926 |
+
import torch
|
| 927 |
+
|
| 928 |
+
tensors_dict = defaultdict(set)
|
| 929 |
+
for k, v in state_dict.items():
|
| 930 |
+
if v.device != torch.device("meta") and storage_ptr(v) != 0 and get_torch_storage_size(v) != 0:
|
| 931 |
+
# Need to add device as key because of multiple GPU.
|
| 932 |
+
tensors_dict[(v.device, storage_ptr(v), get_torch_storage_size(v))].add(k)
|
| 933 |
+
tensors = list(sorted(tensors_dict.values()))
|
| 934 |
+
tensors = _filter_shared_not_shared(tensors, state_dict)
|
| 935 |
+
return tensors
|
| 936 |
+
|
| 937 |
+
|
| 938 |
+
def _is_complete(tensor: "torch.Tensor") -> bool:
|
| 939 |
+
"""
|
| 940 |
+
Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80
|
| 941 |
+
"""
|
| 942 |
+
try:
|
| 943 |
+
# for torch 2.1 and above we can also handle tensor subclasses
|
| 944 |
+
from torch.utils._python_dispatch import is_traceable_wrapper_subclass
|
| 945 |
+
|
| 946 |
+
if is_traceable_wrapper_subclass(tensor):
|
| 947 |
+
attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined]
|
| 948 |
+
return all(_is_complete(getattr(tensor, attr)) for attr in attrs)
|
| 949 |
+
except ImportError:
|
| 950 |
+
# for torch version less than 2.1, we can fall back to original implementation
|
| 951 |
+
pass
|
| 952 |
+
|
| 953 |
+
return tensor.data_ptr() == storage_ptr(tensor) and tensor.nelement() * _get_dtype_size(
|
| 954 |
+
tensor.dtype
|
| 955 |
+
) == get_torch_storage_size(tensor)
|
| 956 |
+
|
| 957 |
+
|
| 958 |
+
def _remove_duplicate_names(
|
| 959 |
+
state_dict: dict[str, "torch.Tensor"],
|
| 960 |
+
*,
|
| 961 |
+
preferred_names: list[str] | None = None,
|
| 962 |
+
discard_names: list[str] | None = None,
|
| 963 |
+
) -> dict[str, list[str]]:
|
| 964 |
+
"""
|
| 965 |
+
Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80
|
| 966 |
+
"""
|
| 967 |
+
if preferred_names is None:
|
| 968 |
+
preferred_names = []
|
| 969 |
+
unique_preferred_names = set(preferred_names)
|
| 970 |
+
if discard_names is None:
|
| 971 |
+
discard_names = []
|
| 972 |
+
unique_discard_names = set(discard_names)
|
| 973 |
+
|
| 974 |
+
shareds = _find_shared_tensors(state_dict)
|
| 975 |
+
to_remove = defaultdict(list)
|
| 976 |
+
for shared in shareds:
|
| 977 |
+
complete_names = {name for name in shared if _is_complete(state_dict[name])}
|
| 978 |
+
if not complete_names:
|
| 979 |
+
raise RuntimeError(
|
| 980 |
+
"Error while trying to find names to remove to save state dict, but found no suitable name to keep"
|
| 981 |
+
f" for saving amongst: {shared}. None is covering the entire storage. Refusing to save/load the model"
|
| 982 |
+
" since you could be storing much more memory than needed. Please refer to"
|
| 983 |
+
" https://huggingface.co/docs/safetensors/torch_shared_tensors for more information. Or open an"
|
| 984 |
+
" issue."
|
| 985 |
+
)
|
| 986 |
+
|
| 987 |
+
keep_name = sorted(list(complete_names))[0]
|
| 988 |
+
|
| 989 |
+
# Mechanism to preferentially select keys to keep
|
| 990 |
+
# coming from the on-disk file to allow
|
| 991 |
+
# loading models saved with a different choice
|
| 992 |
+
# of keep_name
|
| 993 |
+
preferred = complete_names.difference(unique_discard_names)
|
| 994 |
+
if preferred:
|
| 995 |
+
keep_name = sorted(list(preferred))[0]
|
| 996 |
+
|
| 997 |
+
if unique_preferred_names:
|
| 998 |
+
preferred = unique_preferred_names.intersection(complete_names)
|
| 999 |
+
if preferred:
|
| 1000 |
+
keep_name = sorted(list(preferred))[0]
|
| 1001 |
+
for name in sorted(shared):
|
| 1002 |
+
if name != keep_name:
|
| 1003 |
+
to_remove[keep_name].append(name)
|
| 1004 |
+
return to_remove
|
| 1005 |
+
|
| 1006 |
+
|
| 1007 |
+
@lru_cache
|
| 1008 |
+
def _get_dtype_size(dtype: "torch.dtype") -> int:
|
| 1009 |
+
"""
|
| 1010 |
+
Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L344
|
| 1011 |
+
"""
|
| 1012 |
+
import torch
|
| 1013 |
+
|
| 1014 |
+
# torch.float8 formats require 2.1; we do not support these dtypes on earlier versions
|
| 1015 |
+
_float8_e4m3fn = getattr(torch, "float8_e4m3fn", None)
|
| 1016 |
+
_float8_e5m2 = getattr(torch, "float8_e5m2", None)
|
| 1017 |
+
_SIZE = {
|
| 1018 |
+
torch.int64: 8,
|
| 1019 |
+
torch.float32: 4,
|
| 1020 |
+
torch.int32: 4,
|
| 1021 |
+
torch.bfloat16: 2,
|
| 1022 |
+
torch.float16: 2,
|
| 1023 |
+
torch.int16: 2,
|
| 1024 |
+
torch.uint8: 1,
|
| 1025 |
+
torch.int8: 1,
|
| 1026 |
+
torch.bool: 1,
|
| 1027 |
+
torch.float64: 8,
|
| 1028 |
+
_float8_e4m3fn: 1,
|
| 1029 |
+
_float8_e5m2: 1,
|
| 1030 |
+
}
|
| 1031 |
+
return _SIZE[dtype]
|
| 1032 |
+
|
| 1033 |
+
|
| 1034 |
+
class _IncompatibleKeys(namedtuple("IncompatibleKeys", ["missing_keys", "unexpected_keys"])):
|
| 1035 |
+
"""
|
| 1036 |
+
This is used to report missing and unexpected keys in the state dict.
|
| 1037 |
+
Taken from https://github.com/pytorch/pytorch/blob/main/torch/nn/modules/module.py#L52.
|
| 1038 |
+
|
| 1039 |
+
"""
|
| 1040 |
+
|
| 1041 |
+
def __repr__(self) -> str:
|
| 1042 |
+
if not self.missing_keys and not self.unexpected_keys:
|
| 1043 |
+
return "<All keys matched successfully>"
|
| 1044 |
+
return super().__repr__()
|
| 1045 |
+
|
| 1046 |
+
__str__ = __repr__
|
.venv/lib/python3.13/site-packages/huggingface_hub/templates/datasetcard_template.md
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
# For reference on dataset card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/datasetcard.md?plain=1
|
| 3 |
+
# Doc / guide: https://huggingface.co/docs/hub/datasets-cards
|
| 4 |
+
{{ card_data }}
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
# Dataset Card for {{ pretty_name | default("Dataset Name", true) }}
|
| 8 |
+
|
| 9 |
+
<!-- Provide a quick summary of the dataset. -->
|
| 10 |
+
|
| 11 |
+
{{ dataset_summary | default("", true) }}
|
| 12 |
+
|
| 13 |
+
## Dataset Details
|
| 14 |
+
|
| 15 |
+
### Dataset Description
|
| 16 |
+
|
| 17 |
+
<!-- Provide a longer summary of what this dataset is. -->
|
| 18 |
+
|
| 19 |
+
{{ dataset_description | default("", true) }}
|
| 20 |
+
|
| 21 |
+
- **Curated by:** {{ curators | default("[More Information Needed]", true)}}
|
| 22 |
+
- **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}}
|
| 23 |
+
- **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}}
|
| 24 |
+
- **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}}
|
| 25 |
+
- **License:** {{ license | default("[More Information Needed]", true)}}
|
| 26 |
+
|
| 27 |
+
### Dataset Sources [optional]
|
| 28 |
+
|
| 29 |
+
<!-- Provide the basic links for the dataset. -->
|
| 30 |
+
|
| 31 |
+
- **Repository:** {{ repo | default("[More Information Needed]", true)}}
|
| 32 |
+
- **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}}
|
| 33 |
+
- **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}}
|
| 34 |
+
|
| 35 |
+
## Uses
|
| 36 |
+
|
| 37 |
+
<!-- Address questions around how the dataset is intended to be used. -->
|
| 38 |
+
|
| 39 |
+
### Direct Use
|
| 40 |
+
|
| 41 |
+
<!-- This section describes suitable use cases for the dataset. -->
|
| 42 |
+
|
| 43 |
+
{{ direct_use | default("[More Information Needed]", true)}}
|
| 44 |
+
|
| 45 |
+
### Out-of-Scope Use
|
| 46 |
+
|
| 47 |
+
<!-- This section addresses misuse, malicious use, and uses that the dataset will not work well for. -->
|
| 48 |
+
|
| 49 |
+
{{ out_of_scope_use | default("[More Information Needed]", true)}}
|
| 50 |
+
|
| 51 |
+
## Dataset Structure
|
| 52 |
+
|
| 53 |
+
<!-- This section provides a description of the dataset fields, and additional information about the dataset structure such as criteria used to create the splits, relationships between data points, etc. -->
|
| 54 |
+
|
| 55 |
+
{{ dataset_structure | default("[More Information Needed]", true)}}
|
| 56 |
+
|
| 57 |
+
## Dataset Creation
|
| 58 |
+
|
| 59 |
+
### Curation Rationale
|
| 60 |
+
|
| 61 |
+
<!-- Motivation for the creation of this dataset. -->
|
| 62 |
+
|
| 63 |
+
{{ curation_rationale_section | default("[More Information Needed]", true)}}
|
| 64 |
+
|
| 65 |
+
### Source Data
|
| 66 |
+
|
| 67 |
+
<!-- This section describes the source data (e.g. news text and headlines, social media posts, translated sentences, ...). -->
|
| 68 |
+
|
| 69 |
+
#### Data Collection and Processing
|
| 70 |
+
|
| 71 |
+
<!-- This section describes the data collection and processing process such as data selection criteria, filtering and normalization methods, tools and libraries used, etc. -->
|
| 72 |
+
|
| 73 |
+
{{ data_collection_and_processing_section | default("[More Information Needed]", true)}}
|
| 74 |
+
|
| 75 |
+
#### Who are the source data producers?
|
| 76 |
+
|
| 77 |
+
<!-- This section describes the people or systems who originally created the data. It should also include self-reported demographic or identity information for the source data creators if this information is available. -->
|
| 78 |
+
|
| 79 |
+
{{ source_data_producers_section | default("[More Information Needed]", true)}}
|
| 80 |
+
|
| 81 |
+
### Annotations [optional]
|
| 82 |
+
|
| 83 |
+
<!-- If the dataset contains annotations which are not part of the initial data collection, use this section to describe them. -->
|
| 84 |
+
|
| 85 |
+
#### Annotation process
|
| 86 |
+
|
| 87 |
+
<!-- This section describes the annotation process such as annotation tools used in the process, the amount of data annotated, annotation guidelines provided to the annotators, interannotator statistics, annotation validation, etc. -->
|
| 88 |
+
|
| 89 |
+
{{ annotation_process_section | default("[More Information Needed]", true)}}
|
| 90 |
+
|
| 91 |
+
#### Who are the annotators?
|
| 92 |
+
|
| 93 |
+
<!-- This section describes the people or systems who created the annotations. -->
|
| 94 |
+
|
| 95 |
+
{{ who_are_annotators_section | default("[More Information Needed]", true)}}
|
| 96 |
+
|
| 97 |
+
#### Personal and Sensitive Information
|
| 98 |
+
|
| 99 |
+
<!-- State whether the dataset contains data that might be considered personal, sensitive, or private (e.g., data that reveals addresses, uniquely identifiable names or aliases, racial or ethnic origins, sexual orientations, religious beliefs, political opinions, financial or health data, etc.). If efforts were made to anonymize the data, describe the anonymization process. -->
|
| 100 |
+
|
| 101 |
+
{{ personal_and_sensitive_information | default("[More Information Needed]", true)}}
|
| 102 |
+
|
| 103 |
+
## Bias, Risks, and Limitations
|
| 104 |
+
|
| 105 |
+
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
| 106 |
+
|
| 107 |
+
{{ bias_risks_limitations | default("[More Information Needed]", true)}}
|
| 108 |
+
|
| 109 |
+
### Recommendations
|
| 110 |
+
|
| 111 |
+
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
| 112 |
+
|
| 113 |
+
{{ bias_recommendations | default("Users should be made aware of the risks, biases and limitations of the dataset. More information needed for further recommendations.", true)}}
|
| 114 |
+
|
| 115 |
+
## Citation [optional]
|
| 116 |
+
|
| 117 |
+
<!-- If there is a paper or blog post introducing the dataset, the APA and Bibtex information for that should go in this section. -->
|
| 118 |
+
|
| 119 |
+
**BibTeX:**
|
| 120 |
+
|
| 121 |
+
{{ citation_bibtex | default("[More Information Needed]", true)}}
|
| 122 |
+
|
| 123 |
+
**APA:**
|
| 124 |
+
|
| 125 |
+
{{ citation_apa | default("[More Information Needed]", true)}}
|
| 126 |
+
|
| 127 |
+
## Glossary [optional]
|
| 128 |
+
|
| 129 |
+
<!-- If relevant, include terms and calculations in this section that can help readers understand the dataset or dataset card. -->
|
| 130 |
+
|
| 131 |
+
{{ glossary | default("[More Information Needed]", true)}}
|
| 132 |
+
|
| 133 |
+
## More Information [optional]
|
| 134 |
+
|
| 135 |
+
{{ more_information | default("[More Information Needed]", true)}}
|
| 136 |
+
|
| 137 |
+
## Dataset Card Authors [optional]
|
| 138 |
+
|
| 139 |
+
{{ dataset_card_authors | default("[More Information Needed]", true)}}
|
| 140 |
+
|
| 141 |
+
## Dataset Card Contact
|
| 142 |
+
|
| 143 |
+
{{ dataset_card_contact | default("[More Information Needed]", true)}}
|
.venv/lib/python3.13/site-packages/huggingface_hub/templates/modelcard_template.md
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
# For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1
|
| 3 |
+
# Doc / guide: https://huggingface.co/docs/hub/model-cards
|
| 4 |
+
{{ card_data }}
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
# Model Card for {{ model_id | default("Model ID", true) }}
|
| 8 |
+
|
| 9 |
+
<!-- Provide a quick summary of what the model is/does. -->
|
| 10 |
+
|
| 11 |
+
{{ model_summary | default("", true) }}
|
| 12 |
+
|
| 13 |
+
## Model Details
|
| 14 |
+
|
| 15 |
+
### Model Description
|
| 16 |
+
|
| 17 |
+
<!-- Provide a longer summary of what this model is. -->
|
| 18 |
+
|
| 19 |
+
{{ model_description | default("", true) }}
|
| 20 |
+
|
| 21 |
+
- **Developed by:** {{ developers | default("[More Information Needed]", true)}}
|
| 22 |
+
- **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}}
|
| 23 |
+
- **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}}
|
| 24 |
+
- **Model type:** {{ model_type | default("[More Information Needed]", true)}}
|
| 25 |
+
- **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}}
|
| 26 |
+
- **License:** {{ license | default("[More Information Needed]", true)}}
|
| 27 |
+
- **Finetuned from model [optional]:** {{ base_model | default("[More Information Needed]", true)}}
|
| 28 |
+
|
| 29 |
+
### Model Sources [optional]
|
| 30 |
+
|
| 31 |
+
<!-- Provide the basic links for the model. -->
|
| 32 |
+
|
| 33 |
+
- **Repository:** {{ repo | default("[More Information Needed]", true)}}
|
| 34 |
+
- **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}}
|
| 35 |
+
- **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}}
|
| 36 |
+
|
| 37 |
+
## Uses
|
| 38 |
+
|
| 39 |
+
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
| 40 |
+
|
| 41 |
+
### Direct Use
|
| 42 |
+
|
| 43 |
+
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
| 44 |
+
|
| 45 |
+
{{ direct_use | default("[More Information Needed]", true)}}
|
| 46 |
+
|
| 47 |
+
### Downstream Use [optional]
|
| 48 |
+
|
| 49 |
+
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
| 50 |
+
|
| 51 |
+
{{ downstream_use | default("[More Information Needed]", true)}}
|
| 52 |
+
|
| 53 |
+
### Out-of-Scope Use
|
| 54 |
+
|
| 55 |
+
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
| 56 |
+
|
| 57 |
+
{{ out_of_scope_use | default("[More Information Needed]", true)}}
|
| 58 |
+
|
| 59 |
+
## Bias, Risks, and Limitations
|
| 60 |
+
|
| 61 |
+
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
| 62 |
+
|
| 63 |
+
{{ bias_risks_limitations | default("[More Information Needed]", true)}}
|
| 64 |
+
|
| 65 |
+
### Recommendations
|
| 66 |
+
|
| 67 |
+
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
| 68 |
+
|
| 69 |
+
{{ bias_recommendations | default("Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.", true)}}
|
| 70 |
+
|
| 71 |
+
## How to Get Started with the Model
|
| 72 |
+
|
| 73 |
+
Use the code below to get started with the model.
|
| 74 |
+
|
| 75 |
+
{{ get_started_code | default("[More Information Needed]", true)}}
|
| 76 |
+
|
| 77 |
+
## Training Details
|
| 78 |
+
|
| 79 |
+
### Training Data
|
| 80 |
+
|
| 81 |
+
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
| 82 |
+
|
| 83 |
+
{{ training_data | default("[More Information Needed]", true)}}
|
| 84 |
+
|
| 85 |
+
### Training Procedure
|
| 86 |
+
|
| 87 |
+
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
| 88 |
+
|
| 89 |
+
#### Preprocessing [optional]
|
| 90 |
+
|
| 91 |
+
{{ preprocessing | default("[More Information Needed]", true)}}
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
#### Training Hyperparameters
|
| 95 |
+
|
| 96 |
+
- **Training regime:** {{ training_regime | default("[More Information Needed]", true)}} <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
| 97 |
+
|
| 98 |
+
#### Speeds, Sizes, Times [optional]
|
| 99 |
+
|
| 100 |
+
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
| 101 |
+
|
| 102 |
+
{{ speeds_sizes_times | default("[More Information Needed]", true)}}
|
| 103 |
+
|
| 104 |
+
## Evaluation
|
| 105 |
+
|
| 106 |
+
<!-- This section describes the evaluation protocols and provides the results. -->
|
| 107 |
+
|
| 108 |
+
### Testing Data, Factors & Metrics
|
| 109 |
+
|
| 110 |
+
#### Testing Data
|
| 111 |
+
|
| 112 |
+
<!-- This should link to a Dataset Card if possible. -->
|
| 113 |
+
|
| 114 |
+
{{ testing_data | default("[More Information Needed]", true)}}
|
| 115 |
+
|
| 116 |
+
#### Factors
|
| 117 |
+
|
| 118 |
+
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
| 119 |
+
|
| 120 |
+
{{ testing_factors | default("[More Information Needed]", true)}}
|
| 121 |
+
|
| 122 |
+
#### Metrics
|
| 123 |
+
|
| 124 |
+
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
| 125 |
+
|
| 126 |
+
{{ testing_metrics | default("[More Information Needed]", true)}}
|
| 127 |
+
|
| 128 |
+
### Results
|
| 129 |
+
|
| 130 |
+
{{ results | default("[More Information Needed]", true)}}
|
| 131 |
+
|
| 132 |
+
#### Summary
|
| 133 |
+
|
| 134 |
+
{{ results_summary | default("", true) }}
|
| 135 |
+
|
| 136 |
+
## Model Examination [optional]
|
| 137 |
+
|
| 138 |
+
<!-- Relevant interpretability work for the model goes here -->
|
| 139 |
+
|
| 140 |
+
{{ model_examination | default("[More Information Needed]", true)}}
|
| 141 |
+
|
| 142 |
+
## Environmental Impact
|
| 143 |
+
|
| 144 |
+
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
| 145 |
+
|
| 146 |
+
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
| 147 |
+
|
| 148 |
+
- **Hardware Type:** {{ hardware_type | default("[More Information Needed]", true)}}
|
| 149 |
+
- **Hours used:** {{ hours_used | default("[More Information Needed]", true)}}
|
| 150 |
+
- **Cloud Provider:** {{ cloud_provider | default("[More Information Needed]", true)}}
|
| 151 |
+
- **Compute Region:** {{ cloud_region | default("[More Information Needed]", true)}}
|
| 152 |
+
- **Carbon Emitted:** {{ co2_emitted | default("[More Information Needed]", true)}}
|
| 153 |
+
|
| 154 |
+
## Technical Specifications [optional]
|
| 155 |
+
|
| 156 |
+
### Model Architecture and Objective
|
| 157 |
+
|
| 158 |
+
{{ model_specs | default("[More Information Needed]", true)}}
|
| 159 |
+
|
| 160 |
+
### Compute Infrastructure
|
| 161 |
+
|
| 162 |
+
{{ compute_infrastructure | default("[More Information Needed]", true)}}
|
| 163 |
+
|
| 164 |
+
#### Hardware
|
| 165 |
+
|
| 166 |
+
{{ hardware_requirements | default("[More Information Needed]", true)}}
|
| 167 |
+
|
| 168 |
+
#### Software
|
| 169 |
+
|
| 170 |
+
{{ software | default("[More Information Needed]", true)}}
|
| 171 |
+
|
| 172 |
+
## Citation [optional]
|
| 173 |
+
|
| 174 |
+
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
| 175 |
+
|
| 176 |
+
**BibTeX:**
|
| 177 |
+
|
| 178 |
+
{{ citation_bibtex | default("[More Information Needed]", true)}}
|
| 179 |
+
|
| 180 |
+
**APA:**
|
| 181 |
+
|
| 182 |
+
{{ citation_apa | default("[More Information Needed]", true)}}
|
| 183 |
+
|
| 184 |
+
## Glossary [optional]
|
| 185 |
+
|
| 186 |
+
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
| 187 |
+
|
| 188 |
+
{{ glossary | default("[More Information Needed]", true)}}
|
| 189 |
+
|
| 190 |
+
## More Information [optional]
|
| 191 |
+
|
| 192 |
+
{{ more_information | default("[More Information Needed]", true)}}
|
| 193 |
+
|
| 194 |
+
## Model Card Authors [optional]
|
| 195 |
+
|
| 196 |
+
{{ model_card_authors | default("[More Information Needed]", true)}}
|
| 197 |
+
|
| 198 |
+
## Model Card Contact
|
| 199 |
+
|
| 200 |
+
{{ model_card_contact | default("[More Information Needed]", true)}}
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/__init__.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License
|
| 14 |
+
|
| 15 |
+
# ruff: noqa: F401
|
| 16 |
+
from huggingface_hub.errors import (
|
| 17 |
+
BadRequestError,
|
| 18 |
+
BucketNotFoundError,
|
| 19 |
+
CacheNotFound,
|
| 20 |
+
CorruptedCacheException,
|
| 21 |
+
DisabledRepoError,
|
| 22 |
+
EntryNotFoundError,
|
| 23 |
+
FileMetadataError,
|
| 24 |
+
GatedRepoError,
|
| 25 |
+
HfHubHTTPError,
|
| 26 |
+
HFValidationError,
|
| 27 |
+
JobNotFoundError,
|
| 28 |
+
LocalEntryNotFoundError,
|
| 29 |
+
LocalTokenNotFoundError,
|
| 30 |
+
NotASafetensorsRepoError,
|
| 31 |
+
OfflineModeIsEnabled,
|
| 32 |
+
RepositoryNotFoundError,
|
| 33 |
+
RevisionNotFoundError,
|
| 34 |
+
SafetensorsParsingError,
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
from . import tqdm as _tqdm # _tqdm is the module
|
| 38 |
+
from ._auth import get_stored_tokens, get_token
|
| 39 |
+
from ._cache_assets import cached_assets_path
|
| 40 |
+
from ._cache_manager import (
|
| 41 |
+
CachedFileInfo,
|
| 42 |
+
CachedIncompleteFileInfo,
|
| 43 |
+
CachedRepoInfo,
|
| 44 |
+
CachedRevisionInfo,
|
| 45 |
+
DeleteCacheStrategy,
|
| 46 |
+
HFCacheInfo,
|
| 47 |
+
_format_size,
|
| 48 |
+
scan_cache_dir,
|
| 49 |
+
)
|
| 50 |
+
from ._chunk_utils import chunk_iterable
|
| 51 |
+
from ._datetime import parse_datetime
|
| 52 |
+
from ._detect_agent import detect_agent, is_agent
|
| 53 |
+
from ._experimental import experimental
|
| 54 |
+
from ._fixes import SoftTemporaryDirectory, WeakFileLock, yaml_dump
|
| 55 |
+
from ._git_credential import list_credential_helpers, set_git_credential, unset_git_credential
|
| 56 |
+
from ._headers import build_hf_headers, get_token_to_send
|
| 57 |
+
from ._hf_uris import HfMount, HfUri, is_hf_uri, parse_hf_mount, parse_hf_uri
|
| 58 |
+
from ._http import (
|
| 59 |
+
ASYNC_CLIENT_FACTORY_T,
|
| 60 |
+
CLIENT_FACTORY_T,
|
| 61 |
+
RateLimitInfo,
|
| 62 |
+
close_session,
|
| 63 |
+
fix_hf_endpoint_in_url,
|
| 64 |
+
get_async_session,
|
| 65 |
+
get_session,
|
| 66 |
+
hf_raise_for_status,
|
| 67 |
+
http_backoff,
|
| 68 |
+
http_stream_backoff,
|
| 69 |
+
parse_ratelimit_headers,
|
| 70 |
+
set_async_client_factory,
|
| 71 |
+
set_client_factory,
|
| 72 |
+
)
|
| 73 |
+
from ._pagination import paginate
|
| 74 |
+
from ._paths import DEFAULT_IGNORE_PATTERNS, FORBIDDEN_FOLDERS, filter_repo_objects
|
| 75 |
+
from ._runtime import (
|
| 76 |
+
dump_environment_info,
|
| 77 |
+
get_aiohttp_version,
|
| 78 |
+
get_fastai_version,
|
| 79 |
+
get_fastapi_version,
|
| 80 |
+
get_fastcore_version,
|
| 81 |
+
get_gradio_version,
|
| 82 |
+
get_graphviz_version,
|
| 83 |
+
get_hf_hub_version,
|
| 84 |
+
get_jinja_version,
|
| 85 |
+
get_numpy_version,
|
| 86 |
+
get_pillow_version,
|
| 87 |
+
get_pydantic_version,
|
| 88 |
+
get_pydot_version,
|
| 89 |
+
get_python_version,
|
| 90 |
+
get_tensorboard_version,
|
| 91 |
+
get_tf_version,
|
| 92 |
+
get_torch_version,
|
| 93 |
+
installation_method,
|
| 94 |
+
is_aiohttp_available,
|
| 95 |
+
is_colab_enterprise,
|
| 96 |
+
is_fastai_available,
|
| 97 |
+
is_fastapi_available,
|
| 98 |
+
is_fastcore_available,
|
| 99 |
+
is_google_colab,
|
| 100 |
+
is_gradio_available,
|
| 101 |
+
is_graphviz_available,
|
| 102 |
+
is_jinja_available,
|
| 103 |
+
is_notebook,
|
| 104 |
+
is_numpy_available,
|
| 105 |
+
is_package_available,
|
| 106 |
+
is_pillow_available,
|
| 107 |
+
is_pydantic_available,
|
| 108 |
+
is_pydot_available,
|
| 109 |
+
is_safetensors_available,
|
| 110 |
+
is_tensorboard_available,
|
| 111 |
+
is_tf_available,
|
| 112 |
+
is_torch_available,
|
| 113 |
+
)
|
| 114 |
+
from ._safetensors import SafetensorsFileMetadata, SafetensorsRepoMetadata, TensorInfo
|
| 115 |
+
from ._subprocess import capture_output, run_interactive_subprocess, run_subprocess
|
| 116 |
+
from ._telemetry import send_telemetry
|
| 117 |
+
from ._terminal import ANSI, StatusLine, select_choice, tabulate
|
| 118 |
+
from ._typing import is_jsonable, is_simple_optional_type, unwrap_simple_optional_type
|
| 119 |
+
from ._validators import validate_hf_hub_args, validate_repo_id
|
| 120 |
+
from ._xet import (
|
| 121 |
+
XetFileData,
|
| 122 |
+
XetTokenType,
|
| 123 |
+
parse_xet_file_data_from_response,
|
| 124 |
+
)
|
| 125 |
+
from .tqdm import (
|
| 126 |
+
are_progress_bars_disabled,
|
| 127 |
+
disable_progress_bars,
|
| 128 |
+
enable_progress_bars,
|
| 129 |
+
hf_thread_map,
|
| 130 |
+
is_tqdm_disabled,
|
| 131 |
+
silent_tqdm,
|
| 132 |
+
tqdm,
|
| 133 |
+
tqdm_stream_file,
|
| 134 |
+
)
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_auth.py
ADDED
|
@@ -0,0 +1,456 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2023 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains a helper to get the token from machine (env variable, secret or config file)."""
|
| 15 |
+
|
| 16 |
+
import configparser
|
| 17 |
+
import io
|
| 18 |
+
import logging
|
| 19 |
+
import os
|
| 20 |
+
import time
|
| 21 |
+
import warnings
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
from threading import Lock
|
| 24 |
+
from typing import TypedDict
|
| 25 |
+
|
| 26 |
+
from .. import constants
|
| 27 |
+
from ..errors import DeviceCodeError, OAuthErrorCode, OIDCError
|
| 28 |
+
from ._fixes import WeakFileLock
|
| 29 |
+
from ._oauth_device import refresh_access_token
|
| 30 |
+
from ._runtime import is_colab_enterprise, is_google_colab
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
_SECRET_FILE_MODE = 0o600
|
| 34 |
+
_SECRET_DIR_MODE = 0o700
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _write_secret(path: Path, content: str) -> None:
|
| 38 |
+
"""Write content to file, restricting both the file and its parent directory to owner-only on POSIX systems."""
|
| 39 |
+
path.parent.mkdir(parents=True, exist_ok=True, mode=_SECRET_DIR_MODE)
|
| 40 |
+
fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _SECRET_FILE_MODE)
|
| 41 |
+
with os.fdopen(fd, "w") as f:
|
| 42 |
+
f.write(content)
|
| 43 |
+
try:
|
| 44 |
+
path.chmod(_SECRET_FILE_MODE)
|
| 45 |
+
path.parent.chmod(_SECRET_DIR_MODE)
|
| 46 |
+
except (OSError, NotImplementedError):
|
| 47 |
+
# Windows does not support POSIX modes; chmod() will raise. Best-effort.
|
| 48 |
+
pass
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
_IS_GOOGLE_COLAB_CHECKED = False
|
| 52 |
+
_GOOGLE_COLAB_SECRET_LOCK = Lock()
|
| 53 |
+
_GOOGLE_COLAB_SECRET: str | None = None
|
| 54 |
+
|
| 55 |
+
logger = logging.getLogger(__name__)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def get_token() -> str | None:
|
| 59 |
+
"""
|
| 60 |
+
Get token if user is logged in.
|
| 61 |
+
|
| 62 |
+
Note: in most cases, you should use [`huggingface_hub.utils.build_hf_headers`] instead. This method is only useful
|
| 63 |
+
if you want to retrieve the token for other purposes than sending an HTTP request.
|
| 64 |
+
|
| 65 |
+
If `HF_OIDC_RESOURCE` is set (Trusted Publishers, typically in CI), a short-lived token obtained via OIDC token
|
| 66 |
+
exchange takes precedence. Otherwise the token is retrieved from the `HF_TOKEN` environment variable, then from the
|
| 67 |
+
token file in the Hugging Face home folder. Returns None if user is not logged in. To log in, use [`login`] or
|
| 68 |
+
`hf auth login`.
|
| 69 |
+
|
| 70 |
+
OAuth tokens obtained with the browser-based login come with a refresh token: when such a token is close to
|
| 71 |
+
expiry, it is transparently refreshed and persisted before being returned.
|
| 72 |
+
|
| 73 |
+
Note: if `HF_OIDC_RESOURCE` is set but the OIDC token exchange fails, this raises instead of returning `None`,
|
| 74 |
+
opting into OIDC is explicit, so a failure surfaces as a clear error rather than a silent fallback.
|
| 75 |
+
|
| 76 |
+
Returns:
|
| 77 |
+
`str` or `None`: The token, `None` if it doesn't exist.
|
| 78 |
+
"""
|
| 79 |
+
return (
|
| 80 |
+
_get_token_from_oidc()
|
| 81 |
+
or _get_token_from_environment()
|
| 82 |
+
or _get_token_from_file_refreshed()
|
| 83 |
+
or _get_token_from_google_colab()
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _get_token_from_google_colab() -> str | None:
|
| 88 |
+
"""Get token from Google Colab secrets vault using `google.colab.userdata.get(...)`.
|
| 89 |
+
|
| 90 |
+
Token is read from the vault only once per session and then stored in a global variable to avoid re-requesting
|
| 91 |
+
access to the vault.
|
| 92 |
+
"""
|
| 93 |
+
# If it's not a Google Colab or it's Colab Enterprise, fallback to environment variable or token file authentication
|
| 94 |
+
if not is_google_colab() or is_colab_enterprise():
|
| 95 |
+
return None
|
| 96 |
+
|
| 97 |
+
# `google.colab.userdata` is not thread-safe
|
| 98 |
+
# This can lead to a deadlock if multiple threads try to access it at the same time
|
| 99 |
+
# (typically when using `snapshot_download`)
|
| 100 |
+
# => use a lock
|
| 101 |
+
# See https://github.com/huggingface/huggingface_hub/issues/1952 for more details.
|
| 102 |
+
with _GOOGLE_COLAB_SECRET_LOCK:
|
| 103 |
+
global _GOOGLE_COLAB_SECRET
|
| 104 |
+
global _IS_GOOGLE_COLAB_CHECKED
|
| 105 |
+
|
| 106 |
+
if _IS_GOOGLE_COLAB_CHECKED: # request access only once
|
| 107 |
+
return _GOOGLE_COLAB_SECRET
|
| 108 |
+
|
| 109 |
+
try:
|
| 110 |
+
from google.colab import userdata # type: ignore
|
| 111 |
+
from google.colab.errors import Error as ColabError # type: ignore
|
| 112 |
+
except ImportError:
|
| 113 |
+
return None
|
| 114 |
+
|
| 115 |
+
try:
|
| 116 |
+
token = userdata.get("HF_TOKEN")
|
| 117 |
+
_GOOGLE_COLAB_SECRET = _clean_token(token)
|
| 118 |
+
except userdata.NotebookAccessError:
|
| 119 |
+
# Means the user has a secret call `HF_TOKEN` and got a popup "please grand access to HF_TOKEN" and refused it
|
| 120 |
+
# => warn user but ignore error => do not re-request access to user
|
| 121 |
+
warnings.warn(
|
| 122 |
+
"\nAccess to the secret `HF_TOKEN` has not been granted on this notebook."
|
| 123 |
+
"\nYou will not be requested again."
|
| 124 |
+
"\nPlease restart the session if you want to be prompted again."
|
| 125 |
+
)
|
| 126 |
+
_GOOGLE_COLAB_SECRET = None
|
| 127 |
+
except userdata.SecretNotFoundError:
|
| 128 |
+
# No `HF_TOKEN` secret defined: simply not logged in via the Colab vault. Not worth a
|
| 129 |
+
# warning now that `login()` is the primary flow (it would even fire during `login()`
|
| 130 |
+
# itself, telling the user to set up a secret while they are busy authenticating).
|
| 131 |
+
logger.info(
|
| 132 |
+
"The secret `HF_TOKEN` does not exist in your Colab secrets. Run `huggingface_hub.login()` to"
|
| 133 |
+
" authenticate (recommended but still optional to access public models or datasets)."
|
| 134 |
+
)
|
| 135 |
+
_GOOGLE_COLAB_SECRET = None
|
| 136 |
+
except ColabError as e:
|
| 137 |
+
# Something happen but we don't know what => recommend to open a GitHub issue
|
| 138 |
+
warnings.warn(f"\nError while fetching `HF_TOKEN` secret value from your vault: '{str(e)}'.")
|
| 139 |
+
_GOOGLE_COLAB_SECRET = None
|
| 140 |
+
|
| 141 |
+
_IS_GOOGLE_COLAB_CHECKED = True
|
| 142 |
+
return _GOOGLE_COLAB_SECRET
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _get_token_from_environment() -> str | None:
|
| 146 |
+
# `HF_TOKEN` has priority (keep `HUGGING_FACE_HUB_TOKEN` for backward compatibility)
|
| 147 |
+
return _clean_token(os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN"))
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def _get_token_from_file() -> str | None:
|
| 151 |
+
try:
|
| 152 |
+
return _clean_token(Path(constants.HF_TOKEN_PATH).read_text())
|
| 153 |
+
except FileNotFoundError:
|
| 154 |
+
return None
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class _OidcTokenCache(TypedDict):
|
| 158 |
+
resource: str
|
| 159 |
+
token: str
|
| 160 |
+
expires_at: float # monotonic clock value after which the cached token must be re-exchanged
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
# Cache for the OIDC-exchanged token: re-exchanging on every `get_token()` call would be wasteful,
|
| 164 |
+
# and re-exchanging shortly before expiry transparently keeps long-running jobs authenticated.
|
| 165 |
+
_OIDC_TOKEN_LOCK = Lock()
|
| 166 |
+
_OIDC_TOKEN_CACHE: _OidcTokenCache | None = None
|
| 167 |
+
_OIDC_REFRESH_MARGIN = 300 # re-exchange this many seconds before the token actually expires
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _get_token_from_oidc() -> str | None:
|
| 171 |
+
"""Get a short-lived OIDC token in CI (Trusted Publishers).
|
| 172 |
+
|
| 173 |
+
Enabled by setting `HF_OIDC_RESOURCE`, which scopes the token to a repo or user.
|
| 174 |
+
The ID token is read from `HF_OIDC_ID_TOKEN` if available, or minted from a supported CI provider (e.g. GitHub Actions).
|
| 175 |
+
|
| 176 |
+
Returns `None` when OIDC is not enabled.
|
| 177 |
+
If enabled, any failure is raised explicitly rather than falling back silently.
|
| 178 |
+
|
| 179 |
+
See `huggingface_hub._oidc` and https://huggingface.co/docs/hub/trusted-publishers.
|
| 180 |
+
"""
|
| 181 |
+
resource = os.environ.get("HF_OIDC_RESOURCE")
|
| 182 |
+
if not resource:
|
| 183 |
+
return None
|
| 184 |
+
|
| 185 |
+
from .._oidc import detect_provider, oidc_login
|
| 186 |
+
|
| 187 |
+
global _OIDC_TOKEN_CACHE
|
| 188 |
+
with _OIDC_TOKEN_LOCK:
|
| 189 |
+
now = time.monotonic()
|
| 190 |
+
if (
|
| 191 |
+
_OIDC_TOKEN_CACHE is not None
|
| 192 |
+
and _OIDC_TOKEN_CACHE["resource"] == resource
|
| 193 |
+
and now < _OIDC_TOKEN_CACHE["expires_at"]
|
| 194 |
+
):
|
| 195 |
+
return _OIDC_TOKEN_CACHE["token"]
|
| 196 |
+
|
| 197 |
+
# An explicit id token (any provider) takes precedence; otherwise mint from a detected one.
|
| 198 |
+
subject_token = os.environ.get("HF_OIDC_ID_TOKEN") or None
|
| 199 |
+
if subject_token is None and detect_provider() is None:
|
| 200 |
+
raise OIDCError(
|
| 201 |
+
"HF_OIDC_RESOURCE is set but no OIDC id token is available: not running in a supported "
|
| 202 |
+
"CI provider (github) and HF_OIDC_ID_TOKEN is not set. Set HF_OIDC_ID_TOKEN to the id "
|
| 203 |
+
"token minted by your CI provider, or unset HF_OIDC_RESOURCE."
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
result = oidc_login(resource=resource, subject_token=subject_token)
|
| 207 |
+
token = result["access_token"]
|
| 208 |
+
expires_in = int(result.get("expires_in", 3600))
|
| 209 |
+
# A pre-supplied HF_OIDC_ID_TOKEN can't be re-minted, so refreshing early is pointless (the id
|
| 210 |
+
# token is likely already expired by then): cache for the full lifetime. Only the auto-minted
|
| 211 |
+
# path can refresh, so only it gets the safety margin.
|
| 212 |
+
margin = 0 if subject_token is not None else _OIDC_REFRESH_MARGIN
|
| 213 |
+
_OIDC_TOKEN_CACHE = {
|
| 214 |
+
"resource": resource,
|
| 215 |
+
"token": token,
|
| 216 |
+
"expires_at": now + max(expires_in - margin, 0),
|
| 217 |
+
}
|
| 218 |
+
return token
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
class _OAuthRefreshCache(TypedDict):
|
| 222 |
+
file_token: str # token as read from HF_TOKEN_PATH (cache key)
|
| 223 |
+
resolved_token: str # token to return (refreshed, or identical if no refresh was needed)
|
| 224 |
+
recheck_at: float # wall-clock timestamp after which the expiry must be re-evaluated
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
# Cache the refresh decision in-process: `get_token()` is called on every HTTP request and must not
|
| 228 |
+
# re-read the stored tokens file (let alone hit the network) each time.
|
| 229 |
+
_OAUTH_REFRESH_LOCK = Lock()
|
| 230 |
+
_OAUTH_REFRESH_CACHE: _OAuthRefreshCache | None = None
|
| 231 |
+
_OAUTH_REFRESH_MARGIN = 24 * 3600 # refresh when less than 1 day of validity remains
|
| 232 |
+
_OAUTH_RECHECK_INTERVAL = 300 # re-check interval when there is no metadata or the refresh failed
|
| 233 |
+
_OAUTH_REFRESH_WARNED = False # warn at most once per process on refresh failure
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def _get_token_from_file_refreshed() -> str | None:
|
| 237 |
+
"""Get the token from `HF_TOKEN_PATH`, transparently refreshing it if close to expiry."""
|
| 238 |
+
token = _get_token_from_file()
|
| 239 |
+
if token is None:
|
| 240 |
+
return None
|
| 241 |
+
return _refresh_oauth_token_if_needed(token)
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def _refresh_oauth_token_if_needed(token: str) -> str:
|
| 245 |
+
"""Refresh an OAuth access token if it is close to expiry. Best-effort: never raises.
|
| 246 |
+
|
| 247 |
+
OAuth tokens obtained with the browser-based login are stored with a `refresh_token` and an
|
| 248 |
+
`expires_at` timestamp (see `_save_token`). When the active token is one of them and about to
|
| 249 |
+
expire, exchange the refresh token for a new access token and persist it. Any other token is
|
| 250 |
+
returned unchanged.
|
| 251 |
+
"""
|
| 252 |
+
global _OAUTH_REFRESH_CACHE
|
| 253 |
+
with _OAUTH_REFRESH_LOCK:
|
| 254 |
+
now = time.time()
|
| 255 |
+
cache = _OAUTH_REFRESH_CACHE
|
| 256 |
+
if cache is not None and cache["file_token"] == token and now < cache["recheck_at"]:
|
| 257 |
+
return cache["resolved_token"]
|
| 258 |
+
|
| 259 |
+
token_name, fields = next(
|
| 260 |
+
((name, fields) for name, fields in _read_stored_tokens_full().items() if fields.get("hf_token") == token),
|
| 261 |
+
(None, {}),
|
| 262 |
+
)
|
| 263 |
+
refresh_token = fields.get("refresh_token")
|
| 264 |
+
expires_at = _parse_expires_at(fields)
|
| 265 |
+
if token_name is None or refresh_token is None or expires_at is None:
|
| 266 |
+
# `token` may have just been replaced by a concurrent refresh (in which case it no
|
| 267 |
+
# longer appears in the stored tokens): serve the fresh file token without caching.
|
| 268 |
+
current_file_token = _get_token_from_file()
|
| 269 |
+
if current_file_token is not None and current_file_token != token:
|
| 270 |
+
return current_file_token
|
| 271 |
+
# Not a refreshable OAuth token (or its metadata was lost): nothing to do.
|
| 272 |
+
_OAUTH_REFRESH_CACHE = {
|
| 273 |
+
"file_token": token,
|
| 274 |
+
"resolved_token": token,
|
| 275 |
+
"recheck_at": now + _OAUTH_RECHECK_INTERVAL,
|
| 276 |
+
}
|
| 277 |
+
return token
|
| 278 |
+
|
| 279 |
+
if expires_at - _OAUTH_REFRESH_MARGIN > now:
|
| 280 |
+
_OAUTH_REFRESH_CACHE = {
|
| 281 |
+
"file_token": token,
|
| 282 |
+
"resolved_token": token,
|
| 283 |
+
"recheck_at": expires_at - _OAUTH_REFRESH_MARGIN,
|
| 284 |
+
}
|
| 285 |
+
return token
|
| 286 |
+
|
| 287 |
+
try:
|
| 288 |
+
# Cross-process file lock: if the server rotates refresh tokens, two processes
|
| 289 |
+
# refreshing concurrently would invalidate each other's refresh token.
|
| 290 |
+
with WeakFileLock(constants.HF_STORED_TOKENS_PATH + ".lock", timeout=30):
|
| 291 |
+
# Re-read under the lock: another process may have refreshed in the meantime.
|
| 292 |
+
fields = _read_stored_tokens_full().get(token_name, {})
|
| 293 |
+
if fields.get("hf_token") != token:
|
| 294 |
+
# Another process already refreshed this token: adopt its result.
|
| 295 |
+
new_token = fields.get("hf_token") or token
|
| 296 |
+
new_expires_at = _parse_expires_at(fields)
|
| 297 |
+
else:
|
| 298 |
+
response = refresh_access_token(refresh_token)
|
| 299 |
+
new_token = response["access_token"]
|
| 300 |
+
new_expires_at = int(now) + int(response["expires_in"]) if "expires_in" in response else None
|
| 301 |
+
_save_token(
|
| 302 |
+
token=new_token,
|
| 303 |
+
token_name=token_name,
|
| 304 |
+
# The server may rotate the refresh token; keep the old one if it doesn't.
|
| 305 |
+
refresh_token=response.get("refresh_token") or refresh_token,
|
| 306 |
+
expires_at=new_expires_at,
|
| 307 |
+
)
|
| 308 |
+
# Update the active token file, unless another process switched to a different token meanwhile.
|
| 309 |
+
if _get_token_from_file() == token:
|
| 310 |
+
_write_secret(Path(constants.HF_TOKEN_PATH), new_token)
|
| 311 |
+
logger.info(f"Access token `{token_name}` has been refreshed.")
|
| 312 |
+
except Exception as e:
|
| 313 |
+
if isinstance(e, DeviceCodeError) and e.error_code == OAuthErrorCode.INVALID_GRANT:
|
| 314 |
+
# Refresh token expired or revoked: retrying is pointless, a re-login is required.
|
| 315 |
+
# Warned unconditionally (the inf recheck guarantees it fires at most once): an
|
| 316 |
+
# earlier transient warning must not suppress this actionable message.
|
| 317 |
+
logger.warning(
|
| 318 |
+
"Your Hugging Face access token has expired and could not be refreshed "
|
| 319 |
+
f"(session expired or revoked). Run `hf auth login` to re-authenticate. ({e})"
|
| 320 |
+
)
|
| 321 |
+
recheck_at = float("inf")
|
| 322 |
+
else:
|
| 323 |
+
# Transient failure (offline, server error, ...): retry later.
|
| 324 |
+
_warn_refresh_failure_once(f"Could not refresh your Hugging Face access token: {e}. Will retry later.")
|
| 325 |
+
recheck_at = now + _OAUTH_RECHECK_INTERVAL
|
| 326 |
+
# Return the existing token: if it's truly expired, the API will reject it with a clear error.
|
| 327 |
+
_OAUTH_REFRESH_CACHE = {"file_token": token, "resolved_token": token, "recheck_at": recheck_at}
|
| 328 |
+
return token
|
| 329 |
+
|
| 330 |
+
_OAUTH_REFRESH_CACHE = {
|
| 331 |
+
"file_token": new_token,
|
| 332 |
+
"resolved_token": new_token,
|
| 333 |
+
# The floor guards against a token lifetime shorter than the refresh margin, which
|
| 334 |
+
# would otherwise put `recheck_at` in the past and trigger a refresh on every call.
|
| 335 |
+
"recheck_at": max(
|
| 336 |
+
now + _OAUTH_RECHECK_INTERVAL,
|
| 337 |
+
new_expires_at - _OAUTH_REFRESH_MARGIN if new_expires_at else 0,
|
| 338 |
+
),
|
| 339 |
+
}
|
| 340 |
+
return new_token
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def _warn_refresh_failure_once(message: str) -> None:
|
| 344 |
+
global _OAUTH_REFRESH_WARNED
|
| 345 |
+
if not _OAUTH_REFRESH_WARNED:
|
| 346 |
+
logger.warning(message)
|
| 347 |
+
_OAUTH_REFRESH_WARNED = True
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
def _parse_expires_at(fields: dict[str, str]) -> int | None:
|
| 351 |
+
"""Parse the `expires_at` field of a stored-tokens section, `None` if missing or corrupt."""
|
| 352 |
+
try:
|
| 353 |
+
return int(fields["expires_at"])
|
| 354 |
+
except (KeyError, ValueError):
|
| 355 |
+
return None
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def get_stored_tokens() -> dict[str, str]:
|
| 359 |
+
"""
|
| 360 |
+
Returns the parsed INI file containing the access tokens.
|
| 361 |
+
The file is located at `HF_STORED_TOKENS_PATH`, defaulting to `~/.cache/huggingface/stored_tokens`.
|
| 362 |
+
If the file does not exist, an empty dictionary is returned.
|
| 363 |
+
|
| 364 |
+
Returns: `dict[str, str]`
|
| 365 |
+
Key is the token name and value is the token.
|
| 366 |
+
"""
|
| 367 |
+
return {token_name: fields.get("hf_token", "") for token_name, fields in _read_stored_tokens_full().items()}
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
def _read_stored_tokens_full() -> dict[str, dict[str, str]]:
|
| 371 |
+
"""Read all sections of the stored tokens INI file, with all their fields.
|
| 372 |
+
|
| 373 |
+
Beside `hf_token`, sections for OAuth tokens also carry `refresh_token` and `expires_at`
|
| 374 |
+
(unix timestamp), used by [`get_token`] to transparently refresh them.
|
| 375 |
+
"""
|
| 376 |
+
tokens_path = Path(constants.HF_STORED_TOKENS_PATH)
|
| 377 |
+
if not tokens_path.exists():
|
| 378 |
+
return {}
|
| 379 |
+
# interpolation=None: token values are opaque strings, a `%` must not be interpreted.
|
| 380 |
+
config = configparser.ConfigParser(interpolation=None)
|
| 381 |
+
try:
|
| 382 |
+
config.read(tokens_path)
|
| 383 |
+
return {token_name: dict(config.items(token_name)) for token_name in config.sections()}
|
| 384 |
+
except configparser.Error as e:
|
| 385 |
+
logger.error(f"Error parsing stored tokens file: {e}")
|
| 386 |
+
return {}
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def _save_stored_tokens_full(stored_tokens: dict[str, dict[str, str]]) -> None:
|
| 390 |
+
"""Write all sections and their fields to the stored tokens INI file."""
|
| 391 |
+
config = configparser.ConfigParser(interpolation=None)
|
| 392 |
+
for token_name in sorted(stored_tokens.keys()):
|
| 393 |
+
config.add_section(token_name)
|
| 394 |
+
for key, value in stored_tokens[token_name].items():
|
| 395 |
+
config.set(token_name, key, value)
|
| 396 |
+
|
| 397 |
+
buf = io.StringIO()
|
| 398 |
+
config.write(buf)
|
| 399 |
+
_write_secret(Path(constants.HF_STORED_TOKENS_PATH), buf.getvalue())
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
def _get_token_by_name(token_name: str) -> str | None:
|
| 403 |
+
"""
|
| 404 |
+
Get the token by name.
|
| 405 |
+
|
| 406 |
+
Args:
|
| 407 |
+
token_name (`str`):
|
| 408 |
+
The name of the token to get.
|
| 409 |
+
|
| 410 |
+
Returns:
|
| 411 |
+
`str` or `None`: The token, `None` if it doesn't exist.
|
| 412 |
+
|
| 413 |
+
"""
|
| 414 |
+
stored_tokens = get_stored_tokens()
|
| 415 |
+
if token_name not in stored_tokens:
|
| 416 |
+
return None
|
| 417 |
+
return _clean_token(stored_tokens[token_name])
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
def _save_token(
|
| 421 |
+
token: str, token_name: str, *, refresh_token: str | None = None, expires_at: int | None = None
|
| 422 |
+
) -> None:
|
| 423 |
+
"""
|
| 424 |
+
Save the given token.
|
| 425 |
+
|
| 426 |
+
If the stored tokens file does not exist, it will be created.
|
| 427 |
+
Args:
|
| 428 |
+
token (`str`):
|
| 429 |
+
The token to save.
|
| 430 |
+
token_name (`str`):
|
| 431 |
+
The name of the token.
|
| 432 |
+
refresh_token (`str`, *optional*):
|
| 433 |
+
OAuth refresh token used to renew the access token when it expires.
|
| 434 |
+
expires_at (`int`, *optional*):
|
| 435 |
+
Unix timestamp at which the access token expires.
|
| 436 |
+
"""
|
| 437 |
+
stored_tokens = _read_stored_tokens_full()
|
| 438 |
+
fields = {"hf_token": token}
|
| 439 |
+
if refresh_token is not None:
|
| 440 |
+
fields["refresh_token"] = refresh_token
|
| 441 |
+
if expires_at is not None:
|
| 442 |
+
fields["expires_at"] = str(expires_at)
|
| 443 |
+
# Replace the whole section: re-logging in under the same name must drop stale metadata.
|
| 444 |
+
stored_tokens[token_name] = fields
|
| 445 |
+
_save_stored_tokens_full(stored_tokens)
|
| 446 |
+
logger.info(f"The token `{token_name}` has been saved to {constants.HF_STORED_TOKENS_PATH}")
|
| 447 |
+
|
| 448 |
+
|
| 449 |
+
def _clean_token(token: str | None) -> str | None:
|
| 450 |
+
"""Clean token by removing trailing and leading spaces and newlines.
|
| 451 |
+
|
| 452 |
+
If token is an empty string, return None.
|
| 453 |
+
"""
|
| 454 |
+
if token is None:
|
| 455 |
+
return None
|
| 456 |
+
return token.replace("\r", "").replace("\n", "").strip() or None
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_assets.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2019-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
|
| 16 |
+
from ..constants import HF_ASSETS_CACHE
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def cached_assets_path(
|
| 20 |
+
library_name: str,
|
| 21 |
+
namespace: str = "default",
|
| 22 |
+
subfolder: str = "default",
|
| 23 |
+
*,
|
| 24 |
+
assets_dir: str | Path | None = None,
|
| 25 |
+
) -> Path:
|
| 26 |
+
"""Return a folder path to cache arbitrary files.
|
| 27 |
+
|
| 28 |
+
`huggingface_hub` provides a canonical folder path to store assets. This is the
|
| 29 |
+
recommended way to integrate cache in a downstream library as it will benefit from
|
| 30 |
+
the builtins tools to scan and delete the cache properly.
|
| 31 |
+
|
| 32 |
+
The distinction is made between files cached from the Hub and assets. Files from the
|
| 33 |
+
Hub are cached in a git-aware manner and entirely managed by `huggingface_hub`. See
|
| 34 |
+
[related documentation](https://huggingface.co/docs/huggingface_hub/how-to-cache).
|
| 35 |
+
All other files that a downstream library caches are considered to be "assets"
|
| 36 |
+
(files downloaded from external sources, extracted from a .tar archive, preprocessed
|
| 37 |
+
for training,...).
|
| 38 |
+
|
| 39 |
+
Once the folder path is generated, it is guaranteed to exist and to be a directory.
|
| 40 |
+
The path is based on 3 levels of depth: the library name, a namespace and a
|
| 41 |
+
subfolder. Those 3 levels grants flexibility while allowing `huggingface_hub` to
|
| 42 |
+
expect folders when scanning/deleting parts of the assets cache. Within a library,
|
| 43 |
+
it is expected that all namespaces share the same subset of subfolder names but this
|
| 44 |
+
is not a mandatory rule. The downstream library has then full control on which file
|
| 45 |
+
structure to adopt within its cache. Namespace and subfolder are optional (would
|
| 46 |
+
default to a `"default/"` subfolder) but library name is mandatory as we want every
|
| 47 |
+
downstream library to manage its own cache.
|
| 48 |
+
|
| 49 |
+
Expected tree:
|
| 50 |
+
```text
|
| 51 |
+
assets/
|
| 52 |
+
└── datasets/
|
| 53 |
+
│ ├── SQuAD/
|
| 54 |
+
│ │ ├── downloaded/
|
| 55 |
+
│ │ ├── extracted/
|
| 56 |
+
│ │ └── processed/
|
| 57 |
+
│ ├── Helsinki-NLP--tatoeba_mt/
|
| 58 |
+
│ ├── downloaded/
|
| 59 |
+
│ ├── extracted/
|
| 60 |
+
│ └── processed/
|
| 61 |
+
└── transformers/
|
| 62 |
+
├── default/
|
| 63 |
+
│ ├── something/
|
| 64 |
+
├── bert-base-cased/
|
| 65 |
+
│ ├── default/
|
| 66 |
+
│ └── training/
|
| 67 |
+
hub/
|
| 68 |
+
└── models--julien-c--EsperBERTo-small/
|
| 69 |
+
├── blobs/
|
| 70 |
+
│ ├── (...)
|
| 71 |
+
│ ├── (...)
|
| 72 |
+
├── refs/
|
| 73 |
+
│ └── (...)
|
| 74 |
+
└── [ 128] snapshots/
|
| 75 |
+
├── 2439f60ef33a0d46d85da5001d52aeda5b00ce9f/
|
| 76 |
+
│ ├── (...)
|
| 77 |
+
└── bbc77c8132af1cc5cf678da3f1ddf2de43606d48/
|
| 78 |
+
└── (...)
|
| 79 |
+
```
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
Args:
|
| 83 |
+
library_name (`str`):
|
| 84 |
+
Name of the library that will manage the cache folder. Example: `"dataset"`.
|
| 85 |
+
namespace (`str`, *optional*, defaults to "default"):
|
| 86 |
+
Namespace to which the data belongs. Example: `"SQuAD"`.
|
| 87 |
+
subfolder (`str`, *optional*, defaults to "default"):
|
| 88 |
+
Subfolder in which the data will be stored. Example: `extracted`.
|
| 89 |
+
assets_dir (`str`, `Path`, *optional*):
|
| 90 |
+
Path to the folder where assets are cached. This must not be the same folder
|
| 91 |
+
where Hub files are cached. Defaults to `HF_HOME / "assets"` if not provided.
|
| 92 |
+
Can also be set with `HF_ASSETS_CACHE` environment variable.
|
| 93 |
+
|
| 94 |
+
Returns:
|
| 95 |
+
Path to the cache folder (`Path`).
|
| 96 |
+
|
| 97 |
+
Example:
|
| 98 |
+
```py
|
| 99 |
+
>>> from huggingface_hub import cached_assets_path
|
| 100 |
+
|
| 101 |
+
>>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="download")
|
| 102 |
+
PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/download')
|
| 103 |
+
|
| 104 |
+
>>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="extracted")
|
| 105 |
+
PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/extracted')
|
| 106 |
+
|
| 107 |
+
>>> cached_assets_path(library_name="datasets", namespace="Helsinki-NLP/tatoeba_mt")
|
| 108 |
+
PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/Helsinki-NLP--tatoeba_mt/default')
|
| 109 |
+
|
| 110 |
+
>>> cached_assets_path(library_name="datasets", assets_dir="/tmp/tmp123456")
|
| 111 |
+
PosixPath('/tmp/tmp123456/datasets/default/default')
|
| 112 |
+
```
|
| 113 |
+
"""
|
| 114 |
+
# Resolve assets_dir
|
| 115 |
+
if assets_dir is None:
|
| 116 |
+
assets_dir = HF_ASSETS_CACHE
|
| 117 |
+
assets_dir = Path(assets_dir).expanduser().resolve()
|
| 118 |
+
|
| 119 |
+
# Avoid names that could create path issues
|
| 120 |
+
for part in (" ", "/", "\\"):
|
| 121 |
+
library_name = library_name.replace(part, "--")
|
| 122 |
+
namespace = namespace.replace(part, "--")
|
| 123 |
+
subfolder = subfolder.replace(part, "--")
|
| 124 |
+
|
| 125 |
+
# Path to subfolder is created
|
| 126 |
+
path = assets_dir / library_name / namespace / subfolder
|
| 127 |
+
try:
|
| 128 |
+
path.mkdir(exist_ok=True, parents=True)
|
| 129 |
+
except (FileExistsError, NotADirectoryError):
|
| 130 |
+
raise ValueError(f"Corrupted assets folder: cannot create directory because of an existing file ({path}).")
|
| 131 |
+
|
| 132 |
+
# Return
|
| 133 |
+
return path
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_manager.py
ADDED
|
@@ -0,0 +1,901 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains utilities to manage the HF cache directory."""
|
| 15 |
+
|
| 16 |
+
import os
|
| 17 |
+
import shutil
|
| 18 |
+
from collections import defaultdict
|
| 19 |
+
from dataclasses import dataclass
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
from typing import Literal
|
| 22 |
+
|
| 23 |
+
from huggingface_hub.errors import CacheNotFound, CorruptedCacheException
|
| 24 |
+
|
| 25 |
+
from ..constants import HF_HUB_CACHE
|
| 26 |
+
from . import logging
|
| 27 |
+
from ._parsing import format_timesince
|
| 28 |
+
from ._terminal import tabulate
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
logger = logging.get_logger(__name__)
|
| 32 |
+
|
| 33 |
+
REPO_TYPE_T = Literal["model", "dataset", "space"]
|
| 34 |
+
|
| 35 |
+
# List of OS-created helper files that need to be ignored
|
| 36 |
+
FILES_TO_IGNORE = [".DS_Store", "Thumbs.db", "desktop.ini"]
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@dataclass(frozen=True)
|
| 40 |
+
class CachedFileInfo:
|
| 41 |
+
"""Frozen data structure holding information about a single cached file.
|
| 42 |
+
|
| 43 |
+
Args:
|
| 44 |
+
file_name (`str`):
|
| 45 |
+
Name of the file. Example: `config.json`.
|
| 46 |
+
file_path (`Path`):
|
| 47 |
+
Path of the file in the `snapshots` directory. The file path is a symlink
|
| 48 |
+
referring to a blob in the `blobs` folder.
|
| 49 |
+
blob_path (`Path`):
|
| 50 |
+
Path of the blob file. This is equivalent to `file_path.resolve()`.
|
| 51 |
+
size_on_disk (`int`):
|
| 52 |
+
Size of the blob file in bytes.
|
| 53 |
+
blob_last_accessed (`float`):
|
| 54 |
+
Timestamp of the last time the blob file has been accessed (from any
|
| 55 |
+
revision).
|
| 56 |
+
blob_last_modified (`float`):
|
| 57 |
+
Timestamp of the last time the blob file has been modified/created.
|
| 58 |
+
|
| 59 |
+
> [!WARNING]
|
| 60 |
+
> `blob_last_accessed` and `blob_last_modified` reliability can depend on the OS you
|
| 61 |
+
> are using. See [python documentation](https://docs.python.org/3/library/os.html#os.stat_result)
|
| 62 |
+
> for more details.
|
| 63 |
+
"""
|
| 64 |
+
|
| 65 |
+
file_name: str
|
| 66 |
+
file_path: Path
|
| 67 |
+
blob_path: Path
|
| 68 |
+
size_on_disk: int
|
| 69 |
+
|
| 70 |
+
blob_last_accessed: float
|
| 71 |
+
blob_last_modified: float
|
| 72 |
+
|
| 73 |
+
@property
|
| 74 |
+
def blob_last_accessed_str(self) -> str:
|
| 75 |
+
"""
|
| 76 |
+
(property) Timestamp of the last time the blob file has been accessed (from any
|
| 77 |
+
revision), returned as a human-readable string.
|
| 78 |
+
|
| 79 |
+
Example: "2 weeks ago".
|
| 80 |
+
"""
|
| 81 |
+
return format_timesince(self.blob_last_accessed)
|
| 82 |
+
|
| 83 |
+
@property
|
| 84 |
+
def blob_last_modified_str(self) -> str:
|
| 85 |
+
"""
|
| 86 |
+
(property) Timestamp of the last time the blob file has been modified, returned
|
| 87 |
+
as a human-readable string.
|
| 88 |
+
|
| 89 |
+
Example: "2 weeks ago".
|
| 90 |
+
"""
|
| 91 |
+
return format_timesince(self.blob_last_modified)
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def size_on_disk_str(self) -> str:
|
| 95 |
+
"""
|
| 96 |
+
(property) Size of the blob file as a human-readable string.
|
| 97 |
+
|
| 98 |
+
Example: "42.2K".
|
| 99 |
+
"""
|
| 100 |
+
return _format_size(self.size_on_disk)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
@dataclass(frozen=True)
|
| 104 |
+
class CachedRevisionInfo:
|
| 105 |
+
"""Frozen data structure holding information about a revision.
|
| 106 |
+
|
| 107 |
+
A revision correspond to a folder in the `snapshots` folder and is populated with
|
| 108 |
+
the exact tree structure as the repo on the Hub but contains only symlinks. A
|
| 109 |
+
revision can be either referenced by 1 or more `refs` or be "detached" (no refs).
|
| 110 |
+
|
| 111 |
+
Args:
|
| 112 |
+
commit_hash (`str`):
|
| 113 |
+
Hash of the revision (unique).
|
| 114 |
+
Example: `"9338f7b671827df886678df2bdd7cc7b4f36dffd"`.
|
| 115 |
+
snapshot_path (`Path`):
|
| 116 |
+
Path to the revision directory in the `snapshots` folder. It contains the
|
| 117 |
+
exact tree structure as the repo on the Hub.
|
| 118 |
+
files: (`frozenset[CachedFileInfo]`):
|
| 119 |
+
Set of [`~CachedFileInfo`] describing all files contained in the snapshot.
|
| 120 |
+
refs (`frozenset[str]`):
|
| 121 |
+
Set of `refs` pointing to this revision. If the revision has no `refs`, it
|
| 122 |
+
is considered detached.
|
| 123 |
+
Example: `{"main", "2.4.0"}` or `{"refs/pr/1"}`.
|
| 124 |
+
size_on_disk (`int`):
|
| 125 |
+
Sum of the blob file sizes that are symlink-ed by the revision.
|
| 126 |
+
last_modified (`float`):
|
| 127 |
+
Timestamp of the last time the revision has been created/modified.
|
| 128 |
+
|
| 129 |
+
> [!WARNING]
|
| 130 |
+
> `last_accessed` cannot be determined correctly on a single revision as blob files
|
| 131 |
+
> are shared across revisions.
|
| 132 |
+
|
| 133 |
+
> [!WARNING]
|
| 134 |
+
> `size_on_disk` is not necessarily the sum of all file sizes because of possible
|
| 135 |
+
> duplicated files. Besides, only blobs are taken into account, not the (negligible)
|
| 136 |
+
> size of folders and symlinks.
|
| 137 |
+
"""
|
| 138 |
+
|
| 139 |
+
commit_hash: str
|
| 140 |
+
snapshot_path: Path
|
| 141 |
+
size_on_disk: int
|
| 142 |
+
files: frozenset[CachedFileInfo]
|
| 143 |
+
refs: frozenset[str]
|
| 144 |
+
|
| 145 |
+
last_modified: float
|
| 146 |
+
|
| 147 |
+
@property
|
| 148 |
+
def last_modified_str(self) -> str:
|
| 149 |
+
"""
|
| 150 |
+
(property) Timestamp of the last time the revision has been modified, returned
|
| 151 |
+
as a human-readable string.
|
| 152 |
+
|
| 153 |
+
Example: "2 weeks ago".
|
| 154 |
+
"""
|
| 155 |
+
return format_timesince(self.last_modified)
|
| 156 |
+
|
| 157 |
+
@property
|
| 158 |
+
def size_on_disk_str(self) -> str:
|
| 159 |
+
"""
|
| 160 |
+
(property) Sum of the blob file sizes as a human-readable string.
|
| 161 |
+
|
| 162 |
+
Example: "42.2K".
|
| 163 |
+
"""
|
| 164 |
+
return _format_size(self.size_on_disk)
|
| 165 |
+
|
| 166 |
+
@property
|
| 167 |
+
def nb_files(self) -> int:
|
| 168 |
+
"""
|
| 169 |
+
(property) Total number of files in the revision.
|
| 170 |
+
"""
|
| 171 |
+
return len(self.files)
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
@dataclass(frozen=True)
|
| 175 |
+
class CachedRepoInfo:
|
| 176 |
+
"""Frozen data structure holding information about a cached repository.
|
| 177 |
+
|
| 178 |
+
Args:
|
| 179 |
+
repo_id (`str`):
|
| 180 |
+
Repo id of the repo on the Hub. Example: `"google/fleurs"`.
|
| 181 |
+
repo_type (`Literal["dataset", "model", "space"]`):
|
| 182 |
+
Type of the cached repo.
|
| 183 |
+
repo_path (`Path`):
|
| 184 |
+
Local path to the cached repo.
|
| 185 |
+
size_on_disk (`int`):
|
| 186 |
+
Sum of the blob file sizes in the cached repo.
|
| 187 |
+
nb_files (`int`):
|
| 188 |
+
Total number of blob files in the cached repo.
|
| 189 |
+
revisions (`frozenset[CachedRevisionInfo]`):
|
| 190 |
+
Set of [`~CachedRevisionInfo`] describing all revisions cached in the repo.
|
| 191 |
+
last_accessed (`float`):
|
| 192 |
+
Timestamp of the last time a blob file of the repo has been accessed.
|
| 193 |
+
last_modified (`float`):
|
| 194 |
+
Timestamp of the last time a blob file of the repo has been modified/created.
|
| 195 |
+
|
| 196 |
+
> [!WARNING]
|
| 197 |
+
> `size_on_disk` is not necessarily the sum of all revisions sizes because of
|
| 198 |
+
> duplicated files. Besides, only blobs are taken into account, not the (negligible)
|
| 199 |
+
> size of folders and symlinks.
|
| 200 |
+
|
| 201 |
+
> [!WARNING]
|
| 202 |
+
> `last_accessed` and `last_modified` reliability can depend on the OS you are using.
|
| 203 |
+
> See [python documentation](https://docs.python.org/3/library/os.html#os.stat_result)
|
| 204 |
+
> for more details.
|
| 205 |
+
"""
|
| 206 |
+
|
| 207 |
+
repo_id: str
|
| 208 |
+
repo_type: REPO_TYPE_T
|
| 209 |
+
repo_path: Path
|
| 210 |
+
size_on_disk: int
|
| 211 |
+
nb_files: int
|
| 212 |
+
revisions: frozenset[CachedRevisionInfo]
|
| 213 |
+
|
| 214 |
+
last_accessed: float
|
| 215 |
+
last_modified: float
|
| 216 |
+
|
| 217 |
+
@property
|
| 218 |
+
def last_accessed_str(self) -> str:
|
| 219 |
+
"""
|
| 220 |
+
(property) Last time a blob file of the repo has been accessed, returned as a
|
| 221 |
+
human-readable string.
|
| 222 |
+
|
| 223 |
+
Example: "2 weeks ago".
|
| 224 |
+
"""
|
| 225 |
+
return format_timesince(self.last_accessed)
|
| 226 |
+
|
| 227 |
+
@property
|
| 228 |
+
def last_modified_str(self) -> str:
|
| 229 |
+
"""
|
| 230 |
+
(property) Last time a blob file of the repo has been modified, returned as a
|
| 231 |
+
human-readable string.
|
| 232 |
+
|
| 233 |
+
Example: "2 weeks ago".
|
| 234 |
+
"""
|
| 235 |
+
return format_timesince(self.last_modified)
|
| 236 |
+
|
| 237 |
+
@property
|
| 238 |
+
def size_on_disk_str(self) -> str:
|
| 239 |
+
"""
|
| 240 |
+
(property) Sum of the blob file sizes as a human-readable string.
|
| 241 |
+
|
| 242 |
+
Example: "42.2K".
|
| 243 |
+
"""
|
| 244 |
+
return _format_size(self.size_on_disk)
|
| 245 |
+
|
| 246 |
+
@property
|
| 247 |
+
def cache_id(self) -> str:
|
| 248 |
+
"""Canonical `type/id` identifier used across cache tooling."""
|
| 249 |
+
return f"{self.repo_type}/{self.repo_id}"
|
| 250 |
+
|
| 251 |
+
@property
|
| 252 |
+
def refs(self) -> dict[str, CachedRevisionInfo]:
|
| 253 |
+
"""
|
| 254 |
+
(property) Mapping between `refs` and revision data structures.
|
| 255 |
+
"""
|
| 256 |
+
return {ref: revision for revision in self.revisions for ref in revision.refs}
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
@dataclass(frozen=True)
|
| 260 |
+
class DeleteCacheStrategy:
|
| 261 |
+
"""Frozen data structure holding the strategy to delete cached revisions.
|
| 262 |
+
|
| 263 |
+
This object is not meant to be instantiated programmatically but to be returned by
|
| 264 |
+
[`~utils.HFCacheInfo.delete_revisions`]. See documentation for usage example.
|
| 265 |
+
|
| 266 |
+
Args:
|
| 267 |
+
expected_freed_size (`float`):
|
| 268 |
+
Expected freed size once strategy is executed.
|
| 269 |
+
blobs (`frozenset[Path]`):
|
| 270 |
+
Set of blob file paths to be deleted.
|
| 271 |
+
refs (`frozenset[Path]`):
|
| 272 |
+
Set of reference file paths to be deleted.
|
| 273 |
+
repos (`frozenset[Path]`):
|
| 274 |
+
Set of entire repo paths to be deleted.
|
| 275 |
+
snapshots (`frozenset[Path]`):
|
| 276 |
+
Set of snapshots to be deleted (directory of symlinks).
|
| 277 |
+
"""
|
| 278 |
+
|
| 279 |
+
expected_freed_size: int
|
| 280 |
+
blobs: frozenset[Path]
|
| 281 |
+
refs: frozenset[Path]
|
| 282 |
+
repos: frozenset[Path]
|
| 283 |
+
snapshots: frozenset[Path]
|
| 284 |
+
|
| 285 |
+
@property
|
| 286 |
+
def expected_freed_size_str(self) -> str:
|
| 287 |
+
"""
|
| 288 |
+
(property) Expected size that will be freed as a human-readable string.
|
| 289 |
+
|
| 290 |
+
Example: "42.2K".
|
| 291 |
+
"""
|
| 292 |
+
return _format_size(self.expected_freed_size)
|
| 293 |
+
|
| 294 |
+
def execute(self) -> None:
|
| 295 |
+
"""Execute the defined strategy.
|
| 296 |
+
|
| 297 |
+
> [!WARNING]
|
| 298 |
+
> If this method is interrupted, the cache might get corrupted. Deletion order is
|
| 299 |
+
> implemented so that references and symlinks are deleted before the actual blob
|
| 300 |
+
> files.
|
| 301 |
+
|
| 302 |
+
> [!WARNING]
|
| 303 |
+
> This method is irreversible. If executed, cached files are erased and must be
|
| 304 |
+
> downloaded again.
|
| 305 |
+
"""
|
| 306 |
+
# Deletion order matters. Blobs are deleted in last so that the user can't end
|
| 307 |
+
# up in a state where a `ref`` refers to a missing snapshot or a snapshot
|
| 308 |
+
# symlink refers to a deleted blob.
|
| 309 |
+
|
| 310 |
+
# Delete entire repos
|
| 311 |
+
for path in self.repos:
|
| 312 |
+
_try_delete_path(path, path_type="repo")
|
| 313 |
+
|
| 314 |
+
# Delete snapshot directories
|
| 315 |
+
for path in self.snapshots:
|
| 316 |
+
_try_delete_path(path, path_type="snapshot")
|
| 317 |
+
|
| 318 |
+
# Delete refs files
|
| 319 |
+
for path in self.refs:
|
| 320 |
+
_try_delete_path(path, path_type="ref")
|
| 321 |
+
|
| 322 |
+
# Delete blob files
|
| 323 |
+
for path in self.blobs:
|
| 324 |
+
_try_delete_path(path, path_type="blob")
|
| 325 |
+
|
| 326 |
+
logger.info(f"Cache deletion done. Saved {self.expected_freed_size_str}.")
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
@dataclass(frozen=True)
|
| 330 |
+
class CachedIncompleteFileInfo:
|
| 331 |
+
"""Frozen data structure holding information about a single incomplete download.
|
| 332 |
+
|
| 333 |
+
Interrupted downloads leave `<cache>/<repo>/blobs/<etag>.incomplete` files behind.
|
| 334 |
+
These are not part of any committed revision, so they are surfaced separately by
|
| 335 |
+
[`scan_cache_dir`].
|
| 336 |
+
|
| 337 |
+
Args:
|
| 338 |
+
file_path (`Path`):
|
| 339 |
+
Path of the `.incomplete` file in the `blobs` folder.
|
| 340 |
+
size_on_disk (`int`):
|
| 341 |
+
Size of the partially-downloaded file in bytes.
|
| 342 |
+
"""
|
| 343 |
+
|
| 344 |
+
file_path: Path
|
| 345 |
+
size_on_disk: int
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
@dataclass(frozen=True)
|
| 349 |
+
class HFCacheInfo:
|
| 350 |
+
"""Frozen data structure holding information about the entire cache-system.
|
| 351 |
+
|
| 352 |
+
This data structure is returned by [`scan_cache_dir`] and is immutable.
|
| 353 |
+
|
| 354 |
+
Args:
|
| 355 |
+
size_on_disk (`int`):
|
| 356 |
+
Sum of all valid repo sizes in the cache-system.
|
| 357 |
+
repos (`frozenset[CachedRepoInfo]`):
|
| 358 |
+
Set of [`~CachedRepoInfo`] describing all valid cached repos found on the
|
| 359 |
+
cache-system while scanning.
|
| 360 |
+
incomplete_files (`frozenset[CachedIncompleteFileInfo]`):
|
| 361 |
+
Set of [`~CachedIncompleteFileInfo`] describing orphaned `*.incomplete`
|
| 362 |
+
files left behind by interrupted downloads.
|
| 363 |
+
warnings (`list[CorruptedCacheException]`):
|
| 364 |
+
List of [`~CorruptedCacheException`] that occurred while scanning the cache.
|
| 365 |
+
Those exceptions are captured so that the scan can continue. Corrupted repos
|
| 366 |
+
are skipped from the scan.
|
| 367 |
+
|
| 368 |
+
> [!WARNING]
|
| 369 |
+
> Here `size_on_disk` is equal to the sum of all repo sizes (only blobs). However if
|
| 370 |
+
> some cached repos are corrupted, their sizes are not taken into account.
|
| 371 |
+
"""
|
| 372 |
+
|
| 373 |
+
size_on_disk: int
|
| 374 |
+
repos: frozenset[CachedRepoInfo]
|
| 375 |
+
incomplete_files: frozenset[CachedIncompleteFileInfo]
|
| 376 |
+
warnings: list[CorruptedCacheException]
|
| 377 |
+
|
| 378 |
+
@property
|
| 379 |
+
def size_on_disk_str(self) -> str:
|
| 380 |
+
"""
|
| 381 |
+
(property) Sum of all valid repo sizes in the cache-system as a human-readable
|
| 382 |
+
string.
|
| 383 |
+
|
| 384 |
+
Example: "42.2K".
|
| 385 |
+
"""
|
| 386 |
+
return _format_size(self.size_on_disk)
|
| 387 |
+
|
| 388 |
+
@property
|
| 389 |
+
def incomplete_size_on_disk(self) -> int:
|
| 390 |
+
"""(property) Sum of all incomplete download sizes in bytes."""
|
| 391 |
+
return sum(file.size_on_disk for file in self.incomplete_files)
|
| 392 |
+
|
| 393 |
+
def delete_revisions(self, *revisions: str) -> DeleteCacheStrategy:
|
| 394 |
+
"""Prepare the strategy to delete one or more revisions cached locally.
|
| 395 |
+
|
| 396 |
+
Input revisions can be any revision hash. If a revision hash is not found in the
|
| 397 |
+
local cache, a warning is thrown but no error is raised. Revisions can be from
|
| 398 |
+
different cached repos since hashes are unique across repos,
|
| 399 |
+
|
| 400 |
+
Examples:
|
| 401 |
+
```py
|
| 402 |
+
>>> from huggingface_hub import scan_cache_dir
|
| 403 |
+
>>> cache_info = scan_cache_dir()
|
| 404 |
+
>>> delete_strategy = cache_info.delete_revisions(
|
| 405 |
+
... "81fd1d6e7847c99f5862c9fb81387956d99ec7aa"
|
| 406 |
+
... )
|
| 407 |
+
>>> print(f"Will free {delete_strategy.expected_freed_size_str}.")
|
| 408 |
+
Will free 7.9K.
|
| 409 |
+
>>> delete_strategy.execute()
|
| 410 |
+
Cache deletion done. Saved 7.9K.
|
| 411 |
+
```
|
| 412 |
+
|
| 413 |
+
```py
|
| 414 |
+
>>> from huggingface_hub import scan_cache_dir
|
| 415 |
+
>>> scan_cache_dir().delete_revisions(
|
| 416 |
+
... "81fd1d6e7847c99f5862c9fb81387956d99ec7aa",
|
| 417 |
+
... "e2983b237dccf3ab4937c97fa717319a9ca1a96d",
|
| 418 |
+
... "6c0e6080953db56375760c0471a8c5f2929baf11",
|
| 419 |
+
... ).execute()
|
| 420 |
+
Cache deletion done. Saved 8.6G.
|
| 421 |
+
```
|
| 422 |
+
|
| 423 |
+
> [!WARNING]
|
| 424 |
+
> `delete_revisions` returns a [`~utils.DeleteCacheStrategy`] object that needs to
|
| 425 |
+
> be executed. The [`~utils.DeleteCacheStrategy`] is not meant to be modified but
|
| 426 |
+
> allows having a dry run before actually executing the deletion.
|
| 427 |
+
"""
|
| 428 |
+
hashes_to_delete: set[str] = set(revisions)
|
| 429 |
+
|
| 430 |
+
repos_with_revisions: dict[CachedRepoInfo, set[CachedRevisionInfo]] = defaultdict(set)
|
| 431 |
+
|
| 432 |
+
for repo in self.repos:
|
| 433 |
+
for revision in repo.revisions:
|
| 434 |
+
if revision.commit_hash in hashes_to_delete:
|
| 435 |
+
repos_with_revisions[repo].add(revision)
|
| 436 |
+
hashes_to_delete.remove(revision.commit_hash)
|
| 437 |
+
|
| 438 |
+
if len(hashes_to_delete) > 0:
|
| 439 |
+
logger.warning(f"Revision(s) not found - cannot delete them: {', '.join(hashes_to_delete)}")
|
| 440 |
+
|
| 441 |
+
delete_strategy_blobs: set[Path] = set()
|
| 442 |
+
delete_strategy_refs: set[Path] = set()
|
| 443 |
+
delete_strategy_repos: set[Path] = set()
|
| 444 |
+
delete_strategy_snapshots: set[Path] = set()
|
| 445 |
+
delete_strategy_expected_freed_size = 0
|
| 446 |
+
|
| 447 |
+
for affected_repo, revisions_to_delete in repos_with_revisions.items():
|
| 448 |
+
other_revisions = affected_repo.revisions - revisions_to_delete
|
| 449 |
+
|
| 450 |
+
# If no other revisions, it means all revisions are deleted
|
| 451 |
+
# -> delete the entire cached repo
|
| 452 |
+
if len(other_revisions) == 0:
|
| 453 |
+
delete_strategy_repos.add(affected_repo.repo_path)
|
| 454 |
+
delete_strategy_expected_freed_size += affected_repo.size_on_disk
|
| 455 |
+
continue
|
| 456 |
+
|
| 457 |
+
# Some revisions of the repo will be deleted but not all. We need to filter
|
| 458 |
+
# which blob files will not be linked anymore.
|
| 459 |
+
for revision_to_delete in revisions_to_delete:
|
| 460 |
+
# Snapshot dir
|
| 461 |
+
delete_strategy_snapshots.add(revision_to_delete.snapshot_path)
|
| 462 |
+
|
| 463 |
+
# Refs dir
|
| 464 |
+
for ref in revision_to_delete.refs:
|
| 465 |
+
delete_strategy_refs.add(affected_repo.repo_path / "refs" / ref)
|
| 466 |
+
|
| 467 |
+
# Blobs dir
|
| 468 |
+
for file in revision_to_delete.files:
|
| 469 |
+
# Some files are not symlinks to the blobs dir: files copied instead of symlinked
|
| 470 |
+
# (Windows, filesystems without symlink support) or files created by the user
|
| 471 |
+
# inside the snapshot dir (e.g. build artifacts from a `pip install`). They are
|
| 472 |
+
# already removed when deleting the snapshot dir -> count them but don't delete
|
| 473 |
+
# them a second time.
|
| 474 |
+
if revision_to_delete.snapshot_path in file.blob_path.parents:
|
| 475 |
+
delete_strategy_expected_freed_size += file.size_on_disk
|
| 476 |
+
continue
|
| 477 |
+
|
| 478 |
+
if file.blob_path not in delete_strategy_blobs:
|
| 479 |
+
is_file_alone = True
|
| 480 |
+
for revision in other_revisions:
|
| 481 |
+
for rev_file in revision.files:
|
| 482 |
+
if file.blob_path == rev_file.blob_path:
|
| 483 |
+
is_file_alone = False
|
| 484 |
+
break
|
| 485 |
+
if not is_file_alone:
|
| 486 |
+
break
|
| 487 |
+
|
| 488 |
+
# Blob file not referenced by remaining revisions -> delete
|
| 489 |
+
if is_file_alone:
|
| 490 |
+
delete_strategy_blobs.add(file.blob_path)
|
| 491 |
+
delete_strategy_expected_freed_size += file.size_on_disk
|
| 492 |
+
|
| 493 |
+
# Return the strategy instead of executing it.
|
| 494 |
+
return DeleteCacheStrategy(
|
| 495 |
+
blobs=frozenset(delete_strategy_blobs),
|
| 496 |
+
refs=frozenset(delete_strategy_refs),
|
| 497 |
+
repos=frozenset(delete_strategy_repos),
|
| 498 |
+
snapshots=frozenset(delete_strategy_snapshots),
|
| 499 |
+
expected_freed_size=delete_strategy_expected_freed_size,
|
| 500 |
+
)
|
| 501 |
+
|
| 502 |
+
def export_as_table(self, *, verbosity: int = 0) -> str:
|
| 503 |
+
"""Generate a table from the [`HFCacheInfo`] object.
|
| 504 |
+
|
| 505 |
+
Pass `verbosity=0` to get a table with a single row per repo, with columns
|
| 506 |
+
"repo_id", "repo_type", "size_on_disk", "nb_files", "last_accessed", "last_modified", "refs", "local_path".
|
| 507 |
+
|
| 508 |
+
Pass `verbosity=1` to get a table with a row per repo and revision (thus multiple rows can appear for a single repo), with columns
|
| 509 |
+
"repo_id", "repo_type", "revision", "size_on_disk", "nb_files", "last_modified", "refs", "local_path".
|
| 510 |
+
|
| 511 |
+
Example:
|
| 512 |
+
```py
|
| 513 |
+
>>> from huggingface_hub.utils import scan_cache_dir
|
| 514 |
+
|
| 515 |
+
>>> hf_cache_info = scan_cache_dir()
|
| 516 |
+
HFCacheInfo(...)
|
| 517 |
+
|
| 518 |
+
>>> print(hf_cache_info.export_as_table())
|
| 519 |
+
REPO ID REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED LAST_MODIFIED REFS LOCAL PATH
|
| 520 |
+
--------------------------------------------------- --------- ------------ -------- ------------- ------------- ---- --------------------------------------------------------------------------------------------------
|
| 521 |
+
roberta-base model 2.7M 5 1 day ago 1 week ago main ~/.cache/huggingface/hub/models--roberta-base
|
| 522 |
+
suno/bark model 8.8K 1 1 week ago 1 week ago main ~/.cache/huggingface/hub/models--suno--bark
|
| 523 |
+
t5-base model 893.8M 4 4 days ago 7 months ago main ~/.cache/huggingface/hub/models--t5-base
|
| 524 |
+
t5-large model 3.0G 4 5 weeks ago 5 months ago main ~/.cache/huggingface/hub/models--t5-large
|
| 525 |
+
|
| 526 |
+
>>> print(hf_cache_info.export_as_table(verbosity=1))
|
| 527 |
+
REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED REFS LOCAL PATH
|
| 528 |
+
--------------------------------------------------- --------- ---------------------------------------- ------------ -------- ------------- ---- -----------------------------------------------------------------------------------------------------------------------------------------------------
|
| 529 |
+
roberta-base model e2da8e2f811d1448a5b465c236feacd80ffbac7b 2.7M 5 1 week ago main ~/.cache/huggingface/hub/models--roberta-base/snapshots/e2da8e2f811d1448a5b465c236feacd80ffbac7b
|
| 530 |
+
suno/bark model 70a8a7d34168586dc5d028fa9666aceade177992 8.8K 1 1 week ago main ~/.cache/huggingface/hub/models--suno--bark/snapshots/70a8a7d34168586dc5d028fa9666aceade177992
|
| 531 |
+
t5-base model a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 893.8M 4 7 months ago main ~/.cache/huggingface/hub/models--t5-base/snapshots/a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1
|
| 532 |
+
t5-large model 150ebc2c4b72291e770f58e6057481c8d2ed331a 3.0G 4 5 months ago main ~/.cache/huggingface/hub/models--t5-large/snapshots/150ebc2c4b72291e770f58e6057481c8d2ed331a
|
| 533 |
+
```
|
| 534 |
+
|
| 535 |
+
Args:
|
| 536 |
+
verbosity (`int`, *optional*):
|
| 537 |
+
The verbosity level. Defaults to 0.
|
| 538 |
+
|
| 539 |
+
Returns:
|
| 540 |
+
`str`: The table as a string.
|
| 541 |
+
"""
|
| 542 |
+
if verbosity == 0:
|
| 543 |
+
return tabulate(
|
| 544 |
+
rows=[
|
| 545 |
+
[
|
| 546 |
+
repo.repo_id,
|
| 547 |
+
repo.repo_type,
|
| 548 |
+
f"{repo.size_on_disk_str:>12}",
|
| 549 |
+
repo.nb_files,
|
| 550 |
+
repo.last_accessed_str,
|
| 551 |
+
repo.last_modified_str,
|
| 552 |
+
", ".join(sorted(repo.refs)),
|
| 553 |
+
str(repo.repo_path),
|
| 554 |
+
]
|
| 555 |
+
for repo in sorted(self.repos, key=lambda repo: repo.repo_path)
|
| 556 |
+
],
|
| 557 |
+
headers=[
|
| 558 |
+
"REPO ID",
|
| 559 |
+
"REPO TYPE",
|
| 560 |
+
"SIZE ON DISK",
|
| 561 |
+
"NB FILES",
|
| 562 |
+
"LAST_ACCESSED",
|
| 563 |
+
"LAST_MODIFIED",
|
| 564 |
+
"REFS",
|
| 565 |
+
"LOCAL PATH",
|
| 566 |
+
],
|
| 567 |
+
)
|
| 568 |
+
else:
|
| 569 |
+
return tabulate(
|
| 570 |
+
rows=[
|
| 571 |
+
[
|
| 572 |
+
repo.repo_id,
|
| 573 |
+
repo.repo_type,
|
| 574 |
+
revision.commit_hash,
|
| 575 |
+
f"{revision.size_on_disk_str:>12}",
|
| 576 |
+
revision.nb_files,
|
| 577 |
+
revision.last_modified_str,
|
| 578 |
+
", ".join(sorted(revision.refs)),
|
| 579 |
+
str(revision.snapshot_path),
|
| 580 |
+
]
|
| 581 |
+
for repo in sorted(self.repos, key=lambda repo: repo.repo_path)
|
| 582 |
+
for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash)
|
| 583 |
+
],
|
| 584 |
+
headers=[
|
| 585 |
+
"REPO ID",
|
| 586 |
+
"REPO TYPE",
|
| 587 |
+
"REVISION",
|
| 588 |
+
"SIZE ON DISK",
|
| 589 |
+
"NB FILES",
|
| 590 |
+
"LAST_MODIFIED",
|
| 591 |
+
"REFS",
|
| 592 |
+
"LOCAL PATH",
|
| 593 |
+
],
|
| 594 |
+
)
|
| 595 |
+
|
| 596 |
+
|
| 597 |
+
def scan_cache_dir(cache_dir: str | Path | None = None) -> HFCacheInfo:
|
| 598 |
+
"""Scan the entire HF cache-system and return a [`~HFCacheInfo`] structure.
|
| 599 |
+
|
| 600 |
+
Use `scan_cache_dir` in order to programmatically scan your cache-system. The cache
|
| 601 |
+
will be scanned repo by repo. If a repo is corrupted, a [`~CorruptedCacheException`]
|
| 602 |
+
will be thrown internally but captured and returned in the [`~HFCacheInfo`]
|
| 603 |
+
structure. Only valid repos get a proper report.
|
| 604 |
+
|
| 605 |
+
```py
|
| 606 |
+
>>> from huggingface_hub import scan_cache_dir
|
| 607 |
+
|
| 608 |
+
>>> hf_cache_info = scan_cache_dir()
|
| 609 |
+
HFCacheInfo(
|
| 610 |
+
size_on_disk=3398085269,
|
| 611 |
+
repos=frozenset({
|
| 612 |
+
CachedRepoInfo(
|
| 613 |
+
repo_id='t5-small',
|
| 614 |
+
repo_type='model',
|
| 615 |
+
repo_path=PosixPath(...),
|
| 616 |
+
size_on_disk=970726914,
|
| 617 |
+
nb_files=11,
|
| 618 |
+
revisions=frozenset({
|
| 619 |
+
CachedRevisionInfo(
|
| 620 |
+
commit_hash='d78aea13fa7ecd06c29e3e46195d6341255065d5',
|
| 621 |
+
size_on_disk=970726339,
|
| 622 |
+
snapshot_path=PosixPath(...),
|
| 623 |
+
files=frozenset({
|
| 624 |
+
CachedFileInfo(
|
| 625 |
+
file_name='config.json',
|
| 626 |
+
size_on_disk=1197
|
| 627 |
+
file_path=PosixPath(...),
|
| 628 |
+
blob_path=PosixPath(...),
|
| 629 |
+
),
|
| 630 |
+
CachedFileInfo(...),
|
| 631 |
+
...
|
| 632 |
+
}),
|
| 633 |
+
),
|
| 634 |
+
CachedRevisionInfo(...),
|
| 635 |
+
...
|
| 636 |
+
}),
|
| 637 |
+
),
|
| 638 |
+
CachedRepoInfo(...),
|
| 639 |
+
...
|
| 640 |
+
}),
|
| 641 |
+
warnings=[
|
| 642 |
+
CorruptedCacheException("Snapshots dir doesn't exist in cached repo: ..."),
|
| 643 |
+
CorruptedCacheException(...),
|
| 644 |
+
...
|
| 645 |
+
],
|
| 646 |
+
)
|
| 647 |
+
```
|
| 648 |
+
|
| 649 |
+
You can also print a detailed report directly from the `hf` command line using:
|
| 650 |
+
```text
|
| 651 |
+
> hf cache ls
|
| 652 |
+
ID SIZE LAST_ACCESSED LAST_MODIFIED REFS
|
| 653 |
+
--------------------------- -------- ------------- ------------- -----------
|
| 654 |
+
dataset/nyu-mll/glue 157.4M 2 days ago 2 days ago main script
|
| 655 |
+
model/LiquidAI/LFM2-VL-1.6B 3.2G 4 days ago 4 days ago main
|
| 656 |
+
model/microsoft/UserLM-8b 32.1G 4 days ago 4 days ago main
|
| 657 |
+
|
| 658 |
+
Done in 0.0s. Scanned 6 repo(s) for a total of 3.4G.
|
| 659 |
+
Got 1 warning(s) while scanning. Use -vvv to print details.
|
| 660 |
+
```
|
| 661 |
+
|
| 662 |
+
Args:
|
| 663 |
+
cache_dir (`str` or `Path`, `optional`):
|
| 664 |
+
Cache directory to cache. Defaults to the default HF cache directory.
|
| 665 |
+
|
| 666 |
+
> [!WARNING]
|
| 667 |
+
> Raises:
|
| 668 |
+
>
|
| 669 |
+
> `CacheNotFound`
|
| 670 |
+
> If the cache directory does not exist.
|
| 671 |
+
>
|
| 672 |
+
> [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
| 673 |
+
> If the cache directory is a file, instead of a directory.
|
| 674 |
+
|
| 675 |
+
Returns: a [`~HFCacheInfo`] object.
|
| 676 |
+
"""
|
| 677 |
+
if cache_dir is None:
|
| 678 |
+
cache_dir = HF_HUB_CACHE
|
| 679 |
+
|
| 680 |
+
cache_dir = Path(cache_dir).expanduser().resolve()
|
| 681 |
+
if not cache_dir.exists():
|
| 682 |
+
raise CacheNotFound(
|
| 683 |
+
f"Cache directory not found: {cache_dir}. Please use `cache_dir` argument or set `HF_HUB_CACHE` environment variable.",
|
| 684 |
+
cache_dir=cache_dir,
|
| 685 |
+
)
|
| 686 |
+
|
| 687 |
+
if cache_dir.is_file():
|
| 688 |
+
raise ValueError(
|
| 689 |
+
f"Scan cache expects a directory but found a file: {cache_dir}. Please use `cache_dir` argument or set `HF_HUB_CACHE` environment variable."
|
| 690 |
+
)
|
| 691 |
+
|
| 692 |
+
repos: set[CachedRepoInfo] = set()
|
| 693 |
+
warnings: list[CorruptedCacheException] = []
|
| 694 |
+
for repo_path in cache_dir.iterdir():
|
| 695 |
+
if repo_path.name in FILES_TO_IGNORE:
|
| 696 |
+
continue
|
| 697 |
+
if repo_path.name == ".locks": # skip './.locks/' folder
|
| 698 |
+
continue
|
| 699 |
+
if repo_path.name == "CACHEDIR.TAG": # skip CACHEDIR.TAG file
|
| 700 |
+
continue
|
| 701 |
+
try:
|
| 702 |
+
repos.add(_scan_cached_repo(repo_path))
|
| 703 |
+
except CorruptedCacheException as e:
|
| 704 |
+
warnings.append(e)
|
| 705 |
+
|
| 706 |
+
return HFCacheInfo(
|
| 707 |
+
repos=frozenset(repos),
|
| 708 |
+
size_on_disk=sum(repo.size_on_disk for repo in repos),
|
| 709 |
+
incomplete_files=_scan_incomplete_files(cache_dir),
|
| 710 |
+
warnings=warnings,
|
| 711 |
+
)
|
| 712 |
+
|
| 713 |
+
|
| 714 |
+
def _scan_incomplete_files(cache_dir: Path) -> frozenset[CachedIncompleteFileInfo]:
|
| 715 |
+
"""Find orphaned `*.incomplete` partial-download files in the cache.
|
| 716 |
+
|
| 717 |
+
Interrupted downloads leave `<cache>/<repo>/blobs/<etag>.incomplete` files behind.
|
| 718 |
+
These are not part of any committed revision, so they are reported separately in the
|
| 719 |
+
[`~HFCacheInfo`] returned by [`scan_cache_dir`].
|
| 720 |
+
"""
|
| 721 |
+
files: set[CachedIncompleteFileInfo] = set()
|
| 722 |
+
for path in cache_dir.glob("*/blobs/*.incomplete"):
|
| 723 |
+
try:
|
| 724 |
+
size_on_disk = path.stat().st_size
|
| 725 |
+
except OSError:
|
| 726 |
+
continue
|
| 727 |
+
files.add(CachedIncompleteFileInfo(file_path=path, size_on_disk=size_on_disk))
|
| 728 |
+
return frozenset(files)
|
| 729 |
+
|
| 730 |
+
|
| 731 |
+
def _scan_cached_repo(repo_path: Path) -> CachedRepoInfo:
|
| 732 |
+
"""Scan a single cache repo and return information about it.
|
| 733 |
+
|
| 734 |
+
Any unexpected behavior will raise a [`~CorruptedCacheException`].
|
| 735 |
+
"""
|
| 736 |
+
if not repo_path.is_dir():
|
| 737 |
+
raise CorruptedCacheException(f"Repo path is not a directory: {repo_path}")
|
| 738 |
+
|
| 739 |
+
if "--" not in repo_path.name:
|
| 740 |
+
raise CorruptedCacheException(f"Repo path is not a valid HuggingFace cache directory: {repo_path}")
|
| 741 |
+
|
| 742 |
+
repo_type, repo_id = repo_path.name.split("--", maxsplit=1)
|
| 743 |
+
repo_type = repo_type[:-1] # "models" -> "model"
|
| 744 |
+
repo_id = repo_id.replace("--", "/") # google/fleurs -> "google/fleurs"
|
| 745 |
+
|
| 746 |
+
if repo_type not in {"dataset", "model", "space"}:
|
| 747 |
+
raise CorruptedCacheException(
|
| 748 |
+
f"Repo type must be `dataset`, `model` or `space`, found `{repo_type}` ({repo_path})."
|
| 749 |
+
)
|
| 750 |
+
|
| 751 |
+
blob_stats: dict[Path, os.stat_result] = {} # Key is blob_path, value is blob stats
|
| 752 |
+
|
| 753 |
+
snapshots_path = repo_path / "snapshots"
|
| 754 |
+
refs_path = repo_path / "refs"
|
| 755 |
+
|
| 756 |
+
if not snapshots_path.exists() or not snapshots_path.is_dir():
|
| 757 |
+
raise CorruptedCacheException(f"Snapshots dir doesn't exist in cached repo: {snapshots_path}")
|
| 758 |
+
|
| 759 |
+
# Scan over `refs` directory
|
| 760 |
+
|
| 761 |
+
# key is revision hash, value is set of refs
|
| 762 |
+
refs_by_hash: dict[str, set[str]] = defaultdict(set)
|
| 763 |
+
if refs_path.exists():
|
| 764 |
+
# Example of `refs` directory
|
| 765 |
+
# ── refs
|
| 766 |
+
# ├── main
|
| 767 |
+
# └── refs
|
| 768 |
+
# └── pr
|
| 769 |
+
# └── 1
|
| 770 |
+
if refs_path.is_file():
|
| 771 |
+
raise CorruptedCacheException(f"Refs directory cannot be a file: {refs_path}")
|
| 772 |
+
|
| 773 |
+
for ref_path in refs_path.glob("**/*"):
|
| 774 |
+
# glob("**/*") iterates over all files and directories -> skip directories
|
| 775 |
+
if ref_path.is_dir() or ref_path.name in FILES_TO_IGNORE:
|
| 776 |
+
continue
|
| 777 |
+
|
| 778 |
+
ref_name = str(ref_path.relative_to(refs_path))
|
| 779 |
+
with ref_path.open() as f:
|
| 780 |
+
commit_hash = f.read()
|
| 781 |
+
|
| 782 |
+
refs_by_hash[commit_hash].add(ref_name)
|
| 783 |
+
|
| 784 |
+
# Scan snapshots directory
|
| 785 |
+
cached_revisions: set[CachedRevisionInfo] = set()
|
| 786 |
+
for revision_path in snapshots_path.iterdir():
|
| 787 |
+
# Ignore OS-created helper files
|
| 788 |
+
if revision_path.name in FILES_TO_IGNORE:
|
| 789 |
+
continue
|
| 790 |
+
if revision_path.is_file():
|
| 791 |
+
raise CorruptedCacheException(f"Snapshots folder corrupted. Found a file: {revision_path}")
|
| 792 |
+
|
| 793 |
+
cached_files = set()
|
| 794 |
+
for file_path in revision_path.glob("**/*"):
|
| 795 |
+
# glob("**/*") iterates over all files and directories -> skip directories
|
| 796 |
+
if file_path.is_dir():
|
| 797 |
+
continue
|
| 798 |
+
|
| 799 |
+
blob_path = Path(file_path).resolve()
|
| 800 |
+
if not blob_path.exists():
|
| 801 |
+
raise CorruptedCacheException(f"Blob missing (broken symlink): {blob_path}")
|
| 802 |
+
|
| 803 |
+
if blob_path not in blob_stats:
|
| 804 |
+
blob_stats[blob_path] = blob_path.stat()
|
| 805 |
+
|
| 806 |
+
cached_files.add(
|
| 807 |
+
CachedFileInfo(
|
| 808 |
+
file_name=file_path.name,
|
| 809 |
+
file_path=file_path,
|
| 810 |
+
size_on_disk=blob_stats[blob_path].st_size,
|
| 811 |
+
blob_path=blob_path,
|
| 812 |
+
blob_last_accessed=blob_stats[blob_path].st_atime,
|
| 813 |
+
blob_last_modified=blob_stats[blob_path].st_mtime,
|
| 814 |
+
)
|
| 815 |
+
)
|
| 816 |
+
|
| 817 |
+
# Last modified is either the last modified blob file or the revision folder
|
| 818 |
+
# itself if it is empty
|
| 819 |
+
if len(cached_files) > 0:
|
| 820 |
+
revision_last_modified = max(blob_stats[file.blob_path].st_mtime for file in cached_files)
|
| 821 |
+
else:
|
| 822 |
+
revision_last_modified = revision_path.stat().st_mtime
|
| 823 |
+
|
| 824 |
+
cached_revisions.add(
|
| 825 |
+
CachedRevisionInfo(
|
| 826 |
+
commit_hash=revision_path.name,
|
| 827 |
+
files=frozenset(cached_files),
|
| 828 |
+
refs=frozenset(refs_by_hash.pop(revision_path.name, set())),
|
| 829 |
+
size_on_disk=sum(
|
| 830 |
+
blob_stats[blob_path].st_size for blob_path in {file.blob_path for file in cached_files}
|
| 831 |
+
),
|
| 832 |
+
snapshot_path=revision_path,
|
| 833 |
+
last_modified=revision_last_modified,
|
| 834 |
+
)
|
| 835 |
+
)
|
| 836 |
+
|
| 837 |
+
# Check that all refs referred to an existing revision
|
| 838 |
+
if len(refs_by_hash) > 0:
|
| 839 |
+
raise CorruptedCacheException(
|
| 840 |
+
f"Reference(s) refer to missing commit hashes: {dict(refs_by_hash)} ({repo_path})."
|
| 841 |
+
)
|
| 842 |
+
|
| 843 |
+
# Last modified is either the last modified blob file or the repo folder itself if
|
| 844 |
+
# no blob files has been found. Same for last accessed.
|
| 845 |
+
if len(blob_stats) > 0:
|
| 846 |
+
repo_last_accessed = max(stat.st_atime for stat in blob_stats.values())
|
| 847 |
+
repo_last_modified = max(stat.st_mtime for stat in blob_stats.values())
|
| 848 |
+
else:
|
| 849 |
+
repo_stats = repo_path.stat()
|
| 850 |
+
repo_last_accessed = repo_stats.st_atime
|
| 851 |
+
repo_last_modified = repo_stats.st_mtime
|
| 852 |
+
|
| 853 |
+
# Build and return frozen structure
|
| 854 |
+
return CachedRepoInfo(
|
| 855 |
+
nb_files=len(blob_stats),
|
| 856 |
+
repo_id=repo_id,
|
| 857 |
+
repo_path=repo_path,
|
| 858 |
+
repo_type=repo_type, # type: ignore
|
| 859 |
+
revisions=frozenset(cached_revisions),
|
| 860 |
+
size_on_disk=sum(stat.st_size for stat in blob_stats.values()),
|
| 861 |
+
last_accessed=repo_last_accessed,
|
| 862 |
+
last_modified=repo_last_modified,
|
| 863 |
+
)
|
| 864 |
+
|
| 865 |
+
|
| 866 |
+
def _format_size(num: int) -> str:
|
| 867 |
+
"""Format size in bytes into a human-readable string.
|
| 868 |
+
|
| 869 |
+
Taken from https://stackoverflow.com/a/1094933
|
| 870 |
+
"""
|
| 871 |
+
num_f = float(num)
|
| 872 |
+
for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
|
| 873 |
+
if abs(num_f) < 1000.0:
|
| 874 |
+
return f"{num_f:3.1f}{unit}"
|
| 875 |
+
num_f /= 1000.0
|
| 876 |
+
return f"{num_f:.1f}Y"
|
| 877 |
+
|
| 878 |
+
|
| 879 |
+
def _try_delete_path(path: Path, path_type: str) -> None:
|
| 880 |
+
"""Try to delete a local file or folder.
|
| 881 |
+
|
| 882 |
+
If the path does not exist, error is logged as a warning and then ignored.
|
| 883 |
+
|
| 884 |
+
Args:
|
| 885 |
+
path (`Path`)
|
| 886 |
+
Path to delete. Can be a file or a folder.
|
| 887 |
+
path_type (`str`)
|
| 888 |
+
What path are we deleting ? Only for logging purposes. Example: "snapshot".
|
| 889 |
+
"""
|
| 890 |
+
logger.debug(f"Delete {path_type}: {path}")
|
| 891 |
+
try:
|
| 892 |
+
if path.is_file():
|
| 893 |
+
os.remove(path)
|
| 894 |
+
else:
|
| 895 |
+
shutil.rmtree(path)
|
| 896 |
+
except FileNotFoundError:
|
| 897 |
+
# Nothing to delete: the path is already gone. Log without traceback to avoid flooding the
|
| 898 |
+
# terminal when deleting many paths at once.
|
| 899 |
+
logger.warning(f"Couldn't delete {path_type}: file not found ({path})")
|
| 900 |
+
except PermissionError:
|
| 901 |
+
logger.warning(f"Couldn't delete {path_type}: permission denied ({path})", exc_info=True)
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_chunk_utils.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains a utility to iterate by chunks over an iterator."""
|
| 15 |
+
|
| 16 |
+
import itertools
|
| 17 |
+
from collections.abc import Iterable
|
| 18 |
+
from typing import TypeVar
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
T = TypeVar("T")
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def chunk_iterable(iterable: Iterable[T], chunk_size: int) -> Iterable[Iterable[T]]:
|
| 25 |
+
"""Iterates over an iterator chunk by chunk.
|
| 26 |
+
|
| 27 |
+
Taken from https://stackoverflow.com/a/8998040.
|
| 28 |
+
See also https://github.com/huggingface/huggingface_hub/pull/920#discussion_r938793088.
|
| 29 |
+
|
| 30 |
+
Args:
|
| 31 |
+
iterable (`Iterable`):
|
| 32 |
+
The iterable on which we want to iterate.
|
| 33 |
+
chunk_size (`int`):
|
| 34 |
+
Size of the chunks. Must be a strictly positive integer (e.g. >0).
|
| 35 |
+
|
| 36 |
+
Example:
|
| 37 |
+
|
| 38 |
+
```python
|
| 39 |
+
>>> from huggingface_hub.utils import chunk_iterable
|
| 40 |
+
|
| 41 |
+
>>> for items in chunk_iterable(range(17), chunk_size=8):
|
| 42 |
+
... print(items)
|
| 43 |
+
# [0, 1, 2, 3, 4, 5, 6, 7]
|
| 44 |
+
# [8, 9, 10, 11, 12, 13, 14, 15]
|
| 45 |
+
# [16] # smaller last chunk
|
| 46 |
+
```
|
| 47 |
+
|
| 48 |
+
Raises:
|
| 49 |
+
[`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
| 50 |
+
If `chunk_size` <= 0.
|
| 51 |
+
|
| 52 |
+
> [!WARNING]
|
| 53 |
+
> The last chunk can be smaller than `chunk_size`.
|
| 54 |
+
"""
|
| 55 |
+
if not isinstance(chunk_size, int) or chunk_size <= 0:
|
| 56 |
+
raise ValueError("`chunk_size` must be a strictly positive integer (>0).")
|
| 57 |
+
|
| 58 |
+
iterator = iter(iterable)
|
| 59 |
+
while True:
|
| 60 |
+
try:
|
| 61 |
+
next_item = next(iterator)
|
| 62 |
+
except StopIteration:
|
| 63 |
+
return
|
| 64 |
+
yield itertools.chain((next_item,), itertools.islice(iterator, chunk_size - 1))
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_datetime.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains utilities to handle datetimes in Huggingface Hub."""
|
| 15 |
+
|
| 16 |
+
from datetime import datetime, timezone
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def parse_datetime(date_string: str) -> datetime:
|
| 20 |
+
"""
|
| 21 |
+
Parses a date_string returned from the server to a datetime object.
|
| 22 |
+
|
| 23 |
+
This parser is a weak-parser is the sense that it handles only a single format of
|
| 24 |
+
date_string. It is expected that the server format will never change. The
|
| 25 |
+
implementation depends only on the standard lib to avoid an external dependency
|
| 26 |
+
(python-dateutil). See full discussion about this decision on PR:
|
| 27 |
+
https://github.com/huggingface/huggingface_hub/pull/999.
|
| 28 |
+
|
| 29 |
+
Example:
|
| 30 |
+
```py
|
| 31 |
+
> parse_datetime('2022-08-19T07:19:38.123Z')
|
| 32 |
+
datetime.datetime(2022, 8, 19, 7, 19, 38, 123000, tzinfo=timezone.utc)
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
Args:
|
| 36 |
+
date_string (`str`):
|
| 37 |
+
A string representing a datetime returned by the Hub server.
|
| 38 |
+
String is expected to follow '%Y-%m-%dT%H:%M:%S.%fZ' pattern.
|
| 39 |
+
|
| 40 |
+
Returns:
|
| 41 |
+
A python datetime object.
|
| 42 |
+
|
| 43 |
+
Raises:
|
| 44 |
+
:class:`ValueError`:
|
| 45 |
+
If `date_string` cannot be parsed.
|
| 46 |
+
"""
|
| 47 |
+
try:
|
| 48 |
+
# Normalize the string to always have 6 digits of fractional seconds
|
| 49 |
+
if date_string.endswith("Z"):
|
| 50 |
+
# Case 1: No decimal point (e.g., "2024-11-16T00:27:02Z")
|
| 51 |
+
if "." not in date_string:
|
| 52 |
+
# No fractional seconds - insert .000000
|
| 53 |
+
date_string = date_string[:-1] + ".000000Z"
|
| 54 |
+
# Case 2: Has decimal point (e.g., "2022-08-19T07:19:38.123456789Z")
|
| 55 |
+
else:
|
| 56 |
+
# Get the fractional and base parts
|
| 57 |
+
base, fraction = date_string[:-1].split(".")
|
| 58 |
+
# fraction[:6] takes first 6 digits and :0<6 pads with zeros if less than 6 digits
|
| 59 |
+
date_string = f"{base}.{fraction[:6]:0<6}Z"
|
| 60 |
+
|
| 61 |
+
return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
|
| 62 |
+
except ValueError as e:
|
| 63 |
+
raise ValueError(
|
| 64 |
+
f"Cannot parse '{date_string}' as a datetime. Date string is expected to"
|
| 65 |
+
" follow '%Y-%m-%dT%H:%M:%S.%fZ' pattern."
|
| 66 |
+
) from e
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_deprecation.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import warnings
|
| 2 |
+
from collections.abc import Iterable
|
| 3 |
+
from functools import wraps
|
| 4 |
+
from inspect import Parameter, signature
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def _deprecate_positional_args(*, version: str):
|
| 8 |
+
"""Decorator for methods that issues warnings for positional arguments.
|
| 9 |
+
Using the keyword-only argument syntax in pep 3102, arguments after the
|
| 10 |
+
* will issue a warning when passed as a positional argument.
|
| 11 |
+
|
| 12 |
+
Args:
|
| 13 |
+
version (`str`):
|
| 14 |
+
The version when positional arguments will result in error.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
def _inner_deprecate_positional_args(f):
|
| 18 |
+
sig = signature(f)
|
| 19 |
+
kwonly_args = []
|
| 20 |
+
all_args = []
|
| 21 |
+
for name, param in sig.parameters.items():
|
| 22 |
+
if param.kind == Parameter.POSITIONAL_OR_KEYWORD:
|
| 23 |
+
all_args.append(name)
|
| 24 |
+
elif param.kind == Parameter.KEYWORD_ONLY:
|
| 25 |
+
kwonly_args.append(name)
|
| 26 |
+
|
| 27 |
+
@wraps(f)
|
| 28 |
+
def inner_f(*args, **kwargs):
|
| 29 |
+
extra_args = len(args) - len(all_args)
|
| 30 |
+
if extra_args <= 0:
|
| 31 |
+
return f(*args, **kwargs)
|
| 32 |
+
# extra_args > 0
|
| 33 |
+
args_msg = [
|
| 34 |
+
f"{name}='{arg}'" if isinstance(arg, str) else f"{name}={arg}"
|
| 35 |
+
for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:])
|
| 36 |
+
]
|
| 37 |
+
args_msg = ", ".join(args_msg)
|
| 38 |
+
warnings.warn(
|
| 39 |
+
f"Deprecated positional argument(s) used in '{f.__name__}': pass"
|
| 40 |
+
f" {args_msg} as keyword args. From version {version} passing these"
|
| 41 |
+
" as positional arguments will result in an error,",
|
| 42 |
+
FutureWarning,
|
| 43 |
+
)
|
| 44 |
+
kwargs.update(zip(sig.parameters, args))
|
| 45 |
+
return f(**kwargs)
|
| 46 |
+
|
| 47 |
+
return inner_f
|
| 48 |
+
|
| 49 |
+
return _inner_deprecate_positional_args
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _deprecate_arguments(
|
| 53 |
+
*,
|
| 54 |
+
version: str,
|
| 55 |
+
deprecated_args: Iterable[str],
|
| 56 |
+
custom_message: str | None = None,
|
| 57 |
+
):
|
| 58 |
+
"""Decorator to issue warnings when using deprecated arguments.
|
| 59 |
+
|
| 60 |
+
TODO: could be useful to be able to set a custom error message.
|
| 61 |
+
|
| 62 |
+
Args:
|
| 63 |
+
version (`str`):
|
| 64 |
+
The version when deprecated arguments will result in error.
|
| 65 |
+
deprecated_args (`list[str]`):
|
| 66 |
+
List of the arguments to be deprecated.
|
| 67 |
+
custom_message (`str`, *optional*):
|
| 68 |
+
Warning message that is raised. If not passed, a default warning message
|
| 69 |
+
will be created.
|
| 70 |
+
"""
|
| 71 |
+
|
| 72 |
+
def _inner_deprecate_positional_args(f):
|
| 73 |
+
sig = signature(f)
|
| 74 |
+
|
| 75 |
+
@wraps(f)
|
| 76 |
+
def inner_f(*args, **kwargs):
|
| 77 |
+
# Check for used deprecated arguments
|
| 78 |
+
used_deprecated_args = []
|
| 79 |
+
for _, parameter in zip(args, sig.parameters.values()):
|
| 80 |
+
if parameter.name in deprecated_args:
|
| 81 |
+
used_deprecated_args.append(parameter.name)
|
| 82 |
+
for kwarg_name, kwarg_value in kwargs.items():
|
| 83 |
+
if (
|
| 84 |
+
# If argument is deprecated but still used
|
| 85 |
+
kwarg_name in deprecated_args
|
| 86 |
+
# And then the value is not the default value
|
| 87 |
+
and kwarg_value != sig.parameters[kwarg_name].default
|
| 88 |
+
):
|
| 89 |
+
used_deprecated_args.append(kwarg_name)
|
| 90 |
+
|
| 91 |
+
# Warn and proceed
|
| 92 |
+
if len(used_deprecated_args) > 0:
|
| 93 |
+
message = (
|
| 94 |
+
f"Deprecated argument(s) used in '{f.__name__}':"
|
| 95 |
+
f" {', '.join(used_deprecated_args)}. Will not be supported from"
|
| 96 |
+
f" version '{version}'."
|
| 97 |
+
)
|
| 98 |
+
if custom_message is not None:
|
| 99 |
+
message += "\n\n" + custom_message
|
| 100 |
+
warnings.warn(message, FutureWarning)
|
| 101 |
+
return f(*args, **kwargs)
|
| 102 |
+
|
| 103 |
+
return inner_f
|
| 104 |
+
|
| 105 |
+
return _inner_deprecate_positional_args
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _deprecate_method(*, version: str, message: str | None = None):
|
| 109 |
+
"""Decorator to issue warnings when using a deprecated method.
|
| 110 |
+
|
| 111 |
+
Args:
|
| 112 |
+
version (`str`):
|
| 113 |
+
The version when deprecated arguments will result in error.
|
| 114 |
+
message (`str`, *optional*):
|
| 115 |
+
Warning message that is raised. If not passed, a default warning message
|
| 116 |
+
will be created.
|
| 117 |
+
"""
|
| 118 |
+
|
| 119 |
+
def _inner_deprecate_method(f):
|
| 120 |
+
name = f.__name__
|
| 121 |
+
if name == "__init__":
|
| 122 |
+
name = f.__qualname__.split(".")[0] # class name instead of method name
|
| 123 |
+
|
| 124 |
+
@wraps(f)
|
| 125 |
+
def inner_f(*args, **kwargs):
|
| 126 |
+
warning_message = (
|
| 127 |
+
f"'{name}' (from '{f.__module__}') is deprecated and will be removed from version '{version}'."
|
| 128 |
+
)
|
| 129 |
+
if message is not None:
|
| 130 |
+
warning_message += " " + message
|
| 131 |
+
warnings.warn(warning_message, FutureWarning)
|
| 132 |
+
return f(*args, **kwargs)
|
| 133 |
+
|
| 134 |
+
return inner_f
|
| 135 |
+
|
| 136 |
+
return _inner_deprecate_method
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_detect_agent.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Detect whether the process is being invoked by an AI coding agent.
|
| 15 |
+
|
| 16 |
+
Detection is based on environment variables that AI agents set in their shell
|
| 17 |
+
sessions. `AI_AGENT` and `AGENT` are treated as a universal standard (any
|
| 18 |
+
tool can set its harness id there); the remaining checks are tool-specific and
|
| 19 |
+
ordered by priority (first match wins).
|
| 20 |
+
|
| 21 |
+
The list of known harnesses is maintained on the Hub and exposed at
|
| 22 |
+
`{ENDPOINT}/api/agent-harnesses`. We fetch it at most once a day and cache it
|
| 23 |
+
locally so the list can be updated without requiring a new client release.
|
| 24 |
+
|
| 25 |
+
Detection is entirely best-effort: there is no hardcoded list of harnesses. When
|
| 26 |
+
the registry cannot be fetched (and no cached copy is available), detection simply
|
| 27 |
+
reports "no agent". Any error while fetching/reading the registry is swallowed —
|
| 28 |
+
detection must never make a process fail.
|
| 29 |
+
|
| 30 |
+
More details: https://huggingface.co/docs/hub/agents-overview#register-your-agent-harness
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
import json
|
| 34 |
+
import os
|
| 35 |
+
import time
|
| 36 |
+
from pathlib import Path
|
| 37 |
+
from typing import Optional, TypedDict
|
| 38 |
+
|
| 39 |
+
from .. import constants
|
| 40 |
+
from . import logging
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
logger = logging.get_logger(__name__)
|
| 44 |
+
|
| 45 |
+
# Refresh the cached registry at most once every 24 hours.
|
| 46 |
+
_REGISTRY_TTL_SECONDS = 24 * 3600
|
| 47 |
+
|
| 48 |
+
# Short timeout: fetching the registry is best-effort telemetry, never block the caller for long.
|
| 49 |
+
_REGISTRY_FETCH_TIMEOUT = 3
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class HarnessInfo(TypedDict, total=False):
|
| 53 |
+
"""A single harness entry. `envVars` maps an env var name to a match pattern (see `_env_vars_match`)."""
|
| 54 |
+
|
| 55 |
+
envVars: dict[str, str]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class Registry(TypedDict):
|
| 59 |
+
"""The agent harness registry, as served by `{ENDPOINT}/api/agent-harnesses`."""
|
| 60 |
+
|
| 61 |
+
standardEnvVars: list[str]
|
| 62 |
+
harnesses: dict[str, HarnessInfo]
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
# Empty registry: detection is disabled (no agent ever detected). Used when the
|
| 66 |
+
# Hub is unreachable and no cached copy is available.
|
| 67 |
+
_EMPTY_REGISTRY: Registry = {"standardEnvVars": [], "harnesses": {}}
|
| 68 |
+
|
| 69 |
+
# In-process cache of the resolved registry. Populated lazily on first detection.
|
| 70 |
+
_registry: Registry | None = None
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def detect_agent() -> Optional[str]:
|
| 74 |
+
"""Return the id of the detected AI agent harness or `None`.
|
| 75 |
+
|
| 76 |
+
Harnesses are checked in registry order; for each one we match its env var
|
| 77 |
+
pattern(s) and, failing that, the standard `AI_AGENT` / `AGENT` vars
|
| 78 |
+
against the harness id. The first match wins. When a standard var is set to
|
| 79 |
+
an unrecognized value, `"unknown"` is returned.
|
| 80 |
+
"""
|
| 81 |
+
registry = _get_registry()
|
| 82 |
+
standard_vars = registry.get("standardEnvVars") or []
|
| 83 |
+
harnesses = registry.get("harnesses") or {}
|
| 84 |
+
|
| 85 |
+
for harness_id, info in harnesses.items():
|
| 86 |
+
env_vars = (info or {}).get("envVars")
|
| 87 |
+
if env_vars and _env_vars_match(env_vars):
|
| 88 |
+
return harness_id
|
| 89 |
+
for var in standard_vars:
|
| 90 |
+
if os.environ.get(var, "").strip() == harness_id:
|
| 91 |
+
return harness_id
|
| 92 |
+
|
| 93 |
+
# No harness matched but a standard var is set => unrecognized agent.
|
| 94 |
+
lowercased_harnesses = {k.lower() for k in harnesses.keys()}
|
| 95 |
+
for var in standard_vars:
|
| 96 |
+
if value := os.environ.get(var, "").strip().lower():
|
| 97 |
+
if value in lowercased_harnesses:
|
| 98 |
+
return value
|
| 99 |
+
return "unknown"
|
| 100 |
+
|
| 101 |
+
return None
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def is_agent() -> bool:
|
| 105 |
+
"""Return `True` if the process is being invoked by an AI coding agent."""
|
| 106 |
+
return detect_agent() is not None
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _env_vars_match(env_vars: dict[str, str]) -> bool:
|
| 110 |
+
"""Return `True` if any `(var, pattern)` from the harness matches the environment.
|
| 111 |
+
|
| 112 |
+
Supported patterns:
|
| 113 |
+
- `"*"`: the variable is set to any non-empty value
|
| 114 |
+
- `"<value>"`: the variable equals this exact value
|
| 115 |
+
"""
|
| 116 |
+
for var, pattern in env_vars.items():
|
| 117 |
+
value = os.environ.get(var)
|
| 118 |
+
if not value:
|
| 119 |
+
continue
|
| 120 |
+
if pattern == "*":
|
| 121 |
+
return True
|
| 122 |
+
if value == pattern:
|
| 123 |
+
return True
|
| 124 |
+
return False
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _get_registry() -> Registry:
|
| 128 |
+
"""Return the harness registry, loading (and caching in-process) on first call.
|
| 129 |
+
|
| 130 |
+
Best-effort: any unexpected error degrades to an empty registry so detection
|
| 131 |
+
never raises.
|
| 132 |
+
"""
|
| 133 |
+
global _registry
|
| 134 |
+
if _registry is None:
|
| 135 |
+
try:
|
| 136 |
+
_registry = _load_registry()
|
| 137 |
+
except Exception:
|
| 138 |
+
logger.debug("Could not resolve agent harnesses registry.", exc_info=True)
|
| 139 |
+
_registry = _EMPTY_REGISTRY
|
| 140 |
+
return _registry
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _load_registry() -> Registry:
|
| 144 |
+
"""Resolve the registry from the local cache or the Hub.
|
| 145 |
+
|
| 146 |
+
No hardcoded list: if the Hub is unreachable and no cached copy exists, an
|
| 147 |
+
empty registry is returned (i.e. no agent is detected).
|
| 148 |
+
"""
|
| 149 |
+
path = constants.AGENT_HARNESSES_PATH
|
| 150 |
+
|
| 151 |
+
# 1. Use the cached file if it was refreshed within the last 24 hours.
|
| 152 |
+
if cached := _read_cached_registry(path, max_age=_REGISTRY_TTL_SECONDS):
|
| 153 |
+
return cached
|
| 154 |
+
|
| 155 |
+
# 2. Otherwise refresh it from the Hub and persist it for next time.
|
| 156 |
+
if (fetched := _fetch_registry()) is not None:
|
| 157 |
+
_write_cached_registry(path, fetched)
|
| 158 |
+
return fetched
|
| 159 |
+
|
| 160 |
+
# 3. Fetch failed: reuse a stale cache if available, else give up (no detection).
|
| 161 |
+
if stale := _read_cached_registry(path, max_age=None):
|
| 162 |
+
return stale
|
| 163 |
+
return _EMPTY_REGISTRY
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def _read_cached_registry(path: str, max_age: int | None) -> Registry | None:
|
| 167 |
+
"""Return the cached registry, or `None` if missing/stale/unreadable."""
|
| 168 |
+
try:
|
| 169 |
+
if not os.path.exists(path):
|
| 170 |
+
return None
|
| 171 |
+
if max_age is not None and (time.time() - os.path.getmtime(path)) >= max_age:
|
| 172 |
+
return None
|
| 173 |
+
with open(path, encoding="utf-8") as f:
|
| 174 |
+
return json.load(f)
|
| 175 |
+
except Exception:
|
| 176 |
+
logger.debug("Could not read cached agent harnesses registry.", exc_info=True)
|
| 177 |
+
return None
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def _write_cached_registry(path: str, registry: Registry) -> None:
|
| 181 |
+
try:
|
| 182 |
+
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
| 183 |
+
with open(path, "w", encoding="utf-8") as f:
|
| 184 |
+
json.dump(registry, f)
|
| 185 |
+
except Exception:
|
| 186 |
+
logger.debug("Could not cache agent harnesses registry.", exc_info=True)
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def _fetch_registry() -> Registry | None:
|
| 190 |
+
"""Fetch the registry from the Hub. Returns `None` when offline or on any error."""
|
| 191 |
+
if constants.HF_HUB_OFFLINE:
|
| 192 |
+
return None
|
| 193 |
+
try:
|
| 194 |
+
from ._http import get_session
|
| 195 |
+
|
| 196 |
+
response = get_session().get(
|
| 197 |
+
f"{constants.ENDPOINT}/api/agent-harnesses",
|
| 198 |
+
timeout=_REGISTRY_FETCH_TIMEOUT,
|
| 199 |
+
)
|
| 200 |
+
response.raise_for_status()
|
| 201 |
+
return response.json()
|
| 202 |
+
except Exception:
|
| 203 |
+
logger.debug("Could not fetch agent harnesses registry from the Hub.", exc_info=True)
|
| 204 |
+
return None
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_dotenv.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# AI-generated module (ChatGPT)
|
| 2 |
+
import re
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
# Escape sequences expanded inside quoted values. Double-quoted values additionally
|
| 6 |
+
# expand "\$" to "$"; single-quoted values keep it verbatim.
|
| 7 |
+
_ESCAPES = {"n": "\n", "t": "\t", '"': '"', "\\": "\\"}
|
| 8 |
+
_DOUBLE_QUOTE_ESCAPES = {**_ESCAPES, "$": "$"}
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _unescape(value: str, escapes: dict[str, str]) -> str:
|
| 12 |
+
r"""Expand backslash escapes in a single left-to-right pass.
|
| 13 |
+
|
| 14 |
+
Processing in one pass (rather than chained `str.replace` calls) ensures an escaped
|
| 15 |
+
backslash (`\\`) is consumed as a unit and cannot merge with the following character,
|
| 16 |
+
e.g. `\\n` is a backslash followed by `n`, not a newline. Unknown escapes are kept as-is.
|
| 17 |
+
"""
|
| 18 |
+
return re.sub(r"\\(.)", lambda match: escapes.get(match.group(1), match.group(0)), value)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def load_dotenv(dotenv_str: str, environ: dict[str, str] | None = None) -> dict[str, str]:
|
| 22 |
+
"""
|
| 23 |
+
Parse a DOTENV-format string and return a dictionary of key-value pairs.
|
| 24 |
+
Handles quoted values, comments, export keyword, and blank lines.
|
| 25 |
+
"""
|
| 26 |
+
env: dict[str, str] = {}
|
| 27 |
+
line_pattern = re.compile(
|
| 28 |
+
r"""
|
| 29 |
+
^\s*
|
| 30 |
+
(?:export[^\S\n]+)? # optional export
|
| 31 |
+
([A-Za-z_][A-Za-z0-9_]*) # key
|
| 32 |
+
[^\S\n]*(=)?[^\S\n]*
|
| 33 |
+
( # value group
|
| 34 |
+
(?:
|
| 35 |
+
'(?:\\'|[^'])*' # single-quoted value
|
| 36 |
+
| \"(?:\\\"|[^\"])*\" # double-quoted value
|
| 37 |
+
| [^#\n\r]+? # unquoted value
|
| 38 |
+
)
|
| 39 |
+
)?
|
| 40 |
+
[^\S\n]*(?:\#.*)?$ # optional inline comment
|
| 41 |
+
""",
|
| 42 |
+
re.VERBOSE,
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
for line in dotenv_str.splitlines():
|
| 46 |
+
line = line.strip()
|
| 47 |
+
if not line or line.startswith("#"):
|
| 48 |
+
continue # Skip comments and empty lines
|
| 49 |
+
|
| 50 |
+
match = line_pattern.match(line)
|
| 51 |
+
if match:
|
| 52 |
+
key = match.group(1)
|
| 53 |
+
val = None
|
| 54 |
+
if match.group(2): # if there is '='
|
| 55 |
+
raw_val = match.group(3) or ""
|
| 56 |
+
val = raw_val.strip()
|
| 57 |
+
# Remove surrounding quotes if quoted
|
| 58 |
+
if (val.startswith('"') and val.endswith('"')) or (val.startswith("'") and val.endswith("'")):
|
| 59 |
+
escapes = _DOUBLE_QUOTE_ESCAPES if raw_val.startswith('"') else _ESCAPES
|
| 60 |
+
val = _unescape(val[1:-1], escapes)
|
| 61 |
+
elif environ is not None:
|
| 62 |
+
# Get it from the current environment
|
| 63 |
+
val = environ.get(key)
|
| 64 |
+
|
| 65 |
+
if val is not None:
|
| 66 |
+
env[key] = val
|
| 67 |
+
|
| 68 |
+
return env
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_experimental.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2023-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains utilities to flag a feature as "experimental" in Huggingface Hub."""
|
| 15 |
+
|
| 16 |
+
import warnings
|
| 17 |
+
from collections.abc import Callable
|
| 18 |
+
from functools import wraps
|
| 19 |
+
|
| 20 |
+
from .. import constants
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def experimental(fn: Callable) -> Callable:
|
| 24 |
+
"""Decorator to flag a feature as experimental.
|
| 25 |
+
|
| 26 |
+
An experimental feature triggers a warning when used as it might be subject to breaking changes without prior notice
|
| 27 |
+
in the future.
|
| 28 |
+
|
| 29 |
+
Warnings can be disabled by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment variable.
|
| 30 |
+
|
| 31 |
+
Args:
|
| 32 |
+
fn (`Callable`):
|
| 33 |
+
The function to flag as experimental.
|
| 34 |
+
|
| 35 |
+
Returns:
|
| 36 |
+
`Callable`: The decorated function.
|
| 37 |
+
|
| 38 |
+
Example:
|
| 39 |
+
|
| 40 |
+
```python
|
| 41 |
+
>>> from huggingface_hub.utils import experimental
|
| 42 |
+
|
| 43 |
+
>>> @experimental
|
| 44 |
+
... def my_function():
|
| 45 |
+
... print("Hello world!")
|
| 46 |
+
|
| 47 |
+
>>> my_function()
|
| 48 |
+
UserWarning: 'my_function' is experimental and might be subject to breaking changes in the future without prior
|
| 49 |
+
notice. You can disable this warning by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment variable.
|
| 50 |
+
Hello world!
|
| 51 |
+
```
|
| 52 |
+
"""
|
| 53 |
+
# For classes, put the "experimental" around the "__new__" method => __new__ will be removed in warning message
|
| 54 |
+
name = fn.__qualname__[: -len(".__new__")] if fn.__qualname__.endswith(".__new__") else fn.__qualname__
|
| 55 |
+
|
| 56 |
+
@wraps(fn)
|
| 57 |
+
def _inner_fn(*args, **kwargs):
|
| 58 |
+
if not constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING:
|
| 59 |
+
warnings.warn(
|
| 60 |
+
f"'{name}' is experimental and might be subject to breaking changes in the future without prior notice."
|
| 61 |
+
" You can disable this warning by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment"
|
| 62 |
+
" variable.",
|
| 63 |
+
UserWarning,
|
| 64 |
+
)
|
| 65 |
+
return fn(*args, **kwargs)
|
| 66 |
+
|
| 67 |
+
return _inner_fn
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_fixes.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import contextlib
|
| 2 |
+
import os
|
| 3 |
+
import shutil
|
| 4 |
+
import stat
|
| 5 |
+
import tempfile
|
| 6 |
+
import time
|
| 7 |
+
from collections.abc import Callable, Generator
|
| 8 |
+
from functools import partial
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import yaml
|
| 12 |
+
from filelock import BaseFileLock, FileLock, SoftFileLock, Timeout
|
| 13 |
+
|
| 14 |
+
from .. import constants
|
| 15 |
+
from . import logging
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
logger = logging.get_logger(__name__)
|
| 19 |
+
|
| 20 |
+
# Wrap `yaml.dump` to set `allow_unicode=True` by default.
|
| 21 |
+
#
|
| 22 |
+
# Example:
|
| 23 |
+
# ```py
|
| 24 |
+
# >>> yaml.dump({"emoji": "👀", "some unicode": "日本か"})
|
| 25 |
+
# 'emoji: "\\U0001F440"\nsome unicode: "\\u65E5\\u672C\\u304B"\n'
|
| 26 |
+
#
|
| 27 |
+
# >>> yaml_dump({"emoji": "👀", "some unicode": "日本か"})
|
| 28 |
+
# 'emoji: "👀"\nsome unicode: "日本か"\n'
|
| 29 |
+
# ```
|
| 30 |
+
yaml_dump: Callable[..., str] = partial(yaml.dump, stream=None, allow_unicode=True) # type: ignore
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
@contextlib.contextmanager
|
| 34 |
+
def SoftTemporaryDirectory(
|
| 35 |
+
suffix: str | None = None,
|
| 36 |
+
prefix: str | None = None,
|
| 37 |
+
dir: Path | str | None = None,
|
| 38 |
+
**kwargs,
|
| 39 |
+
) -> Generator[Path, None, None]:
|
| 40 |
+
"""
|
| 41 |
+
Context manager to create a temporary directory and safely delete it.
|
| 42 |
+
|
| 43 |
+
If tmp directory cannot be deleted normally, we set the WRITE permission and retry.
|
| 44 |
+
If cleanup still fails, we give up but don't raise an exception. This is equivalent
|
| 45 |
+
to `tempfile.TemporaryDirectory(..., ignore_cleanup_errors=True)` introduced in
|
| 46 |
+
Python 3.10.
|
| 47 |
+
|
| 48 |
+
See https://www.scivision.dev/python-tempfile-permission-error-windows/.
|
| 49 |
+
"""
|
| 50 |
+
tmpdir = tempfile.TemporaryDirectory(prefix=prefix, suffix=suffix, dir=dir, **kwargs)
|
| 51 |
+
yield Path(tmpdir.name).resolve()
|
| 52 |
+
|
| 53 |
+
try:
|
| 54 |
+
# First once with normal cleanup
|
| 55 |
+
shutil.rmtree(tmpdir.name)
|
| 56 |
+
except Exception:
|
| 57 |
+
# If failed, try to set write permission and retry
|
| 58 |
+
try:
|
| 59 |
+
shutil.rmtree(tmpdir.name, onerror=_set_write_permission_and_retry)
|
| 60 |
+
except Exception:
|
| 61 |
+
pass
|
| 62 |
+
|
| 63 |
+
# And finally, cleanup the tmpdir.
|
| 64 |
+
# If it fails again, give up but do not throw error
|
| 65 |
+
try:
|
| 66 |
+
tmpdir.cleanup()
|
| 67 |
+
except Exception:
|
| 68 |
+
pass
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _set_write_permission_and_retry(func, path, excinfo):
|
| 72 |
+
os.chmod(path, stat.S_IWRITE)
|
| 73 |
+
func(path)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
@contextlib.contextmanager
|
| 77 |
+
def WeakFileLock(lock_file: str | Path, *, timeout: float | None = None) -> Generator[BaseFileLock, None, None]:
|
| 78 |
+
"""A filelock with some custom logic.
|
| 79 |
+
|
| 80 |
+
This filelock is weaker than the default filelock in that:
|
| 81 |
+
1. It won't raise an exception if release fails.
|
| 82 |
+
2. It will default to a SoftFileLock if the filesystem does not support flock.
|
| 83 |
+
3. Lock files are created with mode 0o664 (group-writable) instead of the default 0o644.
|
| 84 |
+
This allows multiple users sharing a cache directory to wait for locks.
|
| 85 |
+
|
| 86 |
+
An INFO log message is emitted every 10 seconds if the lock is not acquired immediately.
|
| 87 |
+
If a timeout is provided, a `filelock.Timeout` exception is raised if the lock is not acquired within the timeout.
|
| 88 |
+
"""
|
| 89 |
+
log_interval = constants.FILELOCK_LOG_EVERY_SECONDS
|
| 90 |
+
lock = FileLock(lock_file, timeout=log_interval, mode=0o664)
|
| 91 |
+
start_time = time.time()
|
| 92 |
+
|
| 93 |
+
while True:
|
| 94 |
+
elapsed_time = time.time() - start_time
|
| 95 |
+
if timeout is not None and elapsed_time >= timeout:
|
| 96 |
+
raise Timeout(str(lock_file))
|
| 97 |
+
|
| 98 |
+
try:
|
| 99 |
+
lock.acquire(timeout=min(log_interval, timeout - elapsed_time) if timeout else log_interval)
|
| 100 |
+
except Timeout:
|
| 101 |
+
logger.info(
|
| 102 |
+
f"Still waiting to acquire lock on {lock_file} (elapsed: {time.time() - start_time:.1f} seconds)"
|
| 103 |
+
)
|
| 104 |
+
except NotImplementedError as e:
|
| 105 |
+
if "use SoftFileLock instead" in str(e):
|
| 106 |
+
logger.warning(
|
| 107 |
+
"FileSystem does not appear to support flock. Falling back to SoftFileLock for %s", lock_file
|
| 108 |
+
)
|
| 109 |
+
lock = SoftFileLock(lock_file, timeout=log_interval)
|
| 110 |
+
continue
|
| 111 |
+
else:
|
| 112 |
+
break
|
| 113 |
+
|
| 114 |
+
try:
|
| 115 |
+
yield lock
|
| 116 |
+
finally:
|
| 117 |
+
try:
|
| 118 |
+
lock.release()
|
| 119 |
+
except OSError:
|
| 120 |
+
try:
|
| 121 |
+
Path(lock_file).unlink()
|
| 122 |
+
except OSError:
|
| 123 |
+
pass
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_git_credential.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains utilities to manage Git credentials."""
|
| 15 |
+
|
| 16 |
+
import re
|
| 17 |
+
import subprocess
|
| 18 |
+
|
| 19 |
+
from ..constants import ENDPOINT
|
| 20 |
+
from ._subprocess import run_interactive_subprocess, run_subprocess
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
GIT_CREDENTIAL_REGEX = re.compile(
|
| 24 |
+
r"""
|
| 25 |
+
^\s* # start of line
|
| 26 |
+
credential\.helper # credential.helper value
|
| 27 |
+
\s*=\s* # separator
|
| 28 |
+
([\w\-\/]+) # the helper name or absolute path (group 1)
|
| 29 |
+
(\s|$) # whitespace or end of line
|
| 30 |
+
""",
|
| 31 |
+
flags=re.MULTILINE | re.IGNORECASE | re.VERBOSE,
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def list_credential_helpers(folder: str | None = None) -> list[str]:
|
| 36 |
+
"""Return the list of git credential helpers configured.
|
| 37 |
+
|
| 38 |
+
See https://git-scm.com/docs/gitcredentials.
|
| 39 |
+
|
| 40 |
+
Credentials are saved in all configured helpers (store, cache, macOS keychain,...).
|
| 41 |
+
Calls "`git credential approve`" internally. See https://git-scm.com/docs/git-credential.
|
| 42 |
+
|
| 43 |
+
Args:
|
| 44 |
+
folder (`str`, *optional*):
|
| 45 |
+
The folder in which to check the configured helpers.
|
| 46 |
+
"""
|
| 47 |
+
try:
|
| 48 |
+
output = run_subprocess("git config --list", folder=folder).stdout
|
| 49 |
+
parsed = _parse_credential_output(output)
|
| 50 |
+
return parsed
|
| 51 |
+
except subprocess.CalledProcessError as exc:
|
| 52 |
+
raise OSError(exc.stderr)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def set_git_credential(token: str, username: str = "hf_user", folder: str | None = None) -> None:
|
| 56 |
+
"""Save a username/token pair in git credential for HF Hub registry.
|
| 57 |
+
|
| 58 |
+
Credentials are saved in all configured helpers (store, cache, macOS keychain,...).
|
| 59 |
+
Calls "`git credential approve`" internally. See https://git-scm.com/docs/git-credential.
|
| 60 |
+
|
| 61 |
+
Args:
|
| 62 |
+
username (`str`, defaults to `"hf_user"`):
|
| 63 |
+
A git username. Defaults to `"hf_user"`, the default user used in the Hub.
|
| 64 |
+
token (`str`, defaults to `"hf_user"`):
|
| 65 |
+
A git password. In practice, the User Access Token for the Hub.
|
| 66 |
+
See https://huggingface.co/settings/tokens.
|
| 67 |
+
folder (`str`, *optional*):
|
| 68 |
+
The folder in which to check the configured helpers.
|
| 69 |
+
"""
|
| 70 |
+
with run_interactive_subprocess("git credential approve", folder=folder) as (
|
| 71 |
+
stdin,
|
| 72 |
+
_,
|
| 73 |
+
):
|
| 74 |
+
stdin.write(f"url={ENDPOINT}\nusername={username.lower()}\npassword={token}\n\n")
|
| 75 |
+
stdin.flush()
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def unset_git_credential(username: str = "hf_user", folder: str | None = None) -> None:
|
| 79 |
+
"""Erase credentials from git credential for HF Hub registry.
|
| 80 |
+
|
| 81 |
+
Credentials are erased from the configured helpers (store, cache, macOS
|
| 82 |
+
keychain,...), if any. If `username` is not provided, any credential configured for
|
| 83 |
+
HF Hub endpoint is erased.
|
| 84 |
+
Calls "`git credential erase`" internally. See https://git-scm.com/docs/git-credential.
|
| 85 |
+
|
| 86 |
+
Args:
|
| 87 |
+
username (`str`, defaults to `"hf_user"`):
|
| 88 |
+
A git username. Defaults to `"hf_user"`, the default user used in the Hub.
|
| 89 |
+
folder (`str`, *optional*):
|
| 90 |
+
The folder in which to check the configured helpers.
|
| 91 |
+
"""
|
| 92 |
+
with run_interactive_subprocess("git credential reject", folder=folder) as (
|
| 93 |
+
stdin,
|
| 94 |
+
_,
|
| 95 |
+
):
|
| 96 |
+
standard_input = f"url={ENDPOINT}\n"
|
| 97 |
+
if username is not None:
|
| 98 |
+
standard_input += f"username={username.lower()}\n"
|
| 99 |
+
standard_input += "\n"
|
| 100 |
+
|
| 101 |
+
stdin.write(standard_input)
|
| 102 |
+
stdin.flush()
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _parse_credential_output(output: str) -> list[str]:
|
| 106 |
+
"""Parse the output of `git credential fill` to extract the password.
|
| 107 |
+
|
| 108 |
+
Args:
|
| 109 |
+
output (`str`):
|
| 110 |
+
The output of `git credential fill`.
|
| 111 |
+
"""
|
| 112 |
+
# NOTE: If user has set a helper for a custom URL, it will not be caught here.
|
| 113 |
+
# Example: `credential.https://huggingface.co.helper=store`
|
| 114 |
+
# See: https://github.com/huggingface/huggingface_hub/pull/1138#discussion_r1013324508
|
| 115 |
+
return sorted( # Sort for nice printing
|
| 116 |
+
{ # Might have some duplicates
|
| 117 |
+
match[0] for match in GIT_CREDENTIAL_REGEX.findall(output)
|
| 118 |
+
}
|
| 119 |
+
)
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_headers.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains utilities to handle headers to send in calls to Huggingface Hub."""
|
| 15 |
+
|
| 16 |
+
from huggingface_hub.errors import LocalTokenNotFoundError
|
| 17 |
+
|
| 18 |
+
from .. import constants
|
| 19 |
+
from ._auth import get_token
|
| 20 |
+
from ._detect_agent import detect_agent
|
| 21 |
+
from ._runtime import (
|
| 22 |
+
get_hf_hub_version,
|
| 23 |
+
get_python_version,
|
| 24 |
+
get_torch_version,
|
| 25 |
+
is_torch_available,
|
| 26 |
+
)
|
| 27 |
+
from ._validators import validate_hf_hub_args
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@validate_hf_hub_args
|
| 31 |
+
def build_hf_headers(
|
| 32 |
+
*,
|
| 33 |
+
token: bool | str | None = None,
|
| 34 |
+
library_name: str | None = None,
|
| 35 |
+
library_version: str | None = None,
|
| 36 |
+
user_agent: dict | str | None = None,
|
| 37 |
+
headers: dict[str, str] | None = None,
|
| 38 |
+
) -> dict[str, str]:
|
| 39 |
+
"""
|
| 40 |
+
Build headers dictionary to send in a HF Hub call.
|
| 41 |
+
|
| 42 |
+
By default, authorization token is always provided either from argument (explicit
|
| 43 |
+
use) or retrieved from the cache (implicit use). To explicitly avoid sending the
|
| 44 |
+
token to the Hub, set `token=False` or set the `HF_HUB_DISABLE_IMPLICIT_TOKEN`
|
| 45 |
+
environment variable.
|
| 46 |
+
|
| 47 |
+
In case of an API call that requires write access, an error is thrown if token is
|
| 48 |
+
`None` or token is an organization token (starting with `"api_org***"`).
|
| 49 |
+
|
| 50 |
+
In addition to the auth header, a user-agent is added to provide information about
|
| 51 |
+
the installed packages (versions of python, huggingface_hub, torch).
|
| 52 |
+
|
| 53 |
+
Args:
|
| 54 |
+
token (`str`, `bool`, *optional*):
|
| 55 |
+
The token to be sent in authorization header for the Hub call:
|
| 56 |
+
- if a string, it is used as the Hugging Face token
|
| 57 |
+
- if `True`, the token is read from the machine (cache or env variable)
|
| 58 |
+
- if `False`, authorization header is not set
|
| 59 |
+
- if `None`, the token is read from the machine only except if
|
| 60 |
+
`HF_HUB_DISABLE_IMPLICIT_TOKEN` env variable is set.
|
| 61 |
+
library_name (`str`, *optional*):
|
| 62 |
+
The name of the library that is making the HTTP request. Will be added to
|
| 63 |
+
the user-agent header.
|
| 64 |
+
library_version (`str`, *optional*):
|
| 65 |
+
The version of the library that is making the HTTP request. Will be added
|
| 66 |
+
to the user-agent header.
|
| 67 |
+
user_agent (`str`, `dict`, *optional*):
|
| 68 |
+
The user agent info in the form of a dictionary or a single string. It will
|
| 69 |
+
be completed with information about the installed packages.
|
| 70 |
+
headers (`dict`, *optional*):
|
| 71 |
+
Additional headers to include in the request. Those headers take precedence
|
| 72 |
+
over the ones generated by this function.
|
| 73 |
+
|
| 74 |
+
Returns:
|
| 75 |
+
A `dict` of headers to pass in your API call.
|
| 76 |
+
|
| 77 |
+
Example:
|
| 78 |
+
```py
|
| 79 |
+
>>> build_hf_headers(token="hf_***") # explicit token
|
| 80 |
+
{"authorization": "Bearer hf_***", "user-agent": ""}
|
| 81 |
+
|
| 82 |
+
>>> build_hf_headers(token=True) # explicitly use cached token
|
| 83 |
+
{"authorization": "Bearer hf_***",...}
|
| 84 |
+
|
| 85 |
+
>>> build_hf_headers(token=False) # explicitly don't use cached token
|
| 86 |
+
{"user-agent": ...}
|
| 87 |
+
|
| 88 |
+
>>> build_hf_headers() # implicit use of the cached token
|
| 89 |
+
{"authorization": "Bearer hf_***",...}
|
| 90 |
+
|
| 91 |
+
# HF_HUB_DISABLE_IMPLICIT_TOKEN=True # to set as env variable
|
| 92 |
+
>>> build_hf_headers() # token is not sent
|
| 93 |
+
{"user-agent": ...}
|
| 94 |
+
|
| 95 |
+
>>> build_hf_headers(library_name="transformers", library_version="1.2.3")
|
| 96 |
+
{"authorization": ..., "user-agent": "transformers/1.2.3; hf_hub/0.10.2; python/3.10.4; tensorflow/1.55"}
|
| 97 |
+
```
|
| 98 |
+
|
| 99 |
+
Raises:
|
| 100 |
+
[`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
| 101 |
+
If organization token is passed and "write" access is required.
|
| 102 |
+
[`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
| 103 |
+
If "write" access is required but token is not passed and not saved locally.
|
| 104 |
+
[`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
|
| 105 |
+
If `token=True` but token is not saved locally.
|
| 106 |
+
"""
|
| 107 |
+
# Get auth token to send
|
| 108 |
+
token_to_send = get_token_to_send(token)
|
| 109 |
+
|
| 110 |
+
# Combine headers
|
| 111 |
+
hf_headers = {
|
| 112 |
+
"user-agent": _http_user_agent(
|
| 113 |
+
library_name=library_name,
|
| 114 |
+
library_version=library_version,
|
| 115 |
+
user_agent=user_agent,
|
| 116 |
+
)
|
| 117 |
+
}
|
| 118 |
+
if token_to_send is not None:
|
| 119 |
+
hf_headers["authorization"] = f"Bearer {token_to_send}"
|
| 120 |
+
if headers is not None:
|
| 121 |
+
hf_headers.update(headers)
|
| 122 |
+
return hf_headers
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def get_token_to_send(token: bool | str | None) -> str | None:
|
| 126 |
+
"""Select the token to send from either `token` or the cache."""
|
| 127 |
+
# Case token is explicitly provided
|
| 128 |
+
if isinstance(token, str):
|
| 129 |
+
return token
|
| 130 |
+
|
| 131 |
+
# Case token is explicitly forbidden
|
| 132 |
+
if token is False:
|
| 133 |
+
return None
|
| 134 |
+
|
| 135 |
+
# Case token is explicitly required
|
| 136 |
+
if token is True:
|
| 137 |
+
cached_token = get_token()
|
| 138 |
+
if cached_token is None:
|
| 139 |
+
raise LocalTokenNotFoundError(
|
| 140 |
+
"Token is required (`token=True`), but no token found. You"
|
| 141 |
+
" need to provide a token or be logged in to Hugging Face with"
|
| 142 |
+
" `hf auth login` or `huggingface_hub.login`. See"
|
| 143 |
+
" https://huggingface.co/settings/tokens."
|
| 144 |
+
)
|
| 145 |
+
return cached_token
|
| 146 |
+
|
| 147 |
+
# Case implicit use of the token is forbidden by env variable. Checked before resolving the
|
| 148 |
+
# cached token: `get_token()` may refresh an OAuth token (network call + file writes), which
|
| 149 |
+
# must not happen when the resolved token wouldn't be used anyway.
|
| 150 |
+
if constants.HF_HUB_DISABLE_IMPLICIT_TOKEN:
|
| 151 |
+
return None
|
| 152 |
+
|
| 153 |
+
# Otherwise: we use the cached token as the user has not explicitly forbidden it
|
| 154 |
+
return get_token()
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def _http_user_agent(
|
| 158 |
+
*,
|
| 159 |
+
library_name: str | None = None,
|
| 160 |
+
library_version: str | None = None,
|
| 161 |
+
user_agent: dict | str | None = None,
|
| 162 |
+
) -> str:
|
| 163 |
+
"""Format a user-agent string containing information about the installed packages.
|
| 164 |
+
|
| 165 |
+
Args:
|
| 166 |
+
library_name (`str`, *optional*):
|
| 167 |
+
The name of the library that is making the HTTP request.
|
| 168 |
+
library_version (`str`, *optional*):
|
| 169 |
+
The version of the library that is making the HTTP request.
|
| 170 |
+
user_agent (`str`, `dict`, *optional*):
|
| 171 |
+
The user agent info in the form of a dictionary or a single string.
|
| 172 |
+
|
| 173 |
+
Returns:
|
| 174 |
+
The formatted user-agent string.
|
| 175 |
+
"""
|
| 176 |
+
if library_name is not None:
|
| 177 |
+
ua = f"{library_name}/{library_version}"
|
| 178 |
+
else:
|
| 179 |
+
ua = "unknown/None"
|
| 180 |
+
ua += f"; hf_hub/{get_hf_hub_version()}"
|
| 181 |
+
ua += f"; python/{get_python_version()}"
|
| 182 |
+
|
| 183 |
+
if not constants.HF_HUB_DISABLE_TELEMETRY:
|
| 184 |
+
if is_torch_available():
|
| 185 |
+
ua += f"; torch/{get_torch_version()}"
|
| 186 |
+
|
| 187 |
+
agent = detect_agent()
|
| 188 |
+
if agent:
|
| 189 |
+
ua += f"; agent/{agent}"
|
| 190 |
+
|
| 191 |
+
if isinstance(user_agent, dict):
|
| 192 |
+
ua += "; " + "; ".join(f"{k}/{v}" for k, v in user_agent.items())
|
| 193 |
+
elif isinstance(user_agent, str):
|
| 194 |
+
ua += "; " + user_agent
|
| 195 |
+
|
| 196 |
+
# Retrieve user-agent origin headers from environment variable
|
| 197 |
+
origin = constants.HF_HUB_USER_AGENT_ORIGIN
|
| 198 |
+
if origin is not None:
|
| 199 |
+
ua += "; origin/" + origin
|
| 200 |
+
|
| 201 |
+
return _deduplicate_user_agent(ua)
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def _deduplicate_user_agent(user_agent: str) -> str:
|
| 205 |
+
"""Deduplicate redundant information in the generated user-agent."""
|
| 206 |
+
# Split around ";" > Strip whitespaces > Store as dict keys (ensure unicity) > format back as string
|
| 207 |
+
# Order is implicitly preserved by dictionary structure (see https://stackoverflow.com/a/53657523).
|
| 208 |
+
return "; ".join({key.strip(): None for key in user_agent.split(";")}.keys())
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_hf_uris.py
ADDED
|
@@ -0,0 +1,642 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Centralized parser for Hugging Face Hub URIs ('hf://...') and mount specifications.
|
| 15 |
+
|
| 16 |
+
A HF URI is a URI-like string that identifies a location on the Hugging Face
|
| 17 |
+
Hub: a model/dataset/space/kernel repository, a bucket, optionally a revision,
|
| 18 |
+
and optionally a path inside the repo or bucket.
|
| 19 |
+
|
| 20 |
+
Canonical syntax:
|
| 21 |
+
|
| 22 |
+
```
|
| 23 |
+
hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]
|
| 24 |
+
```
|
| 25 |
+
|
| 26 |
+
For convenience, [`parse_hf_uri`] also accepts Hugging Face **web URLs** (the
|
| 27 |
+
ones you copy-paste from your browser), e.g.
|
| 28 |
+
'https://huggingface.co/datasets/my-org/my-dataset/blob/main/train.csv'. They are
|
| 29 |
+
normalized to the canonical 'hf://' form before parsing. Only unambiguous URLs
|
| 30 |
+
(repository / bucket pages and file/folder viewer routes) are accepted; any other
|
| 31 |
+
route is rejected rather than guessed.
|
| 32 |
+
|
| 33 |
+
A HF mount wraps a HF URI with a local mount path and an optional ':ro'/':rw'
|
| 34 |
+
flag (used by Spaces and Jobs volumes):
|
| 35 |
+
|
| 36 |
+
```
|
| 37 |
+
hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]:<MOUNT_PATH>[:ro|:rw]
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
See 'docs/source/en/package_reference/hf_uris.md' for the full grammar and examples.
|
| 41 |
+
"""
|
| 42 |
+
|
| 43 |
+
import functools
|
| 44 |
+
import re
|
| 45 |
+
from dataclasses import dataclass, field
|
| 46 |
+
from urllib.parse import quote, unquote, urlsplit
|
| 47 |
+
|
| 48 |
+
from huggingface_hub import constants
|
| 49 |
+
from huggingface_hub.errors import HfUriError, HFValidationError
|
| 50 |
+
|
| 51 |
+
from ._validators import validate_repo_id
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
# Inverse map (singular -> plural URI prefix). Built once from the canonical
|
| 55 |
+
# 'constants.HF_URI_TYPE_PREFIXES' and used to render URIs.
|
| 56 |
+
_TYPE_TO_PREFIX: dict[str, str] = {v: k for k, v in constants.HF_URI_TYPE_PREFIXES.items()}
|
| 57 |
+
|
| 58 |
+
# Special revisions that contain a '/'. They take precedence when splitting
|
| 59 |
+
# the part after '@' into '<revision>/<path-in-repo>'. Matches 'refs/pr/N'
|
| 60 |
+
# (Pull Request refs) and 'refs/convert/<name>' (e.g. parquet conversions).
|
| 61 |
+
# The conversion name allows the typical git ref characters '[a-zA-Z0-9_.-]'
|
| 62 |
+
# so names like 'parquet-v2' or 'duckdb.v1' round-trip correctly.
|
| 63 |
+
_SPECIAL_REFS_REVISION_REGEX = re.compile(r"^refs/(?:convert/[\w.-]+|pr/\d+)")
|
| 64 |
+
|
| 65 |
+
# Same as constants.HfUriType, but as a set of strings for easy lookup.)
|
| 66 |
+
_VALID_URI_TYPES: frozenset[str] = frozenset(constants.HF_URI_TYPE_PREFIXES.values())
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
# Web-viewer routes that point at a file or folder and that map cleanly onto a
|
| 70 |
+
# '<revision>/<path>' pair. Other routes (commit, commits, discussions, settings,
|
| 71 |
+
# edit, ...) do not identify a Hub location and are rejected by the URL parser.
|
| 72 |
+
_URL_REPO_LOCATION_ACTIONS: frozenset[str] = frozenset({"blame", "blob", "raw", "resolve", "tree"})
|
| 73 |
+
# Bucket web routes that point at a file or folder. Buckets are not versioned, so
|
| 74 |
+
# these are followed directly by '<path>' (no revision segment).
|
| 75 |
+
_URL_BUCKET_LOCATION_ACTIONS: frozenset[str] = frozenset({"resolve", "tree"})
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
@dataclass(frozen=True)
|
| 79 |
+
class HfUri:
|
| 80 |
+
"""Parsed representation of a Hugging Face Hub URI ('hf://...').
|
| 81 |
+
|
| 82 |
+
Attributes:
|
| 83 |
+
type (`str`):
|
| 84 |
+
One of 'model', 'dataset', 'space', 'kernel' or 'bucket'.
|
| 85 |
+
id (`str`):
|
| 86 |
+
The repository id ('namespace/name', e.g. 'my-org/my-model') for repo URIs, or the bucket id ('namespace/name') for bucket URIs.
|
| 87 |
+
revision (`str`, *optional*):
|
| 88 |
+
The revision specified after '@' in the URI, URL-decoded. 'None' if no revision was specified, or for bucket URIs (which
|
| 89 |
+
never carry a revision). Special refs like 'refs/pr/10' and 'refs/convert/parquet' are preserved as-is.
|
| 90 |
+
path_in_repo (`str`):
|
| 91 |
+
The path inside the repo or bucket. Empty string if the URI points at the root.
|
| 92 |
+
"""
|
| 93 |
+
|
| 94 |
+
type: constants.HfUriType
|
| 95 |
+
id: str
|
| 96 |
+
revision: str | None = None
|
| 97 |
+
path_in_repo: str = ""
|
| 98 |
+
_raw: str | None = field(repr=False, hash=False, compare=False, default=None)
|
| 99 |
+
|
| 100 |
+
def __post_init__(self) -> None:
|
| 101 |
+
uri = self._raw or "" # For error messages
|
| 102 |
+
|
| 103 |
+
# Check valid URI type
|
| 104 |
+
if self.type not in _VALID_URI_TYPES:
|
| 105 |
+
raise HfUriError(uri=uri, msg=f"Invalid type '{self.type}'. Must be one of {sorted(_VALID_URI_TYPES)}.")
|
| 106 |
+
|
| 107 |
+
# Check valid ID
|
| 108 |
+
if not self.id or self.id.count("/") != 1:
|
| 109 |
+
raise HfUriError(uri=uri, msg=f"Id must be 'namespace/name', got '{self.id}'.")
|
| 110 |
+
if self.type != "bucket":
|
| 111 |
+
try:
|
| 112 |
+
validate_repo_id(self.id)
|
| 113 |
+
except HFValidationError as e:
|
| 114 |
+
raise HfUriError(uri=uri, msg=str(e)) from e
|
| 115 |
+
|
| 116 |
+
# Check valid revision
|
| 117 |
+
if self.revision is not None and not self.revision:
|
| 118 |
+
raise HfUriError(uri=uri, msg="Revision must not be an empty string.")
|
| 119 |
+
if self.type == "bucket" and self.revision is not None:
|
| 120 |
+
raise HfUriError(uri=uri, msg="Bucket URIs do not support a revision.")
|
| 121 |
+
|
| 122 |
+
# Check valid path in repo
|
| 123 |
+
if self.path_in_repo:
|
| 124 |
+
if self.path_in_repo.startswith("/") or "//" in self.path_in_repo:
|
| 125 |
+
raise HfUriError(uri=uri, msg=f"Path must not contain empty segments (got '{self.path_in_repo}').")
|
| 126 |
+
|
| 127 |
+
@property
|
| 128 |
+
def is_bucket(self) -> bool:
|
| 129 |
+
"""True if this URI points at a bucket."""
|
| 130 |
+
return self.type == "bucket"
|
| 131 |
+
|
| 132 |
+
@property
|
| 133 |
+
def is_repo(self) -> bool:
|
| 134 |
+
"""True if this URI points at a repository (model, dataset, space or kernel)."""
|
| 135 |
+
return self.type != "bucket"
|
| 136 |
+
|
| 137 |
+
def to_uri(self) -> str:
|
| 138 |
+
"""Render the URI as a canonical 'hf://' string.
|
| 139 |
+
|
| 140 |
+
The type prefix is always written explicitly (e.g. 'hf://models/my-org/my-model').
|
| 141 |
+
"""
|
| 142 |
+
parts: list[str] = [constants.HF_PROTOCOL, _TYPE_TO_PREFIX[self.type], "/", self.id]
|
| 143 |
+
if self.revision is not None:
|
| 144 |
+
# Encode '/' as '%2F' for revisions that would otherwise be split as '<revision>/<path>'
|
| 145 |
+
# at parse time. Special refs ('refs/pr/N', 'refs/convert/<name>') are kept verbatim
|
| 146 |
+
# because the parser matches them eagerly.
|
| 147 |
+
revision = self.revision
|
| 148 |
+
if "/" in revision and _SPECIAL_REFS_REVISION_REGEX.fullmatch(revision) is None:
|
| 149 |
+
revision = revision.replace("/", "%2F")
|
| 150 |
+
parts.append(f"@{revision}")
|
| 151 |
+
if self.path_in_repo:
|
| 152 |
+
parts.append(f"/{self.path_in_repo}")
|
| 153 |
+
return "".join(parts)
|
| 154 |
+
|
| 155 |
+
def to_url(self, endpoint: str | None = None) -> str:
|
| 156 |
+
"""Render the URI as a Hugging Face **web URL** (the kind you open in a browser).
|
| 157 |
+
|
| 158 |
+
This is the inverse of parsing a URL with [`parse_hf_uri`]. The returned URL points at:
|
| 159 |
+
|
| 160 |
+
- the repository / bucket landing page when no path or revision is set;
|
| 161 |
+
- the folder viewer ('/tree/<revision>') when only a revision is set;
|
| 162 |
+
- the file viewer ('/blob/<revision>/<path>') for repository files (revision defaults to 'main');
|
| 163 |
+
- the tree route ('/tree/<path>') for bucket files (buckets are not versioned).
|
| 164 |
+
|
| 165 |
+
Args:
|
| 166 |
+
endpoint (`str`, *optional*):
|
| 167 |
+
Base endpoint to use. Defaults to 'constants.ENDPOINT' (i.e. 'https://huggingface.co').
|
| 168 |
+
|
| 169 |
+
Returns:
|
| 170 |
+
`str`: the web URL.
|
| 171 |
+
|
| 172 |
+
Example:
|
| 173 |
+
```py
|
| 174 |
+
>>> from huggingface_hub import parse_hf_uri
|
| 175 |
+
>>> parse_hf_uri("hf://datasets/my-org/my-dataset@v1/train.csv").to_url()
|
| 176 |
+
'https://huggingface.co/datasets/my-org/my-dataset/blob/v1/train.csv'
|
| 177 |
+
```
|
| 178 |
+
"""
|
| 179 |
+
base = (endpoint or constants.ENDPOINT).rstrip("/")
|
| 180 |
+
# Percent-encode characters that would otherwise break the URL (spaces, '#', '?', ...),
|
| 181 |
+
# keeping '/' as the path separator. This is the inverse of the decoding done when parsing.
|
| 182 |
+
path = quote(self.path_in_repo, safe="/")
|
| 183 |
+
|
| 184 |
+
if self.type == "bucket":
|
| 185 |
+
url = f"{base}/buckets/{self.id}"
|
| 186 |
+
if path:
|
| 187 |
+
url += f"/tree/{path}"
|
| 188 |
+
return url
|
| 189 |
+
|
| 190 |
+
# Models live at the root ('hf.co/<id>'); other repos are namespaced by their plural prefix.
|
| 191 |
+
url = f"{base}/{self.id}" if self.type == "model" else f"{base}/{_TYPE_TO_PREFIX[self.type]}/{self.id}"
|
| 192 |
+
revision = self.revision
|
| 193 |
+
# Percent-encode the branch/tag name so it stays a single, URL-safe segment: a '/' would
|
| 194 |
+
# otherwise open a new path segment and '#'/'?' would be read as a fragment/query when the
|
| 195 |
+
# URL is opened or parsed back. This mirrors the decoding done when parsing. Special refs
|
| 196 |
+
# ('refs/pr/N', 'refs/convert/<name>') are used verbatim by the Hub web routes.
|
| 197 |
+
if revision is not None and _SPECIAL_REFS_REVISION_REGEX.fullmatch(revision) is None:
|
| 198 |
+
revision = quote(revision, safe="")
|
| 199 |
+
if path:
|
| 200 |
+
url += f"/blob/{revision or constants.DEFAULT_REVISION}/{path}"
|
| 201 |
+
elif revision is not None:
|
| 202 |
+
url += f"/tree/{revision}"
|
| 203 |
+
return url
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
@dataclass(frozen=True)
|
| 207 |
+
class HfMount:
|
| 208 |
+
"""A HF URI paired with a local mount path and optional read-only flag.
|
| 209 |
+
|
| 210 |
+
Used by Spaces and Jobs to describe volume mounts. The full syntax is:
|
| 211 |
+
|
| 212 |
+
```
|
| 213 |
+
hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]:<MOUNT_PATH>[:ro|:rw]
|
| 214 |
+
```
|
| 215 |
+
|
| 216 |
+
Attributes:
|
| 217 |
+
source ([`HfUri`]):
|
| 218 |
+
The parsed HF URI identifying the Hub resource to mount.
|
| 219 |
+
mount_path (`str`):
|
| 220 |
+
The local mount path (always starts with '/').
|
| 221 |
+
read_only (`bool`, *optional*):
|
| 222 |
+
True if the mount ends with ':ro', False if it ends with ':rw', 'None' if no flag was provided.
|
| 223 |
+
"""
|
| 224 |
+
|
| 225 |
+
source: HfUri
|
| 226 |
+
mount_path: str
|
| 227 |
+
read_only: bool | None = None
|
| 228 |
+
_raw: str | None = field(repr=False, hash=False, compare=False, default=None)
|
| 229 |
+
|
| 230 |
+
def __post_init__(self) -> None:
|
| 231 |
+
raw = self._raw or ""
|
| 232 |
+
if not self.mount_path.startswith("/") or self.mount_path == "/":
|
| 233 |
+
raise HfUriError(
|
| 234 |
+
uri=raw,
|
| 235 |
+
msg=f"Mount path must be a non-empty absolute path starting with '/', got '{self.mount_path}'.",
|
| 236 |
+
)
|
| 237 |
+
|
| 238 |
+
def to_uri(self) -> str:
|
| 239 |
+
"""Render the mount as a canonical 'hf://' string.
|
| 240 |
+
|
| 241 |
+
Example: 'hf://models/my-org/my-model:/data:ro'
|
| 242 |
+
"""
|
| 243 |
+
parts = [self.source.to_uri(), ":", self.mount_path]
|
| 244 |
+
if self.read_only is not None:
|
| 245 |
+
parts.append(":ro" if self.read_only else ":rw")
|
| 246 |
+
return "".join(parts)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def is_hf_uri(uri: str) -> bool:
|
| 250 |
+
"""Check if a string is a valid HF URI ('hf://...') or a recognized Hugging Face web URL."""
|
| 251 |
+
try:
|
| 252 |
+
parse_hf_uri(uri)
|
| 253 |
+
return True
|
| 254 |
+
except HfUriError:
|
| 255 |
+
return False
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
@functools.lru_cache
|
| 259 |
+
def parse_hf_uri(uri: str, endpoint: str | None = None) -> HfUri:
|
| 260 |
+
"""Parse a Hugging Face Hub URI ('hf://...') or a Hugging Face web URL.
|
| 261 |
+
|
| 262 |
+
A HF URI is a URI-like string identifying a location on the Hugging Face Hub. The full grammar is:
|
| 263 |
+
|
| 264 |
+
```
|
| 265 |
+
hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]
|
| 266 |
+
```
|
| 267 |
+
|
| 268 |
+
For convenience, Hugging Face **web URLs** (the ones you copy-paste from the website) are also
|
| 269 |
+
accepted and normalized to the canonical 'hf://' form, e.g.
|
| 270 |
+
'https://huggingface.co/datasets/my-org/my-dataset/blob/main/train.csv'. Only unambiguous URLs
|
| 271 |
+
(repository / bucket pages and file/folder viewer routes) are accepted; any other route is rejected.
|
| 272 |
+
|
| 273 |
+
See 'docs/source/en/package_reference/hf_uris.md' for the full specification.
|
| 274 |
+
|
| 275 |
+
Args:
|
| 276 |
+
uri (`str`):
|
| 277 |
+
The URI to parse. Must start with 'hf://', or be a Hugging Face URL (e.g. 'https://huggingface.co/...').
|
| 278 |
+
endpoint (`str`, *optional*):
|
| 279 |
+
A custom Hub endpoint (e.g. a self-hosted or proxied Hub like 'https://hub.my-company.com' or
|
| 280 |
+
'http://localhost:8080/hf'). When provided, web URLs on that endpoint are recognized in addition to
|
| 281 |
+
the default Hugging Face hosts. Has no effect on 'hf://' URIs.
|
| 282 |
+
|
| 283 |
+
Returns:
|
| 284 |
+
[`HfUri`]: the parsed URI.
|
| 285 |
+
|
| 286 |
+
Raises:
|
| 287 |
+
[`HfUriError`]:
|
| 288 |
+
If the URI is malformed (missing prefix, invalid type, missing id, unsupported URL route, etc.).
|
| 289 |
+
|
| 290 |
+
Examples:
|
| 291 |
+
```py
|
| 292 |
+
>>> from huggingface_hub.utils import parse_hf_uri
|
| 293 |
+
>>> parse_hf_uri("hf://my-org/my-model")
|
| 294 |
+
HfUri(type='model', id='my-org/my-model', revision=None, path_in_repo='')
|
| 295 |
+
>>> parse_hf_uri("hf://datasets/my-org/my-dataset@refs/pr/3/train.json")
|
| 296 |
+
HfUri(type='dataset', id='my-org/my-dataset', revision='refs/pr/3', path_in_repo='train.json')
|
| 297 |
+
>>> parse_hf_uri("https://huggingface.co/datasets/my-org/my-dataset/blob/main/train.csv")
|
| 298 |
+
HfUri(type='dataset', id='my-org/my-dataset', revision='main', path_in_repo='train.csv')
|
| 299 |
+
```
|
| 300 |
+
"""
|
| 301 |
+
raw = uri
|
| 302 |
+
if uri.startswith(constants.HF_PROTOCOL):
|
| 303 |
+
body = uri[len(constants.HF_PROTOCOL) :]
|
| 304 |
+
if not body:
|
| 305 |
+
raise HfUriError(uri, f"Empty body after '{constants.HF_PROTOCOL}'.")
|
| 306 |
+
elif _looks_like_hf_url(uri, endpoint=endpoint):
|
| 307 |
+
body = _url_to_uri_body(uri, endpoint=endpoint)
|
| 308 |
+
else:
|
| 309 |
+
raise HfUriError(
|
| 310 |
+
uri,
|
| 311 |
+
f"Must start with '{constants.HF_PROTOCOL}' or be a Hugging Face URL (e.g. 'https://huggingface.co/...'). "
|
| 312 |
+
f"Expected format: {constants.HF_PROTOCOL}[<TYPE>/]<ID>[@<REVISION>][/<PATH>]",
|
| 313 |
+
)
|
| 314 |
+
|
| 315 |
+
type_, location = _split_type(body, raw=raw)
|
| 316 |
+
|
| 317 |
+
if type_ == "bucket":
|
| 318 |
+
return _parse_bucket_body(location, type_, raw=raw)
|
| 319 |
+
return _parse_repo_body(location, type_, raw=raw)
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def _endpoint_host_and_path(endpoint: str | None) -> tuple[str | None, str]:
|
| 323 |
+
"""Return the lowercased host and stripped path prefix of a custom Hub 'endpoint'.
|
| 324 |
+
|
| 325 |
+
E.g. 'https://hub.my-company.com' -> ('hub.my-company.com', '') and a self-hosted
|
| 326 |
+
'http://localhost:8080/hf' -> ('localhost', 'hf'). Returns '(None, "")' when 'endpoint' is None.
|
| 327 |
+
"""
|
| 328 |
+
if endpoint is None:
|
| 329 |
+
return None, ""
|
| 330 |
+
# Prefix '//' for scheme-less endpoints so 'urlsplit' populates 'netloc' instead of 'path'.
|
| 331 |
+
parsed = urlsplit(endpoint if "://" in endpoint else "//" + endpoint)
|
| 332 |
+
host = parsed.hostname.lower() if parsed.hostname else None
|
| 333 |
+
return host, parsed.path.strip("/")
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def _recognized_hosts(endpoint: str | None) -> frozenset[str]:
|
| 337 |
+
"""The set of hosts whose web URLs can be parsed: the default Hugging Face hosts plus 'endpoint'."""
|
| 338 |
+
host, _ = _endpoint_host_and_path(endpoint)
|
| 339 |
+
return constants.HF_URL_HOSTS | {host} if host else constants.HF_URL_HOSTS
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def _looks_like_hf_url(uri: str, endpoint: str | None = None) -> bool:
|
| 343 |
+
"""Return True if 'uri' looks like a (possibly scheme-less) Hugging Face web URL."""
|
| 344 |
+
lowered = uri.lower()
|
| 345 |
+
if lowered.startswith(("http://", "https://")):
|
| 346 |
+
return True
|
| 347 |
+
# Scheme-less host (e.g. 'huggingface.co/org/model').
|
| 348 |
+
return any(lowered == host or lowered.startswith(host + "/") for host in _recognized_hosts(endpoint))
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def _decode_url_path_segment(segment: str) -> str:
|
| 352 |
+
"""Percent-decode a single URL path segment (e.g. 'file%20name.txt' -> 'file name.txt').
|
| 353 |
+
|
| 354 |
+
A decoded '/' is re-encoded as '%2F' so the segment stays atomic when the normalized body is
|
| 355 |
+
re-split by the shared parser. This decodes ordinary path characters (spaces, '#', ...) that
|
| 356 |
+
browsers encode, while keeping '%2F'-encoded revisions (e.g. 'feature%2Ffoo') intact.
|
| 357 |
+
"""
|
| 358 |
+
return unquote(segment).replace("/", "%2F")
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def _url_to_uri_body(url: str, endpoint: str | None = None) -> str:
|
| 362 |
+
"""Normalize a Hugging Face web URL into the body of a 'hf://' URI (everything after 'hf://').
|
| 363 |
+
|
| 364 |
+
The returned string is fed back into the regular URI parsing logic, so all validation
|
| 365 |
+
(repo id, revision, empty path segments, ...) is shared with the canonical 'hf://' path.
|
| 366 |
+
Only unambiguous URLs are accepted: any unrecognized route raises [`HfUriError`]. When 'endpoint'
|
| 367 |
+
is provided, URLs on that custom Hub host are recognized too (and its path prefix is stripped).
|
| 368 |
+
"""
|
| 369 |
+
raw = url
|
| 370 |
+
# Prefix '//' for scheme-less inputs so 'urlsplit' populates 'netloc' instead of 'path'.
|
| 371 |
+
parsed = urlsplit(url if "://" in url else "//" + url)
|
| 372 |
+
host = (parsed.hostname or "").lower()
|
| 373 |
+
if host not in _recognized_hosts(endpoint):
|
| 374 |
+
raise HfUriError(
|
| 375 |
+
uri=raw,
|
| 376 |
+
msg=f"Unrecognized host '{host or url}'. Expected a Hugging Face URL (e.g. 'https://huggingface.co/...').",
|
| 377 |
+
)
|
| 378 |
+
|
| 379 |
+
# Query string and fragment are intentionally dropped (e.g. '?download=true').
|
| 380 |
+
path = parsed.path
|
| 381 |
+
# For a self-hosted endpoint with a path prefix (e.g. 'http://localhost:8080/hf'), drop it so the
|
| 382 |
+
# remaining segments are '[<TYPE>/]<namespace>/<name>[/...]' just like on the public Hub.
|
| 383 |
+
endpoint_host, endpoint_path = _endpoint_host_and_path(endpoint)
|
| 384 |
+
if endpoint_path and host == endpoint_host:
|
| 385 |
+
prefix = "/" + endpoint_path
|
| 386 |
+
if path == prefix or path.startswith(prefix + "/"):
|
| 387 |
+
path = path[len(prefix) :]
|
| 388 |
+
segments = [segment for segment in path.split("/") if segment]
|
| 389 |
+
if not segments:
|
| 390 |
+
raise HfUriError(uri=raw, msg=f"Missing repository or bucket identifier in URL '{url}'.")
|
| 391 |
+
|
| 392 |
+
# Optional type prefix ('datasets', 'spaces', 'kernels', 'buckets', 'models').
|
| 393 |
+
type_prefix: str | None = None
|
| 394 |
+
if segments[0] in constants.HF_URI_TYPE_PREFIXES:
|
| 395 |
+
type_prefix = segments[0]
|
| 396 |
+
segments = segments[1:]
|
| 397 |
+
|
| 398 |
+
# Everything in the web UI is namespaced ('<namespace>/<name>'); a single segment is a user or
|
| 399 |
+
# organization page (or a listing page), which we cannot map to a repository -> reject.
|
| 400 |
+
if len(segments) < 2:
|
| 401 |
+
raise HfUriError(
|
| 402 |
+
uri=raw,
|
| 403 |
+
msg=(
|
| 404 |
+
f"Cannot parse URL '{url}': expected a '<namespace>/<name>' repository or bucket. "
|
| 405 |
+
"User/organization pages and single-segment URLs are not supported."
|
| 406 |
+
),
|
| 407 |
+
)
|
| 408 |
+
repo_id = f"{segments[0]}/{segments[1]}"
|
| 409 |
+
rest = segments[2:]
|
| 410 |
+
|
| 411 |
+
if type_prefix == "buckets":
|
| 412 |
+
if not rest:
|
| 413 |
+
return f"buckets/{repo_id}"
|
| 414 |
+
action, *tail = rest
|
| 415 |
+
if action not in _URL_BUCKET_LOCATION_ACTIONS:
|
| 416 |
+
raise HfUriError(uri=raw, msg=f"Cannot parse bucket URL '{url}': unsupported '/{action}/' route.")
|
| 417 |
+
path = "/".join(_decode_url_path_segment(segment) for segment in tail)
|
| 418 |
+
return f"buckets/{repo_id}/{path}" if path else f"buckets/{repo_id}"
|
| 419 |
+
|
| 420 |
+
prefix = f"{type_prefix}/" if type_prefix else ""
|
| 421 |
+
if not rest:
|
| 422 |
+
return f"{prefix}{repo_id}"
|
| 423 |
+
action, *tail = rest
|
| 424 |
+
if action not in _URL_REPO_LOCATION_ACTIONS:
|
| 425 |
+
raise HfUriError(
|
| 426 |
+
uri=raw,
|
| 427 |
+
msg=(
|
| 428 |
+
f"Cannot parse URL '{url}': unsupported '/{action}/' route. "
|
| 429 |
+
"Only repository pages and file/folder viewer routes (blob, resolve, raw, tree, ...) can be parsed."
|
| 430 |
+
),
|
| 431 |
+
)
|
| 432 |
+
if not tail:
|
| 433 |
+
# e.g. '.../tree' with nothing after -> repository root.
|
| 434 |
+
return f"{prefix}{repo_id}"
|
| 435 |
+
# 'tail' is '<revision>/<path>'; reuse the canonical '@<revision>/<path>' splitting logic
|
| 436 |
+
# (special refs, URL-encoded slashes, ...) by handing it back to the URI parser. Each segment
|
| 437 |
+
# is percent-decoded first so file names with spaces, '#', ... resolve correctly; the revision
|
| 438 |
+
# segment's '%2F' survives (re-encoded by '_decode_url_path_segment') and is decoded downstream.
|
| 439 |
+
decoded = "/".join(_decode_url_path_segment(segment) for segment in tail)
|
| 440 |
+
return f"{prefix}{repo_id}@{decoded}"
|
| 441 |
+
|
| 442 |
+
|
| 443 |
+
def parse_hf_mount(mount_str: str) -> HfMount:
|
| 444 |
+
"""Parse a HF mount specification ('hf://...:<MOUNT_PATH>[:ro|:rw]').
|
| 445 |
+
|
| 446 |
+
A mount specification is a HF URI followed by a local mount path and an optional read-only/read-write flag.
|
| 447 |
+
The full grammar is:
|
| 448 |
+
|
| 449 |
+
```
|
| 450 |
+
hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]:<MOUNT_PATH>[:ro|:rw]
|
| 451 |
+
```
|
| 452 |
+
|
| 453 |
+
See 'docs/source/en/package_reference/hf_uris.md' for the full specification.
|
| 454 |
+
|
| 455 |
+
Args:
|
| 456 |
+
mount_str (`str`):
|
| 457 |
+
The mount string to parse. Must start with 'hf://' and contain a ':<MOUNT_PATH>' segment.
|
| 458 |
+
|
| 459 |
+
Returns:
|
| 460 |
+
[`HfMount`]: the parsed mount.
|
| 461 |
+
|
| 462 |
+
Raises:
|
| 463 |
+
[`HfUriError`]:
|
| 464 |
+
If the mount string is malformed (missing mount path, invalid URI, etc.).
|
| 465 |
+
|
| 466 |
+
Examples:
|
| 467 |
+
```py
|
| 468 |
+
>>> from huggingface_hub.utils import parse_hf_mount
|
| 469 |
+
>>> parse_hf_mount("hf://my-org/my-model:/data:ro")
|
| 470 |
+
HfMount(source=HfUri(type='model', id='my-org/my-model', revision=None, path_in_repo=''), mount_path='/data', read_only=True)
|
| 471 |
+
>>> parse_hf_mount("hf://buckets/my-org/my-bucket/sub/dir:/mnt:rw")
|
| 472 |
+
HfMount(source=HfUri(type='bucket', id='my-org/my-bucket', revision=None, path_in_repo='sub/dir'), mount_path='/mnt', read_only=False)
|
| 473 |
+
```
|
| 474 |
+
"""
|
| 475 |
+
if not mount_str.startswith(constants.HF_PROTOCOL):
|
| 476 |
+
raise HfUriError(
|
| 477 |
+
uri=mount_str,
|
| 478 |
+
msg=f"Must start with '{constants.HF_PROTOCOL}'.",
|
| 479 |
+
)
|
| 480 |
+
|
| 481 |
+
raw = mount_str
|
| 482 |
+
body = mount_str[len(constants.HF_PROTOCOL) :]
|
| 483 |
+
if not body:
|
| 484 |
+
raise HfUriError(uri=raw, msg=f"Empty body after '{constants.HF_PROTOCOL}'.")
|
| 485 |
+
|
| 486 |
+
location, mount_path, read_only = _split_mount(body, raw=raw)
|
| 487 |
+
|
| 488 |
+
if mount_path is None:
|
| 489 |
+
raise HfUriError(uri=raw, msg="Missing mount path. Expected ':<MOUNT_PATH>' (e.g. 'hf://org/model:/data').")
|
| 490 |
+
|
| 491 |
+
# Re-assemble the URI part and parse it
|
| 492 |
+
uri_str = constants.HF_PROTOCOL + location
|
| 493 |
+
try:
|
| 494 |
+
source = parse_hf_uri(uri_str)
|
| 495 |
+
except HfUriError as e:
|
| 496 |
+
raise HfUriError(uri=raw, msg=e.msg) from e
|
| 497 |
+
|
| 498 |
+
return HfMount(source=source, mount_path=mount_path, read_only=read_only, _raw=raw)
|
| 499 |
+
|
| 500 |
+
|
| 501 |
+
def _split_mount(body: str, *, raw: str) -> tuple[str, str | None, bool | None]:
|
| 502 |
+
"""Split the ':<MOUNT_PATH>[:ro|:rw]' suffix from 'body'.
|
| 503 |
+
|
| 504 |
+
Returns '(location, mount_path, read_only)' where 'mount_path' is 'None' if no mount segment is present.
|
| 505 |
+
"""
|
| 506 |
+
if body.endswith(":ro"):
|
| 507 |
+
read_only, body = True, body.removesuffix(":ro")
|
| 508 |
+
elif body.endswith(":rw"):
|
| 509 |
+
read_only, body = False, body.removesuffix(":rw")
|
| 510 |
+
else:
|
| 511 |
+
read_only = None
|
| 512 |
+
|
| 513 |
+
# Mount paths always start with '/', so the delimiter is ':/'.
|
| 514 |
+
# We use rfind() because the mount segment is always trailing
|
| 515 |
+
idx = body.rfind(":/")
|
| 516 |
+
if idx == -1:
|
| 517 |
+
if read_only is not None:
|
| 518 |
+
raise HfUriError(
|
| 519 |
+
uri=raw,
|
| 520 |
+
msg="':ro'/':rw' suffix is only valid when a mount path is provided (e.g. 'hf://...:/<MOUNT_PATH>:ro').",
|
| 521 |
+
)
|
| 522 |
+
return body, None, None
|
| 523 |
+
|
| 524 |
+
location = body[:idx]
|
| 525 |
+
mount_path = body[idx + 1 :] # includes the leading '/'
|
| 526 |
+
if not location:
|
| 527 |
+
raise HfUriError(uri=raw, msg="Missing location before mount path.")
|
| 528 |
+
return location, mount_path, read_only
|
| 529 |
+
|
| 530 |
+
|
| 531 |
+
def _split_type(location: str, *, raw: str) -> tuple[constants.HfUriType, str]:
|
| 532 |
+
"""Detect the (optional) type prefix and return '(type, remaining_location)'.
|
| 533 |
+
|
| 534 |
+
A missing type prefix defaults to 'model'. Singular forms ('model/', 'dataset/', etc.) are explicitly rejected with a helpful error.
|
| 535 |
+
"""
|
| 536 |
+
slash_idx = location.find("/")
|
| 537 |
+
if slash_idx == -1:
|
| 538 |
+
# Single segment, no prefix. Reject if it looks like a bare type name.
|
| 539 |
+
if location in constants.HF_URI_TYPE_PREFIXES:
|
| 540 |
+
raise HfUriError(
|
| 541 |
+
uri=raw,
|
| 542 |
+
msg=f"Missing identifier after '{location}'. Expected '{constants.HF_PROTOCOL}{location}/<ID>'.",
|
| 543 |
+
)
|
| 544 |
+
if (singular_plural := _TYPE_TO_PREFIX.get(location)) is not None:
|
| 545 |
+
raise HfUriError(
|
| 546 |
+
uri=raw,
|
| 547 |
+
msg=f"Type prefix must be plural. Did you mean '{constants.HF_PROTOCOL}{singular_plural}/...'?",
|
| 548 |
+
)
|
| 549 |
+
return "model", location
|
| 550 |
+
|
| 551 |
+
first = location[:slash_idx]
|
| 552 |
+
rest = location[slash_idx + 1 :]
|
| 553 |
+
if first in constants.HF_URI_TYPE_PREFIXES:
|
| 554 |
+
return constants.HF_URI_TYPE_PREFIXES[first], rest
|
| 555 |
+
if (singular_plural := _TYPE_TO_PREFIX.get(first)) is not None:
|
| 556 |
+
raise HfUriError(
|
| 557 |
+
uri=raw, msg=f"Type prefix must be plural, got '{first}/'. Did you mean '{singular_plural}/'?"
|
| 558 |
+
)
|
| 559 |
+
return "model", location
|
| 560 |
+
|
| 561 |
+
|
| 562 |
+
def _parse_bucket_body(
|
| 563 |
+
location: str,
|
| 564 |
+
type_: constants.HfUriType,
|
| 565 |
+
*,
|
| 566 |
+
raw: str,
|
| 567 |
+
) -> HfUri:
|
| 568 |
+
"""Parse the body of a bucket URI: 'namespace/name[/path]'."""
|
| 569 |
+
location = location.strip("/")
|
| 570 |
+
parts = location.split("/", 2)
|
| 571 |
+
if len(parts) < 2 or not parts[0] or not parts[1]:
|
| 572 |
+
raise HfUriError(uri=raw, msg=f"Bucket id must be 'namespace/name', got '{location}'.")
|
| 573 |
+
bucket_id = f"{parts[0]}/{parts[1]}"
|
| 574 |
+
if "@" in bucket_id:
|
| 575 |
+
raise HfUriError(uri=raw, msg="Bucket URIs do not support a revision marker ('@').")
|
| 576 |
+
path_in_bucket = parts[2] if len(parts) >= 3 else ""
|
| 577 |
+
return HfUri(
|
| 578 |
+
type=type_,
|
| 579 |
+
id=bucket_id,
|
| 580 |
+
revision=None,
|
| 581 |
+
path_in_repo=path_in_bucket,
|
| 582 |
+
_raw=raw,
|
| 583 |
+
)
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def _parse_repo_body(
|
| 587 |
+
location: str,
|
| 588 |
+
type_: constants.HfUriType,
|
| 589 |
+
*,
|
| 590 |
+
raw: str,
|
| 591 |
+
) -> HfUri:
|
| 592 |
+
"""Parse the body of a repo URI: '<repo_id>[@<revision>][/<path>]'."""
|
| 593 |
+
location = location.strip("/")
|
| 594 |
+
if not location:
|
| 595 |
+
raise HfUriError(uri=raw, msg="Missing repository id.")
|
| 596 |
+
|
| 597 |
+
# The '@' separates the repo_id from the revision, but only when it
|
| 598 |
+
# appears right after 'namespace/name' (at most one '/' before it).
|
| 599 |
+
# An '@' deeper in the path (e.g. in a filename like 'file@1.txt') is literal.
|
| 600 |
+
at_idx = location.find("@")
|
| 601 |
+
revision: str | None
|
| 602 |
+
|
| 603 |
+
if at_idx == -1 or location[:at_idx].count("/") > 1:
|
| 604 |
+
# No '@' at all, or the '@' is past the repo_id portion (in a filename).
|
| 605 |
+
revision = None
|
| 606 |
+
parts = location.split("/", 2)
|
| 607 |
+
if len(parts) < 2:
|
| 608 |
+
raise HfUriError(uri=raw, msg=f"Repository id must be 'namespace/name', got '{location}'. ")
|
| 609 |
+
repo_id = f"{parts[0]}/{parts[1]}"
|
| 610 |
+
path_in_repo = parts[2] if len(parts) > 2 else ""
|
| 611 |
+
else:
|
| 612 |
+
repo_id = location[:at_idx]
|
| 613 |
+
rev_and_path = location[at_idx + 1 :]
|
| 614 |
+
if not repo_id:
|
| 615 |
+
raise HfUriError(uri=raw, msg="Missing repository id before '@'.")
|
| 616 |
+
if repo_id.count("/") != 1:
|
| 617 |
+
raise HfUriError(uri=raw, msg=f"Repository id must be 'namespace/name', got '{repo_id}'.")
|
| 618 |
+
# Special refs like 'refs/pr/10' contain '/' and must be matched eagerly,
|
| 619 |
+
# otherwise we would split them at the first '/' and treat the rest as a path.
|
| 620 |
+
match = _SPECIAL_REFS_REVISION_REGEX.match(rev_and_path)
|
| 621 |
+
if match is not None:
|
| 622 |
+
revision = match.group()
|
| 623 |
+
path_in_repo = rev_and_path[len(revision) :].removeprefix("/")
|
| 624 |
+
else:
|
| 625 |
+
slash_idx = rev_and_path.find("/")
|
| 626 |
+
if slash_idx == -1:
|
| 627 |
+
revision = rev_and_path
|
| 628 |
+
path_in_repo = ""
|
| 629 |
+
else:
|
| 630 |
+
revision = rev_and_path[:slash_idx]
|
| 631 |
+
path_in_repo = rev_and_path[slash_idx + 1 :]
|
| 632 |
+
revision = unquote(revision)
|
| 633 |
+
if not revision:
|
| 634 |
+
raise HfUriError(uri=raw, msg="Empty revision after '@'.")
|
| 635 |
+
|
| 636 |
+
return HfUri(
|
| 637 |
+
type=type_,
|
| 638 |
+
id=repo_id,
|
| 639 |
+
revision=revision,
|
| 640 |
+
path_in_repo=path_in_repo,
|
| 641 |
+
_raw=raw,
|
| 642 |
+
)
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_http.py
ADDED
|
@@ -0,0 +1,1163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains utilities to handle HTTP requests in huggingface_hub."""
|
| 15 |
+
|
| 16 |
+
import atexit
|
| 17 |
+
import io
|
| 18 |
+
import json
|
| 19 |
+
import os
|
| 20 |
+
import re
|
| 21 |
+
import threading
|
| 22 |
+
import time
|
| 23 |
+
import uuid
|
| 24 |
+
from collections.abc import Callable, Generator, Mapping
|
| 25 |
+
from contextlib import contextmanager
|
| 26 |
+
from dataclasses import dataclass
|
| 27 |
+
from shlex import quote
|
| 28 |
+
from typing import Any, TypeVar
|
| 29 |
+
from urllib.parse import urlparse
|
| 30 |
+
|
| 31 |
+
import httpx
|
| 32 |
+
|
| 33 |
+
from huggingface_hub.errors import OfflineModeIsEnabled
|
| 34 |
+
|
| 35 |
+
from .. import constants
|
| 36 |
+
from ..errors import (
|
| 37 |
+
BadRequestError,
|
| 38 |
+
BucketNotFoundError,
|
| 39 |
+
DisabledRepoError,
|
| 40 |
+
GatedRepoError,
|
| 41 |
+
HfHubHTTPError,
|
| 42 |
+
JobNotFoundError,
|
| 43 |
+
RemoteEntryNotFoundError,
|
| 44 |
+
RepositoryNotFoundError,
|
| 45 |
+
RevisionNotFoundError,
|
| 46 |
+
)
|
| 47 |
+
from . import logging
|
| 48 |
+
from ._lfs import SliceFileObj
|
| 49 |
+
from ._typing import HTTP_METHOD_T
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
logger = logging.get_logger(__name__)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
@dataclass(frozen=True)
|
| 56 |
+
class RateLimitInfo:
|
| 57 |
+
"""
|
| 58 |
+
Parsed rate limit information from HTTP response headers.
|
| 59 |
+
|
| 60 |
+
Attributes:
|
| 61 |
+
resource_type (`str`): The type of resource being rate limited.
|
| 62 |
+
remaining (`int`): The number of requests remaining in the current window.
|
| 63 |
+
reset_in_seconds (`int`): The number of seconds until the rate limit resets.
|
| 64 |
+
limit (`int`, *optional*): The maximum number of requests allowed in the current window.
|
| 65 |
+
window_seconds (`int`, *optional*): The number of seconds in the current window.
|
| 66 |
+
|
| 67 |
+
"""
|
| 68 |
+
|
| 69 |
+
resource_type: str
|
| 70 |
+
remaining: int
|
| 71 |
+
reset_in_seconds: int
|
| 72 |
+
limit: int | None = None
|
| 73 |
+
window_seconds: int | None = None
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
# Regex patterns for parsing rate limit headers
|
| 77 |
+
# e.g.: "api";r=0;t=55 --> resource_type="api", r=0, t=55
|
| 78 |
+
_RATELIMIT_REGEX = re.compile(r"\"(?P<resource_type>\w+)\"\s*;\s*r\s*=\s*(?P<r>\d+)\s*;\s*t\s*=\s*(?P<t>\d+)")
|
| 79 |
+
# e.g.: "fixed window";"api";q=500;w=300 --> q=500, w=300
|
| 80 |
+
_RATELIMIT_POLICY_REGEX = re.compile(r"q\s*=\s*(?P<q>\d+).*?w\s*=\s*(?P<w>\d+)")
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def parse_ratelimit_headers(headers: Mapping[str, str]) -> RateLimitInfo | None:
|
| 84 |
+
"""Parse rate limit information from HTTP response headers.
|
| 85 |
+
|
| 86 |
+
Follows IETF draft: https://www.ietf.org/archive/id/draft-ietf-httpapi-ratelimit-headers-09.html
|
| 87 |
+
Only a subset is implemented.
|
| 88 |
+
|
| 89 |
+
Example:
|
| 90 |
+
```python
|
| 91 |
+
>>> from huggingface_hub.utils import parse_ratelimit_headers
|
| 92 |
+
>>> headers = {
|
| 93 |
+
... "ratelimit": '"api";r=0;t=55',
|
| 94 |
+
... "ratelimit-policy": '"fixed window";"api";q=500;w=300',
|
| 95 |
+
... }
|
| 96 |
+
>>> info = parse_ratelimit_headers(headers)
|
| 97 |
+
>>> info.remaining
|
| 98 |
+
0
|
| 99 |
+
>>> info.reset_in_seconds
|
| 100 |
+
55
|
| 101 |
+
```
|
| 102 |
+
"""
|
| 103 |
+
|
| 104 |
+
ratelimit: str | None = None
|
| 105 |
+
policy: str | None = None
|
| 106 |
+
for key in headers:
|
| 107 |
+
lower_key = key.lower()
|
| 108 |
+
if lower_key == "ratelimit":
|
| 109 |
+
ratelimit = headers[key]
|
| 110 |
+
elif lower_key == "ratelimit-policy":
|
| 111 |
+
policy = headers[key]
|
| 112 |
+
|
| 113 |
+
if not ratelimit:
|
| 114 |
+
return None
|
| 115 |
+
|
| 116 |
+
match = _RATELIMIT_REGEX.search(ratelimit)
|
| 117 |
+
if not match:
|
| 118 |
+
return None
|
| 119 |
+
|
| 120 |
+
resource_type = match.group("resource_type")
|
| 121 |
+
remaining = int(match.group("r"))
|
| 122 |
+
reset_in_seconds = int(match.group("t"))
|
| 123 |
+
|
| 124 |
+
limit: int | None = None
|
| 125 |
+
window_seconds: int | None = None
|
| 126 |
+
|
| 127 |
+
if policy:
|
| 128 |
+
policy_match = _RATELIMIT_POLICY_REGEX.search(policy)
|
| 129 |
+
if policy_match:
|
| 130 |
+
limit = int(policy_match.group("q"))
|
| 131 |
+
window_seconds = int(policy_match.group("w"))
|
| 132 |
+
|
| 133 |
+
return RateLimitInfo(
|
| 134 |
+
resource_type=resource_type,
|
| 135 |
+
remaining=remaining,
|
| 136 |
+
reset_in_seconds=reset_in_seconds,
|
| 137 |
+
limit=limit,
|
| 138 |
+
window_seconds=window_seconds,
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def _parse_retry_after(headers: Mapping[str, str]) -> int | None:
|
| 143 |
+
"""Parse the standard `Retry-After` HTTP header into a number of seconds to wait.
|
| 144 |
+
|
| 145 |
+
The `Retry-After` header can be either a non-negative number of seconds (delay-seconds)
|
| 146 |
+
or an HTTP-date after which to retry. We handle only the delay-seconds case.
|
| 147 |
+
|
| 148 |
+
See https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After.
|
| 149 |
+
"""
|
| 150 |
+
value: str | None = None
|
| 151 |
+
for key in headers:
|
| 152 |
+
if key.lower() == "retry-after":
|
| 153 |
+
value = headers[key]
|
| 154 |
+
break
|
| 155 |
+
|
| 156 |
+
if value is None:
|
| 157 |
+
return None
|
| 158 |
+
value = value.strip()
|
| 159 |
+
if not value:
|
| 160 |
+
return None
|
| 161 |
+
|
| 162 |
+
if value.isdigit():
|
| 163 |
+
return int(value) # e.g. "Retry-After: 120"
|
| 164 |
+
return None # e.g. "Retry-After: Wed, 21 Oct 2015 07:28:00 GMT" - not supported
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
# When raising an error, we include the request id in the error message for easier debugging.
|
| 168 |
+
# Request ID is sourced from headers in order of precedence: "X-Request-Id", "X-Amzn-Trace-Id", "X-Amz-Cf-Id".
|
| 169 |
+
X_REQUEST_ID = "x-request-id"
|
| 170 |
+
X_AMZN_TRACE_ID = "X-Amzn-Trace-Id"
|
| 171 |
+
X_AMZ_CF_ID = "x-amz-cf-id"
|
| 172 |
+
|
| 173 |
+
REPO_API_REGEX = re.compile(
|
| 174 |
+
r"""
|
| 175 |
+
# staging or production endpoint
|
| 176 |
+
^https://[^/]+
|
| 177 |
+
(
|
| 178 |
+
# on /api/repo_type/repo_id
|
| 179 |
+
/api/(models|datasets|spaces)/(.+)
|
| 180 |
+
|
|
| 181 |
+
# or /repo_id/resolve/revision/...
|
| 182 |
+
/(.+)/resolve/(.+)
|
| 183 |
+
)
|
| 184 |
+
""",
|
| 185 |
+
flags=re.VERBOSE,
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
BUCKET_API_REGEX = re.compile(
|
| 189 |
+
r"""
|
| 190 |
+
# staging or production endpoint
|
| 191 |
+
^https?://[^/]+
|
| 192 |
+
# on /api/buckets/...
|
| 193 |
+
/api/buckets/
|
| 194 |
+
""",
|
| 195 |
+
flags=re.VERBOSE,
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
# Regex to extract the job_id from a (scheduled) job API URL.
|
| 199 |
+
# Matches /api/jobs/{namespace}/{job_id}[/...] and /api/scheduled-jobs/{namespace}/{job_id}[/...].
|
| 200 |
+
_JOB_ID_FROM_URL_REGEX = re.compile(r"^https?://[^/]+/api/(?:scheduled-jobs|jobs)/[^/]+/([^/?]+)")
|
| 201 |
+
|
| 202 |
+
# Regex to extract repo_type and repo_id from API URLs.
|
| 203 |
+
# Captures: group(1) = repo_type plural (models/datasets/spaces), group(2) = first path segment, group(3) = optional second segment.
|
| 204 |
+
_REPO_ID_FROM_URL_REGEX = re.compile(r"^https?://[^/]+/api/(models|datasets|spaces)/([^/]+)(?:/([^/]+))?")
|
| 205 |
+
|
| 206 |
+
# Regex to extract bucket_id (namespace/name) from bucket API URLs.
|
| 207 |
+
_BUCKET_ID_FROM_URL_REGEX = re.compile(r"^https?://[^/]+/api/buckets/([^/]+/[^/]+)")
|
| 208 |
+
|
| 209 |
+
# Sub-paths that follow a repo_id in API URLs (not part of the repo name).
|
| 210 |
+
_REPO_URL_SUBPATHS = {"resolve", "tree", "blob", "raw", "refs", "commit", "discussions", "settings", "revision"}
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
def _parse_repo_info_from_url(url: str) -> tuple[str | None, str | None]:
|
| 214 |
+
"""Extract (repo_type, repo_id) from an API URL.
|
| 215 |
+
|
| 216 |
+
Returns canonical repo_type values: "model", "dataset", "space" (or None).
|
| 217 |
+
|
| 218 |
+
Examples:
|
| 219 |
+
>>> _parse_repo_info_from_url("https://huggingface.co/api/models/user/repo")
|
| 220 |
+
("model", "user/repo")
|
| 221 |
+
>>> _parse_repo_info_from_url("https://huggingface.co/api/datasets/user/repo/resolve/main/data.csv")
|
| 222 |
+
("dataset", "user/repo")
|
| 223 |
+
>>> _parse_repo_info_from_url("https://huggingface.co/api/models/bert-base-cased/resolve/main/config.json")
|
| 224 |
+
("model", "bert-base-cased")
|
| 225 |
+
"""
|
| 226 |
+
match = _REPO_ID_FROM_URL_REGEX.search(url)
|
| 227 |
+
if not match:
|
| 228 |
+
return None, None
|
| 229 |
+
repo_type = constants.REPO_TYPES_MAPPING.get(match.group(1))
|
| 230 |
+
first, second = match.group(2), match.group(3)
|
| 231 |
+
if second and second not in _REPO_URL_SUBPATHS:
|
| 232 |
+
repo_id = f"{first}/{second}"
|
| 233 |
+
else:
|
| 234 |
+
repo_id = first
|
| 235 |
+
return repo_type, repo_id
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def _parse_bucket_id_from_url(url: str) -> str | None:
|
| 239 |
+
"""Extract bucket_id (namespace/name) from a bucket API URL."""
|
| 240 |
+
match = _BUCKET_ID_FROM_URL_REGEX.search(url)
|
| 241 |
+
return match.group(1) if match else None
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def _parse_job_id_from_url(url: str) -> str | None:
|
| 245 |
+
"""Extract the job_id from a (scheduled) job API URL, if present."""
|
| 246 |
+
match = _JOB_ID_FROM_URL_REGEX.search(url)
|
| 247 |
+
return match.group(1) if match else None
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def hf_request_event_hook(request: httpx.Request) -> None:
|
| 251 |
+
"""
|
| 252 |
+
Event hook that will be used to make HTTP requests to the Hugging Face Hub.
|
| 253 |
+
|
| 254 |
+
What it does:
|
| 255 |
+
- Block requests if offline mode is enabled
|
| 256 |
+
- Add a request ID to the request headers
|
| 257 |
+
- Log the request if debug mode is enabled
|
| 258 |
+
"""
|
| 259 |
+
if constants.is_offline_mode():
|
| 260 |
+
raise OfflineModeIsEnabled(
|
| 261 |
+
f"Cannot reach {request.url}: offline mode is enabled. To disable it, please unset the `HF_HUB_OFFLINE` environment variable."
|
| 262 |
+
)
|
| 263 |
+
|
| 264 |
+
# Add random request ID => easier for server-side debugging
|
| 265 |
+
if X_AMZN_TRACE_ID not in request.headers:
|
| 266 |
+
request.headers[X_AMZN_TRACE_ID] = request.headers.get(X_REQUEST_ID) or str(uuid.uuid4())
|
| 267 |
+
request_id = request.headers.get(X_AMZN_TRACE_ID)
|
| 268 |
+
|
| 269 |
+
# Debug log
|
| 270 |
+
logger.debug(
|
| 271 |
+
"Request %s: %s %s (authenticated: %s)",
|
| 272 |
+
request_id,
|
| 273 |
+
request.method,
|
| 274 |
+
request.url,
|
| 275 |
+
request.headers.get("authorization") is not None,
|
| 276 |
+
)
|
| 277 |
+
if constants.HF_DEBUG:
|
| 278 |
+
logger.debug("Send: %s", _curlify(request))
|
| 279 |
+
|
| 280 |
+
return request_id
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
async def async_hf_request_event_hook(request: httpx.Request) -> None:
|
| 284 |
+
"""
|
| 285 |
+
Async version of `hf_request_event_hook`.
|
| 286 |
+
"""
|
| 287 |
+
return hf_request_event_hook(request)
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
async def async_hf_response_event_hook(response: httpx.Response) -> None:
|
| 291 |
+
if response.status_code >= 400:
|
| 292 |
+
# If response will raise, read content from stream to have it available when raising the exception
|
| 293 |
+
# If content-length is not set or is too large, skip reading the content to avoid OOM
|
| 294 |
+
if "Content-length" in response.headers:
|
| 295 |
+
try:
|
| 296 |
+
length = int(response.headers["Content-length"])
|
| 297 |
+
except ValueError:
|
| 298 |
+
return
|
| 299 |
+
|
| 300 |
+
if length < 1_000_000:
|
| 301 |
+
await response.aread()
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def default_client_factory() -> httpx.Client:
|
| 305 |
+
"""
|
| 306 |
+
Factory function to create a `httpx.Client` with the default transport.
|
| 307 |
+
"""
|
| 308 |
+
return httpx.Client(
|
| 309 |
+
event_hooks={"request": [hf_request_event_hook]},
|
| 310 |
+
follow_redirects=True,
|
| 311 |
+
timeout=None,
|
| 312 |
+
)
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
def default_async_client_factory() -> httpx.AsyncClient:
|
| 316 |
+
"""
|
| 317 |
+
Factory function to create a `httpx.AsyncClient` with the default transport.
|
| 318 |
+
"""
|
| 319 |
+
return httpx.AsyncClient(
|
| 320 |
+
event_hooks={"request": [async_hf_request_event_hook], "response": [async_hf_response_event_hook]},
|
| 321 |
+
follow_redirects=True,
|
| 322 |
+
timeout=None,
|
| 323 |
+
)
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
CLIENT_FACTORY_T = Callable[[], httpx.Client]
|
| 327 |
+
ASYNC_CLIENT_FACTORY_T = Callable[[], httpx.AsyncClient]
|
| 328 |
+
|
| 329 |
+
_CLIENT_LOCK = threading.Lock()
|
| 330 |
+
_GLOBAL_CLIENT_FACTORY: CLIENT_FACTORY_T = default_client_factory
|
| 331 |
+
_GLOBAL_ASYNC_CLIENT_FACTORY: ASYNC_CLIENT_FACTORY_T = default_async_client_factory
|
| 332 |
+
_GLOBAL_CLIENT: httpx.Client | None = None
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def set_client_factory(client_factory: CLIENT_FACTORY_T) -> None:
|
| 336 |
+
"""
|
| 337 |
+
Set the HTTP client factory to be used by `huggingface_hub`.
|
| 338 |
+
|
| 339 |
+
The client factory is a method that returns a `httpx.Client` object. On the first call to [`get_session`] the client factory
|
| 340 |
+
will be used to create a new `httpx.Client` object that will be shared between all calls made by `huggingface_hub`.
|
| 341 |
+
|
| 342 |
+
This can be useful if you are running your scripts in a specific environment requiring custom configuration (e.g. custom proxy or certifications).
|
| 343 |
+
|
| 344 |
+
Use [`get_session`] to get a correctly configured `httpx.Client`.
|
| 345 |
+
"""
|
| 346 |
+
global _GLOBAL_CLIENT_FACTORY
|
| 347 |
+
with _CLIENT_LOCK:
|
| 348 |
+
close_session()
|
| 349 |
+
_GLOBAL_CLIENT_FACTORY = client_factory
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def set_async_client_factory(async_client_factory: ASYNC_CLIENT_FACTORY_T) -> None:
|
| 353 |
+
"""
|
| 354 |
+
Set the HTTP async client factory to be used by `huggingface_hub`.
|
| 355 |
+
|
| 356 |
+
The async client factory is a method that returns a `httpx.AsyncClient` object.
|
| 357 |
+
This can be useful if you are running your scripts in a specific environment requiring custom configuration (e.g. custom proxy or certifications).
|
| 358 |
+
Use [`get_async_client`] to get a correctly configured `httpx.AsyncClient`.
|
| 359 |
+
|
| 360 |
+
> [!WARNING]
|
| 361 |
+
> Contrary to the `httpx.Client` that is shared between all calls made by `huggingface_hub`, the `httpx.AsyncClient` is not shared.
|
| 362 |
+
> It is recommended to use an async context manager to ensure the client is properly closed when the context is exited.
|
| 363 |
+
"""
|
| 364 |
+
global _GLOBAL_ASYNC_CLIENT_FACTORY
|
| 365 |
+
_GLOBAL_ASYNC_CLIENT_FACTORY = async_client_factory
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
def get_session() -> httpx.Client:
|
| 369 |
+
"""
|
| 370 |
+
Get a `httpx.Client` object, using the transport factory from the user.
|
| 371 |
+
|
| 372 |
+
This client is shared between all calls made by `huggingface_hub`. Therefore you should not close it manually.
|
| 373 |
+
|
| 374 |
+
Use [`set_client_factory`] to customize the `httpx.Client`.
|
| 375 |
+
"""
|
| 376 |
+
global _GLOBAL_CLIENT
|
| 377 |
+
if _GLOBAL_CLIENT is None:
|
| 378 |
+
with _CLIENT_LOCK:
|
| 379 |
+
_GLOBAL_CLIENT = _GLOBAL_CLIENT_FACTORY()
|
| 380 |
+
return _GLOBAL_CLIENT
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def get_async_session() -> httpx.AsyncClient:
|
| 384 |
+
"""
|
| 385 |
+
Return a `httpx.AsyncClient` object, using the transport factory from the user.
|
| 386 |
+
|
| 387 |
+
Use [`set_async_client_factory`] to customize the `httpx.AsyncClient`.
|
| 388 |
+
|
| 389 |
+
> [!WARNING]
|
| 390 |
+
> Contrary to the `httpx.Client` that is shared between all calls made by `huggingface_hub`, the `httpx.AsyncClient` is not shared.
|
| 391 |
+
> It is recommended to use an async context manager to ensure the client is properly closed when the context is exited.
|
| 392 |
+
"""
|
| 393 |
+
return _GLOBAL_ASYNC_CLIENT_FACTORY()
|
| 394 |
+
|
| 395 |
+
|
| 396 |
+
def close_session() -> None:
|
| 397 |
+
"""
|
| 398 |
+
Close the global `httpx.Client` used by `huggingface_hub`.
|
| 399 |
+
|
| 400 |
+
If a Client is closed, it will be recreated on the next call to [`get_session`].
|
| 401 |
+
|
| 402 |
+
Can be useful if e.g. an SSL certificate has been updated.
|
| 403 |
+
"""
|
| 404 |
+
global _GLOBAL_CLIENT
|
| 405 |
+
client = _GLOBAL_CLIENT
|
| 406 |
+
|
| 407 |
+
# First, set global client to None
|
| 408 |
+
_GLOBAL_CLIENT = None
|
| 409 |
+
|
| 410 |
+
# Then, close the clients
|
| 411 |
+
if client is not None:
|
| 412 |
+
try:
|
| 413 |
+
client.close()
|
| 414 |
+
except Exception as e:
|
| 415 |
+
logger.warning(f"Error closing client: {e}")
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
atexit.register(close_session)
|
| 419 |
+
if hasattr(os, "register_at_fork"):
|
| 420 |
+
os.register_at_fork(after_in_child=close_session)
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
_DEFAULT_RETRY_ON_EXCEPTIONS: tuple[type[Exception], ...] = (
|
| 424 |
+
httpx.TimeoutException,
|
| 425 |
+
httpx.NetworkError,
|
| 426 |
+
httpx.RemoteProtocolError,
|
| 427 |
+
)
|
| 428 |
+
_DEFAULT_RETRY_ON_STATUS_CODES: tuple[int, ...] = (408, 429, 500, 502, 503, 504)
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def _http_backoff_base(
|
| 432 |
+
method: HTTP_METHOD_T,
|
| 433 |
+
url: str,
|
| 434 |
+
*,
|
| 435 |
+
max_retries: int = 5,
|
| 436 |
+
base_wait_time: float = 1,
|
| 437 |
+
max_wait_time: float = 8,
|
| 438 |
+
retry_on_exceptions: type[Exception] | tuple[type[Exception], ...] = _DEFAULT_RETRY_ON_EXCEPTIONS,
|
| 439 |
+
retry_on_status_codes: int | tuple[int, ...] = _DEFAULT_RETRY_ON_STATUS_CODES,
|
| 440 |
+
stream: bool = False,
|
| 441 |
+
**kwargs,
|
| 442 |
+
) -> Generator[httpx.Response, None, None]:
|
| 443 |
+
"""Internal implementation of HTTP backoff logic shared between `http_backoff` and `http_stream_backoff`."""
|
| 444 |
+
if isinstance(retry_on_exceptions, type): # Tuple from single exception type
|
| 445 |
+
retry_on_exceptions = (retry_on_exceptions,)
|
| 446 |
+
|
| 447 |
+
if isinstance(retry_on_status_codes, int): # Tuple from single status code
|
| 448 |
+
retry_on_status_codes = (retry_on_status_codes,)
|
| 449 |
+
|
| 450 |
+
nb_tries = 0
|
| 451 |
+
sleep_time = base_wait_time
|
| 452 |
+
ratelimit_reset: int | None = None # seconds to wait for rate limit reset if 429 response
|
| 453 |
+
|
| 454 |
+
# If `data` is used and is a file object (or any IO), it will be consumed on the
|
| 455 |
+
# first HTTP request. We need to save the initial position so that the full content
|
| 456 |
+
# of the file is re-sent on http backoff. See warning tip in docstring.
|
| 457 |
+
io_obj_initial_pos = None
|
| 458 |
+
if "data" in kwargs and isinstance(kwargs["data"], (io.IOBase, SliceFileObj)):
|
| 459 |
+
io_obj_initial_pos = kwargs["data"].tell()
|
| 460 |
+
|
| 461 |
+
while True:
|
| 462 |
+
nb_tries += 1
|
| 463 |
+
ratelimit_reset = None
|
| 464 |
+
# Fetched on each attempt: a previous attempt may have closed the shared client (see `close_session` below),
|
| 465 |
+
# and closed `httpx.Client` objects cannot be reused.
|
| 466 |
+
client = get_session()
|
| 467 |
+
try:
|
| 468 |
+
# If `data` is used and is a file object (or any IO), set back cursor to
|
| 469 |
+
# initial position.
|
| 470 |
+
if io_obj_initial_pos is not None:
|
| 471 |
+
kwargs["data"].seek(io_obj_initial_pos)
|
| 472 |
+
|
| 473 |
+
# Perform request and handle response
|
| 474 |
+
def _should_retry(response: httpx.Response) -> bool:
|
| 475 |
+
"""Handle response and return True if should retry, False if should return/yield."""
|
| 476 |
+
nonlocal ratelimit_reset
|
| 477 |
+
|
| 478 |
+
if response.status_code not in retry_on_status_codes:
|
| 479 |
+
return False # Success, don't retry
|
| 480 |
+
|
| 481 |
+
# Wrong status code returned (HTTP 503 for instance)
|
| 482 |
+
logger.warning(f"HTTP Error {response.status_code} thrown while requesting {method} {url}")
|
| 483 |
+
if nb_tries > max_retries:
|
| 484 |
+
hf_raise_for_status(response) # Will raise uncaught exception
|
| 485 |
+
# Return/yield response to avoid infinite loop in the corner case where the
|
| 486 |
+
# user ask for retry on a status code that doesn't raise_for_status.
|
| 487 |
+
return False # Don't retry, return/yield response
|
| 488 |
+
|
| 489 |
+
# Check 'ratelimit' and `Retry-After` headers.
|
| 490 |
+
if (
|
| 491 |
+
response.status_code == 429
|
| 492 |
+
and (ratelimit_info := parse_ratelimit_headers(response.headers)) is not None
|
| 493 |
+
):
|
| 494 |
+
ratelimit_reset = ratelimit_info.reset_in_seconds
|
| 495 |
+
elif (retry_after := _parse_retry_after(response.headers)) is not None:
|
| 496 |
+
ratelimit_reset = retry_after
|
| 497 |
+
|
| 498 |
+
return True # Should retry
|
| 499 |
+
|
| 500 |
+
if stream:
|
| 501 |
+
with client.stream(method=method, url=url, **kwargs) as response:
|
| 502 |
+
if not _should_retry(response):
|
| 503 |
+
yield response
|
| 504 |
+
return
|
| 505 |
+
else:
|
| 506 |
+
response = client.request(method=method, url=url, **kwargs)
|
| 507 |
+
if not _should_retry(response):
|
| 508 |
+
yield response
|
| 509 |
+
return
|
| 510 |
+
|
| 511 |
+
except retry_on_exceptions as err:
|
| 512 |
+
logger.warning(f"'{err}' thrown while requesting {method} {url}")
|
| 513 |
+
|
| 514 |
+
if isinstance(err, httpx.ConnectError):
|
| 515 |
+
close_session() # In case of SSLError it's best to close the shared httpx.Client objects
|
| 516 |
+
|
| 517 |
+
if nb_tries > max_retries:
|
| 518 |
+
raise err
|
| 519 |
+
|
| 520 |
+
if ratelimit_reset is not None:
|
| 521 |
+
actual_sleep = float(ratelimit_reset) + 1 # +1s to avoid rounding issues
|
| 522 |
+
logger.warning(f"Rate limited. Waiting {actual_sleep}s before retry [Retry {nb_tries}/{max_retries}].")
|
| 523 |
+
else:
|
| 524 |
+
actual_sleep = sleep_time
|
| 525 |
+
logger.warning(f"Retrying in {actual_sleep}s [Retry {nb_tries}/{max_retries}].")
|
| 526 |
+
|
| 527 |
+
time.sleep(actual_sleep)
|
| 528 |
+
|
| 529 |
+
# Update sleep time for next retry
|
| 530 |
+
sleep_time = min(max_wait_time, sleep_time * 2) # Exponential backoff
|
| 531 |
+
|
| 532 |
+
|
| 533 |
+
def http_backoff(
|
| 534 |
+
method: HTTP_METHOD_T,
|
| 535 |
+
url: str,
|
| 536 |
+
*,
|
| 537 |
+
max_retries: int = 5,
|
| 538 |
+
base_wait_time: float = 1,
|
| 539 |
+
max_wait_time: float = 8,
|
| 540 |
+
retry_on_exceptions: type[Exception] | tuple[type[Exception], ...] = _DEFAULT_RETRY_ON_EXCEPTIONS,
|
| 541 |
+
retry_on_status_codes: int | tuple[int, ...] = _DEFAULT_RETRY_ON_STATUS_CODES,
|
| 542 |
+
**kwargs,
|
| 543 |
+
) -> httpx.Response:
|
| 544 |
+
"""Wrapper around httpx to retry calls on an endpoint, with exponential backoff.
|
| 545 |
+
|
| 546 |
+
Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...)
|
| 547 |
+
and/or on specific status codes (ex: service unavailable). If the call failed more
|
| 548 |
+
than `max_retries`, the exception is thrown or `raise_for_status` is called on the
|
| 549 |
+
response object.
|
| 550 |
+
|
| 551 |
+
Re-implement mechanisms from the `backoff` library to avoid adding an external
|
| 552 |
+
dependencies to `hugging_face_hub`. See https://github.com/litl/backoff.
|
| 553 |
+
|
| 554 |
+
Args:
|
| 555 |
+
method (`Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]`):
|
| 556 |
+
HTTP method to perform.
|
| 557 |
+
url (`str`):
|
| 558 |
+
The URL of the resource to fetch.
|
| 559 |
+
max_retries (`int`, *optional*, defaults to `5`):
|
| 560 |
+
Maximum number of retries, defaults to 5. Set to `0` to disable retries.
|
| 561 |
+
base_wait_time (`float`, *optional*, defaults to `1`):
|
| 562 |
+
Duration (in seconds) to wait before retrying the first time.
|
| 563 |
+
Wait time between retries then grows exponentially, capped by
|
| 564 |
+
`max_wait_time`.
|
| 565 |
+
max_wait_time (`float`, *optional*, defaults to `8`):
|
| 566 |
+
Maximum duration (in seconds) to wait before retrying.
|
| 567 |
+
retry_on_exceptions (`type[Exception]` or `tuple[type[Exception]]`, *optional*):
|
| 568 |
+
Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types.
|
| 569 |
+
By default, retry on `httpx.TimeoutException`, `httpx.NetworkError` and `httpx.RemoteProtocolError`.
|
| 570 |
+
retry_on_status_codes (`int` or `tuple[int]`, *optional*, defaults to `(429, 500, 502, 503, 504)`):
|
| 571 |
+
Define on which status codes the request must be retried. By default, retries
|
| 572 |
+
on rate limit (429) and server errors (5xx).
|
| 573 |
+
**kwargs (`dict`, *optional*):
|
| 574 |
+
kwargs to pass to `httpx.request`.
|
| 575 |
+
|
| 576 |
+
Example:
|
| 577 |
+
```
|
| 578 |
+
>>> from huggingface_hub.utils import http_backoff
|
| 579 |
+
|
| 580 |
+
# Same usage as "httpx.request".
|
| 581 |
+
>>> response = http_backoff("GET", "https://www.google.com")
|
| 582 |
+
>>> response.raise_for_status()
|
| 583 |
+
|
| 584 |
+
# If you expect a Gateway Timeout from time to time
|
| 585 |
+
>>> http_backoff("PUT", upload_url, data=data, retry_on_status_codes=504)
|
| 586 |
+
>>> response.raise_for_status()
|
| 587 |
+
```
|
| 588 |
+
|
| 589 |
+
> [!WARNING]
|
| 590 |
+
> When using `requests` it is possible to stream data by passing an iterator to the
|
| 591 |
+
> `data` argument. On http backoff this is a problem as the iterator is not reset
|
| 592 |
+
> after a failed call. This issue is mitigated for file objects or any IO streams
|
| 593 |
+
> by saving the initial position of the cursor (with `data.tell()`) and resetting the
|
| 594 |
+
> cursor between each call (with `data.seek()`). For arbitrary iterators, http backoff
|
| 595 |
+
> will fail. If this is a hard constraint for you, please let us know by opening an
|
| 596 |
+
> issue on [Github](https://github.com/huggingface/huggingface_hub).
|
| 597 |
+
"""
|
| 598 |
+
return next(
|
| 599 |
+
_http_backoff_base(
|
| 600 |
+
method=method,
|
| 601 |
+
url=url,
|
| 602 |
+
max_retries=max_retries,
|
| 603 |
+
base_wait_time=base_wait_time,
|
| 604 |
+
max_wait_time=max_wait_time,
|
| 605 |
+
retry_on_exceptions=retry_on_exceptions,
|
| 606 |
+
retry_on_status_codes=retry_on_status_codes,
|
| 607 |
+
stream=False,
|
| 608 |
+
**kwargs,
|
| 609 |
+
)
|
| 610 |
+
)
|
| 611 |
+
|
| 612 |
+
|
| 613 |
+
@contextmanager
|
| 614 |
+
def http_stream_backoff(
|
| 615 |
+
method: HTTP_METHOD_T,
|
| 616 |
+
url: str,
|
| 617 |
+
*,
|
| 618 |
+
max_retries: int = 5,
|
| 619 |
+
base_wait_time: float = 1,
|
| 620 |
+
max_wait_time: float = 8,
|
| 621 |
+
retry_on_exceptions: type[Exception] | tuple[type[Exception], ...] = _DEFAULT_RETRY_ON_EXCEPTIONS,
|
| 622 |
+
retry_on_status_codes: int | tuple[int, ...] = _DEFAULT_RETRY_ON_STATUS_CODES,
|
| 623 |
+
**kwargs,
|
| 624 |
+
) -> Generator[httpx.Response, None, None]:
|
| 625 |
+
"""Wrapper around httpx to retry calls on an endpoint, with exponential backoff.
|
| 626 |
+
|
| 627 |
+
Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...)
|
| 628 |
+
and/or on specific status codes (ex: service unavailable). If the call failed more
|
| 629 |
+
than `max_retries`, the exception is thrown or `raise_for_status` is called on the
|
| 630 |
+
response object.
|
| 631 |
+
|
| 632 |
+
Re-implement mechanisms from the `backoff` library to avoid adding an external
|
| 633 |
+
dependencies to `hugging_face_hub`. See https://github.com/litl/backoff.
|
| 634 |
+
|
| 635 |
+
Args:
|
| 636 |
+
method (`Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]`):
|
| 637 |
+
HTTP method to perform.
|
| 638 |
+
url (`str`):
|
| 639 |
+
The URL of the resource to fetch.
|
| 640 |
+
max_retries (`int`, *optional*, defaults to `5`):
|
| 641 |
+
Maximum number of retries, defaults to 5. Set to `0` to disable retries.
|
| 642 |
+
base_wait_time (`float`, *optional*, defaults to `1`):
|
| 643 |
+
Duration (in seconds) to wait before retrying the first time.
|
| 644 |
+
Wait time between retries then grows exponentially, capped by
|
| 645 |
+
`max_wait_time`.
|
| 646 |
+
max_wait_time (`float`, *optional*, defaults to `8`):
|
| 647 |
+
Maximum duration (in seconds) to wait before retrying.
|
| 648 |
+
retry_on_exceptions (`type[Exception]` or `tuple[type[Exception]]`, *optional*):
|
| 649 |
+
Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types.
|
| 650 |
+
By default, retry on `httpx.TimeoutException`, `httpx.NetworkError` and `httpx.RemoteProtocolError`.
|
| 651 |
+
retry_on_status_codes (`int` or `tuple[int]`, *optional*, defaults to `(429, 500, 502, 503, 504)`):
|
| 652 |
+
Define on which status codes the request must be retried. By default, retries
|
| 653 |
+
on rate limit (429) and server errors (5xx).
|
| 654 |
+
**kwargs (`dict`, *optional*):
|
| 655 |
+
kwargs to pass to `httpx.request`.
|
| 656 |
+
|
| 657 |
+
Example:
|
| 658 |
+
```
|
| 659 |
+
>>> from huggingface_hub.utils import http_stream_backoff
|
| 660 |
+
|
| 661 |
+
# Same usage as "httpx.stream".
|
| 662 |
+
>>> with http_stream_backoff("GET", "https://www.google.com") as response:
|
| 663 |
+
... for chunk in response.iter_bytes():
|
| 664 |
+
... print(chunk)
|
| 665 |
+
|
| 666 |
+
# If you expect a Gateway Timeout from time to time
|
| 667 |
+
>>> with http_stream_backoff("PUT", upload_url, data=data, retry_on_status_codes=504) as response:
|
| 668 |
+
... response.raise_for_status()
|
| 669 |
+
```
|
| 670 |
+
|
| 671 |
+
> [!WARNING]
|
| 672 |
+
> When using `httpx` it is possible to stream data by passing an iterator to the
|
| 673 |
+
> `data` argument. On http backoff this is a problem as the iterator is not reset
|
| 674 |
+
> after a failed call. This issue is mitigated for file objects or any IO streams
|
| 675 |
+
> by saving the initial position of the cursor (with `data.tell()`) and resetting the
|
| 676 |
+
> cursor between each call (with `data.seek()`). For arbitrary iterators, http backoff
|
| 677 |
+
> will fail. If this is a hard constraint for you, please let us know by opening an
|
| 678 |
+
> issue on [Github](https://github.com/huggingface/huggingface_hub).
|
| 679 |
+
"""
|
| 680 |
+
yield from _http_backoff_base(
|
| 681 |
+
method=method,
|
| 682 |
+
url=url,
|
| 683 |
+
max_retries=max_retries,
|
| 684 |
+
base_wait_time=base_wait_time,
|
| 685 |
+
max_wait_time=max_wait_time,
|
| 686 |
+
retry_on_exceptions=retry_on_exceptions,
|
| 687 |
+
retry_on_status_codes=retry_on_status_codes,
|
| 688 |
+
stream=True,
|
| 689 |
+
**kwargs,
|
| 690 |
+
)
|
| 691 |
+
|
| 692 |
+
|
| 693 |
+
def _httpx_follow_relative_redirects_with_backoff(
|
| 694 |
+
method: HTTP_METHOD_T, url: str, *, retry_on_errors: bool = False, **httpx_kwargs
|
| 695 |
+
) -> httpx.Response:
|
| 696 |
+
"""Perform an HTTP request with backoff and follow relative redirects only.
|
| 697 |
+
|
| 698 |
+
Used to fetch HEAD /resolve on repo or bucket files.
|
| 699 |
+
|
| 700 |
+
This is useful to follow a redirection to a renamed repository without following redirection to a CDN.
|
| 701 |
+
|
| 702 |
+
A backoff mechanism retries the HTTP call on errors (429, 5xx, timeout, network errors).
|
| 703 |
+
|
| 704 |
+
Args:
|
| 705 |
+
method (`str`):
|
| 706 |
+
HTTP method, such as 'GET' or 'HEAD'.
|
| 707 |
+
url (`str`):
|
| 708 |
+
The URL of the resource to fetch.
|
| 709 |
+
retry_on_errors (`bool`, *optional*, defaults to `False`):
|
| 710 |
+
Whether to retry on errors. If False, no retry is performed (fast fallback to local cache).
|
| 711 |
+
If True, uses default retry behavior (429, 5xx, timeout, network errors).
|
| 712 |
+
**httpx_kwargs (`dict`, *optional*):
|
| 713 |
+
Params to pass to `httpx.request`.
|
| 714 |
+
"""
|
| 715 |
+
# if `retry_on_errors=False`, disable all retries for fast fallback to cache
|
| 716 |
+
no_retry_kwargs: dict[str, Any] = (
|
| 717 |
+
{} if retry_on_errors else {"retry_on_exceptions": (), "retry_on_status_codes": ()}
|
| 718 |
+
)
|
| 719 |
+
|
| 720 |
+
while True:
|
| 721 |
+
response = http_backoff(
|
| 722 |
+
method=method,
|
| 723 |
+
url=url,
|
| 724 |
+
**httpx_kwargs,
|
| 725 |
+
follow_redirects=False,
|
| 726 |
+
**no_retry_kwargs,
|
| 727 |
+
)
|
| 728 |
+
hf_raise_for_status(response)
|
| 729 |
+
|
| 730 |
+
# Check if response is a relative redirect
|
| 731 |
+
if 300 <= response.status_code <= 399:
|
| 732 |
+
parsed_target = urlparse(response.headers["Location"])
|
| 733 |
+
if parsed_target.netloc == "":
|
| 734 |
+
# Relative redirect -> update URL and retry
|
| 735 |
+
url = urlparse(url)._replace(path=parsed_target.path).geturl()
|
| 736 |
+
continue
|
| 737 |
+
|
| 738 |
+
# Break if no relative redirect
|
| 739 |
+
break
|
| 740 |
+
|
| 741 |
+
return response
|
| 742 |
+
|
| 743 |
+
|
| 744 |
+
def fix_hf_endpoint_in_url(url: str, endpoint: str | None) -> str:
|
| 745 |
+
"""Replace the default endpoint in a URL by a custom one.
|
| 746 |
+
|
| 747 |
+
This is useful when using a proxy and the Hugging Face Hub returns a URL with the default endpoint.
|
| 748 |
+
"""
|
| 749 |
+
endpoint = endpoint.rstrip("/") if endpoint else constants.ENDPOINT
|
| 750 |
+
# check if a proxy has been set => if yes, update the returned URL to use the proxy
|
| 751 |
+
if endpoint not in (constants._HF_DEFAULT_ENDPOINT, constants._HF_DEFAULT_STAGING_ENDPOINT):
|
| 752 |
+
url = url.replace(constants._HF_DEFAULT_ENDPOINT, endpoint)
|
| 753 |
+
url = url.replace(constants._HF_DEFAULT_STAGING_ENDPOINT, endpoint)
|
| 754 |
+
return url
|
| 755 |
+
|
| 756 |
+
|
| 757 |
+
def hf_raise_for_status(response: httpx.Response, endpoint_name: str | None = None) -> None:
|
| 758 |
+
"""
|
| 759 |
+
Internal version of `response.raise_for_status()` that will refine a potential HTTPError.
|
| 760 |
+
Raised exception will be an instance of [`~errors.HfHubHTTPError`].
|
| 761 |
+
|
| 762 |
+
This helper is meant to be the unique method to raise_for_status when making a call to the Hugging Face Hub.
|
| 763 |
+
|
| 764 |
+
Args:
|
| 765 |
+
response (`Response`):
|
| 766 |
+
Response from the server.
|
| 767 |
+
endpoint_name (`str`, *optional*):
|
| 768 |
+
Name of the endpoint that has been called. If provided, the error message will be more complete.
|
| 769 |
+
|
| 770 |
+
> [!WARNING]
|
| 771 |
+
> Raises when the request has failed:
|
| 772 |
+
>
|
| 773 |
+
> - [`~utils.RepositoryNotFoundError`]
|
| 774 |
+
> If the repository to download from cannot be found. This may be because it
|
| 775 |
+
> doesn't exist, because `repo_type` is not set correctly, or because the repo
|
| 776 |
+
> is `private` and you do not have access.
|
| 777 |
+
> - [`~utils.GatedRepoError`]
|
| 778 |
+
> If the repository exists but is gated and the user is not on the authorized
|
| 779 |
+
> list.
|
| 780 |
+
> - [`~utils.RevisionNotFoundError`]
|
| 781 |
+
> If the repository exists but the revision couldn't be found.
|
| 782 |
+
> - [`~utils.EntryNotFoundError`]
|
| 783 |
+
> If the repository exists but the entry (e.g. the requested file) couldn't be
|
| 784 |
+
> find.
|
| 785 |
+
> - [`~utils.BadRequestError`]
|
| 786 |
+
> If request failed with a HTTP 400 BadRequest error.
|
| 787 |
+
> - [`~utils.HfHubHTTPError`]
|
| 788 |
+
> If request failed for a reason not listed above.
|
| 789 |
+
"""
|
| 790 |
+
try:
|
| 791 |
+
_warn_on_warning_headers(response)
|
| 792 |
+
except Exception:
|
| 793 |
+
# Never raise on warning parsing
|
| 794 |
+
logger.debug("Failed to parse warning headers", exc_info=True)
|
| 795 |
+
|
| 796 |
+
try:
|
| 797 |
+
response.raise_for_status()
|
| 798 |
+
except httpx.HTTPStatusError as e:
|
| 799 |
+
if response.status_code // 100 == 3:
|
| 800 |
+
return # Do not raise on redirects to stay consistent with `requests`
|
| 801 |
+
|
| 802 |
+
error_code = response.headers.get("X-Error-Code")
|
| 803 |
+
error_message = response.headers.get("X-Error-Message")
|
| 804 |
+
|
| 805 |
+
# Parse repo info from request URL (used to enrich errors below)
|
| 806 |
+
request_url = (
|
| 807 |
+
str(response.request.url) if response.request is not None and response.request.url is not None else None
|
| 808 |
+
)
|
| 809 |
+
repo_type, repo_id = _parse_repo_info_from_url(request_url) if request_url else (None, None)
|
| 810 |
+
|
| 811 |
+
if error_code == "RevisionNotFound":
|
| 812 |
+
message = f"{response.status_code} Client Error." + "\n\n" + f"Revision Not Found for url: {response.url}."
|
| 813 |
+
raise _format(RevisionNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e
|
| 814 |
+
|
| 815 |
+
elif error_code == "EntryNotFound":
|
| 816 |
+
message = f"{response.status_code} Client Error." + "\n\n" + f"Entry Not Found for url: {response.url}."
|
| 817 |
+
raise _format(RemoteEntryNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e
|
| 818 |
+
|
| 819 |
+
elif error_code == "GatedRepo":
|
| 820 |
+
message = (
|
| 821 |
+
f"{response.status_code} Client Error." + "\n\n" + f"Cannot access gated repo for url {response.url}."
|
| 822 |
+
)
|
| 823 |
+
raise _format(GatedRepoError, message, response, repo_type=repo_type, repo_id=repo_id) from e
|
| 824 |
+
|
| 825 |
+
elif error_message == "Access to this resource is disabled.":
|
| 826 |
+
message = (
|
| 827 |
+
f"{response.status_code} Client Error."
|
| 828 |
+
+ "\n\n"
|
| 829 |
+
+ f"Cannot access repository for url {response.url}."
|
| 830 |
+
+ "\n"
|
| 831 |
+
+ "Access to this resource is disabled."
|
| 832 |
+
)
|
| 833 |
+
raise _format(DisabledRepoError, message, response) from e
|
| 834 |
+
|
| 835 |
+
elif (
|
| 836 |
+
error_code == "RepoNotFound"
|
| 837 |
+
and request_url is not None
|
| 838 |
+
and BUCKET_API_REGEX.search(request_url) is not None
|
| 839 |
+
):
|
| 840 |
+
message = (
|
| 841 |
+
f"{response.status_code} Client Error."
|
| 842 |
+
+ "\n\n"
|
| 843 |
+
+ f"Bucket Not Found for url: {response.url}."
|
| 844 |
+
+ "\nPlease make sure you specified the correct bucket id (namespace/name)."
|
| 845 |
+
+ "\nIf the bucket is private, make sure you are authenticated and your token has the required permissions."
|
| 846 |
+
)
|
| 847 |
+
raise _format(
|
| 848 |
+
BucketNotFoundError, message, response, bucket_id=_parse_bucket_id_from_url(request_url)
|
| 849 |
+
) from e
|
| 850 |
+
|
| 851 |
+
elif (
|
| 852 |
+
response.status_code == 404
|
| 853 |
+
and request_url is not None
|
| 854 |
+
and (job_id := _parse_job_id_from_url(request_url)) is not None
|
| 855 |
+
):
|
| 856 |
+
message = (
|
| 857 |
+
f"{response.status_code} Client Error."
|
| 858 |
+
+ "\n\n"
|
| 859 |
+
+ f"Job Not Found for url: {response.url}."
|
| 860 |
+
+ "\nPlease make sure you specified the correct job ID and namespace."
|
| 861 |
+
)
|
| 862 |
+
raise _format(JobNotFoundError, message, response, job_id=job_id) from e
|
| 863 |
+
|
| 864 |
+
elif error_code == "RepoNotFound" or (
|
| 865 |
+
response.status_code == 401
|
| 866 |
+
and error_message != "Invalid credentials in Authorization header"
|
| 867 |
+
and request_url is not None
|
| 868 |
+
and REPO_API_REGEX.search(request_url) is not None
|
| 869 |
+
):
|
| 870 |
+
# 401 is misleading as it is returned for:
|
| 871 |
+
# - private and gated repos if user is not authenticated
|
| 872 |
+
# - missing repos
|
| 873 |
+
# => for now, we process them as `RepoNotFound` anyway.
|
| 874 |
+
# See https://gist.github.com/Wauplin/46c27ad266b15998ce56a6603796f0b9
|
| 875 |
+
message = (
|
| 876 |
+
f"{response.status_code} Client Error."
|
| 877 |
+
+ "\n\n"
|
| 878 |
+
+ f"Repository Not Found for url: {response.url}."
|
| 879 |
+
+ "\nPlease make sure you specified the correct `repo_id` and"
|
| 880 |
+
" `repo_type`.\nIf you are trying to access a private or gated repo,"
|
| 881 |
+
" make sure you are authenticated and your token has the required permissions."
|
| 882 |
+
+ "\nFor more details, see https://huggingface.co/docs/huggingface_hub/authentication"
|
| 883 |
+
)
|
| 884 |
+
raise _format(RepositoryNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e
|
| 885 |
+
|
| 886 |
+
elif response.status_code == 400:
|
| 887 |
+
message = (
|
| 888 |
+
f"\n\nBad request for {endpoint_name} endpoint:" if endpoint_name is not None else "\n\nBad request:"
|
| 889 |
+
)
|
| 890 |
+
raise _format(BadRequestError, message, response) from e
|
| 891 |
+
|
| 892 |
+
elif response.status_code == 403:
|
| 893 |
+
message = (
|
| 894 |
+
f"\n\n{response.status_code} Forbidden: {error_message}."
|
| 895 |
+
+ f"\nCannot access content at: {response.url}."
|
| 896 |
+
+ "\nMake sure your token has the correct permissions."
|
| 897 |
+
)
|
| 898 |
+
raise _format(HfHubHTTPError, message, response) from e
|
| 899 |
+
|
| 900 |
+
elif response.status_code == 429:
|
| 901 |
+
ratelimit_info = parse_ratelimit_headers(response.headers)
|
| 902 |
+
if ratelimit_info is not None:
|
| 903 |
+
message = (
|
| 904 |
+
f"\n\n429 Too Many Requests: you have reached your '{ratelimit_info.resource_type}' rate limit."
|
| 905 |
+
)
|
| 906 |
+
message += f"\nRetry after {ratelimit_info.reset_in_seconds} seconds"
|
| 907 |
+
if ratelimit_info.limit is not None and ratelimit_info.window_seconds is not None:
|
| 908 |
+
message += (
|
| 909 |
+
f" ({ratelimit_info.remaining}/{ratelimit_info.limit} requests remaining"
|
| 910 |
+
f" in current {ratelimit_info.window_seconds}s window)."
|
| 911 |
+
)
|
| 912 |
+
else:
|
| 913 |
+
message += "."
|
| 914 |
+
message += f"\nUrl: {response.url}."
|
| 915 |
+
else:
|
| 916 |
+
message = f"\n\n429 Too Many Requests for url: {response.url}."
|
| 917 |
+
raise _format(HfHubHTTPError, message, response) from e
|
| 918 |
+
|
| 919 |
+
elif response.status_code == 416:
|
| 920 |
+
range_header = response.request.headers.get("Range")
|
| 921 |
+
message = f"{e}. Requested range: {range_header}. Content-Range: {response.headers.get('Content-Range')}."
|
| 922 |
+
raise _format(HfHubHTTPError, message, response) from e
|
| 923 |
+
|
| 924 |
+
# Convert `HTTPError` into a `HfHubHTTPError` to display request information
|
| 925 |
+
# as well (request id and/or server error message)
|
| 926 |
+
raise _format(HfHubHTTPError, str(e), response) from e
|
| 927 |
+
|
| 928 |
+
|
| 929 |
+
_WARNED_TOPICS = set()
|
| 930 |
+
|
| 931 |
+
|
| 932 |
+
def _warn_on_warning_headers(response: httpx.Response) -> None:
|
| 933 |
+
"""
|
| 934 |
+
Emit warnings if warning headers are present in the HTTP response.
|
| 935 |
+
|
| 936 |
+
Expected header format: 'X-HF-Warning: topic; message'
|
| 937 |
+
|
| 938 |
+
Only the first warning for each topic will be shown. Topic is optional and can be empty. Note that several warning
|
| 939 |
+
headers can be present in a single response.
|
| 940 |
+
|
| 941 |
+
Args:
|
| 942 |
+
response (`httpx.Response`):
|
| 943 |
+
The HTTP response to check for warning headers.
|
| 944 |
+
"""
|
| 945 |
+
server_warnings = response.headers.get_list("X-HF-Warning")
|
| 946 |
+
for server_warning in server_warnings:
|
| 947 |
+
topic, message = server_warning.split(";", 1) if ";" in server_warning else ("", server_warning)
|
| 948 |
+
topic = topic.strip()
|
| 949 |
+
if topic not in _WARNED_TOPICS:
|
| 950 |
+
message = message.strip()
|
| 951 |
+
if message:
|
| 952 |
+
_WARNED_TOPICS.add(topic)
|
| 953 |
+
logger.warning(message)
|
| 954 |
+
|
| 955 |
+
|
| 956 |
+
_HfHubHTTPErrorT = TypeVar("_HfHubHTTPErrorT", bound=HfHubHTTPError)
|
| 957 |
+
|
| 958 |
+
|
| 959 |
+
def _format(
|
| 960 |
+
error_type: type[_HfHubHTTPErrorT], custom_message: str, response: httpx.Response, **attrs: Any
|
| 961 |
+
) -> _HfHubHTTPErrorT:
|
| 962 |
+
server_errors = []
|
| 963 |
+
|
| 964 |
+
# Retrieve server error from header
|
| 965 |
+
from_headers = response.headers.get("X-Error-Message")
|
| 966 |
+
if from_headers is not None:
|
| 967 |
+
server_errors.append(from_headers)
|
| 968 |
+
|
| 969 |
+
# Retrieve server error from body
|
| 970 |
+
try:
|
| 971 |
+
# Case errors are returned in a JSON format
|
| 972 |
+
try:
|
| 973 |
+
data = response.json()
|
| 974 |
+
except httpx.ResponseNotRead:
|
| 975 |
+
try:
|
| 976 |
+
response.read() # In case of streaming response, we need to read the response first
|
| 977 |
+
data = response.json()
|
| 978 |
+
except RuntimeError:
|
| 979 |
+
# In case of async streaming response, we can't read the stream here.
|
| 980 |
+
# In practice if user is using the default async client from `get_async_client`, the stream will have
|
| 981 |
+
# already been read in the async event hook `async_hf_response_event_hook`.
|
| 982 |
+
#
|
| 983 |
+
# Here, we are skipping reading the response to avoid RuntimeError but it happens only if async + stream + used httpx.AsyncClient directly.
|
| 984 |
+
data = {}
|
| 985 |
+
|
| 986 |
+
error = data.get("error")
|
| 987 |
+
error_description = data.get("error_description")
|
| 988 |
+
if error is not None:
|
| 989 |
+
if isinstance(error, list):
|
| 990 |
+
# Case {'error': ['my error 1', 'my error 2']}
|
| 991 |
+
server_errors.extend(error)
|
| 992 |
+
elif error_description is not None:
|
| 993 |
+
# OAuth-style case {'error': 'invalid_grant', 'error_description': 'my description'}
|
| 994 |
+
server_errors.append(f"{error}: {error_description}")
|
| 995 |
+
else:
|
| 996 |
+
# Case {'error': 'my error'}
|
| 997 |
+
server_errors.append(error)
|
| 998 |
+
elif error_description is not None:
|
| 999 |
+
# Case {'error_description': 'my description'} (no 'error' field)
|
| 1000 |
+
server_errors.append(error_description)
|
| 1001 |
+
|
| 1002 |
+
errors = data.get("errors")
|
| 1003 |
+
if errors is not None:
|
| 1004 |
+
# Case {'errors': [{'message': 'my error 1'}, {'message': 'my error 2'}]}
|
| 1005 |
+
for error in errors:
|
| 1006 |
+
if "message" in error:
|
| 1007 |
+
server_errors.append(error["message"])
|
| 1008 |
+
|
| 1009 |
+
except json.JSONDecodeError:
|
| 1010 |
+
# If content is not JSON and not HTML, append the text
|
| 1011 |
+
content_type = response.headers.get("Content-Type", "")
|
| 1012 |
+
if response.text and "html" not in content_type.lower():
|
| 1013 |
+
server_errors.append(response.text)
|
| 1014 |
+
|
| 1015 |
+
# Strip all server messages
|
| 1016 |
+
server_errors = [str(line).strip() for line in server_errors if str(line).strip()]
|
| 1017 |
+
|
| 1018 |
+
# Deduplicate server messages (keep order)
|
| 1019 |
+
# taken from https://stackoverflow.com/a/17016257
|
| 1020 |
+
server_errors = list(dict.fromkeys(server_errors))
|
| 1021 |
+
|
| 1022 |
+
# Format server error
|
| 1023 |
+
server_message = "\n".join(server_errors)
|
| 1024 |
+
|
| 1025 |
+
# Add server error to custom message
|
| 1026 |
+
final_error_message = custom_message
|
| 1027 |
+
if server_message and server_message.lower() not in custom_message.lower():
|
| 1028 |
+
if "\n\n" in custom_message:
|
| 1029 |
+
final_error_message += "\n" + server_message
|
| 1030 |
+
else:
|
| 1031 |
+
final_error_message += "\n\n" + server_message
|
| 1032 |
+
|
| 1033 |
+
# Prepare Request ID message
|
| 1034 |
+
request_id = ""
|
| 1035 |
+
request_id_message = ""
|
| 1036 |
+
for header, label in (
|
| 1037 |
+
(X_REQUEST_ID, "Request ID"),
|
| 1038 |
+
(X_AMZN_TRACE_ID, "Amzn Trace ID"),
|
| 1039 |
+
(X_AMZ_CF_ID, "Amz CF ID"),
|
| 1040 |
+
):
|
| 1041 |
+
value = response.headers.get(header)
|
| 1042 |
+
if value:
|
| 1043 |
+
request_id = str(value)
|
| 1044 |
+
request_id_message = f" ({label}: {value})"
|
| 1045 |
+
break
|
| 1046 |
+
|
| 1047 |
+
# Add Request ID
|
| 1048 |
+
if request_id and request_id.lower() not in final_error_message.lower():
|
| 1049 |
+
if "\n" in final_error_message:
|
| 1050 |
+
newline_index = final_error_message.index("\n")
|
| 1051 |
+
final_error_message = (
|
| 1052 |
+
final_error_message[:newline_index] + request_id_message + final_error_message[newline_index:]
|
| 1053 |
+
)
|
| 1054 |
+
else:
|
| 1055 |
+
final_error_message += request_id_message
|
| 1056 |
+
|
| 1057 |
+
# Return
|
| 1058 |
+
err = error_type(final_error_message.strip(), response=response, server_message=server_message or None)
|
| 1059 |
+
for k, v in attrs.items():
|
| 1060 |
+
setattr(err, k, v)
|
| 1061 |
+
return err
|
| 1062 |
+
|
| 1063 |
+
|
| 1064 |
+
# Request-body fields that carry credentials; redacted from debug-log curl commands (HF_DEBUG).
|
| 1065 |
+
_SENSITIVE_BODY_KEYS = ("subject_token", "access_token", "refresh_token", "client_secret", "device_code")
|
| 1066 |
+
_SENSITIVE_BODY_PATTERNS = [
|
| 1067 |
+
pattern
|
| 1068 |
+
for key in _SENSITIVE_BODY_KEYS
|
| 1069 |
+
for pattern in (
|
| 1070 |
+
(re.compile(rf'("{key}"\s*:\s*")[^"]*(")'), r"\1<REDACTED>\2"), # JSON
|
| 1071 |
+
(re.compile(rf"(^|&)({key}=)[^&]*"), r"\1\2<REDACTED>"), # application/x-www-form-urlencoded
|
| 1072 |
+
)
|
| 1073 |
+
]
|
| 1074 |
+
|
| 1075 |
+
|
| 1076 |
+
def _redact_sensitive_body(body: str) -> str:
|
| 1077 |
+
"""Redact OAuth credential values from a JSON or form-urlencoded request body string."""
|
| 1078 |
+
for pattern, replacement in _SENSITIVE_BODY_PATTERNS:
|
| 1079 |
+
body = pattern.sub(replacement, body)
|
| 1080 |
+
return body
|
| 1081 |
+
|
| 1082 |
+
|
| 1083 |
+
def _curlify(request: httpx.Request) -> str:
|
| 1084 |
+
"""Convert a `httpx.Request` into a curl command (str).
|
| 1085 |
+
|
| 1086 |
+
Used for debug purposes only.
|
| 1087 |
+
|
| 1088 |
+
Implementation vendored from https://github.com/ofw/curlify/blob/master/curlify.py.
|
| 1089 |
+
MIT License Copyright (c) 2016 Egor.
|
| 1090 |
+
"""
|
| 1091 |
+
parts: list[tuple[Any, Any]] = [
|
| 1092 |
+
("curl", None),
|
| 1093 |
+
("-X", request.method),
|
| 1094 |
+
]
|
| 1095 |
+
|
| 1096 |
+
for k, v in sorted(request.headers.items()):
|
| 1097 |
+
if k.lower() == "authorization":
|
| 1098 |
+
v = "<TOKEN>" # Hide authorization header, no matter its value (can be Bearer, Key, etc.)
|
| 1099 |
+
parts += [("-H", f"{k}: {v}")]
|
| 1100 |
+
|
| 1101 |
+
body: str | None = None
|
| 1102 |
+
try:
|
| 1103 |
+
if request.content is not None:
|
| 1104 |
+
body = request.content.decode("utf-8", errors="ignore")
|
| 1105 |
+
if len(body) > 1000:
|
| 1106 |
+
body = f"{body[:1000]} ... [truncated]"
|
| 1107 |
+
body = _redact_sensitive_body(body)
|
| 1108 |
+
except httpx.RequestNotRead:
|
| 1109 |
+
body = "<streaming body>"
|
| 1110 |
+
if body is not None:
|
| 1111 |
+
parts += [("-d", body.replace("\n", ""))]
|
| 1112 |
+
|
| 1113 |
+
parts += [(None, request.url)]
|
| 1114 |
+
|
| 1115 |
+
flat_parts = []
|
| 1116 |
+
for k, v in parts:
|
| 1117 |
+
if k:
|
| 1118 |
+
flat_parts.append(quote(str(k)))
|
| 1119 |
+
if v:
|
| 1120 |
+
flat_parts.append(quote(str(v)))
|
| 1121 |
+
|
| 1122 |
+
return " ".join(flat_parts)
|
| 1123 |
+
|
| 1124 |
+
|
| 1125 |
+
# Regex to parse HTTP Range header
|
| 1126 |
+
RANGE_REGEX = re.compile(r"^\s*bytes\s*=\s*(\d*)\s*-\s*(\d*)\s*$", re.IGNORECASE)
|
| 1127 |
+
|
| 1128 |
+
|
| 1129 |
+
def _adjust_range_header(original_range: str | None, resume_size: int) -> str | None:
|
| 1130 |
+
"""
|
| 1131 |
+
Adjust HTTP Range header to account for resume position.
|
| 1132 |
+
"""
|
| 1133 |
+
if not original_range:
|
| 1134 |
+
return f"bytes={resume_size}-"
|
| 1135 |
+
|
| 1136 |
+
if "," in original_range:
|
| 1137 |
+
raise ValueError(f"Multiple ranges detected - {original_range!r}, not supported yet.")
|
| 1138 |
+
|
| 1139 |
+
match = RANGE_REGEX.match(original_range)
|
| 1140 |
+
if not match:
|
| 1141 |
+
raise RuntimeError(f"Invalid range format - {original_range!r}.")
|
| 1142 |
+
start, end = match.groups()
|
| 1143 |
+
|
| 1144 |
+
if not start:
|
| 1145 |
+
if not end:
|
| 1146 |
+
raise RuntimeError(f"Invalid range format - {original_range!r}.")
|
| 1147 |
+
|
| 1148 |
+
new_suffix = int(end) - resume_size
|
| 1149 |
+
new_range = f"bytes=-{new_suffix}"
|
| 1150 |
+
if new_suffix <= 0:
|
| 1151 |
+
raise RuntimeError(f"Empty new range - {new_range!r}.")
|
| 1152 |
+
return new_range
|
| 1153 |
+
|
| 1154 |
+
start = int(start)
|
| 1155 |
+
new_start = start + resume_size
|
| 1156 |
+
if end:
|
| 1157 |
+
end = int(end)
|
| 1158 |
+
new_range = f"bytes={new_start}-{end}"
|
| 1159 |
+
if new_start > end:
|
| 1160 |
+
raise RuntimeError(f"Empty new range - {new_range!r}.")
|
| 1161 |
+
return new_range
|
| 1162 |
+
|
| 1163 |
+
return f"bytes={new_start}-"
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_lfs.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2019-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Git LFS related utilities"""
|
| 15 |
+
|
| 16 |
+
import io
|
| 17 |
+
import os
|
| 18 |
+
from contextlib import AbstractContextManager
|
| 19 |
+
from typing import BinaryIO
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class SliceFileObj(AbstractContextManager):
|
| 23 |
+
"""
|
| 24 |
+
Utility context manager to read a *slice* of a seekable file-like object as a seekable, file-like object.
|
| 25 |
+
|
| 26 |
+
This is NOT thread safe
|
| 27 |
+
|
| 28 |
+
Inspired by stackoverflow.com/a/29838711/593036
|
| 29 |
+
|
| 30 |
+
Credits to @julien-c
|
| 31 |
+
|
| 32 |
+
Args:
|
| 33 |
+
fileobj (`BinaryIO`):
|
| 34 |
+
A file-like object to slice. MUST implement `tell()` and `seek()` (and `read()` of course).
|
| 35 |
+
`fileobj` will be reset to its original position when exiting the context manager.
|
| 36 |
+
seek_from (`int`):
|
| 37 |
+
The start of the slice (offset from position 0 in bytes).
|
| 38 |
+
read_limit (`int`):
|
| 39 |
+
The maximum number of bytes to read from the slice.
|
| 40 |
+
|
| 41 |
+
Attributes:
|
| 42 |
+
previous_position (`int`):
|
| 43 |
+
The previous position
|
| 44 |
+
|
| 45 |
+
Examples:
|
| 46 |
+
|
| 47 |
+
Reading 200 bytes with an offset of 128 bytes from a file (ie bytes 128 to 327):
|
| 48 |
+
```python
|
| 49 |
+
>>> with open("path/to/file", "rb") as file:
|
| 50 |
+
... with SliceFileObj(file, seek_from=128, read_limit=200) as fslice:
|
| 51 |
+
... fslice.read(...)
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
Reading a file in chunks of 512 bytes
|
| 55 |
+
```python
|
| 56 |
+
>>> import os
|
| 57 |
+
>>> chunk_size = 512
|
| 58 |
+
>>> file_size = os.getsize("path/to/file")
|
| 59 |
+
>>> with open("path/to/file", "rb") as file:
|
| 60 |
+
... for chunk_idx in range(ceil(file_size / chunk_size)):
|
| 61 |
+
... with SliceFileObj(file, seek_from=chunk_idx * chunk_size, read_limit=chunk_size) as fslice:
|
| 62 |
+
... chunk = fslice.read(...)
|
| 63 |
+
|
| 64 |
+
```
|
| 65 |
+
"""
|
| 66 |
+
|
| 67 |
+
def __init__(self, fileobj: BinaryIO, seek_from: int, read_limit: int):
|
| 68 |
+
self.fileobj = fileobj
|
| 69 |
+
self.seek_from = seek_from
|
| 70 |
+
self.read_limit = read_limit
|
| 71 |
+
|
| 72 |
+
def __enter__(self):
|
| 73 |
+
self._previous_position = self.fileobj.tell()
|
| 74 |
+
end_of_stream = self.fileobj.seek(0, os.SEEK_END)
|
| 75 |
+
self._len = min(self.read_limit, end_of_stream - self.seek_from)
|
| 76 |
+
# ^^ The actual number of bytes that can be read from the slice
|
| 77 |
+
self.fileobj.seek(self.seek_from, io.SEEK_SET)
|
| 78 |
+
return self
|
| 79 |
+
|
| 80 |
+
def __exit__(self, exc_type, exc_value, traceback):
|
| 81 |
+
self.fileobj.seek(self._previous_position, io.SEEK_SET)
|
| 82 |
+
|
| 83 |
+
def read(self, n: int = -1):
|
| 84 |
+
pos = self.tell()
|
| 85 |
+
if pos >= self._len:
|
| 86 |
+
return b""
|
| 87 |
+
remaining_amount = self._len - pos
|
| 88 |
+
data = self.fileobj.read(remaining_amount if n < 0 else min(n, remaining_amount))
|
| 89 |
+
return data
|
| 90 |
+
|
| 91 |
+
def tell(self) -> int:
|
| 92 |
+
return self.fileobj.tell() - self.seek_from
|
| 93 |
+
|
| 94 |
+
def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
|
| 95 |
+
start = self.seek_from
|
| 96 |
+
end = start + self._len
|
| 97 |
+
if whence in (os.SEEK_SET, os.SEEK_END):
|
| 98 |
+
offset = start + offset if whence == os.SEEK_SET else end + offset
|
| 99 |
+
offset = max(start, min(offset, end))
|
| 100 |
+
whence = os.SEEK_SET
|
| 101 |
+
elif whence == os.SEEK_CUR:
|
| 102 |
+
cur_pos = self.fileobj.tell()
|
| 103 |
+
offset = max(start - cur_pos, min(offset, end - cur_pos))
|
| 104 |
+
else:
|
| 105 |
+
raise ValueError(f"whence value {whence} is not supported")
|
| 106 |
+
return self.fileobj.seek(offset, whence) - self.seek_from
|
| 107 |
+
|
| 108 |
+
def __iter__(self):
|
| 109 |
+
yield self.read(n=4 * 1024 * 1024)
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_oauth_device.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Device Code OAuth (RFC 8628) for browser-based login, plus access token refresh.
|
| 15 |
+
|
| 16 |
+
The flow: the client requests a device code, displays a URL and a short user code, the user
|
| 17 |
+
authorizes in a browser, and the client polls ``POST {ENDPOINT}/oauth/token`` until a token is
|
| 18 |
+
issued. Access tokens may come with a refresh token, used to renew them transparently (see
|
| 19 |
+
``utils/_auth.py::get_token``).
|
| 20 |
+
|
| 21 |
+
This module is self-contained protocol logic: no printing, no persistence. Interactive flows
|
| 22 |
+
live in ``_login.py`` (human/library) and ``cli/auth.py`` (machine-readable event stream).
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
import time
|
| 26 |
+
from collections.abc import Callable
|
| 27 |
+
from typing import TypedDict, cast
|
| 28 |
+
|
| 29 |
+
import httpx
|
| 30 |
+
|
| 31 |
+
from .. import constants
|
| 32 |
+
from ..errors import DeviceCodeError, OAuthErrorCode
|
| 33 |
+
from ._http import get_session, hf_raise_for_status
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
_DEVICE_CODE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:device_code"
|
| 37 |
+
_REFRESH_TOKEN_GRANT_TYPE = "refresh_token"
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class DeviceCodeInfo(TypedDict):
|
| 41 |
+
device_code: str
|
| 42 |
+
user_code: str
|
| 43 |
+
verification_uri: str
|
| 44 |
+
verification_uri_complete: str # falls back to verification_uri if the server omits it
|
| 45 |
+
interval: int
|
| 46 |
+
expires_in: int
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class OAuthTokenResponse(TypedDict, total=False):
|
| 50 |
+
access_token: str
|
| 51 |
+
refresh_token: str
|
| 52 |
+
expires_in: int
|
| 53 |
+
token_type: str
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def request_device_code() -> DeviceCodeInfo:
|
| 57 |
+
"""Request a device code from the Hub's OAuth device authorization endpoint.
|
| 58 |
+
|
| 59 |
+
The returned dict is normalized: `interval`, `expires_in` and `verification_uri_complete`
|
| 60 |
+
are always set (server values, or sensible defaults).
|
| 61 |
+
|
| 62 |
+
Raises:
|
| 63 |
+
[`DeviceCodeError`]: If the request fails.
|
| 64 |
+
"""
|
| 65 |
+
try:
|
| 66 |
+
response = get_session().post(
|
| 67 |
+
f"{constants.ENDPOINT}/oauth/device",
|
| 68 |
+
data={"client_id": constants.DEVICE_CODE_OAUTH_CLIENT_ID},
|
| 69 |
+
timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT,
|
| 70 |
+
)
|
| 71 |
+
hf_raise_for_status(response)
|
| 72 |
+
except httpx.HTTPError as e:
|
| 73 |
+
raise DeviceCodeError(f"Failed to request device code from {constants.ENDPOINT}/oauth/device: {e}") from e
|
| 74 |
+
info = response.json()
|
| 75 |
+
# `interval` is optional per RFC 8628 (5s is the spec-mandated fallback); `expires_in` is
|
| 76 |
+
# required but defaulted defensively so polling stays bounded if a server omits it.
|
| 77 |
+
info.setdefault("interval", 5)
|
| 78 |
+
info.setdefault("expires_in", 900)
|
| 79 |
+
if not info.get("verification_uri_complete"):
|
| 80 |
+
info["verification_uri_complete"] = info["verification_uri"]
|
| 81 |
+
return cast(DeviceCodeInfo, info)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def poll_device_token(
|
| 85 |
+
device_info: DeviceCodeInfo, *, on_pending: Callable[[], None] | None = None
|
| 86 |
+
) -> OAuthTokenResponse:
|
| 87 |
+
"""Poll the token endpoint until the user authorizes the device.
|
| 88 |
+
|
| 89 |
+
Args:
|
| 90 |
+
device_info (`DeviceCodeInfo`):
|
| 91 |
+
The device authorization response from [`request_device_code`].
|
| 92 |
+
on_pending (`Callable`, *optional*):
|
| 93 |
+
Called after each "authorization pending" response (e.g. to print a progress dot).
|
| 94 |
+
|
| 95 |
+
Returns:
|
| 96 |
+
`OAuthTokenResponse`: the full token response: `access_token`, and optionally
|
| 97 |
+
`refresh_token` and `expires_in`.
|
| 98 |
+
|
| 99 |
+
Raises:
|
| 100 |
+
[`DeviceCodeError`]: If authorization is denied, the device code expires, or the server
|
| 101 |
+
returns an unexpected OAuth error.
|
| 102 |
+
"""
|
| 103 |
+
interval = device_info["interval"]
|
| 104 |
+
deadline = time.monotonic() + device_info["expires_in"]
|
| 105 |
+
while time.monotonic() < deadline:
|
| 106 |
+
# Inconclusive responses (network blip, 5xx, gateway error page, rate limiting) must not
|
| 107 |
+
# abort the login: keep polling until the device code expires (RFC 8628 section 3.5).
|
| 108 |
+
# The deadline bounds the total wait even if the endpoint is genuinely broken.
|
| 109 |
+
data = None
|
| 110 |
+
try:
|
| 111 |
+
response = get_session().post(
|
| 112 |
+
f"{constants.ENDPOINT}/oauth/token",
|
| 113 |
+
data={
|
| 114 |
+
"grant_type": _DEVICE_CODE_GRANT_TYPE,
|
| 115 |
+
"device_code": device_info["device_code"],
|
| 116 |
+
"client_id": constants.DEVICE_CODE_OAUTH_CLIENT_ID,
|
| 117 |
+
},
|
| 118 |
+
timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT,
|
| 119 |
+
)
|
| 120 |
+
if response.status_code < 500:
|
| 121 |
+
data = response.json()
|
| 122 |
+
except (httpx.HTTPError, ValueError):
|
| 123 |
+
pass
|
| 124 |
+
|
| 125 |
+
if data is not None:
|
| 126 |
+
if "access_token" in data:
|
| 127 |
+
return cast(OAuthTokenResponse, data)
|
| 128 |
+
|
| 129 |
+
match data.get("error"):
|
| 130 |
+
case None:
|
| 131 |
+
pass # JSON without an OAuth `error` field (proxy error page, ...): transient
|
| 132 |
+
case OAuthErrorCode.AUTHORIZATION_PENDING:
|
| 133 |
+
if on_pending is not None:
|
| 134 |
+
on_pending()
|
| 135 |
+
case OAuthErrorCode.SLOW_DOWN:
|
| 136 |
+
interval += 5
|
| 137 |
+
case OAuthErrorCode.EXPIRED_TOKEN:
|
| 138 |
+
raise DeviceCodeError(
|
| 139 |
+
"Device code expired. Please try again.", error_code=OAuthErrorCode.EXPIRED_TOKEN
|
| 140 |
+
)
|
| 141 |
+
case OAuthErrorCode.ACCESS_DENIED:
|
| 142 |
+
raise DeviceCodeError(
|
| 143 |
+
"Authorization was denied. Please try again.", error_code=OAuthErrorCode.ACCESS_DENIED
|
| 144 |
+
)
|
| 145 |
+
case error:
|
| 146 |
+
raise DeviceCodeError(
|
| 147 |
+
f"OAuth error: {error} - {data.get('error_description', '')}", error_code=error
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
time.sleep(interval)
|
| 151 |
+
|
| 152 |
+
raise DeviceCodeError("Device code expired (timeout). Please try again.", error_code=OAuthErrorCode.EXPIRED_TOKEN)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def refresh_access_token(refresh_token: str) -> OAuthTokenResponse:
|
| 156 |
+
"""Exchange a refresh token for a new access token.
|
| 157 |
+
|
| 158 |
+
Returns:
|
| 159 |
+
`OAuthTokenResponse`: the full token response: `access_token`, and optionally a rotated
|
| 160 |
+
`refresh_token` and `expires_in`.
|
| 161 |
+
|
| 162 |
+
Raises:
|
| 163 |
+
[`DeviceCodeError`]: If the server rejects the refresh (`error_code="invalid_grant"` when
|
| 164 |
+
the refresh token is expired or revoked) or returns an unexpected response.
|
| 165 |
+
"""
|
| 166 |
+
try:
|
| 167 |
+
response = get_session().post(
|
| 168 |
+
f"{constants.ENDPOINT}/oauth/token",
|
| 169 |
+
data={
|
| 170 |
+
"grant_type": _REFRESH_TOKEN_GRANT_TYPE,
|
| 171 |
+
"refresh_token": refresh_token,
|
| 172 |
+
"client_id": constants.DEVICE_CODE_OAUTH_CLIENT_ID,
|
| 173 |
+
},
|
| 174 |
+
# An explicit timeout is critical here: this runs inside `get_token()`, so a hung
|
| 175 |
+
# request would otherwise block every Hub call in the process.
|
| 176 |
+
timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT,
|
| 177 |
+
)
|
| 178 |
+
except httpx.HTTPError as e:
|
| 179 |
+
raise DeviceCodeError(f"Failed to refresh access token: {e}") from e
|
| 180 |
+
data = _parse_token_response(response)
|
| 181 |
+
if "access_token" in data:
|
| 182 |
+
return cast(OAuthTokenResponse, data)
|
| 183 |
+
error = data.get("error")
|
| 184 |
+
raise DeviceCodeError(
|
| 185 |
+
f"Failed to refresh access token: {error or response.status_code} - {data.get('error_description', '')}",
|
| 186 |
+
error_code=error,
|
| 187 |
+
)
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def _parse_token_response(response: httpx.Response) -> dict:
|
| 191 |
+
try:
|
| 192 |
+
return response.json()
|
| 193 |
+
except ValueError as e:
|
| 194 |
+
raise DeviceCodeError(
|
| 195 |
+
f"Failed to parse response from {constants.ENDPOINT}/oauth/token "
|
| 196 |
+
f"(status {response.status_code}): {response.text[:500]}"
|
| 197 |
+
) from e
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_pagination.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains utilities to handle pagination on Huggingface Hub."""
|
| 15 |
+
|
| 16 |
+
from collections.abc import Iterable
|
| 17 |
+
from typing import Any
|
| 18 |
+
|
| 19 |
+
import httpx
|
| 20 |
+
|
| 21 |
+
from . import get_session, hf_raise_for_status, http_backoff, logging
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
logger = logging.get_logger(__name__)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def paginate(path: str, params: dict | list[tuple[str, Any]], headers: dict, timeout: float | None = None) -> Iterable:
|
| 28 |
+
"""Fetch a list of models/datasets/spaces and paginate through results.
|
| 29 |
+
|
| 30 |
+
This is using the same "Link" header format as GitHub.
|
| 31 |
+
See:
|
| 32 |
+
- https://requests.readthedocs.io/en/latest/api/#requests.Response.links
|
| 33 |
+
- https://docs.github.com/en/rest/guides/traversing-with-pagination#link-header
|
| 34 |
+
"""
|
| 35 |
+
session = get_session()
|
| 36 |
+
r = session.get(path, params=params, headers=headers, timeout=timeout)
|
| 37 |
+
hf_raise_for_status(r)
|
| 38 |
+
yield from r.json()
|
| 39 |
+
|
| 40 |
+
# Follow pages
|
| 41 |
+
# Next link already contains query params
|
| 42 |
+
next_page = _get_next_page(r)
|
| 43 |
+
while next_page is not None:
|
| 44 |
+
logger.debug(f"Pagination detected. Requesting next page: {next_page}")
|
| 45 |
+
r = http_backoff("GET", next_page, headers=headers, timeout=timeout)
|
| 46 |
+
hf_raise_for_status(r)
|
| 47 |
+
yield from r.json()
|
| 48 |
+
next_page = _get_next_page(r)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _get_next_page(response: httpx.Response) -> str | None:
|
| 52 |
+
return response.links.get("next", {}).get("url")
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_parsing.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Parsing helpers shared across modules."""
|
| 15 |
+
|
| 16 |
+
import re
|
| 17 |
+
import time
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
RE_NUMBER_WITH_UNIT = re.compile(r"(\d+)([a-z]+)", re.IGNORECASE)
|
| 21 |
+
|
| 22 |
+
BYTE_UNITS: dict[str, int] = {
|
| 23 |
+
"k": 1_000,
|
| 24 |
+
"kb": 1_000,
|
| 25 |
+
"m": 1_000_000,
|
| 26 |
+
"mb": 1_000_000,
|
| 27 |
+
"g": 1_000_000_000,
|
| 28 |
+
"gb": 1_000_000_000,
|
| 29 |
+
"t": 1_000_000_000_000,
|
| 30 |
+
"tb": 1_000_000_000_000,
|
| 31 |
+
"p": 1_000_000_000_000_000,
|
| 32 |
+
"pb": 1_000_000_000_000_000,
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
TIME_UNITS: dict[str, int] = {
|
| 36 |
+
"s": 1,
|
| 37 |
+
"m": 60,
|
| 38 |
+
"h": 60 * 60,
|
| 39 |
+
"d": 24 * 60 * 60,
|
| 40 |
+
"w": 7 * 24 * 60 * 60,
|
| 41 |
+
"mo": 30 * 24 * 60 * 60,
|
| 42 |
+
"y": 365 * 24 * 60 * 60,
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def parse_size(value: str) -> int:
|
| 47 |
+
"""Parse a size expressed as a string with digits and unit (like `"10MB"`) to an integer (in bytes)."""
|
| 48 |
+
return _parse_with_unit(value, BYTE_UNITS)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def parse_duration(value: str) -> int:
|
| 52 |
+
"""Parse a duration expressed as a string with digits and unit (like `"10s"`) to an integer (in seconds)."""
|
| 53 |
+
return _parse_with_unit(value, TIME_UNITS)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _parse_with_unit(value: str, units: dict[str, int]) -> int:
|
| 57 |
+
"""Parse a numeric value with optional unit."""
|
| 58 |
+
stripped = value.strip()
|
| 59 |
+
if not stripped:
|
| 60 |
+
raise ValueError("Value cannot be empty.")
|
| 61 |
+
try:
|
| 62 |
+
return int(value)
|
| 63 |
+
except ValueError:
|
| 64 |
+
pass
|
| 65 |
+
|
| 66 |
+
match = RE_NUMBER_WITH_UNIT.fullmatch(stripped)
|
| 67 |
+
if not match:
|
| 68 |
+
raise ValueError(f"Invalid value '{value}'. Must match pattern '\\d+[a-z]+' or be a plain number.")
|
| 69 |
+
|
| 70 |
+
number = int(match.group(1))
|
| 71 |
+
unit = match.group(2).lower()
|
| 72 |
+
|
| 73 |
+
if unit not in units:
|
| 74 |
+
raise ValueError(f"Unknown unit '{unit}'. Must be one of {list(units.keys())}.")
|
| 75 |
+
|
| 76 |
+
return number * units[unit]
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def format_duration(secs: int | None) -> str:
|
| 80 |
+
"""Format a duration in seconds as a short human-readable string (e.g. `"1m 32s"`, `"2h 15m"`, `"45s"`).
|
| 81 |
+
|
| 82 |
+
Returns `"--"` when `secs` is `None` so it can be used directly as a CLI table cell.
|
| 83 |
+
"""
|
| 84 |
+
if secs is None:
|
| 85 |
+
return "--"
|
| 86 |
+
secs = int(secs)
|
| 87 |
+
if secs < 60:
|
| 88 |
+
return f"{secs}s"
|
| 89 |
+
if secs < 3600:
|
| 90 |
+
return f"{secs // 60}m {secs % 60}s"
|
| 91 |
+
return f"{secs // 3600}h {(secs % 3600) // 60}m"
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def format_timesince(ts: float) -> str:
|
| 95 |
+
"""Format timestamp in seconds into a human-readable string, relative to now.
|
| 96 |
+
|
| 97 |
+
Vaguely inspired by Django's `timesince` formatter.
|
| 98 |
+
"""
|
| 99 |
+
_TIMESINCE_CHUNKS = (
|
| 100 |
+
# Label, divider, max value
|
| 101 |
+
("second", 1, 60),
|
| 102 |
+
("minute", 60, 60),
|
| 103 |
+
("hour", 60 * 60, 24),
|
| 104 |
+
("day", 60 * 60 * 24, 6),
|
| 105 |
+
("week", 60 * 60 * 24 * 7, 6),
|
| 106 |
+
("month", 60 * 60 * 24 * 30, 11),
|
| 107 |
+
("year", 60 * 60 * 24 * 365, None),
|
| 108 |
+
)
|
| 109 |
+
delta = time.time() - ts
|
| 110 |
+
if delta < 20:
|
| 111 |
+
return "a few seconds ago"
|
| 112 |
+
for label, divider, max_value in _TIMESINCE_CHUNKS: # noqa: B007
|
| 113 |
+
value = round(delta / divider)
|
| 114 |
+
if max_value is not None and value <= max_value:
|
| 115 |
+
break
|
| 116 |
+
return f"{value} {label}{'s' if value > 1 else ''} ago"
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_paths.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Contains utilities to handle paths in Huggingface Hub."""
|
| 15 |
+
|
| 16 |
+
from collections.abc import Callable, Generator, Iterable
|
| 17 |
+
from fnmatch import fnmatchcase
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
from typing import TypeVar
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
T = TypeVar("T")
|
| 23 |
+
|
| 24 |
+
# Always ignore `.git` and `.cache/huggingface` folders in commits
|
| 25 |
+
DEFAULT_IGNORE_PATTERNS = [
|
| 26 |
+
".git",
|
| 27 |
+
".git/*",
|
| 28 |
+
"*/.git",
|
| 29 |
+
"**/.git/**",
|
| 30 |
+
".cache/huggingface",
|
| 31 |
+
".cache/huggingface/*",
|
| 32 |
+
"*/.cache/huggingface",
|
| 33 |
+
"**/.cache/huggingface/**",
|
| 34 |
+
]
|
| 35 |
+
# Forbidden to commit these folders
|
| 36 |
+
FORBIDDEN_FOLDERS = [".git", ".cache"]
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def filter_repo_objects(
|
| 40 |
+
items: Iterable[T],
|
| 41 |
+
*,
|
| 42 |
+
allow_patterns: list[str] | str | None = None,
|
| 43 |
+
ignore_patterns: list[str] | str | None = None,
|
| 44 |
+
key: Callable[[T], str] | None = None,
|
| 45 |
+
) -> Generator[T, None, None]:
|
| 46 |
+
"""Filter repo objects based on an allowlist and a denylist.
|
| 47 |
+
|
| 48 |
+
Input must be a list of paths (`str` or `Path`) or a list of arbitrary objects.
|
| 49 |
+
In the later case, `key` must be provided and specifies a function of one argument
|
| 50 |
+
that is used to extract a path from each element in iterable.
|
| 51 |
+
|
| 52 |
+
Patterns are Standard Wildcards (globbing patterns), NOT regular expressions. The pattern matching is based on
|
| 53 |
+
Python's `fnmatch.fnmatchcase`, so it is case-sensitive on every platform. Backslashes are treated as path separators
|
| 54 |
+
and normalized to forward slashes in both patterns and paths before matching, so patterns built with `os.path.join`
|
| 55 |
+
on Windows work as expected.
|
| 56 |
+
|
| 57 |
+
Note that it matches `*` across path boundaries, unlike traditional Unix shell globbing. For example, `"data/*.json"`
|
| 58 |
+
will match both `data/file.json` and `data/subdir/file.json`.
|
| 59 |
+
|
| 60 |
+
See https://docs.python.org/3/library/fnmatch.html for more details.
|
| 61 |
+
|
| 62 |
+
Args:
|
| 63 |
+
items (`Iterable`):
|
| 64 |
+
List of items to filter.
|
| 65 |
+
allow_patterns (`str` or `list[str]`, *optional*):
|
| 66 |
+
Patterns constituting the allowlist. If provided, item paths must match at
|
| 67 |
+
least one pattern from the allowlist.
|
| 68 |
+
ignore_patterns (`str` or `list[str]`, *optional*):
|
| 69 |
+
Patterns constituting the denylist. If provided, item paths must not match
|
| 70 |
+
any patterns from the denylist.
|
| 71 |
+
key (`Callable[[T], str]`, *optional*):
|
| 72 |
+
Single-argument function to extract a path from each item. If not provided,
|
| 73 |
+
the `items` must already be `str` or `Path`.
|
| 74 |
+
|
| 75 |
+
Returns:
|
| 76 |
+
Filtered list of objects, as a generator.
|
| 77 |
+
|
| 78 |
+
Raises:
|
| 79 |
+
:class:`ValueError`:
|
| 80 |
+
If `key` is not provided and items are not `str` or `Path`.
|
| 81 |
+
|
| 82 |
+
Example usage with paths:
|
| 83 |
+
```python
|
| 84 |
+
>>> # Filter only PDFs that are not hidden.
|
| 85 |
+
>>> list(filter_repo_objects(
|
| 86 |
+
... ["aaa.pdf", "bbb.jpg", ".ccc.pdf", ".ddd.png"],
|
| 87 |
+
... allow_patterns=["*.pdf"],
|
| 88 |
+
... ignore_patterns=[".*"],
|
| 89 |
+
... ))
|
| 90 |
+
["aaa.pdf"]
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
Example usage with objects:
|
| 94 |
+
```python
|
| 95 |
+
>>> list(filter_repo_objects(
|
| 96 |
+
... [
|
| 97 |
+
... CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf")
|
| 98 |
+
... CommitOperationAdd(path_or_fileobj="/tmp/bbb.jpg", path_in_repo="bbb.jpg")
|
| 99 |
+
... CommitOperationAdd(path_or_fileobj="/tmp/.ccc.pdf", path_in_repo=".ccc.pdf")
|
| 100 |
+
... CommitOperationAdd(path_or_fileobj="/tmp/.ddd.png", path_in_repo=".ddd.png")
|
| 101 |
+
... ],
|
| 102 |
+
... allow_patterns=["*.pdf"],
|
| 103 |
+
... ignore_patterns=[".*"],
|
| 104 |
+
... key=lambda x: x.path_in_repo
|
| 105 |
+
... ))
|
| 106 |
+
[CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf")]
|
| 107 |
+
```
|
| 108 |
+
"""
|
| 109 |
+
if isinstance(allow_patterns, str):
|
| 110 |
+
allow_patterns = [allow_patterns]
|
| 111 |
+
|
| 112 |
+
if isinstance(ignore_patterns, str):
|
| 113 |
+
ignore_patterns = [ignore_patterns]
|
| 114 |
+
|
| 115 |
+
if allow_patterns is not None:
|
| 116 |
+
allow_patterns = [_add_wildcard_to_directories(_normalize_separators(p)) for p in allow_patterns]
|
| 117 |
+
if ignore_patterns is not None:
|
| 118 |
+
ignore_patterns = [_add_wildcard_to_directories(_normalize_separators(p)) for p in ignore_patterns]
|
| 119 |
+
|
| 120 |
+
if key is None:
|
| 121 |
+
|
| 122 |
+
def _identity(item: T) -> str:
|
| 123 |
+
if isinstance(item, str):
|
| 124 |
+
return item
|
| 125 |
+
if isinstance(item, Path):
|
| 126 |
+
return str(item)
|
| 127 |
+
raise ValueError(f"Please provide `key` argument in `filter_repo_objects`: `{item}` is not a string.")
|
| 128 |
+
|
| 129 |
+
key = _identity # Items must be `str` or `Path`, otherwise raise ValueError
|
| 130 |
+
|
| 131 |
+
for item in items:
|
| 132 |
+
path = _normalize_separators(key(item))
|
| 133 |
+
|
| 134 |
+
# Skip if there's an allowlist and path doesn't match any
|
| 135 |
+
if allow_patterns is not None and not any(fnmatchcase(path, r) for r in allow_patterns):
|
| 136 |
+
continue
|
| 137 |
+
|
| 138 |
+
# Skip if there's a denylist and path matches any
|
| 139 |
+
if ignore_patterns is not None and any(fnmatchcase(path, r) for r in ignore_patterns):
|
| 140 |
+
continue
|
| 141 |
+
|
| 142 |
+
yield item
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _normalize_separators(value: str | Path) -> str:
|
| 146 |
+
# Repo paths always use `/` and `\` is not an fnmatch escape character, so treating backslashes as path separators
|
| 147 |
+
# (e.g. patterns built with `os.path.join` on Windows) is safe.
|
| 148 |
+
# `value` can be a `Path` if a custom `key` returns one; coerce to `str` first.
|
| 149 |
+
return str(value).replace("\\", "/")
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def _add_wildcard_to_directories(pattern: str) -> str:
|
| 153 |
+
if pattern.endswith("/"):
|
| 154 |
+
return pattern + "*"
|
| 155 |
+
return pattern
|
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_runtime.py
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022-present, the HuggingFace Inc. team.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""Check presence of installed packages at runtime."""
|
| 15 |
+
|
| 16 |
+
import importlib.metadata
|
| 17 |
+
import importlib.util
|
| 18 |
+
import os
|
| 19 |
+
import platform
|
| 20 |
+
import sys
|
| 21 |
+
import warnings
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
from typing import Any, Literal
|
| 24 |
+
|
| 25 |
+
from .. import __version__, constants
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
_PY_VERSION: str = sys.version.split()[0].rstrip("+")
|
| 29 |
+
|
| 30 |
+
_package_versions = {}
|
| 31 |
+
|
| 32 |
+
_CANDIDATES = {
|
| 33 |
+
"aiohttp": {"aiohttp"},
|
| 34 |
+
"fastai": {"fastai"},
|
| 35 |
+
"fastapi": {"fastapi"},
|
| 36 |
+
"fastcore": {"fastcore"},
|
| 37 |
+
"gradio": {"gradio"},
|
| 38 |
+
"graphviz": {"graphviz"},
|
| 39 |
+
"hf_xet": {"hf_xet"},
|
| 40 |
+
"jinja": {"Jinja2"},
|
| 41 |
+
"httpx": {"httpx"},
|
| 42 |
+
"keras": {"keras"},
|
| 43 |
+
"numpy": {"numpy"},
|
| 44 |
+
"pillow": {"Pillow"},
|
| 45 |
+
"pydantic": {"pydantic"},
|
| 46 |
+
"pydot": {"pydot"},
|
| 47 |
+
"safetensors": {"safetensors"},
|
| 48 |
+
"tensorboard": {"tensorboardX"},
|
| 49 |
+
"tensorflow": (
|
| 50 |
+
"tensorflow",
|
| 51 |
+
"tensorflow-cpu",
|
| 52 |
+
"tensorflow-gpu",
|
| 53 |
+
"tf-nightly",
|
| 54 |
+
"tf-nightly-cpu",
|
| 55 |
+
"tf-nightly-gpu",
|
| 56 |
+
"intel-tensorflow",
|
| 57 |
+
"intel-tensorflow-avx512",
|
| 58 |
+
"tensorflow-rocm",
|
| 59 |
+
"tensorflow-macos",
|
| 60 |
+
),
|
| 61 |
+
"torch": {"torch"},
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
# Check once at runtime
|
| 65 |
+
for candidate_name, package_names in _CANDIDATES.items():
|
| 66 |
+
_package_versions[candidate_name] = "N/A"
|
| 67 |
+
for name in package_names:
|
| 68 |
+
try:
|
| 69 |
+
_package_versions[candidate_name] = importlib.metadata.version(name)
|
| 70 |
+
break
|
| 71 |
+
except importlib.metadata.PackageNotFoundError:
|
| 72 |
+
pass
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _get_version(package_name: str) -> str:
|
| 76 |
+
return _package_versions.get(package_name, "N/A")
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def is_package_available(package_name: str) -> bool:
|
| 80 |
+
return _get_version(package_name) != "N/A"
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
# Python
|
| 84 |
+
def get_python_version() -> str:
|
| 85 |
+
return _PY_VERSION
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
# Huggingface Hub
|
| 89 |
+
def get_hf_hub_version() -> str:
|
| 90 |
+
return __version__
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# aiohttp
|
| 94 |
+
def is_aiohttp_available() -> bool:
|
| 95 |
+
return is_package_available("aiohttp")
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def get_aiohttp_version() -> str:
|
| 99 |
+
return _get_version("aiohttp")
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
# FastAI
|
| 103 |
+
def is_fastai_available() -> bool:
|
| 104 |
+
return is_package_available("fastai")
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def get_fastai_version() -> str:
|
| 108 |
+
return _get_version("fastai")
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
# FastAPI
|
| 112 |
+
def is_fastapi_available() -> bool:
|
| 113 |
+
return is_package_available("fastapi")
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def get_fastapi_version() -> str:
|
| 117 |
+
return _get_version("fastapi")
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
# Fastcore
|
| 121 |
+
def is_fastcore_available() -> bool:
|
| 122 |
+
return is_package_available("fastcore")
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def get_fastcore_version() -> str:
|
| 126 |
+
return _get_version("fastcore")
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
# FastAI
|
| 130 |
+
def is_gradio_available() -> bool:
|
| 131 |
+
return is_package_available("gradio")
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def get_gradio_version() -> str:
|
| 135 |
+
return _get_version("gradio")
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
# Graphviz
|
| 139 |
+
def is_graphviz_available() -> bool:
|
| 140 |
+
return is_package_available("graphviz")
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def get_graphviz_version() -> str:
|
| 144 |
+
return _get_version("graphviz")
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
# httpx
|
| 148 |
+
def is_httpx_available() -> bool:
|
| 149 |
+
return is_package_available("httpx")
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def get_httpx_version() -> str:
|
| 153 |
+
return _get_version("httpx")
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
# xet
|
| 157 |
+
def is_xet_available() -> bool:
|
| 158 |
+
# since hf_xet is automatically used if available, allow explicit disabling via environment variable
|
| 159 |
+
if constants.HF_HUB_DISABLE_XET:
|
| 160 |
+
return False
|
| 161 |
+
|
| 162 |
+
return is_package_available("hf_xet")
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def get_xet_version() -> str:
|
| 166 |
+
return _get_version("hf_xet")
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
# keras
|
| 170 |
+
def is_keras_available() -> bool:
|
| 171 |
+
return is_package_available("keras")
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def get_keras_version() -> str:
|
| 175 |
+
return _get_version("keras")
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
# Numpy
|
| 179 |
+
def is_numpy_available() -> bool:
|
| 180 |
+
return is_package_available("numpy")
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def get_numpy_version() -> str:
|
| 184 |
+
return _get_version("numpy")
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
# Jinja
|
| 188 |
+
def is_jinja_available() -> bool:
|
| 189 |
+
return is_package_available("jinja")
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def get_jinja_version() -> str:
|
| 193 |
+
return _get_version("jinja")
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
# Pillow
|
| 197 |
+
def is_pillow_available() -> bool:
|
| 198 |
+
return is_package_available("pillow")
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def get_pillow_version() -> str:
|
| 202 |
+
return _get_version("pillow")
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
# Pydantic
|
| 206 |
+
def is_pydantic_available() -> bool:
|
| 207 |
+
if not is_package_available("pydantic"):
|
| 208 |
+
return False
|
| 209 |
+
# For Pydantic, we add an extra check to test whether it is correctly installed or not. If both pydantic 2.x and
|
| 210 |
+
# typing_extensions<=4.5.0 are installed, then pydantic will fail at import time. This should not happen when
|
| 211 |
+
# it is installed with `pip install huggingface_hub[inference]` but it can happen when it is installed manually
|
| 212 |
+
# by the user in an environment that we don't control.
|
| 213 |
+
#
|
| 214 |
+
# Usually we won't need to do this kind of check on optional dependencies. However, pydantic is a special case
|
| 215 |
+
# as it is automatically imported when doing `from huggingface_hub import ...` even if the user doesn't use it.
|
| 216 |
+
#
|
| 217 |
+
# See https://github.com/huggingface/huggingface_hub/pull/1829 for more details.
|
| 218 |
+
try:
|
| 219 |
+
from pydantic import validator # noqa: F401
|
| 220 |
+
except ImportError:
|
| 221 |
+
# Example: "ImportError: cannot import name 'TypeAliasType' from 'typing_extensions'"
|
| 222 |
+
warnings.warn(
|
| 223 |
+
"Pydantic is installed but cannot be imported. Please check your installation. `huggingface_hub` will "
|
| 224 |
+
"default to not using Pydantic. Error message: '{e}'"
|
| 225 |
+
)
|
| 226 |
+
return False
|
| 227 |
+
return True
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def get_pydantic_version() -> str:
|
| 231 |
+
return _get_version("pydantic")
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
# Pydot
|
| 235 |
+
def is_pydot_available() -> bool:
|
| 236 |
+
return is_package_available("pydot")
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def get_pydot_version() -> str:
|
| 240 |
+
return _get_version("pydot")
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
# Tensorboard
|
| 244 |
+
def is_tensorboard_available() -> bool:
|
| 245 |
+
return is_package_available("tensorboard")
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def get_tensorboard_version() -> str:
|
| 249 |
+
return _get_version("tensorboard")
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
# Tensorflow
|
| 253 |
+
def is_tf_available() -> bool:
|
| 254 |
+
return is_package_available("tensorflow")
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def get_tf_version() -> str:
|
| 258 |
+
return _get_version("tensorflow")
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
# Torch
|
| 262 |
+
def is_torch_available() -> bool:
|
| 263 |
+
return is_package_available("torch")
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def get_torch_version() -> str:
|
| 267 |
+
return _get_version("torch")
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
# Safetensors
|
| 271 |
+
def is_safetensors_available() -> bool:
|
| 272 |
+
return is_package_available("safetensors")
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
# Shell-related helpers
|
| 276 |
+
try:
|
| 277 |
+
# Set to `True` if script is running in a Google Colab notebook.
|
| 278 |
+
# If running in Google Colab, git credential store is set globally which makes the
|
| 279 |
+
# warning disappear. See https://github.com/huggingface/huggingface_hub/issues/1043
|
| 280 |
+
#
|
| 281 |
+
# Taken from https://stackoverflow.com/a/63519730.
|
| 282 |
+
_is_google_colab = "google.colab" in str(get_ipython()) # type: ignore # noqa: F821
|
| 283 |
+
except NameError:
|
| 284 |
+
_is_google_colab = False
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def is_notebook() -> bool:
|
| 288 |
+
"""Return `True` if code is executed in a notebook (Jupyter, Colab, QTconsole).
|
| 289 |
+
|
| 290 |
+
Taken from https://stackoverflow.com/a/39662359.
|
| 291 |
+
Adapted to make it work with Google colab as well.
|
| 292 |
+
"""
|
| 293 |
+
try:
|
| 294 |
+
shell_class = get_ipython().__class__ # type: ignore # noqa: F821
|
| 295 |
+
for parent_class in shell_class.__mro__: # e.g. "is subclass of"
|
| 296 |
+
if parent_class.__name__ == "ZMQInteractiveShell":
|
| 297 |
+
return True # Jupyter notebook, Google colab or qtconsole
|
| 298 |
+
return False
|
| 299 |
+
except NameError:
|
| 300 |
+
return False # Probably standard Python interpreter
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
def is_google_colab() -> bool:
|
| 304 |
+
"""Return `True` if code is executed in a Google colab.
|
| 305 |
+
|
| 306 |
+
Taken from https://stackoverflow.com/a/63519730.
|
| 307 |
+
"""
|
| 308 |
+
return _is_google_colab
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def is_colab_enterprise() -> bool:
|
| 312 |
+
"""Return `True` if code is executed in a Google Colab Enterprise environment."""
|
| 313 |
+
return os.environ.get("VERTEX_PRODUCT") == "COLAB_ENTERPRISE"
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
# Check how huggingface_hub has been installed
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def installation_method() -> Literal["brew", "hf_installer", "pip", "unknown"]:
|
| 320 |
+
"""Return the installation method of the current environment.
|
| 321 |
+
|
| 322 |
+
- "hf_installer" if installed via the official installer script
|
| 323 |
+
- "brew" if installed via Homebrew
|
| 324 |
+
- "pip" if pip is available (default fallback for standard Python environments)
|
| 325 |
+
- "unknown" otherwise
|
| 326 |
+
"""
|
| 327 |
+
# hf_installer check must come first: the installer creates a venv using the
|
| 328 |
+
# system Python, which may be Homebrew's. Checking brew first would false-positive
|
| 329 |
+
# when the resolved sys.executable points to /opt/homebrew/... inside a venv.
|
| 330 |
+
if _is_hf_installer_installation():
|
| 331 |
+
return "hf_installer"
|
| 332 |
+
if _is_brew_installation():
|
| 333 |
+
return "brew"
|
| 334 |
+
if _is_pip_available():
|
| 335 |
+
return "pip"
|
| 336 |
+
return "unknown"
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def _is_brew_installation() -> bool:
|
| 340 |
+
"""Check if running from a Homebrew installation.
|
| 341 |
+
|
| 342 |
+
Homebrew installs the `hf` formula into a Cellar directory and creates a
|
| 343 |
+
libexec virtualenv at e.g. /opt/homebrew/Cellar/hf/0.30.0/libexec/.
|
| 344 |
+
We check `sys.prefix` (the venv/prefix root) for "/Cellar/hf/" rather
|
| 345 |
+
than checking `sys.executable` — the latter resolves to Homebrew's Python
|
| 346 |
+
(e.g. /opt/homebrew/Cellar/python@3.12/...) even for non-brew installs
|
| 347 |
+
when the system Python happens to come from Homebrew.
|
| 348 |
+
"""
|
| 349 |
+
prefix = str(Path(sys.prefix).resolve())
|
| 350 |
+
return "/Cellar/hf/" in prefix
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def _is_hf_installer_installation() -> bool:
|
| 354 |
+
"""Return `True` if the current environment was set up via the official hf installer script.
|
| 355 |
+
|
| 356 |
+
i.e. using one of
|
| 357 |
+
curl -LsSf https://hf.co/cli/install.sh | bash
|
| 358 |
+
powershell -ExecutionPolicy ByPass -c "irm https://hf.co/cli/install.ps1 | iex"
|
| 359 |
+
"""
|
| 360 |
+
venv = sys.prefix # points to venv root if active
|
| 361 |
+
marker = Path(venv) / ".hf_installer_marker"
|
| 362 |
+
return marker.exists()
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
def _is_pip_available() -> bool:
|
| 366 |
+
"""Return `True` if pip is importable in the current environment."""
|
| 367 |
+
return importlib.util.find_spec("pip") is not None
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
def dump_environment_info() -> dict[str, Any]:
|
| 371 |
+
"""Dump information about the machine to help debugging issues.
|
| 372 |
+
|
| 373 |
+
Similar helper exist in:
|
| 374 |
+
- `datasets` (https://github.com/huggingface/datasets/blob/main/src/datasets/commands/env.py)
|
| 375 |
+
- `diffusers` (https://github.com/huggingface/diffusers/blob/main/src/diffusers/commands/env.py)
|
| 376 |
+
- `transformers` (https://github.com/huggingface/transformers/blob/main/src/transformers/commands/env.py)
|
| 377 |
+
"""
|
| 378 |
+
from huggingface_hub import get_token, whoami
|
| 379 |
+
from huggingface_hub.utils import is_agent, list_credential_helpers
|
| 380 |
+
|
| 381 |
+
token = get_token()
|
| 382 |
+
|
| 383 |
+
# Generic machine info
|
| 384 |
+
info: dict[str, Any] = {
|
| 385 |
+
"huggingface_hub version": get_hf_hub_version(),
|
| 386 |
+
"Platform": platform.platform(),
|
| 387 |
+
"Python version": get_python_version(),
|
| 388 |
+
}
|
| 389 |
+
|
| 390 |
+
# Interpreter info
|
| 391 |
+
try:
|
| 392 |
+
shell_class = get_ipython().__class__ # type: ignore # noqa: F821
|
| 393 |
+
info["Running in iPython ?"] = "Yes"
|
| 394 |
+
info["iPython shell"] = shell_class.__name__
|
| 395 |
+
except NameError:
|
| 396 |
+
info["Running in iPython ?"] = "No"
|
| 397 |
+
info["Running in notebook ?"] = "Yes" if is_notebook() else "No"
|
| 398 |
+
info["Running in Google Colab ?"] = "Yes" if is_google_colab() else "No"
|
| 399 |
+
info["Running in Google Colab Enterprise ?"] = "Yes" if is_colab_enterprise() else "No"
|
| 400 |
+
# Login info
|
| 401 |
+
info["Token path ?"] = constants.HF_TOKEN_PATH
|
| 402 |
+
info["Has saved token ?"] = token is not None
|
| 403 |
+
if token is not None:
|
| 404 |
+
try:
|
| 405 |
+
info["Who am I ?"] = whoami()["name"]
|
| 406 |
+
except Exception:
|
| 407 |
+
pass
|
| 408 |
+
|
| 409 |
+
try:
|
| 410 |
+
info["Configured git credential helpers"] = ", ".join(list_credential_helpers())
|
| 411 |
+
except Exception:
|
| 412 |
+
pass
|
| 413 |
+
|
| 414 |
+
info["Run by AI agent ?"] = "Yes" if is_agent() else "No"
|
| 415 |
+
|
| 416 |
+
# How huggingface_hub has been installed?
|
| 417 |
+
info["Installation method"] = installation_method()
|
| 418 |
+
|
| 419 |
+
# Installed dependencies
|
| 420 |
+
info["httpx"] = get_httpx_version()
|
| 421 |
+
info["hf_xet"] = get_xet_version()
|
| 422 |
+
info["gradio"] = get_gradio_version()
|
| 423 |
+
info["tensorboard"] = get_tensorboard_version()
|
| 424 |
+
|
| 425 |
+
# Environment variables
|
| 426 |
+
info["ENDPOINT"] = constants.ENDPOINT
|
| 427 |
+
info["HF_HUB_CACHE"] = constants.HF_HUB_CACHE
|
| 428 |
+
info["HF_ASSETS_CACHE"] = constants.HF_ASSETS_CACHE
|
| 429 |
+
info["HF_TOKEN_PATH"] = constants.HF_TOKEN_PATH
|
| 430 |
+
info["HF_STORED_TOKENS_PATH"] = constants.HF_STORED_TOKENS_PATH
|
| 431 |
+
info["HF_HUB_OFFLINE"] = constants.HF_HUB_OFFLINE
|
| 432 |
+
info["HF_HUB_DISABLE_TELEMETRY"] = constants.HF_HUB_DISABLE_TELEMETRY
|
| 433 |
+
info["HF_HUB_DISABLE_PROGRESS_BARS"] = constants.HF_HUB_DISABLE_PROGRESS_BARS
|
| 434 |
+
info["HF_HUB_DISABLE_SYMLINKS"] = constants.HF_HUB_DISABLE_SYMLINKS
|
| 435 |
+
info["HF_HUB_DISABLE_SYMLINKS_WARNING"] = constants.HF_HUB_DISABLE_SYMLINKS_WARNING
|
| 436 |
+
info["HF_HUB_DISABLE_EXPERIMENTAL_WARNING"] = constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING
|
| 437 |
+
info["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = constants.HF_HUB_DISABLE_IMPLICIT_TOKEN
|
| 438 |
+
info["HF_HUB_DISABLE_XET"] = constants.HF_HUB_DISABLE_XET
|
| 439 |
+
info["HF_HUB_ETAG_TIMEOUT"] = constants.HF_HUB_ETAG_TIMEOUT
|
| 440 |
+
info["HF_HUB_DOWNLOAD_TIMEOUT"] = constants.HF_HUB_DOWNLOAD_TIMEOUT
|
| 441 |
+
info["HF_XET_HIGH_PERFORMANCE"] = constants.HF_XET_HIGH_PERFORMANCE
|
| 442 |
+
|
| 443 |
+
print("\nCopy-and-paste the text below in your GitHub issue.\n")
|
| 444 |
+
print("\n".join([f"- {prop}: {val}" for prop, val in info.items()]) + "\n")
|
| 445 |
+
return info
|