Brettapps commited on
Commit
33d769f
·
verified ·
1 Parent(s): 20e1d92

Upload folder using huggingface_hub (part 8)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +10 -0
  2. .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/INSTALLER +1 -0
  3. .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/METADATA +373 -0
  4. .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/RECORD +380 -0
  5. .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/REQUESTED +0 -0
  6. .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/WHEEL +5 -0
  7. .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/entry_points.txt +7 -0
  8. .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/licenses/LICENSE +201 -0
  9. .venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/top_level.txt +1 -0
  10. .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/novita.py +69 -0
  11. .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/nscale.py +44 -0
  12. .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/openai.py +23 -0
  13. .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/ovhcloud.py +10 -0
  14. .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/publicai.py +6 -0
  15. .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/replicate.py +160 -0
  16. .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/scaleway.py +28 -0
  17. .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/together.py +291 -0
  18. .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/wavespeed.py +138 -0
  19. .venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/zai_org.py +127 -0
  20. .venv/lib/python3.13/site-packages/huggingface_hub/lfs.py +442 -0
  21. .venv/lib/python3.13/site-packages/huggingface_hub/py.typed +0 -0
  22. .venv/lib/python3.13/site-packages/huggingface_hub/repocard.py +835 -0
  23. .venv/lib/python3.13/site-packages/huggingface_hub/repocard_data.py +776 -0
  24. .venv/lib/python3.13/site-packages/huggingface_hub/serialization/__init__.py +26 -0
  25. .venv/lib/python3.13/site-packages/huggingface_hub/serialization/_base.py +212 -0
  26. .venv/lib/python3.13/site-packages/huggingface_hub/serialization/_dduf.py +386 -0
  27. .venv/lib/python3.13/site-packages/huggingface_hub/serialization/_torch.py +1046 -0
  28. .venv/lib/python3.13/site-packages/huggingface_hub/templates/datasetcard_template.md +143 -0
  29. .venv/lib/python3.13/site-packages/huggingface_hub/templates/modelcard_template.md +200 -0
  30. .venv/lib/python3.13/site-packages/huggingface_hub/utils/__init__.py +134 -0
  31. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_auth.py +456 -0
  32. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_assets.py +133 -0
  33. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_manager.py +901 -0
  34. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_chunk_utils.py +64 -0
  35. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_datetime.py +66 -0
  36. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_deprecation.py +136 -0
  37. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_detect_agent.py +204 -0
  38. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_dotenv.py +68 -0
  39. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_experimental.py +67 -0
  40. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_fixes.py +123 -0
  41. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_git_credential.py +119 -0
  42. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_headers.py +208 -0
  43. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_hf_uris.py +642 -0
  44. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_http.py +1163 -0
  45. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_lfs.py +109 -0
  46. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_oauth_device.py +197 -0
  47. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_pagination.py +52 -0
  48. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_parsing.py +116 -0
  49. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_paths.py +155 -0
  50. .venv/lib/python3.13/site-packages/huggingface_hub/utils/_runtime.py +445 -0
.gitattributes CHANGED
@@ -52,3 +52,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
52
  .venv/lib/python3.13/site-packages/ada92cb5d92a588d1b93__mypyc.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
53
  .venv/lib/python3.13/site-packages/cryptography/hazmat/bindings/_rust.abi3.so filter=lfs diff=lfs merge=lfs -text
54
  .venv/lib/python3.13/site-packages/hf_xet/hf_xet.abi3.so filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
52
  .venv/lib/python3.13/site-packages/ada92cb5d92a588d1b93__mypyc.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
53
  .venv/lib/python3.13/site-packages/cryptography/hazmat/bindings/_rust.abi3.so filter=lfs diff=lfs merge=lfs -text
54
  .venv/lib/python3.13/site-packages/hf_xet/hf_xet.abi3.so filter=lfs diff=lfs merge=lfs -text
55
+ .venv/lib/python3.13/site-packages/lxml/_elementpath.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
56
+ .venv/lib/python3.13/site-packages/lxml/builder.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
57
+ .venv/lib/python3.13/site-packages/lxml/etree.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
58
+ .venv/lib/python3.13/site-packages/lxml/html/_difflib.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
59
+ .venv/lib/python3.13/site-packages/lxml/html/diff.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
60
+ .venv/lib/python3.13/site-packages/lxml/objectify.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
61
+ .venv/lib/python3.13/site-packages/lxml/sax.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
62
+ .venv/lib/python3.13/site-packages/numpy/_core/_multiarray_tests.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
63
+ .venv/lib/python3.13/site-packages/numpy/_core/_multiarray_umath.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
64
+ .venv/lib/python3.13/site-packages/numpy/_core/_simd.cpython-313-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/INSTALLER ADDED
@@ -0,0 +1 @@
 
 
1
+ pip
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/METADATA ADDED
@@ -0,0 +1,373 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Metadata-Version: 2.4
2
+ Name: huggingface_hub
3
+ Version: 1.27.0
4
+ Summary: Client library to download and publish models, datasets and other repos on the huggingface.co hub
5
+ Home-page: https://github.com/huggingface/huggingface_hub
6
+ Author: Hugging Face, Inc.
7
+ Author-email: julien@huggingface.co
8
+ License: Apache-2.0
9
+ Keywords: model-hub machine-learning models natural-language-processing deep-learning pytorch pretrained-models
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Intended Audience :: Education
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: License :: OSI Approved :: Apache Software License
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3 :: Only
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Programming Language :: Python :: 3.14
22
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
23
+ Requires-Python: >=3.10.0
24
+ Description-Content-Type: text/markdown
25
+ License-File: LICENSE
26
+ Requires-Dist: click<9.0.0,>=8.4.2
27
+ Requires-Dist: filelock>=3.10.0
28
+ Requires-Dist: fsspec>=2023.5.0
29
+ Requires-Dist: hf-xet<2.0.0,>=1.5.2; platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "arm64" or platform_machine == "aarch64"
30
+ Requires-Dist: httpx<1,>=0.23.0
31
+ Requires-Dist: packaging>=20.9
32
+ Requires-Dist: pyyaml>=5.1
33
+ Requires-Dist: tqdm>=4.42.1
34
+ Requires-Dist: typing-extensions>=4.1.0
35
+ Provides-Extra: oauth
36
+ Requires-Dist: authlib>=1.3.2; extra == "oauth"
37
+ Requires-Dist: fastapi; extra == "oauth"
38
+ Requires-Dist: httpx; extra == "oauth"
39
+ Requires-Dist: itsdangerous; extra == "oauth"
40
+ Provides-Extra: torch
41
+ Requires-Dist: torch; extra == "torch"
42
+ Requires-Dist: safetensors[torch]; extra == "torch"
43
+ Provides-Extra: fastai
44
+ Requires-Dist: toml; extra == "fastai"
45
+ Requires-Dist: fastai>=2.4; extra == "fastai"
46
+ Requires-Dist: fastcore>=1.3.27; extra == "fastai"
47
+ Provides-Extra: hf-xet
48
+ Requires-Dist: hf-xet<2.0.0,>=1.5.2; extra == "hf-xet"
49
+ Provides-Extra: mcp
50
+ Requires-Dist: mcp>=1.8.0; extra == "mcp"
51
+ Provides-Extra: testing
52
+ Requires-Dist: authlib>=1.3.2; extra == "testing"
53
+ Requires-Dist: fastapi; extra == "testing"
54
+ Requires-Dist: httpx; extra == "testing"
55
+ Requires-Dist: itsdangerous; extra == "testing"
56
+ Requires-Dist: jedi; extra == "testing"
57
+ Requires-Dist: Jinja2; extra == "testing"
58
+ Requires-Dist: pytest>=8.4.2; extra == "testing"
59
+ Requires-Dist: pytest-cov; extra == "testing"
60
+ Requires-Dist: pytest-env; extra == "testing"
61
+ Requires-Dist: pytest-xdist; extra == "testing"
62
+ Requires-Dist: pytest-vcr; extra == "testing"
63
+ Requires-Dist: pytest-asyncio; extra == "testing"
64
+ Requires-Dist: pytest-rerunfailures>=16.2; extra == "testing"
65
+ Requires-Dist: pytest-mock; extra == "testing"
66
+ Requires-Dist: urllib3<2.0; extra == "testing"
67
+ Requires-Dist: soundfile; extra == "testing"
68
+ Requires-Dist: Pillow; extra == "testing"
69
+ Requires-Dist: numpy; extra == "testing"
70
+ Requires-Dist: duckdb; extra == "testing"
71
+ Requires-Dist: fastapi; extra == "testing"
72
+ Provides-Extra: gradio
73
+ Requires-Dist: gradio>=5.0.0; extra == "gradio"
74
+ Requires-Dist: requests; extra == "gradio"
75
+ Provides-Extra: typing
76
+ Requires-Dist: typing-extensions>=4.8.0; extra == "typing"
77
+ Requires-Dist: types-PyYAML; extra == "typing"
78
+ Requires-Dist: types-simplejson; extra == "typing"
79
+ Requires-Dist: types-toml; extra == "typing"
80
+ Requires-Dist: types-tqdm; extra == "typing"
81
+ Requires-Dist: types-urllib3; extra == "typing"
82
+ Provides-Extra: quality
83
+ Requires-Dist: ruff>=0.9.0; extra == "quality"
84
+ Requires-Dist: mypy==1.15.0; extra == "quality"
85
+ Requires-Dist: libcst>=1.4.0; extra == "quality"
86
+ Requires-Dist: ty; extra == "quality"
87
+ Provides-Extra: all
88
+ Requires-Dist: authlib>=1.3.2; extra == "all"
89
+ Requires-Dist: fastapi; extra == "all"
90
+ Requires-Dist: httpx; extra == "all"
91
+ Requires-Dist: itsdangerous; extra == "all"
92
+ Requires-Dist: jedi; extra == "all"
93
+ Requires-Dist: Jinja2; extra == "all"
94
+ Requires-Dist: pytest>=8.4.2; extra == "all"
95
+ Requires-Dist: pytest-cov; extra == "all"
96
+ Requires-Dist: pytest-env; extra == "all"
97
+ Requires-Dist: pytest-xdist; extra == "all"
98
+ Requires-Dist: pytest-vcr; extra == "all"
99
+ Requires-Dist: pytest-asyncio; extra == "all"
100
+ Requires-Dist: pytest-rerunfailures>=16.2; extra == "all"
101
+ Requires-Dist: pytest-mock; extra == "all"
102
+ Requires-Dist: urllib3<2.0; extra == "all"
103
+ Requires-Dist: soundfile; extra == "all"
104
+ Requires-Dist: Pillow; extra == "all"
105
+ Requires-Dist: numpy; extra == "all"
106
+ Requires-Dist: duckdb; extra == "all"
107
+ Requires-Dist: fastapi; extra == "all"
108
+ Requires-Dist: ruff>=0.9.0; extra == "all"
109
+ Requires-Dist: mypy==1.15.0; extra == "all"
110
+ Requires-Dist: libcst>=1.4.0; extra == "all"
111
+ Requires-Dist: ty; extra == "all"
112
+ Requires-Dist: typing-extensions>=4.8.0; extra == "all"
113
+ Requires-Dist: types-PyYAML; extra == "all"
114
+ Requires-Dist: types-simplejson; extra == "all"
115
+ Requires-Dist: types-toml; extra == "all"
116
+ Requires-Dist: types-tqdm; extra == "all"
117
+ Requires-Dist: types-urllib3; extra == "all"
118
+ Provides-Extra: dev
119
+ Requires-Dist: authlib>=1.3.2; extra == "dev"
120
+ Requires-Dist: fastapi; extra == "dev"
121
+ Requires-Dist: httpx; extra == "dev"
122
+ Requires-Dist: itsdangerous; extra == "dev"
123
+ Requires-Dist: jedi; extra == "dev"
124
+ Requires-Dist: Jinja2; extra == "dev"
125
+ Requires-Dist: pytest>=8.4.2; extra == "dev"
126
+ Requires-Dist: pytest-cov; extra == "dev"
127
+ Requires-Dist: pytest-env; extra == "dev"
128
+ Requires-Dist: pytest-xdist; extra == "dev"
129
+ Requires-Dist: pytest-vcr; extra == "dev"
130
+ Requires-Dist: pytest-asyncio; extra == "dev"
131
+ Requires-Dist: pytest-rerunfailures>=16.2; extra == "dev"
132
+ Requires-Dist: pytest-mock; extra == "dev"
133
+ Requires-Dist: urllib3<2.0; extra == "dev"
134
+ Requires-Dist: soundfile; extra == "dev"
135
+ Requires-Dist: Pillow; extra == "dev"
136
+ Requires-Dist: numpy; extra == "dev"
137
+ Requires-Dist: duckdb; extra == "dev"
138
+ Requires-Dist: fastapi; extra == "dev"
139
+ Requires-Dist: ruff>=0.9.0; extra == "dev"
140
+ Requires-Dist: mypy==1.15.0; extra == "dev"
141
+ Requires-Dist: libcst>=1.4.0; extra == "dev"
142
+ Requires-Dist: ty; extra == "dev"
143
+ Requires-Dist: typing-extensions>=4.8.0; extra == "dev"
144
+ Requires-Dist: types-PyYAML; extra == "dev"
145
+ Requires-Dist: types-simplejson; extra == "dev"
146
+ Requires-Dist: types-toml; extra == "dev"
147
+ Requires-Dist: types-tqdm; extra == "dev"
148
+ Requires-Dist: types-urllib3; extra == "dev"
149
+ Dynamic: author
150
+ Dynamic: author-email
151
+ Dynamic: classifier
152
+ Dynamic: description
153
+ Dynamic: description-content-type
154
+ Dynamic: home-page
155
+ Dynamic: keywords
156
+ Dynamic: license
157
+ Dynamic: license-file
158
+ Dynamic: provides-extra
159
+ Dynamic: requires-dist
160
+ Dynamic: requires-python
161
+ Dynamic: summary
162
+
163
+ <p align="center">
164
+ <picture>
165
+ <source media="(prefers-color-scheme: dark)" srcset="https://huggingface.co/datasets/huggingface/documentation-images/raw/main/huggingface_hub-dark.svg">
166
+ <source media="(prefers-color-scheme: light)" srcset="https://huggingface.co/datasets/huggingface/documentation-images/raw/main/huggingface_hub.svg">
167
+ <img alt="huggingface_hub library logo" src="https://huggingface.co/datasets/huggingface/documentation-images/raw/main/huggingface_hub.svg" width="352" height="59" style="max-width: 100%">
168
+ </picture>
169
+ <br/>
170
+ <br/>
171
+ </p>
172
+
173
+ <p align="center">
174
+ <i>The official CLI and Python client for the Hugging Face Hub.</i>
175
+ <br/>
176
+ <a href="#what-is-huggingface_hub">About</a>
177
+ ·
178
+ <a href="https://huggingface.co/docs/huggingface_hub">Documentation</a>
179
+ ·
180
+ <a href="https://huggingface.co/docs/huggingface_hub/en/installation">Install</a>
181
+ ·
182
+ <a href="https://huggingface.co/docs/huggingface_hub/en/guides/cli">CLI Guide</a>
183
+ ·
184
+ <a href="https://github.com/huggingface/huggingface_hub/blob/main/CONTRIBUTING.md">Contributing</a>
185
+ </p>
186
+
187
+ <p align="center">
188
+ <a href="https://huggingface.co/docs/huggingface_hub/en/index"><img alt="Documentation" src="https://img.shields.io/website/http/huggingface.co/docs/huggingface_hub/index.svg?down_color=red&down_message=offline&up_message=online&label=doc"></a>
189
+ <a href="https://github.com/huggingface/huggingface_hub/releases"><img alt="GitHub release" src="https://img.shields.io/github/release/huggingface/huggingface_hub.svg"></a>
190
+ <a href="https://github.com/huggingface/huggingface_hub"><img alt="PyPi version" src="https://img.shields.io/pypi/pyversions/huggingface_hub.svg"></a>
191
+ <a href="https://pypi.org/project/huggingface-hub"><img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/huggingface_hub"></a>
192
+ <a href="https://codecov.io/gh/huggingface/huggingface_hub"><img alt="Code coverage" src="https://codecov.io/gh/huggingface/huggingface_hub/branch/main/graph/badge.svg?token=RXP95LE2XL"></a>
193
+ </p>
194
+
195
+ <h4 align="center">
196
+ <p>
197
+ <b>English</b> |
198
+ <a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_de.md">Deutsch</a> |
199
+ <a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_fr.md">Français</a> |
200
+ <a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_hi.md">हिंदी</a> |
201
+ <a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_ko.md">한국어</a> |
202
+ <a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_cn.md">中文 (简体)</a> |
203
+ <a href="https://github.com/huggingface/huggingface_hub/blob/main/i18n/README_kn.md">ಕನ್ನಡ</a>
204
+ </p>
205
+ </h4>
206
+
207
+ ## Quick start
208
+
209
+ Install the [`hf` CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli) with the standalone installer:
210
+
211
+ ```bash
212
+ # On macOS and Linux.
213
+ curl -LsSf https://hf.co/cli/install.sh | bash
214
+ ```
215
+
216
+ ```powershell
217
+ # On Windows.
218
+ powershell -ExecutionPolicy ByPass -c "irm https://hf.co/cli/install.ps1 | iex"
219
+ ```
220
+
221
+ Log in, then start working with the Hub:
222
+
223
+ ```bash
224
+ # Log in (use --token $HF_TOKEN in non-interactive environments)
225
+ hf auth login
226
+
227
+ # Find models served by Inference Providers
228
+ hf models ls --warm
229
+
230
+ # Download a model
231
+ hf download Qwen/Qwen3-0.6B
232
+
233
+ # Upload files to your own repo
234
+ hf upload username/my-cool-model ./model.safetensors
235
+
236
+ # Sync a local folder to a storage bucket
237
+ hf buckets sync ./checkpoints hf://buckets/username/my-bucket
238
+
239
+ # Run a job on Hugging Face infrastructure
240
+ hf jobs run python:3.12 python -c "print('Hello from the cloud!')"
241
+
242
+ # Discover everything else
243
+ hf --help
244
+ ```
245
+
246
+ The Hub uses tokens to authenticate applications (see [docs](https://huggingface.co/docs/hub/security-tokens)). Check out the [CLI guide](https://huggingface.co/docs/huggingface_hub/en/guides/cli) for a tour of the main features.
247
+
248
+ ## What is `huggingface_hub`?
249
+
250
+ The `huggingface_hub` library allows you to interact with the [Hugging Face Hub](https://huggingface.co/), a platform democratizing open-source Machine Learning for creators and collaborators. Discover pre-trained models and datasets for your projects, play with the thousands of machine learning apps hosted on the Hub, or create and share your own models, datasets and demos with the community. Everything ships in one package with two interfaces: the [`hf` CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli) for your terminal and the `huggingface_hub` library for Python — both designed to work well for humans and AI agents. Use them to:
251
+
252
+ - [Download files](https://huggingface.co/docs/huggingface_hub/en/guides/download) from the Hub.
253
+ - [Upload files](https://huggingface.co/docs/huggingface_hub/en/guides/upload) to the Hub.
254
+ - [Manage your repositories](https://huggingface.co/docs/huggingface_hub/en/guides/repository).
255
+ - [Run Inference](https://huggingface.co/docs/huggingface_hub/en/guides/inference) on deployed models.
256
+ - [Run Jobs](https://huggingface.co/docs/huggingface_hub/en/guides/jobs) on Hugging Face infrastructure.
257
+ - [Search](https://huggingface.co/docs/huggingface_hub/en/guides/search) for models, datasets and Spaces.
258
+ - [Share Model Cards](https://huggingface.co/docs/huggingface_hub/en/guides/model-cards) to document your models.
259
+ - [Engage with the community](https://huggingface.co/docs/huggingface_hub/en/guides/community) through PRs and comments.
260
+ - Do all of the above from the terminal with the [`hf` CLI](https://huggingface.co/docs/huggingface_hub/en/guides/cli).
261
+
262
+ ## Built for humans and AI agents
263
+
264
+ The `hf` CLI is designed for people and coding agents alike: the same commands adapt their output when run by an agent. If you use Claude Code, Codex, Cursor, or another coding agent, install the `hf` CLI Skill — a command reference generated from your installed CLI:
265
+
266
+ ```bash
267
+ # for Codex, Cursor, OpenCode, Pi and other agents that load skills from `.agents/skills`
268
+ hf skills add
269
+ # includes the above + Claude Code
270
+ hf skills add --claude
271
+ ```
272
+
273
+ Learn more in the [Hugging Face CLI for AI agents guide](https://huggingface.co/docs/hub/agents-cli) and the [announcement blog post](https://huggingface.co/blog/hf-cli-for-agents).
274
+
275
+ ## Use the Python library
276
+
277
+ Install the `huggingface_hub` package with [pip](https://pypi.org/project/huggingface-hub/) (this also installs the `hf` CLI):
278
+
279
+ ```bash
280
+ pip install huggingface_hub
281
+ ```
282
+
283
+ We recommend using [`uv`](https://docs.astral.sh/uv/) for a fast and reliable install:
284
+
285
+ ```bash
286
+ uv pip install huggingface_hub
287
+ ```
288
+
289
+ In order to keep the package minimal by default, `huggingface_hub` comes with optional dependencies useful for some use cases. For example, if you want to use the MCP module, run:
290
+
291
+ ```bash
292
+ pip install "huggingface_hub[mcp]"
293
+ ```
294
+
295
+ To learn more about installation and optional dependencies, check out the [installation guide](https://huggingface.co/docs/huggingface_hub/en/installation).
296
+
297
+ ### Download files
298
+
299
+ Download a single file
300
+
301
+ ```py
302
+ from huggingface_hub import hf_hub_download
303
+
304
+ hf_hub_download(repo_id="zai-org/GLM-5.2", filename="config.json")
305
+ ```
306
+
307
+ Or an entire repository
308
+
309
+ ```py
310
+ from huggingface_hub import snapshot_download
311
+
312
+ snapshot_download("sentence-transformers/all-MiniLM-L6-v2")
313
+ ```
314
+
315
+ Files will be downloaded in a local cache folder. More details in [this guide](https://huggingface.co/docs/huggingface_hub/en/guides/manage-cache).
316
+
317
+ ### Create a repository
318
+
319
+ ```py
320
+ from huggingface_hub import create_repo
321
+
322
+ create_repo(repo_id="super-cool-model")
323
+ ```
324
+
325
+ ### Upload files
326
+
327
+ Upload a single file
328
+
329
+ ```py
330
+ from huggingface_hub import upload_file
331
+
332
+ upload_file(
333
+ path_or_fileobj="/home/lysandre/dummy-test/README.md",
334
+ path_in_repo="README.md",
335
+ repo_id="lysandre/test-model",
336
+ )
337
+ ```
338
+
339
+ Or an entire folder
340
+
341
+ ```py
342
+ from huggingface_hub import upload_folder
343
+
344
+ upload_folder(
345
+ folder_path="/path/to/local/space",
346
+ repo_id="username/my-cool-space",
347
+ repo_type="space",
348
+ )
349
+ ```
350
+
351
+ More details in the [upload guide](https://huggingface.co/docs/huggingface_hub/en/guides/upload).
352
+
353
+ ## Integrating with the Hub.
354
+
355
+ We're partnering with cool open source ML libraries to provide free model hosting and versioning. You can find the existing integrations [here](https://huggingface.co/docs/hub/libraries).
356
+
357
+ The advantages are:
358
+
359
+ - Free model or dataset hosting for libraries and their users.
360
+ - Built-in file versioning, even with very large files, made possible by [Xet](https://huggingface.co/docs/hub/xet/index), the Hub's chunk-deduplicated storage backend.
361
+ - In-browser widgets to play with the uploaded models.
362
+ - Anyone can upload a new model for your library, they just need to add the corresponding tag for the model to be discoverable.
363
+ - Fast downloads! We use Cloudfront (a CDN) to geo-replicate downloads so they're blazing fast from anywhere on the globe.
364
+ - Usage stats and more features to come.
365
+
366
+ If you would like to integrate your library, feel free to open an issue to begin the discussion. We wrote a [step-by-step guide](https://huggingface.co/docs/hub/adding-a-library) with ❤️ showing how to do this integration.
367
+
368
+ ## Contributions (feature requests, bugs, etc.) are super welcome 💙💚💛💜🧡❤️
369
+
370
+ Everyone is welcome to contribute, and we value everybody's contribution. Code is not the only way to help the community.
371
+ Answering questions, helping others, reaching out and improving the documentations are immensely valuable to the community.
372
+ We wrote a [contribution guide](https://github.com/huggingface/huggingface_hub/blob/main/CONTRIBUTING.md) to summarize
373
+ how to get started to contribute to this repository.
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/RECORD ADDED
@@ -0,0 +1,380 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ../../../bin/hf,sha256=JQijLHciXBpOWGr6WRb8N0z29GfP0tPh2NA2HNavJvo,273
2
+ ../../../bin/huggingface-cli,sha256=nNpWnVatSJHRXWFCuXQb_WZFtQ_HulgkcuNiCSf64xo,285
3
+ ../../../bin/tiny-agents,sha256=0CQNfsvoFKWfo0ngJRPM94Fb2eOQaPFL-J1-Oblf3iA,283
4
+ huggingface_hub-1.27.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
5
+ huggingface_hub-1.27.0.dist-info/METADATA,sha256=kdWT-K9bIzIoJpph_rdX1f5gcwxRWXkZ5PTijS2m2co,16314
6
+ huggingface_hub-1.27.0.dist-info/RECORD,,
7
+ huggingface_hub-1.27.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ huggingface_hub-1.27.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
9
+ huggingface_hub-1.27.0.dist-info/entry_points.txt,sha256=zP7F_bBSdircPQFysHQZ9F3Lcn5_dCSOEZxVlGCsG0w,212
10
+ huggingface_hub-1.27.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
11
+ huggingface_hub-1.27.0.dist-info/top_level.txt,sha256=8KzlQJAY4miUvjAssOAJodqKOw3harNzuiwGQ9qLSSk,16
12
+ huggingface_hub/__init__.py,sha256=eZLm_W9SknOwodC709pz-SVyWudPEfO0-yrtIlL1FMc,60623
13
+ huggingface_hub/__pycache__/__init__.cpython-313.pyc,,
14
+ huggingface_hub/__pycache__/_buckets.cpython-313.pyc,,
15
+ huggingface_hub/__pycache__/_commit_api.cpython-313.pyc,,
16
+ huggingface_hub/__pycache__/_commit_scheduler.cpython-313.pyc,,
17
+ huggingface_hub/__pycache__/_dataset_viewer.cpython-313.pyc,,
18
+ huggingface_hub/__pycache__/_eval_results.cpython-313.pyc,,
19
+ huggingface_hub/__pycache__/_inference_endpoints.cpython-313.pyc,,
20
+ huggingface_hub/__pycache__/_jobs_api.cpython-313.pyc,,
21
+ huggingface_hub/__pycache__/_local_folder.cpython-313.pyc,,
22
+ huggingface_hub/__pycache__/_login.cpython-313.pyc,,
23
+ huggingface_hub/__pycache__/_oauth.cpython-313.pyc,,
24
+ huggingface_hub/__pycache__/_oidc.cpython-313.pyc,,
25
+ huggingface_hub/__pycache__/_revision.cpython-313.pyc,,
26
+ huggingface_hub/__pycache__/_sandbox.cpython-313.pyc,,
27
+ huggingface_hub/__pycache__/_sandbox_cache.cpython-313.pyc,,
28
+ huggingface_hub/__pycache__/_snapshot_download.cpython-313.pyc,,
29
+ huggingface_hub/__pycache__/_space_api.cpython-313.pyc,,
30
+ huggingface_hub/__pycache__/_tensorboard_logger.cpython-313.pyc,,
31
+ huggingface_hub/__pycache__/_tree_cache.cpython-313.pyc,,
32
+ huggingface_hub/__pycache__/_upload_large_folder.cpython-313.pyc,,
33
+ huggingface_hub/__pycache__/_upload_pipeline.cpython-313.pyc,,
34
+ huggingface_hub/__pycache__/_webhooks_payload.cpython-313.pyc,,
35
+ huggingface_hub/__pycache__/_webhooks_server.cpython-313.pyc,,
36
+ huggingface_hub/__pycache__/community.cpython-313.pyc,,
37
+ huggingface_hub/__pycache__/constants.cpython-313.pyc,,
38
+ huggingface_hub/__pycache__/dataclasses.cpython-313.pyc,,
39
+ huggingface_hub/__pycache__/errors.cpython-313.pyc,,
40
+ huggingface_hub/__pycache__/fastai_utils.cpython-313.pyc,,
41
+ huggingface_hub/__pycache__/file_download.cpython-313.pyc,,
42
+ huggingface_hub/__pycache__/hf_api.cpython-313.pyc,,
43
+ huggingface_hub/__pycache__/hf_file_system.cpython-313.pyc,,
44
+ huggingface_hub/__pycache__/hub_mixin.cpython-313.pyc,,
45
+ huggingface_hub/__pycache__/lfs.cpython-313.pyc,,
46
+ huggingface_hub/__pycache__/repocard.cpython-313.pyc,,
47
+ huggingface_hub/__pycache__/repocard_data.cpython-313.pyc,,
48
+ huggingface_hub/_buckets.py,sha256=XyLbsLzS2fpQz0aAIjKzV-1YbQ4XYZ4-HXQTVq0TXoU,45311
49
+ huggingface_hub/_commit_api.py,sha256=lalqhJKl8_RWVHgAdnzZKG_fy4E7fbyHIev6nDLrj4w,45378
50
+ huggingface_hub/_commit_scheduler.py,sha256=VG3zjb7xb66NrFMq_pB9dWIsoqSPoF6CntovV7vUiWg,14684
51
+ huggingface_hub/_dataset_viewer.py,sha256=LE9k4s0WJ0JNdriJ8L6qV_-xfbUVC1VEh7CBd0jOjLI,5675
52
+ huggingface_hub/_eval_results.py,sha256=Og7p7bCl2HQG6ZWlSDyir-CRTXjNWPu3S3q65cTGffs,8126
53
+ huggingface_hub/_hot_reload/__init__.py,sha256=nQEuyVZLAS_463Dff1JH5MorTpkYsBVFCyPhYWY3UyI,606
54
+ huggingface_hub/_hot_reload/__pycache__/__init__.cpython-313.pyc,,
55
+ huggingface_hub/_hot_reload/__pycache__/client.cpython-313.pyc,,
56
+ huggingface_hub/_hot_reload/__pycache__/sse_client.cpython-313.pyc,,
57
+ huggingface_hub/_hot_reload/__pycache__/types.cpython-313.pyc,,
58
+ huggingface_hub/_hot_reload/client.py,sha256=Ax7qT4CUi5wrg3mDzqkAG8ZzDsCwdcIIS1FIXK0SqD8,4269
59
+ huggingface_hub/_hot_reload/sse_client.py,sha256=49kmsB2P9sxgbZ16ed6F4aLDs3rsHrNpbp6iKlvF90c,4948
60
+ huggingface_hub/_hot_reload/types.py,sha256=d3AOJCLUxT41Fhz4jUrFevm9bfpooe_3PQhpBdgNYSs,2725
61
+ huggingface_hub/_inference_endpoints.py,sha256=FWSixNvBQlqLU4Z0yUC7Ch7a8V8Ci6eJBAD4pbTMKhk,18424
62
+ huggingface_hub/_jobs_api.py,sha256=f7NhziUMH6jWpWClnLPFx4XIZ0xVGAdTGfBxIn2IVy0,23679
63
+ huggingface_hub/_local_folder.py,sha256=KsetQw7pW0ZwFI2RLQPqVq0ZnORogPkBwRYcbRqNd5o,20654
64
+ huggingface_hub/_login.py,sha256=wAMdhqMSSh3GBJDozodmk4IIsAvb9J6LaLY9KnXFoyY,21818
65
+ huggingface_hub/_oauth.py,sha256=S8evEd8JLJHz-JiSjGH4WiIi_A7oDox_xH7E9rN1Jic,18628
66
+ huggingface_hub/_oidc.py,sha256=3kcCLavWQ5FQOwizUPzLaBb1ft2FIJo0bSXL24S3r4U,7098
67
+ huggingface_hub/_revision.py,sha256=NCmIVCR-_HjmxeYiDntcsTU-bLSfJLq8PLvvG8UhZtI,1696
68
+ huggingface_hub/_sandbox.py,sha256=XEXZi5S3rYyw_o3td3rB0xOVdmhpAlnyUykBsAw9KP8,80148
69
+ huggingface_hub/_sandbox_cache.py,sha256=UYpvHOWZj7CRhacYs5Ot-5N1wFcAapiAyniYdoX-jX8,5742
70
+ huggingface_hub/_snapshot_download.py,sha256=fOte9xmpaok0LGbubJZhSRd7t0nn-p00mWMebCx56qc,31029
71
+ huggingface_hub/_space_api.py,sha256=MF3ezhgl0Vpt3RP1wuwZ7VI9_rSyIlIJ5SmDzIwQlcQ,13424
72
+ huggingface_hub/_tensorboard_logger.py,sha256=GA-LhoG4Z-0Zq7w5B9vTIDJbWUkAhiG2d8a-ktlwGDY,8377
73
+ huggingface_hub/_tree_cache.py,sha256=M7CHMKhwY1cuhe9fb3WT1X8jmd9BXa2Sx_HS4TIXFgI,5843
74
+ huggingface_hub/_upload_large_folder.py,sha256=BWA1zyOK9W_3YUerX9YWQa3wAYtHRe_ypgsO79JEkqo,31235
75
+ huggingface_hub/_upload_pipeline.py,sha256=6YaQNtubAEaPyRoRBjPQrmsdSnCExg2KzfBuhjNqI5I,28994
76
+ huggingface_hub/_webhooks_payload.py,sha256=B9zqljcm20DGeVxJKba570i6RP-cR9CEupMGBuNDHHA,3544
77
+ huggingface_hub/_webhooks_server.py,sha256=VXOk3FHkbIc28phxknJfaFy5B_EoKOKihoCKyOaWZQk,15767
78
+ huggingface_hub/cli/__init__.py,sha256=tWu3Is-m2tlJwXhM_vT2mPRgpnt9KhU9qvw5aRsnjU0,664
79
+ huggingface_hub/cli/__pycache__/__init__.cpython-313.pyc,,
80
+ huggingface_hub/cli/__pycache__/_city_game.cpython-313.pyc,,
81
+ huggingface_hub/cli/__pycache__/_cli_utils.cpython-313.pyc,,
82
+ huggingface_hub/cli/__pycache__/_completion.cpython-313.pyc,,
83
+ huggingface_hub/cli/__pycache__/_cp.cpython-313.pyc,,
84
+ huggingface_hub/cli/__pycache__/_errors.cpython-313.pyc,,
85
+ huggingface_hub/cli/__pycache__/_file_listing.cpython-313.pyc,,
86
+ huggingface_hub/cli/__pycache__/_framework.cpython-313.pyc,,
87
+ huggingface_hub/cli/__pycache__/_help_formatter.cpython-313.pyc,,
88
+ huggingface_hub/cli/__pycache__/_output.cpython-313.pyc,,
89
+ huggingface_hub/cli/__pycache__/_skills.cpython-313.pyc,,
90
+ huggingface_hub/cli/__pycache__/auth.cpython-313.pyc,,
91
+ huggingface_hub/cli/__pycache__/buckets.cpython-313.pyc,,
92
+ huggingface_hub/cli/__pycache__/cache.cpython-313.pyc,,
93
+ huggingface_hub/cli/__pycache__/collections.cpython-313.pyc,,
94
+ huggingface_hub/cli/__pycache__/datasets.cpython-313.pyc,,
95
+ huggingface_hub/cli/__pycache__/deprecated_cli.cpython-313.pyc,,
96
+ huggingface_hub/cli/__pycache__/discussions.cpython-313.pyc,,
97
+ huggingface_hub/cli/__pycache__/download.cpython-313.pyc,,
98
+ huggingface_hub/cli/__pycache__/extensions.cpython-313.pyc,,
99
+ huggingface_hub/cli/__pycache__/hf.cpython-313.pyc,,
100
+ huggingface_hub/cli/__pycache__/inference_endpoints.cpython-313.pyc,,
101
+ huggingface_hub/cli/__pycache__/jobs.cpython-313.pyc,,
102
+ huggingface_hub/cli/__pycache__/lfs.cpython-313.pyc,,
103
+ huggingface_hub/cli/__pycache__/models.cpython-313.pyc,,
104
+ huggingface_hub/cli/__pycache__/papers.cpython-313.pyc,,
105
+ huggingface_hub/cli/__pycache__/repo_files.cpython-313.pyc,,
106
+ huggingface_hub/cli/__pycache__/repos.cpython-313.pyc,,
107
+ huggingface_hub/cli/__pycache__/sandbox.cpython-313.pyc,,
108
+ huggingface_hub/cli/__pycache__/skills.cpython-313.pyc,,
109
+ huggingface_hub/cli/__pycache__/spaces.cpython-313.pyc,,
110
+ huggingface_hub/cli/__pycache__/system.cpython-313.pyc,,
111
+ huggingface_hub/cli/__pycache__/upload.cpython-313.pyc,,
112
+ huggingface_hub/cli/__pycache__/upload_large_folder.cpython-313.pyc,,
113
+ huggingface_hub/cli/__pycache__/webhooks.cpython-313.pyc,,
114
+ huggingface_hub/cli/_city_game.py,sha256=2qkC9dCtPwf4-aqrnbN-vApf0YqIcGGjn2v-etrTfqg,20585
115
+ huggingface_hub/cli/_cli_utils.py,sha256=uFCEHDWmxTri9ei1kmN4t4ybdm4wO5Cj0e6DvLWWFao,41824
116
+ huggingface_hub/cli/_completion.py,sha256=BDP6Vob9TLFp6K3lr0XOY98FyD-rm22KrjwsO_YGNbE,3759
117
+ huggingface_hub/cli/_cp.py,sha256=Z-2Z_tBs1e4XBqRO5R9QnabU4qTi9HDlbA52jTBaCkY,10436
118
+ huggingface_hub/cli/_errors.py,sha256=X0rUTO-3eCK5v_MLGEza1BxcuXGccPY-h7ShzmuQc34,5907
119
+ huggingface_hub/cli/_file_listing.py,sha256=h_dTEphSqtvknzqJ_9cIK3Cjlz5M6cycYD3cqUW9Fn0,7986
120
+ huggingface_hub/cli/_framework.py,sha256=HqfAEYGWRSjKABAZi2zVNgryyU9QKW8u-tRXfR_4nQ8,27364
121
+ huggingface_hub/cli/_help_formatter.py,sha256=-vsa3HfHQFkKWms77EGyp9i22t44ETmVt2L-e8qP-KM,1288
122
+ huggingface_hub/cli/_output.py,sha256=OxFrvAcDU1TRDV0AbNh4EI1P6tqifSeVT-yqfF92U_0,14754
123
+ huggingface_hub/cli/_skills.py,sha256=s9Q4o4AIerDyLNuGzUB1ISXANQPgSiAs2CRcAY4NckA,15135
124
+ huggingface_hub/cli/auth.py,sha256=Ew79M60pTMCnS1ewKVK3fd8ZyfHrNjREOmNWuFXy37M,6056
125
+ huggingface_hub/cli/buckets.py,sha256=J8xMP88fQ6ri3f65Z8XEImI2GNA4KKWloLJJQCBezZw,21062
126
+ huggingface_hub/cli/cache.py,sha256=2T0oTYXsrtcJxhQENSpXnYYNUkagsvvdrf5gbCACBjA,28574
127
+ huggingface_hub/cli/collections.py,sha256=Ed41zJqrvK-AWkHdkeHWm_aJKR_ZeNIovCGvWG04itk,10120
128
+ huggingface_hub/cli/datasets.py,sha256=X1xMJTFhHH5NJtmjTS6NN1g4ffai6Abh_obDEJM5mvo,10497
129
+ huggingface_hub/cli/deprecated_cli.py,sha256=vAK_CR4NE9uzNJ0I59SyBnIln5nlypG4CPoohMoPMVg,1090
130
+ huggingface_hub/cli/discussions.py,sha256=obvvN4mtsFnfMs67YTZaMvHkVaM6A-FYr5n-wGkF_jE,14029
131
+ huggingface_hub/cli/download.py,sha256=o6fMKJoNQMxCm1RQUnX3blihJ17mantxkf0F68rZIxs,9669
132
+ huggingface_hub/cli/extensions.py,sha256=qNM1DmzcQXt8kPWQfISc8Pvyku8BxAs-BOVP6sKRBOM,25572
133
+ huggingface_hub/cli/hf.py,sha256=84sV7zWF5SB7Q48FMXpA9XwKsMFGaCAzNklrkV7l-vQ,5549
134
+ huggingface_hub/cli/inference_endpoints.py,sha256=F-td5VCPrzgBZbkRwc6HhasJDrJDbLmreHoe4nTgktA,18820
135
+ huggingface_hub/cli/jobs.py,sha256=EVGuxn2GZqkQolHib98PztJeJ64Opq-MjgIub2WTYb4,52840
136
+ huggingface_hub/cli/lfs.py,sha256=MAccazVPX0n-ySRaPocaQUHP7oBnVrvWZt0RhmNIavs,5936
137
+ huggingface_hub/cli/models.py,sha256=eG7H5yDjCVrdL7W_-wpcl70qXlv8bC-roSEu8AroPws,10148
138
+ huggingface_hub/cli/papers.py,sha256=HvT-8zj2fi14ua6awWpHphlbJ362FEJIwSCBJRQZ_cM,4982
139
+ huggingface_hub/cli/repo_files.py,sha256=sGbD5wND3miF71ThfOBJm4xzPRUpJBeLFdvyj229NAM,2443
140
+ huggingface_hub/cli/repos.py,sha256=QS3xYKvspLhCfO5VvpORrSiF6XhSVFclmBxsacVDP2c,20192
141
+ huggingface_hub/cli/sandbox.py,sha256=rTCZf6WhoDa7UiNhsTU_4Lh5uKqKYOs73xXGAAqJGdA,18379
142
+ huggingface_hub/cli/skills.py,sha256=PcZsDiB8XC_-98c9Pw5upmof4FAAlMQPZUmRT0T8kCQ,19926
143
+ huggingface_hub/cli/spaces.py,sha256=AU655GGrQmF1s1yO8yO6ml2Z8veE7zsh4UMuR5b13RI,40800
144
+ huggingface_hub/cli/system.py,sha256=8_xaa-GLGp5tC-slv7cuFnpUQT3zozevsftYGlvFTgs,3071
145
+ huggingface_hub/cli/upload.py,sha256=P6sq8Dj81a2wazUgZEqbliRleVAAmJrf8ajFDDzHrBE,11318
146
+ huggingface_hub/cli/upload_large_folder.py,sha256=A-2qxAdLrYd8IogfZS2nDhSR1cpA_aaxhXst638WkAk,4081
147
+ huggingface_hub/cli/webhooks.py,sha256=HdpPp01c6Ece_EuNNJr46rnRki3vF4j4o9-SqYRUHnM,8550
148
+ huggingface_hub/community.py,sha256=N5OE5Rbc2BMwpNXCFOPQAwn0otqQunsXdEnSE_0J2Xo,12325
149
+ huggingface_hub/constants.py,sha256=leJzcvH7fkoIpb8LLLd21gB_j71AWSonum0MRV18a6c,13792
150
+ huggingface_hub/dataclasses.py,sha256=dkPHOKq1YVlto9TWMkeX8BnErmbZk6tVtqD1S_n6Cw8,25896
151
+ huggingface_hub/errors.py,sha256=MsfOFq6Sxo2bZ8PTPWbQb250v7aqZJO5-mmWSnX_kDk,18695
152
+ huggingface_hub/fastai_utils.py,sha256=jwKeVXH_VP0zxnOKVNQCVCkF-stH70l-bExbbrYkBKQ,16583
153
+ huggingface_hub/file_download.py,sha256=3IPbCapgC-tvlh9ErLCzA-RerYXR_1XIhDKvUOjHvKA,89165
154
+ huggingface_hub/hf_api.py,sha256=RNpf-DpBdBQQ5-lYjB8APlRaCOIcSK4lWUTMkvQiPcA,673285
155
+ huggingface_hub/hf_file_system.py,sha256=3_tdhCz6IaLVbcfk_tpbQUd7VfndHyOmbI_6q4Lq36o,60869
156
+ huggingface_hub/hub_mixin.py,sha256=ljKAtm3qQsY-ARtxY-PCFx0koFXLI37JytMrzpqfnQo,36912
157
+ huggingface_hub/inference/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
158
+ huggingface_hub/inference/__pycache__/__init__.cpython-313.pyc,,
159
+ huggingface_hub/inference/__pycache__/_client.cpython-313.pyc,,
160
+ huggingface_hub/inference/__pycache__/_common.cpython-313.pyc,,
161
+ huggingface_hub/inference/_client.py,sha256=mENW3_gqgJR_9PuC7dwFZYjkQETPvrIrw8rU74PeusQ,157692
162
+ huggingface_hub/inference/_common.py,sha256=X-fBP8s6xuNTAuQsGVIethE8dNQaa-mK7Ds54qVVDz8,14999
163
+ huggingface_hub/inference/_generated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
164
+ huggingface_hub/inference/_generated/__pycache__/__init__.cpython-313.pyc,,
165
+ huggingface_hub/inference/_generated/__pycache__/_async_client.cpython-313.pyc,,
166
+ huggingface_hub/inference/_generated/_async_client.py,sha256=MQNKNIqcgqpGvL-HskuX1yJV6QDDuUEjM0mtZAZv8ss,160903
167
+ huggingface_hub/inference/_generated/types/__init__.py,sha256=e625ENOmTUM4uUL1eAlsZOK7CovNoGBDH-TxSNQhaC4,6869
168
+ huggingface_hub/inference/_generated/types/__pycache__/__init__.cpython-313.pyc,,
169
+ huggingface_hub/inference/_generated/types/__pycache__/audio_classification.cpython-313.pyc,,
170
+ huggingface_hub/inference/_generated/types/__pycache__/audio_to_audio.cpython-313.pyc,,
171
+ huggingface_hub/inference/_generated/types/__pycache__/automatic_speech_recognition.cpython-313.pyc,,
172
+ huggingface_hub/inference/_generated/types/__pycache__/base.cpython-313.pyc,,
173
+ huggingface_hub/inference/_generated/types/__pycache__/chat_completion.cpython-313.pyc,,
174
+ huggingface_hub/inference/_generated/types/__pycache__/depth_estimation.cpython-313.pyc,,
175
+ huggingface_hub/inference/_generated/types/__pycache__/document_question_answering.cpython-313.pyc,,
176
+ huggingface_hub/inference/_generated/types/__pycache__/feature_extraction.cpython-313.pyc,,
177
+ huggingface_hub/inference/_generated/types/__pycache__/fill_mask.cpython-313.pyc,,
178
+ huggingface_hub/inference/_generated/types/__pycache__/image_classification.cpython-313.pyc,,
179
+ huggingface_hub/inference/_generated/types/__pycache__/image_segmentation.cpython-313.pyc,,
180
+ huggingface_hub/inference/_generated/types/__pycache__/image_text_to_image.cpython-313.pyc,,
181
+ huggingface_hub/inference/_generated/types/__pycache__/image_text_to_video.cpython-313.pyc,,
182
+ huggingface_hub/inference/_generated/types/__pycache__/image_to_image.cpython-313.pyc,,
183
+ huggingface_hub/inference/_generated/types/__pycache__/image_to_text.cpython-313.pyc,,
184
+ huggingface_hub/inference/_generated/types/__pycache__/image_to_video.cpython-313.pyc,,
185
+ huggingface_hub/inference/_generated/types/__pycache__/object_detection.cpython-313.pyc,,
186
+ huggingface_hub/inference/_generated/types/__pycache__/question_answering.cpython-313.pyc,,
187
+ huggingface_hub/inference/_generated/types/__pycache__/sentence_similarity.cpython-313.pyc,,
188
+ huggingface_hub/inference/_generated/types/__pycache__/summarization.cpython-313.pyc,,
189
+ huggingface_hub/inference/_generated/types/__pycache__/table_question_answering.cpython-313.pyc,,
190
+ huggingface_hub/inference/_generated/types/__pycache__/text2text_generation.cpython-313.pyc,,
191
+ huggingface_hub/inference/_generated/types/__pycache__/text_classification.cpython-313.pyc,,
192
+ huggingface_hub/inference/_generated/types/__pycache__/text_generation.cpython-313.pyc,,
193
+ huggingface_hub/inference/_generated/types/__pycache__/text_to_audio.cpython-313.pyc,,
194
+ huggingface_hub/inference/_generated/types/__pycache__/text_to_image.cpython-313.pyc,,
195
+ huggingface_hub/inference/_generated/types/__pycache__/text_to_speech.cpython-313.pyc,,
196
+ huggingface_hub/inference/_generated/types/__pycache__/text_to_video.cpython-313.pyc,,
197
+ huggingface_hub/inference/_generated/types/__pycache__/token_classification.cpython-313.pyc,,
198
+ huggingface_hub/inference/_generated/types/__pycache__/translation.cpython-313.pyc,,
199
+ huggingface_hub/inference/_generated/types/__pycache__/video_classification.cpython-313.pyc,,
200
+ huggingface_hub/inference/_generated/types/__pycache__/visual_question_answering.cpython-313.pyc,,
201
+ huggingface_hub/inference/_generated/types/__pycache__/zero_shot_classification.cpython-313.pyc,,
202
+ huggingface_hub/inference/_generated/types/__pycache__/zero_shot_image_classification.cpython-313.pyc,,
203
+ huggingface_hub/inference/_generated/types/__pycache__/zero_shot_object_detection.cpython-313.pyc,,
204
+ huggingface_hub/inference/_generated/types/audio_classification.py,sha256=DGWEvrRPX7xpXlbYY4LMojGcULdEZFTshhNR81rTNcQ,1567
205
+ huggingface_hub/inference/_generated/types/audio_to_audio.py,sha256=2Ep4WkePL7oJwcp5nRJqApwviumGHbft9HhXE9XLHj4,891
206
+ huggingface_hub/inference/_generated/types/automatic_speech_recognition.py,sha256=X0uCj9qwAcOmlKtV3vWH5cAgLGY0PuYIy2eSH7GE1qs,5439
207
+ huggingface_hub/inference/_generated/types/base.py,sha256=EPWp1qudjeEimEAyQ-ptdxpAACoRM8tW342-Ds7pP3I,6938
208
+ huggingface_hub/inference/_generated/types/chat_completion.py,sha256=3yc06ZI0NCZzsaeoxSMWyIHqGc8Vn_6aExncuXa1llE,11100
209
+ huggingface_hub/inference/_generated/types/depth_estimation.py,sha256=5Kn4xNGJg235Cofr1lzO59eRGCuLTZun8z5t6a2do8Q,910
210
+ huggingface_hub/inference/_generated/types/document_question_answering.py,sha256=aUZcrDGexjw8Fnm2rACm_3my77V15NnQHZxj-sa7bjo,3146
211
+ huggingface_hub/inference/_generated/types/feature_extraction.py,sha256=DiO47PqwhdNta_KPDBsXfCPIL6BWNhyCLymxORlqUA0,1509
212
+ huggingface_hub/inference/_generated/types/fill_mask.py,sha256=NhEskgoiMst7U_0cV_S-wryga4_2T5LgpbyQ23vHMp0,1680
213
+ huggingface_hub/inference/_generated/types/image_classification.py,sha256=xWr4OY65EcIIhd7MI610a7xlaAVHQeydi8Gc7tlaPTU,1579
214
+ huggingface_hub/inference/_generated/types/image_segmentation.py,sha256=lZKzbn_gkSi1sYGLb9ryeRr0aDAVrfYh0H5Iiv5JNsY,1935
215
+ huggingface_hub/inference/_generated/types/image_text_to_image.py,sha256=pRQgYKXJbIpEjRBxuIE6NRDtk_km9DSTQ2s2qxkYJRM,2585
216
+ huggingface_hub/inference/_generated/types/image_text_to_video.py,sha256=MVhP8D4eYjpT3hWYkkpuiME-hWY09o7C4rIn_wmnAmY,2450
217
+ huggingface_hub/inference/_generated/types/image_to_image.py,sha256=P9TeMFSnZI_pa6DyinzDPWL0PNm_INJwsqFUEzpwIcU,2262
218
+ huggingface_hub/inference/_generated/types/image_to_text.py,sha256=DCDS0Nqi-4AuATitvZm4EgrzYQuQEQ5lzBcaHuCmHnY,4740
219
+ huggingface_hub/inference/_generated/types/image_to_video.py,sha256=iMCjiFTUza9AW_MyJ1videhYgvMY9QSzQ59kNdTE9QY,2206
220
+ huggingface_hub/inference/_generated/types/object_detection.py,sha256=zNILlyZJh59rNNfqXM5x9uprDCPfEU-bvBtabeKSMzI,1965
221
+ huggingface_hub/inference/_generated/types/question_answering.py,sha256=lIRf8DjRiAsjzWUBEsGt354SB4bvejLokf-8luHw6_o,2845
222
+ huggingface_hub/inference/_generated/types/sentence_similarity.py,sha256=eNGiO4dR-Lft9yOLE0E97jfVyOT_ExDWZW5GeJGdatU,1027
223
+ huggingface_hub/inference/_generated/types/summarization.py,sha256=5Dpt6LRteT-hO9esa3Ty_-xSL5U-zTtVZzy3iKwZRng,1472
224
+ huggingface_hub/inference/_generated/types/table_question_answering.py,sha256=bwn616XJ-eLa5JKzn3m9rZZ3z63INGEUwRSOCWlEh-4,2269
225
+ huggingface_hub/inference/_generated/types/text2text_generation.py,sha256=fotSlPWtIlp_LG26AQvNRXxU-jCUfda5PWsxCkbFeKM,1591
226
+ huggingface_hub/inference/_generated/types/text_classification.py,sha256=uamBzz4YzWQ9mb6yuriykX8r47ZHf0K3eJpOLzB2qJw,1439
227
+ huggingface_hub/inference/_generated/types/text_generation.py,sha256=FRUljypvCSQ2RaiylhlHeLJfHNy8qZvuty6SCB_zcaA,5813
228
+ huggingface_hub/inference/_generated/types/text_to_audio.py,sha256=CiaTUSBdFtOs3gGTosqyQrjCwrN8mVJLRxDDWUtwZpQ,4677
229
+ huggingface_hub/inference/_generated/types/text_to_image.py,sha256=Ju25WIDHE-1n8jlZ_udG4rpAaPYpmghR7-cDgVfuf48,1869
230
+ huggingface_hub/inference/_generated/types/text_to_speech.py,sha256=WqDzWkhZsDcjapkoFrkyjF903SKs7aHhjypsYrn8oBw,4693
231
+ huggingface_hub/inference/_generated/types/text_to_video.py,sha256=KjcDbtChwrIxIdTH6vcesUaMcjL2xjPf2nWEnHazj4Y,1756
232
+ huggingface_hub/inference/_generated/types/token_classification.py,sha256=8KRBlqgFDszaUnPMvdRkZNQaZBtDfBhKZ6SggQldQMM,1894
233
+ huggingface_hub/inference/_generated/types/translation.py,sha256=-j2PC1-qfoxvGMaKesqf5jXelqj0BDlEpfvZ7dsIEJM,1742
234
+ huggingface_hub/inference/_generated/types/video_classification.py,sha256=vfaCqp8MuRMMtGvYCX0x4-AcwEd71kPHgSYJTNIOxLQ,1668
235
+ huggingface_hub/inference/_generated/types/visual_question_answering.py,sha256=sWIB-SuEpW21oU6O5qQF3tt5PavUFsFOa8Xg4upTtek,1654
236
+ huggingface_hub/inference/_generated/types/zero_shot_classification.py,sha256=gx-7WHad7xsNa89iSKSteKHjaHrKbIyzN_nSdJN3HCY,1697
237
+ huggingface_hub/inference/_generated/types/zero_shot_image_classification.py,sha256=JZTQvcbSGrOQlWcPuJvAMT8YziDpBRn1O2RQSw507uY,1449
238
+ huggingface_hub/inference/_generated/types/zero_shot_object_detection.py,sha256=sjdpVUN5zW9aYBymLVUs6i5HVk2qkUBO9ysEjHmsXVM,1605
239
+ huggingface_hub/inference/_mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
240
+ huggingface_hub/inference/_mcp/__pycache__/__init__.cpython-313.pyc,,
241
+ huggingface_hub/inference/_mcp/__pycache__/_cli_hacks.cpython-313.pyc,,
242
+ huggingface_hub/inference/_mcp/__pycache__/agent.cpython-313.pyc,,
243
+ huggingface_hub/inference/_mcp/__pycache__/cli.cpython-313.pyc,,
244
+ huggingface_hub/inference/_mcp/__pycache__/constants.cpython-313.pyc,,
245
+ huggingface_hub/inference/_mcp/__pycache__/mcp_client.cpython-313.pyc,,
246
+ huggingface_hub/inference/_mcp/__pycache__/types.cpython-313.pyc,,
247
+ huggingface_hub/inference/_mcp/__pycache__/utils.cpython-313.pyc,,
248
+ huggingface_hub/inference/_mcp/_cli_hacks.py,sha256=kIA8FfRfOWS3SrZrMTbuVOjE6yxfKfMK75Co5egTWHA,3284
249
+ huggingface_hub/inference/_mcp/agent.py,sha256=ufIzMGHore5n252hV5GZPM0ouDXIl6tv5Jl_5gHXnbg,4250
250
+ huggingface_hub/inference/_mcp/cli.py,sha256=RKdVQ_VdEgf1AL_QBCvwrsUQ0Ux4R1yerrE46DrV5C0,9642
251
+ huggingface_hub/inference/_mcp/constants.py,sha256=0Ih6ivEvG-pfF8NwErFngzP4Rof4lQoz6ms_eKSAkUI,2463
252
+ huggingface_hub/inference/_mcp/mcp_client.py,sha256=dGp8PhN6aVw4bDnuSySFSiguHUiz-nzhgv89CVdO7pI,17243
253
+ huggingface_hub/inference/_mcp/types.py,sha256=yHNfPsM9MhD06oeKdkbmrBsW-3WhUeqA26fyfRfx_bk,929
254
+ huggingface_hub/inference/_mcp/utils.py,sha256=gxSB_rBjQ6VrkApKFsxk6-UzhijxkDVNuZrsjW5pL8k,4318
255
+ huggingface_hub/inference/_providers/__init__.py,sha256=RSm8iJhLmkIvzUdAR9FGwOYuuWauK2RKVnZH_YHnK7s,9979
256
+ huggingface_hub/inference/_providers/__pycache__/__init__.cpython-313.pyc,,
257
+ huggingface_hub/inference/_providers/__pycache__/_common.cpython-313.pyc,,
258
+ huggingface_hub/inference/_providers/__pycache__/baseten.cpython-313.pyc,,
259
+ huggingface_hub/inference/_providers/__pycache__/cerebras.cpython-313.pyc,,
260
+ huggingface_hub/inference/_providers/__pycache__/cohere.cpython-313.pyc,,
261
+ huggingface_hub/inference/_providers/__pycache__/deepinfra.cpython-313.pyc,,
262
+ huggingface_hub/inference/_providers/__pycache__/fal_ai.cpython-313.pyc,,
263
+ huggingface_hub/inference/_providers/__pycache__/featherless_ai.cpython-313.pyc,,
264
+ huggingface_hub/inference/_providers/__pycache__/fireworks_ai.cpython-313.pyc,,
265
+ huggingface_hub/inference/_providers/__pycache__/groq.cpython-313.pyc,,
266
+ huggingface_hub/inference/_providers/__pycache__/hf_inference.cpython-313.pyc,,
267
+ huggingface_hub/inference/_providers/__pycache__/novita.cpython-313.pyc,,
268
+ huggingface_hub/inference/_providers/__pycache__/nscale.cpython-313.pyc,,
269
+ huggingface_hub/inference/_providers/__pycache__/openai.cpython-313.pyc,,
270
+ huggingface_hub/inference/_providers/__pycache__/ovhcloud.cpython-313.pyc,,
271
+ huggingface_hub/inference/_providers/__pycache__/publicai.cpython-313.pyc,,
272
+ huggingface_hub/inference/_providers/__pycache__/replicate.cpython-313.pyc,,
273
+ huggingface_hub/inference/_providers/__pycache__/scaleway.cpython-313.pyc,,
274
+ huggingface_hub/inference/_providers/__pycache__/together.cpython-313.pyc,,
275
+ huggingface_hub/inference/_providers/__pycache__/wavespeed.cpython-313.pyc,,
276
+ huggingface_hub/inference/_providers/__pycache__/zai_org.cpython-313.pyc,,
277
+ huggingface_hub/inference/_providers/_common.py,sha256=T3iKT6McNs7GeVfK3q-l7CBbuih2mTFcqkRluQn7GnQ,14308
278
+ huggingface_hub/inference/_providers/baseten.py,sha256=1uucLZp6HiF8OjfeFOkiuJP6WqunP_xzwk3yWPF8vGE,213
279
+ huggingface_hub/inference/_providers/cerebras.py,sha256=QOJ-1U-os7uE7p6eUnn_P_APq-yQhx28be7c3Tq2EuA,210
280
+ huggingface_hub/inference/_providers/cohere.py,sha256=P9kbIuvQ2rXI1yNmgbw5VKFFTE0huLq2k-BCyDkDico,1226
281
+ huggingface_hub/inference/_providers/deepinfra.py,sha256=4Behgf5V7FTv7CbN6vPEIYwv1z4gztdRRU-KVoOgxVE,4703
282
+ huggingface_hub/inference/_providers/fal_ai.py,sha256=Y2vo5Dl3e8EoehplwNshu8LbDT1e7V4ZyhAMqs0wXws,11705
283
+ huggingface_hub/inference/_providers/featherless_ai.py,sha256=C8OHdFpoFyA--pawLTekUmUSRq4sw_r0D-cSPekD4kE,1347
284
+ huggingface_hub/inference/_providers/fireworks_ai.py,sha256=go6XPum8u0-g768HJ_r0S4Gb445gZJxnuY_acktz-9c,1188
285
+ huggingface_hub/inference/_providers/groq.py,sha256=JTk2JV4ZOlaohho7zLAFQtk92kGVsPmLJ1hmzcwsqvQ,315
286
+ huggingface_hub/inference/_providers/hf_inference.py,sha256=z-NoYyjppc2iBPh6uVGMDikIrhnW8-dD1CJHo665ko0,9803
287
+ huggingface_hub/inference/_providers/novita.py,sha256=ATEoSdPAPLMfx3JpBc0sOyLh4upJYP6xHBuo4YEDYvg,2470
288
+ huggingface_hub/inference/_providers/nscale.py,sha256=T1L2JLI9LqYT9_YEdW76YEtxfDNtdGkoFhH-p2LmSxg,1767
289
+ huggingface_hub/inference/_providers/openai.py,sha256=wwjaaQ55xLmDsDnHpZk52xbuLoGZfWzJkFsE8AdFVaI,1054
290
+ huggingface_hub/inference/_providers/ovhcloud.py,sha256=tdmymlkbddMJKV7NRZ-tH2wymbLPFDTqUSXpWJUXyDQ,314
291
+ huggingface_hub/inference/_providers/publicai.py,sha256=1I2W6rORloB5QHSvky4njZO2XKLTwA-kPdNoauoT5rg,210
292
+ huggingface_hub/inference/_providers/replicate.py,sha256=hte0ZB2RtGFwpAuLrFp2Gbgsn3EOlccCnJkWTDTW__A,6027
293
+ huggingface_hub/inference/_providers/scaleway.py,sha256=MfIc7ZND1sPr__rOmNHZVg0VpECQo0bVyksgIm_32xQ,1174
294
+ huggingface_hub/inference/_providers/together.py,sha256=lrymDZ86n5yoR_Zz7o3AzzSePyo0s7BOBU5digLxb5o,12220
295
+ huggingface_hub/inference/_providers/wavespeed.py,sha256=MGM7Y7r2nQiH_EH0t5UE1o1fT-o3sS7RaQlicS9WHsg,5028
296
+ huggingface_hub/inference/_providers/zai_org.py,sha256=gLJZOEmCPmUZvdM7VDn2nxm4ac3veHdoMgQNR63UeWE,4739
297
+ huggingface_hub/lfs.py,sha256=8ymYNHRXgvgUUM9w0EAnC-vkSOOq_1EFJEJNoevoeE8,16588
298
+ huggingface_hub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
299
+ huggingface_hub/repocard.py,sha256=7u7hiKjNuPYOwa3XPOPUnYPpuv4buya4WM47kVexklY,35602
300
+ huggingface_hub/repocard_data.py,sha256=f834MUSXGqrVcC02668r3ACrPzKKOAg75y3aB_hXm1U,34308
301
+ huggingface_hub/serialization/__init__.py,sha256=jCiw_vVQYW52gwVfWiqgocf2Q19kGTQlRGVpf-4SLP8,963
302
+ huggingface_hub/serialization/__pycache__/__init__.cpython-313.pyc,,
303
+ huggingface_hub/serialization/__pycache__/_base.cpython-313.pyc,,
304
+ huggingface_hub/serialization/__pycache__/_dduf.cpython-313.pyc,,
305
+ huggingface_hub/serialization/__pycache__/_torch.cpython-313.pyc,,
306
+ huggingface_hub/serialization/_base.py,sha256=8vYeDIsqgyOO3I_uii44Bkb50LsMz3kkD4gwAPzhfWU,8436
307
+ huggingface_hub/serialization/_dduf.py,sha256=FmGRg5wkXI5sNCgZgCNJFmqb_nVrrQYOHf4Lqq0d64I,15385
308
+ huggingface_hub/serialization/_torch.py,sha256=BtaCP-G3oYRP6Pj9MhI-ygBsk3JbBPnVDW9huSOgtkk,46897
309
+ huggingface_hub/templates/datasetcard_template.md,sha256=W-EMqR6wndbrnZorkVv56URWPG49l7MATGeI015kTvs,5503
310
+ huggingface_hub/templates/modelcard_template.md,sha256=4AqArS3cqdtbit5Bo-DhjcnDFR-pza5hErLLTPM4Yuc,6870
311
+ huggingface_hub/utils/__init__.py,sha256=pdVhPWWZkv0QM7kAQw7Bf259RZdPtYBLQqQwLQN-VC4,4045
312
+ huggingface_hub/utils/__pycache__/__init__.cpython-313.pyc,,
313
+ huggingface_hub/utils/__pycache__/_auth.cpython-313.pyc,,
314
+ huggingface_hub/utils/__pycache__/_cache_assets.cpython-313.pyc,,
315
+ huggingface_hub/utils/__pycache__/_cache_manager.cpython-313.pyc,,
316
+ huggingface_hub/utils/__pycache__/_chunk_utils.cpython-313.pyc,,
317
+ huggingface_hub/utils/__pycache__/_datetime.cpython-313.pyc,,
318
+ huggingface_hub/utils/__pycache__/_deprecation.cpython-313.pyc,,
319
+ huggingface_hub/utils/__pycache__/_detect_agent.cpython-313.pyc,,
320
+ huggingface_hub/utils/__pycache__/_dotenv.cpython-313.pyc,,
321
+ huggingface_hub/utils/__pycache__/_experimental.cpython-313.pyc,,
322
+ huggingface_hub/utils/__pycache__/_fixes.cpython-313.pyc,,
323
+ huggingface_hub/utils/__pycache__/_git_credential.cpython-313.pyc,,
324
+ huggingface_hub/utils/__pycache__/_headers.cpython-313.pyc,,
325
+ huggingface_hub/utils/__pycache__/_hf_uris.cpython-313.pyc,,
326
+ huggingface_hub/utils/__pycache__/_http.cpython-313.pyc,,
327
+ huggingface_hub/utils/__pycache__/_lfs.cpython-313.pyc,,
328
+ huggingface_hub/utils/__pycache__/_oauth_device.cpython-313.pyc,,
329
+ huggingface_hub/utils/__pycache__/_pagination.cpython-313.pyc,,
330
+ huggingface_hub/utils/__pycache__/_parsing.cpython-313.pyc,,
331
+ huggingface_hub/utils/__pycache__/_paths.cpython-313.pyc,,
332
+ huggingface_hub/utils/__pycache__/_runtime.cpython-313.pyc,,
333
+ huggingface_hub/utils/__pycache__/_safetensors.cpython-313.pyc,,
334
+ huggingface_hub/utils/__pycache__/_subprocess.cpython-313.pyc,,
335
+ huggingface_hub/utils/__pycache__/_telemetry.cpython-313.pyc,,
336
+ huggingface_hub/utils/__pycache__/_terminal.cpython-313.pyc,,
337
+ huggingface_hub/utils/__pycache__/_typing.cpython-313.pyc,,
338
+ huggingface_hub/utils/__pycache__/_validators.cpython-313.pyc,,
339
+ huggingface_hub/utils/__pycache__/_verification.cpython-313.pyc,,
340
+ huggingface_hub/utils/__pycache__/_xet.cpython-313.pyc,,
341
+ huggingface_hub/utils/__pycache__/_xet_progress_reporting.cpython-313.pyc,,
342
+ huggingface_hub/utils/__pycache__/endpoint_helpers.cpython-313.pyc,,
343
+ huggingface_hub/utils/__pycache__/insecure_hashlib.cpython-313.pyc,,
344
+ huggingface_hub/utils/__pycache__/logging.cpython-313.pyc,,
345
+ huggingface_hub/utils/__pycache__/sha.cpython-313.pyc,,
346
+ huggingface_hub/utils/__pycache__/tqdm.cpython-313.pyc,,
347
+ huggingface_hub/utils/_auth.py,sha256=Ykuq3yi67jxHmFdyLSc43xvGuCAIR7-h-dajX8IxpGQ,19958
348
+ huggingface_hub/utils/_cache_assets.py,sha256=nnzHRtQAR50dQeIK6qKddsmjjTW9v9HZ7b9bq7PJqss,5691
349
+ huggingface_hub/utils/_cache_manager.py,sha256=BFixsojrtG1K8rsD89EsGtXSktXbt0-VlJJkPRz5tso,35733
350
+ huggingface_hub/utils/_chunk_utils.py,sha256=pTjy8Z-KLU4W_6D3OUh3E8lCodWDCd6aJwjNDU0C5U8,2121
351
+ huggingface_hub/utils/_datetime.py,sha256=tbNyI0Dkh27oScPUtLIT_8apqIIkXZYbigjOn9S3aMw,2755
352
+ huggingface_hub/utils/_deprecation.py,sha256=n4kNHbGipquSObJ-gxodcfd6lqoe_8s-VIsTuo3Oruk,4865
353
+ huggingface_hub/utils/_detect_agent.py,sha256=Iw2KW9BUZWfIypn9CtiTmEibHwKK9xx5Ar48z1cMU-Q,7337
354
+ huggingface_hub/utils/_dotenv.py,sha256=NdlEM8OXtzwlwmJCmdc6oGubFObrpgr408y8gnSs7ls,2604
355
+ huggingface_hub/utils/_experimental.py,sha256=q9vUvc1JybVFRQ0GRREG-BLouZEJC_40MxcwbAlOud0,2464
356
+ huggingface_hub/utils/_fixes.py,sha256=jTK1VLmc0ZC9ROSXjKoL5F6kOZFByzirRWIdXxhBfWU,4124
357
+ huggingface_hub/utils/_git_credential.py,sha256=1BhjvIScCOAToDORLOKrR3Szs-m0E5AYHPmC0SD7Nrs,4548
358
+ huggingface_hub/utils/_headers.py,sha256=Q9bu9mvOpm-9rfv-YRlr_Ywy_aMZkmRcN3MbVEcJAJA,8234
359
+ huggingface_hub/utils/_hf_uris.py,sha256=RTHrVKLT5vVU-ca8roXHrKwbN3njowanvxHjx5B4sUg,27558
360
+ huggingface_hub/utils/_http.py,sha256=zx1TItko5i0_CzL95zHyt36XXrtVLnn5F9fl6ODq8aw,46197
361
+ huggingface_hub/utils/_lfs.py,sha256=xMU-ROgNAUpDkzbH6yRZsE-eVYUYTNMGrgVb_QdTv-k,3942
362
+ huggingface_hub/utils/_oauth_device.py,sha256=o7ICmI01yDt1oAFNMUJvN1ThSfaCDpC8AHYSQgcnFGw,7983
363
+ huggingface_hub/utils/_pagination.py,sha256=h-TJjFbX3FqkA1T0tGZ0EhxMvuRSCg-UfL_a-BMs54I,1943
364
+ huggingface_hub/utils/_parsing.py,sha256=DVWuO6s_XMiGfXyc0jyX5hi_yGgBHpF9KLoeQM-VVK0,3586
365
+ huggingface_hub/utils/_paths.py,sha256=kWpUMIK9rmKb2LHgHCkk2DnroDzucrwbj-urCJCgfGw,5962
366
+ huggingface_hub/utils/_runtime.py,sha256=GVJ_Dt6y_48TZWl0ui9yPdW8M7rqGblRGyUqKaXxkCo,13493
367
+ huggingface_hub/utils/_safetensors.py,sha256=qFE7OA-vjU8X0zBBtOeNRzLu147KF-PdFoaxkYfWB4M,4426
368
+ huggingface_hub/utils/_subprocess.py,sha256=tFVBBNot_HLVqQ79y873TGb12C4PUTMwub-GhzDTemE,4542
369
+ huggingface_hub/utils/_telemetry.py,sha256=Tpa3YmOLhK_MEnL2i2fRLQy8aqAEZWbj6MKC18Hs8BA,4824
370
+ huggingface_hub/utils/_terminal.py,sha256=whHgs0Co-1LatoCmJE_ojqE_cWP6b5dWPWpU-uSFWUU,8969
371
+ huggingface_hub/utils/_typing.py,sha256=1LeE785YedppXSR9a1fQDu5rhTxX5v9kBYBaHrP65rA,3542
372
+ huggingface_hub/utils/_validators.py,sha256=tuC1U4yxB-4XKIajZMnKZUSKvZ6qPpydY8LItfa0g40,8419
373
+ huggingface_hub/utils/_verification.py,sha256=ZSilnolSkYmHB3vmBlWuuVr53xz59P5uGnxC3H-Hjc0,5434
374
+ huggingface_hub/utils/_xet.py,sha256=lVCUmCxzmBFYRSkJxFLtfhe7_HJWdLzJdENL-ln5Grc,6556
375
+ huggingface_hub/utils/_xet_progress_reporting.py,sha256=MnNCxB9ieflArNNyz3R4kpkjOdt6mQxi8qp8xqzXQTM,14162
376
+ huggingface_hub/utils/endpoint_helpers.py,sha256=9VtIAlxQ5H_4y30sjCAgbu7XCqAtNLC7aRYxaNn0hLI,2366
377
+ huggingface_hub/utils/insecure_hashlib.py,sha256=z3dVUFvdBZ8kQI_8Vzvvlr3ims-EBiY-SYPdnzIKOkw,1008
378
+ huggingface_hub/utils/logging.py,sha256=WaXk5gRa8Ml_LUIH34QCr8suYj8i_8Wot4IDnYJyWyM,4870
379
+ huggingface_hub/utils/sha.py,sha256=h8wxheZpcv671RhtiIFcFmQTDSrogN3kwQx3ZaNEUHg,2121
380
+ huggingface_hub/utils/tqdm.py,sha256=xXjKGy2y2gMqpKeMUqXcQWc66buW39NwkTr3hlWH6MY,16729
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/REQUESTED ADDED
File without changes
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/WHEEL ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (83.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/entry_points.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ [console_scripts]
2
+ hf = huggingface_hub.cli.hf:main
3
+ huggingface-cli = huggingface_hub.cli.deprecated_cli:main
4
+ tiny-agents = huggingface_hub.inference._mcp.cli:app
5
+
6
+ [fsspec.specs]
7
+ hf = huggingface_hub.HfFileSystem
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/licenses/LICENSE ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
.venv/lib/python3.13/site-packages/huggingface_hub-1.27.0.dist-info/top_level.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ huggingface_hub
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/novita.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any
2
+
3
+ from huggingface_hub.hf_api import InferenceProviderMapping
4
+ from huggingface_hub.inference._common import RequestParameters, _as_dict
5
+ from huggingface_hub.inference._providers._common import (
6
+ BaseConversationalTask,
7
+ BaseTextGenerationTask,
8
+ TaskProviderHelper,
9
+ filter_none,
10
+ )
11
+ from huggingface_hub.utils import get_session
12
+
13
+
14
+ _PROVIDER = "novita"
15
+ _BASE_URL = "https://api.novita.ai"
16
+
17
+
18
+ class NovitaTextGenerationTask(BaseTextGenerationTask):
19
+ def __init__(self):
20
+ super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
21
+
22
+ def _prepare_route(self, mapped_model: str, api_key: str) -> str:
23
+ # there is no v1/ route for novita
24
+ return "/v3/openai/completions"
25
+
26
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
27
+ output = _as_dict(response)["choices"][0]
28
+ return {
29
+ "generated_text": output["text"],
30
+ "details": {
31
+ "finish_reason": output.get("finish_reason"),
32
+ "seed": output.get("seed"),
33
+ },
34
+ }
35
+
36
+
37
+ class NovitaConversationalTask(BaseConversationalTask):
38
+ def __init__(self):
39
+ super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
40
+
41
+ def _prepare_route(self, mapped_model: str, api_key: str) -> str:
42
+ # there is no v1/ route for novita
43
+ return "/v3/openai/chat/completions"
44
+
45
+
46
+ class NovitaTextToVideoTask(TaskProviderHelper):
47
+ def __init__(self):
48
+ super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task="text-to-video")
49
+
50
+ def _prepare_route(self, mapped_model: str, api_key: str) -> str:
51
+ return f"/v3/hf/{mapped_model}"
52
+
53
+ def _prepare_payload_as_dict(
54
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
55
+ ) -> dict | None:
56
+ return {"prompt": inputs, **filter_none(parameters)}
57
+
58
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
59
+ response_dict = _as_dict(response)
60
+ if not (
61
+ isinstance(response_dict, dict)
62
+ and "video" in response_dict
63
+ and isinstance(response_dict["video"], dict)
64
+ and "video_url" in response_dict["video"]
65
+ ):
66
+ raise ValueError("Expected response format: { 'video': { 'video_url': string } }")
67
+
68
+ video_url = response_dict["video"]["video_url"]
69
+ return get_session().get(video_url).content
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/nscale.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ from typing import Any
3
+
4
+ from huggingface_hub.hf_api import InferenceProviderMapping
5
+ from huggingface_hub.inference._common import RequestParameters, _as_dict
6
+
7
+ from ._common import BaseConversationalTask, TaskProviderHelper, filter_none
8
+
9
+
10
+ class NscaleConversationalTask(BaseConversationalTask):
11
+ def __init__(self):
12
+ super().__init__(provider="nscale", base_url="https://inference.api.nscale.com")
13
+
14
+
15
+ class NscaleTextToImageTask(TaskProviderHelper):
16
+ def __init__(self):
17
+ super().__init__(provider="nscale", base_url="https://inference.api.nscale.com", task="text-to-image")
18
+
19
+ def _prepare_route(self, mapped_model: str, api_key: str) -> str:
20
+ return "/v1/images/generations"
21
+
22
+ def _prepare_payload_as_dict(
23
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
24
+ ) -> dict | None:
25
+ mapped_model = provider_mapping_info.provider_id
26
+ # Combine all parameters except inputs and parameters
27
+ parameters = filter_none(parameters)
28
+ if "width" in parameters and "height" in parameters:
29
+ parameters["size"] = f"{parameters.pop('width')}x{parameters.pop('height')}"
30
+ if "num_inference_steps" in parameters:
31
+ parameters.pop("num_inference_steps")
32
+ if "cfg_scale" in parameters:
33
+ parameters.pop("cfg_scale")
34
+ payload = {
35
+ "response_format": "b64_json",
36
+ "prompt": inputs,
37
+ "model": mapped_model,
38
+ **parameters,
39
+ }
40
+ return payload
41
+
42
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
43
+ response_dict = _as_dict(response)
44
+ return base64.b64decode(response_dict["data"][0]["b64_json"])
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/openai.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from huggingface_hub.hf_api import InferenceProviderMapping
2
+ from huggingface_hub.inference._providers._common import BaseConversationalTask
3
+
4
+
5
+ class OpenAIConversationalTask(BaseConversationalTask):
6
+ def __init__(self):
7
+ super().__init__(provider="openai", base_url="https://api.openai.com")
8
+
9
+ def _prepare_api_key(self, api_key: str | None) -> str:
10
+ if api_key is None:
11
+ raise ValueError("You must provide an api_key to work with OpenAI API.")
12
+ if api_key.startswith("hf_"):
13
+ raise ValueError(
14
+ "OpenAI provider is not available through Hugging Face routing, please use your own OpenAI API key."
15
+ )
16
+ return api_key
17
+
18
+ def _prepare_mapping_info(self, model: str | None) -> InferenceProviderMapping:
19
+ if model is None:
20
+ raise ValueError("Please provide an OpenAI model ID, e.g. `gpt-4o` or `o1`.")
21
+ return InferenceProviderMapping(
22
+ provider="openai", providerId=model, task="conversational", status="live", hf_model_id=model
23
+ )
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/ovhcloud.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ from huggingface_hub.inference._providers._common import BaseConversationalTask
2
+
3
+
4
+ _PROVIDER = "ovhcloud"
5
+ _BASE_URL = "https://oai.endpoints.kepler.ai.cloud.ovh.net"
6
+
7
+
8
+ class OVHcloudConversationalTask(BaseConversationalTask):
9
+ def __init__(self):
10
+ super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/publicai.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ from ._common import BaseConversationalTask
2
+
3
+
4
+ class PublicAIConversationalTask(BaseConversationalTask):
5
+ def __init__(self):
6
+ super().__init__(provider="publicai", base_url="https://api.publicai.co")
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/replicate.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any
2
+
3
+ from huggingface_hub.hf_api import InferenceProviderMapping
4
+ from huggingface_hub.inference._common import RequestParameters, _as_dict, _as_url
5
+ from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none
6
+ from huggingface_hub.utils import get_session
7
+
8
+
9
+ _PROVIDER = "replicate"
10
+ _BASE_URL = "https://api.replicate.com"
11
+
12
+
13
+ class ReplicateTask(TaskProviderHelper):
14
+ def __init__(self, task: str):
15
+ super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task)
16
+
17
+ def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]:
18
+ headers = super()._prepare_headers(headers, api_key)
19
+ headers["Prefer"] = "wait"
20
+ return headers
21
+
22
+ def _prepare_route(self, mapped_model: str, api_key: str) -> str:
23
+ if ":" in mapped_model:
24
+ return "/v1/predictions"
25
+ return f"/v1/models/{mapped_model}/predictions"
26
+
27
+ def _prepare_payload_as_dict(
28
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
29
+ ) -> dict | None:
30
+ mapped_model = provider_mapping_info.provider_id
31
+ payload: dict[str, Any] = {"input": {"prompt": inputs, **filter_none(parameters)}}
32
+ if ":" in mapped_model:
33
+ version = mapped_model.split(":", 1)[1]
34
+ payload["version"] = version
35
+ return payload
36
+
37
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
38
+ response_dict = _as_dict(response)
39
+ if response_dict.get("output") is None:
40
+ raise TimeoutError(
41
+ f"Inference request timed out after 60 seconds. No output generated for model {response_dict.get('model')}"
42
+ "The model might be in cold state or starting up. Please try again later."
43
+ )
44
+ output_url = (
45
+ response_dict["output"] if isinstance(response_dict["output"], str) else response_dict["output"][0]
46
+ )
47
+ return get_session().get(output_url).content
48
+
49
+
50
+ class ReplicateTextToImageTask(ReplicateTask):
51
+ def __init__(self):
52
+ super().__init__("text-to-image")
53
+
54
+ def _prepare_payload_as_dict(
55
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
56
+ ) -> dict | None:
57
+ payload: dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore
58
+ if provider_mapping_info.adapter_weights_path is not None:
59
+ payload["input"]["lora_weights"] = f"https://huggingface.co/{provider_mapping_info.hf_model_id}"
60
+ return payload
61
+
62
+
63
+ class ReplicateTextToSpeechTask(ReplicateTask):
64
+ def __init__(self):
65
+ super().__init__("text-to-speech")
66
+
67
+ def _prepare_payload_as_dict(
68
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
69
+ ) -> dict | None:
70
+ payload: dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore
71
+ payload["input"]["text"] = payload["input"].pop("prompt") # rename "prompt" to "text" for TTS
72
+ return payload
73
+
74
+
75
+ class ReplicateAutomaticSpeechRecognitionTask(ReplicateTask):
76
+ def __init__(self) -> None:
77
+ super().__init__("automatic-speech-recognition")
78
+
79
+ def _prepare_payload_as_dict(
80
+ self,
81
+ inputs: Any,
82
+ parameters: dict,
83
+ provider_mapping_info: InferenceProviderMapping,
84
+ ) -> dict | None:
85
+ mapped_model = provider_mapping_info.provider_id
86
+ audio_url = _as_url(inputs, default_mime_type="audio/wav")
87
+
88
+ payload: dict[str, Any] = {
89
+ "input": {
90
+ **{"audio": audio_url},
91
+ **filter_none(parameters),
92
+ }
93
+ }
94
+
95
+ if ":" in mapped_model:
96
+ payload["version"] = mapped_model.split(":", 1)[1]
97
+
98
+ return payload
99
+
100
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
101
+ response_dict = _as_dict(response)
102
+ output = response_dict.get("output")
103
+
104
+ if isinstance(output, str):
105
+ return {"text": output}
106
+
107
+ if isinstance(output, list) and output:
108
+ first_item = output[0]
109
+ if isinstance(first_item, str):
110
+ return {"text": first_item}
111
+ if isinstance(first_item, dict):
112
+ output = first_item
113
+
114
+ text: str | None = None
115
+ if isinstance(output, dict):
116
+ transcription = output.get("transcription")
117
+ if isinstance(transcription, str):
118
+ text = transcription
119
+
120
+ translation = output.get("translation")
121
+ if isinstance(translation, str):
122
+ text = translation
123
+
124
+ txt_file = output.get("txt_file")
125
+ if isinstance(txt_file, str):
126
+ text_response = get_session().get(txt_file)
127
+ text_response.raise_for_status()
128
+ text = text_response.text
129
+
130
+ if text is not None:
131
+ return {"text": text}
132
+
133
+ raise ValueError("Received malformed response from Replicate automatic-speech-recognition API")
134
+
135
+
136
+ class ReplicateImageToImageTask(ReplicateTask):
137
+ def __init__(self):
138
+ super().__init__("image-to-image")
139
+
140
+ def _prepare_payload_as_dict(
141
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
142
+ ) -> dict | None:
143
+ image_url = _as_url(inputs, default_mime_type="image/jpeg")
144
+
145
+ # Different Replicate models expect the image in different keys
146
+ payload: dict[str, Any] = {
147
+ "input": {
148
+ "image": image_url,
149
+ "images": [image_url],
150
+ "input_image": image_url,
151
+ "input_images": [image_url],
152
+ **filter_none(parameters),
153
+ }
154
+ }
155
+
156
+ mapped_model = provider_mapping_info.provider_id
157
+ if ":" in mapped_model:
158
+ version = mapped_model.split(":", 1)[1]
159
+ payload["version"] = version
160
+ return payload
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/scaleway.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any
2
+
3
+ from huggingface_hub.inference._common import RequestParameters, _as_dict
4
+
5
+ from ._common import BaseConversationalTask, InferenceProviderMapping, TaskProviderHelper, filter_none
6
+
7
+
8
+ class ScalewayConversationalTask(BaseConversationalTask):
9
+ def __init__(self):
10
+ super().__init__(provider="scaleway", base_url="https://api.scaleway.ai")
11
+
12
+
13
+ class ScalewayFeatureExtractionTask(TaskProviderHelper):
14
+ def __init__(self):
15
+ super().__init__(provider="scaleway", base_url="https://api.scaleway.ai", task="feature-extraction")
16
+
17
+ def _prepare_route(self, mapped_model: str, api_key: str) -> str:
18
+ return "/v1/embeddings"
19
+
20
+ def _prepare_payload_as_dict(
21
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
22
+ ) -> dict | None:
23
+ parameters = filter_none(parameters)
24
+ return {"input": inputs, "model": provider_mapping_info.provider_id, **parameters}
25
+
26
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
27
+ embeddings = _as_dict(response)["data"]
28
+ return [embedding["embedding"] for embedding in embeddings]
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/together.py ADDED
@@ -0,0 +1,291 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ import time
3
+ from abc import ABC
4
+ from typing import Any
5
+
6
+ from huggingface_hub.hf_api import InferenceProviderMapping
7
+ from huggingface_hub.inference._common import (
8
+ RequestParameters,
9
+ _as_dict,
10
+ _as_url,
11
+ )
12
+ from huggingface_hub.inference._providers._common import (
13
+ BaseConversationalTask,
14
+ BaseTextGenerationTask,
15
+ TaskProviderHelper,
16
+ filter_none,
17
+ )
18
+ from huggingface_hub.utils import get_session, hf_raise_for_status, logging
19
+
20
+
21
+ logger = logging.get_logger(__name__)
22
+
23
+ _PROVIDER = "together"
24
+ _BASE_URL = "https://api.together.xyz"
25
+
26
+ # Polling interval for async video generation (in seconds).
27
+ _VIDEO_POLLING_INTERVAL = 2.0
28
+
29
+ # Upper bound on status polls (initial response may already be terminal; each further poll is one attempt).
30
+ _VIDEO_MAX_POLL_ATTEMPTS = 150 # ~5 minutes at _VIDEO_POLLING_INTERVAL
31
+
32
+ # Job statuses that mean "keep polling". Together returns "queued" before transitioning to
33
+ # "in_progress", so we must treat both as pending.
34
+ _VIDEO_PENDING_STATUSES = {"queued", "in_progress"}
35
+
36
+
37
+ class TogetherTask(TaskProviderHelper, ABC):
38
+ """Base class for Together API tasks."""
39
+
40
+ def __init__(self, task: str):
41
+ super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task)
42
+
43
+ def _prepare_route(self, mapped_model: str, api_key: str) -> str:
44
+ match self.task:
45
+ case "text-to-image" | "image-to-image":
46
+ return "/v1/images/generations"
47
+ case "text-to-speech":
48
+ return "/v1/audio/speech"
49
+ case "feature-extraction":
50
+ return "/v1/embeddings"
51
+ case "text-to-video" | "image-to-video":
52
+ # Video creation lives under /v2 (see https://docs.together.ai/reference/create-videos).
53
+ return "/v2/videos"
54
+ raise ValueError(f"Unsupported task '{self.task}' for Together API.")
55
+
56
+
57
+ class TogetherTextGenerationTask(BaseTextGenerationTask):
58
+ def __init__(self):
59
+ super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
60
+
61
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
62
+ output = _as_dict(response)["choices"][0]
63
+ return {
64
+ "generated_text": output["text"],
65
+ "details": {
66
+ "finish_reason": output.get("finish_reason"),
67
+ "seed": output.get("seed"),
68
+ },
69
+ }
70
+
71
+
72
+ class TogetherConversationalTask(BaseConversationalTask):
73
+ def __init__(self):
74
+ super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
75
+
76
+ def _prepare_payload_as_dict(
77
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
78
+ ) -> dict | None:
79
+ payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info)
80
+ if payload is None:
81
+ return None
82
+ # Together accepts response_format `{type: "json_schema", schema: <schema>}` (flattened),
83
+ # so unwrap the OpenAI-style `{type: "json_schema", json_schema: {schema}}` envelope.
84
+ response_format = payload.get("response_format")
85
+ if (
86
+ isinstance(response_format, dict)
87
+ and response_format.get("type") == "json_schema"
88
+ and isinstance(response_format.get("json_schema"), dict)
89
+ and "schema" in response_format["json_schema"]
90
+ ):
91
+ payload["response_format"] = {
92
+ "type": "json_schema",
93
+ "schema": response_format["json_schema"]["schema"],
94
+ }
95
+ return payload
96
+
97
+
98
+ class TogetherTextToImageTask(TogetherTask):
99
+ def __init__(self):
100
+ super().__init__("text-to-image")
101
+
102
+ def _prepare_payload_as_dict(
103
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
104
+ ) -> dict | None:
105
+ mapped_model = provider_mapping_info.provider_id
106
+ parameters = filter_none(parameters)
107
+ if "num_inference_steps" in parameters:
108
+ parameters["steps"] = parameters.pop("num_inference_steps")
109
+
110
+ return {"prompt": inputs, "response_format": "base64", **parameters, "model": mapped_model}
111
+
112
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
113
+ response_dict = _as_dict(response)
114
+ return base64.b64decode(response_dict["data"][0]["b64_json"])
115
+
116
+
117
+ class TogetherImageToImageTask(TogetherTask):
118
+ def __init__(self):
119
+ super().__init__("image-to-image")
120
+
121
+ def _prepare_payload_as_dict(
122
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
123
+ ) -> dict | None:
124
+ mapped_model = provider_mapping_info.provider_id
125
+ image_url = _as_url(inputs, default_mime_type="image/jpeg")
126
+
127
+ # Filter `None` values first: the client always passes `"prompt": None` when the user
128
+ # omits the argument, so popping before filtering would yield `None` instead of the
129
+ # `""` default and send `"prompt": null` to Together (rejected by Flux Kontext).
130
+ parameters = filter_none(parameters)
131
+ prompt = parameters.pop("prompt", "")
132
+ if "num_inference_steps" in parameters:
133
+ parameters["steps"] = parameters.pop("num_inference_steps")
134
+
135
+ # Together exposes two mutually-exclusive image inputs (see
136
+ # https://docs.together.ai/docs/image-to-image): FLUX.1 Kontext only accepts
137
+ # `image_url`; FLUX.2 [dev] and Google models (Gemini 3 Pro Image, Flash Image
138
+ # 2.5) only accept `reference_images`. FLUX.2 [pro]/[flex] accept either but
139
+ # `reference_images` is the documented default. Use `image_url` only for
140
+ # FLUX.1 Kontext models and `reference_images` for everything else.
141
+ lowered = mapped_model.lower()
142
+ use_image_url = "kontext" in lowered and "flux.1" in lowered
143
+ image_field: dict[str, Any] = {"image_url": image_url} if use_image_url else {"reference_images": [image_url]}
144
+ return {
145
+ "prompt": prompt,
146
+ **image_field,
147
+ "response_format": "base64",
148
+ **parameters,
149
+ "model": mapped_model,
150
+ }
151
+
152
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
153
+ response_dict = _as_dict(response)
154
+ return base64.b64decode(response_dict["data"][0]["b64_json"])
155
+
156
+
157
+ class TogetherFeatureExtractionTask(TogetherTask):
158
+ def __init__(self):
159
+ super().__init__("feature-extraction")
160
+
161
+ def _prepare_payload_as_dict(
162
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
163
+ ) -> dict | None:
164
+ return {
165
+ "input": inputs,
166
+ "model": provider_mapping_info.provider_id,
167
+ **filter_none(parameters),
168
+ }
169
+
170
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
171
+ return [item["embedding"] for item in _as_dict(response)["data"]]
172
+
173
+
174
+ class TogetherTextToSpeechTask(TogetherTask):
175
+ def __init__(self):
176
+ super().__init__("text-to-speech")
177
+
178
+ def _prepare_payload_as_dict(
179
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
180
+ ) -> dict | None:
181
+ # `voice` is required by the Together API and is model-specific
182
+ # (see https://docs.together.ai/docs/text-to-speech#supported-voices),
183
+ # so we don't set a default and let the API surface a clear error if missing.
184
+ return {
185
+ "input": inputs,
186
+ "model": provider_mapping_info.provider_id,
187
+ **filter_none(parameters),
188
+ }
189
+
190
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
191
+ if isinstance(response, bytes):
192
+ return response
193
+ raise ValueError(f"Expected raw audio bytes for text-to-speech, got {type(response).__name__}.")
194
+
195
+
196
+ def _normalize_video_parameters(parameters: dict) -> dict:
197
+ """Map HF inference-client conventions onto Together's video API parameter names."""
198
+ parameters = filter_none(parameters)
199
+ if "num_inference_steps" in parameters:
200
+ parameters["steps"] = parameters.pop("num_inference_steps")
201
+ if "target_size" in parameters:
202
+ target_size = parameters.pop("target_size")
203
+ if "width" in target_size:
204
+ parameters["width"] = target_size["width"]
205
+ if "height" in target_size:
206
+ parameters["height"] = target_size["height"]
207
+ return parameters
208
+
209
+
210
+ class TogetherVideoTask(TogetherTask, ABC):
211
+ """Base class for Together's asynchronous video generation tasks."""
212
+
213
+ def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any:
214
+ if request_params is None:
215
+ raise ValueError("A `RequestParameters` object is required to poll Together video jobs.")
216
+
217
+ job = _as_dict(response)
218
+ job_id = job.get("id")
219
+ if not job_id:
220
+ raise ValueError("No job ID found in Together video generation response.")
221
+
222
+ # Status polling lives at the same /v2/videos URL with the job ID appended.
223
+ status_url = f"{request_params.url}/{job_id}"
224
+
225
+ logger.info("Generating video, polling for completion...")
226
+ # Together usually returns `status: "queued"` on the initial POST, but the field is
227
+ # optional per the spec — treat a missing status as "still pending" and poll, rather
228
+ # than falling through to the "unexpected status" error below.
229
+ status = job.get("status")
230
+ for _ in range(_VIDEO_MAX_POLL_ATTEMPTS):
231
+ if status is not None and status not in _VIDEO_PENDING_STATUSES:
232
+ break
233
+ time.sleep(_VIDEO_POLLING_INTERVAL)
234
+ status_response = get_session().get(status_url, headers=request_params.headers)
235
+ hf_raise_for_status(status_response)
236
+ job = status_response.json()
237
+ status = job.get("status")
238
+ if status is not None and status not in _VIDEO_PENDING_STATUSES:
239
+ break
240
+ else:
241
+ raise ValueError(
242
+ "Timed out while waiting for Together video generation "
243
+ f"— aborting after {_VIDEO_MAX_POLL_ATTEMPTS} status polls"
244
+ )
245
+
246
+ if status == "failed":
247
+ error = job.get("error") or {}
248
+ raise RuntimeError(f"Together video generation failed: {error.get('message') or 'Unknown error'}")
249
+ if status != "completed":
250
+ raise RuntimeError(f"Unexpected Together video job status: {status!r}")
251
+
252
+ video_url = (job.get("outputs") or {}).get("video_url")
253
+ if not video_url:
254
+ raise ValueError("No video URL found in completed Together video job.")
255
+
256
+ video_response = get_session().get(video_url)
257
+ hf_raise_for_status(video_response)
258
+ return video_response.content
259
+
260
+
261
+ class TogetherTextToVideoTask(TogetherVideoTask):
262
+ def __init__(self):
263
+ super().__init__("text-to-video")
264
+
265
+ def _prepare_payload_as_dict(
266
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
267
+ ) -> dict | None:
268
+ return {
269
+ "prompt": inputs,
270
+ "model": provider_mapping_info.provider_id,
271
+ **_normalize_video_parameters(parameters),
272
+ }
273
+
274
+
275
+ class TogetherImageToVideoTask(TogetherVideoTask):
276
+ def __init__(self):
277
+ super().__init__("image-to-video")
278
+
279
+ def _prepare_payload_as_dict(
280
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
281
+ ) -> dict | None:
282
+ # Together expects each keyframe as `{input_image, frame: "first" | "last"}`
283
+ # for i2v models. See https://docs.together.ai/docs/inference/videos/reference-and-keyframes.
284
+ # Note: `input_image` accepts a data URL or an HTTP(S) URL but the field is capped
285
+ # at ~60KB — users with larger inputs should host the image and pass `frame_images`
286
+ # directly via `extra_body`.
287
+ return {
288
+ "model": provider_mapping_info.provider_id,
289
+ "frame_images": [{"input_image": _as_url(inputs, default_mime_type="image/png"), "frame": "first"}],
290
+ **_normalize_video_parameters(parameters),
291
+ }
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/wavespeed.py ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ import time
3
+ from abc import ABC
4
+ from typing import Any
5
+ from urllib.parse import urlparse
6
+
7
+ from huggingface_hub.hf_api import InferenceProviderMapping
8
+ from huggingface_hub.inference._common import RequestParameters, _as_dict
9
+ from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none
10
+ from huggingface_hub.utils import get_session, hf_raise_for_status
11
+ from huggingface_hub.utils.logging import get_logger
12
+
13
+
14
+ logger = get_logger(__name__)
15
+
16
+ # Polling interval (in seconds)
17
+ _POLLING_INTERVAL = 0.5
18
+
19
+
20
+ class WavespeedAITask(TaskProviderHelper, ABC):
21
+ def __init__(self, task: str):
22
+ super().__init__(provider="wavespeed", base_url="https://api.wavespeed.ai", task=task)
23
+
24
+ def _prepare_route(self, mapped_model: str, api_key: str) -> str:
25
+ return f"/api/v3/{mapped_model}"
26
+
27
+ def get_response(
28
+ self,
29
+ response: bytes | dict,
30
+ request_params: RequestParameters | None = None,
31
+ ) -> Any:
32
+ response_dict = _as_dict(response)
33
+ data = response_dict.get("data", {})
34
+ result_path = data.get("urls", {}).get("get")
35
+
36
+ if not result_path:
37
+ raise ValueError("No result URL found in the response")
38
+ if request_params is None:
39
+ raise ValueError("A `RequestParameters` object should be provided to get responses with WaveSpeed AI.")
40
+
41
+ # Parse the request URL to determine base URL
42
+ parsed_url = urlparse(request_params.url)
43
+ # Add /wavespeed to base URL if going through HF router
44
+ if parsed_url.netloc == "router.huggingface.co":
45
+ base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/wavespeed"
46
+ else:
47
+ base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
48
+
49
+ # Extract path from result_path URL
50
+ if isinstance(result_path, str):
51
+ result_url_path = urlparse(result_path).path
52
+ else:
53
+ result_url_path = result_path
54
+
55
+ result_url = f"{base_url}{result_url_path}"
56
+
57
+ logger.info("Processing request, polling for results...")
58
+
59
+ # Poll until task is completed
60
+ while True:
61
+ time.sleep(_POLLING_INTERVAL)
62
+ result_response = get_session().get(result_url, headers=request_params.headers)
63
+ hf_raise_for_status(result_response)
64
+
65
+ result = result_response.json()
66
+ task_result = result.get("data", {})
67
+ status = task_result.get("status")
68
+
69
+ if status == "completed":
70
+ # Get content from the first output URL
71
+ if not task_result.get("outputs") or len(task_result["outputs"]) == 0:
72
+ raise ValueError("No output URL in completed response")
73
+
74
+ output_url = task_result["outputs"][0]
75
+ return get_session().get(output_url).content
76
+ elif status == "failed":
77
+ error_msg = task_result.get("error", "Task failed with no specific error message")
78
+ raise ValueError(f"WaveSpeed AI task failed: {error_msg}")
79
+ elif status in ["processing", "created"]:
80
+ continue
81
+ else:
82
+ raise ValueError(f"Unknown status: {status}")
83
+
84
+
85
+ class WavespeedAITextToImageTask(WavespeedAITask):
86
+ def __init__(self):
87
+ super().__init__("text-to-image")
88
+
89
+ def _prepare_payload_as_dict(
90
+ self,
91
+ inputs: Any,
92
+ parameters: dict,
93
+ provider_mapping_info: InferenceProviderMapping,
94
+ ) -> dict | None:
95
+ return {"prompt": inputs, **filter_none(parameters)}
96
+
97
+
98
+ class WavespeedAITextToVideoTask(WavespeedAITextToImageTask):
99
+ def __init__(self):
100
+ WavespeedAITask.__init__(self, "text-to-video")
101
+
102
+
103
+ class WavespeedAIImageToImageTask(WavespeedAITask):
104
+ def __init__(self):
105
+ super().__init__("image-to-image")
106
+
107
+ def _prepare_payload_as_dict(
108
+ self,
109
+ inputs: Any,
110
+ parameters: dict,
111
+ provider_mapping_info: InferenceProviderMapping,
112
+ ) -> dict | None:
113
+ # Convert inputs to image (URL or base64)
114
+ if isinstance(inputs, str) and inputs.startswith(("http://", "https://")):
115
+ image = inputs
116
+ elif isinstance(inputs, str):
117
+ # If input is a file path, read it first
118
+ with open(inputs, "rb") as f:
119
+ file_content = f.read()
120
+ image_b64 = base64.b64encode(file_content).decode("utf-8")
121
+ image = f"data:image/jpeg;base64,{image_b64}"
122
+ else:
123
+ # If input is binary data
124
+ image_b64 = base64.b64encode(inputs).decode("utf-8")
125
+ image = f"data:image/jpeg;base64,{image_b64}"
126
+
127
+ # Extract prompt from parameters if present
128
+ prompt = parameters.pop("prompt", None)
129
+ payload = {"image": image, **filter_none(parameters)}
130
+ if prompt is not None:
131
+ payload["prompt"] = prompt
132
+
133
+ return payload
134
+
135
+
136
+ class WavespeedAIImageToVideoTask(WavespeedAIImageToImageTask):
137
+ def __init__(self):
138
+ WavespeedAITask.__init__(self, "image-to-video")
.venv/lib/python3.13/site-packages/huggingface_hub/inference/_providers/zai_org.py ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import time
2
+ from abc import ABC
3
+ from typing import Any
4
+
5
+ from huggingface_hub.hf_api import InferenceProviderMapping
6
+ from huggingface_hub.inference._common import RequestParameters, _as_dict
7
+ from huggingface_hub.inference._providers._common import BaseConversationalTask, TaskProviderHelper, filter_none
8
+ from huggingface_hub.utils import get_session
9
+
10
+
11
+ _PROVIDER = "zai-org"
12
+ _BASE_URL = "https://api.z.ai"
13
+ _POLLING_INTERVAL = 5 # seconds
14
+ _MAX_POLL_ATTEMPTS = 60
15
+
16
+
17
+ class ZaiTask(TaskProviderHelper, ABC):
18
+ def __init__(self, task: str):
19
+ super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task)
20
+
21
+ def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]:
22
+ headers = super()._prepare_headers(headers, api_key)
23
+ headers["Accept-Language"] = "en-US,en"
24
+ headers["x-source-channel"] = "hugging_face"
25
+ return headers
26
+
27
+
28
+ class ZaiConversationalTask(BaseConversationalTask):
29
+ def __init__(self):
30
+ super().__init__(provider=_PROVIDER, base_url=_BASE_URL)
31
+
32
+ def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]:
33
+ headers = super()._prepare_headers(headers, api_key)
34
+ headers["Accept-Language"] = "en-US,en"
35
+ headers["x-source-channel"] = "hugging_face"
36
+ return headers
37
+
38
+ def _prepare_route(self, mapped_model: str, api_key: str) -> str:
39
+ return "/api/paas/v4/chat/completions"
40
+
41
+
42
+ class ZaiTextToImageTask(ZaiTask):
43
+ """Text-to-image task for ZAI provider using async API."""
44
+
45
+ def __init__(self):
46
+ super().__init__("text-to-image")
47
+
48
+ def _prepare_route(self, mapped_model: str, api_key: str) -> str:
49
+ return "/api/paas/v4/async/images/generations"
50
+
51
+ def _prepare_payload_as_dict(
52
+ self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping
53
+ ) -> dict | None:
54
+ width = parameters.pop("width", None)
55
+ height = parameters.pop("height", None)
56
+ size = None
57
+ if width is not None and height is not None:
58
+ size = f"{width}x{height}"
59
+
60
+ payload: dict[str, Any] = {
61
+ "model": provider_mapping_info.provider_id,
62
+ "prompt": inputs,
63
+ }
64
+ if size is not None:
65
+ payload["size"] = size
66
+
67
+ payload.update(filter_none(parameters))
68
+ return payload
69
+
70
+ def get_response(
71
+ self,
72
+ response: bytes | dict,
73
+ request_params: RequestParameters | None = None,
74
+ ) -> Any:
75
+ """Handle async response by polling for results."""
76
+ response_dict = _as_dict(response)
77
+
78
+ task_id = response_dict.get("id")
79
+ if task_id is None:
80
+ raise ValueError("No task_id in response from ZAI API")
81
+
82
+ task_status = response_dict.get("task_status")
83
+ if task_status == "FAIL":
84
+ raise ValueError(f"ZAI image generation failed for request {task_id}")
85
+
86
+ if task_status == "PROCESSING" and request_params is not None:
87
+ return self._poll_for_result(task_id, request_params)
88
+
89
+ return self._extract_image(response_dict)
90
+
91
+ def _poll_for_result(self, task_id: str, request_params: RequestParameters) -> bytes:
92
+ """Poll the async-result endpoint until completion."""
93
+ session = get_session()
94
+ base_url = request_params.url.rsplit("/api/paas/v4/async/images/generations", 1)[0]
95
+ poll_url = f"{base_url}/api/paas/v4/async-result/{task_id}"
96
+
97
+ for _ in range(_MAX_POLL_ATTEMPTS):
98
+ poll_response = session.get(poll_url, headers=request_params.headers)
99
+ poll_response.raise_for_status()
100
+ result = poll_response.json()
101
+
102
+ task_status = result.get("task_status")
103
+ if task_status == "SUCCESS":
104
+ return self._extract_image(result)
105
+ elif task_status == "FAIL":
106
+ raise ValueError(f"Zai text-to-image generation failed for request {task_id}")
107
+
108
+ time.sleep(_POLLING_INTERVAL)
109
+
110
+ raise ValueError(
111
+ f"Timed out while waiting for the result from Zai API - aborting after {_MAX_POLL_ATTEMPTS} attempts"
112
+ )
113
+
114
+ def _extract_image(self, result: dict) -> bytes:
115
+ """Extract and download the image from the result."""
116
+ image_result = result.get("image_result")
117
+ if not image_result or not isinstance(image_result, list) or len(image_result) == 0:
118
+ raise ValueError("No image_result in response from ZAI API")
119
+
120
+ image_url = image_result[0].get("url")
121
+ if not image_url:
122
+ raise ValueError("No image URL in response from ZAI API")
123
+
124
+ session = get_session()
125
+ image_response = session.get(image_url)
126
+ image_response.raise_for_status()
127
+ return image_response.content
.venv/lib/python3.13/site-packages/huggingface_hub/lfs.py ADDED
@@ -0,0 +1,442 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2019-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Git LFS related type definitions and utilities"""
15
+
16
+ import io
17
+ import re
18
+ from collections.abc import Iterable
19
+ from math import ceil
20
+ from os.path import getsize
21
+ from typing import TYPE_CHECKING, BinaryIO, TypedDict
22
+ from urllib.parse import unquote
23
+
24
+ from huggingface_hub import constants
25
+
26
+ from .utils import (
27
+ build_hf_headers,
28
+ fix_hf_endpoint_in_url,
29
+ hf_raise_for_status,
30
+ http_backoff,
31
+ logging,
32
+ validate_hf_hub_args,
33
+ )
34
+ from .utils._lfs import SliceFileObj
35
+ from .utils.sha import sha256, sha_fileobj
36
+
37
+
38
+ if TYPE_CHECKING:
39
+ from ._commit_api import CommitOperationAdd
40
+
41
+ logger = logging.get_logger(__name__)
42
+
43
+ OID_REGEX = re.compile(r"^[0-9a-f]{40}$")
44
+
45
+ LFS_MULTIPART_UPLOAD_COMMAND = "lfs-multipart-upload"
46
+
47
+ LFS_HEADERS = {
48
+ "Accept": "application/vnd.git-lfs+json",
49
+ "Content-Type": "application/vnd.git-lfs+json",
50
+ }
51
+
52
+
53
+ class UploadInfo:
54
+ """
55
+ Data structure holding required information to determine whether a blob
56
+ should be uploaded to the hub using the LFS protocol or the regular protocol.
57
+
58
+ The SHA256 of the blob is computed lazily: creating an `UploadInfo` from a local path only reads
59
+ the first 512 bytes of the file. The full file is read (and hashed) only if `sha256` is accessed
60
+ before it has been set. When a file is uploaded through the Xet protocol, the SHA256 is computed
61
+ during upload (single read pass) and set afterwards.
62
+
63
+ Args:
64
+ size (`int`):
65
+ Size in bytes of the blob
66
+ sample (`bytes`):
67
+ First 512 bytes of the blob
68
+ sha256 (`bytes`, *optional*):
69
+ SHA256 hash of the blob, if already known. Otherwise computed lazily from `source_path`.
70
+ source_path (`str`, *optional*):
71
+ Path to the local file the blob comes from. Required to lazily compute `sha256` if not provided.
72
+ """
73
+
74
+ def __init__(
75
+ self,
76
+ size: int,
77
+ sample: bytes,
78
+ sha256: bytes | None = None,
79
+ source_path: str | None = None,
80
+ ):
81
+ if sha256 is None and source_path is None:
82
+ raise ValueError("Either `sha256` or `source_path` must be provided.")
83
+ self.size = size
84
+ self.sample = sample
85
+ self._sha256 = sha256
86
+ self._source_path = source_path
87
+
88
+ @property
89
+ def sha256(self) -> bytes:
90
+ """SHA256 of the blob. If not set yet, reads the whole file from `source_path` to compute it."""
91
+ if self._sha256 is None:
92
+ assert self._source_path is not None # guaranteed by __init__
93
+ with open(self._source_path, "rb") as file:
94
+ self._sha256 = sha_fileobj(file)
95
+ return self._sha256
96
+
97
+ @sha256.setter
98
+ def sha256(self, value: bytes) -> None:
99
+ self._sha256 = value
100
+
101
+ @property
102
+ def is_hashed(self) -> bool:
103
+ """Whether the SHA256 is already known (accessing `sha256` will not trigger a file read)."""
104
+ return self._sha256 is not None
105
+
106
+ def __repr__(self) -> str:
107
+ sha = self._sha256.hex() if self._sha256 is not None else "<not computed>"
108
+ return f"UploadInfo(size={self.size}, sha256={sha})"
109
+
110
+ @classmethod
111
+ def from_path(cls, path: str):
112
+ size = getsize(path)
113
+ with open(path, "rb") as file:
114
+ sample = file.peek(512)[:512]
115
+ return cls(size=size, sample=sample, source_path=path)
116
+
117
+ @classmethod
118
+ def from_bytes(cls, data: bytes):
119
+ sha = sha256(data).digest()
120
+ return cls(size=len(data), sample=data[:512], sha256=sha)
121
+
122
+ @classmethod
123
+ def from_fileobj(cls, fileobj: BinaryIO):
124
+ sample = fileobj.read(512)
125
+ fileobj.seek(0, io.SEEK_SET)
126
+ sha = sha_fileobj(fileobj)
127
+ size = fileobj.tell()
128
+ fileobj.seek(0, io.SEEK_SET)
129
+ return cls(size=size, sha256=sha, sample=sample)
130
+
131
+
132
+ @validate_hf_hub_args
133
+ def post_lfs_batch_info(
134
+ upload_infos: Iterable[UploadInfo],
135
+ token: str | None,
136
+ repo_type: str,
137
+ repo_id: str,
138
+ revision: str | None = None,
139
+ endpoint: str | None = None,
140
+ headers: dict[str, str] | None = None,
141
+ transfers: list[str] | None = None,
142
+ ) -> tuple[list[dict], list[dict], str | None]:
143
+ """
144
+ Requests the LFS batch endpoint to retrieve upload instructions
145
+
146
+ Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
147
+
148
+ Args:
149
+ upload_infos (`Iterable` of `UploadInfo`):
150
+ `UploadInfo` for the files that are being uploaded, typically obtained
151
+ from `CommitOperationAdd.upload_info`
152
+ token (`str` or `None`):
153
+ An authentication token (see https://huggingface.co/settings/token).
154
+ Pass `None` to fall back to the local cached token (or no token if unauthenticated).
155
+ repo_type (`str`):
156
+ Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
157
+ repo_id (`str`):
158
+ A namespace (user or an organization) and a repo name separated
159
+ by a `/`.
160
+ revision (`str`, *optional*):
161
+ The git revision to upload to.
162
+ endpoint (`str`, *optional*):
163
+ The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`.
164
+ headers (`dict`, *optional*):
165
+ Additional headers to include in the request
166
+ transfers (`list`, *optional*):
167
+ List of transfer methods to use. Defaults to ["basic", "multipart"].
168
+
169
+ Returns:
170
+ `LfsBatchInfo`: 3-tuple:
171
+ - First element is the list of upload instructions from the server
172
+ - Second element is a list of errors, if any
173
+ - Third element is the chosen transfer adapter if provided by the server (e.g. "basic", "multipart", "xet")
174
+
175
+ Raises:
176
+ [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
177
+ If an argument is invalid or the server response is malformed.
178
+ [`HfHubHTTPError`]
179
+ If the server returned an error.
180
+ """
181
+ endpoint = endpoint if endpoint is not None else constants.ENDPOINT
182
+ url_prefix = ""
183
+ if repo_type in constants.REPO_TYPES_URL_PREFIXES:
184
+ url_prefix = constants.REPO_TYPES_URL_PREFIXES[repo_type]
185
+ batch_url = f"{endpoint}/{url_prefix}{repo_id}.git/info/lfs/objects/batch"
186
+ payload: dict = {
187
+ "operation": "upload",
188
+ "transfers": transfers if transfers is not None else ["basic", "multipart"],
189
+ "objects": [
190
+ {
191
+ "oid": upload.sha256.hex(),
192
+ "size": upload.size,
193
+ }
194
+ for upload in upload_infos
195
+ ],
196
+ "hash_algo": "sha256",
197
+ }
198
+ if revision is not None:
199
+ payload["ref"] = {"name": unquote(revision)} # revision has been previously 'quoted'
200
+
201
+ headers = {
202
+ **LFS_HEADERS,
203
+ **build_hf_headers(token=token),
204
+ **(headers or {}),
205
+ }
206
+ resp = http_backoff("POST", batch_url, headers=headers, json=payload)
207
+ hf_raise_for_status(resp)
208
+ batch_info = resp.json()
209
+
210
+ objects = batch_info.get("objects", None)
211
+ if not isinstance(objects, list):
212
+ raise ValueError("Malformed response from server")
213
+
214
+ chosen_transfer = batch_info.get("transfer")
215
+ chosen_transfer = chosen_transfer if isinstance(chosen_transfer, str) else None
216
+
217
+ return (
218
+ [_validate_batch_actions(obj) for obj in objects if "error" not in obj],
219
+ [_validate_batch_error(obj) for obj in objects if "error" in obj],
220
+ chosen_transfer,
221
+ )
222
+
223
+
224
+ class PayloadPartT(TypedDict):
225
+ partNumber: int
226
+ etag: str
227
+
228
+
229
+ class CompletionPayloadT(TypedDict):
230
+ """Payload that will be sent to the Hub when uploading multi-part."""
231
+
232
+ oid: str
233
+ parts: list[PayloadPartT]
234
+
235
+
236
+ def lfs_upload(
237
+ operation: "CommitOperationAdd",
238
+ lfs_batch_action: dict,
239
+ token: str | None = None,
240
+ headers: dict[str, str] | None = None,
241
+ endpoint: str | None = None,
242
+ ) -> None:
243
+ """
244
+ Handles uploading a given object to the Hub with the LFS protocol.
245
+
246
+ Can be a No-op if the content of the file is already present on the hub large file storage.
247
+
248
+ Args:
249
+ operation (`CommitOperationAdd`):
250
+ The add operation triggering this upload.
251
+ lfs_batch_action (`dict`):
252
+ Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for
253
+ more details.
254
+ token (`str`, *optional*):
255
+ An authentication token (see https://huggingface.co/settings/token). Used to call the
256
+ optional LFS verify step at the end of the upload. If `None`, falls back to the local
257
+ cached token.
258
+ headers (`dict`, *optional*):
259
+ Headers to include in the request, including authentication and user agent headers.
260
+ endpoint (`str`, *optional*):
261
+ The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`.
262
+
263
+ Raises:
264
+ [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
265
+ If `lfs_batch_action` is improperly formatted
266
+ [`HfHubHTTPError`]
267
+ If the upload resulted in an error
268
+ """
269
+ # 0. If LFS file is already present, skip upload
270
+ _validate_batch_actions(lfs_batch_action)
271
+ actions = lfs_batch_action.get("actions")
272
+ if actions is None:
273
+ # The file was already uploaded
274
+ logger.debug(f"Content of file {operation.path_in_repo} is already present upstream - skipping upload")
275
+ return
276
+
277
+ # 1. Validate server response (check required keys in dict)
278
+ upload_action = lfs_batch_action["actions"]["upload"]
279
+ _validate_lfs_action(upload_action)
280
+ verify_action = lfs_batch_action["actions"].get("verify")
281
+ if verify_action is not None:
282
+ _validate_lfs_action(verify_action)
283
+
284
+ # 2. Upload file (either single part or multi-part)
285
+ header = upload_action.get("header", {})
286
+ chunk_size = header.get("chunk_size")
287
+ upload_url = fix_hf_endpoint_in_url(upload_action["href"], endpoint=endpoint)
288
+ if chunk_size is not None:
289
+ try:
290
+ chunk_size = int(chunk_size)
291
+ except (ValueError, TypeError):
292
+ raise ValueError(
293
+ f"Malformed response from LFS batch endpoint: `chunk_size` should be an integer. Got '{chunk_size}'."
294
+ )
295
+ _upload_multi_part(operation=operation, header=header, chunk_size=chunk_size, upload_url=upload_url)
296
+ else:
297
+ _upload_single_part(operation=operation, upload_url=upload_url)
298
+
299
+ # 3. Verify upload went well
300
+ if verify_action is not None:
301
+ _validate_lfs_action(verify_action)
302
+ verify_url = fix_hf_endpoint_in_url(verify_action["href"], endpoint)
303
+ verify_resp = http_backoff(
304
+ "POST",
305
+ verify_url,
306
+ headers=build_hf_headers(token=token, headers=headers),
307
+ json={"oid": operation.upload_info.sha256.hex(), "size": operation.upload_info.size},
308
+ )
309
+ hf_raise_for_status(verify_resp)
310
+ logger.debug(f"{operation.path_in_repo}: Upload successful")
311
+
312
+
313
+ def _validate_lfs_action(lfs_action: dict):
314
+ """validates response from the LFS batch endpoint"""
315
+ if not (
316
+ isinstance(lfs_action.get("href"), str)
317
+ and (lfs_action.get("header") is None or isinstance(lfs_action.get("header"), dict))
318
+ ):
319
+ raise ValueError("lfs_action is improperly formatted")
320
+ return lfs_action
321
+
322
+
323
+ def _validate_batch_actions(lfs_batch_actions: dict):
324
+ """validates response from the LFS batch endpoint"""
325
+ if not (isinstance(lfs_batch_actions.get("oid"), str) and isinstance(lfs_batch_actions.get("size"), int)):
326
+ raise ValueError("lfs_batch_actions is improperly formatted")
327
+
328
+ upload_action = lfs_batch_actions.get("actions", {}).get("upload")
329
+ verify_action = lfs_batch_actions.get("actions", {}).get("verify")
330
+ if upload_action is not None:
331
+ _validate_lfs_action(upload_action)
332
+ if verify_action is not None:
333
+ _validate_lfs_action(verify_action)
334
+ return lfs_batch_actions
335
+
336
+
337
+ def _validate_batch_error(lfs_batch_error: dict):
338
+ """validates response from the LFS batch endpoint"""
339
+ if not (isinstance(lfs_batch_error.get("oid"), str) and isinstance(lfs_batch_error.get("size"), int)):
340
+ raise ValueError("lfs_batch_error is improperly formatted")
341
+ error_info = lfs_batch_error.get("error")
342
+ if not (
343
+ isinstance(error_info, dict)
344
+ and isinstance(error_info.get("message"), str)
345
+ and isinstance(error_info.get("code"), int)
346
+ ):
347
+ raise ValueError("lfs_batch_error is improperly formatted")
348
+ return lfs_batch_error
349
+
350
+
351
+ def _upload_single_part(operation: "CommitOperationAdd", upload_url: str) -> None:
352
+ """
353
+ Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol)
354
+
355
+ Args:
356
+ upload_url (`str`):
357
+ The URL to PUT the file to.
358
+ fileobj:
359
+ The file-like object holding the data to upload.
360
+
361
+ Raises:
362
+ [`HfHubHTTPError`]
363
+ If the upload resulted in an error.
364
+ """
365
+ with operation.as_file(with_tqdm=True) as fileobj:
366
+ # S3 might raise a transient 500 error -> let's retry if that happens
367
+ response = http_backoff("PUT", upload_url, data=fileobj)
368
+ hf_raise_for_status(response)
369
+
370
+
371
+ def _upload_multi_part(operation: "CommitOperationAdd", header: dict, chunk_size: int, upload_url: str) -> None:
372
+ """
373
+ Uploads file using HF multipart LFS transfer protocol.
374
+ """
375
+ # 1. Get upload URLs for each part
376
+ sorted_parts_urls = _get_sorted_parts_urls(header=header, upload_info=operation.upload_info, chunk_size=chunk_size)
377
+
378
+ # 2. Upload parts (pure Python)
379
+ response_headers = _upload_parts_iteratively(
380
+ operation=operation, sorted_parts_urls=sorted_parts_urls, chunk_size=chunk_size
381
+ )
382
+
383
+ # 3. Send completion request
384
+ # NOTE: `upload_url` is the Hub completion endpoint (not the S3 upload URLs).
385
+ completion_res = http_backoff(
386
+ "POST",
387
+ upload_url,
388
+ json=_get_completion_payload(response_headers, operation.upload_info.sha256.hex()),
389
+ headers=LFS_HEADERS,
390
+ )
391
+ hf_raise_for_status(completion_res)
392
+
393
+
394
+ def _get_sorted_parts_urls(header: dict, upload_info: UploadInfo, chunk_size: int) -> list[str]:
395
+ sorted_part_upload_urls = [
396
+ upload_url
397
+ for _, upload_url in sorted(
398
+ [
399
+ (int(part_num, 10), upload_url)
400
+ for part_num, upload_url in header.items()
401
+ if part_num.isdigit() and len(part_num) > 0
402
+ ],
403
+ key=lambda t: t[0],
404
+ )
405
+ ]
406
+ num_parts = len(sorted_part_upload_urls)
407
+ if num_parts != ceil(upload_info.size / chunk_size):
408
+ raise ValueError("Invalid server response to upload large LFS file")
409
+ return sorted_part_upload_urls
410
+
411
+
412
+ def _get_completion_payload(response_headers: list[dict], oid: str) -> CompletionPayloadT:
413
+ parts: list[PayloadPartT] = []
414
+ for part_number, header in enumerate(response_headers):
415
+ etag = header.get("etag")
416
+ if etag is None or etag == "":
417
+ raise ValueError(f"Invalid etag (`{etag}`) returned for part {part_number + 1}")
418
+ parts.append(
419
+ {
420
+ "partNumber": part_number + 1,
421
+ "etag": etag,
422
+ }
423
+ )
424
+ return {"oid": oid, "parts": parts}
425
+
426
+
427
+ def _upload_parts_iteratively(
428
+ operation: "CommitOperationAdd", sorted_parts_urls: list[str], chunk_size: int
429
+ ) -> list[dict]:
430
+ headers = []
431
+ with operation.as_file(with_tqdm=True) as fileobj:
432
+ for part_idx, part_upload_url in enumerate(sorted_parts_urls):
433
+ with SliceFileObj(
434
+ fileobj,
435
+ seek_from=chunk_size * part_idx,
436
+ read_limit=chunk_size,
437
+ ) as fileobj_slice:
438
+ # S3 might raise a transient 500 error -> let's retry if that happens
439
+ part_upload_res = http_backoff("PUT", part_upload_url, data=fileobj_slice)
440
+ hf_raise_for_status(part_upload_res)
441
+ headers.append(part_upload_res.headers)
442
+ return headers # type: ignore
.venv/lib/python3.13/site-packages/huggingface_hub/py.typed ADDED
File without changes
.venv/lib/python3.13/site-packages/huggingface_hub/repocard.py ADDED
@@ -0,0 +1,835 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import re
3
+ from pathlib import Path
4
+ from typing import Any, Literal
5
+
6
+ import yaml
7
+
8
+ from huggingface_hub.file_download import hf_hub_download
9
+ from huggingface_hub.hf_api import upload_file
10
+ from huggingface_hub.repocard_data import (
11
+ CardData,
12
+ DatasetCardData,
13
+ EvalResult,
14
+ ModelCardData,
15
+ SpaceCardData,
16
+ eval_results_to_model_index,
17
+ model_index_to_eval_results,
18
+ )
19
+ from huggingface_hub.utils import HfHubHTTPError, get_session, hf_raise_for_status, is_jinja_available, yaml_dump
20
+
21
+ from . import constants
22
+ from .errors import EntryNotFoundError
23
+ from .utils import SoftTemporaryDirectory, logging, validate_hf_hub_args
24
+
25
+
26
+ logger = logging.get_logger(__name__)
27
+
28
+
29
+ TEMPLATE_MODELCARD_PATH = Path(__file__).parent / "templates" / "modelcard_template.md"
30
+ TEMPLATE_DATASETCARD_PATH = Path(__file__).parent / "templates" / "datasetcard_template.md"
31
+
32
+ # exact same regex as in the Hub server. Please keep in sync.
33
+ # See https://github.com/huggingface/moon-landing/blob/main/server/lib/ViewMarkdown.ts#L18
34
+ REGEX_YAML_BLOCK = re.compile(r"^(\s*---(?:\r\n|\r|\n))([\S\s]*?)((?:\r\n|\r|\n)---[ \t]*(\r\n|\n|$))")
35
+
36
+
37
+ class RepoCard:
38
+ card_data_class = CardData
39
+ default_template_path = TEMPLATE_MODELCARD_PATH
40
+ repo_type = "model"
41
+
42
+ def __init__(self, content: str, ignore_metadata_errors: bool = False):
43
+ """Initialize a RepoCard from string content. The content should be a
44
+ Markdown file with a YAML block at the beginning and a Markdown body.
45
+
46
+ Args:
47
+ content (`str`): The content of the Markdown file.
48
+
49
+ Example:
50
+ ```python
51
+ >>> from huggingface_hub.repocard import RepoCard
52
+ >>> text = '''
53
+ ... ---
54
+ ... language: en
55
+ ... license: mit
56
+ ... ---
57
+ ...
58
+ ... # My repo
59
+ ... '''
60
+ >>> card = RepoCard(text)
61
+ >>> card.data.to_dict()
62
+ {'language': 'en', 'license': 'mit'}
63
+ >>> card.text
64
+ '\\n# My repo\\n'
65
+
66
+ ```
67
+ > [!TIP]
68
+ > Raises the following error:
69
+ >
70
+ > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
71
+ > when the content of the repo card metadata is not a dictionary.
72
+ """
73
+
74
+ # Set the content of the RepoCard, as well as underlying .data and .text attributes.
75
+ # See the `content` property setter for more details.
76
+ self.ignore_metadata_errors = ignore_metadata_errors
77
+ self.content = content
78
+
79
+ @property
80
+ def content(self):
81
+ """The content of the RepoCard, including the YAML block and the Markdown body."""
82
+ line_break = _detect_line_ending(self._content) or "\n"
83
+ return f"---{line_break}{self.data.to_yaml(line_break=line_break, original_order=self._original_order)}{line_break}---{line_break}{self.text}"
84
+
85
+ @content.setter
86
+ def content(self, content: str):
87
+ """Set the content of the RepoCard."""
88
+ self._content = content
89
+
90
+ match = REGEX_YAML_BLOCK.search(content)
91
+ if match:
92
+ # Metadata found in the YAML block
93
+ yaml_block = match.group(2)
94
+ self.text = content[match.end() :]
95
+ data_dict = yaml.safe_load(yaml_block)
96
+
97
+ if data_dict is None:
98
+ data_dict = {}
99
+
100
+ # The YAML block's data should be a dictionary
101
+ if not isinstance(data_dict, dict):
102
+ raise ValueError("repo card metadata block should be a dict")
103
+ else:
104
+ # Model card without metadata... create empty metadata
105
+ logger.warning("Repo card metadata block was not found. Setting CardData to empty.")
106
+ data_dict = {}
107
+ self.text = content
108
+
109
+ self.data = self.card_data_class(**data_dict, ignore_metadata_errors=self.ignore_metadata_errors)
110
+ self._original_order = list(data_dict.keys())
111
+
112
+ def __str__(self):
113
+ return self.content
114
+
115
+ def save(self, filepath: Path | str):
116
+ r"""Save a RepoCard to a file.
117
+
118
+ Args:
119
+ filepath (`Union[Path, str]`): Filepath to the markdown file to save.
120
+
121
+ Example:
122
+ ```python
123
+ >>> from huggingface_hub.repocard import RepoCard
124
+ >>> card = RepoCard("---\nlanguage: en\n---\n# This is a test repo card")
125
+ >>> card.save("/tmp/test.md")
126
+
127
+ ```
128
+ """
129
+ filepath = Path(filepath)
130
+ filepath.parent.mkdir(parents=True, exist_ok=True)
131
+ # Preserve newlines as in the existing file.
132
+ with open(filepath, mode="w", newline="", encoding="utf-8") as f:
133
+ f.write(str(self))
134
+
135
+ @classmethod
136
+ def load(
137
+ cls,
138
+ repo_id_or_path: str | Path,
139
+ repo_type: str | None = None,
140
+ token: str | None = None,
141
+ ignore_metadata_errors: bool = False,
142
+ ):
143
+ """Initialize a RepoCard from a Hugging Face Hub repo's README.md or a local filepath.
144
+
145
+ Args:
146
+ repo_id_or_path (`Union[str, Path]`):
147
+ The repo ID associated with a Hugging Face Hub repo or a local filepath.
148
+ repo_type (`str`, *optional*):
149
+ The type of Hugging Face repo to push to. Defaults to None, which will use "model". Other options
150
+ are "dataset" and "space". Not used when loading from a local filepath. If this is called from a child
151
+ class, the default value will be the child class's `repo_type`.
152
+ token (`str`, *optional*):
153
+ Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to the stored token.
154
+ ignore_metadata_errors (`str`):
155
+ If True, errors while parsing the metadata section will be ignored. Some information might be lost during
156
+ the process. Use it at your own risk.
157
+
158
+ Returns:
159
+ [`huggingface_hub.repocard.RepoCard`]: The RepoCard (or subclass) initialized from the repo's
160
+ README.md file or filepath.
161
+
162
+ Example:
163
+ ```python
164
+ >>> from huggingface_hub.repocard import RepoCard
165
+ >>> card = RepoCard.load("nateraw/food")
166
+ >>> assert card.data.tags == ["generated_from_trainer", "image-classification", "pytorch"]
167
+
168
+ ```
169
+ """
170
+
171
+ if Path(repo_id_or_path).is_file():
172
+ card_path = Path(repo_id_or_path)
173
+ elif isinstance(repo_id_or_path, str):
174
+ card_path = Path(
175
+ hf_hub_download(
176
+ repo_id_or_path,
177
+ constants.REPOCARD_NAME,
178
+ repo_type=repo_type or cls.repo_type,
179
+ token=token,
180
+ )
181
+ )
182
+ else:
183
+ raise ValueError(f"Cannot load RepoCard: path not found on disk ({repo_id_or_path}).")
184
+
185
+ # Preserve newlines in the existing file.
186
+ with card_path.open(mode="r", newline="", encoding="utf-8") as f:
187
+ return cls(f.read(), ignore_metadata_errors=ignore_metadata_errors)
188
+
189
+ def validate(self, repo_type: str | None = None):
190
+ """Validates card against Hugging Face Hub's card validation logic.
191
+ Using this function requires access to the internet, so it is only called
192
+ internally by [`huggingface_hub.repocard.RepoCard.push_to_hub`].
193
+
194
+ Args:
195
+ repo_type (`str`, *optional*, defaults to "model"):
196
+ The type of Hugging Face repo to push to. Options are "model", "dataset", and "space".
197
+ If this function is called from a child class, the default will be the child class's `repo_type`.
198
+
199
+ > [!TIP]
200
+ > Raises the following errors:
201
+ >
202
+ > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
203
+ > if the card fails validation checks.
204
+ > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
205
+ > if the request to the Hub API fails for any other reason.
206
+ """
207
+
208
+ # If repo type is provided, otherwise, use the repo type of the card.
209
+ repo_type = repo_type or self.repo_type
210
+
211
+ body = {
212
+ "repoType": repo_type,
213
+ "content": str(self),
214
+ }
215
+ headers = {"Accept": "text/plain"}
216
+
217
+ try:
218
+ response = get_session().post("https://huggingface.co/api/validate-yaml", json=body, headers=headers)
219
+ hf_raise_for_status(response)
220
+ except HfHubHTTPError as exc:
221
+ if response.status_code == 400:
222
+ raise ValueError(response.text)
223
+ else:
224
+ raise exc
225
+
226
+ def push_to_hub(
227
+ self,
228
+ repo_id: str,
229
+ token: str | None = None,
230
+ repo_type: str | None = None,
231
+ commit_message: str | None = None,
232
+ commit_description: str | None = None,
233
+ revision: str | None = None,
234
+ create_pr: bool | None = None,
235
+ parent_commit: str | None = None,
236
+ ):
237
+ """Push a RepoCard to a Hugging Face Hub repo.
238
+
239
+ Args:
240
+ repo_id (`str`):
241
+ The repo ID of the Hugging Face Hub repo to push to. Example: "nateraw/food".
242
+ token (`str`, *optional*):
243
+ Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to
244
+ the stored token.
245
+ repo_type (`str`, *optional*, defaults to "model"):
246
+ The type of Hugging Face repo to push to. Options are "model", "dataset", and "space". If this
247
+ function is called by a child class, it will default to the child class's `repo_type`.
248
+ commit_message (`str`, *optional*):
249
+ The summary / title / first line of the generated commit.
250
+ commit_description (`str`, *optional*)
251
+ The description of the generated commit.
252
+ revision (`str`, *optional*):
253
+ The git revision to commit from. Defaults to the head of the `"main"` branch.
254
+ create_pr (`bool`, *optional*):
255
+ Whether or not to create a Pull Request with this commit. Defaults to `False`.
256
+ parent_commit (`str`, *optional*):
257
+ The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported.
258
+ If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
259
+ If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
260
+ Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
261
+ especially useful if the repo is updated / committed too concurrently.
262
+ Returns:
263
+ `str`: URL of the commit which updated the card metadata.
264
+ """
265
+
266
+ # If repo type is provided, otherwise, use the repo type of the card.
267
+ repo_type = repo_type or self.repo_type
268
+
269
+ # Validate card before pushing to hub
270
+ self.validate(repo_type=repo_type)
271
+
272
+ with SoftTemporaryDirectory() as tmpdir:
273
+ tmp_path = Path(tmpdir) / constants.REPOCARD_NAME
274
+ tmp_path.write_text(str(self), encoding="utf-8")
275
+ url = upload_file(
276
+ path_or_fileobj=str(tmp_path),
277
+ path_in_repo=constants.REPOCARD_NAME,
278
+ repo_id=repo_id,
279
+ token=token,
280
+ repo_type=repo_type,
281
+ commit_message=commit_message,
282
+ commit_description=commit_description,
283
+ create_pr=create_pr,
284
+ revision=revision,
285
+ parent_commit=parent_commit,
286
+ )
287
+ return url
288
+
289
+ @classmethod
290
+ def from_template(
291
+ cls,
292
+ card_data: CardData,
293
+ template_path: str | None = None,
294
+ template_str: str | None = None,
295
+ **template_kwargs,
296
+ ):
297
+ """Initialize a RepoCard from a template. By default, it uses the default template.
298
+
299
+ Templates are Jinja2 templates that can be customized by passing keyword arguments.
300
+
301
+ Args:
302
+ card_data (`huggingface_hub.CardData`):
303
+ A huggingface_hub.CardData instance containing the metadata you want to include in the YAML
304
+ header of the repo card on the Hugging Face Hub.
305
+ template_path (`str`, *optional*):
306
+ A path to a markdown file with optional Jinja template variables that can be filled
307
+ in with `template_kwargs`. Defaults to the default template.
308
+ template_str (`str`, *optional*):
309
+ A raw Jinja template string with optional variables. Used when neither `template_path`
310
+ nor the default template is appropriate. Ignored if `template_path` is also provided.
311
+
312
+ Returns:
313
+ [`huggingface_hub.repocard.RepoCard`]: A RepoCard instance with the specified card data and content from the
314
+ template.
315
+ """
316
+ if is_jinja_available():
317
+ import jinja2
318
+ else:
319
+ raise ImportError(
320
+ "Using RepoCard.from_template requires Jinja2 to be installed. Please"
321
+ " install it with `pip install Jinja2`."
322
+ )
323
+
324
+ kwargs = card_data.to_dict().copy()
325
+ kwargs.update(template_kwargs) # Template_kwargs have priority
326
+
327
+ if template_path is not None:
328
+ template_str = Path(template_path).read_text()
329
+ if template_str is None:
330
+ template_str = Path(cls.default_template_path).read_text()
331
+ template = jinja2.Template(template_str)
332
+ content = template.render(card_data=card_data.to_yaml(), **kwargs)
333
+ return cls(content)
334
+
335
+
336
+ class ModelCard(RepoCard):
337
+ card_data_class = ModelCardData # type: ignore[assignment]
338
+ default_template_path = TEMPLATE_MODELCARD_PATH
339
+ repo_type = "model"
340
+
341
+ @classmethod
342
+ def from_template( # type: ignore # violates Liskov property but easier to use
343
+ cls,
344
+ card_data: ModelCardData,
345
+ template_path: str | None = None,
346
+ template_str: str | None = None,
347
+ **template_kwargs,
348
+ ):
349
+ """Initialize a ModelCard from a template. By default, it uses the default template, which can be found here:
350
+ https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/modelcard_template.md
351
+
352
+ Templates are Jinja2 templates that can be customized by passing keyword arguments.
353
+
354
+ Args:
355
+ card_data (`huggingface_hub.ModelCardData`):
356
+ A huggingface_hub.ModelCardData instance containing the metadata you want to include in the YAML
357
+ header of the model card on the Hugging Face Hub.
358
+ template_path (`str`, *optional*):
359
+ A path to a markdown file with optional Jinja template variables that can be filled
360
+ in with `template_kwargs`. Defaults to the default template.
361
+ template_str (`str`, *optional*):
362
+ A raw Jinja template string with optional variables. Used when neither `template_path`
363
+ nor the default template is appropriate. Ignored if `template_path` is also provided.
364
+
365
+ Returns:
366
+ [`huggingface_hub.ModelCard`]: A ModelCard instance with the specified card data and content from the
367
+ template.
368
+
369
+ Example:
370
+ ```python
371
+ >>> from huggingface_hub import ModelCard, ModelCardData, EvalResult
372
+
373
+ >>> # Using the Default Template
374
+ >>> card_data = ModelCardData(
375
+ ... language='en',
376
+ ... license='mit',
377
+ ... library_name='timm',
378
+ ... tags=['image-classification', 'resnet'],
379
+ ... datasets=['beans'],
380
+ ... metrics=['accuracy'],
381
+ ... )
382
+ >>> card = ModelCard.from_template(
383
+ ... card_data,
384
+ ... model_description='This model does x + y...'
385
+ ... )
386
+
387
+ >>> # Including Evaluation Results
388
+ >>> card_data = ModelCardData(
389
+ ... language='en',
390
+ ... tags=['image-classification', 'resnet'],
391
+ ... eval_results=[
392
+ ... EvalResult(
393
+ ... task_type='image-classification',
394
+ ... dataset_type='beans',
395
+ ... dataset_name='Beans',
396
+ ... metric_type='accuracy',
397
+ ... metric_value=0.9,
398
+ ... ),
399
+ ... ],
400
+ ... model_name='my-cool-model',
401
+ ... )
402
+ >>> card = ModelCard.from_template(card_data)
403
+
404
+ >>> # Using a Custom Template
405
+ >>> card_data = ModelCardData(
406
+ ... language='en',
407
+ ... tags=['image-classification', 'resnet']
408
+ ... )
409
+ >>> card = ModelCard.from_template(
410
+ ... card_data=card_data,
411
+ ... template_path='./src/huggingface_hub/templates/modelcard_template.md',
412
+ ... custom_template_var='custom value', # will be replaced in template if it exists
413
+ ... )
414
+
415
+ ```
416
+ """
417
+ return super().from_template(card_data, template_path, template_str, **template_kwargs)
418
+
419
+
420
+ class DatasetCard(RepoCard):
421
+ card_data_class = DatasetCardData # type: ignore[assignment]
422
+ default_template_path = TEMPLATE_DATASETCARD_PATH
423
+ repo_type = "dataset"
424
+
425
+ @classmethod
426
+ def from_template( # type: ignore # violates Liskov property but easier to use
427
+ cls,
428
+ card_data: DatasetCardData,
429
+ template_path: str | None = None,
430
+ template_str: str | None = None,
431
+ **template_kwargs,
432
+ ):
433
+ """Initialize a DatasetCard from a template. By default, it uses the default template, which can be found here:
434
+ https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/datasetcard_template.md
435
+
436
+ Templates are Jinja2 templates that can be customized by passing keyword arguments.
437
+
438
+ Args:
439
+ card_data (`huggingface_hub.DatasetCardData`):
440
+ A huggingface_hub.DatasetCardData instance containing the metadata you want to include in the YAML
441
+ header of the dataset card on the Hugging Face Hub.
442
+ template_path (`str`, *optional*):
443
+ A path to a markdown file with optional Jinja template variables that can be filled
444
+ in with `template_kwargs`. Defaults to the default template.
445
+ template_str (`str`, *optional*):
446
+ A raw Jinja template string with optional variables. Used when neither `template_path`
447
+ nor the default template is appropriate. Ignored if `template_path` is also provided.
448
+
449
+ Returns:
450
+ [`huggingface_hub.DatasetCard`]: A DatasetCard instance with the specified card data and content from the
451
+ template.
452
+
453
+ Example:
454
+ ```python
455
+ >>> from huggingface_hub import DatasetCard, DatasetCardData
456
+
457
+ >>> # Using the Default Template
458
+ >>> card_data = DatasetCardData(
459
+ ... language='en',
460
+ ... license='mit',
461
+ ... annotations_creators='crowdsourced',
462
+ ... task_categories=['text-classification'],
463
+ ... task_ids=['sentiment-classification', 'text-scoring'],
464
+ ... multilinguality='monolingual',
465
+ ... pretty_name='My Text Classification Dataset',
466
+ ... )
467
+ >>> card = DatasetCard.from_template(
468
+ ... card_data,
469
+ ... pretty_name=card_data.pretty_name,
470
+ ... )
471
+
472
+ >>> # Using a Custom Template
473
+ >>> card_data = DatasetCardData(
474
+ ... language='en',
475
+ ... license='mit',
476
+ ... )
477
+ >>> card = DatasetCard.from_template(
478
+ ... card_data=card_data,
479
+ ... template_path='./src/huggingface_hub/templates/datasetcard_template.md',
480
+ ... custom_template_var='custom value', # will be replaced in template if it exists
481
+ ... )
482
+
483
+ ```
484
+ """
485
+ return super().from_template(card_data, template_path, template_str, **template_kwargs)
486
+
487
+
488
+ class SpaceCard(RepoCard):
489
+ card_data_class = SpaceCardData # type: ignore[assignment]
490
+ default_template_path = TEMPLATE_MODELCARD_PATH
491
+ repo_type = "space"
492
+
493
+
494
+ def _detect_line_ending(content: str) -> Literal["\r", "\n", "\r\n", None]: # noqa: F722
495
+ """Detect the line ending of a string. Used by RepoCard to avoid making huge diff on newlines.
496
+
497
+ Uses same implementation as in Hub server, keep it in sync.
498
+
499
+ Returns:
500
+ str: The detected line ending of the string.
501
+ """
502
+ cr = content.count("\r")
503
+ lf = content.count("\n")
504
+ crlf = content.count("\r\n")
505
+ if cr + lf == 0:
506
+ return None
507
+ if crlf == cr and crlf == lf:
508
+ return "\r\n"
509
+ if cr > lf:
510
+ return "\r"
511
+ else:
512
+ return "\n"
513
+
514
+
515
+ def metadata_load(local_path: str | Path) -> dict | None:
516
+ content = Path(local_path).read_text()
517
+ match = REGEX_YAML_BLOCK.search(content)
518
+ if match:
519
+ yaml_block = match.group(2)
520
+ data = yaml.safe_load(yaml_block)
521
+ if data is None or isinstance(data, dict):
522
+ return data
523
+ raise ValueError("repo card metadata block should be a dict")
524
+ else:
525
+ return None
526
+
527
+
528
+ def metadata_save(local_path: str | Path, data: dict) -> None:
529
+ """
530
+ Save the metadata dict in the upper YAML part Trying to preserve newlines as
531
+ in the existing file. Docs about open() with newline="" parameter:
532
+ https://docs.python.org/3/library/functions.html?highlight=open#open Does
533
+ not work with "^M" linebreaks, which are replaced by \n
534
+ """
535
+ line_break = "\n"
536
+ content = ""
537
+ # try to detect existing newline character
538
+ if os.path.exists(local_path):
539
+ with open(local_path, newline="", encoding="utf8") as readme:
540
+ content = readme.read()
541
+ if isinstance(readme.newlines, tuple):
542
+ line_break = readme.newlines[0]
543
+ elif isinstance(readme.newlines, str):
544
+ line_break = readme.newlines
545
+
546
+ # creates a new file if it not
547
+ with open(local_path, "w", newline="", encoding="utf8") as readme:
548
+ data_yaml = yaml_dump(data, sort_keys=False, line_break=line_break)
549
+ # sort_keys: keep dict order
550
+ match = REGEX_YAML_BLOCK.search(content)
551
+ if match:
552
+ output = content[: match.start()] + f"---{line_break}{data_yaml}---{line_break}" + content[match.end() :]
553
+ else:
554
+ output = f"---{line_break}{data_yaml}---{line_break}{content}"
555
+
556
+ readme.write(output)
557
+ readme.close()
558
+
559
+
560
+ def metadata_eval_result(
561
+ *,
562
+ model_pretty_name: str,
563
+ task_pretty_name: str,
564
+ task_id: str,
565
+ metrics_pretty_name: str,
566
+ metrics_id: str,
567
+ metrics_value: Any,
568
+ dataset_pretty_name: str,
569
+ dataset_id: str,
570
+ metrics_config: str | None = None,
571
+ metrics_verified: bool = False,
572
+ dataset_config: str | None = None,
573
+ dataset_split: str | None = None,
574
+ dataset_revision: str | None = None,
575
+ metrics_verification_token: str | None = None,
576
+ ) -> dict:
577
+ """
578
+ Creates a metadata dict with the result from a model evaluated on a dataset.
579
+
580
+ Args:
581
+ model_pretty_name (`str`):
582
+ The name of the model in natural language.
583
+ task_pretty_name (`str`):
584
+ The name of a task in natural language.
585
+ task_id (`str`):
586
+ Example: automatic-speech-recognition. A task id.
587
+ metrics_pretty_name (`str`):
588
+ A name for the metric in natural language. Example: Test WER.
589
+ metrics_id (`str`):
590
+ Example: wer. A metric id from https://hf.co/metrics.
591
+ metrics_value (`Any`):
592
+ The value from the metric. Example: 20.0 or "20.0 ± 1.2".
593
+ dataset_pretty_name (`str`):
594
+ The name of the dataset in natural language.
595
+ dataset_id (`str`):
596
+ Example: common_voice. A dataset id from https://hf.co/datasets.
597
+ metrics_config (`str`, *optional*):
598
+ The name of the metric configuration used in `load_metric()`.
599
+ Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.
600
+ metrics_verified (`bool`, *optional*, defaults to `False`):
601
+ Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.
602
+ dataset_config (`str`, *optional*):
603
+ Example: fr. The name of the dataset configuration used in `load_dataset()`.
604
+ dataset_split (`str`, *optional*):
605
+ Example: test. The name of the dataset split used in `load_dataset()`.
606
+ dataset_revision (`str`, *optional*):
607
+ Example: 5503434ddd753f426f4b38109466949a1217c2bb. The name of the dataset dataset revision
608
+ used in `load_dataset()`.
609
+ metrics_verification_token (`bool`, *optional*):
610
+ A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.
611
+
612
+ Returns:
613
+ `dict`: a metadata dict with the result from a model evaluated on a dataset.
614
+
615
+ Example:
616
+ ```python
617
+ >>> from huggingface_hub import metadata_eval_result
618
+ >>> results = metadata_eval_result(
619
+ ... model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
620
+ ... task_pretty_name="Text Classification",
621
+ ... task_id="text-classification",
622
+ ... metrics_pretty_name="Accuracy",
623
+ ... metrics_id="accuracy",
624
+ ... metrics_value=0.2662102282047272,
625
+ ... dataset_pretty_name="ReactionJPEG",
626
+ ... dataset_id="julien-c/reactionjpeg",
627
+ ... dataset_config="default",
628
+ ... dataset_split="test",
629
+ ... )
630
+ >>> results == {
631
+ ... 'model-index': [
632
+ ... {
633
+ ... 'name': 'RoBERTa fine-tuned on ReactionGIF',
634
+ ... 'results': [
635
+ ... {
636
+ ... 'task': {
637
+ ... 'type': 'text-classification',
638
+ ... 'name': 'Text Classification'
639
+ ... },
640
+ ... 'dataset': {
641
+ ... 'name': 'ReactionJPEG',
642
+ ... 'type': 'julien-c/reactionjpeg',
643
+ ... 'config': 'default',
644
+ ... 'split': 'test'
645
+ ... },
646
+ ... 'metrics': [
647
+ ... {
648
+ ... 'type': 'accuracy',
649
+ ... 'value': 0.2662102282047272,
650
+ ... 'name': 'Accuracy',
651
+ ... 'verified': False
652
+ ... }
653
+ ... ]
654
+ ... }
655
+ ... ]
656
+ ... }
657
+ ... ]
658
+ ... }
659
+ True
660
+
661
+ ```
662
+ """
663
+
664
+ return {
665
+ "model-index": eval_results_to_model_index(
666
+ model_name=model_pretty_name,
667
+ eval_results=[
668
+ EvalResult(
669
+ task_name=task_pretty_name,
670
+ task_type=task_id,
671
+ metric_name=metrics_pretty_name,
672
+ metric_type=metrics_id,
673
+ metric_value=metrics_value,
674
+ dataset_name=dataset_pretty_name,
675
+ dataset_type=dataset_id,
676
+ metric_config=metrics_config,
677
+ verified=metrics_verified,
678
+ verify_token=metrics_verification_token,
679
+ dataset_config=dataset_config,
680
+ dataset_split=dataset_split,
681
+ dataset_revision=dataset_revision,
682
+ )
683
+ ],
684
+ )
685
+ }
686
+
687
+
688
+ @validate_hf_hub_args
689
+ def metadata_update(
690
+ repo_id: str,
691
+ metadata: dict,
692
+ *,
693
+ repo_type: str | None = None,
694
+ overwrite: bool = False,
695
+ token: str | None = None,
696
+ commit_message: str | None = None,
697
+ commit_description: str | None = None,
698
+ revision: str | None = None,
699
+ create_pr: bool = False,
700
+ parent_commit: str | None = None,
701
+ ) -> str:
702
+ """
703
+ Updates the metadata in the README.md of a repository on the Hugging Face Hub.
704
+ If the README.md file doesn't exist yet, a new one is created with metadata and
705
+ the default ModelCard or DatasetCard template. For `space` repo, an error is thrown
706
+ as a Space cannot exist without a `README.md` file.
707
+
708
+ Args:
709
+ repo_id (`str`):
710
+ The name of the repository.
711
+ metadata (`dict`):
712
+ A dictionary containing the metadata to be updated.
713
+ repo_type (`str`, *optional*):
714
+ Set to `"dataset"` or `"space"` if updating to a dataset or space,
715
+ `None` or `"model"` if updating to a model. Default is `None`.
716
+ overwrite (`bool`, *optional*, defaults to `False`):
717
+ If set to `True` an existing field can be overwritten, otherwise
718
+ attempting to overwrite an existing field will cause an error.
719
+ token (`str`, *optional*):
720
+ The Hugging Face authentication token.
721
+ commit_message (`str`, *optional*):
722
+ The summary / title / first line of the generated commit. Defaults to
723
+ `f"Update metadata with huggingface_hub"`
724
+ commit_description (`str` *optional*)
725
+ The description of the generated commit
726
+ revision (`str`, *optional*):
727
+ The git revision to commit from. Defaults to the head of the
728
+ `"main"` branch.
729
+ create_pr (`boolean`, *optional*):
730
+ Whether or not to create a Pull Request from `revision` with that commit.
731
+ Defaults to `False`.
732
+ parent_commit (`str`, *optional*):
733
+ The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported.
734
+ If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
735
+ If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
736
+ Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
737
+ especially useful if the repo is updated / committed too concurrently.
738
+ Returns:
739
+ `str`: URL of the commit which updated the card metadata.
740
+
741
+ Example:
742
+ ```python
743
+ >>> from huggingface_hub import metadata_update
744
+ >>> metadata = {'model-index': [{'name': 'RoBERTa fine-tuned on ReactionGIF',
745
+ ... 'results': [{'dataset': {'name': 'ReactionGIF',
746
+ ... 'type': 'julien-c/reactiongif'},
747
+ ... 'metrics': [{'name': 'Recall',
748
+ ... 'type': 'recall',
749
+ ... 'value': 0.7762102282047272}],
750
+ ... 'task': {'name': 'Text Classification',
751
+ ... 'type': 'text-classification'}}]}]}
752
+ >>> url = metadata_update("hf-internal-testing/reactiongif-roberta-card", metadata)
753
+
754
+ ```
755
+ """
756
+ commit_message = commit_message if commit_message is not None else "Update metadata with huggingface_hub"
757
+
758
+ # Card class given repo_type
759
+ card_class: type[RepoCard]
760
+ if repo_type is None or repo_type == "model":
761
+ card_class = ModelCard
762
+ elif repo_type == "dataset":
763
+ card_class = DatasetCard
764
+ elif repo_type == "space":
765
+ card_class = RepoCard
766
+ else:
767
+ raise ValueError(f"Unknown repo_type: {repo_type}")
768
+
769
+ # Either load repo_card from the Hub or create an empty one.
770
+ # NOTE: Will not create the repo if it doesn't exist.
771
+ try:
772
+ card = card_class.load(repo_id, token=token, repo_type=repo_type)
773
+ except EntryNotFoundError:
774
+ if repo_type == "space":
775
+ raise ValueError("Cannot update metadata on a Space that doesn't contain a `README.md` file.")
776
+
777
+ # Initialize a ModelCard or DatasetCard from default template and no data.
778
+ # Cast to the concrete expected card type to satisfy type checkers.
779
+ card = card_class.from_template(CardData()) # type: ignore
780
+
781
+ for key, value in metadata.items():
782
+ if key == "model-index":
783
+ # if the new metadata doesn't include a name, either use existing one or repo name
784
+ if "name" not in value[0]:
785
+ value[0]["name"] = getattr(card, "model_name", repo_id)
786
+ model_name, new_results = model_index_to_eval_results(value)
787
+ if card.data.eval_results is None:
788
+ card.data.eval_results = new_results
789
+ card.data.model_name = model_name
790
+ else:
791
+ existing_results = card.data.eval_results
792
+
793
+ # Iterate over new results
794
+ # Iterate over existing results
795
+ # If both results describe the same metric but value is different:
796
+ # If overwrite=True: overwrite the metric value
797
+ # Else: raise ValueError
798
+ # Else: append new result to existing ones.
799
+ for new_result in new_results:
800
+ result_found = False
801
+ for existing_result in existing_results:
802
+ if new_result.is_equal_except_value(existing_result):
803
+ if new_result != existing_result and not overwrite:
804
+ raise ValueError(
805
+ "You passed a new value for the existing metric"
806
+ f" 'name: {new_result.metric_name}, type: "
807
+ f"{new_result.metric_type}'. Set `overwrite=True`"
808
+ " to overwrite existing metrics."
809
+ )
810
+ result_found = True
811
+ existing_result.metric_value = new_result.metric_value
812
+ if existing_result.verified is True:
813
+ existing_result.verify_token = new_result.verify_token
814
+ if not result_found:
815
+ card.data.eval_results.append(new_result)
816
+ else:
817
+ # Any metadata that is not a result metric
818
+ if card.data.get(key) is not None and not overwrite and card.data.get(key) != value:
819
+ raise ValueError(
820
+ f"You passed a new value for the existing meta data field '{key}'."
821
+ " Set `overwrite=True` to overwrite existing metadata."
822
+ )
823
+ else:
824
+ card.data[key] = value
825
+
826
+ return card.push_to_hub(
827
+ repo_id,
828
+ token=token,
829
+ repo_type=repo_type,
830
+ commit_message=commit_message,
831
+ commit_description=commit_description,
832
+ create_pr=create_pr,
833
+ revision=revision,
834
+ parent_commit=parent_commit,
835
+ )
.venv/lib/python3.13/site-packages/huggingface_hub/repocard_data.py ADDED
@@ -0,0 +1,776 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ from collections import defaultdict
3
+ from dataclasses import dataclass
4
+ from typing import Any
5
+
6
+ from huggingface_hub.utils import logging, yaml_dump
7
+
8
+
9
+ logger = logging.get_logger(__name__)
10
+
11
+
12
+ @dataclass
13
+ class EvalResult:
14
+ """
15
+ Flattened representation of individual evaluation results found in model-index of Model Cards.
16
+
17
+ For more information on the model-index spec, see https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1.
18
+
19
+ Args:
20
+ task_type (`str`):
21
+ The task identifier. Example: "image-classification".
22
+ dataset_type (`str`):
23
+ The dataset identifier. Example: "common_voice". Use dataset id from https://hf.co/datasets.
24
+ dataset_name (`str`):
25
+ A pretty name for the dataset. Example: "Common Voice (French)".
26
+ metric_type (`str`):
27
+ The metric identifier. Example: "wer". Use metric id from https://hf.co/metrics.
28
+ metric_value (`Any`):
29
+ The metric value. Example: 0.9 or "20.0 ± 1.2".
30
+ task_name (`str`, *optional*):
31
+ A pretty name for the task. Example: "Speech Recognition".
32
+ dataset_config (`str`, *optional*):
33
+ The name of the dataset configuration used in `load_dataset()`.
34
+ Example: fr in `load_dataset("common_voice", "fr")`. See the `datasets` docs for more info:
35
+ https://hf.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name
36
+ dataset_split (`str`, *optional*):
37
+ The split used in `load_dataset()`. Example: "test".
38
+ dataset_revision (`str`, *optional*):
39
+ The revision (AKA Git Sha) of the dataset used in `load_dataset()`.
40
+ Example: 5503434ddd753f426f4b38109466949a1217c2bb
41
+ dataset_args (`dict[str, Any]`, *optional*):
42
+ The arguments passed during `Metric.compute()`. Example for `bleu`: `{"max_order": 4}`
43
+ metric_name (`str`, *optional*):
44
+ A pretty name for the metric. Example: "Test WER".
45
+ metric_config (`str`, *optional*):
46
+ The name of the metric configuration used in `load_metric()`.
47
+ Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.
48
+ See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations
49
+ metric_args (`dict[str, Any]`, *optional*):
50
+ The arguments passed during `Metric.compute()`. Example for `bleu`: max_order: 4
51
+ verified (`bool`, *optional*):
52
+ Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.
53
+ verify_token (`str`, *optional*):
54
+ A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.
55
+ source_name (`str`, *optional*):
56
+ The name of the source of the evaluation result. Example: "Open LLM Leaderboard".
57
+ source_url (`str`, *optional*):
58
+ The URL of the source of the evaluation result. Example: "https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard".
59
+ """
60
+
61
+ # Required
62
+
63
+ # The task identifier
64
+ # Example: automatic-speech-recognition
65
+ task_type: str
66
+
67
+ # The dataset identifier
68
+ # Example: common_voice. Use dataset id from https://hf.co/datasets
69
+ dataset_type: str
70
+
71
+ # A pretty name for the dataset.
72
+ # Example: Common Voice (French)
73
+ dataset_name: str
74
+
75
+ # The metric identifier
76
+ # Example: wer. Use metric id from https://hf.co/metrics
77
+ metric_type: str
78
+
79
+ # Value of the metric.
80
+ # Example: 20.0 or "20.0 ± 1.2"
81
+ metric_value: Any
82
+
83
+ # Optional
84
+
85
+ # A pretty name for the task.
86
+ # Example: Speech Recognition
87
+ task_name: str | None = None
88
+
89
+ # The name of the dataset configuration used in `load_dataset()`.
90
+ # Example: fr in `load_dataset("common_voice", "fr")`.
91
+ # See the `datasets` docs for more info:
92
+ # https://huggingface.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name
93
+ dataset_config: str | None = None
94
+
95
+ # The split used in `load_dataset()`.
96
+ # Example: test
97
+ dataset_split: str | None = None
98
+
99
+ # The revision (AKA Git Sha) of the dataset used in `load_dataset()`.
100
+ # Example: 5503434ddd753f426f4b38109466949a1217c2bb
101
+ dataset_revision: str | None = None
102
+
103
+ # The arguments passed during `Metric.compute()`.
104
+ # Example for `bleu`: max_order: 4
105
+ dataset_args: dict[str, Any] | None = None
106
+
107
+ # A pretty name for the metric.
108
+ # Example: Test WER
109
+ metric_name: str | None = None
110
+
111
+ # The name of the metric configuration used in `load_metric()`.
112
+ # Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.
113
+ # See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations
114
+ metric_config: str | None = None
115
+
116
+ # The arguments passed during `Metric.compute()`.
117
+ # Example for `bleu`: max_order: 4
118
+ metric_args: dict[str, Any] | None = None
119
+
120
+ # Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.
121
+ verified: bool | None = None
122
+
123
+ # A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.
124
+ verify_token: str | None = None
125
+
126
+ # The name of the source of the evaluation result.
127
+ # Example: Open LLM Leaderboard
128
+ source_name: str | None = None
129
+
130
+ # The URL of the source of the evaluation result.
131
+ # Example: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard
132
+ source_url: str | None = None
133
+
134
+ @property
135
+ def unique_identifier(self) -> tuple:
136
+ """Returns a tuple that uniquely identifies this evaluation."""
137
+ return (
138
+ self.task_type,
139
+ self.dataset_type,
140
+ self.dataset_config,
141
+ self.dataset_split,
142
+ self.dataset_revision,
143
+ )
144
+
145
+ def is_equal_except_value(self, other: "EvalResult") -> bool:
146
+ """
147
+ Return True if `self` and `other` describe exactly the same metric but with a
148
+ different value.
149
+ """
150
+ for key, _ in self.__dict__.items():
151
+ if key == "metric_value":
152
+ continue
153
+ # For metrics computed by Hugging Face's evaluation service, `verify_token` is derived from `metric_value`,
154
+ # so we exclude it here in the comparison.
155
+ if key != "verify_token" and getattr(self, key) != getattr(other, key):
156
+ return False
157
+ return True
158
+
159
+ def __post_init__(self) -> None:
160
+ if self.source_name is not None and self.source_url is None:
161
+ raise ValueError("If `source_name` is provided, `source_url` must also be provided.")
162
+
163
+
164
+ @dataclass
165
+ class CardData:
166
+ """Structure containing metadata from a RepoCard.
167
+
168
+ [`CardData`] is the parent class of [`ModelCardData`] and [`DatasetCardData`].
169
+
170
+ Metadata can be exported as a dictionary or YAML. Export can be customized to alter the representation of the data
171
+ (example: flatten evaluation results). `CardData` behaves as a dictionary (can get, pop, set values) but do not
172
+ inherit from `dict` to allow this export step.
173
+ """
174
+
175
+ def __init__(self, ignore_metadata_errors: bool = False, **kwargs):
176
+ self.__dict__.update(kwargs)
177
+
178
+ def to_dict(self):
179
+ """Converts CardData to a dict.
180
+
181
+ Returns:
182
+ `dict`: CardData represented as a dictionary ready to be dumped to a YAML
183
+ block for inclusion in a README.md file.
184
+ """
185
+
186
+ data_dict = copy.deepcopy(self.__dict__)
187
+ self._to_dict(data_dict)
188
+ return {key: value for key, value in data_dict.items() if value is not None}
189
+
190
+ def _to_dict(self, data_dict):
191
+ """Use this method in child classes to alter the dict representation of the data. Alter the dict in-place.
192
+
193
+ Args:
194
+ data_dict (`dict`): The raw dict representation of the card data.
195
+ """
196
+ pass
197
+
198
+ def to_yaml(self, line_break=None, original_order: list[str] | None = None) -> str:
199
+ """Dumps CardData to a YAML block for inclusion in a README.md file.
200
+
201
+ Args:
202
+ line_break (str, *optional*):
203
+ The line break to use when dumping to yaml.
204
+ original_order (`list[str]`, *optional*):
205
+ If provided, reorder the metadata fields to match this list before dumping.
206
+ Any keys not in `original_order` are appended after the listed keys, preserving
207
+ their existing relative order. Useful for round-tripping a YAML block without
208
+ shuffling its keys.
209
+
210
+ Returns:
211
+ `str`: CardData represented as a YAML block.
212
+ """
213
+ if original_order:
214
+ original_order_set = set(original_order)
215
+ self.__dict__ = {
216
+ k: self.__dict__[k]
217
+ for k in original_order + [k for k in self.__dict__ if k not in original_order_set]
218
+ if k in self.__dict__
219
+ }
220
+ return yaml_dump(self.to_dict(), sort_keys=False, line_break=line_break).strip()
221
+
222
+ def __repr__(self):
223
+ return repr(self.__dict__)
224
+
225
+ def __str__(self):
226
+ return self.to_yaml()
227
+
228
+ def get(self, key: str, default: Any = None) -> Any:
229
+ """Get value for a given metadata key."""
230
+ value = self.__dict__.get(key)
231
+ return default if value is None else value
232
+
233
+ def pop(self, key: str, default: Any = None) -> Any:
234
+ """Pop value for a given metadata key."""
235
+ return self.__dict__.pop(key, default)
236
+
237
+ def __getitem__(self, key: str) -> Any:
238
+ """Get value for a given metadata key."""
239
+ return self.__dict__[key]
240
+
241
+ def __setitem__(self, key: str, value: Any) -> None:
242
+ """Set value for a given metadata key."""
243
+ self.__dict__[key] = value
244
+
245
+ def __contains__(self, key: str) -> bool:
246
+ """Check if a given metadata key is set."""
247
+ return key in self.__dict__
248
+
249
+ def __len__(self) -> int:
250
+ """Return the number of metadata keys set."""
251
+ return len(self.__dict__)
252
+
253
+
254
+ def _validate_eval_results(
255
+ eval_results: EvalResult | list[EvalResult] | None,
256
+ model_name: str | None,
257
+ ) -> list[EvalResult]:
258
+ if eval_results is None:
259
+ return []
260
+ if isinstance(eval_results, EvalResult):
261
+ eval_results = [eval_results]
262
+ if not isinstance(eval_results, list) or not all(isinstance(r, EvalResult) for r in eval_results):
263
+ raise ValueError(
264
+ f"`eval_results` should be of type `EvalResult` or a list of `EvalResult`, got {type(eval_results)}."
265
+ )
266
+ if model_name is None:
267
+ raise ValueError("Passing `eval_results` requires `model_name` to be set.")
268
+ return eval_results
269
+
270
+
271
+ class ModelCardData(CardData):
272
+ """Model Card Metadata that is used by Hugging Face Hub when included at the top of your README.md
273
+
274
+ Args:
275
+ base_model (`str` or `list[str]`, *optional*):
276
+ The identifier of the base model from which the model derives. This is applicable for example if your model is a
277
+ fine-tune or adapter of an existing model. The value must be the ID of a model on the Hub (or a list of IDs
278
+ if your model derives from multiple models). Defaults to None.
279
+ datasets (`Union[str, list[str]]`, *optional*):
280
+ Dataset or list of datasets that were used to train this model. Should be a dataset ID
281
+ found on https://hf.co/datasets. Defaults to None.
282
+ eval_results (`Union[list[EvalResult], EvalResult]`, *optional*):
283
+ List of `huggingface_hub.EvalResult` that define evaluation results of the model. If provided,
284
+ `model_name` is used to as a name on PapersWithCode's leaderboards. Defaults to `None`.
285
+ language (`Union[str, list[str]]`, *optional*):
286
+ Language of model's training data or metadata. It must be an ISO 639-1, 639-2 or
287
+ 639-3 code (two/three letters), or a special value like "code", "multilingual". Defaults to `None`.
288
+ library_name (`str`, *optional*):
289
+ Name of library used by this model. Example: keras or any library from
290
+ https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/model-libraries.ts.
291
+ Defaults to None.
292
+ license (`str`, *optional*):
293
+ License of this model. Example: apache-2.0 or any license from
294
+ https://huggingface.co/docs/hub/repositories-licenses. Defaults to None.
295
+ license_name (`str`, *optional*):
296
+ Name of the license of this model. Defaults to None. To be used in conjunction with `license_link`.
297
+ Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a name. In that case, use `license` instead.
298
+ license_link (`str`, *optional*):
299
+ Link to the license of this model. Defaults to None. To be used in conjunction with `license_name`.
300
+ Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a link. In that case, use `license` instead.
301
+ metrics (`list[str]`, *optional*):
302
+ List of metrics used to evaluate this model. Should be a metric name that can be found
303
+ at https://hf.co/metrics. Example: 'accuracy'. Defaults to None.
304
+ model_name (`str`, *optional*):
305
+ A name for this model. It is used along with
306
+ `eval_results` to construct the `model-index` within the card's metadata. The name
307
+ you supply here is what will be used on PapersWithCode's leaderboards. If None is provided
308
+ then the repo name is used as a default. Defaults to None.
309
+ pipeline_tag (`str`, *optional*):
310
+ The pipeline tag associated with the model. Example: "text-classification".
311
+ tags (`list[str]`, *optional*):
312
+ List of tags to add to your model that can be used when filtering on the Hugging
313
+ Face Hub. Defaults to None.
314
+ ignore_metadata_errors (`str`):
315
+ If True, errors while parsing the metadata section will be ignored. Some information might be lost during
316
+ the process. Use it at your own risk.
317
+ kwargs (`dict`, *optional*):
318
+ Additional metadata that will be added to the model card. Defaults to None.
319
+
320
+ Example:
321
+ ```python
322
+ >>> from huggingface_hub import ModelCardData
323
+ >>> card_data = ModelCardData(
324
+ ... language="en",
325
+ ... license="mit",
326
+ ... library_name="timm",
327
+ ... tags=['image-classification', 'resnet'],
328
+ ... )
329
+ >>> card_data.to_dict()
330
+ {'language': 'en', 'license': 'mit', 'library_name': 'timm', 'tags': ['image-classification', 'resnet']}
331
+
332
+ ```
333
+ """
334
+
335
+ def __init__(
336
+ self,
337
+ *,
338
+ base_model: str | list[str] | None = None,
339
+ datasets: str | list[str] | None = None,
340
+ eval_results: list[EvalResult] | None = None,
341
+ language: str | list[str] | None = None,
342
+ library_name: str | None = None,
343
+ license: str | None = None,
344
+ license_name: str | None = None,
345
+ license_link: str | None = None,
346
+ metrics: list[str] | None = None,
347
+ model_name: str | None = None,
348
+ pipeline_tag: str | None = None,
349
+ tags: list[str] | None = None,
350
+ ignore_metadata_errors: bool = False,
351
+ **kwargs,
352
+ ):
353
+ self.base_model = base_model
354
+ self.datasets = datasets
355
+ self.eval_results = eval_results
356
+ self.language = language
357
+ self.library_name = library_name
358
+ self.license = license
359
+ self.license_name = license_name
360
+ self.license_link = license_link
361
+ self.metrics = metrics
362
+ self.model_name = model_name
363
+ self.pipeline_tag = pipeline_tag
364
+ self.tags = _to_unique_list(tags)
365
+
366
+ model_index = kwargs.pop("model-index", None)
367
+ if model_index:
368
+ try:
369
+ model_name, eval_results = model_index_to_eval_results(model_index)
370
+ self.model_name = model_name
371
+ self.eval_results = eval_results
372
+ except (KeyError, TypeError) as error:
373
+ if ignore_metadata_errors:
374
+ logger.warning("Invalid model-index. Not loading eval results into CardData.")
375
+ else:
376
+ raise ValueError(
377
+ f"Invalid `model_index` in metadata cannot be parsed: {error.__class__} {error}. Pass"
378
+ " `ignore_metadata_errors=True` to ignore this error while loading a Model Card. Warning:"
379
+ " some information will be lost. Use it at your own risk."
380
+ )
381
+
382
+ super().__init__(**kwargs)
383
+
384
+ if self.eval_results:
385
+ try:
386
+ self.eval_results = _validate_eval_results(self.eval_results, self.model_name)
387
+ except Exception as e:
388
+ if ignore_metadata_errors:
389
+ logger.warning(f"Failed to validate eval_results: {e}. Not loading eval results into CardData.")
390
+ else:
391
+ raise ValueError(f"Failed to validate eval_results: {e}") from e
392
+
393
+ def _to_dict(self, data_dict):
394
+ """Format the internal data dict. In this case, we convert eval results to a valid model index"""
395
+ if self.eval_results is not None:
396
+ data_dict["model-index"] = eval_results_to_model_index(self.model_name, self.eval_results) # type: ignore
397
+ del data_dict["eval_results"], data_dict["model_name"]
398
+
399
+
400
+ class DatasetCardData(CardData):
401
+ """Dataset Card Metadata that is used by Hugging Face Hub when included at the top of your README.md
402
+
403
+ Args:
404
+ language (`list[str]`, *optional*):
405
+ Language of dataset's data or metadata. It must be an ISO 639-1, 639-2 or
406
+ 639-3 code (two/three letters), or a special value like "code", "multilingual".
407
+ license (`Union[str, list[str]]`, *optional*):
408
+ License(s) of this dataset. Example: apache-2.0 or any license from
409
+ https://huggingface.co/docs/hub/repositories-licenses.
410
+ annotations_creators (`Union[str, list[str]]`, *optional*):
411
+ How the annotations for the dataset were created.
412
+ Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'no-annotation', 'other'.
413
+ language_creators (`Union[str, list[str]]`, *optional*):
414
+ How the text-based data in the dataset was created.
415
+ Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'other'
416
+ multilinguality (`Union[str, list[str]]`, *optional*):
417
+ Whether the dataset is multilingual.
418
+ Options are: 'monolingual', 'multilingual', 'translation', 'other'.
419
+ size_categories (`Union[str, list[str]]`, *optional*):
420
+ The number of examples in the dataset. Options are: 'n<1K', '1K<n<10K', '10K<n<100K',
421
+ '100K<n<1M', '1M<n<10M', '10M<n<100M', '100M<n<1B', '1B<n<10B', '10B<n<100B', '100B<n<1T', 'n>1T', and 'other'.
422
+ source_datasets (`list[str]]`, *optional*):
423
+ Indicates whether the dataset is an original dataset or extended from another existing dataset.
424
+ Options are: 'original' and 'extended'.
425
+ task_categories (`Union[str, list[str]]`, *optional*):
426
+ What categories of task does the dataset support?
427
+ task_ids (`Union[str, list[str]]`, *optional*):
428
+ What specific tasks does the dataset support?
429
+ paperswithcode_id (`str`, *optional*):
430
+ ID of the dataset on PapersWithCode.
431
+ pretty_name (`str`, *optional*):
432
+ A more human-readable name for the dataset. (ex. "Cats vs. Dogs")
433
+ train_eval_index (`dict`, *optional*):
434
+ A dictionary that describes the necessary spec for doing evaluation on the Hub.
435
+ If not provided, it will be gathered from the 'train-eval-index' key of the kwargs.
436
+ config_names (`Union[str, list[str]]`, *optional*):
437
+ A list of the available dataset configs for the dataset.
438
+ """
439
+
440
+ def __init__(
441
+ self,
442
+ *,
443
+ language: str | list[str] | None = None,
444
+ license: str | list[str] | None = None,
445
+ annotations_creators: str | list[str] | None = None,
446
+ language_creators: str | list[str] | None = None,
447
+ multilinguality: str | list[str] | None = None,
448
+ size_categories: str | list[str] | None = None,
449
+ source_datasets: list[str] | None = None,
450
+ task_categories: str | list[str] | None = None,
451
+ task_ids: str | list[str] | None = None,
452
+ paperswithcode_id: str | None = None,
453
+ pretty_name: str | None = None,
454
+ train_eval_index: dict | None = None,
455
+ config_names: str | list[str] | None = None,
456
+ ignore_metadata_errors: bool = False,
457
+ **kwargs,
458
+ ):
459
+ self.annotations_creators = annotations_creators
460
+ self.language_creators = language_creators
461
+ self.language = language
462
+ self.license = license
463
+ self.multilinguality = multilinguality
464
+ self.size_categories = size_categories
465
+ self.source_datasets = source_datasets
466
+ self.task_categories = task_categories
467
+ self.task_ids = task_ids
468
+ self.paperswithcode_id = paperswithcode_id
469
+ self.pretty_name = pretty_name
470
+ self.config_names = config_names
471
+
472
+ # TODO - maybe handle this similarly to EvalResult?
473
+ self.train_eval_index = train_eval_index or kwargs.pop("train-eval-index", None)
474
+ super().__init__(**kwargs)
475
+
476
+ def _to_dict(self, data_dict):
477
+ data_dict["train-eval-index"] = data_dict.pop("train_eval_index")
478
+
479
+
480
+ class SpaceCardData(CardData):
481
+ """Space Card Metadata that is used by Hugging Face Hub when included at the top of your README.md
482
+
483
+ To get an exhaustive reference of Spaces configuration, please visit https://huggingface.co/docs/hub/spaces-config-reference#spaces-configuration-reference.
484
+
485
+ Args:
486
+ title (`str`, *optional*)
487
+ Title of the Space.
488
+ sdk (`str`, *optional*)
489
+ SDK of the Space (one of `gradio`, `streamlit`, `docker`, or `static`).
490
+ sdk_version (`str`, *optional*)
491
+ Version of the used SDK (if Gradio/Streamlit sdk).
492
+ python_version (`str`, *optional*)
493
+ Python version used in the Space (if Gradio/Streamlit sdk).
494
+ app_file (`str`, *optional*)
495
+ Path to your main application file (which contains either gradio or streamlit Python code, or static html code).
496
+ Path is relative to the root of the repository.
497
+ app_port (`str`, *optional*)
498
+ Port on which your application is running. Used only if sdk is `docker`.
499
+ license (`str`, *optional*)
500
+ License of this model. Example: apache-2.0 or any license from
501
+ https://huggingface.co/docs/hub/repositories-licenses.
502
+ duplicated_from (`str`, *optional*)
503
+ ID of the original Space if this is a duplicated Space.
504
+ models (list[`str`], *optional*)
505
+ List of models related to this Space. Should be a dataset ID found on https://hf.co/models.
506
+ datasets (`list[str]`, *optional*)
507
+ List of datasets related to this Space. Should be a dataset ID found on https://hf.co/datasets.
508
+ tags (`list[str]`, *optional*)
509
+ List of tags to add to your Space that can be used when filtering on the Hub.
510
+ ignore_metadata_errors (`str`):
511
+ If True, errors while parsing the metadata section will be ignored. Some information might be lost during
512
+ the process. Use it at your own risk.
513
+ kwargs (`dict`, *optional*):
514
+ Additional metadata that will be added to the space card.
515
+
516
+ Example:
517
+ ```python
518
+ >>> from huggingface_hub import SpaceCardData
519
+ >>> card_data = SpaceCardData(
520
+ ... title="Dreambooth Training",
521
+ ... license="mit",
522
+ ... sdk="gradio",
523
+ ... duplicated_from="multimodalart/dreambooth-training"
524
+ ... )
525
+ >>> card_data.to_dict()
526
+ {'title': 'Dreambooth Training', 'sdk': 'gradio', 'license': 'mit', 'duplicated_from': 'multimodalart/dreambooth-training'}
527
+ ```
528
+ """
529
+
530
+ def __init__(
531
+ self,
532
+ *,
533
+ title: str | None = None,
534
+ sdk: str | None = None,
535
+ sdk_version: str | None = None,
536
+ python_version: str | None = None,
537
+ app_file: str | None = None,
538
+ app_port: int | None = None,
539
+ license: str | None = None,
540
+ duplicated_from: str | None = None,
541
+ models: list[str] | None = None,
542
+ datasets: list[str] | None = None,
543
+ tags: list[str] | None = None,
544
+ ignore_metadata_errors: bool = False,
545
+ **kwargs,
546
+ ):
547
+ self.title = title
548
+ self.sdk = sdk
549
+ self.sdk_version = sdk_version
550
+ self.python_version = python_version
551
+ self.app_file = app_file
552
+ self.app_port = app_port
553
+ self.license = license
554
+ self.duplicated_from = duplicated_from
555
+ self.models = models
556
+ self.datasets = datasets
557
+ self.tags = _to_unique_list(tags)
558
+ super().__init__(**kwargs)
559
+
560
+
561
+ def model_index_to_eval_results(model_index: list[dict[str, Any]]) -> tuple[str, list[EvalResult]]:
562
+ """Takes in a model index and returns the model name and a list of `huggingface_hub.EvalResult` objects.
563
+
564
+ A detailed spec of the model index can be found here:
565
+ https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1
566
+
567
+ Args:
568
+ model_index (`list[dict[str, Any]]`):
569
+ A model index data structure, likely coming from a README.md file on the
570
+ Hugging Face Hub.
571
+
572
+ Returns:
573
+ model_name (`str`):
574
+ The name of the model as found in the model index. This is used as the
575
+ identifier for the model on leaderboards like PapersWithCode.
576
+ eval_results (`list[EvalResult]`):
577
+ A list of `huggingface_hub.EvalResult` objects containing the metrics
578
+ reported in the provided model_index.
579
+
580
+ Example:
581
+ ```python
582
+ >>> from huggingface_hub.repocard_data import model_index_to_eval_results
583
+ >>> # Define a minimal model index
584
+ >>> model_index = [
585
+ ... {
586
+ ... "name": "my-cool-model",
587
+ ... "results": [
588
+ ... {
589
+ ... "task": {
590
+ ... "type": "image-classification"
591
+ ... },
592
+ ... "dataset": {
593
+ ... "type": "beans",
594
+ ... "name": "Beans"
595
+ ... },
596
+ ... "metrics": [
597
+ ... {
598
+ ... "type": "accuracy",
599
+ ... "value": 0.9
600
+ ... }
601
+ ... ]
602
+ ... }
603
+ ... ]
604
+ ... }
605
+ ... ]
606
+ >>> model_name, eval_results = model_index_to_eval_results(model_index)
607
+ >>> model_name
608
+ 'my-cool-model'
609
+ >>> eval_results[0].task_type
610
+ 'image-classification'
611
+ >>> eval_results[0].metric_type
612
+ 'accuracy'
613
+
614
+ ```
615
+ """
616
+
617
+ eval_results = []
618
+ for elem in model_index:
619
+ name = elem["name"]
620
+ results = elem["results"]
621
+ for result in results:
622
+ task_type = result["task"]["type"]
623
+ task_name = result["task"].get("name")
624
+ dataset_type = result["dataset"]["type"]
625
+ dataset_name = result["dataset"]["name"]
626
+ dataset_config = result["dataset"].get("config")
627
+ dataset_split = result["dataset"].get("split")
628
+ dataset_revision = result["dataset"].get("revision")
629
+ dataset_args = result["dataset"].get("args")
630
+ source_name = result.get("source", {}).get("name")
631
+ source_url = result.get("source", {}).get("url")
632
+
633
+ for metric in result["metrics"]:
634
+ metric_type = metric["type"]
635
+ metric_value = metric["value"]
636
+ metric_name = metric.get("name")
637
+ metric_args = metric.get("args")
638
+ metric_config = metric.get("config")
639
+ verified = metric.get("verified")
640
+ verify_token = metric.get("verifyToken")
641
+
642
+ eval_result = EvalResult(
643
+ task_type=task_type, # Required
644
+ dataset_type=dataset_type, # Required
645
+ dataset_name=dataset_name, # Required
646
+ metric_type=metric_type, # Required
647
+ metric_value=metric_value, # Required
648
+ task_name=task_name,
649
+ dataset_config=dataset_config,
650
+ dataset_split=dataset_split,
651
+ dataset_revision=dataset_revision,
652
+ dataset_args=dataset_args,
653
+ metric_name=metric_name,
654
+ metric_args=metric_args,
655
+ metric_config=metric_config,
656
+ verified=verified,
657
+ verify_token=verify_token,
658
+ source_name=source_name,
659
+ source_url=source_url,
660
+ )
661
+ eval_results.append(eval_result)
662
+ return name, eval_results
663
+
664
+
665
+ def _remove_none(obj):
666
+ """
667
+ Recursively remove `None` values from a dict. Borrowed from: https://stackoverflow.com/a/20558778
668
+ """
669
+ if isinstance(obj, (list, tuple, set)):
670
+ return type(obj)(_remove_none(x) for x in obj if x is not None)
671
+ elif isinstance(obj, dict):
672
+ return type(obj)((_remove_none(k), _remove_none(v)) for k, v in obj.items() if k is not None and v is not None)
673
+ else:
674
+ return obj
675
+
676
+
677
+ def eval_results_to_model_index(model_name: str, eval_results: list[EvalResult]) -> list[dict[str, Any]]:
678
+ """Takes in given model name and list of `huggingface_hub.EvalResult` and returns a
679
+ valid model-index that will be compatible with the format expected by the
680
+ Hugging Face Hub.
681
+
682
+ Args:
683
+ model_name (`str`):
684
+ Name of the model (ex. "my-cool-model"). This is used as the identifier
685
+ for the model on leaderboards like PapersWithCode.
686
+ eval_results (`list[EvalResult]`):
687
+ List of `huggingface_hub.EvalResult` objects containing the metrics to be
688
+ reported in the model-index.
689
+
690
+ Returns:
691
+ model_index (`list[dict[str, Any]]`): The eval_results converted to a model-index.
692
+
693
+ Example:
694
+ ```python
695
+ >>> from huggingface_hub.repocard_data import eval_results_to_model_index, EvalResult
696
+ >>> # Define minimal eval_results
697
+ >>> eval_results = [
698
+ ... EvalResult(
699
+ ... task_type="image-classification", # Required
700
+ ... dataset_type="beans", # Required
701
+ ... dataset_name="Beans", # Required
702
+ ... metric_type="accuracy", # Required
703
+ ... metric_value=0.9, # Required
704
+ ... )
705
+ ... ]
706
+ >>> eval_results_to_model_index("my-cool-model", eval_results)
707
+ [{'name': 'my-cool-model', 'results': [{'task': {'type': 'image-classification'}, 'dataset': {'name': 'Beans', 'type': 'beans'}, 'metrics': [{'type': 'accuracy', 'value': 0.9}]}]}]
708
+
709
+ ```
710
+ """
711
+
712
+ # Metrics are reported on a unique task-and-dataset basis.
713
+ # Here, we make a map of those pairs and the associated EvalResults.
714
+ task_and_ds_types_map: dict[Any, list[EvalResult]] = defaultdict(list)
715
+ for eval_result in eval_results:
716
+ task_and_ds_types_map[eval_result.unique_identifier].append(eval_result)
717
+
718
+ # Use the map from above to generate the model index data.
719
+ model_index_data: list[dict[str, Any]] = []
720
+ for results in task_and_ds_types_map.values():
721
+ # All items from `results` share same metadata
722
+ sample_result = results[0]
723
+ data: dict[str, Any] = {
724
+ "task": {
725
+ "type": sample_result.task_type,
726
+ "name": sample_result.task_name,
727
+ },
728
+ "dataset": {
729
+ "name": sample_result.dataset_name,
730
+ "type": sample_result.dataset_type,
731
+ "config": sample_result.dataset_config,
732
+ "split": sample_result.dataset_split,
733
+ "revision": sample_result.dataset_revision,
734
+ "args": sample_result.dataset_args,
735
+ },
736
+ "metrics": [
737
+ {
738
+ "type": result.metric_type,
739
+ "value": result.metric_value,
740
+ "name": result.metric_name,
741
+ "config": result.metric_config,
742
+ "args": result.metric_args,
743
+ "verified": result.verified,
744
+ "verifyToken": result.verify_token,
745
+ }
746
+ for result in results
747
+ ],
748
+ }
749
+ if sample_result.source_url is not None:
750
+ source: dict[str, str] = {
751
+ "url": sample_result.source_url,
752
+ }
753
+ if sample_result.source_name is not None:
754
+ source["name"] = sample_result.source_name
755
+ data["source"] = source
756
+ model_index_data.append(data)
757
+
758
+ # TODO - Check if there cases where this list is longer than one?
759
+ # Finally, the model index itself is list of dicts.
760
+ model_index = [
761
+ {
762
+ "name": model_name,
763
+ "results": model_index_data,
764
+ }
765
+ ]
766
+ return _remove_none(model_index)
767
+
768
+
769
+ def _to_unique_list(tags: list[str] | None) -> list[str] | None:
770
+ if tags is None:
771
+ return tags
772
+ unique_tags = [] # make tags unique + keep order explicitly
773
+ for tag in tags:
774
+ if tag not in unique_tags:
775
+ unique_tags.append(tag)
776
+ return unique_tags
.venv/lib/python3.13/site-packages/huggingface_hub/serialization/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2024 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ # ruff: noqa: F401
15
+ """Contains helpers to serialize tensors."""
16
+
17
+ from ._base import StateDictSplit, split_state_dict_into_shards_factory
18
+ from ._torch import (
19
+ get_torch_storage_id,
20
+ get_torch_storage_size,
21
+ load_state_dict_from_file,
22
+ load_torch_model,
23
+ save_torch_model,
24
+ save_torch_state_dict,
25
+ split_torch_state_dict_into_shards,
26
+ )
.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_base.py ADDED
@@ -0,0 +1,212 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2024 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains helpers to split tensors into shards."""
15
+
16
+ from collections.abc import Callable
17
+ from dataclasses import dataclass, field
18
+ from typing import Any, TypeVar
19
+
20
+ from .. import logging
21
+
22
+
23
+ TensorT = TypeVar("TensorT")
24
+ TensorSizeFn_T = Callable[[TensorT], int]
25
+ StorageIDFn_T = Callable[[TensorT], Any | None]
26
+
27
+ MAX_SHARD_SIZE = "5GB"
28
+ SIZE_UNITS = {
29
+ "TB": 10**12,
30
+ "GB": 10**9,
31
+ "MB": 10**6,
32
+ "KB": 10**3,
33
+ }
34
+
35
+
36
+ logger = logging.get_logger(__file__)
37
+
38
+
39
+ @dataclass
40
+ class StateDictSplit:
41
+ is_sharded: bool = field(init=False)
42
+ metadata: dict[str, Any]
43
+ filename_to_tensors: dict[str, list[str]]
44
+ tensor_to_filename: dict[str, str]
45
+
46
+ def __post_init__(self):
47
+ self.is_sharded = len(self.filename_to_tensors) > 1
48
+
49
+
50
+ def split_state_dict_into_shards_factory(
51
+ state_dict: dict[str, TensorT],
52
+ *,
53
+ get_storage_size: TensorSizeFn_T,
54
+ filename_pattern: str,
55
+ get_storage_id: StorageIDFn_T = lambda tensor: None,
56
+ max_shard_size: int | str = MAX_SHARD_SIZE,
57
+ ) -> StateDictSplit:
58
+ """
59
+ Split a model state dictionary in shards so that each shard is smaller than a given size.
60
+
61
+ The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization
62
+ made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we
63
+ have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not
64
+ [6+2+2GB], [6+2GB], [6GB].
65
+
66
+ > [!WARNING]
67
+ > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a
68
+ > size greater than `max_shard_size`.
69
+
70
+ Args:
71
+ state_dict (`dict[str, Tensor]`):
72
+ The state dictionary to save.
73
+ get_storage_size (`Callable[[Tensor], int]`):
74
+ A function that returns the size of a tensor when saved on disk in bytes.
75
+ get_storage_id (`Callable[[Tensor], Optional[Any]]`, *optional*):
76
+ A function that returns a unique identifier to a tensor storage. Multiple different tensors can share the
77
+ same underlying storage. This identifier is guaranteed to be unique and constant for this tensor's storage
78
+ during its lifetime. Two tensor storages with non-overlapping lifetimes may have the same id.
79
+ filename_pattern (`str`, *optional*):
80
+ The pattern to generate the files names in which the model will be saved. Pattern must be a string that
81
+ can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
82
+ max_shard_size (`int` or `str`, *optional*):
83
+ The maximum size of each shard, in bytes. Defaults to 5GB.
84
+
85
+ Returns:
86
+ [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them.
87
+ """
88
+ storage_id_to_tensors: dict[Any, list[str]] = {}
89
+
90
+ shard_list: list[dict[str, TensorT]] = []
91
+ current_shard: dict[str, TensorT] = {}
92
+ current_shard_size = 0
93
+ total_size = 0
94
+
95
+ if isinstance(max_shard_size, str):
96
+ max_shard_size = parse_size_to_int(max_shard_size)
97
+
98
+ for key, tensor in state_dict.items():
99
+ # when bnb serialization is used the weights in the state dict can be strings
100
+ # check: https://github.com/huggingface/transformers/pull/24416 for more details
101
+ if isinstance(tensor, str):
102
+ logger.info("Skipping tensor %s as it is a string (bnb serialization)", key)
103
+ continue
104
+
105
+ # If a `tensor` shares the same underlying storage as another tensor, we put `tensor` in the same `block`
106
+ storage_id = get_storage_id(tensor) # type: ignore[invalid-argument-type]
107
+ if storage_id is not None:
108
+ if storage_id in storage_id_to_tensors:
109
+ # We skip this tensor for now and will reassign to correct shard later
110
+ storage_id_to_tensors[storage_id].append(key)
111
+ continue
112
+ else:
113
+ # This is the first tensor with this storage_id, we create a new entry
114
+ # in the storage_id_to_tensors dict => we will assign the shard id later
115
+ storage_id_to_tensors[storage_id] = [key]
116
+
117
+ # Compute tensor size
118
+ tensor_size = get_storage_size(tensor) # type: ignore[invalid-argument-type]
119
+
120
+ # If this tensor is bigger than the maximal size, we put it in its own shard
121
+ if tensor_size > max_shard_size:
122
+ total_size += tensor_size
123
+ shard_list.append({key: tensor})
124
+ continue
125
+
126
+ # If this tensor is going to tip up over the maximal size, we split.
127
+ # Current shard already has some tensors, we add it to the list of shards and create a new one.
128
+ if current_shard_size + tensor_size > max_shard_size:
129
+ shard_list.append(current_shard)
130
+ current_shard = {}
131
+ current_shard_size = 0
132
+
133
+ # Add the tensor to the current shard
134
+ current_shard[key] = tensor
135
+ current_shard_size += tensor_size
136
+ total_size += tensor_size
137
+
138
+ # Add the last shard
139
+ if len(current_shard) > 0:
140
+ shard_list.append(current_shard)
141
+ nb_shards = len(shard_list)
142
+
143
+ # Loop over the tensors that share the same storage and assign them together
144
+ for storage_id, keys in storage_id_to_tensors.items():
145
+ # Let's try to find the shard where the first tensor of this storage is and put all tensors in the same shard
146
+ for shard in shard_list:
147
+ if keys[0] in shard:
148
+ for key in keys:
149
+ shard[key] = state_dict[key]
150
+ break
151
+
152
+ # If we only have one shard, we return it => no need to build the index
153
+ if nb_shards == 1:
154
+ filename = filename_pattern.format(suffix="")
155
+ # Use the keys from the shard itself rather than `state_dict` directly, so that tensors
156
+ # skipped above (e.g. string tensors from bnb serialization) are excluded from the index,
157
+ # consistently with the multi-shard path below.
158
+ keys = list(shard_list[0].keys())
159
+ return StateDictSplit(
160
+ metadata={"total_size": total_size},
161
+ filename_to_tensors={filename: keys},
162
+ tensor_to_filename={key: filename for key in keys},
163
+ )
164
+
165
+ # Now that each tensor is assigned to a shard, let's assign a filename to each shard
166
+ tensor_name_to_filename = {}
167
+ filename_to_tensors = {}
168
+ for idx, shard in enumerate(shard_list):
169
+ filename = filename_pattern.format(suffix=f"-{idx + 1:05d}-of-{nb_shards:05d}")
170
+ for key in shard:
171
+ tensor_name_to_filename[key] = filename
172
+ filename_to_tensors[filename] = list(shard.keys())
173
+
174
+ # Build the index and return
175
+ return StateDictSplit(
176
+ metadata={"total_size": total_size},
177
+ filename_to_tensors=filename_to_tensors,
178
+ tensor_to_filename=tensor_name_to_filename,
179
+ )
180
+
181
+
182
+ def parse_size_to_int(size_as_str: str) -> int:
183
+ """
184
+ Parse a size expressed as a string with digits and unit (like `"5MB"`) to an integer (in bytes).
185
+
186
+ Supported units are "TB", "GB", "MB", "KB".
187
+
188
+ Args:
189
+ size_as_str (`str`): The size to convert. Will be directly returned if an `int`.
190
+
191
+ Example:
192
+
193
+ ```py
194
+ >>> parse_size_to_int("5MB")
195
+ 5000000
196
+ ```
197
+ """
198
+ size_as_str = size_as_str.strip()
199
+
200
+ # Parse unit
201
+ unit = size_as_str[-2:].upper()
202
+ if unit not in SIZE_UNITS:
203
+ raise ValueError(f"Unit '{unit}' not supported. Supported units are TB, GB, MB, KB. Got '{size_as_str}'.")
204
+ multiplier = SIZE_UNITS[unit]
205
+
206
+ # Parse value
207
+ try:
208
+ value = float(size_as_str[:-2].strip())
209
+ except ValueError as e:
210
+ raise ValueError(f"Could not parse the size value from '{size_as_str}': {e}") from e
211
+
212
+ return int(value * multiplier)
.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_dduf.py ADDED
@@ -0,0 +1,386 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import logging
3
+ import mmap
4
+ import os
5
+ import shutil
6
+ import zipfile
7
+ from collections.abc import Generator, Iterable
8
+ from contextlib import contextmanager
9
+ from dataclasses import dataclass, field
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from ..errors import DDUFCorruptedFileError, DDUFExportError, DDUFInvalidEntryNameError
14
+
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ DDUF_ALLOWED_ENTRIES = {
19
+ # Allowed file extensions in a DDUF file
20
+ ".json",
21
+ ".model",
22
+ ".safetensors",
23
+ ".txt",
24
+ }
25
+
26
+ DDUF_FOLDER_REQUIRED_ENTRIES = {
27
+ # Each folder must contain at least one of these entries
28
+ "config.json",
29
+ "tokenizer_config.json",
30
+ "preprocessor_config.json",
31
+ "scheduler_config.json",
32
+ }
33
+
34
+
35
+ @dataclass
36
+ class DDUFEntry:
37
+ """Object representing a file entry in a DDUF file.
38
+
39
+ See [`read_dduf_file`] for how to read a DDUF file.
40
+
41
+ Attributes:
42
+ filename (str):
43
+ The name of the file in the DDUF archive.
44
+ offset (int):
45
+ The offset of the file in the DDUF archive.
46
+ length (int):
47
+ The length of the file in the DDUF archive.
48
+ dduf_path (str):
49
+ The path to the DDUF archive (for internal use).
50
+ """
51
+
52
+ filename: str
53
+ length: int
54
+ offset: int
55
+
56
+ dduf_path: Path = field(repr=False)
57
+
58
+ @contextmanager
59
+ def as_mmap(self) -> Generator[bytes, None, None]:
60
+ """Open the file as a memory-mapped file.
61
+
62
+ Useful to load safetensors directly from the file.
63
+
64
+ Example:
65
+ ```py
66
+ >>> import safetensors.torch
67
+ >>> with entry.as_mmap() as mm:
68
+ ... tensors = safetensors.torch.load(mm)
69
+ ```
70
+ """
71
+ with self.dduf_path.open("rb") as f:
72
+ with mmap.mmap(f.fileno(), length=0, access=mmap.ACCESS_READ) as mm:
73
+ yield mm[self.offset : self.offset + self.length]
74
+
75
+ def read_text(self, encoding: str = "utf-8") -> str:
76
+ """Read the file as text.
77
+
78
+ Useful for '.txt' and '.json' entries.
79
+
80
+ Example:
81
+ ```py
82
+ >>> import json
83
+ >>> index = json.loads(entry.read_text())
84
+ ```
85
+ """
86
+ with self.dduf_path.open("rb") as f:
87
+ f.seek(self.offset)
88
+ return f.read(self.length).decode(encoding=encoding)
89
+
90
+
91
+ def read_dduf_file(dduf_path: os.PathLike | str) -> dict[str, DDUFEntry]:
92
+ """
93
+ Read a DDUF file and return a dictionary of entries.
94
+
95
+ Only the metadata is read, the data is not loaded in memory.
96
+
97
+ Args:
98
+ dduf_path (`str` or `os.PathLike`):
99
+ The path to the DDUF file to read.
100
+
101
+ Returns:
102
+ `dict[str, DDUFEntry]`:
103
+ A dictionary of [`DDUFEntry`] indexed by filename.
104
+
105
+ Raises:
106
+ - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format).
107
+
108
+ Example:
109
+ ```python
110
+ >>> import json
111
+ >>> import safetensors.torch
112
+ >>> from huggingface_hub import read_dduf_file
113
+
114
+ # Read DDUF metadata
115
+ >>> dduf_entries = read_dduf_file("FLUX.1-dev.dduf")
116
+
117
+ # Returns a mapping filename <> DDUFEntry
118
+ >>> dduf_entries["model_index.json"]
119
+ DDUFEntry(filename='model_index.json', offset=66, length=587)
120
+
121
+ # Load model index as JSON
122
+ >>> json.loads(dduf_entries["model_index.json"].read_text())
123
+ {'_class_name': 'FluxPipeline', '_diffusers_version': '0.32.0.dev0', '_name_or_path': 'black-forest-labs/FLUX.1-dev', ...
124
+
125
+ # Load VAE weights using safetensors
126
+ >>> with dduf_entries["vae/diffusion_pytorch_model.safetensors"].as_mmap() as mm:
127
+ ... state_dict = safetensors.torch.load(mm)
128
+ ```
129
+ """
130
+ entries = {}
131
+ dduf_path = Path(dduf_path)
132
+ logger.info(f"Reading DDUF file {dduf_path}")
133
+ with zipfile.ZipFile(str(dduf_path), "r") as zf:
134
+ for info in zf.infolist():
135
+ logger.debug(f"Reading entry {info.filename}")
136
+ if info.compress_type != zipfile.ZIP_STORED:
137
+ raise DDUFCorruptedFileError("Data must not be compressed in DDUF file.")
138
+
139
+ try:
140
+ _validate_dduf_entry_name(info.filename)
141
+ except DDUFInvalidEntryNameError as e:
142
+ raise DDUFCorruptedFileError(f"Invalid entry name in DDUF file: {info.filename}") from e
143
+
144
+ offset = _get_data_offset(zf, info)
145
+
146
+ entries[info.filename] = DDUFEntry(
147
+ filename=info.filename, offset=offset, length=info.file_size, dduf_path=dduf_path
148
+ )
149
+
150
+ # Consistency checks on the DDUF file
151
+ if "model_index.json" not in entries:
152
+ raise DDUFCorruptedFileError("Missing required 'model_index.json' entry in DDUF file.")
153
+ index = json.loads(entries["model_index.json"].read_text())
154
+ _validate_dduf_structure(index, entries.keys())
155
+
156
+ logger.info(f"Done reading DDUF file {dduf_path}. Found {len(entries)} entries")
157
+ return entries
158
+
159
+
160
+ def export_entries_as_dduf(dduf_path: str | os.PathLike, entries: Iterable[tuple[str, str | Path | bytes]]) -> None:
161
+ """Write a DDUF file from an iterable of entries.
162
+
163
+ This is a lower-level helper than [`export_folder_as_dduf`] that allows more flexibility when serializing data.
164
+ In particular, you don't need to save the data on disk before exporting it in the DDUF file.
165
+
166
+ Args:
167
+ dduf_path (`str` or `os.PathLike`):
168
+ The path to the DDUF file to write.
169
+ entries (`Iterable[tuple[str, Union[str, Path, bytes]]]`):
170
+ An iterable of entries to write in the DDUF file. Each entry is a tuple with the filename and the content.
171
+ The filename should be the path to the file in the DDUF archive.
172
+ The content can be a string or a pathlib.Path representing a path to a file on the local disk or directly the content as bytes.
173
+
174
+ Raises:
175
+ - [`DDUFExportError`]: If anything goes wrong during the export (e.g. invalid entry name, missing 'model_index.json', etc.).
176
+
177
+ Example:
178
+ ```python
179
+ # Export specific files from the local disk.
180
+ >>> from huggingface_hub import export_entries_as_dduf
181
+ >>> export_entries_as_dduf(
182
+ ... dduf_path="stable-diffusion-v1-4-FP16.dduf",
183
+ ... entries=[ # List entries to add to the DDUF file (here, only FP16 weights)
184
+ ... ("model_index.json", "path/to/model_index.json"),
185
+ ... ("vae/config.json", "path/to/vae/config.json"),
186
+ ... ("vae/diffusion_pytorch_model.fp16.safetensors", "path/to/vae/diffusion_pytorch_model.fp16.safetensors"),
187
+ ... ("text_encoder/config.json", "path/to/text_encoder/config.json"),
188
+ ... ("text_encoder/model.fp16.safetensors", "path/to/text_encoder/model.fp16.safetensors"),
189
+ ... # ... add more entries here
190
+ ... ]
191
+ ... )
192
+ ```
193
+
194
+ ```python
195
+ # Export state_dicts one by one from a loaded pipeline
196
+ >>> from diffusers import DiffusionPipeline
197
+ >>> from typing import Generator, Tuple
198
+ >>> import safetensors.torch
199
+ >>> from huggingface_hub import export_entries_as_dduf
200
+ >>> pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4")
201
+ ... # ... do some work with the pipeline
202
+
203
+ >>> def as_entries(pipe: DiffusionPipeline) -> Generator[tuple[str, bytes], None, None]:
204
+ ... # Build a generator that yields the entries to add to the DDUF file.
205
+ ... # The first element of the tuple is the filename in the DDUF archive (must use UNIX separator!). The second element is the content of the file.
206
+ ... # Entries will be evaluated lazily when the DDUF file is created (only 1 entry is loaded in memory at a time)
207
+ ... yield "vae/config.json", pipe.vae.to_json_string().encode()
208
+ ... yield "vae/diffusion_pytorch_model.safetensors", safetensors.torch.save(pipe.vae.state_dict())
209
+ ... yield "text_encoder/config.json", pipe.text_encoder.config.to_json_string().encode()
210
+ ... yield "text_encoder/model.safetensors", safetensors.torch.save(pipe.text_encoder.state_dict())
211
+ ... # ... add more entries here
212
+
213
+ >>> export_entries_as_dduf(dduf_path="stable-diffusion-v1-4.dduf", entries=as_entries(pipe))
214
+ ```
215
+ """
216
+ logger.info(f"Exporting DDUF file '{dduf_path}'")
217
+ filenames = set()
218
+ index = None
219
+ with zipfile.ZipFile(str(dduf_path), "w", zipfile.ZIP_STORED) as archive:
220
+ for filename, content in entries:
221
+ if filename in filenames:
222
+ raise DDUFExportError(f"Can't add duplicate entry: {filename}")
223
+ filenames.add(filename)
224
+
225
+ if filename == "model_index.json":
226
+ try:
227
+ index = json.loads(_load_content(content).decode())
228
+ except json.JSONDecodeError as e:
229
+ raise DDUFExportError("Failed to parse 'model_index.json'.") from e
230
+
231
+ try:
232
+ filename = _validate_dduf_entry_name(filename)
233
+ except DDUFInvalidEntryNameError as e:
234
+ raise DDUFExportError(f"Invalid entry name: {filename}") from e
235
+ logger.debug(f"Adding entry '{filename}' to DDUF file")
236
+ _dump_content_in_archive(archive, filename, content)
237
+
238
+ # Consistency checks on the DDUF file
239
+ if index is None:
240
+ raise DDUFExportError("Missing required 'model_index.json' entry in DDUF file.")
241
+ try:
242
+ _validate_dduf_structure(index, filenames)
243
+ except DDUFCorruptedFileError as e:
244
+ raise DDUFExportError("Invalid DDUF file structure.") from e
245
+
246
+ logger.info(f"Done writing DDUF file {dduf_path}")
247
+
248
+
249
+ def export_folder_as_dduf(dduf_path: str | os.PathLike, folder_path: str | os.PathLike) -> None:
250
+ """
251
+ Export a folder as a DDUF file.
252
+
253
+ AUses [`export_entries_as_dduf`] under the hood.
254
+
255
+ Args:
256
+ dduf_path (`str` or `os.PathLike`):
257
+ The path to the DDUF file to write.
258
+ folder_path (`str` or `os.PathLike`):
259
+ The path to the folder containing the diffusion model.
260
+
261
+ Example:
262
+ ```python
263
+ >>> from huggingface_hub import export_folder_as_dduf
264
+ >>> export_folder_as_dduf(dduf_path="FLUX.1-dev.dduf", folder_path="path/to/FLUX.1-dev")
265
+ ```
266
+ """
267
+ folder_path = Path(folder_path)
268
+
269
+ def _iterate_over_folder() -> Iterable[tuple[str, Path]]:
270
+ for path in Path(folder_path).glob("**/*"):
271
+ if not path.is_file():
272
+ continue
273
+ if path.suffix not in DDUF_ALLOWED_ENTRIES:
274
+ logger.debug(f"Skipping file '{path}' (file type not allowed)")
275
+ continue
276
+ path_in_archive = path.relative_to(folder_path)
277
+ if len(path_in_archive.parts) >= 3:
278
+ logger.debug(f"Skipping file '{path}' (nested directories not allowed)")
279
+ continue
280
+ yield path_in_archive.as_posix(), path
281
+
282
+ export_entries_as_dduf(dduf_path, _iterate_over_folder())
283
+
284
+
285
+ def _dump_content_in_archive(archive: zipfile.ZipFile, filename: str, content: str | os.PathLike | bytes) -> None:
286
+ with archive.open(filename, "w", force_zip64=True) as archive_fh:
287
+ if isinstance(content, (str, Path)):
288
+ content_path = Path(content)
289
+ with content_path.open("rb") as content_fh:
290
+ shutil.copyfileobj(content_fh, archive_fh, 1024 * 1024 * 8) # type: ignore[misc]
291
+ elif isinstance(content, bytes):
292
+ archive_fh.write(content)
293
+ else:
294
+ raise DDUFExportError(f"Invalid content type for {filename}. Must be str, Path or bytes.")
295
+
296
+
297
+ def _load_content(content: str | Path | bytes) -> bytes:
298
+ """Load the content of an entry as bytes.
299
+
300
+ Used only for small checks (not to dump content into archive).
301
+ """
302
+ if isinstance(content, (str, Path)):
303
+ return Path(content).read_bytes()
304
+ elif isinstance(content, bytes):
305
+ return content
306
+ else:
307
+ raise DDUFExportError(f"Invalid content type. Must be str, Path or bytes. Got {type(content)}.")
308
+
309
+
310
+ def _validate_dduf_entry_name(entry_name: str) -> str:
311
+ if "." + entry_name.split(".")[-1] not in DDUF_ALLOWED_ENTRIES:
312
+ raise DDUFInvalidEntryNameError(f"File type not allowed: {entry_name}")
313
+ if "\\" in entry_name:
314
+ raise DDUFInvalidEntryNameError(f"Entry names must use UNIX separators ('/'). Got {entry_name}.")
315
+ entry_name = entry_name.strip("/")
316
+ if entry_name.count("/") > 1:
317
+ raise DDUFInvalidEntryNameError(f"DDUF only supports 1 level of directory. Got {entry_name}.")
318
+ return entry_name
319
+
320
+
321
+ def _validate_dduf_structure(index: Any, entry_names: Iterable[str]) -> None:
322
+ """
323
+ Consistency checks on the DDUF file structure.
324
+
325
+ Rules:
326
+ - The 'model_index.json' entry is required and must contain a dictionary.
327
+ - Each folder name must correspond to an entry in 'model_index.json'.
328
+ - Each folder must contain at least a config file ('config.json', 'tokenizer_config.json', 'preprocessor_config.json', 'scheduler_config.json').
329
+
330
+ Args:
331
+ index (Any):
332
+ The content of the 'model_index.json' entry.
333
+ entry_names (Iterable[str]):
334
+ The list of entry names in the DDUF file.
335
+
336
+ Raises:
337
+ - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format).
338
+ """
339
+ if not isinstance(index, dict):
340
+ raise DDUFCorruptedFileError(f"Invalid 'model_index.json' content. Must be a dictionary. Got {type(index)}.")
341
+
342
+ dduf_folders = {entry.split("/")[0] for entry in entry_names if "/" in entry}
343
+ for folder in dduf_folders:
344
+ if folder not in index:
345
+ raise DDUFCorruptedFileError(f"Missing required entry '{folder}' in 'model_index.json'.")
346
+ if not any(f"{folder}/{required_entry}" in entry_names for required_entry in DDUF_FOLDER_REQUIRED_ENTRIES):
347
+ raise DDUFCorruptedFileError(
348
+ f"Missing required file in folder '{folder}'. Must contains at least one of {DDUF_FOLDER_REQUIRED_ENTRIES}."
349
+ )
350
+
351
+
352
+ def _get_data_offset(zf: zipfile.ZipFile, info: zipfile.ZipInfo) -> int:
353
+ """
354
+ Calculate the data offset for a file in a ZIP archive.
355
+
356
+ Args:
357
+ zf (`zipfile.ZipFile`):
358
+ The opened ZIP file. Must be opened in read mode.
359
+ info (`zipfile.ZipInfo`):
360
+ The file info.
361
+
362
+ Returns:
363
+ int: The offset of the file data in the ZIP archive.
364
+ """
365
+ if zf.fp is None:
366
+ raise DDUFCorruptedFileError("ZipFile object must be opened in read mode.")
367
+
368
+ # Step 1: Get the local file header offset
369
+ header_offset = info.header_offset
370
+
371
+ # Step 2: Read the local file header
372
+ zf.fp.seek(header_offset)
373
+ local_file_header = zf.fp.read(30) # Fixed-size part of the local header
374
+
375
+ if len(local_file_header) < 30:
376
+ raise DDUFCorruptedFileError("Incomplete local file header.")
377
+
378
+ # Step 3: Parse the header fields to calculate the start of file data
379
+ # Local file header: https://en.wikipedia.org/wiki/ZIP_(file_format)#File_headers
380
+ filename_len = int.from_bytes(local_file_header[26:28], "little")
381
+ extra_field_len = int.from_bytes(local_file_header[28:30], "little")
382
+
383
+ # Data offset is after the fixed header, filename, and extra fields
384
+ data_offset = header_offset + 30 + filename_len + extra_field_len
385
+
386
+ return data_offset
.venv/lib/python3.13/site-packages/huggingface_hub/serialization/_torch.py ADDED
@@ -0,0 +1,1046 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2024 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains pytorch-specific helpers."""
15
+
16
+ import importlib
17
+ import importlib.util
18
+ import json
19
+ import os
20
+ import re
21
+ from collections import defaultdict, namedtuple
22
+ from collections.abc import Iterable
23
+ from functools import lru_cache
24
+ from pathlib import Path, PureWindowsPath
25
+ from typing import TYPE_CHECKING, Any, NamedTuple, Union
26
+
27
+ from packaging import version
28
+
29
+ from .. import constants, logging
30
+ from ._base import MAX_SHARD_SIZE, StateDictSplit, split_state_dict_into_shards_factory
31
+
32
+
33
+ logger = logging.get_logger(__file__)
34
+
35
+ if TYPE_CHECKING:
36
+ import torch
37
+
38
+ # SAVING
39
+
40
+
41
+ def save_torch_model(
42
+ model: "torch.nn.Module",
43
+ save_directory: str | Path,
44
+ *,
45
+ filename_pattern: str | None = None,
46
+ force_contiguous: bool = True,
47
+ max_shard_size: int | str = MAX_SHARD_SIZE,
48
+ metadata: dict[str, str] | None = None,
49
+ safe_serialization: bool = True,
50
+ is_main_process: bool = True,
51
+ shared_tensors_to_discard: list[str] | None = None,
52
+ ):
53
+ """
54
+ Saves a given torch model to disk, handling sharding and shared tensors issues.
55
+
56
+ See also [`save_torch_state_dict`] to save a state dict with more flexibility.
57
+
58
+ For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors).
59
+
60
+ The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are
61
+ saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard,
62
+ an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses
63
+ [`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as
64
+ safetensors (the default). Otherwise, the shards are saved as pickle.
65
+
66
+ Before saving the model, the `save_directory` is cleaned from any previous shard files.
67
+
68
+ > [!WARNING]
69
+ > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a
70
+ > size greater than `max_shard_size`.
71
+
72
+ > [!WARNING]
73
+ > If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving.
74
+
75
+ Args:
76
+ model (`torch.nn.Module`):
77
+ The model to save on disk.
78
+ save_directory (`str` or `Path`):
79
+ The directory in which the model will be saved.
80
+ filename_pattern (`str`, *optional*):
81
+ The pattern to generate the files names in which the model will be saved. Pattern must be a string that
82
+ can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
83
+ Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization`
84
+ parameter.
85
+ force_contiguous (`boolean`, *optional*):
86
+ Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the
87
+ model, but it could potentially change performance if the layout of the tensor was chosen specifically for
88
+ that reason. Defaults to `True`.
89
+ max_shard_size (`int` or `str`, *optional*):
90
+ The maximum size of each shard, in bytes. Defaults to 5GB.
91
+ metadata (`dict[str, str]`, *optional*):
92
+ Extra information to save along with the model. Some metadata will be added for each dropped tensors.
93
+ This information will not be enough to recover the entire shared structure but might help understanding
94
+ things.
95
+ safe_serialization (`bool`, *optional*):
96
+ Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle.
97
+ Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed
98
+ in a future version.
99
+ is_main_process (`bool`, *optional*):
100
+ Whether the process calling this is the main process or not. Useful when in distributed training like
101
+ TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on
102
+ the main process to avoid race conditions. Defaults to True.
103
+ shared_tensors_to_discard (`list[str]`, *optional*):
104
+ List of tensor names to drop when saving shared tensors. If not provided and shared tensors are
105
+ detected, it will drop the first name alphabetically.
106
+
107
+ Example:
108
+
109
+ ```py
110
+ >>> from huggingface_hub import save_torch_model
111
+ >>> model = ... # A PyTorch model
112
+
113
+ # Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors.
114
+ >>> save_torch_model(model, "path/to/folder")
115
+
116
+ # Load model back
117
+ >>> from huggingface_hub import load_torch_model # TODO
118
+ >>> load_torch_model(model, "path/to/folder")
119
+ >>>
120
+ ```
121
+ """
122
+ save_torch_state_dict(
123
+ state_dict=model.state_dict(),
124
+ filename_pattern=filename_pattern,
125
+ force_contiguous=force_contiguous,
126
+ max_shard_size=max_shard_size,
127
+ metadata=metadata,
128
+ safe_serialization=safe_serialization,
129
+ save_directory=save_directory,
130
+ is_main_process=is_main_process,
131
+ shared_tensors_to_discard=shared_tensors_to_discard,
132
+ )
133
+
134
+
135
+ def save_torch_state_dict(
136
+ state_dict: dict[str, "torch.Tensor"],
137
+ save_directory: str | Path,
138
+ *,
139
+ filename_pattern: str | None = None,
140
+ force_contiguous: bool = True,
141
+ max_shard_size: int | str = MAX_SHARD_SIZE,
142
+ metadata: dict[str, str] | None = None,
143
+ safe_serialization: bool = True,
144
+ is_main_process: bool = True,
145
+ shared_tensors_to_discard: list[str] | None = None,
146
+ ) -> None:
147
+ """
148
+ Save a model state dictionary to the disk, handling sharding and shared tensors issues.
149
+
150
+ See also [`save_torch_model`] to directly save a PyTorch model.
151
+
152
+ For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors).
153
+
154
+ The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are
155
+ saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard,
156
+ an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses
157
+ [`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as
158
+ safetensors (the default). Otherwise, the shards are saved as pickle.
159
+
160
+ Before saving the model, the `save_directory` is cleaned from any previous shard files.
161
+
162
+ > [!WARNING]
163
+ > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a
164
+ > size greater than `max_shard_size`.
165
+
166
+ > [!WARNING]
167
+ > If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving.
168
+
169
+ Args:
170
+ state_dict (`dict[str, torch.Tensor]`):
171
+ The state dictionary to save.
172
+ save_directory (`str` or `Path`):
173
+ The directory in which the model will be saved.
174
+ filename_pattern (`str`, *optional*):
175
+ The pattern to generate the files names in which the model will be saved. Pattern must be a string that
176
+ can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
177
+ Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization`
178
+ parameter.
179
+ force_contiguous (`boolean`, *optional*):
180
+ Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the
181
+ model, but it could potentially change performance if the layout of the tensor was chosen specifically for
182
+ that reason. Defaults to `True`.
183
+ max_shard_size (`int` or `str`, *optional*):
184
+ The maximum size of each shard, in bytes. Defaults to 5GB.
185
+ metadata (`dict[str, str]`, *optional*):
186
+ Extra information to save along with the model. Some metadata will be added for each dropped tensors.
187
+ This information will not be enough to recover the entire shared structure but might help understanding
188
+ things.
189
+ safe_serialization (`bool`, *optional*):
190
+ Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle.
191
+ Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed
192
+ in a future version.
193
+ is_main_process (`bool`, *optional*):
194
+ Whether the process calling this is the main process or not. Useful when in distributed training like
195
+ TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on
196
+ the main process to avoid race conditions. Defaults to True.
197
+ shared_tensors_to_discard (`list[str]`, *optional*):
198
+ List of tensor names to drop when saving shared tensors. If not provided and shared tensors are
199
+ detected, it will drop the first name alphabetically.
200
+
201
+ Example:
202
+
203
+ ```py
204
+ >>> from huggingface_hub import save_torch_state_dict
205
+ >>> model = ... # A PyTorch model
206
+
207
+ # Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors.
208
+ >>> state_dict = model_to_save.state_dict()
209
+ >>> save_torch_state_dict(state_dict, "path/to/folder")
210
+ ```
211
+ """
212
+ save_directory = str(save_directory)
213
+
214
+ if filename_pattern is None:
215
+ filename_pattern = (
216
+ constants.SAFETENSORS_WEIGHTS_FILE_PATTERN
217
+ if safe_serialization
218
+ else constants.PYTORCH_WEIGHTS_FILE_PATTERN
219
+ )
220
+
221
+ if metadata is None:
222
+ metadata = {}
223
+ if safe_serialization:
224
+ try:
225
+ from safetensors.torch import save_file as save_file_fn
226
+ except ImportError as e:
227
+ raise ImportError(
228
+ "Please install `safetensors` to use safe serialization. "
229
+ "You can install it with `pip install safetensors`."
230
+ ) from e
231
+ # Clean state dict for safetensors
232
+ state_dict = _clean_state_dict_for_safetensors(
233
+ state_dict,
234
+ metadata,
235
+ force_contiguous=force_contiguous,
236
+ shared_tensors_to_discard=shared_tensors_to_discard,
237
+ )
238
+ else:
239
+ from torch import save as save_file_fn # type: ignore[assignment, no-redef]
240
+
241
+ logger.warning(
242
+ "You are using unsafe serialization. Due to security reasons, it is recommended not to load "
243
+ "pickled models from untrusted sources. If you intend to share your model, we strongly recommend "
244
+ "using safe serialization by installing `safetensors` with `pip install safetensors`."
245
+ )
246
+ # Split dict
247
+ state_dict_split = split_torch_state_dict_into_shards(
248
+ state_dict, filename_pattern=filename_pattern, max_shard_size=max_shard_size
249
+ )
250
+
251
+ # Only main process should clean up existing files to avoid race conditions in distributed environment
252
+ if is_main_process:
253
+ existing_files_regex = re.compile(filename_pattern.format(suffix=r"(-\d{5}-of-\d{5})?") + r"(\.index\.json)?")
254
+ for filename in os.listdir(save_directory):
255
+ if existing_files_regex.match(filename):
256
+ try:
257
+ logger.debug(f"Removing existing file '{filename}' from folder.")
258
+ os.remove(os.path.join(save_directory, filename))
259
+ except Exception as e:
260
+ logger.warning(
261
+ f"Error when trying to remove existing '{filename}' from folder: {e}. Continuing..."
262
+ )
263
+
264
+ # Save each shard
265
+ per_file_metadata = {"format": "pt"}
266
+ if not state_dict_split.is_sharded:
267
+ per_file_metadata.update(metadata)
268
+ safe_file_kwargs = {"metadata": per_file_metadata} if safe_serialization else {}
269
+ for filename, tensors in state_dict_split.filename_to_tensors.items():
270
+ shard = {tensor: state_dict[tensor] for tensor in tensors}
271
+ save_file_fn(shard, os.path.join(save_directory, filename), **safe_file_kwargs) # ty: ignore[invalid-argument-type]
272
+ logger.debug(f"Shard saved to {filename}")
273
+
274
+ # Save the index (if any)
275
+ if state_dict_split.is_sharded:
276
+ index_path = filename_pattern.format(suffix="") + ".index.json"
277
+ index = {
278
+ "metadata": {**state_dict_split.metadata, **metadata},
279
+ "weight_map": state_dict_split.tensor_to_filename,
280
+ }
281
+ with open(os.path.join(save_directory, index_path), "w") as f:
282
+ json.dump(index, f, indent=2)
283
+ logger.info(
284
+ f"The model is bigger than the maximum size per checkpoint ({max_shard_size}). "
285
+ f"Model weighs have been saved in {len(state_dict_split.filename_to_tensors)} checkpoint shards. "
286
+ f"You can find where each parameters has been saved in the index located at {index_path}."
287
+ )
288
+
289
+ logger.info(f"Model weights successfully saved to {save_directory}!")
290
+
291
+
292
+ def split_torch_state_dict_into_shards(
293
+ state_dict: dict[str, "torch.Tensor"],
294
+ *,
295
+ filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN,
296
+ max_shard_size: int | str = MAX_SHARD_SIZE,
297
+ ) -> StateDictSplit:
298
+ """
299
+ Split a model state dictionary in shards so that each shard is smaller than a given size.
300
+
301
+ The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization
302
+ made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we
303
+ have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not
304
+ [6+2+2GB], [6+2GB], [6GB].
305
+
306
+
307
+ > [!TIP]
308
+ > To save a model state dictionary to the disk, see [`save_torch_state_dict`]. This helper uses
309
+ > `split_torch_state_dict_into_shards` under the hood.
310
+
311
+ > [!WARNING]
312
+ > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a
313
+ > size greater than `max_shard_size`.
314
+
315
+ Args:
316
+ state_dict (`dict[str, torch.Tensor]`):
317
+ The state dictionary to save.
318
+ filename_pattern (`str`, *optional*):
319
+ The pattern to generate the files names in which the model will be saved. Pattern must be a string that
320
+ can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
321
+ Defaults to `"model{suffix}.safetensors"`.
322
+ max_shard_size (`int` or `str`, *optional*):
323
+ The maximum size of each shard, in bytes. Defaults to 5GB.
324
+
325
+ Returns:
326
+ [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them.
327
+
328
+ Example:
329
+ ```py
330
+ >>> import json
331
+ >>> import os
332
+ >>> from safetensors.torch import save_file as safe_save_file
333
+ >>> from huggingface_hub import split_torch_state_dict_into_shards
334
+
335
+ >>> def save_state_dict(state_dict: dict[str, torch.Tensor], save_directory: str):
336
+ ... state_dict_split = split_torch_state_dict_into_shards(state_dict)
337
+ ... for filename, tensors in state_dict_split.filename_to_tensors.items():
338
+ ... shard = {tensor: state_dict[tensor] for tensor in tensors}
339
+ ... safe_save_file(
340
+ ... shard,
341
+ ... os.path.join(save_directory, filename),
342
+ ... metadata={"format": "pt"},
343
+ ... )
344
+ ... if state_dict_split.is_sharded:
345
+ ... index = {
346
+ ... "metadata": state_dict_split.metadata,
347
+ ... "weight_map": state_dict_split.tensor_to_filename,
348
+ ... }
349
+ ... with open(os.path.join(save_directory, "model.safetensors.index.json"), "w") as f:
350
+ ... f.write(json.dumps(index, indent=2))
351
+ ```
352
+ """
353
+ return split_state_dict_into_shards_factory(
354
+ state_dict,
355
+ max_shard_size=max_shard_size,
356
+ filename_pattern=filename_pattern,
357
+ get_storage_size=get_torch_storage_size,
358
+ get_storage_id=get_torch_storage_id,
359
+ )
360
+
361
+
362
+ # LOADING
363
+
364
+
365
+ def load_torch_model(
366
+ model: "torch.nn.Module",
367
+ checkpoint_path: str | os.PathLike,
368
+ *,
369
+ strict: bool = False,
370
+ safe: bool = True,
371
+ weights_only: bool = False,
372
+ map_location: Union[str, "torch.device"] | None = None,
373
+ mmap: bool = False,
374
+ filename_pattern: str | None = None,
375
+ ) -> NamedTuple:
376
+ """
377
+ Load a checkpoint into a model, handling both sharded and non-sharded checkpoints.
378
+
379
+ Args:
380
+ model (`torch.nn.Module`):
381
+ The model in which to load the checkpoint.
382
+ checkpoint_path (`str` or `os.PathLike`):
383
+ Path to either the checkpoint file or directory containing the checkpoint(s).
384
+ strict (`bool`, *optional*, defaults to `False`):
385
+ Whether to strictly enforce that the keys in the model state dict match the keys in the checkpoint.
386
+ safe (`bool`, *optional*, defaults to `True`):
387
+ If `safe` is True, the safetensors files will be loaded. If `safe` is False, the function
388
+ will first attempt to load safetensors files if they are available, otherwise it will fall back to loading
389
+ pickle files. `filename_pattern` parameter takes precedence over `safe` parameter.
390
+ weights_only (`bool`, *optional*, defaults to `False`):
391
+ If True, only loads the model weights without optimizer states and other metadata.
392
+ Only supported in PyTorch >= 1.13.
393
+ map_location (`str` or `torch.device`, *optional*):
394
+ A `torch.device` object, string or a dict specifying how to remap storage locations. It
395
+ indicates the location where all tensors should be loaded.
396
+ mmap (`bool`, *optional*, defaults to `False`):
397
+ Whether to use memory-mapped file loading. Memory mapping can improve loading performance
398
+ for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints.
399
+ filename_pattern (`str`, *optional*):
400
+ The pattern to look for the index file. Pattern must be a string that
401
+ can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
402
+ Defaults to `"model{suffix}.safetensors"`.
403
+ Returns:
404
+ `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields.
405
+ - `missing_keys` is a list of str containing the missing keys, i.e. keys that are in the model but not in the checkpoint.
406
+ - `unexpected_keys` is a list of str containing the unexpected keys, i.e. keys that are in the checkpoint but not in the model.
407
+
408
+ Raises:
409
+ [`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError)
410
+ If the checkpoint file or directory does not exist.
411
+ [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
412
+ If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively.
413
+ [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
414
+ If the checkpoint path is invalid or if the checkpoint format cannot be determined.
415
+
416
+ Example:
417
+ ```python
418
+ >>> from huggingface_hub import load_torch_model
419
+ >>> model = ... # A PyTorch model
420
+ >>> load_torch_model(model, "path/to/checkpoint")
421
+ ```
422
+ """
423
+ checkpoint_path = Path(checkpoint_path)
424
+
425
+ if not checkpoint_path.exists():
426
+ raise ValueError(f"Checkpoint path {checkpoint_path} does not exist")
427
+ # 1. Check if checkpoint is a single file
428
+ if checkpoint_path.is_file():
429
+ state_dict = load_state_dict_from_file(
430
+ checkpoint_file=checkpoint_path,
431
+ map_location=map_location,
432
+ weights_only=weights_only,
433
+ )
434
+ return model.load_state_dict(state_dict, strict=strict)
435
+
436
+ # 2. If not, checkpoint_path is a directory
437
+ if filename_pattern is None:
438
+ filename_pattern = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN
439
+ index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json")
440
+ # Only fallback to pickle format if safetensors index is not found and safe is False.
441
+ if not index_path.is_file() and not safe:
442
+ filename_pattern = constants.PYTORCH_WEIGHTS_FILE_PATTERN
443
+
444
+ index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json")
445
+
446
+ if index_path.is_file():
447
+ return _load_sharded_checkpoint(
448
+ model=model,
449
+ save_directory=checkpoint_path,
450
+ strict=strict,
451
+ weights_only=weights_only,
452
+ filename_pattern=filename_pattern,
453
+ )
454
+
455
+ # Look for single model file
456
+ model_files = list(checkpoint_path.glob("*.safetensors" if safe else "*.bin"))
457
+ if len(model_files) == 1:
458
+ state_dict = load_state_dict_from_file(
459
+ checkpoint_file=model_files[0],
460
+ map_location=map_location,
461
+ weights_only=weights_only,
462
+ mmap=mmap,
463
+ )
464
+ return model.load_state_dict(state_dict, strict=strict)
465
+
466
+ raise ValueError(
467
+ f"Directory '{checkpoint_path}' does not contain a valid checkpoint. "
468
+ "Expected either a sharded checkpoint with an index file, or a single model file."
469
+ )
470
+
471
+
472
+ def _load_sharded_checkpoint(
473
+ model: "torch.nn.Module",
474
+ save_directory: os.PathLike,
475
+ *,
476
+ strict: bool = False,
477
+ weights_only: bool = False,
478
+ filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN,
479
+ ) -> NamedTuple:
480
+ """
481
+ Loads a sharded checkpoint into a model. This is the same as
482
+ [`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict)
483
+ but for a sharded checkpoint. Each shard is loaded one by one and removed from memory after being loaded into the model.
484
+
485
+ Args:
486
+ model (`torch.nn.Module`):
487
+ The model in which to load the checkpoint.
488
+ save_directory (`str` or `os.PathLike`):
489
+ A path to a folder containing the sharded checkpoint.
490
+ strict (`bool`, *optional*, defaults to `False`):
491
+ Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint.
492
+ weights_only (`bool`, *optional*, defaults to `False`):
493
+ If True, only loads the model weights without optimizer states and other metadata.
494
+ Only supported in PyTorch >= 1.13.
495
+ filename_pattern (`str`, *optional*, defaults to `"model{suffix}.safetensors"`):
496
+ The pattern to look for the index file. Pattern must be a string that
497
+ can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix`
498
+ Defaults to `"model{suffix}.safetensors"`.
499
+
500
+ Returns:
501
+ `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields,
502
+ - `missing_keys` is a list of str containing the missing keys
503
+ - `unexpected_keys` is a list of str containing the unexpected keys
504
+ """
505
+
506
+ # 1. Load and validate index file
507
+ # The index file contains mapping of parameter names to shard files
508
+ index_path = filename_pattern.format(suffix="") + ".index.json"
509
+ index_file = os.path.join(save_directory, index_path)
510
+ with open(index_file, encoding="utf-8") as f:
511
+ index = json.load(f)
512
+
513
+ # 2. Validate shard filenames from the index
514
+ # This prevents path traversal attacks and extension confusion attacks
515
+ # (e.g. a safetensors index referencing .bin pickle files)
516
+ expected_extension = Path(filename_pattern.format(suffix="")).suffix # e.g. ".safetensors"
517
+ shard_files = list(set(index["weight_map"].values()))
518
+ for shard_file in shard_files:
519
+ # Reject anything that could escape `save_directory` on any host OS:
520
+ # POSIX absolute ("/tmp/x"), Windows drive ("C:x", "C:\\x"), UNC
521
+ # ("\\\\server\\share\\x"), rooted-without-drive ("\\x", "/x"), or
522
+ # ".." traversal — including "..\\x" which `os.path.isabs` never caught on POSIX.
523
+ #
524
+ # We parse with `PureWindowsPath` *regardless of host OS*: it treats both "/" and
525
+ # "\\" as separators and exposes `drive` / `root`, so a single check rejects a
526
+ # malicious index file on Linux too (e.g. if it's later opened on Windows). The
527
+ # only over-strict case is a POSIX filename like "a:foo" which would be parsed as
528
+ # drive "a:" — such names are never produced for safetensors shards and would
529
+ # break on Windows anyway, so rejecting them is fine.
530
+ win_path = PureWindowsPath(shard_file)
531
+ if win_path.drive or win_path.root or ".." in win_path.parts:
532
+ raise ValueError(
533
+ f"Invalid shard filename '{shard_file}' in index file '{index_file}'. "
534
+ "Shard filenames must be relative paths without '..' components."
535
+ )
536
+ # Reject extension mismatch (e.g. .bin shard in a .safetensors index)
537
+ if not shard_file.endswith(expected_extension):
538
+ raise ValueError(
539
+ f"Invalid shard filename '{shard_file}' in index file '{index_file}'. "
540
+ f"Expected '{expected_extension}' extension to match the index format."
541
+ )
542
+
543
+ # 3. Validate keys if in strict mode
544
+ # This is done before loading any shards to fail fast
545
+ if strict:
546
+ _validate_keys_for_strict_loading(model, index["weight_map"].keys())
547
+
548
+ # 4. Load each shard using `load_state_dict`
549
+ # Get unique shard files (multiple parameters can be in same shard)
550
+ for shard_file in shard_files:
551
+ # Load shard into memory
552
+ shard_path = os.path.join(save_directory, shard_file)
553
+ state_dict = load_state_dict_from_file(
554
+ shard_path,
555
+ map_location="cpu",
556
+ weights_only=weights_only,
557
+ )
558
+ # Update model with parameters from this shard
559
+ model.load_state_dict(state_dict, strict=strict)
560
+ # Explicitly remove the state dict from memory
561
+ del state_dict
562
+
563
+ # 5. Return compatibility info
564
+ loaded_keys = set(index["weight_map"].keys())
565
+ model_keys = set(model.state_dict().keys())
566
+ return _IncompatibleKeys(
567
+ missing_keys=list(model_keys - loaded_keys), unexpected_keys=list(loaded_keys - model_keys)
568
+ )
569
+
570
+
571
+ def load_state_dict_from_file(
572
+ checkpoint_file: str | os.PathLike,
573
+ map_location: Union[str, "torch.device"] | None = None,
574
+ weights_only: bool = False,
575
+ mmap: bool = False,
576
+ ) -> dict[str, "torch.Tensor"] | Any:
577
+ """
578
+ Loads a checkpoint file, handling both safetensors and pickle checkpoint formats.
579
+
580
+ Args:
581
+ checkpoint_file (`str` or `os.PathLike`):
582
+ Path to the checkpoint file to load. Can be either a safetensors or pickle (`.bin`) checkpoint.
583
+ map_location (`str` or `torch.device`, *optional*):
584
+ A `torch.device` object, string or a dict specifying how to remap storage locations. It
585
+ indicates the location where all tensors should be loaded.
586
+ weights_only (`bool`, *optional*, defaults to `False`):
587
+ If True, only loads the model weights without optimizer states and other metadata.
588
+ Only supported for pickle (`.bin`) checkpoints with PyTorch >= 1.13. Has no effect when
589
+ loading safetensors files.
590
+ mmap (`bool`, *optional*, defaults to `False`):
591
+ Whether to use memory-mapped file loading. Memory mapping can improve loading performance
592
+ for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints. Has no effect when
593
+ loading safetensors files, as the `safetensors` library uses memory mapping by default.
594
+
595
+ Returns:
596
+ `Union[dict[str, "torch.Tensor"], Any]`: The loaded checkpoint.
597
+ - For safetensors files: always returns a dictionary mapping parameter names to tensors.
598
+ - For pickle files: returns any Python object that was pickled (commonly a state dict, but could be
599
+ an entire model, optimizer state, or any other Python object).
600
+
601
+ Raises:
602
+ [`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError)
603
+ If the checkpoint file does not exist.
604
+ [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
605
+ If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively.
606
+ [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError)
607
+ If the checkpoint file format is invalid or if git-lfs files are not properly downloaded.
608
+ [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
609
+ If the checkpoint file path is empty or invalid.
610
+
611
+ Example:
612
+ ```python
613
+ >>> from huggingface_hub import load_state_dict_from_file
614
+
615
+ # Load a PyTorch checkpoint
616
+ >>> state_dict = load_state_dict_from_file("path/to/model.bin", map_location="cpu")
617
+ >>> model.load_state_dict(state_dict)
618
+
619
+ # Load a safetensors checkpoint
620
+ >>> state_dict = load_state_dict_from_file("path/to/model.safetensors")
621
+ >>> model.load_state_dict(state_dict)
622
+ ```
623
+ """
624
+ checkpoint_path = Path(checkpoint_file)
625
+
626
+ # Check if file exists and is a regular file (not a directory)
627
+ if not checkpoint_path.is_file():
628
+ raise FileNotFoundError(
629
+ f"No checkpoint file found at '{checkpoint_path}'. Please verify the path is correct and "
630
+ "the file has been properly downloaded."
631
+ )
632
+
633
+ # Load safetensors checkpoint
634
+ if checkpoint_path.suffix == ".safetensors":
635
+ try:
636
+ from safetensors import safe_open
637
+ from safetensors.torch import load_file
638
+ except ImportError as e:
639
+ raise ImportError(
640
+ "Please install `safetensors` to load safetensors checkpoint. "
641
+ "You can install it with `pip install safetensors`."
642
+ ) from e
643
+
644
+ # Check format of the archive
645
+ with safe_open(checkpoint_file, framework="pt") as f: # type: ignore[attr-defined]
646
+ metadata = f.metadata()
647
+ # see comment: https://github.com/huggingface/transformers/blob/3d213b57fe74302e5902d68ed9478c3ad1aaa713/src/transformers/modeling_utils.py#L3966
648
+ if metadata is not None and metadata.get("format") not in ["pt", "mlx"]:
649
+ raise OSError(
650
+ f"The safetensors archive passed at {checkpoint_file} does not contain the valid metadata. Make sure "
651
+ "you save your model with the `save_torch_model` method."
652
+ )
653
+ device = str(map_location.type) if map_location is not None and hasattr(map_location, "type") else map_location
654
+ # meta device is not supported with safetensors, falling back to CPU
655
+ if device == "meta":
656
+ logger.warning("Meta device is not supported with safetensors. Falling back to CPU device.")
657
+ device = "cpu"
658
+ return load_file(checkpoint_file, device=device) # type: ignore[arg-type]
659
+ # Otherwise, load from pickle
660
+ try:
661
+ import torch
662
+ from torch import load
663
+ except ImportError as e:
664
+ raise ImportError(
665
+ "Please install `torch` to load torch tensors. You can install it with `pip install torch`."
666
+ ) from e
667
+ # Add additional kwargs, mmap is only supported in torch >= 2.1.0
668
+ additional_kwargs = {}
669
+ if version.parse(torch.__version__) >= version.parse("2.1.0"):
670
+ additional_kwargs["mmap"] = mmap
671
+
672
+ # weights_only is only supported in torch >= 1.13.0
673
+ if version.parse(torch.__version__) >= version.parse("1.13.0"):
674
+ additional_kwargs["weights_only"] = weights_only
675
+
676
+ return load(
677
+ checkpoint_file,
678
+ map_location=map_location,
679
+ **additional_kwargs,
680
+ )
681
+
682
+
683
+ # HELPERS
684
+
685
+
686
+ def _validate_keys_for_strict_loading(
687
+ model: "torch.nn.Module",
688
+ loaded_keys: Iterable[str],
689
+ ) -> None:
690
+ """
691
+ Validate that model keys match loaded keys when strict loading is enabled.
692
+
693
+ Args:
694
+ model: The PyTorch model being loaded
695
+ loaded_keys: The keys present in the checkpoint
696
+
697
+ Raises:
698
+ RuntimeError: If there are missing or unexpected keys in strict mode
699
+ """
700
+ loaded_keys_set = set(loaded_keys)
701
+ model_keys = set(model.state_dict().keys())
702
+ missing_keys = model_keys - loaded_keys_set # Keys in model but not in checkpoint
703
+ unexpected_keys = loaded_keys_set - model_keys # Keys in checkpoint but not in model
704
+
705
+ if missing_keys or unexpected_keys:
706
+ error_message = f"Error(s) in loading state_dict for {model.__class__.__name__}"
707
+ if missing_keys:
708
+ str_missing_keys = ",".join([f'"{k}"' for k in sorted(missing_keys)])
709
+ error_message += f"\nMissing key(s): {str_missing_keys}."
710
+ if unexpected_keys:
711
+ str_unexpected_keys = ",".join([f'"{k}"' for k in sorted(unexpected_keys)])
712
+ error_message += f"\nUnexpected key(s): {str_unexpected_keys}."
713
+ raise RuntimeError(error_message)
714
+
715
+
716
+ def _get_unique_id(tensor: "torch.Tensor") -> int | tuple[Any, ...]:
717
+ """Returns a unique id for plain tensor
718
+ or a (potentially nested) Tuple of unique id for the flattened Tensor
719
+ if the input is a wrapper tensor subclass Tensor
720
+ """
721
+
722
+ try:
723
+ from torch.distributed.tensor import DTensor
724
+
725
+ if isinstance(tensor, DTensor):
726
+ local_tensor = tensor.to_local()
727
+ return local_tensor.storage().data_ptr()
728
+ except ImportError:
729
+ pass
730
+
731
+ try:
732
+ # for torch 2.1 and above we can also handle tensor subclasses
733
+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass
734
+
735
+ if is_traceable_wrapper_subclass(tensor):
736
+ attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined]
737
+ return tuple(_get_unique_id(getattr(tensor, attr)) for attr in attrs)
738
+
739
+ except ImportError:
740
+ # for torch version less than 2.1, we can fall back to original implementation
741
+ pass
742
+
743
+ if tensor.device.type == "xla" and is_torch_tpu_available():
744
+ # NOTE: xla tensors don't have storage
745
+ # use some other unique id to distinguish.
746
+ # this is a XLA tensor, it must be created using torch_xla's
747
+ # device. So the following import is safe:
748
+ import torch_xla # type: ignore[import]
749
+
750
+ unique_id = torch_xla._XLAC._xla_get_tensor_id(tensor)
751
+ else:
752
+ unique_id = storage_ptr(tensor)
753
+
754
+ return unique_id
755
+
756
+
757
+ def get_torch_storage_id(tensor: "torch.Tensor") -> tuple["torch.device", int | tuple[Any, ...], int] | None:
758
+ """
759
+ Return unique identifier to a tensor storage.
760
+
761
+ Multiple different tensors can share the same underlying storage. This identifier is
762
+ guaranteed to be unique and constant for this tensor's storage during its lifetime. Two tensor storages with
763
+ non-overlapping lifetimes may have the same id.
764
+ In the case of meta tensors, we return None since we can't tell if they share the same storage.
765
+
766
+ Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/pytorch_utils.py#L278.
767
+ """
768
+ if tensor.device.type == "meta":
769
+ return None
770
+ else:
771
+ return tensor.device, _get_unique_id(tensor), get_torch_storage_size(tensor)
772
+
773
+
774
+ def get_torch_storage_size(tensor: "torch.Tensor") -> int:
775
+ """
776
+ Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L31C1-L41C59
777
+ """
778
+ try:
779
+ from torch.distributed.tensor import DTensor
780
+
781
+ if isinstance(tensor, DTensor):
782
+ # this returns the size of the FULL tensor in bytes
783
+ return tensor.nbytes
784
+ except ImportError:
785
+ pass
786
+
787
+ try:
788
+ # for torch 2.1 and above we can also handle tensor subclasses
789
+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass
790
+
791
+ if is_traceable_wrapper_subclass(tensor):
792
+ attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined]
793
+ return sum(get_torch_storage_size(getattr(tensor, attr)) for attr in attrs)
794
+ except ImportError:
795
+ # for torch version less than 2.1, we can fall back to original implementation
796
+ pass
797
+
798
+ try:
799
+ return tensor.untyped_storage().nbytes()
800
+ except AttributeError:
801
+ # Fallback for torch==1.10
802
+ try:
803
+ return tensor.storage().size() * _get_dtype_size(tensor.dtype)
804
+ except NotImplementedError:
805
+ # Fallback for meta storage
806
+ # On torch >=2.0 this is the tensor size
807
+ return tensor.nelement() * _get_dtype_size(tensor.dtype)
808
+
809
+
810
+ @lru_cache
811
+ def is_torch_tpu_available(check_device=True):
812
+ """
813
+ Checks if `torch_xla` is installed and potentially if a TPU is in the environment
814
+
815
+ Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/utils/import_utils.py#L463.
816
+ """
817
+ if importlib.util.find_spec("torch_xla") is not None:
818
+ if check_device:
819
+ # We need to check if `xla_device` can be found, will raise a RuntimeError if not
820
+ try:
821
+ import torch_xla.core.xla_model as xm # type: ignore[import]
822
+
823
+ _ = xm.xla_device()
824
+ return True
825
+ except RuntimeError:
826
+ return False
827
+ return True
828
+ return False
829
+
830
+
831
+ def storage_ptr(tensor: "torch.Tensor") -> int | tuple[Any, ...]:
832
+ """
833
+ Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L11.
834
+ """
835
+ try:
836
+ # for torch 2.1 and above we can also handle tensor subclasses
837
+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass
838
+
839
+ if is_traceable_wrapper_subclass(tensor):
840
+ return _get_unique_id(tensor) # type: ignore
841
+ except ImportError:
842
+ # for torch version less than 2.1, we can fall back to original implementation
843
+ pass
844
+
845
+ try:
846
+ return tensor.untyped_storage().data_ptr()
847
+ except Exception:
848
+ # Fallback for torch==1.10
849
+ try:
850
+ return tensor.storage().data_ptr()
851
+ except NotImplementedError:
852
+ # Fallback for meta storage
853
+ return 0
854
+
855
+
856
+ def _clean_state_dict_for_safetensors(
857
+ state_dict: dict[str, "torch.Tensor"],
858
+ metadata: dict[str, str],
859
+ force_contiguous: bool = True,
860
+ shared_tensors_to_discard: list[str] | None = None,
861
+ ):
862
+ """Remove shared tensors from state_dict and update metadata accordingly (for reloading).
863
+
864
+ Warning: `state_dict` and `metadata` are mutated in-place!
865
+
866
+ Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L155.
867
+ """
868
+ to_removes = _remove_duplicate_names(state_dict, discard_names=shared_tensors_to_discard)
869
+ for kept_name, to_remove_group in to_removes.items():
870
+ for to_remove in to_remove_group:
871
+ if metadata is None:
872
+ metadata = {}
873
+
874
+ if to_remove not in metadata:
875
+ # Do not override user data
876
+ metadata[to_remove] = kept_name
877
+ del state_dict[to_remove]
878
+ if force_contiguous:
879
+ state_dict = {k: v.contiguous() for k, v in state_dict.items()}
880
+ return state_dict
881
+
882
+
883
+ def _end_ptr(tensor: "torch.Tensor") -> int:
884
+ """
885
+ Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L23.
886
+ """
887
+ if tensor.nelement():
888
+ stop = tensor.view(-1)[-1].data_ptr() + _get_dtype_size(tensor.dtype)
889
+ else:
890
+ stop = tensor.data_ptr()
891
+ return stop
892
+
893
+
894
+ def _filter_shared_not_shared(tensors: list[set[str]], state_dict: dict[str, "torch.Tensor"]) -> list[set[str]]:
895
+ """
896
+ Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L44
897
+ """
898
+ filtered_tensors = []
899
+ for shared in tensors:
900
+ if len(shared) < 2:
901
+ filtered_tensors.append(shared)
902
+ continue
903
+
904
+ areas = []
905
+ for name in shared:
906
+ tensor = state_dict[name]
907
+ areas.append((tensor.data_ptr(), _end_ptr(tensor), name))
908
+ areas.sort()
909
+
910
+ _, last_stop, last_name = areas[0]
911
+ filtered_tensors.append({last_name})
912
+ for start, stop, name in areas[1:]:
913
+ if start >= last_stop:
914
+ filtered_tensors.append({name})
915
+ else:
916
+ filtered_tensors[-1].add(name)
917
+ last_stop = stop
918
+
919
+ return filtered_tensors
920
+
921
+
922
+ def _find_shared_tensors(state_dict: dict[str, "torch.Tensor"]) -> list[set[str]]:
923
+ """
924
+ Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L69.
925
+ """
926
+ import torch
927
+
928
+ tensors_dict = defaultdict(set)
929
+ for k, v in state_dict.items():
930
+ if v.device != torch.device("meta") and storage_ptr(v) != 0 and get_torch_storage_size(v) != 0:
931
+ # Need to add device as key because of multiple GPU.
932
+ tensors_dict[(v.device, storage_ptr(v), get_torch_storage_size(v))].add(k)
933
+ tensors = list(sorted(tensors_dict.values()))
934
+ tensors = _filter_shared_not_shared(tensors, state_dict)
935
+ return tensors
936
+
937
+
938
+ def _is_complete(tensor: "torch.Tensor") -> bool:
939
+ """
940
+ Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80
941
+ """
942
+ try:
943
+ # for torch 2.1 and above we can also handle tensor subclasses
944
+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass
945
+
946
+ if is_traceable_wrapper_subclass(tensor):
947
+ attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined]
948
+ return all(_is_complete(getattr(tensor, attr)) for attr in attrs)
949
+ except ImportError:
950
+ # for torch version less than 2.1, we can fall back to original implementation
951
+ pass
952
+
953
+ return tensor.data_ptr() == storage_ptr(tensor) and tensor.nelement() * _get_dtype_size(
954
+ tensor.dtype
955
+ ) == get_torch_storage_size(tensor)
956
+
957
+
958
+ def _remove_duplicate_names(
959
+ state_dict: dict[str, "torch.Tensor"],
960
+ *,
961
+ preferred_names: list[str] | None = None,
962
+ discard_names: list[str] | None = None,
963
+ ) -> dict[str, list[str]]:
964
+ """
965
+ Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80
966
+ """
967
+ if preferred_names is None:
968
+ preferred_names = []
969
+ unique_preferred_names = set(preferred_names)
970
+ if discard_names is None:
971
+ discard_names = []
972
+ unique_discard_names = set(discard_names)
973
+
974
+ shareds = _find_shared_tensors(state_dict)
975
+ to_remove = defaultdict(list)
976
+ for shared in shareds:
977
+ complete_names = {name for name in shared if _is_complete(state_dict[name])}
978
+ if not complete_names:
979
+ raise RuntimeError(
980
+ "Error while trying to find names to remove to save state dict, but found no suitable name to keep"
981
+ f" for saving amongst: {shared}. None is covering the entire storage. Refusing to save/load the model"
982
+ " since you could be storing much more memory than needed. Please refer to"
983
+ " https://huggingface.co/docs/safetensors/torch_shared_tensors for more information. Or open an"
984
+ " issue."
985
+ )
986
+
987
+ keep_name = sorted(list(complete_names))[0]
988
+
989
+ # Mechanism to preferentially select keys to keep
990
+ # coming from the on-disk file to allow
991
+ # loading models saved with a different choice
992
+ # of keep_name
993
+ preferred = complete_names.difference(unique_discard_names)
994
+ if preferred:
995
+ keep_name = sorted(list(preferred))[0]
996
+
997
+ if unique_preferred_names:
998
+ preferred = unique_preferred_names.intersection(complete_names)
999
+ if preferred:
1000
+ keep_name = sorted(list(preferred))[0]
1001
+ for name in sorted(shared):
1002
+ if name != keep_name:
1003
+ to_remove[keep_name].append(name)
1004
+ return to_remove
1005
+
1006
+
1007
+ @lru_cache
1008
+ def _get_dtype_size(dtype: "torch.dtype") -> int:
1009
+ """
1010
+ Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L344
1011
+ """
1012
+ import torch
1013
+
1014
+ # torch.float8 formats require 2.1; we do not support these dtypes on earlier versions
1015
+ _float8_e4m3fn = getattr(torch, "float8_e4m3fn", None)
1016
+ _float8_e5m2 = getattr(torch, "float8_e5m2", None)
1017
+ _SIZE = {
1018
+ torch.int64: 8,
1019
+ torch.float32: 4,
1020
+ torch.int32: 4,
1021
+ torch.bfloat16: 2,
1022
+ torch.float16: 2,
1023
+ torch.int16: 2,
1024
+ torch.uint8: 1,
1025
+ torch.int8: 1,
1026
+ torch.bool: 1,
1027
+ torch.float64: 8,
1028
+ _float8_e4m3fn: 1,
1029
+ _float8_e5m2: 1,
1030
+ }
1031
+ return _SIZE[dtype]
1032
+
1033
+
1034
+ class _IncompatibleKeys(namedtuple("IncompatibleKeys", ["missing_keys", "unexpected_keys"])):
1035
+ """
1036
+ This is used to report missing and unexpected keys in the state dict.
1037
+ Taken from https://github.com/pytorch/pytorch/blob/main/torch/nn/modules/module.py#L52.
1038
+
1039
+ """
1040
+
1041
+ def __repr__(self) -> str:
1042
+ if not self.missing_keys and not self.unexpected_keys:
1043
+ return "<All keys matched successfully>"
1044
+ return super().__repr__()
1045
+
1046
+ __str__ = __repr__
.venv/lib/python3.13/site-packages/huggingface_hub/templates/datasetcard_template.md ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ # For reference on dataset card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/datasetcard.md?plain=1
3
+ # Doc / guide: https://huggingface.co/docs/hub/datasets-cards
4
+ {{ card_data }}
5
+ ---
6
+
7
+ # Dataset Card for {{ pretty_name | default("Dataset Name", true) }}
8
+
9
+ <!-- Provide a quick summary of the dataset. -->
10
+
11
+ {{ dataset_summary | default("", true) }}
12
+
13
+ ## Dataset Details
14
+
15
+ ### Dataset Description
16
+
17
+ <!-- Provide a longer summary of what this dataset is. -->
18
+
19
+ {{ dataset_description | default("", true) }}
20
+
21
+ - **Curated by:** {{ curators | default("[More Information Needed]", true)}}
22
+ - **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}}
23
+ - **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}}
24
+ - **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}}
25
+ - **License:** {{ license | default("[More Information Needed]", true)}}
26
+
27
+ ### Dataset Sources [optional]
28
+
29
+ <!-- Provide the basic links for the dataset. -->
30
+
31
+ - **Repository:** {{ repo | default("[More Information Needed]", true)}}
32
+ - **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}}
33
+ - **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}}
34
+
35
+ ## Uses
36
+
37
+ <!-- Address questions around how the dataset is intended to be used. -->
38
+
39
+ ### Direct Use
40
+
41
+ <!-- This section describes suitable use cases for the dataset. -->
42
+
43
+ {{ direct_use | default("[More Information Needed]", true)}}
44
+
45
+ ### Out-of-Scope Use
46
+
47
+ <!-- This section addresses misuse, malicious use, and uses that the dataset will not work well for. -->
48
+
49
+ {{ out_of_scope_use | default("[More Information Needed]", true)}}
50
+
51
+ ## Dataset Structure
52
+
53
+ <!-- This section provides a description of the dataset fields, and additional information about the dataset structure such as criteria used to create the splits, relationships between data points, etc. -->
54
+
55
+ {{ dataset_structure | default("[More Information Needed]", true)}}
56
+
57
+ ## Dataset Creation
58
+
59
+ ### Curation Rationale
60
+
61
+ <!-- Motivation for the creation of this dataset. -->
62
+
63
+ {{ curation_rationale_section | default("[More Information Needed]", true)}}
64
+
65
+ ### Source Data
66
+
67
+ <!-- This section describes the source data (e.g. news text and headlines, social media posts, translated sentences, ...). -->
68
+
69
+ #### Data Collection and Processing
70
+
71
+ <!-- This section describes the data collection and processing process such as data selection criteria, filtering and normalization methods, tools and libraries used, etc. -->
72
+
73
+ {{ data_collection_and_processing_section | default("[More Information Needed]", true)}}
74
+
75
+ #### Who are the source data producers?
76
+
77
+ <!-- This section describes the people or systems who originally created the data. It should also include self-reported demographic or identity information for the source data creators if this information is available. -->
78
+
79
+ {{ source_data_producers_section | default("[More Information Needed]", true)}}
80
+
81
+ ### Annotations [optional]
82
+
83
+ <!-- If the dataset contains annotations which are not part of the initial data collection, use this section to describe them. -->
84
+
85
+ #### Annotation process
86
+
87
+ <!-- This section describes the annotation process such as annotation tools used in the process, the amount of data annotated, annotation guidelines provided to the annotators, interannotator statistics, annotation validation, etc. -->
88
+
89
+ {{ annotation_process_section | default("[More Information Needed]", true)}}
90
+
91
+ #### Who are the annotators?
92
+
93
+ <!-- This section describes the people or systems who created the annotations. -->
94
+
95
+ {{ who_are_annotators_section | default("[More Information Needed]", true)}}
96
+
97
+ #### Personal and Sensitive Information
98
+
99
+ <!-- State whether the dataset contains data that might be considered personal, sensitive, or private (e.g., data that reveals addresses, uniquely identifiable names or aliases, racial or ethnic origins, sexual orientations, religious beliefs, political opinions, financial or health data, etc.). If efforts were made to anonymize the data, describe the anonymization process. -->
100
+
101
+ {{ personal_and_sensitive_information | default("[More Information Needed]", true)}}
102
+
103
+ ## Bias, Risks, and Limitations
104
+
105
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
106
+
107
+ {{ bias_risks_limitations | default("[More Information Needed]", true)}}
108
+
109
+ ### Recommendations
110
+
111
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
112
+
113
+ {{ bias_recommendations | default("Users should be made aware of the risks, biases and limitations of the dataset. More information needed for further recommendations.", true)}}
114
+
115
+ ## Citation [optional]
116
+
117
+ <!-- If there is a paper or blog post introducing the dataset, the APA and Bibtex information for that should go in this section. -->
118
+
119
+ **BibTeX:**
120
+
121
+ {{ citation_bibtex | default("[More Information Needed]", true)}}
122
+
123
+ **APA:**
124
+
125
+ {{ citation_apa | default("[More Information Needed]", true)}}
126
+
127
+ ## Glossary [optional]
128
+
129
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the dataset or dataset card. -->
130
+
131
+ {{ glossary | default("[More Information Needed]", true)}}
132
+
133
+ ## More Information [optional]
134
+
135
+ {{ more_information | default("[More Information Needed]", true)}}
136
+
137
+ ## Dataset Card Authors [optional]
138
+
139
+ {{ dataset_card_authors | default("[More Information Needed]", true)}}
140
+
141
+ ## Dataset Card Contact
142
+
143
+ {{ dataset_card_contact | default("[More Information Needed]", true)}}
.venv/lib/python3.13/site-packages/huggingface_hub/templates/modelcard_template.md ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ # For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1
3
+ # Doc / guide: https://huggingface.co/docs/hub/model-cards
4
+ {{ card_data }}
5
+ ---
6
+
7
+ # Model Card for {{ model_id | default("Model ID", true) }}
8
+
9
+ <!-- Provide a quick summary of what the model is/does. -->
10
+
11
+ {{ model_summary | default("", true) }}
12
+
13
+ ## Model Details
14
+
15
+ ### Model Description
16
+
17
+ <!-- Provide a longer summary of what this model is. -->
18
+
19
+ {{ model_description | default("", true) }}
20
+
21
+ - **Developed by:** {{ developers | default("[More Information Needed]", true)}}
22
+ - **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}}
23
+ - **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}}
24
+ - **Model type:** {{ model_type | default("[More Information Needed]", true)}}
25
+ - **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}}
26
+ - **License:** {{ license | default("[More Information Needed]", true)}}
27
+ - **Finetuned from model [optional]:** {{ base_model | default("[More Information Needed]", true)}}
28
+
29
+ ### Model Sources [optional]
30
+
31
+ <!-- Provide the basic links for the model. -->
32
+
33
+ - **Repository:** {{ repo | default("[More Information Needed]", true)}}
34
+ - **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}}
35
+ - **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}}
36
+
37
+ ## Uses
38
+
39
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
40
+
41
+ ### Direct Use
42
+
43
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
44
+
45
+ {{ direct_use | default("[More Information Needed]", true)}}
46
+
47
+ ### Downstream Use [optional]
48
+
49
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
50
+
51
+ {{ downstream_use | default("[More Information Needed]", true)}}
52
+
53
+ ### Out-of-Scope Use
54
+
55
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
56
+
57
+ {{ out_of_scope_use | default("[More Information Needed]", true)}}
58
+
59
+ ## Bias, Risks, and Limitations
60
+
61
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
62
+
63
+ {{ bias_risks_limitations | default("[More Information Needed]", true)}}
64
+
65
+ ### Recommendations
66
+
67
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
68
+
69
+ {{ bias_recommendations | default("Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.", true)}}
70
+
71
+ ## How to Get Started with the Model
72
+
73
+ Use the code below to get started with the model.
74
+
75
+ {{ get_started_code | default("[More Information Needed]", true)}}
76
+
77
+ ## Training Details
78
+
79
+ ### Training Data
80
+
81
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
82
+
83
+ {{ training_data | default("[More Information Needed]", true)}}
84
+
85
+ ### Training Procedure
86
+
87
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
88
+
89
+ #### Preprocessing [optional]
90
+
91
+ {{ preprocessing | default("[More Information Needed]", true)}}
92
+
93
+
94
+ #### Training Hyperparameters
95
+
96
+ - **Training regime:** {{ training_regime | default("[More Information Needed]", true)}} <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
97
+
98
+ #### Speeds, Sizes, Times [optional]
99
+
100
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
101
+
102
+ {{ speeds_sizes_times | default("[More Information Needed]", true)}}
103
+
104
+ ## Evaluation
105
+
106
+ <!-- This section describes the evaluation protocols and provides the results. -->
107
+
108
+ ### Testing Data, Factors & Metrics
109
+
110
+ #### Testing Data
111
+
112
+ <!-- This should link to a Dataset Card if possible. -->
113
+
114
+ {{ testing_data | default("[More Information Needed]", true)}}
115
+
116
+ #### Factors
117
+
118
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
119
+
120
+ {{ testing_factors | default("[More Information Needed]", true)}}
121
+
122
+ #### Metrics
123
+
124
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
125
+
126
+ {{ testing_metrics | default("[More Information Needed]", true)}}
127
+
128
+ ### Results
129
+
130
+ {{ results | default("[More Information Needed]", true)}}
131
+
132
+ #### Summary
133
+
134
+ {{ results_summary | default("", true) }}
135
+
136
+ ## Model Examination [optional]
137
+
138
+ <!-- Relevant interpretability work for the model goes here -->
139
+
140
+ {{ model_examination | default("[More Information Needed]", true)}}
141
+
142
+ ## Environmental Impact
143
+
144
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
145
+
146
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
147
+
148
+ - **Hardware Type:** {{ hardware_type | default("[More Information Needed]", true)}}
149
+ - **Hours used:** {{ hours_used | default("[More Information Needed]", true)}}
150
+ - **Cloud Provider:** {{ cloud_provider | default("[More Information Needed]", true)}}
151
+ - **Compute Region:** {{ cloud_region | default("[More Information Needed]", true)}}
152
+ - **Carbon Emitted:** {{ co2_emitted | default("[More Information Needed]", true)}}
153
+
154
+ ## Technical Specifications [optional]
155
+
156
+ ### Model Architecture and Objective
157
+
158
+ {{ model_specs | default("[More Information Needed]", true)}}
159
+
160
+ ### Compute Infrastructure
161
+
162
+ {{ compute_infrastructure | default("[More Information Needed]", true)}}
163
+
164
+ #### Hardware
165
+
166
+ {{ hardware_requirements | default("[More Information Needed]", true)}}
167
+
168
+ #### Software
169
+
170
+ {{ software | default("[More Information Needed]", true)}}
171
+
172
+ ## Citation [optional]
173
+
174
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
175
+
176
+ **BibTeX:**
177
+
178
+ {{ citation_bibtex | default("[More Information Needed]", true)}}
179
+
180
+ **APA:**
181
+
182
+ {{ citation_apa | default("[More Information Needed]", true)}}
183
+
184
+ ## Glossary [optional]
185
+
186
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
187
+
188
+ {{ glossary | default("[More Information Needed]", true)}}
189
+
190
+ ## More Information [optional]
191
+
192
+ {{ more_information | default("[More Information Needed]", true)}}
193
+
194
+ ## Model Card Authors [optional]
195
+
196
+ {{ model_card_authors | default("[More Information Needed]", true)}}
197
+
198
+ ## Model Card Contact
199
+
200
+ {{ model_card_contact | default("[More Information Needed]", true)}}
.venv/lib/python3.13/site-packages/huggingface_hub/utils/__init__.py ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2021 The HuggingFace Inc. team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License
14
+
15
+ # ruff: noqa: F401
16
+ from huggingface_hub.errors import (
17
+ BadRequestError,
18
+ BucketNotFoundError,
19
+ CacheNotFound,
20
+ CorruptedCacheException,
21
+ DisabledRepoError,
22
+ EntryNotFoundError,
23
+ FileMetadataError,
24
+ GatedRepoError,
25
+ HfHubHTTPError,
26
+ HFValidationError,
27
+ JobNotFoundError,
28
+ LocalEntryNotFoundError,
29
+ LocalTokenNotFoundError,
30
+ NotASafetensorsRepoError,
31
+ OfflineModeIsEnabled,
32
+ RepositoryNotFoundError,
33
+ RevisionNotFoundError,
34
+ SafetensorsParsingError,
35
+ )
36
+
37
+ from . import tqdm as _tqdm # _tqdm is the module
38
+ from ._auth import get_stored_tokens, get_token
39
+ from ._cache_assets import cached_assets_path
40
+ from ._cache_manager import (
41
+ CachedFileInfo,
42
+ CachedIncompleteFileInfo,
43
+ CachedRepoInfo,
44
+ CachedRevisionInfo,
45
+ DeleteCacheStrategy,
46
+ HFCacheInfo,
47
+ _format_size,
48
+ scan_cache_dir,
49
+ )
50
+ from ._chunk_utils import chunk_iterable
51
+ from ._datetime import parse_datetime
52
+ from ._detect_agent import detect_agent, is_agent
53
+ from ._experimental import experimental
54
+ from ._fixes import SoftTemporaryDirectory, WeakFileLock, yaml_dump
55
+ from ._git_credential import list_credential_helpers, set_git_credential, unset_git_credential
56
+ from ._headers import build_hf_headers, get_token_to_send
57
+ from ._hf_uris import HfMount, HfUri, is_hf_uri, parse_hf_mount, parse_hf_uri
58
+ from ._http import (
59
+ ASYNC_CLIENT_FACTORY_T,
60
+ CLIENT_FACTORY_T,
61
+ RateLimitInfo,
62
+ close_session,
63
+ fix_hf_endpoint_in_url,
64
+ get_async_session,
65
+ get_session,
66
+ hf_raise_for_status,
67
+ http_backoff,
68
+ http_stream_backoff,
69
+ parse_ratelimit_headers,
70
+ set_async_client_factory,
71
+ set_client_factory,
72
+ )
73
+ from ._pagination import paginate
74
+ from ._paths import DEFAULT_IGNORE_PATTERNS, FORBIDDEN_FOLDERS, filter_repo_objects
75
+ from ._runtime import (
76
+ dump_environment_info,
77
+ get_aiohttp_version,
78
+ get_fastai_version,
79
+ get_fastapi_version,
80
+ get_fastcore_version,
81
+ get_gradio_version,
82
+ get_graphviz_version,
83
+ get_hf_hub_version,
84
+ get_jinja_version,
85
+ get_numpy_version,
86
+ get_pillow_version,
87
+ get_pydantic_version,
88
+ get_pydot_version,
89
+ get_python_version,
90
+ get_tensorboard_version,
91
+ get_tf_version,
92
+ get_torch_version,
93
+ installation_method,
94
+ is_aiohttp_available,
95
+ is_colab_enterprise,
96
+ is_fastai_available,
97
+ is_fastapi_available,
98
+ is_fastcore_available,
99
+ is_google_colab,
100
+ is_gradio_available,
101
+ is_graphviz_available,
102
+ is_jinja_available,
103
+ is_notebook,
104
+ is_numpy_available,
105
+ is_package_available,
106
+ is_pillow_available,
107
+ is_pydantic_available,
108
+ is_pydot_available,
109
+ is_safetensors_available,
110
+ is_tensorboard_available,
111
+ is_tf_available,
112
+ is_torch_available,
113
+ )
114
+ from ._safetensors import SafetensorsFileMetadata, SafetensorsRepoMetadata, TensorInfo
115
+ from ._subprocess import capture_output, run_interactive_subprocess, run_subprocess
116
+ from ._telemetry import send_telemetry
117
+ from ._terminal import ANSI, StatusLine, select_choice, tabulate
118
+ from ._typing import is_jsonable, is_simple_optional_type, unwrap_simple_optional_type
119
+ from ._validators import validate_hf_hub_args, validate_repo_id
120
+ from ._xet import (
121
+ XetFileData,
122
+ XetTokenType,
123
+ parse_xet_file_data_from_response,
124
+ )
125
+ from .tqdm import (
126
+ are_progress_bars_disabled,
127
+ disable_progress_bars,
128
+ enable_progress_bars,
129
+ hf_thread_map,
130
+ is_tqdm_disabled,
131
+ silent_tqdm,
132
+ tqdm,
133
+ tqdm_stream_file,
134
+ )
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_auth.py ADDED
@@ -0,0 +1,456 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2023 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains a helper to get the token from machine (env variable, secret or config file)."""
15
+
16
+ import configparser
17
+ import io
18
+ import logging
19
+ import os
20
+ import time
21
+ import warnings
22
+ from pathlib import Path
23
+ from threading import Lock
24
+ from typing import TypedDict
25
+
26
+ from .. import constants
27
+ from ..errors import DeviceCodeError, OAuthErrorCode, OIDCError
28
+ from ._fixes import WeakFileLock
29
+ from ._oauth_device import refresh_access_token
30
+ from ._runtime import is_colab_enterprise, is_google_colab
31
+
32
+
33
+ _SECRET_FILE_MODE = 0o600
34
+ _SECRET_DIR_MODE = 0o700
35
+
36
+
37
+ def _write_secret(path: Path, content: str) -> None:
38
+ """Write content to file, restricting both the file and its parent directory to owner-only on POSIX systems."""
39
+ path.parent.mkdir(parents=True, exist_ok=True, mode=_SECRET_DIR_MODE)
40
+ fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, _SECRET_FILE_MODE)
41
+ with os.fdopen(fd, "w") as f:
42
+ f.write(content)
43
+ try:
44
+ path.chmod(_SECRET_FILE_MODE)
45
+ path.parent.chmod(_SECRET_DIR_MODE)
46
+ except (OSError, NotImplementedError):
47
+ # Windows does not support POSIX modes; chmod() will raise. Best-effort.
48
+ pass
49
+
50
+
51
+ _IS_GOOGLE_COLAB_CHECKED = False
52
+ _GOOGLE_COLAB_SECRET_LOCK = Lock()
53
+ _GOOGLE_COLAB_SECRET: str | None = None
54
+
55
+ logger = logging.getLogger(__name__)
56
+
57
+
58
+ def get_token() -> str | None:
59
+ """
60
+ Get token if user is logged in.
61
+
62
+ Note: in most cases, you should use [`huggingface_hub.utils.build_hf_headers`] instead. This method is only useful
63
+ if you want to retrieve the token for other purposes than sending an HTTP request.
64
+
65
+ If `HF_OIDC_RESOURCE` is set (Trusted Publishers, typically in CI), a short-lived token obtained via OIDC token
66
+ exchange takes precedence. Otherwise the token is retrieved from the `HF_TOKEN` environment variable, then from the
67
+ token file in the Hugging Face home folder. Returns None if user is not logged in. To log in, use [`login`] or
68
+ `hf auth login`.
69
+
70
+ OAuth tokens obtained with the browser-based login come with a refresh token: when such a token is close to
71
+ expiry, it is transparently refreshed and persisted before being returned.
72
+
73
+ Note: if `HF_OIDC_RESOURCE` is set but the OIDC token exchange fails, this raises instead of returning `None`,
74
+ opting into OIDC is explicit, so a failure surfaces as a clear error rather than a silent fallback.
75
+
76
+ Returns:
77
+ `str` or `None`: The token, `None` if it doesn't exist.
78
+ """
79
+ return (
80
+ _get_token_from_oidc()
81
+ or _get_token_from_environment()
82
+ or _get_token_from_file_refreshed()
83
+ or _get_token_from_google_colab()
84
+ )
85
+
86
+
87
+ def _get_token_from_google_colab() -> str | None:
88
+ """Get token from Google Colab secrets vault using `google.colab.userdata.get(...)`.
89
+
90
+ Token is read from the vault only once per session and then stored in a global variable to avoid re-requesting
91
+ access to the vault.
92
+ """
93
+ # If it's not a Google Colab or it's Colab Enterprise, fallback to environment variable or token file authentication
94
+ if not is_google_colab() or is_colab_enterprise():
95
+ return None
96
+
97
+ # `google.colab.userdata` is not thread-safe
98
+ # This can lead to a deadlock if multiple threads try to access it at the same time
99
+ # (typically when using `snapshot_download`)
100
+ # => use a lock
101
+ # See https://github.com/huggingface/huggingface_hub/issues/1952 for more details.
102
+ with _GOOGLE_COLAB_SECRET_LOCK:
103
+ global _GOOGLE_COLAB_SECRET
104
+ global _IS_GOOGLE_COLAB_CHECKED
105
+
106
+ if _IS_GOOGLE_COLAB_CHECKED: # request access only once
107
+ return _GOOGLE_COLAB_SECRET
108
+
109
+ try:
110
+ from google.colab import userdata # type: ignore
111
+ from google.colab.errors import Error as ColabError # type: ignore
112
+ except ImportError:
113
+ return None
114
+
115
+ try:
116
+ token = userdata.get("HF_TOKEN")
117
+ _GOOGLE_COLAB_SECRET = _clean_token(token)
118
+ except userdata.NotebookAccessError:
119
+ # Means the user has a secret call `HF_TOKEN` and got a popup "please grand access to HF_TOKEN" and refused it
120
+ # => warn user but ignore error => do not re-request access to user
121
+ warnings.warn(
122
+ "\nAccess to the secret `HF_TOKEN` has not been granted on this notebook."
123
+ "\nYou will not be requested again."
124
+ "\nPlease restart the session if you want to be prompted again."
125
+ )
126
+ _GOOGLE_COLAB_SECRET = None
127
+ except userdata.SecretNotFoundError:
128
+ # No `HF_TOKEN` secret defined: simply not logged in via the Colab vault. Not worth a
129
+ # warning now that `login()` is the primary flow (it would even fire during `login()`
130
+ # itself, telling the user to set up a secret while they are busy authenticating).
131
+ logger.info(
132
+ "The secret `HF_TOKEN` does not exist in your Colab secrets. Run `huggingface_hub.login()` to"
133
+ " authenticate (recommended but still optional to access public models or datasets)."
134
+ )
135
+ _GOOGLE_COLAB_SECRET = None
136
+ except ColabError as e:
137
+ # Something happen but we don't know what => recommend to open a GitHub issue
138
+ warnings.warn(f"\nError while fetching `HF_TOKEN` secret value from your vault: '{str(e)}'.")
139
+ _GOOGLE_COLAB_SECRET = None
140
+
141
+ _IS_GOOGLE_COLAB_CHECKED = True
142
+ return _GOOGLE_COLAB_SECRET
143
+
144
+
145
+ def _get_token_from_environment() -> str | None:
146
+ # `HF_TOKEN` has priority (keep `HUGGING_FACE_HUB_TOKEN` for backward compatibility)
147
+ return _clean_token(os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN"))
148
+
149
+
150
+ def _get_token_from_file() -> str | None:
151
+ try:
152
+ return _clean_token(Path(constants.HF_TOKEN_PATH).read_text())
153
+ except FileNotFoundError:
154
+ return None
155
+
156
+
157
+ class _OidcTokenCache(TypedDict):
158
+ resource: str
159
+ token: str
160
+ expires_at: float # monotonic clock value after which the cached token must be re-exchanged
161
+
162
+
163
+ # Cache for the OIDC-exchanged token: re-exchanging on every `get_token()` call would be wasteful,
164
+ # and re-exchanging shortly before expiry transparently keeps long-running jobs authenticated.
165
+ _OIDC_TOKEN_LOCK = Lock()
166
+ _OIDC_TOKEN_CACHE: _OidcTokenCache | None = None
167
+ _OIDC_REFRESH_MARGIN = 300 # re-exchange this many seconds before the token actually expires
168
+
169
+
170
+ def _get_token_from_oidc() -> str | None:
171
+ """Get a short-lived OIDC token in CI (Trusted Publishers).
172
+
173
+ Enabled by setting `HF_OIDC_RESOURCE`, which scopes the token to a repo or user.
174
+ The ID token is read from `HF_OIDC_ID_TOKEN` if available, or minted from a supported CI provider (e.g. GitHub Actions).
175
+
176
+ Returns `None` when OIDC is not enabled.
177
+ If enabled, any failure is raised explicitly rather than falling back silently.
178
+
179
+ See `huggingface_hub._oidc` and https://huggingface.co/docs/hub/trusted-publishers.
180
+ """
181
+ resource = os.environ.get("HF_OIDC_RESOURCE")
182
+ if not resource:
183
+ return None
184
+
185
+ from .._oidc import detect_provider, oidc_login
186
+
187
+ global _OIDC_TOKEN_CACHE
188
+ with _OIDC_TOKEN_LOCK:
189
+ now = time.monotonic()
190
+ if (
191
+ _OIDC_TOKEN_CACHE is not None
192
+ and _OIDC_TOKEN_CACHE["resource"] == resource
193
+ and now < _OIDC_TOKEN_CACHE["expires_at"]
194
+ ):
195
+ return _OIDC_TOKEN_CACHE["token"]
196
+
197
+ # An explicit id token (any provider) takes precedence; otherwise mint from a detected one.
198
+ subject_token = os.environ.get("HF_OIDC_ID_TOKEN") or None
199
+ if subject_token is None and detect_provider() is None:
200
+ raise OIDCError(
201
+ "HF_OIDC_RESOURCE is set but no OIDC id token is available: not running in a supported "
202
+ "CI provider (github) and HF_OIDC_ID_TOKEN is not set. Set HF_OIDC_ID_TOKEN to the id "
203
+ "token minted by your CI provider, or unset HF_OIDC_RESOURCE."
204
+ )
205
+
206
+ result = oidc_login(resource=resource, subject_token=subject_token)
207
+ token = result["access_token"]
208
+ expires_in = int(result.get("expires_in", 3600))
209
+ # A pre-supplied HF_OIDC_ID_TOKEN can't be re-minted, so refreshing early is pointless (the id
210
+ # token is likely already expired by then): cache for the full lifetime. Only the auto-minted
211
+ # path can refresh, so only it gets the safety margin.
212
+ margin = 0 if subject_token is not None else _OIDC_REFRESH_MARGIN
213
+ _OIDC_TOKEN_CACHE = {
214
+ "resource": resource,
215
+ "token": token,
216
+ "expires_at": now + max(expires_in - margin, 0),
217
+ }
218
+ return token
219
+
220
+
221
+ class _OAuthRefreshCache(TypedDict):
222
+ file_token: str # token as read from HF_TOKEN_PATH (cache key)
223
+ resolved_token: str # token to return (refreshed, or identical if no refresh was needed)
224
+ recheck_at: float # wall-clock timestamp after which the expiry must be re-evaluated
225
+
226
+
227
+ # Cache the refresh decision in-process: `get_token()` is called on every HTTP request and must not
228
+ # re-read the stored tokens file (let alone hit the network) each time.
229
+ _OAUTH_REFRESH_LOCK = Lock()
230
+ _OAUTH_REFRESH_CACHE: _OAuthRefreshCache | None = None
231
+ _OAUTH_REFRESH_MARGIN = 24 * 3600 # refresh when less than 1 day of validity remains
232
+ _OAUTH_RECHECK_INTERVAL = 300 # re-check interval when there is no metadata or the refresh failed
233
+ _OAUTH_REFRESH_WARNED = False # warn at most once per process on refresh failure
234
+
235
+
236
+ def _get_token_from_file_refreshed() -> str | None:
237
+ """Get the token from `HF_TOKEN_PATH`, transparently refreshing it if close to expiry."""
238
+ token = _get_token_from_file()
239
+ if token is None:
240
+ return None
241
+ return _refresh_oauth_token_if_needed(token)
242
+
243
+
244
+ def _refresh_oauth_token_if_needed(token: str) -> str:
245
+ """Refresh an OAuth access token if it is close to expiry. Best-effort: never raises.
246
+
247
+ OAuth tokens obtained with the browser-based login are stored with a `refresh_token` and an
248
+ `expires_at` timestamp (see `_save_token`). When the active token is one of them and about to
249
+ expire, exchange the refresh token for a new access token and persist it. Any other token is
250
+ returned unchanged.
251
+ """
252
+ global _OAUTH_REFRESH_CACHE
253
+ with _OAUTH_REFRESH_LOCK:
254
+ now = time.time()
255
+ cache = _OAUTH_REFRESH_CACHE
256
+ if cache is not None and cache["file_token"] == token and now < cache["recheck_at"]:
257
+ return cache["resolved_token"]
258
+
259
+ token_name, fields = next(
260
+ ((name, fields) for name, fields in _read_stored_tokens_full().items() if fields.get("hf_token") == token),
261
+ (None, {}),
262
+ )
263
+ refresh_token = fields.get("refresh_token")
264
+ expires_at = _parse_expires_at(fields)
265
+ if token_name is None or refresh_token is None or expires_at is None:
266
+ # `token` may have just been replaced by a concurrent refresh (in which case it no
267
+ # longer appears in the stored tokens): serve the fresh file token without caching.
268
+ current_file_token = _get_token_from_file()
269
+ if current_file_token is not None and current_file_token != token:
270
+ return current_file_token
271
+ # Not a refreshable OAuth token (or its metadata was lost): nothing to do.
272
+ _OAUTH_REFRESH_CACHE = {
273
+ "file_token": token,
274
+ "resolved_token": token,
275
+ "recheck_at": now + _OAUTH_RECHECK_INTERVAL,
276
+ }
277
+ return token
278
+
279
+ if expires_at - _OAUTH_REFRESH_MARGIN > now:
280
+ _OAUTH_REFRESH_CACHE = {
281
+ "file_token": token,
282
+ "resolved_token": token,
283
+ "recheck_at": expires_at - _OAUTH_REFRESH_MARGIN,
284
+ }
285
+ return token
286
+
287
+ try:
288
+ # Cross-process file lock: if the server rotates refresh tokens, two processes
289
+ # refreshing concurrently would invalidate each other's refresh token.
290
+ with WeakFileLock(constants.HF_STORED_TOKENS_PATH + ".lock", timeout=30):
291
+ # Re-read under the lock: another process may have refreshed in the meantime.
292
+ fields = _read_stored_tokens_full().get(token_name, {})
293
+ if fields.get("hf_token") != token:
294
+ # Another process already refreshed this token: adopt its result.
295
+ new_token = fields.get("hf_token") or token
296
+ new_expires_at = _parse_expires_at(fields)
297
+ else:
298
+ response = refresh_access_token(refresh_token)
299
+ new_token = response["access_token"]
300
+ new_expires_at = int(now) + int(response["expires_in"]) if "expires_in" in response else None
301
+ _save_token(
302
+ token=new_token,
303
+ token_name=token_name,
304
+ # The server may rotate the refresh token; keep the old one if it doesn't.
305
+ refresh_token=response.get("refresh_token") or refresh_token,
306
+ expires_at=new_expires_at,
307
+ )
308
+ # Update the active token file, unless another process switched to a different token meanwhile.
309
+ if _get_token_from_file() == token:
310
+ _write_secret(Path(constants.HF_TOKEN_PATH), new_token)
311
+ logger.info(f"Access token `{token_name}` has been refreshed.")
312
+ except Exception as e:
313
+ if isinstance(e, DeviceCodeError) and e.error_code == OAuthErrorCode.INVALID_GRANT:
314
+ # Refresh token expired or revoked: retrying is pointless, a re-login is required.
315
+ # Warned unconditionally (the inf recheck guarantees it fires at most once): an
316
+ # earlier transient warning must not suppress this actionable message.
317
+ logger.warning(
318
+ "Your Hugging Face access token has expired and could not be refreshed "
319
+ f"(session expired or revoked). Run `hf auth login` to re-authenticate. ({e})"
320
+ )
321
+ recheck_at = float("inf")
322
+ else:
323
+ # Transient failure (offline, server error, ...): retry later.
324
+ _warn_refresh_failure_once(f"Could not refresh your Hugging Face access token: {e}. Will retry later.")
325
+ recheck_at = now + _OAUTH_RECHECK_INTERVAL
326
+ # Return the existing token: if it's truly expired, the API will reject it with a clear error.
327
+ _OAUTH_REFRESH_CACHE = {"file_token": token, "resolved_token": token, "recheck_at": recheck_at}
328
+ return token
329
+
330
+ _OAUTH_REFRESH_CACHE = {
331
+ "file_token": new_token,
332
+ "resolved_token": new_token,
333
+ # The floor guards against a token lifetime shorter than the refresh margin, which
334
+ # would otherwise put `recheck_at` in the past and trigger a refresh on every call.
335
+ "recheck_at": max(
336
+ now + _OAUTH_RECHECK_INTERVAL,
337
+ new_expires_at - _OAUTH_REFRESH_MARGIN if new_expires_at else 0,
338
+ ),
339
+ }
340
+ return new_token
341
+
342
+
343
+ def _warn_refresh_failure_once(message: str) -> None:
344
+ global _OAUTH_REFRESH_WARNED
345
+ if not _OAUTH_REFRESH_WARNED:
346
+ logger.warning(message)
347
+ _OAUTH_REFRESH_WARNED = True
348
+
349
+
350
+ def _parse_expires_at(fields: dict[str, str]) -> int | None:
351
+ """Parse the `expires_at` field of a stored-tokens section, `None` if missing or corrupt."""
352
+ try:
353
+ return int(fields["expires_at"])
354
+ except (KeyError, ValueError):
355
+ return None
356
+
357
+
358
+ def get_stored_tokens() -> dict[str, str]:
359
+ """
360
+ Returns the parsed INI file containing the access tokens.
361
+ The file is located at `HF_STORED_TOKENS_PATH`, defaulting to `~/.cache/huggingface/stored_tokens`.
362
+ If the file does not exist, an empty dictionary is returned.
363
+
364
+ Returns: `dict[str, str]`
365
+ Key is the token name and value is the token.
366
+ """
367
+ return {token_name: fields.get("hf_token", "") for token_name, fields in _read_stored_tokens_full().items()}
368
+
369
+
370
+ def _read_stored_tokens_full() -> dict[str, dict[str, str]]:
371
+ """Read all sections of the stored tokens INI file, with all their fields.
372
+
373
+ Beside `hf_token`, sections for OAuth tokens also carry `refresh_token` and `expires_at`
374
+ (unix timestamp), used by [`get_token`] to transparently refresh them.
375
+ """
376
+ tokens_path = Path(constants.HF_STORED_TOKENS_PATH)
377
+ if not tokens_path.exists():
378
+ return {}
379
+ # interpolation=None: token values are opaque strings, a `%` must not be interpreted.
380
+ config = configparser.ConfigParser(interpolation=None)
381
+ try:
382
+ config.read(tokens_path)
383
+ return {token_name: dict(config.items(token_name)) for token_name in config.sections()}
384
+ except configparser.Error as e:
385
+ logger.error(f"Error parsing stored tokens file: {e}")
386
+ return {}
387
+
388
+
389
+ def _save_stored_tokens_full(stored_tokens: dict[str, dict[str, str]]) -> None:
390
+ """Write all sections and their fields to the stored tokens INI file."""
391
+ config = configparser.ConfigParser(interpolation=None)
392
+ for token_name in sorted(stored_tokens.keys()):
393
+ config.add_section(token_name)
394
+ for key, value in stored_tokens[token_name].items():
395
+ config.set(token_name, key, value)
396
+
397
+ buf = io.StringIO()
398
+ config.write(buf)
399
+ _write_secret(Path(constants.HF_STORED_TOKENS_PATH), buf.getvalue())
400
+
401
+
402
+ def _get_token_by_name(token_name: str) -> str | None:
403
+ """
404
+ Get the token by name.
405
+
406
+ Args:
407
+ token_name (`str`):
408
+ The name of the token to get.
409
+
410
+ Returns:
411
+ `str` or `None`: The token, `None` if it doesn't exist.
412
+
413
+ """
414
+ stored_tokens = get_stored_tokens()
415
+ if token_name not in stored_tokens:
416
+ return None
417
+ return _clean_token(stored_tokens[token_name])
418
+
419
+
420
+ def _save_token(
421
+ token: str, token_name: str, *, refresh_token: str | None = None, expires_at: int | None = None
422
+ ) -> None:
423
+ """
424
+ Save the given token.
425
+
426
+ If the stored tokens file does not exist, it will be created.
427
+ Args:
428
+ token (`str`):
429
+ The token to save.
430
+ token_name (`str`):
431
+ The name of the token.
432
+ refresh_token (`str`, *optional*):
433
+ OAuth refresh token used to renew the access token when it expires.
434
+ expires_at (`int`, *optional*):
435
+ Unix timestamp at which the access token expires.
436
+ """
437
+ stored_tokens = _read_stored_tokens_full()
438
+ fields = {"hf_token": token}
439
+ if refresh_token is not None:
440
+ fields["refresh_token"] = refresh_token
441
+ if expires_at is not None:
442
+ fields["expires_at"] = str(expires_at)
443
+ # Replace the whole section: re-logging in under the same name must drop stale metadata.
444
+ stored_tokens[token_name] = fields
445
+ _save_stored_tokens_full(stored_tokens)
446
+ logger.info(f"The token `{token_name}` has been saved to {constants.HF_STORED_TOKENS_PATH}")
447
+
448
+
449
+ def _clean_token(token: str | None) -> str | None:
450
+ """Clean token by removing trailing and leading spaces and newlines.
451
+
452
+ If token is an empty string, return None.
453
+ """
454
+ if token is None:
455
+ return None
456
+ return token.replace("\r", "").replace("\n", "").strip() or None
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_assets.py ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2019-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ from pathlib import Path
15
+
16
+ from ..constants import HF_ASSETS_CACHE
17
+
18
+
19
+ def cached_assets_path(
20
+ library_name: str,
21
+ namespace: str = "default",
22
+ subfolder: str = "default",
23
+ *,
24
+ assets_dir: str | Path | None = None,
25
+ ) -> Path:
26
+ """Return a folder path to cache arbitrary files.
27
+
28
+ `huggingface_hub` provides a canonical folder path to store assets. This is the
29
+ recommended way to integrate cache in a downstream library as it will benefit from
30
+ the builtins tools to scan and delete the cache properly.
31
+
32
+ The distinction is made between files cached from the Hub and assets. Files from the
33
+ Hub are cached in a git-aware manner and entirely managed by `huggingface_hub`. See
34
+ [related documentation](https://huggingface.co/docs/huggingface_hub/how-to-cache).
35
+ All other files that a downstream library caches are considered to be "assets"
36
+ (files downloaded from external sources, extracted from a .tar archive, preprocessed
37
+ for training,...).
38
+
39
+ Once the folder path is generated, it is guaranteed to exist and to be a directory.
40
+ The path is based on 3 levels of depth: the library name, a namespace and a
41
+ subfolder. Those 3 levels grants flexibility while allowing `huggingface_hub` to
42
+ expect folders when scanning/deleting parts of the assets cache. Within a library,
43
+ it is expected that all namespaces share the same subset of subfolder names but this
44
+ is not a mandatory rule. The downstream library has then full control on which file
45
+ structure to adopt within its cache. Namespace and subfolder are optional (would
46
+ default to a `"default/"` subfolder) but library name is mandatory as we want every
47
+ downstream library to manage its own cache.
48
+
49
+ Expected tree:
50
+ ```text
51
+ assets/
52
+ └── datasets/
53
+ │ ├── SQuAD/
54
+ │ │ ├── downloaded/
55
+ │ │ ├── extracted/
56
+ │ │ └── processed/
57
+ │ ├── Helsinki-NLP--tatoeba_mt/
58
+ │ ├── downloaded/
59
+ │ ├── extracted/
60
+ │ └── processed/
61
+ └── transformers/
62
+ ├── default/
63
+ │ ├── something/
64
+ ├── bert-base-cased/
65
+ │ ├── default/
66
+ │ └── training/
67
+ hub/
68
+ └── models--julien-c--EsperBERTo-small/
69
+ ├── blobs/
70
+ │ ├── (...)
71
+ │ ├── (...)
72
+ ├── refs/
73
+ │ └── (...)
74
+ └── [ 128] snapshots/
75
+ ├── 2439f60ef33a0d46d85da5001d52aeda5b00ce9f/
76
+ │ ├── (...)
77
+ └── bbc77c8132af1cc5cf678da3f1ddf2de43606d48/
78
+ └── (...)
79
+ ```
80
+
81
+
82
+ Args:
83
+ library_name (`str`):
84
+ Name of the library that will manage the cache folder. Example: `"dataset"`.
85
+ namespace (`str`, *optional*, defaults to "default"):
86
+ Namespace to which the data belongs. Example: `"SQuAD"`.
87
+ subfolder (`str`, *optional*, defaults to "default"):
88
+ Subfolder in which the data will be stored. Example: `extracted`.
89
+ assets_dir (`str`, `Path`, *optional*):
90
+ Path to the folder where assets are cached. This must not be the same folder
91
+ where Hub files are cached. Defaults to `HF_HOME / "assets"` if not provided.
92
+ Can also be set with `HF_ASSETS_CACHE` environment variable.
93
+
94
+ Returns:
95
+ Path to the cache folder (`Path`).
96
+
97
+ Example:
98
+ ```py
99
+ >>> from huggingface_hub import cached_assets_path
100
+
101
+ >>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="download")
102
+ PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/download')
103
+
104
+ >>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="extracted")
105
+ PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/extracted')
106
+
107
+ >>> cached_assets_path(library_name="datasets", namespace="Helsinki-NLP/tatoeba_mt")
108
+ PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/Helsinki-NLP--tatoeba_mt/default')
109
+
110
+ >>> cached_assets_path(library_name="datasets", assets_dir="/tmp/tmp123456")
111
+ PosixPath('/tmp/tmp123456/datasets/default/default')
112
+ ```
113
+ """
114
+ # Resolve assets_dir
115
+ if assets_dir is None:
116
+ assets_dir = HF_ASSETS_CACHE
117
+ assets_dir = Path(assets_dir).expanduser().resolve()
118
+
119
+ # Avoid names that could create path issues
120
+ for part in (" ", "/", "\\"):
121
+ library_name = library_name.replace(part, "--")
122
+ namespace = namespace.replace(part, "--")
123
+ subfolder = subfolder.replace(part, "--")
124
+
125
+ # Path to subfolder is created
126
+ path = assets_dir / library_name / namespace / subfolder
127
+ try:
128
+ path.mkdir(exist_ok=True, parents=True)
129
+ except (FileExistsError, NotADirectoryError):
130
+ raise ValueError(f"Corrupted assets folder: cannot create directory because of an existing file ({path}).")
131
+
132
+ # Return
133
+ return path
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_cache_manager.py ADDED
@@ -0,0 +1,901 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains utilities to manage the HF cache directory."""
15
+
16
+ import os
17
+ import shutil
18
+ from collections import defaultdict
19
+ from dataclasses import dataclass
20
+ from pathlib import Path
21
+ from typing import Literal
22
+
23
+ from huggingface_hub.errors import CacheNotFound, CorruptedCacheException
24
+
25
+ from ..constants import HF_HUB_CACHE
26
+ from . import logging
27
+ from ._parsing import format_timesince
28
+ from ._terminal import tabulate
29
+
30
+
31
+ logger = logging.get_logger(__name__)
32
+
33
+ REPO_TYPE_T = Literal["model", "dataset", "space"]
34
+
35
+ # List of OS-created helper files that need to be ignored
36
+ FILES_TO_IGNORE = [".DS_Store", "Thumbs.db", "desktop.ini"]
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class CachedFileInfo:
41
+ """Frozen data structure holding information about a single cached file.
42
+
43
+ Args:
44
+ file_name (`str`):
45
+ Name of the file. Example: `config.json`.
46
+ file_path (`Path`):
47
+ Path of the file in the `snapshots` directory. The file path is a symlink
48
+ referring to a blob in the `blobs` folder.
49
+ blob_path (`Path`):
50
+ Path of the blob file. This is equivalent to `file_path.resolve()`.
51
+ size_on_disk (`int`):
52
+ Size of the blob file in bytes.
53
+ blob_last_accessed (`float`):
54
+ Timestamp of the last time the blob file has been accessed (from any
55
+ revision).
56
+ blob_last_modified (`float`):
57
+ Timestamp of the last time the blob file has been modified/created.
58
+
59
+ > [!WARNING]
60
+ > `blob_last_accessed` and `blob_last_modified` reliability can depend on the OS you
61
+ > are using. See [python documentation](https://docs.python.org/3/library/os.html#os.stat_result)
62
+ > for more details.
63
+ """
64
+
65
+ file_name: str
66
+ file_path: Path
67
+ blob_path: Path
68
+ size_on_disk: int
69
+
70
+ blob_last_accessed: float
71
+ blob_last_modified: float
72
+
73
+ @property
74
+ def blob_last_accessed_str(self) -> str:
75
+ """
76
+ (property) Timestamp of the last time the blob file has been accessed (from any
77
+ revision), returned as a human-readable string.
78
+
79
+ Example: "2 weeks ago".
80
+ """
81
+ return format_timesince(self.blob_last_accessed)
82
+
83
+ @property
84
+ def blob_last_modified_str(self) -> str:
85
+ """
86
+ (property) Timestamp of the last time the blob file has been modified, returned
87
+ as a human-readable string.
88
+
89
+ Example: "2 weeks ago".
90
+ """
91
+ return format_timesince(self.blob_last_modified)
92
+
93
+ @property
94
+ def size_on_disk_str(self) -> str:
95
+ """
96
+ (property) Size of the blob file as a human-readable string.
97
+
98
+ Example: "42.2K".
99
+ """
100
+ return _format_size(self.size_on_disk)
101
+
102
+
103
+ @dataclass(frozen=True)
104
+ class CachedRevisionInfo:
105
+ """Frozen data structure holding information about a revision.
106
+
107
+ A revision correspond to a folder in the `snapshots` folder and is populated with
108
+ the exact tree structure as the repo on the Hub but contains only symlinks. A
109
+ revision can be either referenced by 1 or more `refs` or be "detached" (no refs).
110
+
111
+ Args:
112
+ commit_hash (`str`):
113
+ Hash of the revision (unique).
114
+ Example: `"9338f7b671827df886678df2bdd7cc7b4f36dffd"`.
115
+ snapshot_path (`Path`):
116
+ Path to the revision directory in the `snapshots` folder. It contains the
117
+ exact tree structure as the repo on the Hub.
118
+ files: (`frozenset[CachedFileInfo]`):
119
+ Set of [`~CachedFileInfo`] describing all files contained in the snapshot.
120
+ refs (`frozenset[str]`):
121
+ Set of `refs` pointing to this revision. If the revision has no `refs`, it
122
+ is considered detached.
123
+ Example: `{"main", "2.4.0"}` or `{"refs/pr/1"}`.
124
+ size_on_disk (`int`):
125
+ Sum of the blob file sizes that are symlink-ed by the revision.
126
+ last_modified (`float`):
127
+ Timestamp of the last time the revision has been created/modified.
128
+
129
+ > [!WARNING]
130
+ > `last_accessed` cannot be determined correctly on a single revision as blob files
131
+ > are shared across revisions.
132
+
133
+ > [!WARNING]
134
+ > `size_on_disk` is not necessarily the sum of all file sizes because of possible
135
+ > duplicated files. Besides, only blobs are taken into account, not the (negligible)
136
+ > size of folders and symlinks.
137
+ """
138
+
139
+ commit_hash: str
140
+ snapshot_path: Path
141
+ size_on_disk: int
142
+ files: frozenset[CachedFileInfo]
143
+ refs: frozenset[str]
144
+
145
+ last_modified: float
146
+
147
+ @property
148
+ def last_modified_str(self) -> str:
149
+ """
150
+ (property) Timestamp of the last time the revision has been modified, returned
151
+ as a human-readable string.
152
+
153
+ Example: "2 weeks ago".
154
+ """
155
+ return format_timesince(self.last_modified)
156
+
157
+ @property
158
+ def size_on_disk_str(self) -> str:
159
+ """
160
+ (property) Sum of the blob file sizes as a human-readable string.
161
+
162
+ Example: "42.2K".
163
+ """
164
+ return _format_size(self.size_on_disk)
165
+
166
+ @property
167
+ def nb_files(self) -> int:
168
+ """
169
+ (property) Total number of files in the revision.
170
+ """
171
+ return len(self.files)
172
+
173
+
174
+ @dataclass(frozen=True)
175
+ class CachedRepoInfo:
176
+ """Frozen data structure holding information about a cached repository.
177
+
178
+ Args:
179
+ repo_id (`str`):
180
+ Repo id of the repo on the Hub. Example: `"google/fleurs"`.
181
+ repo_type (`Literal["dataset", "model", "space"]`):
182
+ Type of the cached repo.
183
+ repo_path (`Path`):
184
+ Local path to the cached repo.
185
+ size_on_disk (`int`):
186
+ Sum of the blob file sizes in the cached repo.
187
+ nb_files (`int`):
188
+ Total number of blob files in the cached repo.
189
+ revisions (`frozenset[CachedRevisionInfo]`):
190
+ Set of [`~CachedRevisionInfo`] describing all revisions cached in the repo.
191
+ last_accessed (`float`):
192
+ Timestamp of the last time a blob file of the repo has been accessed.
193
+ last_modified (`float`):
194
+ Timestamp of the last time a blob file of the repo has been modified/created.
195
+
196
+ > [!WARNING]
197
+ > `size_on_disk` is not necessarily the sum of all revisions sizes because of
198
+ > duplicated files. Besides, only blobs are taken into account, not the (negligible)
199
+ > size of folders and symlinks.
200
+
201
+ > [!WARNING]
202
+ > `last_accessed` and `last_modified` reliability can depend on the OS you are using.
203
+ > See [python documentation](https://docs.python.org/3/library/os.html#os.stat_result)
204
+ > for more details.
205
+ """
206
+
207
+ repo_id: str
208
+ repo_type: REPO_TYPE_T
209
+ repo_path: Path
210
+ size_on_disk: int
211
+ nb_files: int
212
+ revisions: frozenset[CachedRevisionInfo]
213
+
214
+ last_accessed: float
215
+ last_modified: float
216
+
217
+ @property
218
+ def last_accessed_str(self) -> str:
219
+ """
220
+ (property) Last time a blob file of the repo has been accessed, returned as a
221
+ human-readable string.
222
+
223
+ Example: "2 weeks ago".
224
+ """
225
+ return format_timesince(self.last_accessed)
226
+
227
+ @property
228
+ def last_modified_str(self) -> str:
229
+ """
230
+ (property) Last time a blob file of the repo has been modified, returned as a
231
+ human-readable string.
232
+
233
+ Example: "2 weeks ago".
234
+ """
235
+ return format_timesince(self.last_modified)
236
+
237
+ @property
238
+ def size_on_disk_str(self) -> str:
239
+ """
240
+ (property) Sum of the blob file sizes as a human-readable string.
241
+
242
+ Example: "42.2K".
243
+ """
244
+ return _format_size(self.size_on_disk)
245
+
246
+ @property
247
+ def cache_id(self) -> str:
248
+ """Canonical `type/id` identifier used across cache tooling."""
249
+ return f"{self.repo_type}/{self.repo_id}"
250
+
251
+ @property
252
+ def refs(self) -> dict[str, CachedRevisionInfo]:
253
+ """
254
+ (property) Mapping between `refs` and revision data structures.
255
+ """
256
+ return {ref: revision for revision in self.revisions for ref in revision.refs}
257
+
258
+
259
+ @dataclass(frozen=True)
260
+ class DeleteCacheStrategy:
261
+ """Frozen data structure holding the strategy to delete cached revisions.
262
+
263
+ This object is not meant to be instantiated programmatically but to be returned by
264
+ [`~utils.HFCacheInfo.delete_revisions`]. See documentation for usage example.
265
+
266
+ Args:
267
+ expected_freed_size (`float`):
268
+ Expected freed size once strategy is executed.
269
+ blobs (`frozenset[Path]`):
270
+ Set of blob file paths to be deleted.
271
+ refs (`frozenset[Path]`):
272
+ Set of reference file paths to be deleted.
273
+ repos (`frozenset[Path]`):
274
+ Set of entire repo paths to be deleted.
275
+ snapshots (`frozenset[Path]`):
276
+ Set of snapshots to be deleted (directory of symlinks).
277
+ """
278
+
279
+ expected_freed_size: int
280
+ blobs: frozenset[Path]
281
+ refs: frozenset[Path]
282
+ repos: frozenset[Path]
283
+ snapshots: frozenset[Path]
284
+
285
+ @property
286
+ def expected_freed_size_str(self) -> str:
287
+ """
288
+ (property) Expected size that will be freed as a human-readable string.
289
+
290
+ Example: "42.2K".
291
+ """
292
+ return _format_size(self.expected_freed_size)
293
+
294
+ def execute(self) -> None:
295
+ """Execute the defined strategy.
296
+
297
+ > [!WARNING]
298
+ > If this method is interrupted, the cache might get corrupted. Deletion order is
299
+ > implemented so that references and symlinks are deleted before the actual blob
300
+ > files.
301
+
302
+ > [!WARNING]
303
+ > This method is irreversible. If executed, cached files are erased and must be
304
+ > downloaded again.
305
+ """
306
+ # Deletion order matters. Blobs are deleted in last so that the user can't end
307
+ # up in a state where a `ref`` refers to a missing snapshot or a snapshot
308
+ # symlink refers to a deleted blob.
309
+
310
+ # Delete entire repos
311
+ for path in self.repos:
312
+ _try_delete_path(path, path_type="repo")
313
+
314
+ # Delete snapshot directories
315
+ for path in self.snapshots:
316
+ _try_delete_path(path, path_type="snapshot")
317
+
318
+ # Delete refs files
319
+ for path in self.refs:
320
+ _try_delete_path(path, path_type="ref")
321
+
322
+ # Delete blob files
323
+ for path in self.blobs:
324
+ _try_delete_path(path, path_type="blob")
325
+
326
+ logger.info(f"Cache deletion done. Saved {self.expected_freed_size_str}.")
327
+
328
+
329
+ @dataclass(frozen=True)
330
+ class CachedIncompleteFileInfo:
331
+ """Frozen data structure holding information about a single incomplete download.
332
+
333
+ Interrupted downloads leave `<cache>/<repo>/blobs/<etag>.incomplete` files behind.
334
+ These are not part of any committed revision, so they are surfaced separately by
335
+ [`scan_cache_dir`].
336
+
337
+ Args:
338
+ file_path (`Path`):
339
+ Path of the `.incomplete` file in the `blobs` folder.
340
+ size_on_disk (`int`):
341
+ Size of the partially-downloaded file in bytes.
342
+ """
343
+
344
+ file_path: Path
345
+ size_on_disk: int
346
+
347
+
348
+ @dataclass(frozen=True)
349
+ class HFCacheInfo:
350
+ """Frozen data structure holding information about the entire cache-system.
351
+
352
+ This data structure is returned by [`scan_cache_dir`] and is immutable.
353
+
354
+ Args:
355
+ size_on_disk (`int`):
356
+ Sum of all valid repo sizes in the cache-system.
357
+ repos (`frozenset[CachedRepoInfo]`):
358
+ Set of [`~CachedRepoInfo`] describing all valid cached repos found on the
359
+ cache-system while scanning.
360
+ incomplete_files (`frozenset[CachedIncompleteFileInfo]`):
361
+ Set of [`~CachedIncompleteFileInfo`] describing orphaned `*.incomplete`
362
+ files left behind by interrupted downloads.
363
+ warnings (`list[CorruptedCacheException]`):
364
+ List of [`~CorruptedCacheException`] that occurred while scanning the cache.
365
+ Those exceptions are captured so that the scan can continue. Corrupted repos
366
+ are skipped from the scan.
367
+
368
+ > [!WARNING]
369
+ > Here `size_on_disk` is equal to the sum of all repo sizes (only blobs). However if
370
+ > some cached repos are corrupted, their sizes are not taken into account.
371
+ """
372
+
373
+ size_on_disk: int
374
+ repos: frozenset[CachedRepoInfo]
375
+ incomplete_files: frozenset[CachedIncompleteFileInfo]
376
+ warnings: list[CorruptedCacheException]
377
+
378
+ @property
379
+ def size_on_disk_str(self) -> str:
380
+ """
381
+ (property) Sum of all valid repo sizes in the cache-system as a human-readable
382
+ string.
383
+
384
+ Example: "42.2K".
385
+ """
386
+ return _format_size(self.size_on_disk)
387
+
388
+ @property
389
+ def incomplete_size_on_disk(self) -> int:
390
+ """(property) Sum of all incomplete download sizes in bytes."""
391
+ return sum(file.size_on_disk for file in self.incomplete_files)
392
+
393
+ def delete_revisions(self, *revisions: str) -> DeleteCacheStrategy:
394
+ """Prepare the strategy to delete one or more revisions cached locally.
395
+
396
+ Input revisions can be any revision hash. If a revision hash is not found in the
397
+ local cache, a warning is thrown but no error is raised. Revisions can be from
398
+ different cached repos since hashes are unique across repos,
399
+
400
+ Examples:
401
+ ```py
402
+ >>> from huggingface_hub import scan_cache_dir
403
+ >>> cache_info = scan_cache_dir()
404
+ >>> delete_strategy = cache_info.delete_revisions(
405
+ ... "81fd1d6e7847c99f5862c9fb81387956d99ec7aa"
406
+ ... )
407
+ >>> print(f"Will free {delete_strategy.expected_freed_size_str}.")
408
+ Will free 7.9K.
409
+ >>> delete_strategy.execute()
410
+ Cache deletion done. Saved 7.9K.
411
+ ```
412
+
413
+ ```py
414
+ >>> from huggingface_hub import scan_cache_dir
415
+ >>> scan_cache_dir().delete_revisions(
416
+ ... "81fd1d6e7847c99f5862c9fb81387956d99ec7aa",
417
+ ... "e2983b237dccf3ab4937c97fa717319a9ca1a96d",
418
+ ... "6c0e6080953db56375760c0471a8c5f2929baf11",
419
+ ... ).execute()
420
+ Cache deletion done. Saved 8.6G.
421
+ ```
422
+
423
+ > [!WARNING]
424
+ > `delete_revisions` returns a [`~utils.DeleteCacheStrategy`] object that needs to
425
+ > be executed. The [`~utils.DeleteCacheStrategy`] is not meant to be modified but
426
+ > allows having a dry run before actually executing the deletion.
427
+ """
428
+ hashes_to_delete: set[str] = set(revisions)
429
+
430
+ repos_with_revisions: dict[CachedRepoInfo, set[CachedRevisionInfo]] = defaultdict(set)
431
+
432
+ for repo in self.repos:
433
+ for revision in repo.revisions:
434
+ if revision.commit_hash in hashes_to_delete:
435
+ repos_with_revisions[repo].add(revision)
436
+ hashes_to_delete.remove(revision.commit_hash)
437
+
438
+ if len(hashes_to_delete) > 0:
439
+ logger.warning(f"Revision(s) not found - cannot delete them: {', '.join(hashes_to_delete)}")
440
+
441
+ delete_strategy_blobs: set[Path] = set()
442
+ delete_strategy_refs: set[Path] = set()
443
+ delete_strategy_repos: set[Path] = set()
444
+ delete_strategy_snapshots: set[Path] = set()
445
+ delete_strategy_expected_freed_size = 0
446
+
447
+ for affected_repo, revisions_to_delete in repos_with_revisions.items():
448
+ other_revisions = affected_repo.revisions - revisions_to_delete
449
+
450
+ # If no other revisions, it means all revisions are deleted
451
+ # -> delete the entire cached repo
452
+ if len(other_revisions) == 0:
453
+ delete_strategy_repos.add(affected_repo.repo_path)
454
+ delete_strategy_expected_freed_size += affected_repo.size_on_disk
455
+ continue
456
+
457
+ # Some revisions of the repo will be deleted but not all. We need to filter
458
+ # which blob files will not be linked anymore.
459
+ for revision_to_delete in revisions_to_delete:
460
+ # Snapshot dir
461
+ delete_strategy_snapshots.add(revision_to_delete.snapshot_path)
462
+
463
+ # Refs dir
464
+ for ref in revision_to_delete.refs:
465
+ delete_strategy_refs.add(affected_repo.repo_path / "refs" / ref)
466
+
467
+ # Blobs dir
468
+ for file in revision_to_delete.files:
469
+ # Some files are not symlinks to the blobs dir: files copied instead of symlinked
470
+ # (Windows, filesystems without symlink support) or files created by the user
471
+ # inside the snapshot dir (e.g. build artifacts from a `pip install`). They are
472
+ # already removed when deleting the snapshot dir -> count them but don't delete
473
+ # them a second time.
474
+ if revision_to_delete.snapshot_path in file.blob_path.parents:
475
+ delete_strategy_expected_freed_size += file.size_on_disk
476
+ continue
477
+
478
+ if file.blob_path not in delete_strategy_blobs:
479
+ is_file_alone = True
480
+ for revision in other_revisions:
481
+ for rev_file in revision.files:
482
+ if file.blob_path == rev_file.blob_path:
483
+ is_file_alone = False
484
+ break
485
+ if not is_file_alone:
486
+ break
487
+
488
+ # Blob file not referenced by remaining revisions -> delete
489
+ if is_file_alone:
490
+ delete_strategy_blobs.add(file.blob_path)
491
+ delete_strategy_expected_freed_size += file.size_on_disk
492
+
493
+ # Return the strategy instead of executing it.
494
+ return DeleteCacheStrategy(
495
+ blobs=frozenset(delete_strategy_blobs),
496
+ refs=frozenset(delete_strategy_refs),
497
+ repos=frozenset(delete_strategy_repos),
498
+ snapshots=frozenset(delete_strategy_snapshots),
499
+ expected_freed_size=delete_strategy_expected_freed_size,
500
+ )
501
+
502
+ def export_as_table(self, *, verbosity: int = 0) -> str:
503
+ """Generate a table from the [`HFCacheInfo`] object.
504
+
505
+ Pass `verbosity=0` to get a table with a single row per repo, with columns
506
+ "repo_id", "repo_type", "size_on_disk", "nb_files", "last_accessed", "last_modified", "refs", "local_path".
507
+
508
+ Pass `verbosity=1` to get a table with a row per repo and revision (thus multiple rows can appear for a single repo), with columns
509
+ "repo_id", "repo_type", "revision", "size_on_disk", "nb_files", "last_modified", "refs", "local_path".
510
+
511
+ Example:
512
+ ```py
513
+ >>> from huggingface_hub.utils import scan_cache_dir
514
+
515
+ >>> hf_cache_info = scan_cache_dir()
516
+ HFCacheInfo(...)
517
+
518
+ >>> print(hf_cache_info.export_as_table())
519
+ REPO ID REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED LAST_MODIFIED REFS LOCAL PATH
520
+ --------------------------------------------------- --------- ------------ -------- ------------- ------------- ---- --------------------------------------------------------------------------------------------------
521
+ roberta-base model 2.7M 5 1 day ago 1 week ago main ~/.cache/huggingface/hub/models--roberta-base
522
+ suno/bark model 8.8K 1 1 week ago 1 week ago main ~/.cache/huggingface/hub/models--suno--bark
523
+ t5-base model 893.8M 4 4 days ago 7 months ago main ~/.cache/huggingface/hub/models--t5-base
524
+ t5-large model 3.0G 4 5 weeks ago 5 months ago main ~/.cache/huggingface/hub/models--t5-large
525
+
526
+ >>> print(hf_cache_info.export_as_table(verbosity=1))
527
+ REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED REFS LOCAL PATH
528
+ --------------------------------------------------- --------- ---------------------------------------- ------------ -------- ------------- ---- -----------------------------------------------------------------------------------------------------------------------------------------------------
529
+ roberta-base model e2da8e2f811d1448a5b465c236feacd80ffbac7b 2.7M 5 1 week ago main ~/.cache/huggingface/hub/models--roberta-base/snapshots/e2da8e2f811d1448a5b465c236feacd80ffbac7b
530
+ suno/bark model 70a8a7d34168586dc5d028fa9666aceade177992 8.8K 1 1 week ago main ~/.cache/huggingface/hub/models--suno--bark/snapshots/70a8a7d34168586dc5d028fa9666aceade177992
531
+ t5-base model a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 893.8M 4 7 months ago main ~/.cache/huggingface/hub/models--t5-base/snapshots/a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1
532
+ t5-large model 150ebc2c4b72291e770f58e6057481c8d2ed331a 3.0G 4 5 months ago main ~/.cache/huggingface/hub/models--t5-large/snapshots/150ebc2c4b72291e770f58e6057481c8d2ed331a
533
+ ```
534
+
535
+ Args:
536
+ verbosity (`int`, *optional*):
537
+ The verbosity level. Defaults to 0.
538
+
539
+ Returns:
540
+ `str`: The table as a string.
541
+ """
542
+ if verbosity == 0:
543
+ return tabulate(
544
+ rows=[
545
+ [
546
+ repo.repo_id,
547
+ repo.repo_type,
548
+ f"{repo.size_on_disk_str:>12}",
549
+ repo.nb_files,
550
+ repo.last_accessed_str,
551
+ repo.last_modified_str,
552
+ ", ".join(sorted(repo.refs)),
553
+ str(repo.repo_path),
554
+ ]
555
+ for repo in sorted(self.repos, key=lambda repo: repo.repo_path)
556
+ ],
557
+ headers=[
558
+ "REPO ID",
559
+ "REPO TYPE",
560
+ "SIZE ON DISK",
561
+ "NB FILES",
562
+ "LAST_ACCESSED",
563
+ "LAST_MODIFIED",
564
+ "REFS",
565
+ "LOCAL PATH",
566
+ ],
567
+ )
568
+ else:
569
+ return tabulate(
570
+ rows=[
571
+ [
572
+ repo.repo_id,
573
+ repo.repo_type,
574
+ revision.commit_hash,
575
+ f"{revision.size_on_disk_str:>12}",
576
+ revision.nb_files,
577
+ revision.last_modified_str,
578
+ ", ".join(sorted(revision.refs)),
579
+ str(revision.snapshot_path),
580
+ ]
581
+ for repo in sorted(self.repos, key=lambda repo: repo.repo_path)
582
+ for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash)
583
+ ],
584
+ headers=[
585
+ "REPO ID",
586
+ "REPO TYPE",
587
+ "REVISION",
588
+ "SIZE ON DISK",
589
+ "NB FILES",
590
+ "LAST_MODIFIED",
591
+ "REFS",
592
+ "LOCAL PATH",
593
+ ],
594
+ )
595
+
596
+
597
+ def scan_cache_dir(cache_dir: str | Path | None = None) -> HFCacheInfo:
598
+ """Scan the entire HF cache-system and return a [`~HFCacheInfo`] structure.
599
+
600
+ Use `scan_cache_dir` in order to programmatically scan your cache-system. The cache
601
+ will be scanned repo by repo. If a repo is corrupted, a [`~CorruptedCacheException`]
602
+ will be thrown internally but captured and returned in the [`~HFCacheInfo`]
603
+ structure. Only valid repos get a proper report.
604
+
605
+ ```py
606
+ >>> from huggingface_hub import scan_cache_dir
607
+
608
+ >>> hf_cache_info = scan_cache_dir()
609
+ HFCacheInfo(
610
+ size_on_disk=3398085269,
611
+ repos=frozenset({
612
+ CachedRepoInfo(
613
+ repo_id='t5-small',
614
+ repo_type='model',
615
+ repo_path=PosixPath(...),
616
+ size_on_disk=970726914,
617
+ nb_files=11,
618
+ revisions=frozenset({
619
+ CachedRevisionInfo(
620
+ commit_hash='d78aea13fa7ecd06c29e3e46195d6341255065d5',
621
+ size_on_disk=970726339,
622
+ snapshot_path=PosixPath(...),
623
+ files=frozenset({
624
+ CachedFileInfo(
625
+ file_name='config.json',
626
+ size_on_disk=1197
627
+ file_path=PosixPath(...),
628
+ blob_path=PosixPath(...),
629
+ ),
630
+ CachedFileInfo(...),
631
+ ...
632
+ }),
633
+ ),
634
+ CachedRevisionInfo(...),
635
+ ...
636
+ }),
637
+ ),
638
+ CachedRepoInfo(...),
639
+ ...
640
+ }),
641
+ warnings=[
642
+ CorruptedCacheException("Snapshots dir doesn't exist in cached repo: ..."),
643
+ CorruptedCacheException(...),
644
+ ...
645
+ ],
646
+ )
647
+ ```
648
+
649
+ You can also print a detailed report directly from the `hf` command line using:
650
+ ```text
651
+ > hf cache ls
652
+ ID SIZE LAST_ACCESSED LAST_MODIFIED REFS
653
+ --------------------------- -------- ------------- ------------- -----------
654
+ dataset/nyu-mll/glue 157.4M 2 days ago 2 days ago main script
655
+ model/LiquidAI/LFM2-VL-1.6B 3.2G 4 days ago 4 days ago main
656
+ model/microsoft/UserLM-8b 32.1G 4 days ago 4 days ago main
657
+
658
+ Done in 0.0s. Scanned 6 repo(s) for a total of 3.4G.
659
+ Got 1 warning(s) while scanning. Use -vvv to print details.
660
+ ```
661
+
662
+ Args:
663
+ cache_dir (`str` or `Path`, `optional`):
664
+ Cache directory to cache. Defaults to the default HF cache directory.
665
+
666
+ > [!WARNING]
667
+ > Raises:
668
+ >
669
+ > `CacheNotFound`
670
+ > If the cache directory does not exist.
671
+ >
672
+ > [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
673
+ > If the cache directory is a file, instead of a directory.
674
+
675
+ Returns: a [`~HFCacheInfo`] object.
676
+ """
677
+ if cache_dir is None:
678
+ cache_dir = HF_HUB_CACHE
679
+
680
+ cache_dir = Path(cache_dir).expanduser().resolve()
681
+ if not cache_dir.exists():
682
+ raise CacheNotFound(
683
+ f"Cache directory not found: {cache_dir}. Please use `cache_dir` argument or set `HF_HUB_CACHE` environment variable.",
684
+ cache_dir=cache_dir,
685
+ )
686
+
687
+ if cache_dir.is_file():
688
+ raise ValueError(
689
+ f"Scan cache expects a directory but found a file: {cache_dir}. Please use `cache_dir` argument or set `HF_HUB_CACHE` environment variable."
690
+ )
691
+
692
+ repos: set[CachedRepoInfo] = set()
693
+ warnings: list[CorruptedCacheException] = []
694
+ for repo_path in cache_dir.iterdir():
695
+ if repo_path.name in FILES_TO_IGNORE:
696
+ continue
697
+ if repo_path.name == ".locks": # skip './.locks/' folder
698
+ continue
699
+ if repo_path.name == "CACHEDIR.TAG": # skip CACHEDIR.TAG file
700
+ continue
701
+ try:
702
+ repos.add(_scan_cached_repo(repo_path))
703
+ except CorruptedCacheException as e:
704
+ warnings.append(e)
705
+
706
+ return HFCacheInfo(
707
+ repos=frozenset(repos),
708
+ size_on_disk=sum(repo.size_on_disk for repo in repos),
709
+ incomplete_files=_scan_incomplete_files(cache_dir),
710
+ warnings=warnings,
711
+ )
712
+
713
+
714
+ def _scan_incomplete_files(cache_dir: Path) -> frozenset[CachedIncompleteFileInfo]:
715
+ """Find orphaned `*.incomplete` partial-download files in the cache.
716
+
717
+ Interrupted downloads leave `<cache>/<repo>/blobs/<etag>.incomplete` files behind.
718
+ These are not part of any committed revision, so they are reported separately in the
719
+ [`~HFCacheInfo`] returned by [`scan_cache_dir`].
720
+ """
721
+ files: set[CachedIncompleteFileInfo] = set()
722
+ for path in cache_dir.glob("*/blobs/*.incomplete"):
723
+ try:
724
+ size_on_disk = path.stat().st_size
725
+ except OSError:
726
+ continue
727
+ files.add(CachedIncompleteFileInfo(file_path=path, size_on_disk=size_on_disk))
728
+ return frozenset(files)
729
+
730
+
731
+ def _scan_cached_repo(repo_path: Path) -> CachedRepoInfo:
732
+ """Scan a single cache repo and return information about it.
733
+
734
+ Any unexpected behavior will raise a [`~CorruptedCacheException`].
735
+ """
736
+ if not repo_path.is_dir():
737
+ raise CorruptedCacheException(f"Repo path is not a directory: {repo_path}")
738
+
739
+ if "--" not in repo_path.name:
740
+ raise CorruptedCacheException(f"Repo path is not a valid HuggingFace cache directory: {repo_path}")
741
+
742
+ repo_type, repo_id = repo_path.name.split("--", maxsplit=1)
743
+ repo_type = repo_type[:-1] # "models" -> "model"
744
+ repo_id = repo_id.replace("--", "/") # google/fleurs -> "google/fleurs"
745
+
746
+ if repo_type not in {"dataset", "model", "space"}:
747
+ raise CorruptedCacheException(
748
+ f"Repo type must be `dataset`, `model` or `space`, found `{repo_type}` ({repo_path})."
749
+ )
750
+
751
+ blob_stats: dict[Path, os.stat_result] = {} # Key is blob_path, value is blob stats
752
+
753
+ snapshots_path = repo_path / "snapshots"
754
+ refs_path = repo_path / "refs"
755
+
756
+ if not snapshots_path.exists() or not snapshots_path.is_dir():
757
+ raise CorruptedCacheException(f"Snapshots dir doesn't exist in cached repo: {snapshots_path}")
758
+
759
+ # Scan over `refs` directory
760
+
761
+ # key is revision hash, value is set of refs
762
+ refs_by_hash: dict[str, set[str]] = defaultdict(set)
763
+ if refs_path.exists():
764
+ # Example of `refs` directory
765
+ # ── refs
766
+ # ├── main
767
+ # └── refs
768
+ # └── pr
769
+ # └── 1
770
+ if refs_path.is_file():
771
+ raise CorruptedCacheException(f"Refs directory cannot be a file: {refs_path}")
772
+
773
+ for ref_path in refs_path.glob("**/*"):
774
+ # glob("**/*") iterates over all files and directories -> skip directories
775
+ if ref_path.is_dir() or ref_path.name in FILES_TO_IGNORE:
776
+ continue
777
+
778
+ ref_name = str(ref_path.relative_to(refs_path))
779
+ with ref_path.open() as f:
780
+ commit_hash = f.read()
781
+
782
+ refs_by_hash[commit_hash].add(ref_name)
783
+
784
+ # Scan snapshots directory
785
+ cached_revisions: set[CachedRevisionInfo] = set()
786
+ for revision_path in snapshots_path.iterdir():
787
+ # Ignore OS-created helper files
788
+ if revision_path.name in FILES_TO_IGNORE:
789
+ continue
790
+ if revision_path.is_file():
791
+ raise CorruptedCacheException(f"Snapshots folder corrupted. Found a file: {revision_path}")
792
+
793
+ cached_files = set()
794
+ for file_path in revision_path.glob("**/*"):
795
+ # glob("**/*") iterates over all files and directories -> skip directories
796
+ if file_path.is_dir():
797
+ continue
798
+
799
+ blob_path = Path(file_path).resolve()
800
+ if not blob_path.exists():
801
+ raise CorruptedCacheException(f"Blob missing (broken symlink): {blob_path}")
802
+
803
+ if blob_path not in blob_stats:
804
+ blob_stats[blob_path] = blob_path.stat()
805
+
806
+ cached_files.add(
807
+ CachedFileInfo(
808
+ file_name=file_path.name,
809
+ file_path=file_path,
810
+ size_on_disk=blob_stats[blob_path].st_size,
811
+ blob_path=blob_path,
812
+ blob_last_accessed=blob_stats[blob_path].st_atime,
813
+ blob_last_modified=blob_stats[blob_path].st_mtime,
814
+ )
815
+ )
816
+
817
+ # Last modified is either the last modified blob file or the revision folder
818
+ # itself if it is empty
819
+ if len(cached_files) > 0:
820
+ revision_last_modified = max(blob_stats[file.blob_path].st_mtime for file in cached_files)
821
+ else:
822
+ revision_last_modified = revision_path.stat().st_mtime
823
+
824
+ cached_revisions.add(
825
+ CachedRevisionInfo(
826
+ commit_hash=revision_path.name,
827
+ files=frozenset(cached_files),
828
+ refs=frozenset(refs_by_hash.pop(revision_path.name, set())),
829
+ size_on_disk=sum(
830
+ blob_stats[blob_path].st_size for blob_path in {file.blob_path for file in cached_files}
831
+ ),
832
+ snapshot_path=revision_path,
833
+ last_modified=revision_last_modified,
834
+ )
835
+ )
836
+
837
+ # Check that all refs referred to an existing revision
838
+ if len(refs_by_hash) > 0:
839
+ raise CorruptedCacheException(
840
+ f"Reference(s) refer to missing commit hashes: {dict(refs_by_hash)} ({repo_path})."
841
+ )
842
+
843
+ # Last modified is either the last modified blob file or the repo folder itself if
844
+ # no blob files has been found. Same for last accessed.
845
+ if len(blob_stats) > 0:
846
+ repo_last_accessed = max(stat.st_atime for stat in blob_stats.values())
847
+ repo_last_modified = max(stat.st_mtime for stat in blob_stats.values())
848
+ else:
849
+ repo_stats = repo_path.stat()
850
+ repo_last_accessed = repo_stats.st_atime
851
+ repo_last_modified = repo_stats.st_mtime
852
+
853
+ # Build and return frozen structure
854
+ return CachedRepoInfo(
855
+ nb_files=len(blob_stats),
856
+ repo_id=repo_id,
857
+ repo_path=repo_path,
858
+ repo_type=repo_type, # type: ignore
859
+ revisions=frozenset(cached_revisions),
860
+ size_on_disk=sum(stat.st_size for stat in blob_stats.values()),
861
+ last_accessed=repo_last_accessed,
862
+ last_modified=repo_last_modified,
863
+ )
864
+
865
+
866
+ def _format_size(num: int) -> str:
867
+ """Format size in bytes into a human-readable string.
868
+
869
+ Taken from https://stackoverflow.com/a/1094933
870
+ """
871
+ num_f = float(num)
872
+ for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
873
+ if abs(num_f) < 1000.0:
874
+ return f"{num_f:3.1f}{unit}"
875
+ num_f /= 1000.0
876
+ return f"{num_f:.1f}Y"
877
+
878
+
879
+ def _try_delete_path(path: Path, path_type: str) -> None:
880
+ """Try to delete a local file or folder.
881
+
882
+ If the path does not exist, error is logged as a warning and then ignored.
883
+
884
+ Args:
885
+ path (`Path`)
886
+ Path to delete. Can be a file or a folder.
887
+ path_type (`str`)
888
+ What path are we deleting ? Only for logging purposes. Example: "snapshot".
889
+ """
890
+ logger.debug(f"Delete {path_type}: {path}")
891
+ try:
892
+ if path.is_file():
893
+ os.remove(path)
894
+ else:
895
+ shutil.rmtree(path)
896
+ except FileNotFoundError:
897
+ # Nothing to delete: the path is already gone. Log without traceback to avoid flooding the
898
+ # terminal when deleting many paths at once.
899
+ logger.warning(f"Couldn't delete {path_type}: file not found ({path})")
900
+ except PermissionError:
901
+ logger.warning(f"Couldn't delete {path_type}: permission denied ({path})", exc_info=True)
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_chunk_utils.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains a utility to iterate by chunks over an iterator."""
15
+
16
+ import itertools
17
+ from collections.abc import Iterable
18
+ from typing import TypeVar
19
+
20
+
21
+ T = TypeVar("T")
22
+
23
+
24
+ def chunk_iterable(iterable: Iterable[T], chunk_size: int) -> Iterable[Iterable[T]]:
25
+ """Iterates over an iterator chunk by chunk.
26
+
27
+ Taken from https://stackoverflow.com/a/8998040.
28
+ See also https://github.com/huggingface/huggingface_hub/pull/920#discussion_r938793088.
29
+
30
+ Args:
31
+ iterable (`Iterable`):
32
+ The iterable on which we want to iterate.
33
+ chunk_size (`int`):
34
+ Size of the chunks. Must be a strictly positive integer (e.g. >0).
35
+
36
+ Example:
37
+
38
+ ```python
39
+ >>> from huggingface_hub.utils import chunk_iterable
40
+
41
+ >>> for items in chunk_iterable(range(17), chunk_size=8):
42
+ ... print(items)
43
+ # [0, 1, 2, 3, 4, 5, 6, 7]
44
+ # [8, 9, 10, 11, 12, 13, 14, 15]
45
+ # [16] # smaller last chunk
46
+ ```
47
+
48
+ Raises:
49
+ [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
50
+ If `chunk_size` <= 0.
51
+
52
+ > [!WARNING]
53
+ > The last chunk can be smaller than `chunk_size`.
54
+ """
55
+ if not isinstance(chunk_size, int) or chunk_size <= 0:
56
+ raise ValueError("`chunk_size` must be a strictly positive integer (>0).")
57
+
58
+ iterator = iter(iterable)
59
+ while True:
60
+ try:
61
+ next_item = next(iterator)
62
+ except StopIteration:
63
+ return
64
+ yield itertools.chain((next_item,), itertools.islice(iterator, chunk_size - 1))
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_datetime.py ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains utilities to handle datetimes in Huggingface Hub."""
15
+
16
+ from datetime import datetime, timezone
17
+
18
+
19
+ def parse_datetime(date_string: str) -> datetime:
20
+ """
21
+ Parses a date_string returned from the server to a datetime object.
22
+
23
+ This parser is a weak-parser is the sense that it handles only a single format of
24
+ date_string. It is expected that the server format will never change. The
25
+ implementation depends only on the standard lib to avoid an external dependency
26
+ (python-dateutil). See full discussion about this decision on PR:
27
+ https://github.com/huggingface/huggingface_hub/pull/999.
28
+
29
+ Example:
30
+ ```py
31
+ > parse_datetime('2022-08-19T07:19:38.123Z')
32
+ datetime.datetime(2022, 8, 19, 7, 19, 38, 123000, tzinfo=timezone.utc)
33
+ ```
34
+
35
+ Args:
36
+ date_string (`str`):
37
+ A string representing a datetime returned by the Hub server.
38
+ String is expected to follow '%Y-%m-%dT%H:%M:%S.%fZ' pattern.
39
+
40
+ Returns:
41
+ A python datetime object.
42
+
43
+ Raises:
44
+ :class:`ValueError`:
45
+ If `date_string` cannot be parsed.
46
+ """
47
+ try:
48
+ # Normalize the string to always have 6 digits of fractional seconds
49
+ if date_string.endswith("Z"):
50
+ # Case 1: No decimal point (e.g., "2024-11-16T00:27:02Z")
51
+ if "." not in date_string:
52
+ # No fractional seconds - insert .000000
53
+ date_string = date_string[:-1] + ".000000Z"
54
+ # Case 2: Has decimal point (e.g., "2022-08-19T07:19:38.123456789Z")
55
+ else:
56
+ # Get the fractional and base parts
57
+ base, fraction = date_string[:-1].split(".")
58
+ # fraction[:6] takes first 6 digits and :0<6 pads with zeros if less than 6 digits
59
+ date_string = f"{base}.{fraction[:6]:0<6}Z"
60
+
61
+ return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
62
+ except ValueError as e:
63
+ raise ValueError(
64
+ f"Cannot parse '{date_string}' as a datetime. Date string is expected to"
65
+ " follow '%Y-%m-%dT%H:%M:%S.%fZ' pattern."
66
+ ) from e
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_deprecation.py ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+ from collections.abc import Iterable
3
+ from functools import wraps
4
+ from inspect import Parameter, signature
5
+
6
+
7
+ def _deprecate_positional_args(*, version: str):
8
+ """Decorator for methods that issues warnings for positional arguments.
9
+ Using the keyword-only argument syntax in pep 3102, arguments after the
10
+ * will issue a warning when passed as a positional argument.
11
+
12
+ Args:
13
+ version (`str`):
14
+ The version when positional arguments will result in error.
15
+ """
16
+
17
+ def _inner_deprecate_positional_args(f):
18
+ sig = signature(f)
19
+ kwonly_args = []
20
+ all_args = []
21
+ for name, param in sig.parameters.items():
22
+ if param.kind == Parameter.POSITIONAL_OR_KEYWORD:
23
+ all_args.append(name)
24
+ elif param.kind == Parameter.KEYWORD_ONLY:
25
+ kwonly_args.append(name)
26
+
27
+ @wraps(f)
28
+ def inner_f(*args, **kwargs):
29
+ extra_args = len(args) - len(all_args)
30
+ if extra_args <= 0:
31
+ return f(*args, **kwargs)
32
+ # extra_args > 0
33
+ args_msg = [
34
+ f"{name}='{arg}'" if isinstance(arg, str) else f"{name}={arg}"
35
+ for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:])
36
+ ]
37
+ args_msg = ", ".join(args_msg)
38
+ warnings.warn(
39
+ f"Deprecated positional argument(s) used in '{f.__name__}': pass"
40
+ f" {args_msg} as keyword args. From version {version} passing these"
41
+ " as positional arguments will result in an error,",
42
+ FutureWarning,
43
+ )
44
+ kwargs.update(zip(sig.parameters, args))
45
+ return f(**kwargs)
46
+
47
+ return inner_f
48
+
49
+ return _inner_deprecate_positional_args
50
+
51
+
52
+ def _deprecate_arguments(
53
+ *,
54
+ version: str,
55
+ deprecated_args: Iterable[str],
56
+ custom_message: str | None = None,
57
+ ):
58
+ """Decorator to issue warnings when using deprecated arguments.
59
+
60
+ TODO: could be useful to be able to set a custom error message.
61
+
62
+ Args:
63
+ version (`str`):
64
+ The version when deprecated arguments will result in error.
65
+ deprecated_args (`list[str]`):
66
+ List of the arguments to be deprecated.
67
+ custom_message (`str`, *optional*):
68
+ Warning message that is raised. If not passed, a default warning message
69
+ will be created.
70
+ """
71
+
72
+ def _inner_deprecate_positional_args(f):
73
+ sig = signature(f)
74
+
75
+ @wraps(f)
76
+ def inner_f(*args, **kwargs):
77
+ # Check for used deprecated arguments
78
+ used_deprecated_args = []
79
+ for _, parameter in zip(args, sig.parameters.values()):
80
+ if parameter.name in deprecated_args:
81
+ used_deprecated_args.append(parameter.name)
82
+ for kwarg_name, kwarg_value in kwargs.items():
83
+ if (
84
+ # If argument is deprecated but still used
85
+ kwarg_name in deprecated_args
86
+ # And then the value is not the default value
87
+ and kwarg_value != sig.parameters[kwarg_name].default
88
+ ):
89
+ used_deprecated_args.append(kwarg_name)
90
+
91
+ # Warn and proceed
92
+ if len(used_deprecated_args) > 0:
93
+ message = (
94
+ f"Deprecated argument(s) used in '{f.__name__}':"
95
+ f" {', '.join(used_deprecated_args)}. Will not be supported from"
96
+ f" version '{version}'."
97
+ )
98
+ if custom_message is not None:
99
+ message += "\n\n" + custom_message
100
+ warnings.warn(message, FutureWarning)
101
+ return f(*args, **kwargs)
102
+
103
+ return inner_f
104
+
105
+ return _inner_deprecate_positional_args
106
+
107
+
108
+ def _deprecate_method(*, version: str, message: str | None = None):
109
+ """Decorator to issue warnings when using a deprecated method.
110
+
111
+ Args:
112
+ version (`str`):
113
+ The version when deprecated arguments will result in error.
114
+ message (`str`, *optional*):
115
+ Warning message that is raised. If not passed, a default warning message
116
+ will be created.
117
+ """
118
+
119
+ def _inner_deprecate_method(f):
120
+ name = f.__name__
121
+ if name == "__init__":
122
+ name = f.__qualname__.split(".")[0] # class name instead of method name
123
+
124
+ @wraps(f)
125
+ def inner_f(*args, **kwargs):
126
+ warning_message = (
127
+ f"'{name}' (from '{f.__module__}') is deprecated and will be removed from version '{version}'."
128
+ )
129
+ if message is not None:
130
+ warning_message += " " + message
131
+ warnings.warn(warning_message, FutureWarning)
132
+ return f(*args, **kwargs)
133
+
134
+ return inner_f
135
+
136
+ return _inner_deprecate_method
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_detect_agent.py ADDED
@@ -0,0 +1,204 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Detect whether the process is being invoked by an AI coding agent.
15
+
16
+ Detection is based on environment variables that AI agents set in their shell
17
+ sessions. `AI_AGENT` and `AGENT` are treated as a universal standard (any
18
+ tool can set its harness id there); the remaining checks are tool-specific and
19
+ ordered by priority (first match wins).
20
+
21
+ The list of known harnesses is maintained on the Hub and exposed at
22
+ `{ENDPOINT}/api/agent-harnesses`. We fetch it at most once a day and cache it
23
+ locally so the list can be updated without requiring a new client release.
24
+
25
+ Detection is entirely best-effort: there is no hardcoded list of harnesses. When
26
+ the registry cannot be fetched (and no cached copy is available), detection simply
27
+ reports "no agent". Any error while fetching/reading the registry is swallowed —
28
+ detection must never make a process fail.
29
+
30
+ More details: https://huggingface.co/docs/hub/agents-overview#register-your-agent-harness
31
+ """
32
+
33
+ import json
34
+ import os
35
+ import time
36
+ from pathlib import Path
37
+ from typing import Optional, TypedDict
38
+
39
+ from .. import constants
40
+ from . import logging
41
+
42
+
43
+ logger = logging.get_logger(__name__)
44
+
45
+ # Refresh the cached registry at most once every 24 hours.
46
+ _REGISTRY_TTL_SECONDS = 24 * 3600
47
+
48
+ # Short timeout: fetching the registry is best-effort telemetry, never block the caller for long.
49
+ _REGISTRY_FETCH_TIMEOUT = 3
50
+
51
+
52
+ class HarnessInfo(TypedDict, total=False):
53
+ """A single harness entry. `envVars` maps an env var name to a match pattern (see `_env_vars_match`)."""
54
+
55
+ envVars: dict[str, str]
56
+
57
+
58
+ class Registry(TypedDict):
59
+ """The agent harness registry, as served by `{ENDPOINT}/api/agent-harnesses`."""
60
+
61
+ standardEnvVars: list[str]
62
+ harnesses: dict[str, HarnessInfo]
63
+
64
+
65
+ # Empty registry: detection is disabled (no agent ever detected). Used when the
66
+ # Hub is unreachable and no cached copy is available.
67
+ _EMPTY_REGISTRY: Registry = {"standardEnvVars": [], "harnesses": {}}
68
+
69
+ # In-process cache of the resolved registry. Populated lazily on first detection.
70
+ _registry: Registry | None = None
71
+
72
+
73
+ def detect_agent() -> Optional[str]:
74
+ """Return the id of the detected AI agent harness or `None`.
75
+
76
+ Harnesses are checked in registry order; for each one we match its env var
77
+ pattern(s) and, failing that, the standard `AI_AGENT` / `AGENT` vars
78
+ against the harness id. The first match wins. When a standard var is set to
79
+ an unrecognized value, `"unknown"` is returned.
80
+ """
81
+ registry = _get_registry()
82
+ standard_vars = registry.get("standardEnvVars") or []
83
+ harnesses = registry.get("harnesses") or {}
84
+
85
+ for harness_id, info in harnesses.items():
86
+ env_vars = (info or {}).get("envVars")
87
+ if env_vars and _env_vars_match(env_vars):
88
+ return harness_id
89
+ for var in standard_vars:
90
+ if os.environ.get(var, "").strip() == harness_id:
91
+ return harness_id
92
+
93
+ # No harness matched but a standard var is set => unrecognized agent.
94
+ lowercased_harnesses = {k.lower() for k in harnesses.keys()}
95
+ for var in standard_vars:
96
+ if value := os.environ.get(var, "").strip().lower():
97
+ if value in lowercased_harnesses:
98
+ return value
99
+ return "unknown"
100
+
101
+ return None
102
+
103
+
104
+ def is_agent() -> bool:
105
+ """Return `True` if the process is being invoked by an AI coding agent."""
106
+ return detect_agent() is not None
107
+
108
+
109
+ def _env_vars_match(env_vars: dict[str, str]) -> bool:
110
+ """Return `True` if any `(var, pattern)` from the harness matches the environment.
111
+
112
+ Supported patterns:
113
+ - `"*"`: the variable is set to any non-empty value
114
+ - `"<value>"`: the variable equals this exact value
115
+ """
116
+ for var, pattern in env_vars.items():
117
+ value = os.environ.get(var)
118
+ if not value:
119
+ continue
120
+ if pattern == "*":
121
+ return True
122
+ if value == pattern:
123
+ return True
124
+ return False
125
+
126
+
127
+ def _get_registry() -> Registry:
128
+ """Return the harness registry, loading (and caching in-process) on first call.
129
+
130
+ Best-effort: any unexpected error degrades to an empty registry so detection
131
+ never raises.
132
+ """
133
+ global _registry
134
+ if _registry is None:
135
+ try:
136
+ _registry = _load_registry()
137
+ except Exception:
138
+ logger.debug("Could not resolve agent harnesses registry.", exc_info=True)
139
+ _registry = _EMPTY_REGISTRY
140
+ return _registry
141
+
142
+
143
+ def _load_registry() -> Registry:
144
+ """Resolve the registry from the local cache or the Hub.
145
+
146
+ No hardcoded list: if the Hub is unreachable and no cached copy exists, an
147
+ empty registry is returned (i.e. no agent is detected).
148
+ """
149
+ path = constants.AGENT_HARNESSES_PATH
150
+
151
+ # 1. Use the cached file if it was refreshed within the last 24 hours.
152
+ if cached := _read_cached_registry(path, max_age=_REGISTRY_TTL_SECONDS):
153
+ return cached
154
+
155
+ # 2. Otherwise refresh it from the Hub and persist it for next time.
156
+ if (fetched := _fetch_registry()) is not None:
157
+ _write_cached_registry(path, fetched)
158
+ return fetched
159
+
160
+ # 3. Fetch failed: reuse a stale cache if available, else give up (no detection).
161
+ if stale := _read_cached_registry(path, max_age=None):
162
+ return stale
163
+ return _EMPTY_REGISTRY
164
+
165
+
166
+ def _read_cached_registry(path: str, max_age: int | None) -> Registry | None:
167
+ """Return the cached registry, or `None` if missing/stale/unreadable."""
168
+ try:
169
+ if not os.path.exists(path):
170
+ return None
171
+ if max_age is not None and (time.time() - os.path.getmtime(path)) >= max_age:
172
+ return None
173
+ with open(path, encoding="utf-8") as f:
174
+ return json.load(f)
175
+ except Exception:
176
+ logger.debug("Could not read cached agent harnesses registry.", exc_info=True)
177
+ return None
178
+
179
+
180
+ def _write_cached_registry(path: str, registry: Registry) -> None:
181
+ try:
182
+ Path(path).parent.mkdir(parents=True, exist_ok=True)
183
+ with open(path, "w", encoding="utf-8") as f:
184
+ json.dump(registry, f)
185
+ except Exception:
186
+ logger.debug("Could not cache agent harnesses registry.", exc_info=True)
187
+
188
+
189
+ def _fetch_registry() -> Registry | None:
190
+ """Fetch the registry from the Hub. Returns `None` when offline or on any error."""
191
+ if constants.HF_HUB_OFFLINE:
192
+ return None
193
+ try:
194
+ from ._http import get_session
195
+
196
+ response = get_session().get(
197
+ f"{constants.ENDPOINT}/api/agent-harnesses",
198
+ timeout=_REGISTRY_FETCH_TIMEOUT,
199
+ )
200
+ response.raise_for_status()
201
+ return response.json()
202
+ except Exception:
203
+ logger.debug("Could not fetch agent harnesses registry from the Hub.", exc_info=True)
204
+ return None
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_dotenv.py ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # AI-generated module (ChatGPT)
2
+ import re
3
+
4
+
5
+ # Escape sequences expanded inside quoted values. Double-quoted values additionally
6
+ # expand "\$" to "$"; single-quoted values keep it verbatim.
7
+ _ESCAPES = {"n": "\n", "t": "\t", '"': '"', "\\": "\\"}
8
+ _DOUBLE_QUOTE_ESCAPES = {**_ESCAPES, "$": "$"}
9
+
10
+
11
+ def _unescape(value: str, escapes: dict[str, str]) -> str:
12
+ r"""Expand backslash escapes in a single left-to-right pass.
13
+
14
+ Processing in one pass (rather than chained `str.replace` calls) ensures an escaped
15
+ backslash (`\\`) is consumed as a unit and cannot merge with the following character,
16
+ e.g. `\\n` is a backslash followed by `n`, not a newline. Unknown escapes are kept as-is.
17
+ """
18
+ return re.sub(r"\\(.)", lambda match: escapes.get(match.group(1), match.group(0)), value)
19
+
20
+
21
+ def load_dotenv(dotenv_str: str, environ: dict[str, str] | None = None) -> dict[str, str]:
22
+ """
23
+ Parse a DOTENV-format string and return a dictionary of key-value pairs.
24
+ Handles quoted values, comments, export keyword, and blank lines.
25
+ """
26
+ env: dict[str, str] = {}
27
+ line_pattern = re.compile(
28
+ r"""
29
+ ^\s*
30
+ (?:export[^\S\n]+)? # optional export
31
+ ([A-Za-z_][A-Za-z0-9_]*) # key
32
+ [^\S\n]*(=)?[^\S\n]*
33
+ ( # value group
34
+ (?:
35
+ '(?:\\'|[^'])*' # single-quoted value
36
+ | \"(?:\\\"|[^\"])*\" # double-quoted value
37
+ | [^#\n\r]+? # unquoted value
38
+ )
39
+ )?
40
+ [^\S\n]*(?:\#.*)?$ # optional inline comment
41
+ """,
42
+ re.VERBOSE,
43
+ )
44
+
45
+ for line in dotenv_str.splitlines():
46
+ line = line.strip()
47
+ if not line or line.startswith("#"):
48
+ continue # Skip comments and empty lines
49
+
50
+ match = line_pattern.match(line)
51
+ if match:
52
+ key = match.group(1)
53
+ val = None
54
+ if match.group(2): # if there is '='
55
+ raw_val = match.group(3) or ""
56
+ val = raw_val.strip()
57
+ # Remove surrounding quotes if quoted
58
+ if (val.startswith('"') and val.endswith('"')) or (val.startswith("'") and val.endswith("'")):
59
+ escapes = _DOUBLE_QUOTE_ESCAPES if raw_val.startswith('"') else _ESCAPES
60
+ val = _unescape(val[1:-1], escapes)
61
+ elif environ is not None:
62
+ # Get it from the current environment
63
+ val = environ.get(key)
64
+
65
+ if val is not None:
66
+ env[key] = val
67
+
68
+ return env
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_experimental.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2023-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains utilities to flag a feature as "experimental" in Huggingface Hub."""
15
+
16
+ import warnings
17
+ from collections.abc import Callable
18
+ from functools import wraps
19
+
20
+ from .. import constants
21
+
22
+
23
+ def experimental(fn: Callable) -> Callable:
24
+ """Decorator to flag a feature as experimental.
25
+
26
+ An experimental feature triggers a warning when used as it might be subject to breaking changes without prior notice
27
+ in the future.
28
+
29
+ Warnings can be disabled by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment variable.
30
+
31
+ Args:
32
+ fn (`Callable`):
33
+ The function to flag as experimental.
34
+
35
+ Returns:
36
+ `Callable`: The decorated function.
37
+
38
+ Example:
39
+
40
+ ```python
41
+ >>> from huggingface_hub.utils import experimental
42
+
43
+ >>> @experimental
44
+ ... def my_function():
45
+ ... print("Hello world!")
46
+
47
+ >>> my_function()
48
+ UserWarning: 'my_function' is experimental and might be subject to breaking changes in the future without prior
49
+ notice. You can disable this warning by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment variable.
50
+ Hello world!
51
+ ```
52
+ """
53
+ # For classes, put the "experimental" around the "__new__" method => __new__ will be removed in warning message
54
+ name = fn.__qualname__[: -len(".__new__")] if fn.__qualname__.endswith(".__new__") else fn.__qualname__
55
+
56
+ @wraps(fn)
57
+ def _inner_fn(*args, **kwargs):
58
+ if not constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING:
59
+ warnings.warn(
60
+ f"'{name}' is experimental and might be subject to breaking changes in the future without prior notice."
61
+ " You can disable this warning by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment"
62
+ " variable.",
63
+ UserWarning,
64
+ )
65
+ return fn(*args, **kwargs)
66
+
67
+ return _inner_fn
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_fixes.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import contextlib
2
+ import os
3
+ import shutil
4
+ import stat
5
+ import tempfile
6
+ import time
7
+ from collections.abc import Callable, Generator
8
+ from functools import partial
9
+ from pathlib import Path
10
+
11
+ import yaml
12
+ from filelock import BaseFileLock, FileLock, SoftFileLock, Timeout
13
+
14
+ from .. import constants
15
+ from . import logging
16
+
17
+
18
+ logger = logging.get_logger(__name__)
19
+
20
+ # Wrap `yaml.dump` to set `allow_unicode=True` by default.
21
+ #
22
+ # Example:
23
+ # ```py
24
+ # >>> yaml.dump({"emoji": "👀", "some unicode": "日本か"})
25
+ # 'emoji: "\\U0001F440"\nsome unicode: "\\u65E5\\u672C\\u304B"\n'
26
+ #
27
+ # >>> yaml_dump({"emoji": "👀", "some unicode": "日本か"})
28
+ # 'emoji: "👀"\nsome unicode: "日本か"\n'
29
+ # ```
30
+ yaml_dump: Callable[..., str] = partial(yaml.dump, stream=None, allow_unicode=True) # type: ignore
31
+
32
+
33
+ @contextlib.contextmanager
34
+ def SoftTemporaryDirectory(
35
+ suffix: str | None = None,
36
+ prefix: str | None = None,
37
+ dir: Path | str | None = None,
38
+ **kwargs,
39
+ ) -> Generator[Path, None, None]:
40
+ """
41
+ Context manager to create a temporary directory and safely delete it.
42
+
43
+ If tmp directory cannot be deleted normally, we set the WRITE permission and retry.
44
+ If cleanup still fails, we give up but don't raise an exception. This is equivalent
45
+ to `tempfile.TemporaryDirectory(..., ignore_cleanup_errors=True)` introduced in
46
+ Python 3.10.
47
+
48
+ See https://www.scivision.dev/python-tempfile-permission-error-windows/.
49
+ """
50
+ tmpdir = tempfile.TemporaryDirectory(prefix=prefix, suffix=suffix, dir=dir, **kwargs)
51
+ yield Path(tmpdir.name).resolve()
52
+
53
+ try:
54
+ # First once with normal cleanup
55
+ shutil.rmtree(tmpdir.name)
56
+ except Exception:
57
+ # If failed, try to set write permission and retry
58
+ try:
59
+ shutil.rmtree(tmpdir.name, onerror=_set_write_permission_and_retry)
60
+ except Exception:
61
+ pass
62
+
63
+ # And finally, cleanup the tmpdir.
64
+ # If it fails again, give up but do not throw error
65
+ try:
66
+ tmpdir.cleanup()
67
+ except Exception:
68
+ pass
69
+
70
+
71
+ def _set_write_permission_and_retry(func, path, excinfo):
72
+ os.chmod(path, stat.S_IWRITE)
73
+ func(path)
74
+
75
+
76
+ @contextlib.contextmanager
77
+ def WeakFileLock(lock_file: str | Path, *, timeout: float | None = None) -> Generator[BaseFileLock, None, None]:
78
+ """A filelock with some custom logic.
79
+
80
+ This filelock is weaker than the default filelock in that:
81
+ 1. It won't raise an exception if release fails.
82
+ 2. It will default to a SoftFileLock if the filesystem does not support flock.
83
+ 3. Lock files are created with mode 0o664 (group-writable) instead of the default 0o644.
84
+ This allows multiple users sharing a cache directory to wait for locks.
85
+
86
+ An INFO log message is emitted every 10 seconds if the lock is not acquired immediately.
87
+ If a timeout is provided, a `filelock.Timeout` exception is raised if the lock is not acquired within the timeout.
88
+ """
89
+ log_interval = constants.FILELOCK_LOG_EVERY_SECONDS
90
+ lock = FileLock(lock_file, timeout=log_interval, mode=0o664)
91
+ start_time = time.time()
92
+
93
+ while True:
94
+ elapsed_time = time.time() - start_time
95
+ if timeout is not None and elapsed_time >= timeout:
96
+ raise Timeout(str(lock_file))
97
+
98
+ try:
99
+ lock.acquire(timeout=min(log_interval, timeout - elapsed_time) if timeout else log_interval)
100
+ except Timeout:
101
+ logger.info(
102
+ f"Still waiting to acquire lock on {lock_file} (elapsed: {time.time() - start_time:.1f} seconds)"
103
+ )
104
+ except NotImplementedError as e:
105
+ if "use SoftFileLock instead" in str(e):
106
+ logger.warning(
107
+ "FileSystem does not appear to support flock. Falling back to SoftFileLock for %s", lock_file
108
+ )
109
+ lock = SoftFileLock(lock_file, timeout=log_interval)
110
+ continue
111
+ else:
112
+ break
113
+
114
+ try:
115
+ yield lock
116
+ finally:
117
+ try:
118
+ lock.release()
119
+ except OSError:
120
+ try:
121
+ Path(lock_file).unlink()
122
+ except OSError:
123
+ pass
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_git_credential.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains utilities to manage Git credentials."""
15
+
16
+ import re
17
+ import subprocess
18
+
19
+ from ..constants import ENDPOINT
20
+ from ._subprocess import run_interactive_subprocess, run_subprocess
21
+
22
+
23
+ GIT_CREDENTIAL_REGEX = re.compile(
24
+ r"""
25
+ ^\s* # start of line
26
+ credential\.helper # credential.helper value
27
+ \s*=\s* # separator
28
+ ([\w\-\/]+) # the helper name or absolute path (group 1)
29
+ (\s|$) # whitespace or end of line
30
+ """,
31
+ flags=re.MULTILINE | re.IGNORECASE | re.VERBOSE,
32
+ )
33
+
34
+
35
+ def list_credential_helpers(folder: str | None = None) -> list[str]:
36
+ """Return the list of git credential helpers configured.
37
+
38
+ See https://git-scm.com/docs/gitcredentials.
39
+
40
+ Credentials are saved in all configured helpers (store, cache, macOS keychain,...).
41
+ Calls "`git credential approve`" internally. See https://git-scm.com/docs/git-credential.
42
+
43
+ Args:
44
+ folder (`str`, *optional*):
45
+ The folder in which to check the configured helpers.
46
+ """
47
+ try:
48
+ output = run_subprocess("git config --list", folder=folder).stdout
49
+ parsed = _parse_credential_output(output)
50
+ return parsed
51
+ except subprocess.CalledProcessError as exc:
52
+ raise OSError(exc.stderr)
53
+
54
+
55
+ def set_git_credential(token: str, username: str = "hf_user", folder: str | None = None) -> None:
56
+ """Save a username/token pair in git credential for HF Hub registry.
57
+
58
+ Credentials are saved in all configured helpers (store, cache, macOS keychain,...).
59
+ Calls "`git credential approve`" internally. See https://git-scm.com/docs/git-credential.
60
+
61
+ Args:
62
+ username (`str`, defaults to `"hf_user"`):
63
+ A git username. Defaults to `"hf_user"`, the default user used in the Hub.
64
+ token (`str`, defaults to `"hf_user"`):
65
+ A git password. In practice, the User Access Token for the Hub.
66
+ See https://huggingface.co/settings/tokens.
67
+ folder (`str`, *optional*):
68
+ The folder in which to check the configured helpers.
69
+ """
70
+ with run_interactive_subprocess("git credential approve", folder=folder) as (
71
+ stdin,
72
+ _,
73
+ ):
74
+ stdin.write(f"url={ENDPOINT}\nusername={username.lower()}\npassword={token}\n\n")
75
+ stdin.flush()
76
+
77
+
78
+ def unset_git_credential(username: str = "hf_user", folder: str | None = None) -> None:
79
+ """Erase credentials from git credential for HF Hub registry.
80
+
81
+ Credentials are erased from the configured helpers (store, cache, macOS
82
+ keychain,...), if any. If `username` is not provided, any credential configured for
83
+ HF Hub endpoint is erased.
84
+ Calls "`git credential erase`" internally. See https://git-scm.com/docs/git-credential.
85
+
86
+ Args:
87
+ username (`str`, defaults to `"hf_user"`):
88
+ A git username. Defaults to `"hf_user"`, the default user used in the Hub.
89
+ folder (`str`, *optional*):
90
+ The folder in which to check the configured helpers.
91
+ """
92
+ with run_interactive_subprocess("git credential reject", folder=folder) as (
93
+ stdin,
94
+ _,
95
+ ):
96
+ standard_input = f"url={ENDPOINT}\n"
97
+ if username is not None:
98
+ standard_input += f"username={username.lower()}\n"
99
+ standard_input += "\n"
100
+
101
+ stdin.write(standard_input)
102
+ stdin.flush()
103
+
104
+
105
+ def _parse_credential_output(output: str) -> list[str]:
106
+ """Parse the output of `git credential fill` to extract the password.
107
+
108
+ Args:
109
+ output (`str`):
110
+ The output of `git credential fill`.
111
+ """
112
+ # NOTE: If user has set a helper for a custom URL, it will not be caught here.
113
+ # Example: `credential.https://huggingface.co.helper=store`
114
+ # See: https://github.com/huggingface/huggingface_hub/pull/1138#discussion_r1013324508
115
+ return sorted( # Sort for nice printing
116
+ { # Might have some duplicates
117
+ match[0] for match in GIT_CREDENTIAL_REGEX.findall(output)
118
+ }
119
+ )
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_headers.py ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains utilities to handle headers to send in calls to Huggingface Hub."""
15
+
16
+ from huggingface_hub.errors import LocalTokenNotFoundError
17
+
18
+ from .. import constants
19
+ from ._auth import get_token
20
+ from ._detect_agent import detect_agent
21
+ from ._runtime import (
22
+ get_hf_hub_version,
23
+ get_python_version,
24
+ get_torch_version,
25
+ is_torch_available,
26
+ )
27
+ from ._validators import validate_hf_hub_args
28
+
29
+
30
+ @validate_hf_hub_args
31
+ def build_hf_headers(
32
+ *,
33
+ token: bool | str | None = None,
34
+ library_name: str | None = None,
35
+ library_version: str | None = None,
36
+ user_agent: dict | str | None = None,
37
+ headers: dict[str, str] | None = None,
38
+ ) -> dict[str, str]:
39
+ """
40
+ Build headers dictionary to send in a HF Hub call.
41
+
42
+ By default, authorization token is always provided either from argument (explicit
43
+ use) or retrieved from the cache (implicit use). To explicitly avoid sending the
44
+ token to the Hub, set `token=False` or set the `HF_HUB_DISABLE_IMPLICIT_TOKEN`
45
+ environment variable.
46
+
47
+ In case of an API call that requires write access, an error is thrown if token is
48
+ `None` or token is an organization token (starting with `"api_org***"`).
49
+
50
+ In addition to the auth header, a user-agent is added to provide information about
51
+ the installed packages (versions of python, huggingface_hub, torch).
52
+
53
+ Args:
54
+ token (`str`, `bool`, *optional*):
55
+ The token to be sent in authorization header for the Hub call:
56
+ - if a string, it is used as the Hugging Face token
57
+ - if `True`, the token is read from the machine (cache or env variable)
58
+ - if `False`, authorization header is not set
59
+ - if `None`, the token is read from the machine only except if
60
+ `HF_HUB_DISABLE_IMPLICIT_TOKEN` env variable is set.
61
+ library_name (`str`, *optional*):
62
+ The name of the library that is making the HTTP request. Will be added to
63
+ the user-agent header.
64
+ library_version (`str`, *optional*):
65
+ The version of the library that is making the HTTP request. Will be added
66
+ to the user-agent header.
67
+ user_agent (`str`, `dict`, *optional*):
68
+ The user agent info in the form of a dictionary or a single string. It will
69
+ be completed with information about the installed packages.
70
+ headers (`dict`, *optional*):
71
+ Additional headers to include in the request. Those headers take precedence
72
+ over the ones generated by this function.
73
+
74
+ Returns:
75
+ A `dict` of headers to pass in your API call.
76
+
77
+ Example:
78
+ ```py
79
+ >>> build_hf_headers(token="hf_***") # explicit token
80
+ {"authorization": "Bearer hf_***", "user-agent": ""}
81
+
82
+ >>> build_hf_headers(token=True) # explicitly use cached token
83
+ {"authorization": "Bearer hf_***",...}
84
+
85
+ >>> build_hf_headers(token=False) # explicitly don't use cached token
86
+ {"user-agent": ...}
87
+
88
+ >>> build_hf_headers() # implicit use of the cached token
89
+ {"authorization": "Bearer hf_***",...}
90
+
91
+ # HF_HUB_DISABLE_IMPLICIT_TOKEN=True # to set as env variable
92
+ >>> build_hf_headers() # token is not sent
93
+ {"user-agent": ...}
94
+
95
+ >>> build_hf_headers(library_name="transformers", library_version="1.2.3")
96
+ {"authorization": ..., "user-agent": "transformers/1.2.3; hf_hub/0.10.2; python/3.10.4; tensorflow/1.55"}
97
+ ```
98
+
99
+ Raises:
100
+ [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
101
+ If organization token is passed and "write" access is required.
102
+ [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
103
+ If "write" access is required but token is not passed and not saved locally.
104
+ [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
105
+ If `token=True` but token is not saved locally.
106
+ """
107
+ # Get auth token to send
108
+ token_to_send = get_token_to_send(token)
109
+
110
+ # Combine headers
111
+ hf_headers = {
112
+ "user-agent": _http_user_agent(
113
+ library_name=library_name,
114
+ library_version=library_version,
115
+ user_agent=user_agent,
116
+ )
117
+ }
118
+ if token_to_send is not None:
119
+ hf_headers["authorization"] = f"Bearer {token_to_send}"
120
+ if headers is not None:
121
+ hf_headers.update(headers)
122
+ return hf_headers
123
+
124
+
125
+ def get_token_to_send(token: bool | str | None) -> str | None:
126
+ """Select the token to send from either `token` or the cache."""
127
+ # Case token is explicitly provided
128
+ if isinstance(token, str):
129
+ return token
130
+
131
+ # Case token is explicitly forbidden
132
+ if token is False:
133
+ return None
134
+
135
+ # Case token is explicitly required
136
+ if token is True:
137
+ cached_token = get_token()
138
+ if cached_token is None:
139
+ raise LocalTokenNotFoundError(
140
+ "Token is required (`token=True`), but no token found. You"
141
+ " need to provide a token or be logged in to Hugging Face with"
142
+ " `hf auth login` or `huggingface_hub.login`. See"
143
+ " https://huggingface.co/settings/tokens."
144
+ )
145
+ return cached_token
146
+
147
+ # Case implicit use of the token is forbidden by env variable. Checked before resolving the
148
+ # cached token: `get_token()` may refresh an OAuth token (network call + file writes), which
149
+ # must not happen when the resolved token wouldn't be used anyway.
150
+ if constants.HF_HUB_DISABLE_IMPLICIT_TOKEN:
151
+ return None
152
+
153
+ # Otherwise: we use the cached token as the user has not explicitly forbidden it
154
+ return get_token()
155
+
156
+
157
+ def _http_user_agent(
158
+ *,
159
+ library_name: str | None = None,
160
+ library_version: str | None = None,
161
+ user_agent: dict | str | None = None,
162
+ ) -> str:
163
+ """Format a user-agent string containing information about the installed packages.
164
+
165
+ Args:
166
+ library_name (`str`, *optional*):
167
+ The name of the library that is making the HTTP request.
168
+ library_version (`str`, *optional*):
169
+ The version of the library that is making the HTTP request.
170
+ user_agent (`str`, `dict`, *optional*):
171
+ The user agent info in the form of a dictionary or a single string.
172
+
173
+ Returns:
174
+ The formatted user-agent string.
175
+ """
176
+ if library_name is not None:
177
+ ua = f"{library_name}/{library_version}"
178
+ else:
179
+ ua = "unknown/None"
180
+ ua += f"; hf_hub/{get_hf_hub_version()}"
181
+ ua += f"; python/{get_python_version()}"
182
+
183
+ if not constants.HF_HUB_DISABLE_TELEMETRY:
184
+ if is_torch_available():
185
+ ua += f"; torch/{get_torch_version()}"
186
+
187
+ agent = detect_agent()
188
+ if agent:
189
+ ua += f"; agent/{agent}"
190
+
191
+ if isinstance(user_agent, dict):
192
+ ua += "; " + "; ".join(f"{k}/{v}" for k, v in user_agent.items())
193
+ elif isinstance(user_agent, str):
194
+ ua += "; " + user_agent
195
+
196
+ # Retrieve user-agent origin headers from environment variable
197
+ origin = constants.HF_HUB_USER_AGENT_ORIGIN
198
+ if origin is not None:
199
+ ua += "; origin/" + origin
200
+
201
+ return _deduplicate_user_agent(ua)
202
+
203
+
204
+ def _deduplicate_user_agent(user_agent: str) -> str:
205
+ """Deduplicate redundant information in the generated user-agent."""
206
+ # Split around ";" > Strip whitespaces > Store as dict keys (ensure unicity) > format back as string
207
+ # Order is implicitly preserved by dictionary structure (see https://stackoverflow.com/a/53657523).
208
+ return "; ".join({key.strip(): None for key in user_agent.split(";")}.keys())
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_hf_uris.py ADDED
@@ -0,0 +1,642 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Centralized parser for Hugging Face Hub URIs ('hf://...') and mount specifications.
15
+
16
+ A HF URI is a URI-like string that identifies a location on the Hugging Face
17
+ Hub: a model/dataset/space/kernel repository, a bucket, optionally a revision,
18
+ and optionally a path inside the repo or bucket.
19
+
20
+ Canonical syntax:
21
+
22
+ ```
23
+ hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]
24
+ ```
25
+
26
+ For convenience, [`parse_hf_uri`] also accepts Hugging Face **web URLs** (the
27
+ ones you copy-paste from your browser), e.g.
28
+ 'https://huggingface.co/datasets/my-org/my-dataset/blob/main/train.csv'. They are
29
+ normalized to the canonical 'hf://' form before parsing. Only unambiguous URLs
30
+ (repository / bucket pages and file/folder viewer routes) are accepted; any other
31
+ route is rejected rather than guessed.
32
+
33
+ A HF mount wraps a HF URI with a local mount path and an optional ':ro'/':rw'
34
+ flag (used by Spaces and Jobs volumes):
35
+
36
+ ```
37
+ hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]:<MOUNT_PATH>[:ro|:rw]
38
+ ```
39
+
40
+ See 'docs/source/en/package_reference/hf_uris.md' for the full grammar and examples.
41
+ """
42
+
43
+ import functools
44
+ import re
45
+ from dataclasses import dataclass, field
46
+ from urllib.parse import quote, unquote, urlsplit
47
+
48
+ from huggingface_hub import constants
49
+ from huggingface_hub.errors import HfUriError, HFValidationError
50
+
51
+ from ._validators import validate_repo_id
52
+
53
+
54
+ # Inverse map (singular -> plural URI prefix). Built once from the canonical
55
+ # 'constants.HF_URI_TYPE_PREFIXES' and used to render URIs.
56
+ _TYPE_TO_PREFIX: dict[str, str] = {v: k for k, v in constants.HF_URI_TYPE_PREFIXES.items()}
57
+
58
+ # Special revisions that contain a '/'. They take precedence when splitting
59
+ # the part after '@' into '<revision>/<path-in-repo>'. Matches 'refs/pr/N'
60
+ # (Pull Request refs) and 'refs/convert/<name>' (e.g. parquet conversions).
61
+ # The conversion name allows the typical git ref characters '[a-zA-Z0-9_.-]'
62
+ # so names like 'parquet-v2' or 'duckdb.v1' round-trip correctly.
63
+ _SPECIAL_REFS_REVISION_REGEX = re.compile(r"^refs/(?:convert/[\w.-]+|pr/\d+)")
64
+
65
+ # Same as constants.HfUriType, but as a set of strings for easy lookup.)
66
+ _VALID_URI_TYPES: frozenset[str] = frozenset(constants.HF_URI_TYPE_PREFIXES.values())
67
+
68
+
69
+ # Web-viewer routes that point at a file or folder and that map cleanly onto a
70
+ # '<revision>/<path>' pair. Other routes (commit, commits, discussions, settings,
71
+ # edit, ...) do not identify a Hub location and are rejected by the URL parser.
72
+ _URL_REPO_LOCATION_ACTIONS: frozenset[str] = frozenset({"blame", "blob", "raw", "resolve", "tree"})
73
+ # Bucket web routes that point at a file or folder. Buckets are not versioned, so
74
+ # these are followed directly by '<path>' (no revision segment).
75
+ _URL_BUCKET_LOCATION_ACTIONS: frozenset[str] = frozenset({"resolve", "tree"})
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class HfUri:
80
+ """Parsed representation of a Hugging Face Hub URI ('hf://...').
81
+
82
+ Attributes:
83
+ type (`str`):
84
+ One of 'model', 'dataset', 'space', 'kernel' or 'bucket'.
85
+ id (`str`):
86
+ The repository id ('namespace/name', e.g. 'my-org/my-model') for repo URIs, or the bucket id ('namespace/name') for bucket URIs.
87
+ revision (`str`, *optional*):
88
+ The revision specified after '@' in the URI, URL-decoded. 'None' if no revision was specified, or for bucket URIs (which
89
+ never carry a revision). Special refs like 'refs/pr/10' and 'refs/convert/parquet' are preserved as-is.
90
+ path_in_repo (`str`):
91
+ The path inside the repo or bucket. Empty string if the URI points at the root.
92
+ """
93
+
94
+ type: constants.HfUriType
95
+ id: str
96
+ revision: str | None = None
97
+ path_in_repo: str = ""
98
+ _raw: str | None = field(repr=False, hash=False, compare=False, default=None)
99
+
100
+ def __post_init__(self) -> None:
101
+ uri = self._raw or "" # For error messages
102
+
103
+ # Check valid URI type
104
+ if self.type not in _VALID_URI_TYPES:
105
+ raise HfUriError(uri=uri, msg=f"Invalid type '{self.type}'. Must be one of {sorted(_VALID_URI_TYPES)}.")
106
+
107
+ # Check valid ID
108
+ if not self.id or self.id.count("/") != 1:
109
+ raise HfUriError(uri=uri, msg=f"Id must be 'namespace/name', got '{self.id}'.")
110
+ if self.type != "bucket":
111
+ try:
112
+ validate_repo_id(self.id)
113
+ except HFValidationError as e:
114
+ raise HfUriError(uri=uri, msg=str(e)) from e
115
+
116
+ # Check valid revision
117
+ if self.revision is not None and not self.revision:
118
+ raise HfUriError(uri=uri, msg="Revision must not be an empty string.")
119
+ if self.type == "bucket" and self.revision is not None:
120
+ raise HfUriError(uri=uri, msg="Bucket URIs do not support a revision.")
121
+
122
+ # Check valid path in repo
123
+ if self.path_in_repo:
124
+ if self.path_in_repo.startswith("/") or "//" in self.path_in_repo:
125
+ raise HfUriError(uri=uri, msg=f"Path must not contain empty segments (got '{self.path_in_repo}').")
126
+
127
+ @property
128
+ def is_bucket(self) -> bool:
129
+ """True if this URI points at a bucket."""
130
+ return self.type == "bucket"
131
+
132
+ @property
133
+ def is_repo(self) -> bool:
134
+ """True if this URI points at a repository (model, dataset, space or kernel)."""
135
+ return self.type != "bucket"
136
+
137
+ def to_uri(self) -> str:
138
+ """Render the URI as a canonical 'hf://' string.
139
+
140
+ The type prefix is always written explicitly (e.g. 'hf://models/my-org/my-model').
141
+ """
142
+ parts: list[str] = [constants.HF_PROTOCOL, _TYPE_TO_PREFIX[self.type], "/", self.id]
143
+ if self.revision is not None:
144
+ # Encode '/' as '%2F' for revisions that would otherwise be split as '<revision>/<path>'
145
+ # at parse time. Special refs ('refs/pr/N', 'refs/convert/<name>') are kept verbatim
146
+ # because the parser matches them eagerly.
147
+ revision = self.revision
148
+ if "/" in revision and _SPECIAL_REFS_REVISION_REGEX.fullmatch(revision) is None:
149
+ revision = revision.replace("/", "%2F")
150
+ parts.append(f"@{revision}")
151
+ if self.path_in_repo:
152
+ parts.append(f"/{self.path_in_repo}")
153
+ return "".join(parts)
154
+
155
+ def to_url(self, endpoint: str | None = None) -> str:
156
+ """Render the URI as a Hugging Face **web URL** (the kind you open in a browser).
157
+
158
+ This is the inverse of parsing a URL with [`parse_hf_uri`]. The returned URL points at:
159
+
160
+ - the repository / bucket landing page when no path or revision is set;
161
+ - the folder viewer ('/tree/<revision>') when only a revision is set;
162
+ - the file viewer ('/blob/<revision>/<path>') for repository files (revision defaults to 'main');
163
+ - the tree route ('/tree/<path>') for bucket files (buckets are not versioned).
164
+
165
+ Args:
166
+ endpoint (`str`, *optional*):
167
+ Base endpoint to use. Defaults to 'constants.ENDPOINT' (i.e. 'https://huggingface.co').
168
+
169
+ Returns:
170
+ `str`: the web URL.
171
+
172
+ Example:
173
+ ```py
174
+ >>> from huggingface_hub import parse_hf_uri
175
+ >>> parse_hf_uri("hf://datasets/my-org/my-dataset@v1/train.csv").to_url()
176
+ 'https://huggingface.co/datasets/my-org/my-dataset/blob/v1/train.csv'
177
+ ```
178
+ """
179
+ base = (endpoint or constants.ENDPOINT).rstrip("/")
180
+ # Percent-encode characters that would otherwise break the URL (spaces, '#', '?', ...),
181
+ # keeping '/' as the path separator. This is the inverse of the decoding done when parsing.
182
+ path = quote(self.path_in_repo, safe="/")
183
+
184
+ if self.type == "bucket":
185
+ url = f"{base}/buckets/{self.id}"
186
+ if path:
187
+ url += f"/tree/{path}"
188
+ return url
189
+
190
+ # Models live at the root ('hf.co/<id>'); other repos are namespaced by their plural prefix.
191
+ url = f"{base}/{self.id}" if self.type == "model" else f"{base}/{_TYPE_TO_PREFIX[self.type]}/{self.id}"
192
+ revision = self.revision
193
+ # Percent-encode the branch/tag name so it stays a single, URL-safe segment: a '/' would
194
+ # otherwise open a new path segment and '#'/'?' would be read as a fragment/query when the
195
+ # URL is opened or parsed back. This mirrors the decoding done when parsing. Special refs
196
+ # ('refs/pr/N', 'refs/convert/<name>') are used verbatim by the Hub web routes.
197
+ if revision is not None and _SPECIAL_REFS_REVISION_REGEX.fullmatch(revision) is None:
198
+ revision = quote(revision, safe="")
199
+ if path:
200
+ url += f"/blob/{revision or constants.DEFAULT_REVISION}/{path}"
201
+ elif revision is not None:
202
+ url += f"/tree/{revision}"
203
+ return url
204
+
205
+
206
+ @dataclass(frozen=True)
207
+ class HfMount:
208
+ """A HF URI paired with a local mount path and optional read-only flag.
209
+
210
+ Used by Spaces and Jobs to describe volume mounts. The full syntax is:
211
+
212
+ ```
213
+ hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]:<MOUNT_PATH>[:ro|:rw]
214
+ ```
215
+
216
+ Attributes:
217
+ source ([`HfUri`]):
218
+ The parsed HF URI identifying the Hub resource to mount.
219
+ mount_path (`str`):
220
+ The local mount path (always starts with '/').
221
+ read_only (`bool`, *optional*):
222
+ True if the mount ends with ':ro', False if it ends with ':rw', 'None' if no flag was provided.
223
+ """
224
+
225
+ source: HfUri
226
+ mount_path: str
227
+ read_only: bool | None = None
228
+ _raw: str | None = field(repr=False, hash=False, compare=False, default=None)
229
+
230
+ def __post_init__(self) -> None:
231
+ raw = self._raw or ""
232
+ if not self.mount_path.startswith("/") or self.mount_path == "/":
233
+ raise HfUriError(
234
+ uri=raw,
235
+ msg=f"Mount path must be a non-empty absolute path starting with '/', got '{self.mount_path}'.",
236
+ )
237
+
238
+ def to_uri(self) -> str:
239
+ """Render the mount as a canonical 'hf://' string.
240
+
241
+ Example: 'hf://models/my-org/my-model:/data:ro'
242
+ """
243
+ parts = [self.source.to_uri(), ":", self.mount_path]
244
+ if self.read_only is not None:
245
+ parts.append(":ro" if self.read_only else ":rw")
246
+ return "".join(parts)
247
+
248
+
249
+ def is_hf_uri(uri: str) -> bool:
250
+ """Check if a string is a valid HF URI ('hf://...') or a recognized Hugging Face web URL."""
251
+ try:
252
+ parse_hf_uri(uri)
253
+ return True
254
+ except HfUriError:
255
+ return False
256
+
257
+
258
+ @functools.lru_cache
259
+ def parse_hf_uri(uri: str, endpoint: str | None = None) -> HfUri:
260
+ """Parse a Hugging Face Hub URI ('hf://...') or a Hugging Face web URL.
261
+
262
+ A HF URI is a URI-like string identifying a location on the Hugging Face Hub. The full grammar is:
263
+
264
+ ```
265
+ hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]
266
+ ```
267
+
268
+ For convenience, Hugging Face **web URLs** (the ones you copy-paste from the website) are also
269
+ accepted and normalized to the canonical 'hf://' form, e.g.
270
+ 'https://huggingface.co/datasets/my-org/my-dataset/blob/main/train.csv'. Only unambiguous URLs
271
+ (repository / bucket pages and file/folder viewer routes) are accepted; any other route is rejected.
272
+
273
+ See 'docs/source/en/package_reference/hf_uris.md' for the full specification.
274
+
275
+ Args:
276
+ uri (`str`):
277
+ The URI to parse. Must start with 'hf://', or be a Hugging Face URL (e.g. 'https://huggingface.co/...').
278
+ endpoint (`str`, *optional*):
279
+ A custom Hub endpoint (e.g. a self-hosted or proxied Hub like 'https://hub.my-company.com' or
280
+ 'http://localhost:8080/hf'). When provided, web URLs on that endpoint are recognized in addition to
281
+ the default Hugging Face hosts. Has no effect on 'hf://' URIs.
282
+
283
+ Returns:
284
+ [`HfUri`]: the parsed URI.
285
+
286
+ Raises:
287
+ [`HfUriError`]:
288
+ If the URI is malformed (missing prefix, invalid type, missing id, unsupported URL route, etc.).
289
+
290
+ Examples:
291
+ ```py
292
+ >>> from huggingface_hub.utils import parse_hf_uri
293
+ >>> parse_hf_uri("hf://my-org/my-model")
294
+ HfUri(type='model', id='my-org/my-model', revision=None, path_in_repo='')
295
+ >>> parse_hf_uri("hf://datasets/my-org/my-dataset@refs/pr/3/train.json")
296
+ HfUri(type='dataset', id='my-org/my-dataset', revision='refs/pr/3', path_in_repo='train.json')
297
+ >>> parse_hf_uri("https://huggingface.co/datasets/my-org/my-dataset/blob/main/train.csv")
298
+ HfUri(type='dataset', id='my-org/my-dataset', revision='main', path_in_repo='train.csv')
299
+ ```
300
+ """
301
+ raw = uri
302
+ if uri.startswith(constants.HF_PROTOCOL):
303
+ body = uri[len(constants.HF_PROTOCOL) :]
304
+ if not body:
305
+ raise HfUriError(uri, f"Empty body after '{constants.HF_PROTOCOL}'.")
306
+ elif _looks_like_hf_url(uri, endpoint=endpoint):
307
+ body = _url_to_uri_body(uri, endpoint=endpoint)
308
+ else:
309
+ raise HfUriError(
310
+ uri,
311
+ f"Must start with '{constants.HF_PROTOCOL}' or be a Hugging Face URL (e.g. 'https://huggingface.co/...'). "
312
+ f"Expected format: {constants.HF_PROTOCOL}[<TYPE>/]<ID>[@<REVISION>][/<PATH>]",
313
+ )
314
+
315
+ type_, location = _split_type(body, raw=raw)
316
+
317
+ if type_ == "bucket":
318
+ return _parse_bucket_body(location, type_, raw=raw)
319
+ return _parse_repo_body(location, type_, raw=raw)
320
+
321
+
322
+ def _endpoint_host_and_path(endpoint: str | None) -> tuple[str | None, str]:
323
+ """Return the lowercased host and stripped path prefix of a custom Hub 'endpoint'.
324
+
325
+ E.g. 'https://hub.my-company.com' -> ('hub.my-company.com', '') and a self-hosted
326
+ 'http://localhost:8080/hf' -> ('localhost', 'hf'). Returns '(None, "")' when 'endpoint' is None.
327
+ """
328
+ if endpoint is None:
329
+ return None, ""
330
+ # Prefix '//' for scheme-less endpoints so 'urlsplit' populates 'netloc' instead of 'path'.
331
+ parsed = urlsplit(endpoint if "://" in endpoint else "//" + endpoint)
332
+ host = parsed.hostname.lower() if parsed.hostname else None
333
+ return host, parsed.path.strip("/")
334
+
335
+
336
+ def _recognized_hosts(endpoint: str | None) -> frozenset[str]:
337
+ """The set of hosts whose web URLs can be parsed: the default Hugging Face hosts plus 'endpoint'."""
338
+ host, _ = _endpoint_host_and_path(endpoint)
339
+ return constants.HF_URL_HOSTS | {host} if host else constants.HF_URL_HOSTS
340
+
341
+
342
+ def _looks_like_hf_url(uri: str, endpoint: str | None = None) -> bool:
343
+ """Return True if 'uri' looks like a (possibly scheme-less) Hugging Face web URL."""
344
+ lowered = uri.lower()
345
+ if lowered.startswith(("http://", "https://")):
346
+ return True
347
+ # Scheme-less host (e.g. 'huggingface.co/org/model').
348
+ return any(lowered == host or lowered.startswith(host + "/") for host in _recognized_hosts(endpoint))
349
+
350
+
351
+ def _decode_url_path_segment(segment: str) -> str:
352
+ """Percent-decode a single URL path segment (e.g. 'file%20name.txt' -> 'file name.txt').
353
+
354
+ A decoded '/' is re-encoded as '%2F' so the segment stays atomic when the normalized body is
355
+ re-split by the shared parser. This decodes ordinary path characters (spaces, '#', ...) that
356
+ browsers encode, while keeping '%2F'-encoded revisions (e.g. 'feature%2Ffoo') intact.
357
+ """
358
+ return unquote(segment).replace("/", "%2F")
359
+
360
+
361
+ def _url_to_uri_body(url: str, endpoint: str | None = None) -> str:
362
+ """Normalize a Hugging Face web URL into the body of a 'hf://' URI (everything after 'hf://').
363
+
364
+ The returned string is fed back into the regular URI parsing logic, so all validation
365
+ (repo id, revision, empty path segments, ...) is shared with the canonical 'hf://' path.
366
+ Only unambiguous URLs are accepted: any unrecognized route raises [`HfUriError`]. When 'endpoint'
367
+ is provided, URLs on that custom Hub host are recognized too (and its path prefix is stripped).
368
+ """
369
+ raw = url
370
+ # Prefix '//' for scheme-less inputs so 'urlsplit' populates 'netloc' instead of 'path'.
371
+ parsed = urlsplit(url if "://" in url else "//" + url)
372
+ host = (parsed.hostname or "").lower()
373
+ if host not in _recognized_hosts(endpoint):
374
+ raise HfUriError(
375
+ uri=raw,
376
+ msg=f"Unrecognized host '{host or url}'. Expected a Hugging Face URL (e.g. 'https://huggingface.co/...').",
377
+ )
378
+
379
+ # Query string and fragment are intentionally dropped (e.g. '?download=true').
380
+ path = parsed.path
381
+ # For a self-hosted endpoint with a path prefix (e.g. 'http://localhost:8080/hf'), drop it so the
382
+ # remaining segments are '[<TYPE>/]<namespace>/<name>[/...]' just like on the public Hub.
383
+ endpoint_host, endpoint_path = _endpoint_host_and_path(endpoint)
384
+ if endpoint_path and host == endpoint_host:
385
+ prefix = "/" + endpoint_path
386
+ if path == prefix or path.startswith(prefix + "/"):
387
+ path = path[len(prefix) :]
388
+ segments = [segment for segment in path.split("/") if segment]
389
+ if not segments:
390
+ raise HfUriError(uri=raw, msg=f"Missing repository or bucket identifier in URL '{url}'.")
391
+
392
+ # Optional type prefix ('datasets', 'spaces', 'kernels', 'buckets', 'models').
393
+ type_prefix: str | None = None
394
+ if segments[0] in constants.HF_URI_TYPE_PREFIXES:
395
+ type_prefix = segments[0]
396
+ segments = segments[1:]
397
+
398
+ # Everything in the web UI is namespaced ('<namespace>/<name>'); a single segment is a user or
399
+ # organization page (or a listing page), which we cannot map to a repository -> reject.
400
+ if len(segments) < 2:
401
+ raise HfUriError(
402
+ uri=raw,
403
+ msg=(
404
+ f"Cannot parse URL '{url}': expected a '<namespace>/<name>' repository or bucket. "
405
+ "User/organization pages and single-segment URLs are not supported."
406
+ ),
407
+ )
408
+ repo_id = f"{segments[0]}/{segments[1]}"
409
+ rest = segments[2:]
410
+
411
+ if type_prefix == "buckets":
412
+ if not rest:
413
+ return f"buckets/{repo_id}"
414
+ action, *tail = rest
415
+ if action not in _URL_BUCKET_LOCATION_ACTIONS:
416
+ raise HfUriError(uri=raw, msg=f"Cannot parse bucket URL '{url}': unsupported '/{action}/' route.")
417
+ path = "/".join(_decode_url_path_segment(segment) for segment in tail)
418
+ return f"buckets/{repo_id}/{path}" if path else f"buckets/{repo_id}"
419
+
420
+ prefix = f"{type_prefix}/" if type_prefix else ""
421
+ if not rest:
422
+ return f"{prefix}{repo_id}"
423
+ action, *tail = rest
424
+ if action not in _URL_REPO_LOCATION_ACTIONS:
425
+ raise HfUriError(
426
+ uri=raw,
427
+ msg=(
428
+ f"Cannot parse URL '{url}': unsupported '/{action}/' route. "
429
+ "Only repository pages and file/folder viewer routes (blob, resolve, raw, tree, ...) can be parsed."
430
+ ),
431
+ )
432
+ if not tail:
433
+ # e.g. '.../tree' with nothing after -> repository root.
434
+ return f"{prefix}{repo_id}"
435
+ # 'tail' is '<revision>/<path>'; reuse the canonical '@<revision>/<path>' splitting logic
436
+ # (special refs, URL-encoded slashes, ...) by handing it back to the URI parser. Each segment
437
+ # is percent-decoded first so file names with spaces, '#', ... resolve correctly; the revision
438
+ # segment's '%2F' survives (re-encoded by '_decode_url_path_segment') and is decoded downstream.
439
+ decoded = "/".join(_decode_url_path_segment(segment) for segment in tail)
440
+ return f"{prefix}{repo_id}@{decoded}"
441
+
442
+
443
+ def parse_hf_mount(mount_str: str) -> HfMount:
444
+ """Parse a HF mount specification ('hf://...:<MOUNT_PATH>[:ro|:rw]').
445
+
446
+ A mount specification is a HF URI followed by a local mount path and an optional read-only/read-write flag.
447
+ The full grammar is:
448
+
449
+ ```
450
+ hf://[<TYPE>/]<ID>[@<REVISION>][/<PATH>]:<MOUNT_PATH>[:ro|:rw]
451
+ ```
452
+
453
+ See 'docs/source/en/package_reference/hf_uris.md' for the full specification.
454
+
455
+ Args:
456
+ mount_str (`str`):
457
+ The mount string to parse. Must start with 'hf://' and contain a ':<MOUNT_PATH>' segment.
458
+
459
+ Returns:
460
+ [`HfMount`]: the parsed mount.
461
+
462
+ Raises:
463
+ [`HfUriError`]:
464
+ If the mount string is malformed (missing mount path, invalid URI, etc.).
465
+
466
+ Examples:
467
+ ```py
468
+ >>> from huggingface_hub.utils import parse_hf_mount
469
+ >>> parse_hf_mount("hf://my-org/my-model:/data:ro")
470
+ HfMount(source=HfUri(type='model', id='my-org/my-model', revision=None, path_in_repo=''), mount_path='/data', read_only=True)
471
+ >>> parse_hf_mount("hf://buckets/my-org/my-bucket/sub/dir:/mnt:rw")
472
+ HfMount(source=HfUri(type='bucket', id='my-org/my-bucket', revision=None, path_in_repo='sub/dir'), mount_path='/mnt', read_only=False)
473
+ ```
474
+ """
475
+ if not mount_str.startswith(constants.HF_PROTOCOL):
476
+ raise HfUriError(
477
+ uri=mount_str,
478
+ msg=f"Must start with '{constants.HF_PROTOCOL}'.",
479
+ )
480
+
481
+ raw = mount_str
482
+ body = mount_str[len(constants.HF_PROTOCOL) :]
483
+ if not body:
484
+ raise HfUriError(uri=raw, msg=f"Empty body after '{constants.HF_PROTOCOL}'.")
485
+
486
+ location, mount_path, read_only = _split_mount(body, raw=raw)
487
+
488
+ if mount_path is None:
489
+ raise HfUriError(uri=raw, msg="Missing mount path. Expected ':<MOUNT_PATH>' (e.g. 'hf://org/model:/data').")
490
+
491
+ # Re-assemble the URI part and parse it
492
+ uri_str = constants.HF_PROTOCOL + location
493
+ try:
494
+ source = parse_hf_uri(uri_str)
495
+ except HfUriError as e:
496
+ raise HfUriError(uri=raw, msg=e.msg) from e
497
+
498
+ return HfMount(source=source, mount_path=mount_path, read_only=read_only, _raw=raw)
499
+
500
+
501
+ def _split_mount(body: str, *, raw: str) -> tuple[str, str | None, bool | None]:
502
+ """Split the ':<MOUNT_PATH>[:ro|:rw]' suffix from 'body'.
503
+
504
+ Returns '(location, mount_path, read_only)' where 'mount_path' is 'None' if no mount segment is present.
505
+ """
506
+ if body.endswith(":ro"):
507
+ read_only, body = True, body.removesuffix(":ro")
508
+ elif body.endswith(":rw"):
509
+ read_only, body = False, body.removesuffix(":rw")
510
+ else:
511
+ read_only = None
512
+
513
+ # Mount paths always start with '/', so the delimiter is ':/'.
514
+ # We use rfind() because the mount segment is always trailing
515
+ idx = body.rfind(":/")
516
+ if idx == -1:
517
+ if read_only is not None:
518
+ raise HfUriError(
519
+ uri=raw,
520
+ msg="':ro'/':rw' suffix is only valid when a mount path is provided (e.g. 'hf://...:/<MOUNT_PATH>:ro').",
521
+ )
522
+ return body, None, None
523
+
524
+ location = body[:idx]
525
+ mount_path = body[idx + 1 :] # includes the leading '/'
526
+ if not location:
527
+ raise HfUriError(uri=raw, msg="Missing location before mount path.")
528
+ return location, mount_path, read_only
529
+
530
+
531
+ def _split_type(location: str, *, raw: str) -> tuple[constants.HfUriType, str]:
532
+ """Detect the (optional) type prefix and return '(type, remaining_location)'.
533
+
534
+ A missing type prefix defaults to 'model'. Singular forms ('model/', 'dataset/', etc.) are explicitly rejected with a helpful error.
535
+ """
536
+ slash_idx = location.find("/")
537
+ if slash_idx == -1:
538
+ # Single segment, no prefix. Reject if it looks like a bare type name.
539
+ if location in constants.HF_URI_TYPE_PREFIXES:
540
+ raise HfUriError(
541
+ uri=raw,
542
+ msg=f"Missing identifier after '{location}'. Expected '{constants.HF_PROTOCOL}{location}/<ID>'.",
543
+ )
544
+ if (singular_plural := _TYPE_TO_PREFIX.get(location)) is not None:
545
+ raise HfUriError(
546
+ uri=raw,
547
+ msg=f"Type prefix must be plural. Did you mean '{constants.HF_PROTOCOL}{singular_plural}/...'?",
548
+ )
549
+ return "model", location
550
+
551
+ first = location[:slash_idx]
552
+ rest = location[slash_idx + 1 :]
553
+ if first in constants.HF_URI_TYPE_PREFIXES:
554
+ return constants.HF_URI_TYPE_PREFIXES[first], rest
555
+ if (singular_plural := _TYPE_TO_PREFIX.get(first)) is not None:
556
+ raise HfUriError(
557
+ uri=raw, msg=f"Type prefix must be plural, got '{first}/'. Did you mean '{singular_plural}/'?"
558
+ )
559
+ return "model", location
560
+
561
+
562
+ def _parse_bucket_body(
563
+ location: str,
564
+ type_: constants.HfUriType,
565
+ *,
566
+ raw: str,
567
+ ) -> HfUri:
568
+ """Parse the body of a bucket URI: 'namespace/name[/path]'."""
569
+ location = location.strip("/")
570
+ parts = location.split("/", 2)
571
+ if len(parts) < 2 or not parts[0] or not parts[1]:
572
+ raise HfUriError(uri=raw, msg=f"Bucket id must be 'namespace/name', got '{location}'.")
573
+ bucket_id = f"{parts[0]}/{parts[1]}"
574
+ if "@" in bucket_id:
575
+ raise HfUriError(uri=raw, msg="Bucket URIs do not support a revision marker ('@').")
576
+ path_in_bucket = parts[2] if len(parts) >= 3 else ""
577
+ return HfUri(
578
+ type=type_,
579
+ id=bucket_id,
580
+ revision=None,
581
+ path_in_repo=path_in_bucket,
582
+ _raw=raw,
583
+ )
584
+
585
+
586
+ def _parse_repo_body(
587
+ location: str,
588
+ type_: constants.HfUriType,
589
+ *,
590
+ raw: str,
591
+ ) -> HfUri:
592
+ """Parse the body of a repo URI: '<repo_id>[@<revision>][/<path>]'."""
593
+ location = location.strip("/")
594
+ if not location:
595
+ raise HfUriError(uri=raw, msg="Missing repository id.")
596
+
597
+ # The '@' separates the repo_id from the revision, but only when it
598
+ # appears right after 'namespace/name' (at most one '/' before it).
599
+ # An '@' deeper in the path (e.g. in a filename like 'file@1.txt') is literal.
600
+ at_idx = location.find("@")
601
+ revision: str | None
602
+
603
+ if at_idx == -1 or location[:at_idx].count("/") > 1:
604
+ # No '@' at all, or the '@' is past the repo_id portion (in a filename).
605
+ revision = None
606
+ parts = location.split("/", 2)
607
+ if len(parts) < 2:
608
+ raise HfUriError(uri=raw, msg=f"Repository id must be 'namespace/name', got '{location}'. ")
609
+ repo_id = f"{parts[0]}/{parts[1]}"
610
+ path_in_repo = parts[2] if len(parts) > 2 else ""
611
+ else:
612
+ repo_id = location[:at_idx]
613
+ rev_and_path = location[at_idx + 1 :]
614
+ if not repo_id:
615
+ raise HfUriError(uri=raw, msg="Missing repository id before '@'.")
616
+ if repo_id.count("/") != 1:
617
+ raise HfUriError(uri=raw, msg=f"Repository id must be 'namespace/name', got '{repo_id}'.")
618
+ # Special refs like 'refs/pr/10' contain '/' and must be matched eagerly,
619
+ # otherwise we would split them at the first '/' and treat the rest as a path.
620
+ match = _SPECIAL_REFS_REVISION_REGEX.match(rev_and_path)
621
+ if match is not None:
622
+ revision = match.group()
623
+ path_in_repo = rev_and_path[len(revision) :].removeprefix("/")
624
+ else:
625
+ slash_idx = rev_and_path.find("/")
626
+ if slash_idx == -1:
627
+ revision = rev_and_path
628
+ path_in_repo = ""
629
+ else:
630
+ revision = rev_and_path[:slash_idx]
631
+ path_in_repo = rev_and_path[slash_idx + 1 :]
632
+ revision = unquote(revision)
633
+ if not revision:
634
+ raise HfUriError(uri=raw, msg="Empty revision after '@'.")
635
+
636
+ return HfUri(
637
+ type=type_,
638
+ id=repo_id,
639
+ revision=revision,
640
+ path_in_repo=path_in_repo,
641
+ _raw=raw,
642
+ )
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_http.py ADDED
@@ -0,0 +1,1163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains utilities to handle HTTP requests in huggingface_hub."""
15
+
16
+ import atexit
17
+ import io
18
+ import json
19
+ import os
20
+ import re
21
+ import threading
22
+ import time
23
+ import uuid
24
+ from collections.abc import Callable, Generator, Mapping
25
+ from contextlib import contextmanager
26
+ from dataclasses import dataclass
27
+ from shlex import quote
28
+ from typing import Any, TypeVar
29
+ from urllib.parse import urlparse
30
+
31
+ import httpx
32
+
33
+ from huggingface_hub.errors import OfflineModeIsEnabled
34
+
35
+ from .. import constants
36
+ from ..errors import (
37
+ BadRequestError,
38
+ BucketNotFoundError,
39
+ DisabledRepoError,
40
+ GatedRepoError,
41
+ HfHubHTTPError,
42
+ JobNotFoundError,
43
+ RemoteEntryNotFoundError,
44
+ RepositoryNotFoundError,
45
+ RevisionNotFoundError,
46
+ )
47
+ from . import logging
48
+ from ._lfs import SliceFileObj
49
+ from ._typing import HTTP_METHOD_T
50
+
51
+
52
+ logger = logging.get_logger(__name__)
53
+
54
+
55
+ @dataclass(frozen=True)
56
+ class RateLimitInfo:
57
+ """
58
+ Parsed rate limit information from HTTP response headers.
59
+
60
+ Attributes:
61
+ resource_type (`str`): The type of resource being rate limited.
62
+ remaining (`int`): The number of requests remaining in the current window.
63
+ reset_in_seconds (`int`): The number of seconds until the rate limit resets.
64
+ limit (`int`, *optional*): The maximum number of requests allowed in the current window.
65
+ window_seconds (`int`, *optional*): The number of seconds in the current window.
66
+
67
+ """
68
+
69
+ resource_type: str
70
+ remaining: int
71
+ reset_in_seconds: int
72
+ limit: int | None = None
73
+ window_seconds: int | None = None
74
+
75
+
76
+ # Regex patterns for parsing rate limit headers
77
+ # e.g.: "api";r=0;t=55 --> resource_type="api", r=0, t=55
78
+ _RATELIMIT_REGEX = re.compile(r"\"(?P<resource_type>\w+)\"\s*;\s*r\s*=\s*(?P<r>\d+)\s*;\s*t\s*=\s*(?P<t>\d+)")
79
+ # e.g.: "fixed window";"api";q=500;w=300 --> q=500, w=300
80
+ _RATELIMIT_POLICY_REGEX = re.compile(r"q\s*=\s*(?P<q>\d+).*?w\s*=\s*(?P<w>\d+)")
81
+
82
+
83
+ def parse_ratelimit_headers(headers: Mapping[str, str]) -> RateLimitInfo | None:
84
+ """Parse rate limit information from HTTP response headers.
85
+
86
+ Follows IETF draft: https://www.ietf.org/archive/id/draft-ietf-httpapi-ratelimit-headers-09.html
87
+ Only a subset is implemented.
88
+
89
+ Example:
90
+ ```python
91
+ >>> from huggingface_hub.utils import parse_ratelimit_headers
92
+ >>> headers = {
93
+ ... "ratelimit": '"api";r=0;t=55',
94
+ ... "ratelimit-policy": '"fixed window";"api";q=500;w=300',
95
+ ... }
96
+ >>> info = parse_ratelimit_headers(headers)
97
+ >>> info.remaining
98
+ 0
99
+ >>> info.reset_in_seconds
100
+ 55
101
+ ```
102
+ """
103
+
104
+ ratelimit: str | None = None
105
+ policy: str | None = None
106
+ for key in headers:
107
+ lower_key = key.lower()
108
+ if lower_key == "ratelimit":
109
+ ratelimit = headers[key]
110
+ elif lower_key == "ratelimit-policy":
111
+ policy = headers[key]
112
+
113
+ if not ratelimit:
114
+ return None
115
+
116
+ match = _RATELIMIT_REGEX.search(ratelimit)
117
+ if not match:
118
+ return None
119
+
120
+ resource_type = match.group("resource_type")
121
+ remaining = int(match.group("r"))
122
+ reset_in_seconds = int(match.group("t"))
123
+
124
+ limit: int | None = None
125
+ window_seconds: int | None = None
126
+
127
+ if policy:
128
+ policy_match = _RATELIMIT_POLICY_REGEX.search(policy)
129
+ if policy_match:
130
+ limit = int(policy_match.group("q"))
131
+ window_seconds = int(policy_match.group("w"))
132
+
133
+ return RateLimitInfo(
134
+ resource_type=resource_type,
135
+ remaining=remaining,
136
+ reset_in_seconds=reset_in_seconds,
137
+ limit=limit,
138
+ window_seconds=window_seconds,
139
+ )
140
+
141
+
142
+ def _parse_retry_after(headers: Mapping[str, str]) -> int | None:
143
+ """Parse the standard `Retry-After` HTTP header into a number of seconds to wait.
144
+
145
+ The `Retry-After` header can be either a non-negative number of seconds (delay-seconds)
146
+ or an HTTP-date after which to retry. We handle only the delay-seconds case.
147
+
148
+ See https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After.
149
+ """
150
+ value: str | None = None
151
+ for key in headers:
152
+ if key.lower() == "retry-after":
153
+ value = headers[key]
154
+ break
155
+
156
+ if value is None:
157
+ return None
158
+ value = value.strip()
159
+ if not value:
160
+ return None
161
+
162
+ if value.isdigit():
163
+ return int(value) # e.g. "Retry-After: 120"
164
+ return None # e.g. "Retry-After: Wed, 21 Oct 2015 07:28:00 GMT" - not supported
165
+
166
+
167
+ # When raising an error, we include the request id in the error message for easier debugging.
168
+ # Request ID is sourced from headers in order of precedence: "X-Request-Id", "X-Amzn-Trace-Id", "X-Amz-Cf-Id".
169
+ X_REQUEST_ID = "x-request-id"
170
+ X_AMZN_TRACE_ID = "X-Amzn-Trace-Id"
171
+ X_AMZ_CF_ID = "x-amz-cf-id"
172
+
173
+ REPO_API_REGEX = re.compile(
174
+ r"""
175
+ # staging or production endpoint
176
+ ^https://[^/]+
177
+ (
178
+ # on /api/repo_type/repo_id
179
+ /api/(models|datasets|spaces)/(.+)
180
+ |
181
+ # or /repo_id/resolve/revision/...
182
+ /(.+)/resolve/(.+)
183
+ )
184
+ """,
185
+ flags=re.VERBOSE,
186
+ )
187
+
188
+ BUCKET_API_REGEX = re.compile(
189
+ r"""
190
+ # staging or production endpoint
191
+ ^https?://[^/]+
192
+ # on /api/buckets/...
193
+ /api/buckets/
194
+ """,
195
+ flags=re.VERBOSE,
196
+ )
197
+
198
+ # Regex to extract the job_id from a (scheduled) job API URL.
199
+ # Matches /api/jobs/{namespace}/{job_id}[/...] and /api/scheduled-jobs/{namespace}/{job_id}[/...].
200
+ _JOB_ID_FROM_URL_REGEX = re.compile(r"^https?://[^/]+/api/(?:scheduled-jobs|jobs)/[^/]+/([^/?]+)")
201
+
202
+ # Regex to extract repo_type and repo_id from API URLs.
203
+ # Captures: group(1) = repo_type plural (models/datasets/spaces), group(2) = first path segment, group(3) = optional second segment.
204
+ _REPO_ID_FROM_URL_REGEX = re.compile(r"^https?://[^/]+/api/(models|datasets|spaces)/([^/]+)(?:/([^/]+))?")
205
+
206
+ # Regex to extract bucket_id (namespace/name) from bucket API URLs.
207
+ _BUCKET_ID_FROM_URL_REGEX = re.compile(r"^https?://[^/]+/api/buckets/([^/]+/[^/]+)")
208
+
209
+ # Sub-paths that follow a repo_id in API URLs (not part of the repo name).
210
+ _REPO_URL_SUBPATHS = {"resolve", "tree", "blob", "raw", "refs", "commit", "discussions", "settings", "revision"}
211
+
212
+
213
+ def _parse_repo_info_from_url(url: str) -> tuple[str | None, str | None]:
214
+ """Extract (repo_type, repo_id) from an API URL.
215
+
216
+ Returns canonical repo_type values: "model", "dataset", "space" (or None).
217
+
218
+ Examples:
219
+ >>> _parse_repo_info_from_url("https://huggingface.co/api/models/user/repo")
220
+ ("model", "user/repo")
221
+ >>> _parse_repo_info_from_url("https://huggingface.co/api/datasets/user/repo/resolve/main/data.csv")
222
+ ("dataset", "user/repo")
223
+ >>> _parse_repo_info_from_url("https://huggingface.co/api/models/bert-base-cased/resolve/main/config.json")
224
+ ("model", "bert-base-cased")
225
+ """
226
+ match = _REPO_ID_FROM_URL_REGEX.search(url)
227
+ if not match:
228
+ return None, None
229
+ repo_type = constants.REPO_TYPES_MAPPING.get(match.group(1))
230
+ first, second = match.group(2), match.group(3)
231
+ if second and second not in _REPO_URL_SUBPATHS:
232
+ repo_id = f"{first}/{second}"
233
+ else:
234
+ repo_id = first
235
+ return repo_type, repo_id
236
+
237
+
238
+ def _parse_bucket_id_from_url(url: str) -> str | None:
239
+ """Extract bucket_id (namespace/name) from a bucket API URL."""
240
+ match = _BUCKET_ID_FROM_URL_REGEX.search(url)
241
+ return match.group(1) if match else None
242
+
243
+
244
+ def _parse_job_id_from_url(url: str) -> str | None:
245
+ """Extract the job_id from a (scheduled) job API URL, if present."""
246
+ match = _JOB_ID_FROM_URL_REGEX.search(url)
247
+ return match.group(1) if match else None
248
+
249
+
250
+ def hf_request_event_hook(request: httpx.Request) -> None:
251
+ """
252
+ Event hook that will be used to make HTTP requests to the Hugging Face Hub.
253
+
254
+ What it does:
255
+ - Block requests if offline mode is enabled
256
+ - Add a request ID to the request headers
257
+ - Log the request if debug mode is enabled
258
+ """
259
+ if constants.is_offline_mode():
260
+ raise OfflineModeIsEnabled(
261
+ f"Cannot reach {request.url}: offline mode is enabled. To disable it, please unset the `HF_HUB_OFFLINE` environment variable."
262
+ )
263
+
264
+ # Add random request ID => easier for server-side debugging
265
+ if X_AMZN_TRACE_ID not in request.headers:
266
+ request.headers[X_AMZN_TRACE_ID] = request.headers.get(X_REQUEST_ID) or str(uuid.uuid4())
267
+ request_id = request.headers.get(X_AMZN_TRACE_ID)
268
+
269
+ # Debug log
270
+ logger.debug(
271
+ "Request %s: %s %s (authenticated: %s)",
272
+ request_id,
273
+ request.method,
274
+ request.url,
275
+ request.headers.get("authorization") is not None,
276
+ )
277
+ if constants.HF_DEBUG:
278
+ logger.debug("Send: %s", _curlify(request))
279
+
280
+ return request_id
281
+
282
+
283
+ async def async_hf_request_event_hook(request: httpx.Request) -> None:
284
+ """
285
+ Async version of `hf_request_event_hook`.
286
+ """
287
+ return hf_request_event_hook(request)
288
+
289
+
290
+ async def async_hf_response_event_hook(response: httpx.Response) -> None:
291
+ if response.status_code >= 400:
292
+ # If response will raise, read content from stream to have it available when raising the exception
293
+ # If content-length is not set or is too large, skip reading the content to avoid OOM
294
+ if "Content-length" in response.headers:
295
+ try:
296
+ length = int(response.headers["Content-length"])
297
+ except ValueError:
298
+ return
299
+
300
+ if length < 1_000_000:
301
+ await response.aread()
302
+
303
+
304
+ def default_client_factory() -> httpx.Client:
305
+ """
306
+ Factory function to create a `httpx.Client` with the default transport.
307
+ """
308
+ return httpx.Client(
309
+ event_hooks={"request": [hf_request_event_hook]},
310
+ follow_redirects=True,
311
+ timeout=None,
312
+ )
313
+
314
+
315
+ def default_async_client_factory() -> httpx.AsyncClient:
316
+ """
317
+ Factory function to create a `httpx.AsyncClient` with the default transport.
318
+ """
319
+ return httpx.AsyncClient(
320
+ event_hooks={"request": [async_hf_request_event_hook], "response": [async_hf_response_event_hook]},
321
+ follow_redirects=True,
322
+ timeout=None,
323
+ )
324
+
325
+
326
+ CLIENT_FACTORY_T = Callable[[], httpx.Client]
327
+ ASYNC_CLIENT_FACTORY_T = Callable[[], httpx.AsyncClient]
328
+
329
+ _CLIENT_LOCK = threading.Lock()
330
+ _GLOBAL_CLIENT_FACTORY: CLIENT_FACTORY_T = default_client_factory
331
+ _GLOBAL_ASYNC_CLIENT_FACTORY: ASYNC_CLIENT_FACTORY_T = default_async_client_factory
332
+ _GLOBAL_CLIENT: httpx.Client | None = None
333
+
334
+
335
+ def set_client_factory(client_factory: CLIENT_FACTORY_T) -> None:
336
+ """
337
+ Set the HTTP client factory to be used by `huggingface_hub`.
338
+
339
+ The client factory is a method that returns a `httpx.Client` object. On the first call to [`get_session`] the client factory
340
+ will be used to create a new `httpx.Client` object that will be shared between all calls made by `huggingface_hub`.
341
+
342
+ This can be useful if you are running your scripts in a specific environment requiring custom configuration (e.g. custom proxy or certifications).
343
+
344
+ Use [`get_session`] to get a correctly configured `httpx.Client`.
345
+ """
346
+ global _GLOBAL_CLIENT_FACTORY
347
+ with _CLIENT_LOCK:
348
+ close_session()
349
+ _GLOBAL_CLIENT_FACTORY = client_factory
350
+
351
+
352
+ def set_async_client_factory(async_client_factory: ASYNC_CLIENT_FACTORY_T) -> None:
353
+ """
354
+ Set the HTTP async client factory to be used by `huggingface_hub`.
355
+
356
+ The async client factory is a method that returns a `httpx.AsyncClient` object.
357
+ This can be useful if you are running your scripts in a specific environment requiring custom configuration (e.g. custom proxy or certifications).
358
+ Use [`get_async_client`] to get a correctly configured `httpx.AsyncClient`.
359
+
360
+ > [!WARNING]
361
+ > Contrary to the `httpx.Client` that is shared between all calls made by `huggingface_hub`, the `httpx.AsyncClient` is not shared.
362
+ > It is recommended to use an async context manager to ensure the client is properly closed when the context is exited.
363
+ """
364
+ global _GLOBAL_ASYNC_CLIENT_FACTORY
365
+ _GLOBAL_ASYNC_CLIENT_FACTORY = async_client_factory
366
+
367
+
368
+ def get_session() -> httpx.Client:
369
+ """
370
+ Get a `httpx.Client` object, using the transport factory from the user.
371
+
372
+ This client is shared between all calls made by `huggingface_hub`. Therefore you should not close it manually.
373
+
374
+ Use [`set_client_factory`] to customize the `httpx.Client`.
375
+ """
376
+ global _GLOBAL_CLIENT
377
+ if _GLOBAL_CLIENT is None:
378
+ with _CLIENT_LOCK:
379
+ _GLOBAL_CLIENT = _GLOBAL_CLIENT_FACTORY()
380
+ return _GLOBAL_CLIENT
381
+
382
+
383
+ def get_async_session() -> httpx.AsyncClient:
384
+ """
385
+ Return a `httpx.AsyncClient` object, using the transport factory from the user.
386
+
387
+ Use [`set_async_client_factory`] to customize the `httpx.AsyncClient`.
388
+
389
+ > [!WARNING]
390
+ > Contrary to the `httpx.Client` that is shared between all calls made by `huggingface_hub`, the `httpx.AsyncClient` is not shared.
391
+ > It is recommended to use an async context manager to ensure the client is properly closed when the context is exited.
392
+ """
393
+ return _GLOBAL_ASYNC_CLIENT_FACTORY()
394
+
395
+
396
+ def close_session() -> None:
397
+ """
398
+ Close the global `httpx.Client` used by `huggingface_hub`.
399
+
400
+ If a Client is closed, it will be recreated on the next call to [`get_session`].
401
+
402
+ Can be useful if e.g. an SSL certificate has been updated.
403
+ """
404
+ global _GLOBAL_CLIENT
405
+ client = _GLOBAL_CLIENT
406
+
407
+ # First, set global client to None
408
+ _GLOBAL_CLIENT = None
409
+
410
+ # Then, close the clients
411
+ if client is not None:
412
+ try:
413
+ client.close()
414
+ except Exception as e:
415
+ logger.warning(f"Error closing client: {e}")
416
+
417
+
418
+ atexit.register(close_session)
419
+ if hasattr(os, "register_at_fork"):
420
+ os.register_at_fork(after_in_child=close_session)
421
+
422
+
423
+ _DEFAULT_RETRY_ON_EXCEPTIONS: tuple[type[Exception], ...] = (
424
+ httpx.TimeoutException,
425
+ httpx.NetworkError,
426
+ httpx.RemoteProtocolError,
427
+ )
428
+ _DEFAULT_RETRY_ON_STATUS_CODES: tuple[int, ...] = (408, 429, 500, 502, 503, 504)
429
+
430
+
431
+ def _http_backoff_base(
432
+ method: HTTP_METHOD_T,
433
+ url: str,
434
+ *,
435
+ max_retries: int = 5,
436
+ base_wait_time: float = 1,
437
+ max_wait_time: float = 8,
438
+ retry_on_exceptions: type[Exception] | tuple[type[Exception], ...] = _DEFAULT_RETRY_ON_EXCEPTIONS,
439
+ retry_on_status_codes: int | tuple[int, ...] = _DEFAULT_RETRY_ON_STATUS_CODES,
440
+ stream: bool = False,
441
+ **kwargs,
442
+ ) -> Generator[httpx.Response, None, None]:
443
+ """Internal implementation of HTTP backoff logic shared between `http_backoff` and `http_stream_backoff`."""
444
+ if isinstance(retry_on_exceptions, type): # Tuple from single exception type
445
+ retry_on_exceptions = (retry_on_exceptions,)
446
+
447
+ if isinstance(retry_on_status_codes, int): # Tuple from single status code
448
+ retry_on_status_codes = (retry_on_status_codes,)
449
+
450
+ nb_tries = 0
451
+ sleep_time = base_wait_time
452
+ ratelimit_reset: int | None = None # seconds to wait for rate limit reset if 429 response
453
+
454
+ # If `data` is used and is a file object (or any IO), it will be consumed on the
455
+ # first HTTP request. We need to save the initial position so that the full content
456
+ # of the file is re-sent on http backoff. See warning tip in docstring.
457
+ io_obj_initial_pos = None
458
+ if "data" in kwargs and isinstance(kwargs["data"], (io.IOBase, SliceFileObj)):
459
+ io_obj_initial_pos = kwargs["data"].tell()
460
+
461
+ while True:
462
+ nb_tries += 1
463
+ ratelimit_reset = None
464
+ # Fetched on each attempt: a previous attempt may have closed the shared client (see `close_session` below),
465
+ # and closed `httpx.Client` objects cannot be reused.
466
+ client = get_session()
467
+ try:
468
+ # If `data` is used and is a file object (or any IO), set back cursor to
469
+ # initial position.
470
+ if io_obj_initial_pos is not None:
471
+ kwargs["data"].seek(io_obj_initial_pos)
472
+
473
+ # Perform request and handle response
474
+ def _should_retry(response: httpx.Response) -> bool:
475
+ """Handle response and return True if should retry, False if should return/yield."""
476
+ nonlocal ratelimit_reset
477
+
478
+ if response.status_code not in retry_on_status_codes:
479
+ return False # Success, don't retry
480
+
481
+ # Wrong status code returned (HTTP 503 for instance)
482
+ logger.warning(f"HTTP Error {response.status_code} thrown while requesting {method} {url}")
483
+ if nb_tries > max_retries:
484
+ hf_raise_for_status(response) # Will raise uncaught exception
485
+ # Return/yield response to avoid infinite loop in the corner case where the
486
+ # user ask for retry on a status code that doesn't raise_for_status.
487
+ return False # Don't retry, return/yield response
488
+
489
+ # Check 'ratelimit' and `Retry-After` headers.
490
+ if (
491
+ response.status_code == 429
492
+ and (ratelimit_info := parse_ratelimit_headers(response.headers)) is not None
493
+ ):
494
+ ratelimit_reset = ratelimit_info.reset_in_seconds
495
+ elif (retry_after := _parse_retry_after(response.headers)) is not None:
496
+ ratelimit_reset = retry_after
497
+
498
+ return True # Should retry
499
+
500
+ if stream:
501
+ with client.stream(method=method, url=url, **kwargs) as response:
502
+ if not _should_retry(response):
503
+ yield response
504
+ return
505
+ else:
506
+ response = client.request(method=method, url=url, **kwargs)
507
+ if not _should_retry(response):
508
+ yield response
509
+ return
510
+
511
+ except retry_on_exceptions as err:
512
+ logger.warning(f"'{err}' thrown while requesting {method} {url}")
513
+
514
+ if isinstance(err, httpx.ConnectError):
515
+ close_session() # In case of SSLError it's best to close the shared httpx.Client objects
516
+
517
+ if nb_tries > max_retries:
518
+ raise err
519
+
520
+ if ratelimit_reset is not None:
521
+ actual_sleep = float(ratelimit_reset) + 1 # +1s to avoid rounding issues
522
+ logger.warning(f"Rate limited. Waiting {actual_sleep}s before retry [Retry {nb_tries}/{max_retries}].")
523
+ else:
524
+ actual_sleep = sleep_time
525
+ logger.warning(f"Retrying in {actual_sleep}s [Retry {nb_tries}/{max_retries}].")
526
+
527
+ time.sleep(actual_sleep)
528
+
529
+ # Update sleep time for next retry
530
+ sleep_time = min(max_wait_time, sleep_time * 2) # Exponential backoff
531
+
532
+
533
+ def http_backoff(
534
+ method: HTTP_METHOD_T,
535
+ url: str,
536
+ *,
537
+ max_retries: int = 5,
538
+ base_wait_time: float = 1,
539
+ max_wait_time: float = 8,
540
+ retry_on_exceptions: type[Exception] | tuple[type[Exception], ...] = _DEFAULT_RETRY_ON_EXCEPTIONS,
541
+ retry_on_status_codes: int | tuple[int, ...] = _DEFAULT_RETRY_ON_STATUS_CODES,
542
+ **kwargs,
543
+ ) -> httpx.Response:
544
+ """Wrapper around httpx to retry calls on an endpoint, with exponential backoff.
545
+
546
+ Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...)
547
+ and/or on specific status codes (ex: service unavailable). If the call failed more
548
+ than `max_retries`, the exception is thrown or `raise_for_status` is called on the
549
+ response object.
550
+
551
+ Re-implement mechanisms from the `backoff` library to avoid adding an external
552
+ dependencies to `hugging_face_hub`. See https://github.com/litl/backoff.
553
+
554
+ Args:
555
+ method (`Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]`):
556
+ HTTP method to perform.
557
+ url (`str`):
558
+ The URL of the resource to fetch.
559
+ max_retries (`int`, *optional*, defaults to `5`):
560
+ Maximum number of retries, defaults to 5. Set to `0` to disable retries.
561
+ base_wait_time (`float`, *optional*, defaults to `1`):
562
+ Duration (in seconds) to wait before retrying the first time.
563
+ Wait time between retries then grows exponentially, capped by
564
+ `max_wait_time`.
565
+ max_wait_time (`float`, *optional*, defaults to `8`):
566
+ Maximum duration (in seconds) to wait before retrying.
567
+ retry_on_exceptions (`type[Exception]` or `tuple[type[Exception]]`, *optional*):
568
+ Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types.
569
+ By default, retry on `httpx.TimeoutException`, `httpx.NetworkError` and `httpx.RemoteProtocolError`.
570
+ retry_on_status_codes (`int` or `tuple[int]`, *optional*, defaults to `(429, 500, 502, 503, 504)`):
571
+ Define on which status codes the request must be retried. By default, retries
572
+ on rate limit (429) and server errors (5xx).
573
+ **kwargs (`dict`, *optional*):
574
+ kwargs to pass to `httpx.request`.
575
+
576
+ Example:
577
+ ```
578
+ >>> from huggingface_hub.utils import http_backoff
579
+
580
+ # Same usage as "httpx.request".
581
+ >>> response = http_backoff("GET", "https://www.google.com")
582
+ >>> response.raise_for_status()
583
+
584
+ # If you expect a Gateway Timeout from time to time
585
+ >>> http_backoff("PUT", upload_url, data=data, retry_on_status_codes=504)
586
+ >>> response.raise_for_status()
587
+ ```
588
+
589
+ > [!WARNING]
590
+ > When using `requests` it is possible to stream data by passing an iterator to the
591
+ > `data` argument. On http backoff this is a problem as the iterator is not reset
592
+ > after a failed call. This issue is mitigated for file objects or any IO streams
593
+ > by saving the initial position of the cursor (with `data.tell()`) and resetting the
594
+ > cursor between each call (with `data.seek()`). For arbitrary iterators, http backoff
595
+ > will fail. If this is a hard constraint for you, please let us know by opening an
596
+ > issue on [Github](https://github.com/huggingface/huggingface_hub).
597
+ """
598
+ return next(
599
+ _http_backoff_base(
600
+ method=method,
601
+ url=url,
602
+ max_retries=max_retries,
603
+ base_wait_time=base_wait_time,
604
+ max_wait_time=max_wait_time,
605
+ retry_on_exceptions=retry_on_exceptions,
606
+ retry_on_status_codes=retry_on_status_codes,
607
+ stream=False,
608
+ **kwargs,
609
+ )
610
+ )
611
+
612
+
613
+ @contextmanager
614
+ def http_stream_backoff(
615
+ method: HTTP_METHOD_T,
616
+ url: str,
617
+ *,
618
+ max_retries: int = 5,
619
+ base_wait_time: float = 1,
620
+ max_wait_time: float = 8,
621
+ retry_on_exceptions: type[Exception] | tuple[type[Exception], ...] = _DEFAULT_RETRY_ON_EXCEPTIONS,
622
+ retry_on_status_codes: int | tuple[int, ...] = _DEFAULT_RETRY_ON_STATUS_CODES,
623
+ **kwargs,
624
+ ) -> Generator[httpx.Response, None, None]:
625
+ """Wrapper around httpx to retry calls on an endpoint, with exponential backoff.
626
+
627
+ Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...)
628
+ and/or on specific status codes (ex: service unavailable). If the call failed more
629
+ than `max_retries`, the exception is thrown or `raise_for_status` is called on the
630
+ response object.
631
+
632
+ Re-implement mechanisms from the `backoff` library to avoid adding an external
633
+ dependencies to `hugging_face_hub`. See https://github.com/litl/backoff.
634
+
635
+ Args:
636
+ method (`Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]`):
637
+ HTTP method to perform.
638
+ url (`str`):
639
+ The URL of the resource to fetch.
640
+ max_retries (`int`, *optional*, defaults to `5`):
641
+ Maximum number of retries, defaults to 5. Set to `0` to disable retries.
642
+ base_wait_time (`float`, *optional*, defaults to `1`):
643
+ Duration (in seconds) to wait before retrying the first time.
644
+ Wait time between retries then grows exponentially, capped by
645
+ `max_wait_time`.
646
+ max_wait_time (`float`, *optional*, defaults to `8`):
647
+ Maximum duration (in seconds) to wait before retrying.
648
+ retry_on_exceptions (`type[Exception]` or `tuple[type[Exception]]`, *optional*):
649
+ Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types.
650
+ By default, retry on `httpx.TimeoutException`, `httpx.NetworkError` and `httpx.RemoteProtocolError`.
651
+ retry_on_status_codes (`int` or `tuple[int]`, *optional*, defaults to `(429, 500, 502, 503, 504)`):
652
+ Define on which status codes the request must be retried. By default, retries
653
+ on rate limit (429) and server errors (5xx).
654
+ **kwargs (`dict`, *optional*):
655
+ kwargs to pass to `httpx.request`.
656
+
657
+ Example:
658
+ ```
659
+ >>> from huggingface_hub.utils import http_stream_backoff
660
+
661
+ # Same usage as "httpx.stream".
662
+ >>> with http_stream_backoff("GET", "https://www.google.com") as response:
663
+ ... for chunk in response.iter_bytes():
664
+ ... print(chunk)
665
+
666
+ # If you expect a Gateway Timeout from time to time
667
+ >>> with http_stream_backoff("PUT", upload_url, data=data, retry_on_status_codes=504) as response:
668
+ ... response.raise_for_status()
669
+ ```
670
+
671
+ > [!WARNING]
672
+ > When using `httpx` it is possible to stream data by passing an iterator to the
673
+ > `data` argument. On http backoff this is a problem as the iterator is not reset
674
+ > after a failed call. This issue is mitigated for file objects or any IO streams
675
+ > by saving the initial position of the cursor (with `data.tell()`) and resetting the
676
+ > cursor between each call (with `data.seek()`). For arbitrary iterators, http backoff
677
+ > will fail. If this is a hard constraint for you, please let us know by opening an
678
+ > issue on [Github](https://github.com/huggingface/huggingface_hub).
679
+ """
680
+ yield from _http_backoff_base(
681
+ method=method,
682
+ url=url,
683
+ max_retries=max_retries,
684
+ base_wait_time=base_wait_time,
685
+ max_wait_time=max_wait_time,
686
+ retry_on_exceptions=retry_on_exceptions,
687
+ retry_on_status_codes=retry_on_status_codes,
688
+ stream=True,
689
+ **kwargs,
690
+ )
691
+
692
+
693
+ def _httpx_follow_relative_redirects_with_backoff(
694
+ method: HTTP_METHOD_T, url: str, *, retry_on_errors: bool = False, **httpx_kwargs
695
+ ) -> httpx.Response:
696
+ """Perform an HTTP request with backoff and follow relative redirects only.
697
+
698
+ Used to fetch HEAD /resolve on repo or bucket files.
699
+
700
+ This is useful to follow a redirection to a renamed repository without following redirection to a CDN.
701
+
702
+ A backoff mechanism retries the HTTP call on errors (429, 5xx, timeout, network errors).
703
+
704
+ Args:
705
+ method (`str`):
706
+ HTTP method, such as 'GET' or 'HEAD'.
707
+ url (`str`):
708
+ The URL of the resource to fetch.
709
+ retry_on_errors (`bool`, *optional*, defaults to `False`):
710
+ Whether to retry on errors. If False, no retry is performed (fast fallback to local cache).
711
+ If True, uses default retry behavior (429, 5xx, timeout, network errors).
712
+ **httpx_kwargs (`dict`, *optional*):
713
+ Params to pass to `httpx.request`.
714
+ """
715
+ # if `retry_on_errors=False`, disable all retries for fast fallback to cache
716
+ no_retry_kwargs: dict[str, Any] = (
717
+ {} if retry_on_errors else {"retry_on_exceptions": (), "retry_on_status_codes": ()}
718
+ )
719
+
720
+ while True:
721
+ response = http_backoff(
722
+ method=method,
723
+ url=url,
724
+ **httpx_kwargs,
725
+ follow_redirects=False,
726
+ **no_retry_kwargs,
727
+ )
728
+ hf_raise_for_status(response)
729
+
730
+ # Check if response is a relative redirect
731
+ if 300 <= response.status_code <= 399:
732
+ parsed_target = urlparse(response.headers["Location"])
733
+ if parsed_target.netloc == "":
734
+ # Relative redirect -> update URL and retry
735
+ url = urlparse(url)._replace(path=parsed_target.path).geturl()
736
+ continue
737
+
738
+ # Break if no relative redirect
739
+ break
740
+
741
+ return response
742
+
743
+
744
+ def fix_hf_endpoint_in_url(url: str, endpoint: str | None) -> str:
745
+ """Replace the default endpoint in a URL by a custom one.
746
+
747
+ This is useful when using a proxy and the Hugging Face Hub returns a URL with the default endpoint.
748
+ """
749
+ endpoint = endpoint.rstrip("/") if endpoint else constants.ENDPOINT
750
+ # check if a proxy has been set => if yes, update the returned URL to use the proxy
751
+ if endpoint not in (constants._HF_DEFAULT_ENDPOINT, constants._HF_DEFAULT_STAGING_ENDPOINT):
752
+ url = url.replace(constants._HF_DEFAULT_ENDPOINT, endpoint)
753
+ url = url.replace(constants._HF_DEFAULT_STAGING_ENDPOINT, endpoint)
754
+ return url
755
+
756
+
757
+ def hf_raise_for_status(response: httpx.Response, endpoint_name: str | None = None) -> None:
758
+ """
759
+ Internal version of `response.raise_for_status()` that will refine a potential HTTPError.
760
+ Raised exception will be an instance of [`~errors.HfHubHTTPError`].
761
+
762
+ This helper is meant to be the unique method to raise_for_status when making a call to the Hugging Face Hub.
763
+
764
+ Args:
765
+ response (`Response`):
766
+ Response from the server.
767
+ endpoint_name (`str`, *optional*):
768
+ Name of the endpoint that has been called. If provided, the error message will be more complete.
769
+
770
+ > [!WARNING]
771
+ > Raises when the request has failed:
772
+ >
773
+ > - [`~utils.RepositoryNotFoundError`]
774
+ > If the repository to download from cannot be found. This may be because it
775
+ > doesn't exist, because `repo_type` is not set correctly, or because the repo
776
+ > is `private` and you do not have access.
777
+ > - [`~utils.GatedRepoError`]
778
+ > If the repository exists but is gated and the user is not on the authorized
779
+ > list.
780
+ > - [`~utils.RevisionNotFoundError`]
781
+ > If the repository exists but the revision couldn't be found.
782
+ > - [`~utils.EntryNotFoundError`]
783
+ > If the repository exists but the entry (e.g. the requested file) couldn't be
784
+ > find.
785
+ > - [`~utils.BadRequestError`]
786
+ > If request failed with a HTTP 400 BadRequest error.
787
+ > - [`~utils.HfHubHTTPError`]
788
+ > If request failed for a reason not listed above.
789
+ """
790
+ try:
791
+ _warn_on_warning_headers(response)
792
+ except Exception:
793
+ # Never raise on warning parsing
794
+ logger.debug("Failed to parse warning headers", exc_info=True)
795
+
796
+ try:
797
+ response.raise_for_status()
798
+ except httpx.HTTPStatusError as e:
799
+ if response.status_code // 100 == 3:
800
+ return # Do not raise on redirects to stay consistent with `requests`
801
+
802
+ error_code = response.headers.get("X-Error-Code")
803
+ error_message = response.headers.get("X-Error-Message")
804
+
805
+ # Parse repo info from request URL (used to enrich errors below)
806
+ request_url = (
807
+ str(response.request.url) if response.request is not None and response.request.url is not None else None
808
+ )
809
+ repo_type, repo_id = _parse_repo_info_from_url(request_url) if request_url else (None, None)
810
+
811
+ if error_code == "RevisionNotFound":
812
+ message = f"{response.status_code} Client Error." + "\n\n" + f"Revision Not Found for url: {response.url}."
813
+ raise _format(RevisionNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e
814
+
815
+ elif error_code == "EntryNotFound":
816
+ message = f"{response.status_code} Client Error." + "\n\n" + f"Entry Not Found for url: {response.url}."
817
+ raise _format(RemoteEntryNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e
818
+
819
+ elif error_code == "GatedRepo":
820
+ message = (
821
+ f"{response.status_code} Client Error." + "\n\n" + f"Cannot access gated repo for url {response.url}."
822
+ )
823
+ raise _format(GatedRepoError, message, response, repo_type=repo_type, repo_id=repo_id) from e
824
+
825
+ elif error_message == "Access to this resource is disabled.":
826
+ message = (
827
+ f"{response.status_code} Client Error."
828
+ + "\n\n"
829
+ + f"Cannot access repository for url {response.url}."
830
+ + "\n"
831
+ + "Access to this resource is disabled."
832
+ )
833
+ raise _format(DisabledRepoError, message, response) from e
834
+
835
+ elif (
836
+ error_code == "RepoNotFound"
837
+ and request_url is not None
838
+ and BUCKET_API_REGEX.search(request_url) is not None
839
+ ):
840
+ message = (
841
+ f"{response.status_code} Client Error."
842
+ + "\n\n"
843
+ + f"Bucket Not Found for url: {response.url}."
844
+ + "\nPlease make sure you specified the correct bucket id (namespace/name)."
845
+ + "\nIf the bucket is private, make sure you are authenticated and your token has the required permissions."
846
+ )
847
+ raise _format(
848
+ BucketNotFoundError, message, response, bucket_id=_parse_bucket_id_from_url(request_url)
849
+ ) from e
850
+
851
+ elif (
852
+ response.status_code == 404
853
+ and request_url is not None
854
+ and (job_id := _parse_job_id_from_url(request_url)) is not None
855
+ ):
856
+ message = (
857
+ f"{response.status_code} Client Error."
858
+ + "\n\n"
859
+ + f"Job Not Found for url: {response.url}."
860
+ + "\nPlease make sure you specified the correct job ID and namespace."
861
+ )
862
+ raise _format(JobNotFoundError, message, response, job_id=job_id) from e
863
+
864
+ elif error_code == "RepoNotFound" or (
865
+ response.status_code == 401
866
+ and error_message != "Invalid credentials in Authorization header"
867
+ and request_url is not None
868
+ and REPO_API_REGEX.search(request_url) is not None
869
+ ):
870
+ # 401 is misleading as it is returned for:
871
+ # - private and gated repos if user is not authenticated
872
+ # - missing repos
873
+ # => for now, we process them as `RepoNotFound` anyway.
874
+ # See https://gist.github.com/Wauplin/46c27ad266b15998ce56a6603796f0b9
875
+ message = (
876
+ f"{response.status_code} Client Error."
877
+ + "\n\n"
878
+ + f"Repository Not Found for url: {response.url}."
879
+ + "\nPlease make sure you specified the correct `repo_id` and"
880
+ " `repo_type`.\nIf you are trying to access a private or gated repo,"
881
+ " make sure you are authenticated and your token has the required permissions."
882
+ + "\nFor more details, see https://huggingface.co/docs/huggingface_hub/authentication"
883
+ )
884
+ raise _format(RepositoryNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e
885
+
886
+ elif response.status_code == 400:
887
+ message = (
888
+ f"\n\nBad request for {endpoint_name} endpoint:" if endpoint_name is not None else "\n\nBad request:"
889
+ )
890
+ raise _format(BadRequestError, message, response) from e
891
+
892
+ elif response.status_code == 403:
893
+ message = (
894
+ f"\n\n{response.status_code} Forbidden: {error_message}."
895
+ + f"\nCannot access content at: {response.url}."
896
+ + "\nMake sure your token has the correct permissions."
897
+ )
898
+ raise _format(HfHubHTTPError, message, response) from e
899
+
900
+ elif response.status_code == 429:
901
+ ratelimit_info = parse_ratelimit_headers(response.headers)
902
+ if ratelimit_info is not None:
903
+ message = (
904
+ f"\n\n429 Too Many Requests: you have reached your '{ratelimit_info.resource_type}' rate limit."
905
+ )
906
+ message += f"\nRetry after {ratelimit_info.reset_in_seconds} seconds"
907
+ if ratelimit_info.limit is not None and ratelimit_info.window_seconds is not None:
908
+ message += (
909
+ f" ({ratelimit_info.remaining}/{ratelimit_info.limit} requests remaining"
910
+ f" in current {ratelimit_info.window_seconds}s window)."
911
+ )
912
+ else:
913
+ message += "."
914
+ message += f"\nUrl: {response.url}."
915
+ else:
916
+ message = f"\n\n429 Too Many Requests for url: {response.url}."
917
+ raise _format(HfHubHTTPError, message, response) from e
918
+
919
+ elif response.status_code == 416:
920
+ range_header = response.request.headers.get("Range")
921
+ message = f"{e}. Requested range: {range_header}. Content-Range: {response.headers.get('Content-Range')}."
922
+ raise _format(HfHubHTTPError, message, response) from e
923
+
924
+ # Convert `HTTPError` into a `HfHubHTTPError` to display request information
925
+ # as well (request id and/or server error message)
926
+ raise _format(HfHubHTTPError, str(e), response) from e
927
+
928
+
929
+ _WARNED_TOPICS = set()
930
+
931
+
932
+ def _warn_on_warning_headers(response: httpx.Response) -> None:
933
+ """
934
+ Emit warnings if warning headers are present in the HTTP response.
935
+
936
+ Expected header format: 'X-HF-Warning: topic; message'
937
+
938
+ Only the first warning for each topic will be shown. Topic is optional and can be empty. Note that several warning
939
+ headers can be present in a single response.
940
+
941
+ Args:
942
+ response (`httpx.Response`):
943
+ The HTTP response to check for warning headers.
944
+ """
945
+ server_warnings = response.headers.get_list("X-HF-Warning")
946
+ for server_warning in server_warnings:
947
+ topic, message = server_warning.split(";", 1) if ";" in server_warning else ("", server_warning)
948
+ topic = topic.strip()
949
+ if topic not in _WARNED_TOPICS:
950
+ message = message.strip()
951
+ if message:
952
+ _WARNED_TOPICS.add(topic)
953
+ logger.warning(message)
954
+
955
+
956
+ _HfHubHTTPErrorT = TypeVar("_HfHubHTTPErrorT", bound=HfHubHTTPError)
957
+
958
+
959
+ def _format(
960
+ error_type: type[_HfHubHTTPErrorT], custom_message: str, response: httpx.Response, **attrs: Any
961
+ ) -> _HfHubHTTPErrorT:
962
+ server_errors = []
963
+
964
+ # Retrieve server error from header
965
+ from_headers = response.headers.get("X-Error-Message")
966
+ if from_headers is not None:
967
+ server_errors.append(from_headers)
968
+
969
+ # Retrieve server error from body
970
+ try:
971
+ # Case errors are returned in a JSON format
972
+ try:
973
+ data = response.json()
974
+ except httpx.ResponseNotRead:
975
+ try:
976
+ response.read() # In case of streaming response, we need to read the response first
977
+ data = response.json()
978
+ except RuntimeError:
979
+ # In case of async streaming response, we can't read the stream here.
980
+ # In practice if user is using the default async client from `get_async_client`, the stream will have
981
+ # already been read in the async event hook `async_hf_response_event_hook`.
982
+ #
983
+ # Here, we are skipping reading the response to avoid RuntimeError but it happens only if async + stream + used httpx.AsyncClient directly.
984
+ data = {}
985
+
986
+ error = data.get("error")
987
+ error_description = data.get("error_description")
988
+ if error is not None:
989
+ if isinstance(error, list):
990
+ # Case {'error': ['my error 1', 'my error 2']}
991
+ server_errors.extend(error)
992
+ elif error_description is not None:
993
+ # OAuth-style case {'error': 'invalid_grant', 'error_description': 'my description'}
994
+ server_errors.append(f"{error}: {error_description}")
995
+ else:
996
+ # Case {'error': 'my error'}
997
+ server_errors.append(error)
998
+ elif error_description is not None:
999
+ # Case {'error_description': 'my description'} (no 'error' field)
1000
+ server_errors.append(error_description)
1001
+
1002
+ errors = data.get("errors")
1003
+ if errors is not None:
1004
+ # Case {'errors': [{'message': 'my error 1'}, {'message': 'my error 2'}]}
1005
+ for error in errors:
1006
+ if "message" in error:
1007
+ server_errors.append(error["message"])
1008
+
1009
+ except json.JSONDecodeError:
1010
+ # If content is not JSON and not HTML, append the text
1011
+ content_type = response.headers.get("Content-Type", "")
1012
+ if response.text and "html" not in content_type.lower():
1013
+ server_errors.append(response.text)
1014
+
1015
+ # Strip all server messages
1016
+ server_errors = [str(line).strip() for line in server_errors if str(line).strip()]
1017
+
1018
+ # Deduplicate server messages (keep order)
1019
+ # taken from https://stackoverflow.com/a/17016257
1020
+ server_errors = list(dict.fromkeys(server_errors))
1021
+
1022
+ # Format server error
1023
+ server_message = "\n".join(server_errors)
1024
+
1025
+ # Add server error to custom message
1026
+ final_error_message = custom_message
1027
+ if server_message and server_message.lower() not in custom_message.lower():
1028
+ if "\n\n" in custom_message:
1029
+ final_error_message += "\n" + server_message
1030
+ else:
1031
+ final_error_message += "\n\n" + server_message
1032
+
1033
+ # Prepare Request ID message
1034
+ request_id = ""
1035
+ request_id_message = ""
1036
+ for header, label in (
1037
+ (X_REQUEST_ID, "Request ID"),
1038
+ (X_AMZN_TRACE_ID, "Amzn Trace ID"),
1039
+ (X_AMZ_CF_ID, "Amz CF ID"),
1040
+ ):
1041
+ value = response.headers.get(header)
1042
+ if value:
1043
+ request_id = str(value)
1044
+ request_id_message = f" ({label}: {value})"
1045
+ break
1046
+
1047
+ # Add Request ID
1048
+ if request_id and request_id.lower() not in final_error_message.lower():
1049
+ if "\n" in final_error_message:
1050
+ newline_index = final_error_message.index("\n")
1051
+ final_error_message = (
1052
+ final_error_message[:newline_index] + request_id_message + final_error_message[newline_index:]
1053
+ )
1054
+ else:
1055
+ final_error_message += request_id_message
1056
+
1057
+ # Return
1058
+ err = error_type(final_error_message.strip(), response=response, server_message=server_message or None)
1059
+ for k, v in attrs.items():
1060
+ setattr(err, k, v)
1061
+ return err
1062
+
1063
+
1064
+ # Request-body fields that carry credentials; redacted from debug-log curl commands (HF_DEBUG).
1065
+ _SENSITIVE_BODY_KEYS = ("subject_token", "access_token", "refresh_token", "client_secret", "device_code")
1066
+ _SENSITIVE_BODY_PATTERNS = [
1067
+ pattern
1068
+ for key in _SENSITIVE_BODY_KEYS
1069
+ for pattern in (
1070
+ (re.compile(rf'("{key}"\s*:\s*")[^"]*(")'), r"\1<REDACTED>\2"), # JSON
1071
+ (re.compile(rf"(^|&)({key}=)[^&]*"), r"\1\2<REDACTED>"), # application/x-www-form-urlencoded
1072
+ )
1073
+ ]
1074
+
1075
+
1076
+ def _redact_sensitive_body(body: str) -> str:
1077
+ """Redact OAuth credential values from a JSON or form-urlencoded request body string."""
1078
+ for pattern, replacement in _SENSITIVE_BODY_PATTERNS:
1079
+ body = pattern.sub(replacement, body)
1080
+ return body
1081
+
1082
+
1083
+ def _curlify(request: httpx.Request) -> str:
1084
+ """Convert a `httpx.Request` into a curl command (str).
1085
+
1086
+ Used for debug purposes only.
1087
+
1088
+ Implementation vendored from https://github.com/ofw/curlify/blob/master/curlify.py.
1089
+ MIT License Copyright (c) 2016 Egor.
1090
+ """
1091
+ parts: list[tuple[Any, Any]] = [
1092
+ ("curl", None),
1093
+ ("-X", request.method),
1094
+ ]
1095
+
1096
+ for k, v in sorted(request.headers.items()):
1097
+ if k.lower() == "authorization":
1098
+ v = "<TOKEN>" # Hide authorization header, no matter its value (can be Bearer, Key, etc.)
1099
+ parts += [("-H", f"{k}: {v}")]
1100
+
1101
+ body: str | None = None
1102
+ try:
1103
+ if request.content is not None:
1104
+ body = request.content.decode("utf-8", errors="ignore")
1105
+ if len(body) > 1000:
1106
+ body = f"{body[:1000]} ... [truncated]"
1107
+ body = _redact_sensitive_body(body)
1108
+ except httpx.RequestNotRead:
1109
+ body = "<streaming body>"
1110
+ if body is not None:
1111
+ parts += [("-d", body.replace("\n", ""))]
1112
+
1113
+ parts += [(None, request.url)]
1114
+
1115
+ flat_parts = []
1116
+ for k, v in parts:
1117
+ if k:
1118
+ flat_parts.append(quote(str(k)))
1119
+ if v:
1120
+ flat_parts.append(quote(str(v)))
1121
+
1122
+ return " ".join(flat_parts)
1123
+
1124
+
1125
+ # Regex to parse HTTP Range header
1126
+ RANGE_REGEX = re.compile(r"^\s*bytes\s*=\s*(\d*)\s*-\s*(\d*)\s*$", re.IGNORECASE)
1127
+
1128
+
1129
+ def _adjust_range_header(original_range: str | None, resume_size: int) -> str | None:
1130
+ """
1131
+ Adjust HTTP Range header to account for resume position.
1132
+ """
1133
+ if not original_range:
1134
+ return f"bytes={resume_size}-"
1135
+
1136
+ if "," in original_range:
1137
+ raise ValueError(f"Multiple ranges detected - {original_range!r}, not supported yet.")
1138
+
1139
+ match = RANGE_REGEX.match(original_range)
1140
+ if not match:
1141
+ raise RuntimeError(f"Invalid range format - {original_range!r}.")
1142
+ start, end = match.groups()
1143
+
1144
+ if not start:
1145
+ if not end:
1146
+ raise RuntimeError(f"Invalid range format - {original_range!r}.")
1147
+
1148
+ new_suffix = int(end) - resume_size
1149
+ new_range = f"bytes=-{new_suffix}"
1150
+ if new_suffix <= 0:
1151
+ raise RuntimeError(f"Empty new range - {new_range!r}.")
1152
+ return new_range
1153
+
1154
+ start = int(start)
1155
+ new_start = start + resume_size
1156
+ if end:
1157
+ end = int(end)
1158
+ new_range = f"bytes={new_start}-{end}"
1159
+ if new_start > end:
1160
+ raise RuntimeError(f"Empty new range - {new_range!r}.")
1161
+ return new_range
1162
+
1163
+ return f"bytes={new_start}-"
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_lfs.py ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2019-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Git LFS related utilities"""
15
+
16
+ import io
17
+ import os
18
+ from contextlib import AbstractContextManager
19
+ from typing import BinaryIO
20
+
21
+
22
+ class SliceFileObj(AbstractContextManager):
23
+ """
24
+ Utility context manager to read a *slice* of a seekable file-like object as a seekable, file-like object.
25
+
26
+ This is NOT thread safe
27
+
28
+ Inspired by stackoverflow.com/a/29838711/593036
29
+
30
+ Credits to @julien-c
31
+
32
+ Args:
33
+ fileobj (`BinaryIO`):
34
+ A file-like object to slice. MUST implement `tell()` and `seek()` (and `read()` of course).
35
+ `fileobj` will be reset to its original position when exiting the context manager.
36
+ seek_from (`int`):
37
+ The start of the slice (offset from position 0 in bytes).
38
+ read_limit (`int`):
39
+ The maximum number of bytes to read from the slice.
40
+
41
+ Attributes:
42
+ previous_position (`int`):
43
+ The previous position
44
+
45
+ Examples:
46
+
47
+ Reading 200 bytes with an offset of 128 bytes from a file (ie bytes 128 to 327):
48
+ ```python
49
+ >>> with open("path/to/file", "rb") as file:
50
+ ... with SliceFileObj(file, seek_from=128, read_limit=200) as fslice:
51
+ ... fslice.read(...)
52
+ ```
53
+
54
+ Reading a file in chunks of 512 bytes
55
+ ```python
56
+ >>> import os
57
+ >>> chunk_size = 512
58
+ >>> file_size = os.getsize("path/to/file")
59
+ >>> with open("path/to/file", "rb") as file:
60
+ ... for chunk_idx in range(ceil(file_size / chunk_size)):
61
+ ... with SliceFileObj(file, seek_from=chunk_idx * chunk_size, read_limit=chunk_size) as fslice:
62
+ ... chunk = fslice.read(...)
63
+
64
+ ```
65
+ """
66
+
67
+ def __init__(self, fileobj: BinaryIO, seek_from: int, read_limit: int):
68
+ self.fileobj = fileobj
69
+ self.seek_from = seek_from
70
+ self.read_limit = read_limit
71
+
72
+ def __enter__(self):
73
+ self._previous_position = self.fileobj.tell()
74
+ end_of_stream = self.fileobj.seek(0, os.SEEK_END)
75
+ self._len = min(self.read_limit, end_of_stream - self.seek_from)
76
+ # ^^ The actual number of bytes that can be read from the slice
77
+ self.fileobj.seek(self.seek_from, io.SEEK_SET)
78
+ return self
79
+
80
+ def __exit__(self, exc_type, exc_value, traceback):
81
+ self.fileobj.seek(self._previous_position, io.SEEK_SET)
82
+
83
+ def read(self, n: int = -1):
84
+ pos = self.tell()
85
+ if pos >= self._len:
86
+ return b""
87
+ remaining_amount = self._len - pos
88
+ data = self.fileobj.read(remaining_amount if n < 0 else min(n, remaining_amount))
89
+ return data
90
+
91
+ def tell(self) -> int:
92
+ return self.fileobj.tell() - self.seek_from
93
+
94
+ def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
95
+ start = self.seek_from
96
+ end = start + self._len
97
+ if whence in (os.SEEK_SET, os.SEEK_END):
98
+ offset = start + offset if whence == os.SEEK_SET else end + offset
99
+ offset = max(start, min(offset, end))
100
+ whence = os.SEEK_SET
101
+ elif whence == os.SEEK_CUR:
102
+ cur_pos = self.fileobj.tell()
103
+ offset = max(start - cur_pos, min(offset, end - cur_pos))
104
+ else:
105
+ raise ValueError(f"whence value {whence} is not supported")
106
+ return self.fileobj.seek(offset, whence) - self.seek_from
107
+
108
+ def __iter__(self):
109
+ yield self.read(n=4 * 1024 * 1024)
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_oauth_device.py ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2026 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Device Code OAuth (RFC 8628) for browser-based login, plus access token refresh.
15
+
16
+ The flow: the client requests a device code, displays a URL and a short user code, the user
17
+ authorizes in a browser, and the client polls ``POST {ENDPOINT}/oauth/token`` until a token is
18
+ issued. Access tokens may come with a refresh token, used to renew them transparently (see
19
+ ``utils/_auth.py::get_token``).
20
+
21
+ This module is self-contained protocol logic: no printing, no persistence. Interactive flows
22
+ live in ``_login.py`` (human/library) and ``cli/auth.py`` (machine-readable event stream).
23
+ """
24
+
25
+ import time
26
+ from collections.abc import Callable
27
+ from typing import TypedDict, cast
28
+
29
+ import httpx
30
+
31
+ from .. import constants
32
+ from ..errors import DeviceCodeError, OAuthErrorCode
33
+ from ._http import get_session, hf_raise_for_status
34
+
35
+
36
+ _DEVICE_CODE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:device_code"
37
+ _REFRESH_TOKEN_GRANT_TYPE = "refresh_token"
38
+
39
+
40
+ class DeviceCodeInfo(TypedDict):
41
+ device_code: str
42
+ user_code: str
43
+ verification_uri: str
44
+ verification_uri_complete: str # falls back to verification_uri if the server omits it
45
+ interval: int
46
+ expires_in: int
47
+
48
+
49
+ class OAuthTokenResponse(TypedDict, total=False):
50
+ access_token: str
51
+ refresh_token: str
52
+ expires_in: int
53
+ token_type: str
54
+
55
+
56
+ def request_device_code() -> DeviceCodeInfo:
57
+ """Request a device code from the Hub's OAuth device authorization endpoint.
58
+
59
+ The returned dict is normalized: `interval`, `expires_in` and `verification_uri_complete`
60
+ are always set (server values, or sensible defaults).
61
+
62
+ Raises:
63
+ [`DeviceCodeError`]: If the request fails.
64
+ """
65
+ try:
66
+ response = get_session().post(
67
+ f"{constants.ENDPOINT}/oauth/device",
68
+ data={"client_id": constants.DEVICE_CODE_OAUTH_CLIENT_ID},
69
+ timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT,
70
+ )
71
+ hf_raise_for_status(response)
72
+ except httpx.HTTPError as e:
73
+ raise DeviceCodeError(f"Failed to request device code from {constants.ENDPOINT}/oauth/device: {e}") from e
74
+ info = response.json()
75
+ # `interval` is optional per RFC 8628 (5s is the spec-mandated fallback); `expires_in` is
76
+ # required but defaulted defensively so polling stays bounded if a server omits it.
77
+ info.setdefault("interval", 5)
78
+ info.setdefault("expires_in", 900)
79
+ if not info.get("verification_uri_complete"):
80
+ info["verification_uri_complete"] = info["verification_uri"]
81
+ return cast(DeviceCodeInfo, info)
82
+
83
+
84
+ def poll_device_token(
85
+ device_info: DeviceCodeInfo, *, on_pending: Callable[[], None] | None = None
86
+ ) -> OAuthTokenResponse:
87
+ """Poll the token endpoint until the user authorizes the device.
88
+
89
+ Args:
90
+ device_info (`DeviceCodeInfo`):
91
+ The device authorization response from [`request_device_code`].
92
+ on_pending (`Callable`, *optional*):
93
+ Called after each "authorization pending" response (e.g. to print a progress dot).
94
+
95
+ Returns:
96
+ `OAuthTokenResponse`: the full token response: `access_token`, and optionally
97
+ `refresh_token` and `expires_in`.
98
+
99
+ Raises:
100
+ [`DeviceCodeError`]: If authorization is denied, the device code expires, or the server
101
+ returns an unexpected OAuth error.
102
+ """
103
+ interval = device_info["interval"]
104
+ deadline = time.monotonic() + device_info["expires_in"]
105
+ while time.monotonic() < deadline:
106
+ # Inconclusive responses (network blip, 5xx, gateway error page, rate limiting) must not
107
+ # abort the login: keep polling until the device code expires (RFC 8628 section 3.5).
108
+ # The deadline bounds the total wait even if the endpoint is genuinely broken.
109
+ data = None
110
+ try:
111
+ response = get_session().post(
112
+ f"{constants.ENDPOINT}/oauth/token",
113
+ data={
114
+ "grant_type": _DEVICE_CODE_GRANT_TYPE,
115
+ "device_code": device_info["device_code"],
116
+ "client_id": constants.DEVICE_CODE_OAUTH_CLIENT_ID,
117
+ },
118
+ timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT,
119
+ )
120
+ if response.status_code < 500:
121
+ data = response.json()
122
+ except (httpx.HTTPError, ValueError):
123
+ pass
124
+
125
+ if data is not None:
126
+ if "access_token" in data:
127
+ return cast(OAuthTokenResponse, data)
128
+
129
+ match data.get("error"):
130
+ case None:
131
+ pass # JSON without an OAuth `error` field (proxy error page, ...): transient
132
+ case OAuthErrorCode.AUTHORIZATION_PENDING:
133
+ if on_pending is not None:
134
+ on_pending()
135
+ case OAuthErrorCode.SLOW_DOWN:
136
+ interval += 5
137
+ case OAuthErrorCode.EXPIRED_TOKEN:
138
+ raise DeviceCodeError(
139
+ "Device code expired. Please try again.", error_code=OAuthErrorCode.EXPIRED_TOKEN
140
+ )
141
+ case OAuthErrorCode.ACCESS_DENIED:
142
+ raise DeviceCodeError(
143
+ "Authorization was denied. Please try again.", error_code=OAuthErrorCode.ACCESS_DENIED
144
+ )
145
+ case error:
146
+ raise DeviceCodeError(
147
+ f"OAuth error: {error} - {data.get('error_description', '')}", error_code=error
148
+ )
149
+
150
+ time.sleep(interval)
151
+
152
+ raise DeviceCodeError("Device code expired (timeout). Please try again.", error_code=OAuthErrorCode.EXPIRED_TOKEN)
153
+
154
+
155
+ def refresh_access_token(refresh_token: str) -> OAuthTokenResponse:
156
+ """Exchange a refresh token for a new access token.
157
+
158
+ Returns:
159
+ `OAuthTokenResponse`: the full token response: `access_token`, and optionally a rotated
160
+ `refresh_token` and `expires_in`.
161
+
162
+ Raises:
163
+ [`DeviceCodeError`]: If the server rejects the refresh (`error_code="invalid_grant"` when
164
+ the refresh token is expired or revoked) or returns an unexpected response.
165
+ """
166
+ try:
167
+ response = get_session().post(
168
+ f"{constants.ENDPOINT}/oauth/token",
169
+ data={
170
+ "grant_type": _REFRESH_TOKEN_GRANT_TYPE,
171
+ "refresh_token": refresh_token,
172
+ "client_id": constants.DEVICE_CODE_OAUTH_CLIENT_ID,
173
+ },
174
+ # An explicit timeout is critical here: this runs inside `get_token()`, so a hung
175
+ # request would otherwise block every Hub call in the process.
176
+ timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT,
177
+ )
178
+ except httpx.HTTPError as e:
179
+ raise DeviceCodeError(f"Failed to refresh access token: {e}") from e
180
+ data = _parse_token_response(response)
181
+ if "access_token" in data:
182
+ return cast(OAuthTokenResponse, data)
183
+ error = data.get("error")
184
+ raise DeviceCodeError(
185
+ f"Failed to refresh access token: {error or response.status_code} - {data.get('error_description', '')}",
186
+ error_code=error,
187
+ )
188
+
189
+
190
+ def _parse_token_response(response: httpx.Response) -> dict:
191
+ try:
192
+ return response.json()
193
+ except ValueError as e:
194
+ raise DeviceCodeError(
195
+ f"Failed to parse response from {constants.ENDPOINT}/oauth/token "
196
+ f"(status {response.status_code}): {response.text[:500]}"
197
+ ) from e
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_pagination.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains utilities to handle pagination on Huggingface Hub."""
15
+
16
+ from collections.abc import Iterable
17
+ from typing import Any
18
+
19
+ import httpx
20
+
21
+ from . import get_session, hf_raise_for_status, http_backoff, logging
22
+
23
+
24
+ logger = logging.get_logger(__name__)
25
+
26
+
27
+ def paginate(path: str, params: dict | list[tuple[str, Any]], headers: dict, timeout: float | None = None) -> Iterable:
28
+ """Fetch a list of models/datasets/spaces and paginate through results.
29
+
30
+ This is using the same "Link" header format as GitHub.
31
+ See:
32
+ - https://requests.readthedocs.io/en/latest/api/#requests.Response.links
33
+ - https://docs.github.com/en/rest/guides/traversing-with-pagination#link-header
34
+ """
35
+ session = get_session()
36
+ r = session.get(path, params=params, headers=headers, timeout=timeout)
37
+ hf_raise_for_status(r)
38
+ yield from r.json()
39
+
40
+ # Follow pages
41
+ # Next link already contains query params
42
+ next_page = _get_next_page(r)
43
+ while next_page is not None:
44
+ logger.debug(f"Pagination detected. Requesting next page: {next_page}")
45
+ r = http_backoff("GET", next_page, headers=headers, timeout=timeout)
46
+ hf_raise_for_status(r)
47
+ yield from r.json()
48
+ next_page = _get_next_page(r)
49
+
50
+
51
+ def _get_next_page(response: httpx.Response) -> str | None:
52
+ return response.links.get("next", {}).get("url")
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_parsing.py ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Parsing helpers shared across modules."""
15
+
16
+ import re
17
+ import time
18
+
19
+
20
+ RE_NUMBER_WITH_UNIT = re.compile(r"(\d+)([a-z]+)", re.IGNORECASE)
21
+
22
+ BYTE_UNITS: dict[str, int] = {
23
+ "k": 1_000,
24
+ "kb": 1_000,
25
+ "m": 1_000_000,
26
+ "mb": 1_000_000,
27
+ "g": 1_000_000_000,
28
+ "gb": 1_000_000_000,
29
+ "t": 1_000_000_000_000,
30
+ "tb": 1_000_000_000_000,
31
+ "p": 1_000_000_000_000_000,
32
+ "pb": 1_000_000_000_000_000,
33
+ }
34
+
35
+ TIME_UNITS: dict[str, int] = {
36
+ "s": 1,
37
+ "m": 60,
38
+ "h": 60 * 60,
39
+ "d": 24 * 60 * 60,
40
+ "w": 7 * 24 * 60 * 60,
41
+ "mo": 30 * 24 * 60 * 60,
42
+ "y": 365 * 24 * 60 * 60,
43
+ }
44
+
45
+
46
+ def parse_size(value: str) -> int:
47
+ """Parse a size expressed as a string with digits and unit (like `"10MB"`) to an integer (in bytes)."""
48
+ return _parse_with_unit(value, BYTE_UNITS)
49
+
50
+
51
+ def parse_duration(value: str) -> int:
52
+ """Parse a duration expressed as a string with digits and unit (like `"10s"`) to an integer (in seconds)."""
53
+ return _parse_with_unit(value, TIME_UNITS)
54
+
55
+
56
+ def _parse_with_unit(value: str, units: dict[str, int]) -> int:
57
+ """Parse a numeric value with optional unit."""
58
+ stripped = value.strip()
59
+ if not stripped:
60
+ raise ValueError("Value cannot be empty.")
61
+ try:
62
+ return int(value)
63
+ except ValueError:
64
+ pass
65
+
66
+ match = RE_NUMBER_WITH_UNIT.fullmatch(stripped)
67
+ if not match:
68
+ raise ValueError(f"Invalid value '{value}'. Must match pattern '\\d+[a-z]+' or be a plain number.")
69
+
70
+ number = int(match.group(1))
71
+ unit = match.group(2).lower()
72
+
73
+ if unit not in units:
74
+ raise ValueError(f"Unknown unit '{unit}'. Must be one of {list(units.keys())}.")
75
+
76
+ return number * units[unit]
77
+
78
+
79
+ def format_duration(secs: int | None) -> str:
80
+ """Format a duration in seconds as a short human-readable string (e.g. `"1m 32s"`, `"2h 15m"`, `"45s"`).
81
+
82
+ Returns `"--"` when `secs` is `None` so it can be used directly as a CLI table cell.
83
+ """
84
+ if secs is None:
85
+ return "--"
86
+ secs = int(secs)
87
+ if secs < 60:
88
+ return f"{secs}s"
89
+ if secs < 3600:
90
+ return f"{secs // 60}m {secs % 60}s"
91
+ return f"{secs // 3600}h {(secs % 3600) // 60}m"
92
+
93
+
94
+ def format_timesince(ts: float) -> str:
95
+ """Format timestamp in seconds into a human-readable string, relative to now.
96
+
97
+ Vaguely inspired by Django's `timesince` formatter.
98
+ """
99
+ _TIMESINCE_CHUNKS = (
100
+ # Label, divider, max value
101
+ ("second", 1, 60),
102
+ ("minute", 60, 60),
103
+ ("hour", 60 * 60, 24),
104
+ ("day", 60 * 60 * 24, 6),
105
+ ("week", 60 * 60 * 24 * 7, 6),
106
+ ("month", 60 * 60 * 24 * 30, 11),
107
+ ("year", 60 * 60 * 24 * 365, None),
108
+ )
109
+ delta = time.time() - ts
110
+ if delta < 20:
111
+ return "a few seconds ago"
112
+ for label, divider, max_value in _TIMESINCE_CHUNKS: # noqa: B007
113
+ value = round(delta / divider)
114
+ if max_value is not None and value <= max_value:
115
+ break
116
+ return f"{value} {label}{'s' if value > 1 else ''} ago"
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_paths.py ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Contains utilities to handle paths in Huggingface Hub."""
15
+
16
+ from collections.abc import Callable, Generator, Iterable
17
+ from fnmatch import fnmatchcase
18
+ from pathlib import Path
19
+ from typing import TypeVar
20
+
21
+
22
+ T = TypeVar("T")
23
+
24
+ # Always ignore `.git` and `.cache/huggingface` folders in commits
25
+ DEFAULT_IGNORE_PATTERNS = [
26
+ ".git",
27
+ ".git/*",
28
+ "*/.git",
29
+ "**/.git/**",
30
+ ".cache/huggingface",
31
+ ".cache/huggingface/*",
32
+ "*/.cache/huggingface",
33
+ "**/.cache/huggingface/**",
34
+ ]
35
+ # Forbidden to commit these folders
36
+ FORBIDDEN_FOLDERS = [".git", ".cache"]
37
+
38
+
39
+ def filter_repo_objects(
40
+ items: Iterable[T],
41
+ *,
42
+ allow_patterns: list[str] | str | None = None,
43
+ ignore_patterns: list[str] | str | None = None,
44
+ key: Callable[[T], str] | None = None,
45
+ ) -> Generator[T, None, None]:
46
+ """Filter repo objects based on an allowlist and a denylist.
47
+
48
+ Input must be a list of paths (`str` or `Path`) or a list of arbitrary objects.
49
+ In the later case, `key` must be provided and specifies a function of one argument
50
+ that is used to extract a path from each element in iterable.
51
+
52
+ Patterns are Standard Wildcards (globbing patterns), NOT regular expressions. The pattern matching is based on
53
+ Python's `fnmatch.fnmatchcase`, so it is case-sensitive on every platform. Backslashes are treated as path separators
54
+ and normalized to forward slashes in both patterns and paths before matching, so patterns built with `os.path.join`
55
+ on Windows work as expected.
56
+
57
+ Note that it matches `*` across path boundaries, unlike traditional Unix shell globbing. For example, `"data/*.json"`
58
+ will match both `data/file.json` and `data/subdir/file.json`.
59
+
60
+ See https://docs.python.org/3/library/fnmatch.html for more details.
61
+
62
+ Args:
63
+ items (`Iterable`):
64
+ List of items to filter.
65
+ allow_patterns (`str` or `list[str]`, *optional*):
66
+ Patterns constituting the allowlist. If provided, item paths must match at
67
+ least one pattern from the allowlist.
68
+ ignore_patterns (`str` or `list[str]`, *optional*):
69
+ Patterns constituting the denylist. If provided, item paths must not match
70
+ any patterns from the denylist.
71
+ key (`Callable[[T], str]`, *optional*):
72
+ Single-argument function to extract a path from each item. If not provided,
73
+ the `items` must already be `str` or `Path`.
74
+
75
+ Returns:
76
+ Filtered list of objects, as a generator.
77
+
78
+ Raises:
79
+ :class:`ValueError`:
80
+ If `key` is not provided and items are not `str` or `Path`.
81
+
82
+ Example usage with paths:
83
+ ```python
84
+ >>> # Filter only PDFs that are not hidden.
85
+ >>> list(filter_repo_objects(
86
+ ... ["aaa.pdf", "bbb.jpg", ".ccc.pdf", ".ddd.png"],
87
+ ... allow_patterns=["*.pdf"],
88
+ ... ignore_patterns=[".*"],
89
+ ... ))
90
+ ["aaa.pdf"]
91
+ ```
92
+
93
+ Example usage with objects:
94
+ ```python
95
+ >>> list(filter_repo_objects(
96
+ ... [
97
+ ... CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf")
98
+ ... CommitOperationAdd(path_or_fileobj="/tmp/bbb.jpg", path_in_repo="bbb.jpg")
99
+ ... CommitOperationAdd(path_or_fileobj="/tmp/.ccc.pdf", path_in_repo=".ccc.pdf")
100
+ ... CommitOperationAdd(path_or_fileobj="/tmp/.ddd.png", path_in_repo=".ddd.png")
101
+ ... ],
102
+ ... allow_patterns=["*.pdf"],
103
+ ... ignore_patterns=[".*"],
104
+ ... key=lambda x: x.path_in_repo
105
+ ... ))
106
+ [CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf")]
107
+ ```
108
+ """
109
+ if isinstance(allow_patterns, str):
110
+ allow_patterns = [allow_patterns]
111
+
112
+ if isinstance(ignore_patterns, str):
113
+ ignore_patterns = [ignore_patterns]
114
+
115
+ if allow_patterns is not None:
116
+ allow_patterns = [_add_wildcard_to_directories(_normalize_separators(p)) for p in allow_patterns]
117
+ if ignore_patterns is not None:
118
+ ignore_patterns = [_add_wildcard_to_directories(_normalize_separators(p)) for p in ignore_patterns]
119
+
120
+ if key is None:
121
+
122
+ def _identity(item: T) -> str:
123
+ if isinstance(item, str):
124
+ return item
125
+ if isinstance(item, Path):
126
+ return str(item)
127
+ raise ValueError(f"Please provide `key` argument in `filter_repo_objects`: `{item}` is not a string.")
128
+
129
+ key = _identity # Items must be `str` or `Path`, otherwise raise ValueError
130
+
131
+ for item in items:
132
+ path = _normalize_separators(key(item))
133
+
134
+ # Skip if there's an allowlist and path doesn't match any
135
+ if allow_patterns is not None and not any(fnmatchcase(path, r) for r in allow_patterns):
136
+ continue
137
+
138
+ # Skip if there's a denylist and path matches any
139
+ if ignore_patterns is not None and any(fnmatchcase(path, r) for r in ignore_patterns):
140
+ continue
141
+
142
+ yield item
143
+
144
+
145
+ def _normalize_separators(value: str | Path) -> str:
146
+ # Repo paths always use `/` and `\` is not an fnmatch escape character, so treating backslashes as path separators
147
+ # (e.g. patterns built with `os.path.join` on Windows) is safe.
148
+ # `value` can be a `Path` if a custom `key` returns one; coerce to `str` first.
149
+ return str(value).replace("\\", "/")
150
+
151
+
152
+ def _add_wildcard_to_directories(pattern: str) -> str:
153
+ if pattern.endswith("/"):
154
+ return pattern + "*"
155
+ return pattern
.venv/lib/python3.13/site-packages/huggingface_hub/utils/_runtime.py ADDED
@@ -0,0 +1,445 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022-present, the HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Check presence of installed packages at runtime."""
15
+
16
+ import importlib.metadata
17
+ import importlib.util
18
+ import os
19
+ import platform
20
+ import sys
21
+ import warnings
22
+ from pathlib import Path
23
+ from typing import Any, Literal
24
+
25
+ from .. import __version__, constants
26
+
27
+
28
+ _PY_VERSION: str = sys.version.split()[0].rstrip("+")
29
+
30
+ _package_versions = {}
31
+
32
+ _CANDIDATES = {
33
+ "aiohttp": {"aiohttp"},
34
+ "fastai": {"fastai"},
35
+ "fastapi": {"fastapi"},
36
+ "fastcore": {"fastcore"},
37
+ "gradio": {"gradio"},
38
+ "graphviz": {"graphviz"},
39
+ "hf_xet": {"hf_xet"},
40
+ "jinja": {"Jinja2"},
41
+ "httpx": {"httpx"},
42
+ "keras": {"keras"},
43
+ "numpy": {"numpy"},
44
+ "pillow": {"Pillow"},
45
+ "pydantic": {"pydantic"},
46
+ "pydot": {"pydot"},
47
+ "safetensors": {"safetensors"},
48
+ "tensorboard": {"tensorboardX"},
49
+ "tensorflow": (
50
+ "tensorflow",
51
+ "tensorflow-cpu",
52
+ "tensorflow-gpu",
53
+ "tf-nightly",
54
+ "tf-nightly-cpu",
55
+ "tf-nightly-gpu",
56
+ "intel-tensorflow",
57
+ "intel-tensorflow-avx512",
58
+ "tensorflow-rocm",
59
+ "tensorflow-macos",
60
+ ),
61
+ "torch": {"torch"},
62
+ }
63
+
64
+ # Check once at runtime
65
+ for candidate_name, package_names in _CANDIDATES.items():
66
+ _package_versions[candidate_name] = "N/A"
67
+ for name in package_names:
68
+ try:
69
+ _package_versions[candidate_name] = importlib.metadata.version(name)
70
+ break
71
+ except importlib.metadata.PackageNotFoundError:
72
+ pass
73
+
74
+
75
+ def _get_version(package_name: str) -> str:
76
+ return _package_versions.get(package_name, "N/A")
77
+
78
+
79
+ def is_package_available(package_name: str) -> bool:
80
+ return _get_version(package_name) != "N/A"
81
+
82
+
83
+ # Python
84
+ def get_python_version() -> str:
85
+ return _PY_VERSION
86
+
87
+
88
+ # Huggingface Hub
89
+ def get_hf_hub_version() -> str:
90
+ return __version__
91
+
92
+
93
+ # aiohttp
94
+ def is_aiohttp_available() -> bool:
95
+ return is_package_available("aiohttp")
96
+
97
+
98
+ def get_aiohttp_version() -> str:
99
+ return _get_version("aiohttp")
100
+
101
+
102
+ # FastAI
103
+ def is_fastai_available() -> bool:
104
+ return is_package_available("fastai")
105
+
106
+
107
+ def get_fastai_version() -> str:
108
+ return _get_version("fastai")
109
+
110
+
111
+ # FastAPI
112
+ def is_fastapi_available() -> bool:
113
+ return is_package_available("fastapi")
114
+
115
+
116
+ def get_fastapi_version() -> str:
117
+ return _get_version("fastapi")
118
+
119
+
120
+ # Fastcore
121
+ def is_fastcore_available() -> bool:
122
+ return is_package_available("fastcore")
123
+
124
+
125
+ def get_fastcore_version() -> str:
126
+ return _get_version("fastcore")
127
+
128
+
129
+ # FastAI
130
+ def is_gradio_available() -> bool:
131
+ return is_package_available("gradio")
132
+
133
+
134
+ def get_gradio_version() -> str:
135
+ return _get_version("gradio")
136
+
137
+
138
+ # Graphviz
139
+ def is_graphviz_available() -> bool:
140
+ return is_package_available("graphviz")
141
+
142
+
143
+ def get_graphviz_version() -> str:
144
+ return _get_version("graphviz")
145
+
146
+
147
+ # httpx
148
+ def is_httpx_available() -> bool:
149
+ return is_package_available("httpx")
150
+
151
+
152
+ def get_httpx_version() -> str:
153
+ return _get_version("httpx")
154
+
155
+
156
+ # xet
157
+ def is_xet_available() -> bool:
158
+ # since hf_xet is automatically used if available, allow explicit disabling via environment variable
159
+ if constants.HF_HUB_DISABLE_XET:
160
+ return False
161
+
162
+ return is_package_available("hf_xet")
163
+
164
+
165
+ def get_xet_version() -> str:
166
+ return _get_version("hf_xet")
167
+
168
+
169
+ # keras
170
+ def is_keras_available() -> bool:
171
+ return is_package_available("keras")
172
+
173
+
174
+ def get_keras_version() -> str:
175
+ return _get_version("keras")
176
+
177
+
178
+ # Numpy
179
+ def is_numpy_available() -> bool:
180
+ return is_package_available("numpy")
181
+
182
+
183
+ def get_numpy_version() -> str:
184
+ return _get_version("numpy")
185
+
186
+
187
+ # Jinja
188
+ def is_jinja_available() -> bool:
189
+ return is_package_available("jinja")
190
+
191
+
192
+ def get_jinja_version() -> str:
193
+ return _get_version("jinja")
194
+
195
+
196
+ # Pillow
197
+ def is_pillow_available() -> bool:
198
+ return is_package_available("pillow")
199
+
200
+
201
+ def get_pillow_version() -> str:
202
+ return _get_version("pillow")
203
+
204
+
205
+ # Pydantic
206
+ def is_pydantic_available() -> bool:
207
+ if not is_package_available("pydantic"):
208
+ return False
209
+ # For Pydantic, we add an extra check to test whether it is correctly installed or not. If both pydantic 2.x and
210
+ # typing_extensions<=4.5.0 are installed, then pydantic will fail at import time. This should not happen when
211
+ # it is installed with `pip install huggingface_hub[inference]` but it can happen when it is installed manually
212
+ # by the user in an environment that we don't control.
213
+ #
214
+ # Usually we won't need to do this kind of check on optional dependencies. However, pydantic is a special case
215
+ # as it is automatically imported when doing `from huggingface_hub import ...` even if the user doesn't use it.
216
+ #
217
+ # See https://github.com/huggingface/huggingface_hub/pull/1829 for more details.
218
+ try:
219
+ from pydantic import validator # noqa: F401
220
+ except ImportError:
221
+ # Example: "ImportError: cannot import name 'TypeAliasType' from 'typing_extensions'"
222
+ warnings.warn(
223
+ "Pydantic is installed but cannot be imported. Please check your installation. `huggingface_hub` will "
224
+ "default to not using Pydantic. Error message: '{e}'"
225
+ )
226
+ return False
227
+ return True
228
+
229
+
230
+ def get_pydantic_version() -> str:
231
+ return _get_version("pydantic")
232
+
233
+
234
+ # Pydot
235
+ def is_pydot_available() -> bool:
236
+ return is_package_available("pydot")
237
+
238
+
239
+ def get_pydot_version() -> str:
240
+ return _get_version("pydot")
241
+
242
+
243
+ # Tensorboard
244
+ def is_tensorboard_available() -> bool:
245
+ return is_package_available("tensorboard")
246
+
247
+
248
+ def get_tensorboard_version() -> str:
249
+ return _get_version("tensorboard")
250
+
251
+
252
+ # Tensorflow
253
+ def is_tf_available() -> bool:
254
+ return is_package_available("tensorflow")
255
+
256
+
257
+ def get_tf_version() -> str:
258
+ return _get_version("tensorflow")
259
+
260
+
261
+ # Torch
262
+ def is_torch_available() -> bool:
263
+ return is_package_available("torch")
264
+
265
+
266
+ def get_torch_version() -> str:
267
+ return _get_version("torch")
268
+
269
+
270
+ # Safetensors
271
+ def is_safetensors_available() -> bool:
272
+ return is_package_available("safetensors")
273
+
274
+
275
+ # Shell-related helpers
276
+ try:
277
+ # Set to `True` if script is running in a Google Colab notebook.
278
+ # If running in Google Colab, git credential store is set globally which makes the
279
+ # warning disappear. See https://github.com/huggingface/huggingface_hub/issues/1043
280
+ #
281
+ # Taken from https://stackoverflow.com/a/63519730.
282
+ _is_google_colab = "google.colab" in str(get_ipython()) # type: ignore # noqa: F821
283
+ except NameError:
284
+ _is_google_colab = False
285
+
286
+
287
+ def is_notebook() -> bool:
288
+ """Return `True` if code is executed in a notebook (Jupyter, Colab, QTconsole).
289
+
290
+ Taken from https://stackoverflow.com/a/39662359.
291
+ Adapted to make it work with Google colab as well.
292
+ """
293
+ try:
294
+ shell_class = get_ipython().__class__ # type: ignore # noqa: F821
295
+ for parent_class in shell_class.__mro__: # e.g. "is subclass of"
296
+ if parent_class.__name__ == "ZMQInteractiveShell":
297
+ return True # Jupyter notebook, Google colab or qtconsole
298
+ return False
299
+ except NameError:
300
+ return False # Probably standard Python interpreter
301
+
302
+
303
+ def is_google_colab() -> bool:
304
+ """Return `True` if code is executed in a Google colab.
305
+
306
+ Taken from https://stackoverflow.com/a/63519730.
307
+ """
308
+ return _is_google_colab
309
+
310
+
311
+ def is_colab_enterprise() -> bool:
312
+ """Return `True` if code is executed in a Google Colab Enterprise environment."""
313
+ return os.environ.get("VERTEX_PRODUCT") == "COLAB_ENTERPRISE"
314
+
315
+
316
+ # Check how huggingface_hub has been installed
317
+
318
+
319
+ def installation_method() -> Literal["brew", "hf_installer", "pip", "unknown"]:
320
+ """Return the installation method of the current environment.
321
+
322
+ - "hf_installer" if installed via the official installer script
323
+ - "brew" if installed via Homebrew
324
+ - "pip" if pip is available (default fallback for standard Python environments)
325
+ - "unknown" otherwise
326
+ """
327
+ # hf_installer check must come first: the installer creates a venv using the
328
+ # system Python, which may be Homebrew's. Checking brew first would false-positive
329
+ # when the resolved sys.executable points to /opt/homebrew/... inside a venv.
330
+ if _is_hf_installer_installation():
331
+ return "hf_installer"
332
+ if _is_brew_installation():
333
+ return "brew"
334
+ if _is_pip_available():
335
+ return "pip"
336
+ return "unknown"
337
+
338
+
339
+ def _is_brew_installation() -> bool:
340
+ """Check if running from a Homebrew installation.
341
+
342
+ Homebrew installs the `hf` formula into a Cellar directory and creates a
343
+ libexec virtualenv at e.g. /opt/homebrew/Cellar/hf/0.30.0/libexec/.
344
+ We check `sys.prefix` (the venv/prefix root) for "/Cellar/hf/" rather
345
+ than checking `sys.executable` — the latter resolves to Homebrew's Python
346
+ (e.g. /opt/homebrew/Cellar/python@3.12/...) even for non-brew installs
347
+ when the system Python happens to come from Homebrew.
348
+ """
349
+ prefix = str(Path(sys.prefix).resolve())
350
+ return "/Cellar/hf/" in prefix
351
+
352
+
353
+ def _is_hf_installer_installation() -> bool:
354
+ """Return `True` if the current environment was set up via the official hf installer script.
355
+
356
+ i.e. using one of
357
+ curl -LsSf https://hf.co/cli/install.sh | bash
358
+ powershell -ExecutionPolicy ByPass -c "irm https://hf.co/cli/install.ps1 | iex"
359
+ """
360
+ venv = sys.prefix # points to venv root if active
361
+ marker = Path(venv) / ".hf_installer_marker"
362
+ return marker.exists()
363
+
364
+
365
+ def _is_pip_available() -> bool:
366
+ """Return `True` if pip is importable in the current environment."""
367
+ return importlib.util.find_spec("pip") is not None
368
+
369
+
370
+ def dump_environment_info() -> dict[str, Any]:
371
+ """Dump information about the machine to help debugging issues.
372
+
373
+ Similar helper exist in:
374
+ - `datasets` (https://github.com/huggingface/datasets/blob/main/src/datasets/commands/env.py)
375
+ - `diffusers` (https://github.com/huggingface/diffusers/blob/main/src/diffusers/commands/env.py)
376
+ - `transformers` (https://github.com/huggingface/transformers/blob/main/src/transformers/commands/env.py)
377
+ """
378
+ from huggingface_hub import get_token, whoami
379
+ from huggingface_hub.utils import is_agent, list_credential_helpers
380
+
381
+ token = get_token()
382
+
383
+ # Generic machine info
384
+ info: dict[str, Any] = {
385
+ "huggingface_hub version": get_hf_hub_version(),
386
+ "Platform": platform.platform(),
387
+ "Python version": get_python_version(),
388
+ }
389
+
390
+ # Interpreter info
391
+ try:
392
+ shell_class = get_ipython().__class__ # type: ignore # noqa: F821
393
+ info["Running in iPython ?"] = "Yes"
394
+ info["iPython shell"] = shell_class.__name__
395
+ except NameError:
396
+ info["Running in iPython ?"] = "No"
397
+ info["Running in notebook ?"] = "Yes" if is_notebook() else "No"
398
+ info["Running in Google Colab ?"] = "Yes" if is_google_colab() else "No"
399
+ info["Running in Google Colab Enterprise ?"] = "Yes" if is_colab_enterprise() else "No"
400
+ # Login info
401
+ info["Token path ?"] = constants.HF_TOKEN_PATH
402
+ info["Has saved token ?"] = token is not None
403
+ if token is not None:
404
+ try:
405
+ info["Who am I ?"] = whoami()["name"]
406
+ except Exception:
407
+ pass
408
+
409
+ try:
410
+ info["Configured git credential helpers"] = ", ".join(list_credential_helpers())
411
+ except Exception:
412
+ pass
413
+
414
+ info["Run by AI agent ?"] = "Yes" if is_agent() else "No"
415
+
416
+ # How huggingface_hub has been installed?
417
+ info["Installation method"] = installation_method()
418
+
419
+ # Installed dependencies
420
+ info["httpx"] = get_httpx_version()
421
+ info["hf_xet"] = get_xet_version()
422
+ info["gradio"] = get_gradio_version()
423
+ info["tensorboard"] = get_tensorboard_version()
424
+
425
+ # Environment variables
426
+ info["ENDPOINT"] = constants.ENDPOINT
427
+ info["HF_HUB_CACHE"] = constants.HF_HUB_CACHE
428
+ info["HF_ASSETS_CACHE"] = constants.HF_ASSETS_CACHE
429
+ info["HF_TOKEN_PATH"] = constants.HF_TOKEN_PATH
430
+ info["HF_STORED_TOKENS_PATH"] = constants.HF_STORED_TOKENS_PATH
431
+ info["HF_HUB_OFFLINE"] = constants.HF_HUB_OFFLINE
432
+ info["HF_HUB_DISABLE_TELEMETRY"] = constants.HF_HUB_DISABLE_TELEMETRY
433
+ info["HF_HUB_DISABLE_PROGRESS_BARS"] = constants.HF_HUB_DISABLE_PROGRESS_BARS
434
+ info["HF_HUB_DISABLE_SYMLINKS"] = constants.HF_HUB_DISABLE_SYMLINKS
435
+ info["HF_HUB_DISABLE_SYMLINKS_WARNING"] = constants.HF_HUB_DISABLE_SYMLINKS_WARNING
436
+ info["HF_HUB_DISABLE_EXPERIMENTAL_WARNING"] = constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING
437
+ info["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = constants.HF_HUB_DISABLE_IMPLICIT_TOKEN
438
+ info["HF_HUB_DISABLE_XET"] = constants.HF_HUB_DISABLE_XET
439
+ info["HF_HUB_ETAG_TIMEOUT"] = constants.HF_HUB_ETAG_TIMEOUT
440
+ info["HF_HUB_DOWNLOAD_TIMEOUT"] = constants.HF_HUB_DOWNLOAD_TIMEOUT
441
+ info["HF_XET_HIGH_PERFORMANCE"] = constants.HF_XET_HIGH_PERFORMANCE
442
+
443
+ print("\nCopy-and-paste the text below in your GitHub issue.\n")
444
+ print("\n".join([f"- {prop}: {val}" for prop, val in info.items()]) + "\n")
445
+ return info