diff --git a/.gitattributes b/.gitattributes index fbd4edc459621463df86b999d9359230b9d1aa63..f291a2fd56ab12a3fa7cc0eb61a5913e274d393a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -59,3 +59,6 @@ venv/lib/python3.12/site-packages/cuda/bindings/nvrtc.cpython-312-x86_64-linux-g venv/lib/python3.12/site-packages/cuda/bindings/nvvm.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text venv/lib/python3.12/site-packages/cuda/bindings/runtime.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text venv/lib/python3.12/site-packages/hf_xet/hf_xet.abi3.so filter=lfs diff=lfs merge=lfs -text +venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hf_api.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text +venv/lib/python3.12/site-packages/huggingface_hub/inference/__pycache__/_client.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text +venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__pycache__/_async_client.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text diff --git a/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/INSTALLER b/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/METADATA b/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..8056834e6714b089ef49847820064a1ae4b041fd --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/METADATA @@ -0,0 +1,625 @@ +Metadata-Version: 2.4 +Name: httpcore +Version: 1.0.9 +Summary: A minimal low-level HTTP client. +Project-URL: Documentation, https://www.encode.io/httpcore +Project-URL: Homepage, https://www.encode.io/httpcore/ +Project-URL: Source, https://github.com/encode/httpcore +Author-email: Tom Christie +License-Expression: BSD-3-Clause +License-File: LICENSE.md +Classifier: Development Status :: 3 - Alpha +Classifier: Environment :: Web Environment +Classifier: Framework :: AsyncIO +Classifier: Framework :: Trio +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Topic :: Internet :: WWW/HTTP +Requires-Python: >=3.8 +Requires-Dist: certifi +Requires-Dist: h11>=0.16 +Provides-Extra: asyncio +Requires-Dist: anyio<5.0,>=4.0; extra == 'asyncio' +Provides-Extra: http2 +Requires-Dist: h2<5,>=3; extra == 'http2' +Provides-Extra: socks +Requires-Dist: socksio==1.*; extra == 'socks' +Provides-Extra: trio +Requires-Dist: trio<1.0,>=0.22.0; extra == 'trio' +Description-Content-Type: text/markdown + +# HTTP Core + +[![Test Suite](https://github.com/encode/httpcore/workflows/Test%20Suite/badge.svg)](https://github.com/encode/httpcore/actions) +[![Package version](https://badge.fury.io/py/httpcore.svg)](https://pypi.org/project/httpcore/) + +> *Do one thing, and do it well.* + +The HTTP Core package provides a minimal low-level HTTP client, which does +one thing only. Sending HTTP requests. + +It does not provide any high level model abstractions over the API, +does not handle redirects, multipart uploads, building authentication headers, +transparent HTTP caching, URL parsing, session cookie handling, +content or charset decoding, handling JSON, environment based configuration +defaults, or any of that Jazz. + +Some things HTTP Core does do: + +* Sending HTTP requests. +* Thread-safe / task-safe connection pooling. +* HTTP(S) proxy & SOCKS proxy support. +* Supports HTTP/1.1 and HTTP/2. +* Provides both sync and async interfaces. +* Async backend support for `asyncio` and `trio`. + +## Requirements + +Python 3.8+ + +## Installation + +For HTTP/1.1 only support, install with: + +```shell +$ pip install httpcore +``` + +There are also a number of optional extras available... + +```shell +$ pip install httpcore['asyncio,trio,http2,socks'] +``` + +## Sending requests + +Send an HTTP request: + +```python +import httpcore + +response = httpcore.request("GET", "https://www.example.com/") + +print(response) +# +print(response.status) +# 200 +print(response.headers) +# [(b'Accept-Ranges', b'bytes'), (b'Age', b'557328'), (b'Cache-Control', b'max-age=604800'), ...] +print(response.content) +# b'\n\n\nExample Domain\n\n\n ...' +``` + +The top-level `httpcore.request()` function is provided for convenience. In practice whenever you're working with `httpcore` you'll want to use the connection pooling functionality that it provides. + +```python +import httpcore + +http = httpcore.ConnectionPool() +response = http.request("GET", "https://www.example.com/") +``` + +Once you're ready to get going, [head over to the documentation](https://www.encode.io/httpcore/). + +## Motivation + +You *probably* don't want to be using HTTP Core directly. It might make sense if +you're writing something like a proxy service in Python, and you just want +something at the lowest possible level, but more typically you'll want to use +a higher level client library, such as `httpx`. + +The motivation for `httpcore` is: + +* To provide a reusable low-level client library, that other packages can then build on top of. +* To provide a *really clear interface split* between the networking code and client logic, + so that each is easier to understand and reason about in isolation. + +## Dependencies + +The `httpcore` package has the following dependencies... + +* `h11` +* `certifi` + +And the following optional extras... + +* `anyio` - Required by `pip install httpcore['asyncio']`. +* `trio` - Required by `pip install httpcore['trio']`. +* `h2` - Required by `pip install httpcore['http2']`. +* `socksio` - Required by `pip install httpcore['socks']`. + +## Versioning + +We use [SEMVER for our versioning policy](https://semver.org/). + +For changes between package versions please see our [project changelog](CHANGELOG.md). + +We recommend pinning your requirements either the most current major version, or a more specific version range: + +```python +pip install 'httpcore==1.*' +``` +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## Version 1.0.9 (April 24th, 2025) + +- Resolve https://github.com/advisories/GHSA-vqfr-h8mv-ghfj with h11 dependency update. (#1008) + +## Version 1.0.8 (April 11th, 2025) + +- Fix `AttributeError` when importing on Python 3.14. (#1005) + +## Version 1.0.7 (November 15th, 2024) + +- Support `proxy=…` configuration on `ConnectionPool()`. (#974) + +## Version 1.0.6 (October 1st, 2024) + +- Relax `trio` dependency pinning. (#956) +- Handle `trio` raising `NotImplementedError` on unsupported platforms. (#955) +- Handle mapping `ssl.SSLError` to `httpcore.ConnectError`. (#918) + +## 1.0.5 (March 27th, 2024) + +- Handle `EndOfStream` exception for anyio backend. (#899) +- Allow trio `0.25.*` series in package dependancies. (#903) + +## 1.0.4 (February 21st, 2024) + +- Add `target` request extension. (#888) +- Fix support for connection `Upgrade` and `CONNECT` when some data in the stream has been read. (#882) + +## 1.0.3 (February 13th, 2024) + +- Fix support for async cancellations. (#880) +- Fix trace extension when used with socks proxy. (#849) +- Fix SSL context for connections using the "wss" scheme (#869) + +## 1.0.2 (November 10th, 2023) + +- Fix `float("inf")` timeouts in `Event.wait` function. (#846) + +## 1.0.1 (November 3rd, 2023) + +- Fix pool timeout to account for the total time spent retrying. (#823) +- Raise a neater RuntimeError when the correct async deps are not installed. (#826) +- Add support for synchronous TLS-in-TLS streams. (#840) + +## 1.0.0 (October 6th, 2023) + +From version 1.0 our async support is now optional, as the package has minimal dependencies by default. + +For async support use either `pip install 'httpcore[asyncio]'` or `pip install 'httpcore[trio]'`. + +The project versioning policy is now explicitly governed by SEMVER. See https://semver.org/. + +- Async support becomes fully optional. (#809) +- Add support for Python 3.12. (#807) + +## 0.18.0 (September 8th, 2023) + +- Add support for HTTPS proxies. (#745, #786) +- Drop Python 3.7 support. (#727) +- Handle `sni_hostname` extension with SOCKS proxy. (#774) +- Handle HTTP/1.1 half-closed connections gracefully. (#641) +- Change the type of `Extensions` from `Mapping[Str, Any]` to `MutableMapping[Str, Any]`. (#762) + +## 0.17.3 (July 5th, 2023) + +- Support async cancellations, ensuring that the connection pool is left in a clean state when cancellations occur. (#726) +- The networking backend interface has [been added to the public API](https://www.encode.io/httpcore/network-backends). Some classes which were previously private implementation detail are now part of the top-level public API. (#699) +- Graceful handling of HTTP/2 GoAway frames, with requests being transparently retried on a new connection. (#730) +- Add exceptions when a synchronous `trace callback` is passed to an asynchronous request or an asynchronous `trace callback` is passed to a synchronous request. (#717) +- Drop Python 3.7 support. (#727) + +## 0.17.2 (May 23th, 2023) + +- Add `socket_options` argument to `ConnectionPool` and `HTTProxy` classes. (#668) +- Improve logging with per-module logger names. (#690) +- Add `sni_hostname` request extension. (#696) +- Resolve race condition during import of `anyio` package. (#692) +- Enable TCP_NODELAY for all synchronous sockets. (#651) + +## 0.17.1 (May 17th, 2023) + +- If 'retries' is set, then allow retries if an SSL handshake error occurs. (#669) +- Improve correctness of tracebacks on network exceptions, by raising properly chained exceptions. (#678) +- Prevent connection-hanging behaviour when HTTP/2 connections are closed by a server-sent 'GoAway' frame. (#679) +- Fix edge-case exception when removing requests from the connection pool. (#680) +- Fix pool timeout edge-case. (#688) + +## 0.17.0 (March 16th, 2023) + +- Add DEBUG level logging. (#648) +- Respect HTTP/2 max concurrent streams when settings updates are sent by server. (#652) +- Increase the allowable HTTP header size to 100kB. (#647) +- Add `retries` option to SOCKS proxy classes. (#643) + +## 0.16.3 (December 20th, 2022) + +- Allow `ws` and `wss` schemes. Allows us to properly support websocket upgrade connections. (#625) +- Forwarding HTTP proxies use a connection-per-remote-host. Required by some proxy implementations. (#637) +- Don't raise `RuntimeError` when closing a connection pool with active connections. Removes some error cases when cancellations are used. (#631) +- Lazy import `anyio`, so that it's no longer a hard dependancy, and isn't imported if unused. (#639) + +## 0.16.2 (November 25th, 2022) + +- Revert 'Fix async cancellation behaviour', which introduced race conditions. (#627) +- Raise `RuntimeError` if attempting to us UNIX domain sockets on Windows. (#619) + +## 0.16.1 (November 17th, 2022) + +- Fix HTTP/1.1 interim informational responses, such as "100 Continue". (#605) + +## 0.16.0 (October 11th, 2022) + +- Support HTTP/1.1 informational responses. (#581) +- Fix async cancellation behaviour. (#580) +- Support `h11` 0.14. (#579) + +## 0.15.0 (May 17th, 2022) + +- Drop Python 3.6 support (#535) +- Ensure HTTP proxy CONNECT requests include `timeout` configuration. (#506) +- Switch to explicit `typing.Optional` for type hints. (#513) +- For `trio` map OSError exceptions to `ConnectError`. (#543) + +## 0.14.7 (February 4th, 2022) + +- Requests which raise a PoolTimeout need to be removed from the pool queue. (#502) +- Fix AttributeError that happened when Socks5Connection were terminated. (#501) + +## 0.14.6 (February 1st, 2022) + +- Fix SOCKS support for `http://` URLs. (#492) +- Resolve race condition around exceptions during streaming a response. (#491) + +## 0.14.5 (January 18th, 2022) + +- SOCKS proxy support. (#478) +- Add proxy_auth argument to HTTPProxy. (#481) +- Improve error message on 'RemoteProtocolError' exception when server disconnects without sending a response. (#479) + +## 0.14.4 (January 5th, 2022) + +- Support HTTP/2 on HTTPS tunnelling proxies. (#468) +- Fix proxy headers missing on HTTP forwarding. (#456) +- Only instantiate SSL context if required. (#457) +- More robust HTTP/2 handling. (#253, #439, #440, #441) + +## 0.14.3 (November 17th, 2021) + +- Fix race condition when removing closed connections from the pool. (#437) + +## 0.14.2 (November 16th, 2021) + +- Failed connections no longer remain in the pool. (Pull #433) + +## 0.14.1 (November 12th, 2021) + +- `max_connections` becomes optional. (Pull #429) +- `certifi` is now included in the install dependancies. (Pull #428) +- `h2` is now strictly optional. (Pull #428) + +## 0.14.0 (November 11th, 2021) + +The 0.14 release is a complete reworking of `httpcore`, comprehensively addressing some underlying issues in the connection pooling, as well as substantially redesigning the API to be more user friendly. + +Some of the lower-level API design also makes the components more easily testable in isolation, and the package now has 100% test coverage. + +See [discussion #419](https://github.com/encode/httpcore/discussions/419) for a little more background. + +There's some other neat bits in there too, such as the "trace" extension, which gives a hook into inspecting the internal events that occur during the request/response cycle. This extension is needed for the HTTPX cli, in order to... + +* Log the point at which the connection is established, and the IP/port on which it is made. +* Determine if the outgoing request should log as HTTP/1.1 or HTTP/2, rather than having to assume it's HTTP/2 if the --http2 flag was passed. (Which may not actually be true.) +* Log SSL version info / certificate info. + +Note that `curio` support is not currently available in 0.14.0. If you're using `httpcore` with `curio` please get in touch, so we can assess if we ought to prioritize it as a feature or not. + +## 0.13.7 (September 13th, 2021) + +- Fix broken error messaging when URL scheme is missing, or a non HTTP(S) scheme is used. (Pull #403) + +## 0.13.6 (June 15th, 2021) + +### Fixed + +- Close sockets when read or write timeouts occur. (Pull #365) + +## 0.13.5 (June 14th, 2021) + +### Fixed + +- Resolved niggles with AnyIO EOF behaviours. (Pull #358, #362) + +## 0.13.4 (June 9th, 2021) + +### Added + +- Improved error messaging when URL scheme is missing, or a non HTTP(S) scheme is used. (Pull #354) + +### Fixed + +- Switched to `anyio` as the default backend implementation when running with `asyncio`. Resolves some awkward [TLS timeout issues](https://github.com/encode/httpx/discussions/1511). + +## 0.13.3 (May 6th, 2021) + +### Added + +- Support HTTP/2 prior knowledge, using `httpcore.SyncConnectionPool(http1=False)`. (Pull #333) + +### Fixed + +- Handle cases where environment does not provide `select.poll` support. (Pull #331) + +## 0.13.2 (April 29th, 2021) + +### Added + +- Improve error message for specific case of `RemoteProtocolError` where server disconnects without sending a response. (Pull #313) + +## 0.13.1 (April 28th, 2021) + +### Fixed + +- More resiliant testing for closed connections. (Pull #311) +- Don't raise exceptions on ungraceful connection closes. (Pull #310) + +## 0.13.0 (April 21st, 2021) + +The 0.13 release updates the core API in order to match the HTTPX Transport API, +introduced in HTTPX 0.18 onwards. + +An example of making requests with the new interface is: + +```python +with httpcore.SyncConnectionPool() as http: + status_code, headers, stream, extensions = http.handle_request( + method=b'GET', + url=(b'https', b'example.org', 443, b'/'), + headers=[(b'host', b'example.org'), (b'user-agent', b'httpcore')] + stream=httpcore.ByteStream(b''), + extensions={} + ) + body = stream.read() + print(status_code, body) +``` + +### Changed + +- The `.request()` method is now `handle_request()`. (Pull #296) +- The `.arequest()` method is now `.handle_async_request()`. (Pull #296) +- The `headers` argument is no longer optional. (Pull #296) +- The `stream` argument is no longer optional. (Pull #296) +- The `ext` argument is now named `extensions`, and is no longer optional. (Pull #296) +- The `"reason"` extension keyword is now named `"reason_phrase"`. (Pull #296) +- The `"reason_phrase"` and `"http_version"` extensions now use byte strings for their values. (Pull #296) +- The `httpcore.PlainByteStream()` class becomes `httpcore.ByteStream()`. (Pull #296) + +### Added + +- Streams now support a `.read()` interface. (Pull #296) + +### Fixed + +- Task cancellation no longer leaks connections from the connection pool. (Pull #305) + +## 0.12.3 (December 7th, 2020) + +### Fixed + +- Abort SSL connections on close rather than waiting for remote EOF when using `asyncio`. (Pull #167) +- Fix exception raised in case of connect timeouts when using the `anyio` backend. (Pull #236) +- Fix `Host` header precedence for `:authority` in HTTP/2. (Pull #241, #243) +- Handle extra edge case when detecting for socket readability when using `asyncio`. (Pull #242, #244) +- Fix `asyncio` SSL warning when using proxy tunneling. (Pull #249) + +## 0.12.2 (November 20th, 2020) + +### Fixed + +- Properly wrap connect errors on the asyncio backend. (Pull #235) +- Fix `ImportError` occurring on Python 3.9 when using the HTTP/1.1 sync client in a multithreaded context. (Pull #237) + +## 0.12.1 (November 7th, 2020) + +### Added + +- Add connect retries. (Pull #221) + +### Fixed + +- Tweak detection of dropped connections, resolving an issue with open files limits on Linux. (Pull #185) +- Avoid leaking connections when establishing an HTTP tunnel to a proxy has failed. (Pull #223) +- Properly wrap OS errors when using `trio`. (Pull #225) + +## 0.12.0 (October 6th, 2020) + +### Changed + +- HTTP header casing is now preserved, rather than always sent in lowercase. (#216 and python-hyper/h11#104) + +### Added + +- Add Python 3.9 to officially supported versions. + +### Fixed + +- Gracefully handle a stdlib asyncio bug when a connection is closed while it is in a paused-for-reading state. (#201) + +## 0.11.1 (September 28nd, 2020) + +### Fixed + +- Add await to async semaphore release() coroutine (#197) +- Drop incorrect curio classifier (#192) + +## 0.11.0 (September 22nd, 2020) + +The Transport API with 0.11.0 has a couple of significant changes. + +Firstly we've moved changed the request interface in order to allow extensions, which will later enable us to support features +such as trailing headers, HTTP/2 server push, and CONNECT/Upgrade connections. + +The interface changes from: + +```python +def request(method, url, headers, stream, timeout): + return (http_version, status_code, reason, headers, stream) +``` + +To instead including an optional dictionary of extensions on the request and response: + +```python +def request(method, url, headers, stream, ext): + return (status_code, headers, stream, ext) +``` + +Having an open-ended extensions point will allow us to add later support for various optional features, that wouldn't otherwise be supported without these API changes. + +In particular: + +* Trailing headers support. +* HTTP/2 Server Push +* sendfile. +* Exposing raw connection on CONNECT, Upgrade, HTTP/2 bi-di streaming. +* Exposing debug information out of the API, including template name, template context. + +Currently extensions are limited to: + +* request: `timeout` - Optional. Timeout dictionary. +* response: `http_version` - Optional. Include the HTTP version used on the response. +* response: `reason` - Optional. Include the reason phrase used on the response. Only valid with HTTP/1.*. + +See https://github.com/encode/httpx/issues/1274#issuecomment-694884553 for the history behind this. + +Secondly, the async version of `request` is now namespaced as `arequest`. + +This allows concrete transports to support both sync and async implementations on the same class. + +### Added + +- Add curio support. (Pull #168) +- Add anyio support, with `backend="anyio"`. (Pull #169) + +### Changed + +- Update the Transport API to use 'ext' for optional extensions. (Pull #190) +- Update the Transport API to use `.request` and `.arequest` so implementations can support both sync and async. (Pull #189) + +## 0.10.2 (August 20th, 2020) + +### Added + +- Added Unix Domain Socket support. (Pull #139) + +### Fixed + +- Always include the port on proxy CONNECT requests. (Pull #154) +- Fix `max_keepalive_connections` configuration. (Pull #153) +- Fixes behaviour in HTTP/1.1 where server disconnects can be used to signal the end of the response body. (Pull #164) + +## 0.10.1 (August 7th, 2020) + +- Include `max_keepalive_connections` on `AsyncHTTPProxy`/`SyncHTTPProxy` classes. + +## 0.10.0 (August 7th, 2020) + +The most notable change in the 0.10.0 release is that HTTP/2 support is now fully optional. + +Use either `pip install httpcore` for HTTP/1.1 support only, or `pip install httpcore[http2]` for HTTP/1.1 and HTTP/2 support. + +### Added + +- HTTP/2 support becomes optional. (Pull #121, #130) +- Add `local_address=...` support. (Pull #100, #134) +- Add `PlainByteStream`, `IteratorByteStream`, `AsyncIteratorByteStream`. The `AsyncByteSteam` and `SyncByteStream` classes are now pure interface classes. (#133) +- Add `LocalProtocolError`, `RemoteProtocolError` exceptions. (Pull #129) +- Add `UnsupportedProtocol` exception. (Pull #128) +- Add `.get_connection_info()` method. (Pull #102, #137) +- Add better TRACE logs. (Pull #101) + +### Changed + +- `max_keepalive` is deprecated in favour of `max_keepalive_connections`. (Pull #140) + +### Fixed + +- Improve handling of server disconnects. (Pull #112) + +## 0.9.1 (May 27th, 2020) + +### Fixed + +- Proper host resolution for sync case, including IPv6 support. (Pull #97) +- Close outstanding connections when connection pool is closed. (Pull #98) + +## 0.9.0 (May 21th, 2020) + +### Changed + +- URL port becomes an `Optional[int]` instead of `int`. (Pull #92) + +### Fixed + +- Honor HTTP/2 max concurrent streams settings. (Pull #89, #90) +- Remove incorrect debug log. (Pull #83) + +## 0.8.4 (May 11th, 2020) + +### Added + +- Logging via HTTPCORE_LOG_LEVEL and HTTPX_LOG_LEVEL environment variables +and TRACE level logging. (Pull #79) + +### Fixed + +- Reuse of connections on HTTP/2 in close concurrency situations. (Pull #81) + +## 0.8.3 (May 6rd, 2020) + +### Fixed + +- Include `Host` and `Accept` headers on proxy "CONNECT" requests. +- De-duplicate any headers also contained in proxy_headers. +- HTTP/2 flag not being passed down to proxy connections. + +## 0.8.2 (May 3rd, 2020) + +### Fixed + +- Fix connections using proxy forwarding requests not being added to the +connection pool properly. (Pull #70) + +## 0.8.1 (April 30th, 2020) + +### Changed + +- Allow inherintance of both `httpcore.AsyncByteStream`, `httpcore.SyncByteStream` without type conflicts. + +## 0.8.0 (April 30th, 2020) + +### Fixed + +- Fixed tunnel proxy support. + +### Added + +- New `TimeoutException` base class. + +## 0.7.0 (March 5th, 2020) + +- First integration with HTTPX. diff --git a/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/RECORD b/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..44d4769e791312e8572bc5c24ed2efdc86819835 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/RECORD @@ -0,0 +1,68 @@ +httpcore-1.0.9.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +httpcore-1.0.9.dist-info/METADATA,sha256=_i1P2mGZEol4d54M8n88BFxTGGP83Zh-rMdPOhjUHCE,21529 +httpcore-1.0.9.dist-info/RECORD,, +httpcore-1.0.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87 +httpcore-1.0.9.dist-info/licenses/LICENSE.md,sha256=_ctZFUx0y6uhahEkL3dAvqnyPW_rVUeRfYxflKgDkqU,1518 +httpcore/__init__.py,sha256=9kT_kqChCCJUTHww24ZmR_ezcdbpRYWksD-gYNzkZP8,3445 +httpcore/__pycache__/__init__.cpython-312.pyc,, +httpcore/__pycache__/_api.cpython-312.pyc,, +httpcore/__pycache__/_exceptions.cpython-312.pyc,, +httpcore/__pycache__/_models.cpython-312.pyc,, +httpcore/__pycache__/_ssl.cpython-312.pyc,, +httpcore/__pycache__/_synchronization.cpython-312.pyc,, +httpcore/__pycache__/_trace.cpython-312.pyc,, +httpcore/__pycache__/_utils.cpython-312.pyc,, +httpcore/_api.py,sha256=unZmeDschBWCGCPCwkS3Wot9euK6bg_kKxLtGTxw214,3146 +httpcore/_async/__init__.py,sha256=EWdl2v4thnAHzJpqjU4h2a8DUiGAvNiWrkii9pfhTf0,1221 +httpcore/_async/__pycache__/__init__.cpython-312.pyc,, +httpcore/_async/__pycache__/connection.cpython-312.pyc,, +httpcore/_async/__pycache__/connection_pool.cpython-312.pyc,, +httpcore/_async/__pycache__/http11.cpython-312.pyc,, +httpcore/_async/__pycache__/http2.cpython-312.pyc,, +httpcore/_async/__pycache__/http_proxy.cpython-312.pyc,, +httpcore/_async/__pycache__/interfaces.cpython-312.pyc,, +httpcore/_async/__pycache__/socks_proxy.cpython-312.pyc,, +httpcore/_async/connection.py,sha256=6OcPXqMEfc0BU38_-iHUNDd1vKSTc2UVT09XqNb_BOk,8449 +httpcore/_async/connection_pool.py,sha256=DOIQ2s2ZCf9qfwxhzMprTPLqCL8OxGXiKF6qRHxvVyY,17307 +httpcore/_async/http11.py,sha256=-qM9bV7PjSQF5vxs37-eUXOIFwbIjPcZbNliuX9TtBw,13880 +httpcore/_async/http2.py,sha256=azX1fcmtXaIwjputFlZ4vd92J8xwjGOa9ax9QIv4394,23936 +httpcore/_async/http_proxy.py,sha256=2zVkrlv-Ds-rWGaqaXlrhEJiAQFPo23BT3Gq_sWoBXU,14701 +httpcore/_async/interfaces.py,sha256=jTiaWL83pgpGC9ziv90ZfwaKNMmHwmOalzaKiuTxATo,4455 +httpcore/_async/socks_proxy.py,sha256=lLKgLlggPfhFlqi0ODeBkOWvt9CghBBUyqsnsU1tx6Q,13841 +httpcore/_backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +httpcore/_backends/__pycache__/__init__.cpython-312.pyc,, +httpcore/_backends/__pycache__/anyio.cpython-312.pyc,, +httpcore/_backends/__pycache__/auto.cpython-312.pyc,, +httpcore/_backends/__pycache__/base.cpython-312.pyc,, +httpcore/_backends/__pycache__/mock.cpython-312.pyc,, +httpcore/_backends/__pycache__/sync.cpython-312.pyc,, +httpcore/_backends/__pycache__/trio.cpython-312.pyc,, +httpcore/_backends/anyio.py,sha256=x8PgEhXRC8bVqsdzk_YJx8Y6d9Tub06CuUSwnbmtqoY,5252 +httpcore/_backends/auto.py,sha256=zO136PKZmsaTDK-HRk84eA-MUg8_2wJf4NvmK432Aio,1662 +httpcore/_backends/base.py,sha256=aShgRdZnMmRhFWHetjumlM73f8Kz1YOAyCUP_4kHslA,3042 +httpcore/_backends/mock.py,sha256=er9T436uSe7NLrfiLa4x6Nuqg5ivQ693CxWYCWsgbH4,4077 +httpcore/_backends/sync.py,sha256=bhE4d9iK9Umxdsdsgm2EfKnXaBms2WggGYU-7jmUujU,7977 +httpcore/_backends/trio.py,sha256=LHu4_Mr5MswQmmT3yE4oLgf9b_JJfeVS4BjDxeJc7Ro,5996 +httpcore/_exceptions.py,sha256=looCKga3_YVYu3s-d3L9RMPRJyhsY7fiuuGxvkOD0c0,1184 +httpcore/_models.py,sha256=IO2CcXcdpovRcLTdGFGB6RyBZdEm2h_TOmoCc4rEKho,17623 +httpcore/_ssl.py,sha256=srqmSNU4iOUvWF-SrJvb8G_YEbHFELOXQOwdDIBTS9c,187 +httpcore/_sync/__init__.py,sha256=JBDIgXt5la1LCJ1sLQeKhjKFpLnpNr8Svs6z2ni3fgg,1141 +httpcore/_sync/__pycache__/__init__.cpython-312.pyc,, +httpcore/_sync/__pycache__/connection.cpython-312.pyc,, +httpcore/_sync/__pycache__/connection_pool.cpython-312.pyc,, +httpcore/_sync/__pycache__/http11.cpython-312.pyc,, +httpcore/_sync/__pycache__/http2.cpython-312.pyc,, +httpcore/_sync/__pycache__/http_proxy.cpython-312.pyc,, +httpcore/_sync/__pycache__/interfaces.cpython-312.pyc,, +httpcore/_sync/__pycache__/socks_proxy.cpython-312.pyc,, +httpcore/_sync/connection.py,sha256=9exGOb3PB-Mp2T1-sckSeL2t-tJ_9-NXomV8ihmWCgU,8238 +httpcore/_sync/connection_pool.py,sha256=a-T8LTsUxc7r0Ww1atfHSDoWPjQ0fA8Ul7S3-F0Mj70,16955 +httpcore/_sync/http11.py,sha256=IFobD1Md5JFlJGKWnh1_Q3epikUryI8qo09v8MiJIEA,13476 +httpcore/_sync/http2.py,sha256=AxU4yhcq68Bn5vqdJYtiXKYUj7nvhYbxz3v4rT4xnvA,23400 +httpcore/_sync/http_proxy.py,sha256=_al_6crKuEZu2wyvu493RZImJdBJnj5oGKNjLOJL2Zo,14463 +httpcore/_sync/interfaces.py,sha256=snXON42vUDHO5JBJvo8D4VWk2Wat44z2OXXHDrjbl94,4344 +httpcore/_sync/socks_proxy.py,sha256=zegZW9Snqj2_992DFJa8_CppOVBkVL4AgwduRkStakQ,13614 +httpcore/_synchronization.py,sha256=zSi13mAColBnknjZBknUC6hKNDQT4C6ijnezZ-r0T2s,9434 +httpcore/_trace.py,sha256=ck6ZoIzYTkdNAIfq5MGeKqBXDtqjOX-qfYwmZFbrGco,3952 +httpcore/_utils.py,sha256=_RLgXYOAYC350ikALV59GZ68IJrdocRZxPs9PjmzdFY,1537 +httpcore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/WHEEL b/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..12228d414b6cfed7c39d3781c85c63256a1d7fb5 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.27.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/licenses/LICENSE.md b/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/licenses/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..311b2b56c53f678ab95fc0def708c675d521a807 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore-1.0.9.dist-info/licenses/LICENSE.md @@ -0,0 +1,27 @@ +Copyright © 2020, [Encode OSS Ltd](https://www.encode.io/). +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/venv/lib/python3.12/site-packages/httpcore/_async/http2.py b/venv/lib/python3.12/site-packages/httpcore/_async/http2.py new file mode 100644 index 0000000000000000000000000000000000000000..dbd0beeb4da32d8c0175d412fa442eae8f837723 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_async/http2.py @@ -0,0 +1,592 @@ +from __future__ import annotations + +import enum +import logging +import time +import types +import typing + +import h2.config +import h2.connection +import h2.events +import h2.exceptions +import h2.settings + +from .._backends.base import AsyncNetworkStream +from .._exceptions import ( + ConnectionNotAvailable, + LocalProtocolError, + RemoteProtocolError, +) +from .._models import Origin, Request, Response +from .._synchronization import AsyncLock, AsyncSemaphore, AsyncShieldCancellation +from .._trace import Trace +from .interfaces import AsyncConnectionInterface + +logger = logging.getLogger("httpcore.http2") + + +def has_body_headers(request: Request) -> bool: + return any( + k.lower() == b"content-length" or k.lower() == b"transfer-encoding" + for k, v in request.headers + ) + + +class HTTPConnectionState(enum.IntEnum): + ACTIVE = 1 + IDLE = 2 + CLOSED = 3 + + +class AsyncHTTP2Connection(AsyncConnectionInterface): + READ_NUM_BYTES = 64 * 1024 + CONFIG = h2.config.H2Configuration(validate_inbound_headers=False) + + def __init__( + self, + origin: Origin, + stream: AsyncNetworkStream, + keepalive_expiry: float | None = None, + ): + self._origin = origin + self._network_stream = stream + self._keepalive_expiry: float | None = keepalive_expiry + self._h2_state = h2.connection.H2Connection(config=self.CONFIG) + self._state = HTTPConnectionState.IDLE + self._expire_at: float | None = None + self._request_count = 0 + self._init_lock = AsyncLock() + self._state_lock = AsyncLock() + self._read_lock = AsyncLock() + self._write_lock = AsyncLock() + self._sent_connection_init = False + self._used_all_stream_ids = False + self._connection_error = False + + # Mapping from stream ID to response stream events. + self._events: dict[ + int, + list[ + h2.events.ResponseReceived + | h2.events.DataReceived + | h2.events.StreamEnded + | h2.events.StreamReset, + ], + ] = {} + + # Connection terminated events are stored as state since + # we need to handle them for all streams. + self._connection_terminated: h2.events.ConnectionTerminated | None = None + + self._read_exception: Exception | None = None + self._write_exception: Exception | None = None + + async def handle_async_request(self, request: Request) -> Response: + if not self.can_handle_request(request.url.origin): + # This cannot occur in normal operation, since the connection pool + # will only send requests on connections that handle them. + # It's in place simply for resilience as a guard against incorrect + # usage, for anyone working directly with httpcore connections. + raise RuntimeError( + f"Attempted to send request to {request.url.origin} on connection " + f"to {self._origin}" + ) + + async with self._state_lock: + if self._state in (HTTPConnectionState.ACTIVE, HTTPConnectionState.IDLE): + self._request_count += 1 + self._expire_at = None + self._state = HTTPConnectionState.ACTIVE + else: + raise ConnectionNotAvailable() + + async with self._init_lock: + if not self._sent_connection_init: + try: + sci_kwargs = {"request": request} + async with Trace( + "send_connection_init", logger, request, sci_kwargs + ): + await self._send_connection_init(**sci_kwargs) + except BaseException as exc: + with AsyncShieldCancellation(): + await self.aclose() + raise exc + + self._sent_connection_init = True + + # Initially start with just 1 until the remote server provides + # its max_concurrent_streams value + self._max_streams = 1 + + local_settings_max_streams = ( + self._h2_state.local_settings.max_concurrent_streams + ) + self._max_streams_semaphore = AsyncSemaphore(local_settings_max_streams) + + for _ in range(local_settings_max_streams - self._max_streams): + await self._max_streams_semaphore.acquire() + + await self._max_streams_semaphore.acquire() + + try: + stream_id = self._h2_state.get_next_available_stream_id() + self._events[stream_id] = [] + except h2.exceptions.NoAvailableStreamIDError: # pragma: nocover + self._used_all_stream_ids = True + self._request_count -= 1 + raise ConnectionNotAvailable() + + try: + kwargs = {"request": request, "stream_id": stream_id} + async with Trace("send_request_headers", logger, request, kwargs): + await self._send_request_headers(request=request, stream_id=stream_id) + async with Trace("send_request_body", logger, request, kwargs): + await self._send_request_body(request=request, stream_id=stream_id) + async with Trace( + "receive_response_headers", logger, request, kwargs + ) as trace: + status, headers = await self._receive_response( + request=request, stream_id=stream_id + ) + trace.return_value = (status, headers) + + return Response( + status=status, + headers=headers, + content=HTTP2ConnectionByteStream(self, request, stream_id=stream_id), + extensions={ + "http_version": b"HTTP/2", + "network_stream": self._network_stream, + "stream_id": stream_id, + }, + ) + except BaseException as exc: # noqa: PIE786 + with AsyncShieldCancellation(): + kwargs = {"stream_id": stream_id} + async with Trace("response_closed", logger, request, kwargs): + await self._response_closed(stream_id=stream_id) + + if isinstance(exc, h2.exceptions.ProtocolError): + # One case where h2 can raise a protocol error is when a + # closed frame has been seen by the state machine. + # + # This happens when one stream is reading, and encounters + # a GOAWAY event. Other flows of control may then raise + # a protocol error at any point they interact with the 'h2_state'. + # + # In this case we'll have stored the event, and should raise + # it as a RemoteProtocolError. + if self._connection_terminated: # pragma: nocover + raise RemoteProtocolError(self._connection_terminated) + # If h2 raises a protocol error in some other state then we + # must somehow have made a protocol violation. + raise LocalProtocolError(exc) # pragma: nocover + + raise exc + + async def _send_connection_init(self, request: Request) -> None: + """ + The HTTP/2 connection requires some initial setup before we can start + using individual request/response streams on it. + """ + # Need to set these manually here instead of manipulating via + # __setitem__() otherwise the H2Connection will emit SettingsUpdate + # frames in addition to sending the undesired defaults. + self._h2_state.local_settings = h2.settings.Settings( + client=True, + initial_values={ + # Disable PUSH_PROMISE frames from the server since we don't do anything + # with them for now. Maybe when we support caching? + h2.settings.SettingCodes.ENABLE_PUSH: 0, + # These two are taken from h2 for safe defaults + h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS: 100, + h2.settings.SettingCodes.MAX_HEADER_LIST_SIZE: 65536, + }, + ) + + # Some websites (*cough* Yahoo *cough*) balk at this setting being + # present in the initial handshake since it's not defined in the original + # RFC despite the RFC mandating ignoring settings you don't know about. + del self._h2_state.local_settings[ + h2.settings.SettingCodes.ENABLE_CONNECT_PROTOCOL + ] + + self._h2_state.initiate_connection() + self._h2_state.increment_flow_control_window(2**24) + await self._write_outgoing_data(request) + + # Sending the request... + + async def _send_request_headers(self, request: Request, stream_id: int) -> None: + """ + Send the request headers to a given stream ID. + """ + end_stream = not has_body_headers(request) + + # In HTTP/2 the ':authority' pseudo-header is used instead of 'Host'. + # In order to gracefully handle HTTP/1.1 and HTTP/2 we always require + # HTTP/1.1 style headers, and map them appropriately if we end up on + # an HTTP/2 connection. + authority = [v for k, v in request.headers if k.lower() == b"host"][0] + + headers = [ + (b":method", request.method), + (b":authority", authority), + (b":scheme", request.url.scheme), + (b":path", request.url.target), + ] + [ + (k.lower(), v) + for k, v in request.headers + if k.lower() + not in ( + b"host", + b"transfer-encoding", + ) + ] + + self._h2_state.send_headers(stream_id, headers, end_stream=end_stream) + self._h2_state.increment_flow_control_window(2**24, stream_id=stream_id) + await self._write_outgoing_data(request) + + async def _send_request_body(self, request: Request, stream_id: int) -> None: + """ + Iterate over the request body sending it to a given stream ID. + """ + if not has_body_headers(request): + return + + assert isinstance(request.stream, typing.AsyncIterable) + async for data in request.stream: + await self._send_stream_data(request, stream_id, data) + await self._send_end_stream(request, stream_id) + + async def _send_stream_data( + self, request: Request, stream_id: int, data: bytes + ) -> None: + """ + Send a single chunk of data in one or more data frames. + """ + while data: + max_flow = await self._wait_for_outgoing_flow(request, stream_id) + chunk_size = min(len(data), max_flow) + chunk, data = data[:chunk_size], data[chunk_size:] + self._h2_state.send_data(stream_id, chunk) + await self._write_outgoing_data(request) + + async def _send_end_stream(self, request: Request, stream_id: int) -> None: + """ + Send an empty data frame on on a given stream ID with the END_STREAM flag set. + """ + self._h2_state.end_stream(stream_id) + await self._write_outgoing_data(request) + + # Receiving the response... + + async def _receive_response( + self, request: Request, stream_id: int + ) -> tuple[int, list[tuple[bytes, bytes]]]: + """ + Return the response status code and headers for a given stream ID. + """ + while True: + event = await self._receive_stream_event(request, stream_id) + if isinstance(event, h2.events.ResponseReceived): + break + + status_code = 200 + headers = [] + assert event.headers is not None + for k, v in event.headers: + if k == b":status": + status_code = int(v.decode("ascii", errors="ignore")) + elif not k.startswith(b":"): + headers.append((k, v)) + + return (status_code, headers) + + async def _receive_response_body( + self, request: Request, stream_id: int + ) -> typing.AsyncIterator[bytes]: + """ + Iterator that returns the bytes of the response body for a given stream ID. + """ + while True: + event = await self._receive_stream_event(request, stream_id) + if isinstance(event, h2.events.DataReceived): + assert event.flow_controlled_length is not None + assert event.data is not None + amount = event.flow_controlled_length + self._h2_state.acknowledge_received_data(amount, stream_id) + await self._write_outgoing_data(request) + yield event.data + elif isinstance(event, h2.events.StreamEnded): + break + + async def _receive_stream_event( + self, request: Request, stream_id: int + ) -> h2.events.ResponseReceived | h2.events.DataReceived | h2.events.StreamEnded: + """ + Return the next available event for a given stream ID. + + Will read more data from the network if required. + """ + while not self._events.get(stream_id): + await self._receive_events(request, stream_id) + event = self._events[stream_id].pop(0) + if isinstance(event, h2.events.StreamReset): + raise RemoteProtocolError(event) + return event + + async def _receive_events( + self, request: Request, stream_id: int | None = None + ) -> None: + """ + Read some data from the network until we see one or more events + for a given stream ID. + """ + async with self._read_lock: + if self._connection_terminated is not None: + last_stream_id = self._connection_terminated.last_stream_id + if stream_id and last_stream_id and stream_id > last_stream_id: + self._request_count -= 1 + raise ConnectionNotAvailable() + raise RemoteProtocolError(self._connection_terminated) + + # This conditional is a bit icky. We don't want to block reading if we've + # actually got an event to return for a given stream. We need to do that + # check *within* the atomic read lock. Though it also need to be optional, + # because when we call it from `_wait_for_outgoing_flow` we *do* want to + # block until we've available flow control, event when we have events + # pending for the stream ID we're attempting to send on. + if stream_id is None or not self._events.get(stream_id): + events = await self._read_incoming_data(request) + for event in events: + if isinstance(event, h2.events.RemoteSettingsChanged): + async with Trace( + "receive_remote_settings", logger, request + ) as trace: + await self._receive_remote_settings_change(event) + trace.return_value = event + + elif isinstance( + event, + ( + h2.events.ResponseReceived, + h2.events.DataReceived, + h2.events.StreamEnded, + h2.events.StreamReset, + ), + ): + if event.stream_id in self._events: + self._events[event.stream_id].append(event) + + elif isinstance(event, h2.events.ConnectionTerminated): + self._connection_terminated = event + + await self._write_outgoing_data(request) + + async def _receive_remote_settings_change( + self, event: h2.events.RemoteSettingsChanged + ) -> None: + max_concurrent_streams = event.changed_settings.get( + h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS + ) + if max_concurrent_streams: + new_max_streams = min( + max_concurrent_streams.new_value, + self._h2_state.local_settings.max_concurrent_streams, + ) + if new_max_streams and new_max_streams != self._max_streams: + while new_max_streams > self._max_streams: + await self._max_streams_semaphore.release() + self._max_streams += 1 + while new_max_streams < self._max_streams: + await self._max_streams_semaphore.acquire() + self._max_streams -= 1 + + async def _response_closed(self, stream_id: int) -> None: + await self._max_streams_semaphore.release() + del self._events[stream_id] + async with self._state_lock: + if self._connection_terminated and not self._events: + await self.aclose() + + elif self._state == HTTPConnectionState.ACTIVE and not self._events: + self._state = HTTPConnectionState.IDLE + if self._keepalive_expiry is not None: + now = time.monotonic() + self._expire_at = now + self._keepalive_expiry + if self._used_all_stream_ids: # pragma: nocover + await self.aclose() + + async def aclose(self) -> None: + # Note that this method unilaterally closes the connection, and does + # not have any kind of locking in place around it. + self._h2_state.close_connection() + self._state = HTTPConnectionState.CLOSED + await self._network_stream.aclose() + + # Wrappers around network read/write operations... + + async def _read_incoming_data(self, request: Request) -> list[h2.events.Event]: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("read", None) + + if self._read_exception is not None: + raise self._read_exception # pragma: nocover + + try: + data = await self._network_stream.read(self.READ_NUM_BYTES, timeout) + if data == b"": + raise RemoteProtocolError("Server disconnected") + except Exception as exc: + # If we get a network error we should: + # + # 1. Save the exception and just raise it immediately on any future reads. + # (For example, this means that a single read timeout or disconnect will + # immediately close all pending streams. Without requiring multiple + # sequential timeouts.) + # 2. Mark the connection as errored, so that we don't accept any other + # incoming requests. + self._read_exception = exc + self._connection_error = True + raise exc + + events: list[h2.events.Event] = self._h2_state.receive_data(data) + + return events + + async def _write_outgoing_data(self, request: Request) -> None: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("write", None) + + async with self._write_lock: + data_to_send = self._h2_state.data_to_send() + + if self._write_exception is not None: + raise self._write_exception # pragma: nocover + + try: + await self._network_stream.write(data_to_send, timeout) + except Exception as exc: # pragma: nocover + # If we get a network error we should: + # + # 1. Save the exception and just raise it immediately on any future write. + # (For example, this means that a single write timeout or disconnect will + # immediately close all pending streams. Without requiring multiple + # sequential timeouts.) + # 2. Mark the connection as errored, so that we don't accept any other + # incoming requests. + self._write_exception = exc + self._connection_error = True + raise exc + + # Flow control... + + async def _wait_for_outgoing_flow(self, request: Request, stream_id: int) -> int: + """ + Returns the maximum allowable outgoing flow for a given stream. + + If the allowable flow is zero, then waits on the network until + WindowUpdated frames have increased the flow rate. + https://tools.ietf.org/html/rfc7540#section-6.9 + """ + local_flow: int = self._h2_state.local_flow_control_window(stream_id) + max_frame_size: int = self._h2_state.max_outbound_frame_size + flow = min(local_flow, max_frame_size) + while flow == 0: + await self._receive_events(request) + local_flow = self._h2_state.local_flow_control_window(stream_id) + max_frame_size = self._h2_state.max_outbound_frame_size + flow = min(local_flow, max_frame_size) + return flow + + # Interface for connection pooling... + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._origin + + def is_available(self) -> bool: + return ( + self._state != HTTPConnectionState.CLOSED + and not self._connection_error + and not self._used_all_stream_ids + and not ( + self._h2_state.state_machine.state + == h2.connection.ConnectionState.CLOSED + ) + ) + + def has_expired(self) -> bool: + now = time.monotonic() + return self._expire_at is not None and now > self._expire_at + + def is_idle(self) -> bool: + return self._state == HTTPConnectionState.IDLE + + def is_closed(self) -> bool: + return self._state == HTTPConnectionState.CLOSED + + def info(self) -> str: + origin = str(self._origin) + return ( + f"{origin!r}, HTTP/2, {self._state.name}, " + f"Request Count: {self._request_count}" + ) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + origin = str(self._origin) + return ( + f"<{class_name} [{origin!r}, {self._state.name}, " + f"Request Count: {self._request_count}]>" + ) + + # These context managers are not used in the standard flow, but are + # useful for testing or working with connection instances directly. + + async def __aenter__(self) -> AsyncHTTP2Connection: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + await self.aclose() + + +class HTTP2ConnectionByteStream: + def __init__( + self, connection: AsyncHTTP2Connection, request: Request, stream_id: int + ) -> None: + self._connection = connection + self._request = request + self._stream_id = stream_id + self._closed = False + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + kwargs = {"request": self._request, "stream_id": self._stream_id} + try: + async with Trace("receive_response_body", logger, self._request, kwargs): + async for chunk in self._connection._receive_response_body( + request=self._request, stream_id=self._stream_id + ): + yield chunk + except BaseException as exc: + # If we get an exception while streaming the response, + # we want to close the response (and possibly the connection) + # before raising that exception. + with AsyncShieldCancellation(): + await self.aclose() + raise exc + + async def aclose(self) -> None: + if not self._closed: + self._closed = True + kwargs = {"stream_id": self._stream_id} + async with Trace("response_closed", logger, self._request, kwargs): + await self._connection._response_closed(stream_id=self._stream_id) diff --git a/venv/lib/python3.12/site-packages/httpcore/_async/http_proxy.py b/venv/lib/python3.12/site-packages/httpcore/_async/http_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..cc9d92066e1680576846e46ccdf645a2b1dd5718 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_async/http_proxy.py @@ -0,0 +1,367 @@ +from __future__ import annotations + +import base64 +import logging +import ssl +import typing + +from .._backends.base import SOCKET_OPTION, AsyncNetworkBackend +from .._exceptions import ProxyError +from .._models import ( + URL, + Origin, + Request, + Response, + enforce_bytes, + enforce_headers, + enforce_url, +) +from .._ssl import default_ssl_context +from .._synchronization import AsyncLock +from .._trace import Trace +from .connection import AsyncHTTPConnection +from .connection_pool import AsyncConnectionPool +from .http11 import AsyncHTTP11Connection +from .interfaces import AsyncConnectionInterface + +ByteOrStr = typing.Union[bytes, str] +HeadersAsSequence = typing.Sequence[typing.Tuple[ByteOrStr, ByteOrStr]] +HeadersAsMapping = typing.Mapping[ByteOrStr, ByteOrStr] + + +logger = logging.getLogger("httpcore.proxy") + + +def merge_headers( + default_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, + override_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, +) -> list[tuple[bytes, bytes]]: + """ + Append default_headers and override_headers, de-duplicating if a key exists + in both cases. + """ + default_headers = [] if default_headers is None else list(default_headers) + override_headers = [] if override_headers is None else list(override_headers) + has_override = set(key.lower() for key, value in override_headers) + default_headers = [ + (key, value) + for key, value in default_headers + if key.lower() not in has_override + ] + return default_headers + override_headers + + +class AsyncHTTPProxy(AsyncConnectionPool): # pragma: nocover + """ + A connection pool that sends requests via an HTTP proxy. + """ + + def __init__( + self, + proxy_url: URL | bytes | str, + proxy_auth: tuple[bytes | str, bytes | str] | None = None, + proxy_headers: HeadersAsMapping | HeadersAsSequence | None = None, + ssl_context: ssl.SSLContext | None = None, + proxy_ssl_context: ssl.SSLContext | None = None, + max_connections: int | None = 10, + max_keepalive_connections: int | None = None, + keepalive_expiry: float | None = None, + http1: bool = True, + http2: bool = False, + retries: int = 0, + local_address: str | None = None, + uds: str | None = None, + network_backend: AsyncNetworkBackend | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + """ + A connection pool for making HTTP requests. + + Parameters: + proxy_url: The URL to use when connecting to the proxy server. + For example `"http://127.0.0.1:8080/"`. + proxy_auth: Any proxy authentication as a two-tuple of + (username, password). May be either bytes or ascii-only str. + proxy_headers: Any HTTP headers to use for the proxy requests. + For example `{"Proxy-Authorization": "Basic :"}`. + ssl_context: An SSL context to use for verifying connections. + If not specified, the default `httpcore.default_ssl_context()` + will be used. + proxy_ssl_context: The same as `ssl_context`, but for a proxy server rather than a remote origin. + max_connections: The maximum number of concurrent HTTP connections that + the pool should allow. Any attempt to send a request on a pool that + would exceed this amount will block until a connection is available. + max_keepalive_connections: The maximum number of idle HTTP connections + that will be maintained in the pool. + keepalive_expiry: The duration in seconds that an idle HTTP connection + may be maintained for before being expired from the pool. + http1: A boolean indicating if HTTP/1.1 requests should be supported + by the connection pool. Defaults to True. + http2: A boolean indicating if HTTP/2 requests should be supported by + the connection pool. Defaults to False. + retries: The maximum number of retries when trying to establish + a connection. + local_address: Local address to connect from. Can also be used to + connect using a particular address family. Using + `local_address="0.0.0.0"` will connect using an `AF_INET` address + (IPv4), while using `local_address="::"` will connect using an + `AF_INET6` address (IPv6). + uds: Path to a Unix Domain Socket to use instead of TCP sockets. + network_backend: A backend instance to use for handling network I/O. + """ + super().__init__( + ssl_context=ssl_context, + max_connections=max_connections, + max_keepalive_connections=max_keepalive_connections, + keepalive_expiry=keepalive_expiry, + http1=http1, + http2=http2, + network_backend=network_backend, + retries=retries, + local_address=local_address, + uds=uds, + socket_options=socket_options, + ) + + self._proxy_url = enforce_url(proxy_url, name="proxy_url") + if ( + self._proxy_url.scheme == b"http" and proxy_ssl_context is not None + ): # pragma: no cover + raise RuntimeError( + "The `proxy_ssl_context` argument is not allowed for the http scheme" + ) + + self._ssl_context = ssl_context + self._proxy_ssl_context = proxy_ssl_context + self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") + if proxy_auth is not None: + username = enforce_bytes(proxy_auth[0], name="proxy_auth") + password = enforce_bytes(proxy_auth[1], name="proxy_auth") + userpass = username + b":" + password + authorization = b"Basic " + base64.b64encode(userpass) + self._proxy_headers = [ + (b"Proxy-Authorization", authorization) + ] + self._proxy_headers + + def create_connection(self, origin: Origin) -> AsyncConnectionInterface: + if origin.scheme == b"http": + return AsyncForwardHTTPConnection( + proxy_origin=self._proxy_url.origin, + proxy_headers=self._proxy_headers, + remote_origin=origin, + keepalive_expiry=self._keepalive_expiry, + network_backend=self._network_backend, + proxy_ssl_context=self._proxy_ssl_context, + ) + return AsyncTunnelHTTPConnection( + proxy_origin=self._proxy_url.origin, + proxy_headers=self._proxy_headers, + remote_origin=origin, + ssl_context=self._ssl_context, + proxy_ssl_context=self._proxy_ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + + +class AsyncForwardHTTPConnection(AsyncConnectionInterface): + def __init__( + self, + proxy_origin: Origin, + remote_origin: Origin, + proxy_headers: HeadersAsMapping | HeadersAsSequence | None = None, + keepalive_expiry: float | None = None, + network_backend: AsyncNetworkBackend | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + proxy_ssl_context: ssl.SSLContext | None = None, + ) -> None: + self._connection = AsyncHTTPConnection( + origin=proxy_origin, + keepalive_expiry=keepalive_expiry, + network_backend=network_backend, + socket_options=socket_options, + ssl_context=proxy_ssl_context, + ) + self._proxy_origin = proxy_origin + self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") + self._remote_origin = remote_origin + + async def handle_async_request(self, request: Request) -> Response: + headers = merge_headers(self._proxy_headers, request.headers) + url = URL( + scheme=self._proxy_origin.scheme, + host=self._proxy_origin.host, + port=self._proxy_origin.port, + target=bytes(request.url), + ) + proxy_request = Request( + method=request.method, + url=url, + headers=headers, + content=request.stream, + extensions=request.extensions, + ) + return await self._connection.handle_async_request(proxy_request) + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._remote_origin + + async def aclose(self) -> None: + await self._connection.aclose() + + def info(self) -> str: + return self._connection.info() + + def is_available(self) -> bool: + return self._connection.is_available() + + def has_expired(self) -> bool: + return self._connection.has_expired() + + def is_idle(self) -> bool: + return self._connection.is_idle() + + def is_closed(self) -> bool: + return self._connection.is_closed() + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{self.info()}]>" + + +class AsyncTunnelHTTPConnection(AsyncConnectionInterface): + def __init__( + self, + proxy_origin: Origin, + remote_origin: Origin, + ssl_context: ssl.SSLContext | None = None, + proxy_ssl_context: ssl.SSLContext | None = None, + proxy_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, + keepalive_expiry: float | None = None, + http1: bool = True, + http2: bool = False, + network_backend: AsyncNetworkBackend | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + self._connection: AsyncConnectionInterface = AsyncHTTPConnection( + origin=proxy_origin, + keepalive_expiry=keepalive_expiry, + network_backend=network_backend, + socket_options=socket_options, + ssl_context=proxy_ssl_context, + ) + self._proxy_origin = proxy_origin + self._remote_origin = remote_origin + self._ssl_context = ssl_context + self._proxy_ssl_context = proxy_ssl_context + self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") + self._keepalive_expiry = keepalive_expiry + self._http1 = http1 + self._http2 = http2 + self._connect_lock = AsyncLock() + self._connected = False + + async def handle_async_request(self, request: Request) -> Response: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("connect", None) + + async with self._connect_lock: + if not self._connected: + target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) + + connect_url = URL( + scheme=self._proxy_origin.scheme, + host=self._proxy_origin.host, + port=self._proxy_origin.port, + target=target, + ) + connect_headers = merge_headers( + [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers + ) + connect_request = Request( + method=b"CONNECT", + url=connect_url, + headers=connect_headers, + extensions=request.extensions, + ) + connect_response = await self._connection.handle_async_request( + connect_request + ) + + if connect_response.status < 200 or connect_response.status > 299: + reason_bytes = connect_response.extensions.get("reason_phrase", b"") + reason_str = reason_bytes.decode("ascii", errors="ignore") + msg = "%d %s" % (connect_response.status, reason_str) + await self._connection.aclose() + raise ProxyError(msg) + + stream = connect_response.extensions["network_stream"] + + # Upgrade the stream to SSL + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + async with Trace("start_tls", logger, request, kwargs) as trace: + stream = await stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) + + # Create the HTTP/1.1 or HTTP/2 connection + if http2_negotiated or (self._http2 and not self._http1): + from .http2 import AsyncHTTP2Connection + + self._connection = AsyncHTTP2Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + else: + self._connection = AsyncHTTP11Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + + self._connected = True + return await self._connection.handle_async_request(request) + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._remote_origin + + async def aclose(self) -> None: + await self._connection.aclose() + + def info(self) -> str: + return self._connection.info() + + def is_available(self) -> bool: + return self._connection.is_available() + + def has_expired(self) -> bool: + return self._connection.has_expired() + + def is_idle(self) -> bool: + return self._connection.is_idle() + + def is_closed(self) -> bool: + return self._connection.is_closed() + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{self.info()}]>" diff --git a/venv/lib/python3.12/site-packages/httpcore/_async/interfaces.py b/venv/lib/python3.12/site-packages/httpcore/_async/interfaces.py new file mode 100644 index 0000000000000000000000000000000000000000..361583bede6b2b84088b38054d5d8116ef9f1597 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_async/interfaces.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import contextlib +import typing + +from .._models import ( + URL, + Extensions, + HeaderTypes, + Origin, + Request, + Response, + enforce_bytes, + enforce_headers, + enforce_url, + include_request_headers, +) + + +class AsyncRequestInterface: + async def request( + self, + method: bytes | str, + url: URL | bytes | str, + *, + headers: HeaderTypes = None, + content: bytes | typing.AsyncIterator[bytes] | None = None, + extensions: Extensions | None = None, + ) -> Response: + # Strict type checking on our parameters. + method = enforce_bytes(method, name="method") + url = enforce_url(url, name="url") + headers = enforce_headers(headers, name="headers") + + # Include Host header, and optionally Content-Length or Transfer-Encoding. + headers = include_request_headers(headers, url=url, content=content) + + request = Request( + method=method, + url=url, + headers=headers, + content=content, + extensions=extensions, + ) + response = await self.handle_async_request(request) + try: + await response.aread() + finally: + await response.aclose() + return response + + @contextlib.asynccontextmanager + async def stream( + self, + method: bytes | str, + url: URL | bytes | str, + *, + headers: HeaderTypes = None, + content: bytes | typing.AsyncIterator[bytes] | None = None, + extensions: Extensions | None = None, + ) -> typing.AsyncIterator[Response]: + # Strict type checking on our parameters. + method = enforce_bytes(method, name="method") + url = enforce_url(url, name="url") + headers = enforce_headers(headers, name="headers") + + # Include Host header, and optionally Content-Length or Transfer-Encoding. + headers = include_request_headers(headers, url=url, content=content) + + request = Request( + method=method, + url=url, + headers=headers, + content=content, + extensions=extensions, + ) + response = await self.handle_async_request(request) + try: + yield response + finally: + await response.aclose() + + async def handle_async_request(self, request: Request) -> Response: + raise NotImplementedError() # pragma: nocover + + +class AsyncConnectionInterface(AsyncRequestInterface): + async def aclose(self) -> None: + raise NotImplementedError() # pragma: nocover + + def info(self) -> str: + raise NotImplementedError() # pragma: nocover + + def can_handle_request(self, origin: Origin) -> bool: + raise NotImplementedError() # pragma: nocover + + def is_available(self) -> bool: + """ + Return `True` if the connection is currently able to accept an + outgoing request. + + An HTTP/1.1 connection will only be available if it is currently idle. + + An HTTP/2 connection will be available so long as the stream ID space is + not yet exhausted, and the connection is not in an error state. + + While the connection is being established we may not yet know if it is going + to result in an HTTP/1.1 or HTTP/2 connection. The connection should be + treated as being available, but might ultimately raise `NewConnectionRequired` + required exceptions if multiple requests are attempted over a connection + that ends up being established as HTTP/1.1. + """ + raise NotImplementedError() # pragma: nocover + + def has_expired(self) -> bool: + """ + Return `True` if the connection is in a state where it should be closed. + + This either means that the connection is idle and it has passed the + expiry time on its keep-alive, or that server has sent an EOF. + """ + raise NotImplementedError() # pragma: nocover + + def is_idle(self) -> bool: + """ + Return `True` if the connection is currently idle. + """ + raise NotImplementedError() # pragma: nocover + + def is_closed(self) -> bool: + """ + Return `True` if the connection has been closed. + + Used when a response is closed to determine if the connection may be + returned to the connection pool or not. + """ + raise NotImplementedError() # pragma: nocover diff --git a/venv/lib/python3.12/site-packages/httpcore/_async/socks_proxy.py b/venv/lib/python3.12/site-packages/httpcore/_async/socks_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..b363f55a0b071de6c5f377726be82dc2110e373c --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_async/socks_proxy.py @@ -0,0 +1,341 @@ +from __future__ import annotations + +import logging +import ssl + +import socksio + +from .._backends.auto import AutoBackend +from .._backends.base import AsyncNetworkBackend, AsyncNetworkStream +from .._exceptions import ConnectionNotAvailable, ProxyError +from .._models import URL, Origin, Request, Response, enforce_bytes, enforce_url +from .._ssl import default_ssl_context +from .._synchronization import AsyncLock +from .._trace import Trace +from .connection_pool import AsyncConnectionPool +from .http11 import AsyncHTTP11Connection +from .interfaces import AsyncConnectionInterface + +logger = logging.getLogger("httpcore.socks") + + +AUTH_METHODS = { + b"\x00": "NO AUTHENTICATION REQUIRED", + b"\x01": "GSSAPI", + b"\x02": "USERNAME/PASSWORD", + b"\xff": "NO ACCEPTABLE METHODS", +} + +REPLY_CODES = { + b"\x00": "Succeeded", + b"\x01": "General SOCKS server failure", + b"\x02": "Connection not allowed by ruleset", + b"\x03": "Network unreachable", + b"\x04": "Host unreachable", + b"\x05": "Connection refused", + b"\x06": "TTL expired", + b"\x07": "Command not supported", + b"\x08": "Address type not supported", +} + + +async def _init_socks5_connection( + stream: AsyncNetworkStream, + *, + host: bytes, + port: int, + auth: tuple[bytes, bytes] | None = None, +) -> None: + conn = socksio.socks5.SOCKS5Connection() + + # Auth method request + auth_method = ( + socksio.socks5.SOCKS5AuthMethod.NO_AUTH_REQUIRED + if auth is None + else socksio.socks5.SOCKS5AuthMethod.USERNAME_PASSWORD + ) + conn.send(socksio.socks5.SOCKS5AuthMethodsRequest([auth_method])) + outgoing_bytes = conn.data_to_send() + await stream.write(outgoing_bytes) + + # Auth method response + incoming_bytes = await stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socksio.socks5.SOCKS5AuthReply) + if response.method != auth_method: + requested = AUTH_METHODS.get(auth_method, "UNKNOWN") + responded = AUTH_METHODS.get(response.method, "UNKNOWN") + raise ProxyError( + f"Requested {requested} from proxy server, but got {responded}." + ) + + if response.method == socksio.socks5.SOCKS5AuthMethod.USERNAME_PASSWORD: + # Username/password request + assert auth is not None + username, password = auth + conn.send(socksio.socks5.SOCKS5UsernamePasswordRequest(username, password)) + outgoing_bytes = conn.data_to_send() + await stream.write(outgoing_bytes) + + # Username/password response + incoming_bytes = await stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socksio.socks5.SOCKS5UsernamePasswordReply) + if not response.success: + raise ProxyError("Invalid username/password") + + # Connect request + conn.send( + socksio.socks5.SOCKS5CommandRequest.from_address( + socksio.socks5.SOCKS5Command.CONNECT, (host, port) + ) + ) + outgoing_bytes = conn.data_to_send() + await stream.write(outgoing_bytes) + + # Connect response + incoming_bytes = await stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socksio.socks5.SOCKS5Reply) + if response.reply_code != socksio.socks5.SOCKS5ReplyCode.SUCCEEDED: + reply_code = REPLY_CODES.get(response.reply_code, "UNKOWN") + raise ProxyError(f"Proxy Server could not connect: {reply_code}.") + + +class AsyncSOCKSProxy(AsyncConnectionPool): # pragma: nocover + """ + A connection pool that sends requests via an HTTP proxy. + """ + + def __init__( + self, + proxy_url: URL | bytes | str, + proxy_auth: tuple[bytes | str, bytes | str] | None = None, + ssl_context: ssl.SSLContext | None = None, + max_connections: int | None = 10, + max_keepalive_connections: int | None = None, + keepalive_expiry: float | None = None, + http1: bool = True, + http2: bool = False, + retries: int = 0, + network_backend: AsyncNetworkBackend | None = None, + ) -> None: + """ + A connection pool for making HTTP requests. + + Parameters: + proxy_url: The URL to use when connecting to the proxy server. + For example `"http://127.0.0.1:8080/"`. + ssl_context: An SSL context to use for verifying connections. + If not specified, the default `httpcore.default_ssl_context()` + will be used. + max_connections: The maximum number of concurrent HTTP connections that + the pool should allow. Any attempt to send a request on a pool that + would exceed this amount will block until a connection is available. + max_keepalive_connections: The maximum number of idle HTTP connections + that will be maintained in the pool. + keepalive_expiry: The duration in seconds that an idle HTTP connection + may be maintained for before being expired from the pool. + http1: A boolean indicating if HTTP/1.1 requests should be supported + by the connection pool. Defaults to True. + http2: A boolean indicating if HTTP/2 requests should be supported by + the connection pool. Defaults to False. + retries: The maximum number of retries when trying to establish + a connection. + local_address: Local address to connect from. Can also be used to + connect using a particular address family. Using + `local_address="0.0.0.0"` will connect using an `AF_INET` address + (IPv4), while using `local_address="::"` will connect using an + `AF_INET6` address (IPv6). + uds: Path to a Unix Domain Socket to use instead of TCP sockets. + network_backend: A backend instance to use for handling network I/O. + """ + super().__init__( + ssl_context=ssl_context, + max_connections=max_connections, + max_keepalive_connections=max_keepalive_connections, + keepalive_expiry=keepalive_expiry, + http1=http1, + http2=http2, + network_backend=network_backend, + retries=retries, + ) + self._ssl_context = ssl_context + self._proxy_url = enforce_url(proxy_url, name="proxy_url") + if proxy_auth is not None: + username, password = proxy_auth + username_bytes = enforce_bytes(username, name="proxy_auth") + password_bytes = enforce_bytes(password, name="proxy_auth") + self._proxy_auth: tuple[bytes, bytes] | None = ( + username_bytes, + password_bytes, + ) + else: + self._proxy_auth = None + + def create_connection(self, origin: Origin) -> AsyncConnectionInterface: + return AsyncSocks5Connection( + proxy_origin=self._proxy_url.origin, + remote_origin=origin, + proxy_auth=self._proxy_auth, + ssl_context=self._ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + + +class AsyncSocks5Connection(AsyncConnectionInterface): + def __init__( + self, + proxy_origin: Origin, + remote_origin: Origin, + proxy_auth: tuple[bytes, bytes] | None = None, + ssl_context: ssl.SSLContext | None = None, + keepalive_expiry: float | None = None, + http1: bool = True, + http2: bool = False, + network_backend: AsyncNetworkBackend | None = None, + ) -> None: + self._proxy_origin = proxy_origin + self._remote_origin = remote_origin + self._proxy_auth = proxy_auth + self._ssl_context = ssl_context + self._keepalive_expiry = keepalive_expiry + self._http1 = http1 + self._http2 = http2 + + self._network_backend: AsyncNetworkBackend = ( + AutoBackend() if network_backend is None else network_backend + ) + self._connect_lock = AsyncLock() + self._connection: AsyncConnectionInterface | None = None + self._connect_failed = False + + async def handle_async_request(self, request: Request) -> Response: + timeouts = request.extensions.get("timeout", {}) + sni_hostname = request.extensions.get("sni_hostname", None) + timeout = timeouts.get("connect", None) + + async with self._connect_lock: + if self._connection is None: + try: + # Connect to the proxy + kwargs = { + "host": self._proxy_origin.host.decode("ascii"), + "port": self._proxy_origin.port, + "timeout": timeout, + } + async with Trace("connect_tcp", logger, request, kwargs) as trace: + stream = await self._network_backend.connect_tcp(**kwargs) + trace.return_value = stream + + # Connect to the remote host using socks5 + kwargs = { + "stream": stream, + "host": self._remote_origin.host.decode("ascii"), + "port": self._remote_origin.port, + "auth": self._proxy_auth, + } + async with Trace( + "setup_socks5_connection", logger, request, kwargs + ) as trace: + await _init_socks5_connection(**kwargs) + trace.return_value = stream + + # Upgrade the stream to SSL + if self._remote_origin.scheme == b"https": + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ( + ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ) + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": sni_hostname + or self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + async with Trace("start_tls", logger, request, kwargs) as trace: + stream = await stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) + + # Create the HTTP/1.1 or HTTP/2 connection + if http2_negotiated or ( + self._http2 and not self._http1 + ): # pragma: nocover + from .http2 import AsyncHTTP2Connection + + self._connection = AsyncHTTP2Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + else: + self._connection = AsyncHTTP11Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + except Exception as exc: + self._connect_failed = True + raise exc + elif not self._connection.is_available(): # pragma: nocover + raise ConnectionNotAvailable() + + return await self._connection.handle_async_request(request) + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._remote_origin + + async def aclose(self) -> None: + if self._connection is not None: + await self._connection.aclose() + + def is_available(self) -> bool: + if self._connection is None: # pragma: nocover + # If HTTP/2 support is enabled, and the resulting connection could + # end up as HTTP/2 then we should indicate the connection as being + # available to service multiple requests. + return ( + self._http2 + and (self._remote_origin.scheme == b"https" or not self._http1) + and not self._connect_failed + ) + return self._connection.is_available() + + def has_expired(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.has_expired() + + def is_idle(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.is_idle() + + def is_closed(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.is_closed() + + def info(self) -> str: + if self._connection is None: # pragma: nocover + return "CONNECTION FAILED" if self._connect_failed else "CONNECTING" + return self._connection.info() + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{self.info()}]>" diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/__init__.py b/venv/lib/python3.12/site-packages/httpcore/_backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..495e1c4463ac62be5fbe6e2ce1bbbee20f47e94a Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/anyio.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/anyio.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..940659bdc43a8faa1343aa97497ba45db80c9881 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/anyio.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/auto.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/auto.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0f81649c3983c11c84bf89f0cb06170f03d714a Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/auto.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/base.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/base.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d613aeab18f3b3b1f13dc7849b0c0cb40c8842f9 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/base.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/mock.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/mock.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..04cbe45d31de8e65268d7e0f050b5324f426a0f7 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/mock.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/sync.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/sync.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..85b378293939aa4ace2a98091b5e893685a1f7a7 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/sync.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/trio.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/trio.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bc3060b2b0674606c40658b786abdb28b0b62c44 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_backends/__pycache__/trio.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/anyio.py b/venv/lib/python3.12/site-packages/httpcore/_backends/anyio.py new file mode 100644 index 0000000000000000000000000000000000000000..a140095e1b8de022f321a41c0125e0e5febc0749 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_backends/anyio.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import ssl +import typing + +import anyio + +from .._exceptions import ( + ConnectError, + ConnectTimeout, + ReadError, + ReadTimeout, + WriteError, + WriteTimeout, + map_exceptions, +) +from .._utils import is_socket_readable +from .base import SOCKET_OPTION, AsyncNetworkBackend, AsyncNetworkStream + + +class AnyIOStream(AsyncNetworkStream): + def __init__(self, stream: anyio.abc.ByteStream) -> None: + self._stream = stream + + async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + exc_map = { + TimeoutError: ReadTimeout, + anyio.BrokenResourceError: ReadError, + anyio.ClosedResourceError: ReadError, + anyio.EndOfStream: ReadError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + try: + return await self._stream.receive(max_bytes=max_bytes) + except anyio.EndOfStream: # pragma: nocover + return b"" + + async def write(self, buffer: bytes, timeout: float | None = None) -> None: + if not buffer: + return + + exc_map = { + TimeoutError: WriteTimeout, + anyio.BrokenResourceError: WriteError, + anyio.ClosedResourceError: WriteError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + await self._stream.send(item=buffer) + + async def aclose(self) -> None: + await self._stream.aclose() + + async def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> AsyncNetworkStream: + exc_map = { + TimeoutError: ConnectTimeout, + anyio.BrokenResourceError: ConnectError, + anyio.EndOfStream: ConnectError, + ssl.SSLError: ConnectError, + } + with map_exceptions(exc_map): + try: + with anyio.fail_after(timeout): + ssl_stream = await anyio.streams.tls.TLSStream.wrap( + self._stream, + ssl_context=ssl_context, + hostname=server_hostname, + standard_compatible=False, + server_side=False, + ) + except Exception as exc: # pragma: nocover + await self.aclose() + raise exc + return AnyIOStream(ssl_stream) + + def get_extra_info(self, info: str) -> typing.Any: + if info == "ssl_object": + return self._stream.extra(anyio.streams.tls.TLSAttribute.ssl_object, None) + if info == "client_addr": + return self._stream.extra(anyio.abc.SocketAttribute.local_address, None) + if info == "server_addr": + return self._stream.extra(anyio.abc.SocketAttribute.remote_address, None) + if info == "socket": + return self._stream.extra(anyio.abc.SocketAttribute.raw_socket, None) + if info == "is_readable": + sock = self._stream.extra(anyio.abc.SocketAttribute.raw_socket, None) + return is_socket_readable(sock) + return None + + +class AnyIOBackend(AsyncNetworkBackend): + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: # pragma: nocover + if socket_options is None: + socket_options = [] + exc_map = { + TimeoutError: ConnectTimeout, + OSError: ConnectError, + anyio.BrokenResourceError: ConnectError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + stream: anyio.abc.ByteStream = await anyio.connect_tcp( + remote_host=host, + remote_port=port, + local_host=local_address, + ) + # By default TCP sockets opened in `asyncio` include TCP_NODELAY. + for option in socket_options: + stream._raw_socket.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover + return AnyIOStream(stream) + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: # pragma: nocover + if socket_options is None: + socket_options = [] + exc_map = { + TimeoutError: ConnectTimeout, + OSError: ConnectError, + anyio.BrokenResourceError: ConnectError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + stream: anyio.abc.ByteStream = await anyio.connect_unix(path) + for option in socket_options: + stream._raw_socket.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover + return AnyIOStream(stream) + + async def sleep(self, seconds: float) -> None: + await anyio.sleep(seconds) # pragma: nocover diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/auto.py b/venv/lib/python3.12/site-packages/httpcore/_backends/auto.py new file mode 100644 index 0000000000000000000000000000000000000000..49f0e698c97ad5623f376d8182675352e21c2c3c --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_backends/auto.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import typing + +from .._synchronization import current_async_library +from .base import SOCKET_OPTION, AsyncNetworkBackend, AsyncNetworkStream + + +class AutoBackend(AsyncNetworkBackend): + async def _init_backend(self) -> None: + if not (hasattr(self, "_backend")): + backend = current_async_library() + if backend == "trio": + from .trio import TrioBackend + + self._backend: AsyncNetworkBackend = TrioBackend() + else: + from .anyio import AnyIOBackend + + self._backend = AnyIOBackend() + + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + await self._init_backend() + return await self._backend.connect_tcp( + host, + port, + timeout=timeout, + local_address=local_address, + socket_options=socket_options, + ) + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: # pragma: nocover + await self._init_backend() + return await self._backend.connect_unix_socket( + path, timeout=timeout, socket_options=socket_options + ) + + async def sleep(self, seconds: float) -> None: # pragma: nocover + await self._init_backend() + return await self._backend.sleep(seconds) diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/base.py b/venv/lib/python3.12/site-packages/httpcore/_backends/base.py new file mode 100644 index 0000000000000000000000000000000000000000..cf55c8b10eb543872550be863206fe2f760d0d8d --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_backends/base.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import ssl +import time +import typing + +SOCKET_OPTION = typing.Union[ + typing.Tuple[int, int, int], + typing.Tuple[int, int, typing.Union[bytes, bytearray]], + typing.Tuple[int, int, None, int], +] + + +class NetworkStream: + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + raise NotImplementedError() # pragma: nocover + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + raise NotImplementedError() # pragma: nocover + + def close(self) -> None: + raise NotImplementedError() # pragma: nocover + + def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> NetworkStream: + raise NotImplementedError() # pragma: nocover + + def get_extra_info(self, info: str) -> typing.Any: + return None # pragma: nocover + + +class NetworkBackend: + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + raise NotImplementedError() # pragma: nocover + + def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + raise NotImplementedError() # pragma: nocover + + def sleep(self, seconds: float) -> None: + time.sleep(seconds) # pragma: nocover + + +class AsyncNetworkStream: + async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + raise NotImplementedError() # pragma: nocover + + async def write(self, buffer: bytes, timeout: float | None = None) -> None: + raise NotImplementedError() # pragma: nocover + + async def aclose(self) -> None: + raise NotImplementedError() # pragma: nocover + + async def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> AsyncNetworkStream: + raise NotImplementedError() # pragma: nocover + + def get_extra_info(self, info: str) -> typing.Any: + return None # pragma: nocover + + +class AsyncNetworkBackend: + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + raise NotImplementedError() # pragma: nocover + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + raise NotImplementedError() # pragma: nocover + + async def sleep(self, seconds: float) -> None: + raise NotImplementedError() # pragma: nocover diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/mock.py b/venv/lib/python3.12/site-packages/httpcore/_backends/mock.py new file mode 100644 index 0000000000000000000000000000000000000000..9b6edca03d4d4b34f355fd53e49d4b4c699c972c --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_backends/mock.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import ssl +import typing + +from .._exceptions import ReadError +from .base import ( + SOCKET_OPTION, + AsyncNetworkBackend, + AsyncNetworkStream, + NetworkBackend, + NetworkStream, +) + + +class MockSSLObject: + def __init__(self, http2: bool): + self._http2 = http2 + + def selected_alpn_protocol(self) -> str: + return "h2" if self._http2 else "http/1.1" + + +class MockStream(NetworkStream): + def __init__(self, buffer: list[bytes], http2: bool = False) -> None: + self._buffer = buffer + self._http2 = http2 + self._closed = False + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + if self._closed: + raise ReadError("Connection closed") + if not self._buffer: + return b"" + return self._buffer.pop(0) + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + pass + + def close(self) -> None: + self._closed = True + + def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> NetworkStream: + return self + + def get_extra_info(self, info: str) -> typing.Any: + return MockSSLObject(http2=self._http2) if info == "ssl_object" else None + + def __repr__(self) -> str: + return "" + + +class MockBackend(NetworkBackend): + def __init__(self, buffer: list[bytes], http2: bool = False) -> None: + self._buffer = buffer + self._http2 = http2 + + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + return MockStream(list(self._buffer), http2=self._http2) + + def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + return MockStream(list(self._buffer), http2=self._http2) + + def sleep(self, seconds: float) -> None: + pass + + +class AsyncMockStream(AsyncNetworkStream): + def __init__(self, buffer: list[bytes], http2: bool = False) -> None: + self._buffer = buffer + self._http2 = http2 + self._closed = False + + async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + if self._closed: + raise ReadError("Connection closed") + if not self._buffer: + return b"" + return self._buffer.pop(0) + + async def write(self, buffer: bytes, timeout: float | None = None) -> None: + pass + + async def aclose(self) -> None: + self._closed = True + + async def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> AsyncNetworkStream: + return self + + def get_extra_info(self, info: str) -> typing.Any: + return MockSSLObject(http2=self._http2) if info == "ssl_object" else None + + def __repr__(self) -> str: + return "" + + +class AsyncMockBackend(AsyncNetworkBackend): + def __init__(self, buffer: list[bytes], http2: bool = False) -> None: + self._buffer = buffer + self._http2 = http2 + + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + return AsyncMockStream(list(self._buffer), http2=self._http2) + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + return AsyncMockStream(list(self._buffer), http2=self._http2) + + async def sleep(self, seconds: float) -> None: + pass diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/sync.py b/venv/lib/python3.12/site-packages/httpcore/_backends/sync.py new file mode 100644 index 0000000000000000000000000000000000000000..4018a09c6fb1e0ef1b03ab8d84b13ebef4031f7c --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_backends/sync.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +import functools +import socket +import ssl +import sys +import typing + +from .._exceptions import ( + ConnectError, + ConnectTimeout, + ExceptionMapping, + ReadError, + ReadTimeout, + WriteError, + WriteTimeout, + map_exceptions, +) +from .._utils import is_socket_readable +from .base import SOCKET_OPTION, NetworkBackend, NetworkStream + + +class TLSinTLSStream(NetworkStream): # pragma: no cover + """ + Because the standard `SSLContext.wrap_socket` method does + not work for `SSLSocket` objects, we need this class + to implement TLS stream using an underlying `SSLObject` + instance in order to support TLS on top of TLS. + """ + + # Defined in RFC 8449 + TLS_RECORD_SIZE = 16384 + + def __init__( + self, + sock: socket.socket, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ): + self._sock = sock + self._incoming = ssl.MemoryBIO() + self._outgoing = ssl.MemoryBIO() + + self.ssl_obj = ssl_context.wrap_bio( + incoming=self._incoming, + outgoing=self._outgoing, + server_hostname=server_hostname, + ) + + self._sock.settimeout(timeout) + self._perform_io(self.ssl_obj.do_handshake) + + def _perform_io( + self, + func: typing.Callable[..., typing.Any], + ) -> typing.Any: + ret = None + + while True: + errno = None + try: + ret = func() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError) as e: + errno = e.errno + + self._sock.sendall(self._outgoing.read()) + + if errno == ssl.SSL_ERROR_WANT_READ: + buf = self._sock.recv(self.TLS_RECORD_SIZE) + + if buf: + self._incoming.write(buf) + else: + self._incoming.write_eof() + if errno is None: + return ret + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + exc_map: ExceptionMapping = {socket.timeout: ReadTimeout, OSError: ReadError} + with map_exceptions(exc_map): + self._sock.settimeout(timeout) + return typing.cast( + bytes, self._perform_io(functools.partial(self.ssl_obj.read, max_bytes)) + ) + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + exc_map: ExceptionMapping = {socket.timeout: WriteTimeout, OSError: WriteError} + with map_exceptions(exc_map): + self._sock.settimeout(timeout) + while buffer: + nsent = self._perform_io(functools.partial(self.ssl_obj.write, buffer)) + buffer = buffer[nsent:] + + def close(self) -> None: + self._sock.close() + + def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> NetworkStream: + raise NotImplementedError() + + def get_extra_info(self, info: str) -> typing.Any: + if info == "ssl_object": + return self.ssl_obj + if info == "client_addr": + return self._sock.getsockname() + if info == "server_addr": + return self._sock.getpeername() + if info == "socket": + return self._sock + if info == "is_readable": + return is_socket_readable(self._sock) + return None + + +class SyncStream(NetworkStream): + def __init__(self, sock: socket.socket) -> None: + self._sock = sock + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + exc_map: ExceptionMapping = {socket.timeout: ReadTimeout, OSError: ReadError} + with map_exceptions(exc_map): + self._sock.settimeout(timeout) + return self._sock.recv(max_bytes) + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + if not buffer: + return + + exc_map: ExceptionMapping = {socket.timeout: WriteTimeout, OSError: WriteError} + with map_exceptions(exc_map): + while buffer: + self._sock.settimeout(timeout) + n = self._sock.send(buffer) + buffer = buffer[n:] + + def close(self) -> None: + self._sock.close() + + def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> NetworkStream: + exc_map: ExceptionMapping = { + socket.timeout: ConnectTimeout, + OSError: ConnectError, + } + with map_exceptions(exc_map): + try: + if isinstance(self._sock, ssl.SSLSocket): # pragma: no cover + # If the underlying socket has already been upgraded + # to the TLS layer (i.e. is an instance of SSLSocket), + # we need some additional smarts to support TLS-in-TLS. + return TLSinTLSStream( + self._sock, ssl_context, server_hostname, timeout + ) + else: + self._sock.settimeout(timeout) + sock = ssl_context.wrap_socket( + self._sock, server_hostname=server_hostname + ) + except Exception as exc: # pragma: nocover + self.close() + raise exc + return SyncStream(sock) + + def get_extra_info(self, info: str) -> typing.Any: + if info == "ssl_object" and isinstance(self._sock, ssl.SSLSocket): + return self._sock._sslobj # type: ignore + if info == "client_addr": + return self._sock.getsockname() + if info == "server_addr": + return self._sock.getpeername() + if info == "socket": + return self._sock + if info == "is_readable": + return is_socket_readable(self._sock) + return None + + +class SyncBackend(NetworkBackend): + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + # Note that we automatically include `TCP_NODELAY` + # in addition to any other custom socket options. + if socket_options is None: + socket_options = [] # pragma: no cover + address = (host, port) + source_address = None if local_address is None else (local_address, 0) + exc_map: ExceptionMapping = { + socket.timeout: ConnectTimeout, + OSError: ConnectError, + } + + with map_exceptions(exc_map): + sock = socket.create_connection( + address, + timeout, + source_address=source_address, + ) + for option in socket_options: + sock.setsockopt(*option) # pragma: no cover + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + return SyncStream(sock) + + def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: # pragma: nocover + if sys.platform == "win32": + raise RuntimeError( + "Attempted to connect to a UNIX socket on a Windows system." + ) + if socket_options is None: + socket_options = [] + + exc_map: ExceptionMapping = { + socket.timeout: ConnectTimeout, + OSError: ConnectError, + } + with map_exceptions(exc_map): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + for option in socket_options: + sock.setsockopt(*option) + sock.settimeout(timeout) + sock.connect(path) + return SyncStream(sock) diff --git a/venv/lib/python3.12/site-packages/httpcore/_backends/trio.py b/venv/lib/python3.12/site-packages/httpcore/_backends/trio.py new file mode 100644 index 0000000000000000000000000000000000000000..6f53f5f2a025e01e9949e2530bd9ca6928859251 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_backends/trio.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import ssl +import typing + +import trio + +from .._exceptions import ( + ConnectError, + ConnectTimeout, + ExceptionMapping, + ReadError, + ReadTimeout, + WriteError, + WriteTimeout, + map_exceptions, +) +from .base import SOCKET_OPTION, AsyncNetworkBackend, AsyncNetworkStream + + +class TrioStream(AsyncNetworkStream): + def __init__(self, stream: trio.abc.Stream) -> None: + self._stream = stream + + async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + timeout_or_inf = float("inf") if timeout is None else timeout + exc_map: ExceptionMapping = { + trio.TooSlowError: ReadTimeout, + trio.BrokenResourceError: ReadError, + trio.ClosedResourceError: ReadError, + } + with map_exceptions(exc_map): + with trio.fail_after(timeout_or_inf): + data: bytes = await self._stream.receive_some(max_bytes=max_bytes) + return data + + async def write(self, buffer: bytes, timeout: float | None = None) -> None: + if not buffer: + return + + timeout_or_inf = float("inf") if timeout is None else timeout + exc_map: ExceptionMapping = { + trio.TooSlowError: WriteTimeout, + trio.BrokenResourceError: WriteError, + trio.ClosedResourceError: WriteError, + } + with map_exceptions(exc_map): + with trio.fail_after(timeout_or_inf): + await self._stream.send_all(data=buffer) + + async def aclose(self) -> None: + await self._stream.aclose() + + async def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> AsyncNetworkStream: + timeout_or_inf = float("inf") if timeout is None else timeout + exc_map: ExceptionMapping = { + trio.TooSlowError: ConnectTimeout, + trio.BrokenResourceError: ConnectError, + } + ssl_stream = trio.SSLStream( + self._stream, + ssl_context=ssl_context, + server_hostname=server_hostname, + https_compatible=True, + server_side=False, + ) + with map_exceptions(exc_map): + try: + with trio.fail_after(timeout_or_inf): + await ssl_stream.do_handshake() + except Exception as exc: # pragma: nocover + await self.aclose() + raise exc + return TrioStream(ssl_stream) + + def get_extra_info(self, info: str) -> typing.Any: + if info == "ssl_object" and isinstance(self._stream, trio.SSLStream): + # Type checkers cannot see `_ssl_object` attribute because trio._ssl.SSLStream uses __getattr__/__setattr__. + # Tracked at https://github.com/python-trio/trio/issues/542 + return self._stream._ssl_object # type: ignore[attr-defined] + if info == "client_addr": + return self._get_socket_stream().socket.getsockname() + if info == "server_addr": + return self._get_socket_stream().socket.getpeername() + if info == "socket": + stream = self._stream + while isinstance(stream, trio.SSLStream): + stream = stream.transport_stream + assert isinstance(stream, trio.SocketStream) + return stream.socket + if info == "is_readable": + socket = self.get_extra_info("socket") + return socket.is_readable() + return None + + def _get_socket_stream(self) -> trio.SocketStream: + stream = self._stream + while isinstance(stream, trio.SSLStream): + stream = stream.transport_stream + assert isinstance(stream, trio.SocketStream) + return stream + + +class TrioBackend(AsyncNetworkBackend): + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + # By default for TCP sockets, trio enables TCP_NODELAY. + # https://trio.readthedocs.io/en/stable/reference-io.html#trio.SocketStream + if socket_options is None: + socket_options = [] # pragma: no cover + timeout_or_inf = float("inf") if timeout is None else timeout + exc_map: ExceptionMapping = { + trio.TooSlowError: ConnectTimeout, + trio.BrokenResourceError: ConnectError, + OSError: ConnectError, + } + with map_exceptions(exc_map): + with trio.fail_after(timeout_or_inf): + stream: trio.abc.Stream = await trio.open_tcp_stream( + host=host, port=port, local_address=local_address + ) + for option in socket_options: + stream.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover + return TrioStream(stream) + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: # pragma: nocover + if socket_options is None: + socket_options = [] + timeout_or_inf = float("inf") if timeout is None else timeout + exc_map: ExceptionMapping = { + trio.TooSlowError: ConnectTimeout, + trio.BrokenResourceError: ConnectError, + OSError: ConnectError, + } + with map_exceptions(exc_map): + with trio.fail_after(timeout_or_inf): + stream: trio.abc.Stream = await trio.open_unix_socket(path) + for option in socket_options: + stream.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover + return TrioStream(stream) + + async def sleep(self, seconds: float) -> None: + await trio.sleep(seconds) # pragma: nocover diff --git a/venv/lib/python3.12/site-packages/httpcore/_exceptions.py b/venv/lib/python3.12/site-packages/httpcore/_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..bc28d44f55bdc4b872951a74780469a3999d9ab4 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_exceptions.py @@ -0,0 +1,81 @@ +import contextlib +import typing + +ExceptionMapping = typing.Mapping[typing.Type[Exception], typing.Type[Exception]] + + +@contextlib.contextmanager +def map_exceptions(map: ExceptionMapping) -> typing.Iterator[None]: + try: + yield + except Exception as exc: # noqa: PIE786 + for from_exc, to_exc in map.items(): + if isinstance(exc, from_exc): + raise to_exc(exc) from exc + raise # pragma: nocover + + +class ConnectionNotAvailable(Exception): + pass + + +class ProxyError(Exception): + pass + + +class UnsupportedProtocol(Exception): + pass + + +class ProtocolError(Exception): + pass + + +class RemoteProtocolError(ProtocolError): + pass + + +class LocalProtocolError(ProtocolError): + pass + + +# Timeout errors + + +class TimeoutException(Exception): + pass + + +class PoolTimeout(TimeoutException): + pass + + +class ConnectTimeout(TimeoutException): + pass + + +class ReadTimeout(TimeoutException): + pass + + +class WriteTimeout(TimeoutException): + pass + + +# Network errors + + +class NetworkError(Exception): + pass + + +class ConnectError(NetworkError): + pass + + +class ReadError(NetworkError): + pass + + +class WriteError(NetworkError): + pass diff --git a/venv/lib/python3.12/site-packages/httpcore/_models.py b/venv/lib/python3.12/site-packages/httpcore/_models.py new file mode 100644 index 0000000000000000000000000000000000000000..8a65f13347d6621289a166d08123cbc8e1ad0157 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_models.py @@ -0,0 +1,516 @@ +from __future__ import annotations + +import base64 +import ssl +import typing +import urllib.parse + +# Functions for typechecking... + + +ByteOrStr = typing.Union[bytes, str] +HeadersAsSequence = typing.Sequence[typing.Tuple[ByteOrStr, ByteOrStr]] +HeadersAsMapping = typing.Mapping[ByteOrStr, ByteOrStr] +HeaderTypes = typing.Union[HeadersAsSequence, HeadersAsMapping, None] + +Extensions = typing.MutableMapping[str, typing.Any] + + +def enforce_bytes(value: bytes | str, *, name: str) -> bytes: + """ + Any arguments that are ultimately represented as bytes can be specified + either as bytes or as strings. + + However we enforce that any string arguments must only contain characters in + the plain ASCII range. chr(0)...chr(127). If you need to use characters + outside that range then be precise, and use a byte-wise argument. + """ + if isinstance(value, str): + try: + return value.encode("ascii") + except UnicodeEncodeError: + raise TypeError(f"{name} strings may not include unicode characters.") + elif isinstance(value, bytes): + return value + + seen_type = type(value).__name__ + raise TypeError(f"{name} must be bytes or str, but got {seen_type}.") + + +def enforce_url(value: URL | bytes | str, *, name: str) -> URL: + """ + Type check for URL parameters. + """ + if isinstance(value, (bytes, str)): + return URL(value) + elif isinstance(value, URL): + return value + + seen_type = type(value).__name__ + raise TypeError(f"{name} must be a URL, bytes, or str, but got {seen_type}.") + + +def enforce_headers( + value: HeadersAsMapping | HeadersAsSequence | None = None, *, name: str +) -> list[tuple[bytes, bytes]]: + """ + Convienence function that ensure all items in request or response headers + are either bytes or strings in the plain ASCII range. + """ + if value is None: + return [] + elif isinstance(value, typing.Mapping): + return [ + ( + enforce_bytes(k, name="header name"), + enforce_bytes(v, name="header value"), + ) + for k, v in value.items() + ] + elif isinstance(value, typing.Sequence): + return [ + ( + enforce_bytes(k, name="header name"), + enforce_bytes(v, name="header value"), + ) + for k, v in value + ] + + seen_type = type(value).__name__ + raise TypeError( + f"{name} must be a mapping or sequence of two-tuples, but got {seen_type}." + ) + + +def enforce_stream( + value: bytes | typing.Iterable[bytes] | typing.AsyncIterable[bytes] | None, + *, + name: str, +) -> typing.Iterable[bytes] | typing.AsyncIterable[bytes]: + if value is None: + return ByteStream(b"") + elif isinstance(value, bytes): + return ByteStream(value) + return value + + +# * https://tools.ietf.org/html/rfc3986#section-3.2.3 +# * https://url.spec.whatwg.org/#url-miscellaneous +# * https://url.spec.whatwg.org/#scheme-state +DEFAULT_PORTS = { + b"ftp": 21, + b"http": 80, + b"https": 443, + b"ws": 80, + b"wss": 443, +} + + +def include_request_headers( + headers: list[tuple[bytes, bytes]], + *, + url: "URL", + content: None | bytes | typing.Iterable[bytes] | typing.AsyncIterable[bytes], +) -> list[tuple[bytes, bytes]]: + headers_set = set(k.lower() for k, v in headers) + + if b"host" not in headers_set: + default_port = DEFAULT_PORTS.get(url.scheme) + if url.port is None or url.port == default_port: + header_value = url.host + else: + header_value = b"%b:%d" % (url.host, url.port) + headers = [(b"Host", header_value)] + headers + + if ( + content is not None + and b"content-length" not in headers_set + and b"transfer-encoding" not in headers_set + ): + if isinstance(content, bytes): + content_length = str(len(content)).encode("ascii") + headers += [(b"Content-Length", content_length)] + else: + headers += [(b"Transfer-Encoding", b"chunked")] # pragma: nocover + + return headers + + +# Interfaces for byte streams... + + +class ByteStream: + """ + A container for non-streaming content, and that supports both sync and async + stream iteration. + """ + + def __init__(self, content: bytes) -> None: + self._content = content + + def __iter__(self) -> typing.Iterator[bytes]: + yield self._content + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + yield self._content + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{len(self._content)} bytes]>" + + +class Origin: + def __init__(self, scheme: bytes, host: bytes, port: int) -> None: + self.scheme = scheme + self.host = host + self.port = port + + def __eq__(self, other: typing.Any) -> bool: + return ( + isinstance(other, Origin) + and self.scheme == other.scheme + and self.host == other.host + and self.port == other.port + ) + + def __str__(self) -> str: + scheme = self.scheme.decode("ascii") + host = self.host.decode("ascii") + port = str(self.port) + return f"{scheme}://{host}:{port}" + + +class URL: + """ + Represents the URL against which an HTTP request may be made. + + The URL may either be specified as a plain string, for convienence: + + ```python + url = httpcore.URL("https://www.example.com/") + ``` + + Or be constructed with explicitily pre-parsed components: + + ```python + url = httpcore.URL(scheme=b'https', host=b'www.example.com', port=None, target=b'/') + ``` + + Using this second more explicit style allows integrations that are using + `httpcore` to pass through URLs that have already been parsed in order to use + libraries such as `rfc-3986` rather than relying on the stdlib. It also ensures + that URL parsing is treated identically at both the networking level and at any + higher layers of abstraction. + + The four components are important here, as they allow the URL to be precisely + specified in a pre-parsed format. They also allow certain types of request to + be created that could not otherwise be expressed. + + For example, an HTTP request to `http://www.example.com/` forwarded via a proxy + at `http://localhost:8080`... + + ```python + # Constructs an HTTP request with a complete URL as the target: + # GET https://www.example.com/ HTTP/1.1 + url = httpcore.URL( + scheme=b'http', + host=b'localhost', + port=8080, + target=b'https://www.example.com/' + ) + request = httpcore.Request( + method="GET", + url=url + ) + ``` + + Another example is constructing an `OPTIONS *` request... + + ```python + # Constructs an 'OPTIONS *' HTTP request: + # OPTIONS * HTTP/1.1 + url = httpcore.URL(scheme=b'https', host=b'www.example.com', target=b'*') + request = httpcore.Request(method="OPTIONS", url=url) + ``` + + This kind of request is not possible to formulate with a URL string, + because the `/` delimiter is always used to demark the target from the + host/port portion of the URL. + + For convenience, string-like arguments may be specified either as strings or + as bytes. However, once a request is being issue over-the-wire, the URL + components are always ultimately required to be a bytewise representation. + + In order to avoid any ambiguity over character encodings, when strings are used + as arguments, they must be strictly limited to the ASCII range `chr(0)`-`chr(127)`. + If you require a bytewise representation that is outside this range you must + handle the character encoding directly, and pass a bytes instance. + """ + + def __init__( + self, + url: bytes | str = "", + *, + scheme: bytes | str = b"", + host: bytes | str = b"", + port: int | None = None, + target: bytes | str = b"", + ) -> None: + """ + Parameters: + url: The complete URL as a string or bytes. + scheme: The URL scheme as a string or bytes. + Typically either `"http"` or `"https"`. + host: The URL host as a string or bytes. Such as `"www.example.com"`. + port: The port to connect to. Either an integer or `None`. + target: The target of the HTTP request. Such as `"/items?search=red"`. + """ + if url: + parsed = urllib.parse.urlparse(enforce_bytes(url, name="url")) + self.scheme = parsed.scheme + self.host = parsed.hostname or b"" + self.port = parsed.port + self.target = (parsed.path or b"/") + ( + b"?" + parsed.query if parsed.query else b"" + ) + else: + self.scheme = enforce_bytes(scheme, name="scheme") + self.host = enforce_bytes(host, name="host") + self.port = port + self.target = enforce_bytes(target, name="target") + + @property + def origin(self) -> Origin: + default_port = { + b"http": 80, + b"https": 443, + b"ws": 80, + b"wss": 443, + b"socks5": 1080, + b"socks5h": 1080, + }[self.scheme] + return Origin( + scheme=self.scheme, host=self.host, port=self.port or default_port + ) + + def __eq__(self, other: typing.Any) -> bool: + return ( + isinstance(other, URL) + and other.scheme == self.scheme + and other.host == self.host + and other.port == self.port + and other.target == self.target + ) + + def __bytes__(self) -> bytes: + if self.port is None: + return b"%b://%b%b" % (self.scheme, self.host, self.target) + return b"%b://%b:%d%b" % (self.scheme, self.host, self.port, self.target) + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(scheme={self.scheme!r}, " + f"host={self.host!r}, port={self.port!r}, target={self.target!r})" + ) + + +class Request: + """ + An HTTP request. + """ + + def __init__( + self, + method: bytes | str, + url: URL | bytes | str, + *, + headers: HeaderTypes = None, + content: bytes + | typing.Iterable[bytes] + | typing.AsyncIterable[bytes] + | None = None, + extensions: Extensions | None = None, + ) -> None: + """ + Parameters: + method: The HTTP request method, either as a string or bytes. + For example: `GET`. + url: The request URL, either as a `URL` instance, or as a string or bytes. + For example: `"https://www.example.com".` + headers: The HTTP request headers. + content: The content of the request body. + extensions: A dictionary of optional extra information included on + the request. Possible keys include `"timeout"`, and `"trace"`. + """ + self.method: bytes = enforce_bytes(method, name="method") + self.url: URL = enforce_url(url, name="url") + self.headers: list[tuple[bytes, bytes]] = enforce_headers( + headers, name="headers" + ) + self.stream: typing.Iterable[bytes] | typing.AsyncIterable[bytes] = ( + enforce_stream(content, name="content") + ) + self.extensions = {} if extensions is None else extensions + + if "target" in self.extensions: + self.url = URL( + scheme=self.url.scheme, + host=self.url.host, + port=self.url.port, + target=self.extensions["target"], + ) + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{self.method!r}]>" + + +class Response: + """ + An HTTP response. + """ + + def __init__( + self, + status: int, + *, + headers: HeaderTypes = None, + content: bytes + | typing.Iterable[bytes] + | typing.AsyncIterable[bytes] + | None = None, + extensions: Extensions | None = None, + ) -> None: + """ + Parameters: + status: The HTTP status code of the response. For example `200`. + headers: The HTTP response headers. + content: The content of the response body. + extensions: A dictionary of optional extra information included on + the responseself.Possible keys include `"http_version"`, + `"reason_phrase"`, and `"network_stream"`. + """ + self.status: int = status + self.headers: list[tuple[bytes, bytes]] = enforce_headers( + headers, name="headers" + ) + self.stream: typing.Iterable[bytes] | typing.AsyncIterable[bytes] = ( + enforce_stream(content, name="content") + ) + self.extensions = {} if extensions is None else extensions + + self._stream_consumed = False + + @property + def content(self) -> bytes: + if not hasattr(self, "_content"): + if isinstance(self.stream, typing.Iterable): + raise RuntimeError( + "Attempted to access 'response.content' on a streaming response. " + "Call 'response.read()' first." + ) + else: + raise RuntimeError( + "Attempted to access 'response.content' on a streaming response. " + "Call 'await response.aread()' first." + ) + return self._content + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{self.status}]>" + + # Sync interface... + + def read(self) -> bytes: + if not isinstance(self.stream, typing.Iterable): # pragma: nocover + raise RuntimeError( + "Attempted to read an asynchronous response using 'response.read()'. " + "You should use 'await response.aread()' instead." + ) + if not hasattr(self, "_content"): + self._content = b"".join([part for part in self.iter_stream()]) + return self._content + + def iter_stream(self) -> typing.Iterator[bytes]: + if not isinstance(self.stream, typing.Iterable): # pragma: nocover + raise RuntimeError( + "Attempted to stream an asynchronous response using 'for ... in " + "response.iter_stream()'. " + "You should use 'async for ... in response.aiter_stream()' instead." + ) + if self._stream_consumed: + raise RuntimeError( + "Attempted to call 'for ... in response.iter_stream()' more than once." + ) + self._stream_consumed = True + for chunk in self.stream: + yield chunk + + def close(self) -> None: + if not isinstance(self.stream, typing.Iterable): # pragma: nocover + raise RuntimeError( + "Attempted to close an asynchronous response using 'response.close()'. " + "You should use 'await response.aclose()' instead." + ) + if hasattr(self.stream, "close"): + self.stream.close() + + # Async interface... + + async def aread(self) -> bytes: + if not isinstance(self.stream, typing.AsyncIterable): # pragma: nocover + raise RuntimeError( + "Attempted to read an synchronous response using " + "'await response.aread()'. " + "You should use 'response.read()' instead." + ) + if not hasattr(self, "_content"): + self._content = b"".join([part async for part in self.aiter_stream()]) + return self._content + + async def aiter_stream(self) -> typing.AsyncIterator[bytes]: + if not isinstance(self.stream, typing.AsyncIterable): # pragma: nocover + raise RuntimeError( + "Attempted to stream an synchronous response using 'async for ... in " + "response.aiter_stream()'. " + "You should use 'for ... in response.iter_stream()' instead." + ) + if self._stream_consumed: + raise RuntimeError( + "Attempted to call 'async for ... in response.aiter_stream()' " + "more than once." + ) + self._stream_consumed = True + async for chunk in self.stream: + yield chunk + + async def aclose(self) -> None: + if not isinstance(self.stream, typing.AsyncIterable): # pragma: nocover + raise RuntimeError( + "Attempted to close a synchronous response using " + "'await response.aclose()'. " + "You should use 'response.close()' instead." + ) + if hasattr(self.stream, "aclose"): + await self.stream.aclose() + + +class Proxy: + def __init__( + self, + url: URL | bytes | str, + auth: tuple[bytes | str, bytes | str] | None = None, + headers: HeadersAsMapping | HeadersAsSequence | None = None, + ssl_context: ssl.SSLContext | None = None, + ): + self.url = enforce_url(url, name="url") + self.headers = enforce_headers(headers, name="headers") + self.ssl_context = ssl_context + + if auth is not None: + username = enforce_bytes(auth[0], name="auth") + password = enforce_bytes(auth[1], name="auth") + userpass = username + b":" + password + authorization = b"Basic " + base64.b64encode(userpass) + self.auth: tuple[bytes, bytes] | None = (username, password) + self.headers = [(b"Proxy-Authorization", authorization)] + self.headers + else: + self.auth = None diff --git a/venv/lib/python3.12/site-packages/httpcore/_ssl.py b/venv/lib/python3.12/site-packages/httpcore/_ssl.py new file mode 100644 index 0000000000000000000000000000000000000000..c99c5a67945b8a3a3544d481e979c791ab45fe23 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_ssl.py @@ -0,0 +1,9 @@ +import ssl + +import certifi + + +def default_ssl_context() -> ssl.SSLContext: + context = ssl.create_default_context() + context.load_verify_locations(certifi.where()) + return context diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/__init__.py b/venv/lib/python3.12/site-packages/httpcore/_sync/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b476d76d9a7ff45de8d18ec22d33d6af2982f92e --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_sync/__init__.py @@ -0,0 +1,39 @@ +from .connection import HTTPConnection +from .connection_pool import ConnectionPool +from .http11 import HTTP11Connection +from .http_proxy import HTTPProxy +from .interfaces import ConnectionInterface + +try: + from .http2 import HTTP2Connection +except ImportError: # pragma: nocover + + class HTTP2Connection: # type: ignore + def __init__(self, *args, **kwargs) -> None: # type: ignore + raise RuntimeError( + "Attempted to use http2 support, but the `h2` package is not " + "installed. Use 'pip install httpcore[http2]'." + ) + + +try: + from .socks_proxy import SOCKSProxy +except ImportError: # pragma: nocover + + class SOCKSProxy: # type: ignore + def __init__(self, *args, **kwargs) -> None: # type: ignore + raise RuntimeError( + "Attempted to use SOCKS support, but the `socksio` package is not " + "installed. Use 'pip install httpcore[socks]'." + ) + + +__all__ = [ + "HTTPConnection", + "ConnectionPool", + "HTTPProxy", + "HTTP11Connection", + "HTTP2Connection", + "ConnectionInterface", + "SOCKSProxy", +] diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..80600e6a2dc584c1d9f54753553aa359af910e2b Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/connection.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/connection.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..97003e602ef4214995bc069e50baf5de8834389b Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/connection.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/connection_pool.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/connection_pool.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..392c880c90e709d3675c9490b322210d1e2f5e06 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/connection_pool.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/http11.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/http11.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5692c5f93b902bba8dea988c7f3d6cd625c98bf7 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/http11.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/http2.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/http2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..79e4646c39d6328884f34611830943f1deb9b815 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/http2.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/http_proxy.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/http_proxy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb7cfc5535b8d495de1d67cc61fd6ced712c5447 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/http_proxy.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/interfaces.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/interfaces.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b2c7302d820112b302a1b1b3cebd0c9b712c9d5 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/interfaces.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/socks_proxy.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/socks_proxy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..182b0ec8b162a7c026e41b285cf12bb9c4b4e9dc Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpcore/_sync/__pycache__/socks_proxy.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/connection.py b/venv/lib/python3.12/site-packages/httpcore/_sync/connection.py new file mode 100644 index 0000000000000000000000000000000000000000..363f8be819d2576ea65365e625dd1596ea40429a --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_sync/connection.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import itertools +import logging +import ssl +import types +import typing + +from .._backends.sync import SyncBackend +from .._backends.base import SOCKET_OPTION, NetworkBackend, NetworkStream +from .._exceptions import ConnectError, ConnectTimeout +from .._models import Origin, Request, Response +from .._ssl import default_ssl_context +from .._synchronization import Lock +from .._trace import Trace +from .http11 import HTTP11Connection +from .interfaces import ConnectionInterface + +RETRIES_BACKOFF_FACTOR = 0.5 # 0s, 0.5s, 1s, 2s, 4s, etc. + + +logger = logging.getLogger("httpcore.connection") + + +def exponential_backoff(factor: float) -> typing.Iterator[float]: + """ + Generate a geometric sequence that has a ratio of 2 and starts with 0. + + For example: + - `factor = 2`: `0, 2, 4, 8, 16, 32, 64, ...` + - `factor = 3`: `0, 3, 6, 12, 24, 48, 96, ...` + """ + yield 0 + for n in itertools.count(): + yield factor * 2**n + + +class HTTPConnection(ConnectionInterface): + def __init__( + self, + origin: Origin, + ssl_context: ssl.SSLContext | None = None, + keepalive_expiry: float | None = None, + http1: bool = True, + http2: bool = False, + retries: int = 0, + local_address: str | None = None, + uds: str | None = None, + network_backend: NetworkBackend | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + self._origin = origin + self._ssl_context = ssl_context + self._keepalive_expiry = keepalive_expiry + self._http1 = http1 + self._http2 = http2 + self._retries = retries + self._local_address = local_address + self._uds = uds + + self._network_backend: NetworkBackend = ( + SyncBackend() if network_backend is None else network_backend + ) + self._connection: ConnectionInterface | None = None + self._connect_failed: bool = False + self._request_lock = Lock() + self._socket_options = socket_options + + def handle_request(self, request: Request) -> Response: + if not self.can_handle_request(request.url.origin): + raise RuntimeError( + f"Attempted to send request to {request.url.origin} on connection to {self._origin}" + ) + + try: + with self._request_lock: + if self._connection is None: + stream = self._connect(request) + + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) + if http2_negotiated or (self._http2 and not self._http1): + from .http2 import HTTP2Connection + + self._connection = HTTP2Connection( + origin=self._origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + else: + self._connection = HTTP11Connection( + origin=self._origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + except BaseException as exc: + self._connect_failed = True + raise exc + + return self._connection.handle_request(request) + + def _connect(self, request: Request) -> NetworkStream: + timeouts = request.extensions.get("timeout", {}) + sni_hostname = request.extensions.get("sni_hostname", None) + timeout = timeouts.get("connect", None) + + retries_left = self._retries + delays = exponential_backoff(factor=RETRIES_BACKOFF_FACTOR) + + while True: + try: + if self._uds is None: + kwargs = { + "host": self._origin.host.decode("ascii"), + "port": self._origin.port, + "local_address": self._local_address, + "timeout": timeout, + "socket_options": self._socket_options, + } + with Trace("connect_tcp", logger, request, kwargs) as trace: + stream = self._network_backend.connect_tcp(**kwargs) + trace.return_value = stream + else: + kwargs = { + "path": self._uds, + "timeout": timeout, + "socket_options": self._socket_options, + } + with Trace( + "connect_unix_socket", logger, request, kwargs + ) as trace: + stream = self._network_backend.connect_unix_socket( + **kwargs + ) + trace.return_value = stream + + if self._origin.scheme in (b"https", b"wss"): + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": sni_hostname + or self._origin.host.decode("ascii"), + "timeout": timeout, + } + with Trace("start_tls", logger, request, kwargs) as trace: + stream = stream.start_tls(**kwargs) + trace.return_value = stream + return stream + except (ConnectError, ConnectTimeout): + if retries_left <= 0: + raise + retries_left -= 1 + delay = next(delays) + with Trace("retry", logger, request, kwargs) as trace: + self._network_backend.sleep(delay) + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._origin + + def close(self) -> None: + if self._connection is not None: + with Trace("close", logger, None, {}): + self._connection.close() + + def is_available(self) -> bool: + if self._connection is None: + # If HTTP/2 support is enabled, and the resulting connection could + # end up as HTTP/2 then we should indicate the connection as being + # available to service multiple requests. + return ( + self._http2 + and (self._origin.scheme == b"https" or not self._http1) + and not self._connect_failed + ) + return self._connection.is_available() + + def has_expired(self) -> bool: + if self._connection is None: + return self._connect_failed + return self._connection.has_expired() + + def is_idle(self) -> bool: + if self._connection is None: + return self._connect_failed + return self._connection.is_idle() + + def is_closed(self) -> bool: + if self._connection is None: + return self._connect_failed + return self._connection.is_closed() + + def info(self) -> str: + if self._connection is None: + return "CONNECTION FAILED" if self._connect_failed else "CONNECTING" + return self._connection.info() + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{self.info()}]>" + + # These context managers are not used in the standard flow, but are + # useful for testing or working with connection instances directly. + + def __enter__(self) -> HTTPConnection: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + self.close() diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/connection_pool.py b/venv/lib/python3.12/site-packages/httpcore/_sync/connection_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..9ccfa53e597a29ee387f9d16f3af4f695ac0d33a --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_sync/connection_pool.py @@ -0,0 +1,420 @@ +from __future__ import annotations + +import ssl +import sys +import types +import typing + +from .._backends.sync import SyncBackend +from .._backends.base import SOCKET_OPTION, NetworkBackend +from .._exceptions import ConnectionNotAvailable, UnsupportedProtocol +from .._models import Origin, Proxy, Request, Response +from .._synchronization import Event, ShieldCancellation, ThreadLock +from .connection import HTTPConnection +from .interfaces import ConnectionInterface, RequestInterface + + +class PoolRequest: + def __init__(self, request: Request) -> None: + self.request = request + self.connection: ConnectionInterface | None = None + self._connection_acquired = Event() + + def assign_to_connection(self, connection: ConnectionInterface | None) -> None: + self.connection = connection + self._connection_acquired.set() + + def clear_connection(self) -> None: + self.connection = None + self._connection_acquired = Event() + + def wait_for_connection( + self, timeout: float | None = None + ) -> ConnectionInterface: + if self.connection is None: + self._connection_acquired.wait(timeout=timeout) + assert self.connection is not None + return self.connection + + def is_queued(self) -> bool: + return self.connection is None + + +class ConnectionPool(RequestInterface): + """ + A connection pool for making HTTP requests. + """ + + def __init__( + self, + ssl_context: ssl.SSLContext | None = None, + proxy: Proxy | None = None, + max_connections: int | None = 10, + max_keepalive_connections: int | None = None, + keepalive_expiry: float | None = None, + http1: bool = True, + http2: bool = False, + retries: int = 0, + local_address: str | None = None, + uds: str | None = None, + network_backend: NetworkBackend | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + """ + A connection pool for making HTTP requests. + + Parameters: + ssl_context: An SSL context to use for verifying connections. + If not specified, the default `httpcore.default_ssl_context()` + will be used. + max_connections: The maximum number of concurrent HTTP connections that + the pool should allow. Any attempt to send a request on a pool that + would exceed this amount will block until a connection is available. + max_keepalive_connections: The maximum number of idle HTTP connections + that will be maintained in the pool. + keepalive_expiry: The duration in seconds that an idle HTTP connection + may be maintained for before being expired from the pool. + http1: A boolean indicating if HTTP/1.1 requests should be supported + by the connection pool. Defaults to True. + http2: A boolean indicating if HTTP/2 requests should be supported by + the connection pool. Defaults to False. + retries: The maximum number of retries when trying to establish a + connection. + local_address: Local address to connect from. Can also be used to connect + using a particular address family. Using `local_address="0.0.0.0"` + will connect using an `AF_INET` address (IPv4), while using + `local_address="::"` will connect using an `AF_INET6` address (IPv6). + uds: Path to a Unix Domain Socket to use instead of TCP sockets. + network_backend: A backend instance to use for handling network I/O. + socket_options: Socket options that have to be included + in the TCP socket when the connection was established. + """ + self._ssl_context = ssl_context + self._proxy = proxy + self._max_connections = ( + sys.maxsize if max_connections is None else max_connections + ) + self._max_keepalive_connections = ( + sys.maxsize + if max_keepalive_connections is None + else max_keepalive_connections + ) + self._max_keepalive_connections = min( + self._max_connections, self._max_keepalive_connections + ) + + self._keepalive_expiry = keepalive_expiry + self._http1 = http1 + self._http2 = http2 + self._retries = retries + self._local_address = local_address + self._uds = uds + + self._network_backend = ( + SyncBackend() if network_backend is None else network_backend + ) + self._socket_options = socket_options + + # The mutable state on a connection pool is the queue of incoming requests, + # and the set of connections that are servicing those requests. + self._connections: list[ConnectionInterface] = [] + self._requests: list[PoolRequest] = [] + + # We only mutate the state of the connection pool within an 'optional_thread_lock' + # context. This holds a threading lock unless we're running in async mode, + # in which case it is a no-op. + self._optional_thread_lock = ThreadLock() + + def create_connection(self, origin: Origin) -> ConnectionInterface: + if self._proxy is not None: + if self._proxy.url.scheme in (b"socks5", b"socks5h"): + from .socks_proxy import Socks5Connection + + return Socks5Connection( + proxy_origin=self._proxy.url.origin, + proxy_auth=self._proxy.auth, + remote_origin=origin, + ssl_context=self._ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + elif origin.scheme == b"http": + from .http_proxy import ForwardHTTPConnection + + return ForwardHTTPConnection( + proxy_origin=self._proxy.url.origin, + proxy_headers=self._proxy.headers, + proxy_ssl_context=self._proxy.ssl_context, + remote_origin=origin, + keepalive_expiry=self._keepalive_expiry, + network_backend=self._network_backend, + ) + from .http_proxy import TunnelHTTPConnection + + return TunnelHTTPConnection( + proxy_origin=self._proxy.url.origin, + proxy_headers=self._proxy.headers, + proxy_ssl_context=self._proxy.ssl_context, + remote_origin=origin, + ssl_context=self._ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + + return HTTPConnection( + origin=origin, + ssl_context=self._ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + retries=self._retries, + local_address=self._local_address, + uds=self._uds, + network_backend=self._network_backend, + socket_options=self._socket_options, + ) + + @property + def connections(self) -> list[ConnectionInterface]: + """ + Return a list of the connections currently in the pool. + + For example: + + ```python + >>> pool.connections + [ + , + , + , + ] + ``` + """ + return list(self._connections) + + def handle_request(self, request: Request) -> Response: + """ + Send an HTTP request, and return an HTTP response. + + This is the core implementation that is called into by `.request()` or `.stream()`. + """ + scheme = request.url.scheme.decode() + if scheme == "": + raise UnsupportedProtocol( + "Request URL is missing an 'http://' or 'https://' protocol." + ) + if scheme not in ("http", "https", "ws", "wss"): + raise UnsupportedProtocol( + f"Request URL has an unsupported protocol '{scheme}://'." + ) + + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("pool", None) + + with self._optional_thread_lock: + # Add the incoming request to our request queue. + pool_request = PoolRequest(request) + self._requests.append(pool_request) + + try: + while True: + with self._optional_thread_lock: + # Assign incoming requests to available connections, + # closing or creating new connections as required. + closing = self._assign_requests_to_connections() + self._close_connections(closing) + + # Wait until this request has an assigned connection. + connection = pool_request.wait_for_connection(timeout=timeout) + + try: + # Send the request on the assigned connection. + response = connection.handle_request( + pool_request.request + ) + except ConnectionNotAvailable: + # In some cases a connection may initially be available to + # handle a request, but then become unavailable. + # + # In this case we clear the connection and try again. + pool_request.clear_connection() + else: + break # pragma: nocover + + except BaseException as exc: + with self._optional_thread_lock: + # For any exception or cancellation we remove the request from + # the queue, and then re-assign requests to connections. + self._requests.remove(pool_request) + closing = self._assign_requests_to_connections() + + self._close_connections(closing) + raise exc from None + + # Return the response. Note that in this case we still have to manage + # the point at which the response is closed. + assert isinstance(response.stream, typing.Iterable) + return Response( + status=response.status, + headers=response.headers, + content=PoolByteStream( + stream=response.stream, pool_request=pool_request, pool=self + ), + extensions=response.extensions, + ) + + def _assign_requests_to_connections(self) -> list[ConnectionInterface]: + """ + Manage the state of the connection pool, assigning incoming + requests to connections as available. + + Called whenever a new request is added or removed from the pool. + + Any closing connections are returned, allowing the I/O for closing + those connections to be handled seperately. + """ + closing_connections = [] + + # First we handle cleaning up any connections that are closed, + # have expired their keep-alive, or surplus idle connections. + for connection in list(self._connections): + if connection.is_closed(): + # log: "removing closed connection" + self._connections.remove(connection) + elif connection.has_expired(): + # log: "closing expired connection" + self._connections.remove(connection) + closing_connections.append(connection) + elif ( + connection.is_idle() + and len([connection.is_idle() for connection in self._connections]) + > self._max_keepalive_connections + ): + # log: "closing idle connection" + self._connections.remove(connection) + closing_connections.append(connection) + + # Assign queued requests to connections. + queued_requests = [request for request in self._requests if request.is_queued()] + for pool_request in queued_requests: + origin = pool_request.request.url.origin + available_connections = [ + connection + for connection in self._connections + if connection.can_handle_request(origin) and connection.is_available() + ] + idle_connections = [ + connection for connection in self._connections if connection.is_idle() + ] + + # There are three cases for how we may be able to handle the request: + # + # 1. There is an existing connection that can handle the request. + # 2. We can create a new connection to handle the request. + # 3. We can close an idle connection and then create a new connection + # to handle the request. + if available_connections: + # log: "reusing existing connection" + connection = available_connections[0] + pool_request.assign_to_connection(connection) + elif len(self._connections) < self._max_connections: + # log: "creating new connection" + connection = self.create_connection(origin) + self._connections.append(connection) + pool_request.assign_to_connection(connection) + elif idle_connections: + # log: "closing idle connection" + connection = idle_connections[0] + self._connections.remove(connection) + closing_connections.append(connection) + # log: "creating new connection" + connection = self.create_connection(origin) + self._connections.append(connection) + pool_request.assign_to_connection(connection) + + return closing_connections + + def _close_connections(self, closing: list[ConnectionInterface]) -> None: + # Close connections which have been removed from the pool. + with ShieldCancellation(): + for connection in closing: + connection.close() + + def close(self) -> None: + # Explicitly close the connection pool. + # Clears all existing requests and connections. + with self._optional_thread_lock: + closing_connections = list(self._connections) + self._connections = [] + self._close_connections(closing_connections) + + def __enter__(self) -> ConnectionPool: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + self.close() + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + with self._optional_thread_lock: + request_is_queued = [request.is_queued() for request in self._requests] + connection_is_idle = [ + connection.is_idle() for connection in self._connections + ] + + num_active_requests = request_is_queued.count(False) + num_queued_requests = request_is_queued.count(True) + num_active_connections = connection_is_idle.count(False) + num_idle_connections = connection_is_idle.count(True) + + requests_info = ( + f"Requests: {num_active_requests} active, {num_queued_requests} queued" + ) + connection_info = ( + f"Connections: {num_active_connections} active, {num_idle_connections} idle" + ) + + return f"<{class_name} [{requests_info} | {connection_info}]>" + + +class PoolByteStream: + def __init__( + self, + stream: typing.Iterable[bytes], + pool_request: PoolRequest, + pool: ConnectionPool, + ) -> None: + self._stream = stream + self._pool_request = pool_request + self._pool = pool + self._closed = False + + def __iter__(self) -> typing.Iterator[bytes]: + try: + for part in self._stream: + yield part + except BaseException as exc: + self.close() + raise exc from None + + def close(self) -> None: + if not self._closed: + self._closed = True + with ShieldCancellation(): + if hasattr(self._stream, "close"): + self._stream.close() + + with self._pool._optional_thread_lock: + self._pool._requests.remove(self._pool_request) + closing = self._pool._assign_requests_to_connections() + + self._pool._close_connections(closing) diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/http11.py b/venv/lib/python3.12/site-packages/httpcore/_sync/http11.py new file mode 100644 index 0000000000000000000000000000000000000000..ebd3a97480c720d418acb1285a7b75da19b62c8c --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_sync/http11.py @@ -0,0 +1,379 @@ +from __future__ import annotations + +import enum +import logging +import ssl +import time +import types +import typing + +import h11 + +from .._backends.base import NetworkStream +from .._exceptions import ( + ConnectionNotAvailable, + LocalProtocolError, + RemoteProtocolError, + WriteError, + map_exceptions, +) +from .._models import Origin, Request, Response +from .._synchronization import Lock, ShieldCancellation +from .._trace import Trace +from .interfaces import ConnectionInterface + +logger = logging.getLogger("httpcore.http11") + + +# A subset of `h11.Event` types supported by `_send_event` +H11SendEvent = typing.Union[ + h11.Request, + h11.Data, + h11.EndOfMessage, +] + + +class HTTPConnectionState(enum.IntEnum): + NEW = 0 + ACTIVE = 1 + IDLE = 2 + CLOSED = 3 + + +class HTTP11Connection(ConnectionInterface): + READ_NUM_BYTES = 64 * 1024 + MAX_INCOMPLETE_EVENT_SIZE = 100 * 1024 + + def __init__( + self, + origin: Origin, + stream: NetworkStream, + keepalive_expiry: float | None = None, + ) -> None: + self._origin = origin + self._network_stream = stream + self._keepalive_expiry: float | None = keepalive_expiry + self._expire_at: float | None = None + self._state = HTTPConnectionState.NEW + self._state_lock = Lock() + self._request_count = 0 + self._h11_state = h11.Connection( + our_role=h11.CLIENT, + max_incomplete_event_size=self.MAX_INCOMPLETE_EVENT_SIZE, + ) + + def handle_request(self, request: Request) -> Response: + if not self.can_handle_request(request.url.origin): + raise RuntimeError( + f"Attempted to send request to {request.url.origin} on connection " + f"to {self._origin}" + ) + + with self._state_lock: + if self._state in (HTTPConnectionState.NEW, HTTPConnectionState.IDLE): + self._request_count += 1 + self._state = HTTPConnectionState.ACTIVE + self._expire_at = None + else: + raise ConnectionNotAvailable() + + try: + kwargs = {"request": request} + try: + with Trace( + "send_request_headers", logger, request, kwargs + ) as trace: + self._send_request_headers(**kwargs) + with Trace("send_request_body", logger, request, kwargs) as trace: + self._send_request_body(**kwargs) + except WriteError: + # If we get a write error while we're writing the request, + # then we supress this error and move on to attempting to + # read the response. Servers can sometimes close the request + # pre-emptively and then respond with a well formed HTTP + # error response. + pass + + with Trace( + "receive_response_headers", logger, request, kwargs + ) as trace: + ( + http_version, + status, + reason_phrase, + headers, + trailing_data, + ) = self._receive_response_headers(**kwargs) + trace.return_value = ( + http_version, + status, + reason_phrase, + headers, + ) + + network_stream = self._network_stream + + # CONNECT or Upgrade request + if (status == 101) or ( + (request.method == b"CONNECT") and (200 <= status < 300) + ): + network_stream = HTTP11UpgradeStream(network_stream, trailing_data) + + return Response( + status=status, + headers=headers, + content=HTTP11ConnectionByteStream(self, request), + extensions={ + "http_version": http_version, + "reason_phrase": reason_phrase, + "network_stream": network_stream, + }, + ) + except BaseException as exc: + with ShieldCancellation(): + with Trace("response_closed", logger, request) as trace: + self._response_closed() + raise exc + + # Sending the request... + + def _send_request_headers(self, request: Request) -> None: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("write", None) + + with map_exceptions({h11.LocalProtocolError: LocalProtocolError}): + event = h11.Request( + method=request.method, + target=request.url.target, + headers=request.headers, + ) + self._send_event(event, timeout=timeout) + + def _send_request_body(self, request: Request) -> None: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("write", None) + + assert isinstance(request.stream, typing.Iterable) + for chunk in request.stream: + event = h11.Data(data=chunk) + self._send_event(event, timeout=timeout) + + self._send_event(h11.EndOfMessage(), timeout=timeout) + + def _send_event(self, event: h11.Event, timeout: float | None = None) -> None: + bytes_to_send = self._h11_state.send(event) + if bytes_to_send is not None: + self._network_stream.write(bytes_to_send, timeout=timeout) + + # Receiving the response... + + def _receive_response_headers( + self, request: Request + ) -> tuple[bytes, int, bytes, list[tuple[bytes, bytes]], bytes]: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("read", None) + + while True: + event = self._receive_event(timeout=timeout) + if isinstance(event, h11.Response): + break + if ( + isinstance(event, h11.InformationalResponse) + and event.status_code == 101 + ): + break + + http_version = b"HTTP/" + event.http_version + + # h11 version 0.11+ supports a `raw_items` interface to get the + # raw header casing, rather than the enforced lowercase headers. + headers = event.headers.raw_items() + + trailing_data, _ = self._h11_state.trailing_data + + return http_version, event.status_code, event.reason, headers, trailing_data + + def _receive_response_body( + self, request: Request + ) -> typing.Iterator[bytes]: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("read", None) + + while True: + event = self._receive_event(timeout=timeout) + if isinstance(event, h11.Data): + yield bytes(event.data) + elif isinstance(event, (h11.EndOfMessage, h11.PAUSED)): + break + + def _receive_event( + self, timeout: float | None = None + ) -> h11.Event | type[h11.PAUSED]: + while True: + with map_exceptions({h11.RemoteProtocolError: RemoteProtocolError}): + event = self._h11_state.next_event() + + if event is h11.NEED_DATA: + data = self._network_stream.read( + self.READ_NUM_BYTES, timeout=timeout + ) + + # If we feed this case through h11 we'll raise an exception like: + # + # httpcore.RemoteProtocolError: can't handle event type + # ConnectionClosed when role=SERVER and state=SEND_RESPONSE + # + # Which is accurate, but not very informative from an end-user + # perspective. Instead we handle this case distinctly and treat + # it as a ConnectError. + if data == b"" and self._h11_state.their_state == h11.SEND_RESPONSE: + msg = "Server disconnected without sending a response." + raise RemoteProtocolError(msg) + + self._h11_state.receive_data(data) + else: + # mypy fails to narrow the type in the above if statement above + return event # type: ignore[return-value] + + def _response_closed(self) -> None: + with self._state_lock: + if ( + self._h11_state.our_state is h11.DONE + and self._h11_state.their_state is h11.DONE + ): + self._state = HTTPConnectionState.IDLE + self._h11_state.start_next_cycle() + if self._keepalive_expiry is not None: + now = time.monotonic() + self._expire_at = now + self._keepalive_expiry + else: + self.close() + + # Once the connection is no longer required... + + def close(self) -> None: + # Note that this method unilaterally closes the connection, and does + # not have any kind of locking in place around it. + self._state = HTTPConnectionState.CLOSED + self._network_stream.close() + + # The ConnectionInterface methods provide information about the state of + # the connection, allowing for a connection pooling implementation to + # determine when to reuse and when to close the connection... + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._origin + + def is_available(self) -> bool: + # Note that HTTP/1.1 connections in the "NEW" state are not treated as + # being "available". The control flow which created the connection will + # be able to send an outgoing request, but the connection will not be + # acquired from the connection pool for any other request. + return self._state == HTTPConnectionState.IDLE + + def has_expired(self) -> bool: + now = time.monotonic() + keepalive_expired = self._expire_at is not None and now > self._expire_at + + # If the HTTP connection is idle but the socket is readable, then the + # only valid state is that the socket is about to return b"", indicating + # a server-initiated disconnect. + server_disconnected = ( + self._state == HTTPConnectionState.IDLE + and self._network_stream.get_extra_info("is_readable") + ) + + return keepalive_expired or server_disconnected + + def is_idle(self) -> bool: + return self._state == HTTPConnectionState.IDLE + + def is_closed(self) -> bool: + return self._state == HTTPConnectionState.CLOSED + + def info(self) -> str: + origin = str(self._origin) + return ( + f"{origin!r}, HTTP/1.1, {self._state.name}, " + f"Request Count: {self._request_count}" + ) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + origin = str(self._origin) + return ( + f"<{class_name} [{origin!r}, {self._state.name}, " + f"Request Count: {self._request_count}]>" + ) + + # These context managers are not used in the standard flow, but are + # useful for testing or working with connection instances directly. + + def __enter__(self) -> HTTP11Connection: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + self.close() + + +class HTTP11ConnectionByteStream: + def __init__(self, connection: HTTP11Connection, request: Request) -> None: + self._connection = connection + self._request = request + self._closed = False + + def __iter__(self) -> typing.Iterator[bytes]: + kwargs = {"request": self._request} + try: + with Trace("receive_response_body", logger, self._request, kwargs): + for chunk in self._connection._receive_response_body(**kwargs): + yield chunk + except BaseException as exc: + # If we get an exception while streaming the response, + # we want to close the response (and possibly the connection) + # before raising that exception. + with ShieldCancellation(): + self.close() + raise exc + + def close(self) -> None: + if not self._closed: + self._closed = True + with Trace("response_closed", logger, self._request): + self._connection._response_closed() + + +class HTTP11UpgradeStream(NetworkStream): + def __init__(self, stream: NetworkStream, leading_data: bytes) -> None: + self._stream = stream + self._leading_data = leading_data + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + if self._leading_data: + buffer = self._leading_data[:max_bytes] + self._leading_data = self._leading_data[max_bytes:] + return buffer + else: + return self._stream.read(max_bytes, timeout) + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + self._stream.write(buffer, timeout) + + def close(self) -> None: + self._stream.close() + + def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> NetworkStream: + return self._stream.start_tls(ssl_context, server_hostname, timeout) + + def get_extra_info(self, info: str) -> typing.Any: + return self._stream.get_extra_info(info) diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/http2.py b/venv/lib/python3.12/site-packages/httpcore/_sync/http2.py new file mode 100644 index 0000000000000000000000000000000000000000..ddcc189001c50c37c6a03810dc21d955df919f10 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_sync/http2.py @@ -0,0 +1,592 @@ +from __future__ import annotations + +import enum +import logging +import time +import types +import typing + +import h2.config +import h2.connection +import h2.events +import h2.exceptions +import h2.settings + +from .._backends.base import NetworkStream +from .._exceptions import ( + ConnectionNotAvailable, + LocalProtocolError, + RemoteProtocolError, +) +from .._models import Origin, Request, Response +from .._synchronization import Lock, Semaphore, ShieldCancellation +from .._trace import Trace +from .interfaces import ConnectionInterface + +logger = logging.getLogger("httpcore.http2") + + +def has_body_headers(request: Request) -> bool: + return any( + k.lower() == b"content-length" or k.lower() == b"transfer-encoding" + for k, v in request.headers + ) + + +class HTTPConnectionState(enum.IntEnum): + ACTIVE = 1 + IDLE = 2 + CLOSED = 3 + + +class HTTP2Connection(ConnectionInterface): + READ_NUM_BYTES = 64 * 1024 + CONFIG = h2.config.H2Configuration(validate_inbound_headers=False) + + def __init__( + self, + origin: Origin, + stream: NetworkStream, + keepalive_expiry: float | None = None, + ): + self._origin = origin + self._network_stream = stream + self._keepalive_expiry: float | None = keepalive_expiry + self._h2_state = h2.connection.H2Connection(config=self.CONFIG) + self._state = HTTPConnectionState.IDLE + self._expire_at: float | None = None + self._request_count = 0 + self._init_lock = Lock() + self._state_lock = Lock() + self._read_lock = Lock() + self._write_lock = Lock() + self._sent_connection_init = False + self._used_all_stream_ids = False + self._connection_error = False + + # Mapping from stream ID to response stream events. + self._events: dict[ + int, + list[ + h2.events.ResponseReceived + | h2.events.DataReceived + | h2.events.StreamEnded + | h2.events.StreamReset, + ], + ] = {} + + # Connection terminated events are stored as state since + # we need to handle them for all streams. + self._connection_terminated: h2.events.ConnectionTerminated | None = None + + self._read_exception: Exception | None = None + self._write_exception: Exception | None = None + + def handle_request(self, request: Request) -> Response: + if not self.can_handle_request(request.url.origin): + # This cannot occur in normal operation, since the connection pool + # will only send requests on connections that handle them. + # It's in place simply for resilience as a guard against incorrect + # usage, for anyone working directly with httpcore connections. + raise RuntimeError( + f"Attempted to send request to {request.url.origin} on connection " + f"to {self._origin}" + ) + + with self._state_lock: + if self._state in (HTTPConnectionState.ACTIVE, HTTPConnectionState.IDLE): + self._request_count += 1 + self._expire_at = None + self._state = HTTPConnectionState.ACTIVE + else: + raise ConnectionNotAvailable() + + with self._init_lock: + if not self._sent_connection_init: + try: + sci_kwargs = {"request": request} + with Trace( + "send_connection_init", logger, request, sci_kwargs + ): + self._send_connection_init(**sci_kwargs) + except BaseException as exc: + with ShieldCancellation(): + self.close() + raise exc + + self._sent_connection_init = True + + # Initially start with just 1 until the remote server provides + # its max_concurrent_streams value + self._max_streams = 1 + + local_settings_max_streams = ( + self._h2_state.local_settings.max_concurrent_streams + ) + self._max_streams_semaphore = Semaphore(local_settings_max_streams) + + for _ in range(local_settings_max_streams - self._max_streams): + self._max_streams_semaphore.acquire() + + self._max_streams_semaphore.acquire() + + try: + stream_id = self._h2_state.get_next_available_stream_id() + self._events[stream_id] = [] + except h2.exceptions.NoAvailableStreamIDError: # pragma: nocover + self._used_all_stream_ids = True + self._request_count -= 1 + raise ConnectionNotAvailable() + + try: + kwargs = {"request": request, "stream_id": stream_id} + with Trace("send_request_headers", logger, request, kwargs): + self._send_request_headers(request=request, stream_id=stream_id) + with Trace("send_request_body", logger, request, kwargs): + self._send_request_body(request=request, stream_id=stream_id) + with Trace( + "receive_response_headers", logger, request, kwargs + ) as trace: + status, headers = self._receive_response( + request=request, stream_id=stream_id + ) + trace.return_value = (status, headers) + + return Response( + status=status, + headers=headers, + content=HTTP2ConnectionByteStream(self, request, stream_id=stream_id), + extensions={ + "http_version": b"HTTP/2", + "network_stream": self._network_stream, + "stream_id": stream_id, + }, + ) + except BaseException as exc: # noqa: PIE786 + with ShieldCancellation(): + kwargs = {"stream_id": stream_id} + with Trace("response_closed", logger, request, kwargs): + self._response_closed(stream_id=stream_id) + + if isinstance(exc, h2.exceptions.ProtocolError): + # One case where h2 can raise a protocol error is when a + # closed frame has been seen by the state machine. + # + # This happens when one stream is reading, and encounters + # a GOAWAY event. Other flows of control may then raise + # a protocol error at any point they interact with the 'h2_state'. + # + # In this case we'll have stored the event, and should raise + # it as a RemoteProtocolError. + if self._connection_terminated: # pragma: nocover + raise RemoteProtocolError(self._connection_terminated) + # If h2 raises a protocol error in some other state then we + # must somehow have made a protocol violation. + raise LocalProtocolError(exc) # pragma: nocover + + raise exc + + def _send_connection_init(self, request: Request) -> None: + """ + The HTTP/2 connection requires some initial setup before we can start + using individual request/response streams on it. + """ + # Need to set these manually here instead of manipulating via + # __setitem__() otherwise the H2Connection will emit SettingsUpdate + # frames in addition to sending the undesired defaults. + self._h2_state.local_settings = h2.settings.Settings( + client=True, + initial_values={ + # Disable PUSH_PROMISE frames from the server since we don't do anything + # with them for now. Maybe when we support caching? + h2.settings.SettingCodes.ENABLE_PUSH: 0, + # These two are taken from h2 for safe defaults + h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS: 100, + h2.settings.SettingCodes.MAX_HEADER_LIST_SIZE: 65536, + }, + ) + + # Some websites (*cough* Yahoo *cough*) balk at this setting being + # present in the initial handshake since it's not defined in the original + # RFC despite the RFC mandating ignoring settings you don't know about. + del self._h2_state.local_settings[ + h2.settings.SettingCodes.ENABLE_CONNECT_PROTOCOL + ] + + self._h2_state.initiate_connection() + self._h2_state.increment_flow_control_window(2**24) + self._write_outgoing_data(request) + + # Sending the request... + + def _send_request_headers(self, request: Request, stream_id: int) -> None: + """ + Send the request headers to a given stream ID. + """ + end_stream = not has_body_headers(request) + + # In HTTP/2 the ':authority' pseudo-header is used instead of 'Host'. + # In order to gracefully handle HTTP/1.1 and HTTP/2 we always require + # HTTP/1.1 style headers, and map them appropriately if we end up on + # an HTTP/2 connection. + authority = [v for k, v in request.headers if k.lower() == b"host"][0] + + headers = [ + (b":method", request.method), + (b":authority", authority), + (b":scheme", request.url.scheme), + (b":path", request.url.target), + ] + [ + (k.lower(), v) + for k, v in request.headers + if k.lower() + not in ( + b"host", + b"transfer-encoding", + ) + ] + + self._h2_state.send_headers(stream_id, headers, end_stream=end_stream) + self._h2_state.increment_flow_control_window(2**24, stream_id=stream_id) + self._write_outgoing_data(request) + + def _send_request_body(self, request: Request, stream_id: int) -> None: + """ + Iterate over the request body sending it to a given stream ID. + """ + if not has_body_headers(request): + return + + assert isinstance(request.stream, typing.Iterable) + for data in request.stream: + self._send_stream_data(request, stream_id, data) + self._send_end_stream(request, stream_id) + + def _send_stream_data( + self, request: Request, stream_id: int, data: bytes + ) -> None: + """ + Send a single chunk of data in one or more data frames. + """ + while data: + max_flow = self._wait_for_outgoing_flow(request, stream_id) + chunk_size = min(len(data), max_flow) + chunk, data = data[:chunk_size], data[chunk_size:] + self._h2_state.send_data(stream_id, chunk) + self._write_outgoing_data(request) + + def _send_end_stream(self, request: Request, stream_id: int) -> None: + """ + Send an empty data frame on on a given stream ID with the END_STREAM flag set. + """ + self._h2_state.end_stream(stream_id) + self._write_outgoing_data(request) + + # Receiving the response... + + def _receive_response( + self, request: Request, stream_id: int + ) -> tuple[int, list[tuple[bytes, bytes]]]: + """ + Return the response status code and headers for a given stream ID. + """ + while True: + event = self._receive_stream_event(request, stream_id) + if isinstance(event, h2.events.ResponseReceived): + break + + status_code = 200 + headers = [] + assert event.headers is not None + for k, v in event.headers: + if k == b":status": + status_code = int(v.decode("ascii", errors="ignore")) + elif not k.startswith(b":"): + headers.append((k, v)) + + return (status_code, headers) + + def _receive_response_body( + self, request: Request, stream_id: int + ) -> typing.Iterator[bytes]: + """ + Iterator that returns the bytes of the response body for a given stream ID. + """ + while True: + event = self._receive_stream_event(request, stream_id) + if isinstance(event, h2.events.DataReceived): + assert event.flow_controlled_length is not None + assert event.data is not None + amount = event.flow_controlled_length + self._h2_state.acknowledge_received_data(amount, stream_id) + self._write_outgoing_data(request) + yield event.data + elif isinstance(event, h2.events.StreamEnded): + break + + def _receive_stream_event( + self, request: Request, stream_id: int + ) -> h2.events.ResponseReceived | h2.events.DataReceived | h2.events.StreamEnded: + """ + Return the next available event for a given stream ID. + + Will read more data from the network if required. + """ + while not self._events.get(stream_id): + self._receive_events(request, stream_id) + event = self._events[stream_id].pop(0) + if isinstance(event, h2.events.StreamReset): + raise RemoteProtocolError(event) + return event + + def _receive_events( + self, request: Request, stream_id: int | None = None + ) -> None: + """ + Read some data from the network until we see one or more events + for a given stream ID. + """ + with self._read_lock: + if self._connection_terminated is not None: + last_stream_id = self._connection_terminated.last_stream_id + if stream_id and last_stream_id and stream_id > last_stream_id: + self._request_count -= 1 + raise ConnectionNotAvailable() + raise RemoteProtocolError(self._connection_terminated) + + # This conditional is a bit icky. We don't want to block reading if we've + # actually got an event to return for a given stream. We need to do that + # check *within* the atomic read lock. Though it also need to be optional, + # because when we call it from `_wait_for_outgoing_flow` we *do* want to + # block until we've available flow control, event when we have events + # pending for the stream ID we're attempting to send on. + if stream_id is None or not self._events.get(stream_id): + events = self._read_incoming_data(request) + for event in events: + if isinstance(event, h2.events.RemoteSettingsChanged): + with Trace( + "receive_remote_settings", logger, request + ) as trace: + self._receive_remote_settings_change(event) + trace.return_value = event + + elif isinstance( + event, + ( + h2.events.ResponseReceived, + h2.events.DataReceived, + h2.events.StreamEnded, + h2.events.StreamReset, + ), + ): + if event.stream_id in self._events: + self._events[event.stream_id].append(event) + + elif isinstance(event, h2.events.ConnectionTerminated): + self._connection_terminated = event + + self._write_outgoing_data(request) + + def _receive_remote_settings_change( + self, event: h2.events.RemoteSettingsChanged + ) -> None: + max_concurrent_streams = event.changed_settings.get( + h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS + ) + if max_concurrent_streams: + new_max_streams = min( + max_concurrent_streams.new_value, + self._h2_state.local_settings.max_concurrent_streams, + ) + if new_max_streams and new_max_streams != self._max_streams: + while new_max_streams > self._max_streams: + self._max_streams_semaphore.release() + self._max_streams += 1 + while new_max_streams < self._max_streams: + self._max_streams_semaphore.acquire() + self._max_streams -= 1 + + def _response_closed(self, stream_id: int) -> None: + self._max_streams_semaphore.release() + del self._events[stream_id] + with self._state_lock: + if self._connection_terminated and not self._events: + self.close() + + elif self._state == HTTPConnectionState.ACTIVE and not self._events: + self._state = HTTPConnectionState.IDLE + if self._keepalive_expiry is not None: + now = time.monotonic() + self._expire_at = now + self._keepalive_expiry + if self._used_all_stream_ids: # pragma: nocover + self.close() + + def close(self) -> None: + # Note that this method unilaterally closes the connection, and does + # not have any kind of locking in place around it. + self._h2_state.close_connection() + self._state = HTTPConnectionState.CLOSED + self._network_stream.close() + + # Wrappers around network read/write operations... + + def _read_incoming_data(self, request: Request) -> list[h2.events.Event]: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("read", None) + + if self._read_exception is not None: + raise self._read_exception # pragma: nocover + + try: + data = self._network_stream.read(self.READ_NUM_BYTES, timeout) + if data == b"": + raise RemoteProtocolError("Server disconnected") + except Exception as exc: + # If we get a network error we should: + # + # 1. Save the exception and just raise it immediately on any future reads. + # (For example, this means that a single read timeout or disconnect will + # immediately close all pending streams. Without requiring multiple + # sequential timeouts.) + # 2. Mark the connection as errored, so that we don't accept any other + # incoming requests. + self._read_exception = exc + self._connection_error = True + raise exc + + events: list[h2.events.Event] = self._h2_state.receive_data(data) + + return events + + def _write_outgoing_data(self, request: Request) -> None: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("write", None) + + with self._write_lock: + data_to_send = self._h2_state.data_to_send() + + if self._write_exception is not None: + raise self._write_exception # pragma: nocover + + try: + self._network_stream.write(data_to_send, timeout) + except Exception as exc: # pragma: nocover + # If we get a network error we should: + # + # 1. Save the exception and just raise it immediately on any future write. + # (For example, this means that a single write timeout or disconnect will + # immediately close all pending streams. Without requiring multiple + # sequential timeouts.) + # 2. Mark the connection as errored, so that we don't accept any other + # incoming requests. + self._write_exception = exc + self._connection_error = True + raise exc + + # Flow control... + + def _wait_for_outgoing_flow(self, request: Request, stream_id: int) -> int: + """ + Returns the maximum allowable outgoing flow for a given stream. + + If the allowable flow is zero, then waits on the network until + WindowUpdated frames have increased the flow rate. + https://tools.ietf.org/html/rfc7540#section-6.9 + """ + local_flow: int = self._h2_state.local_flow_control_window(stream_id) + max_frame_size: int = self._h2_state.max_outbound_frame_size + flow = min(local_flow, max_frame_size) + while flow == 0: + self._receive_events(request) + local_flow = self._h2_state.local_flow_control_window(stream_id) + max_frame_size = self._h2_state.max_outbound_frame_size + flow = min(local_flow, max_frame_size) + return flow + + # Interface for connection pooling... + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._origin + + def is_available(self) -> bool: + return ( + self._state != HTTPConnectionState.CLOSED + and not self._connection_error + and not self._used_all_stream_ids + and not ( + self._h2_state.state_machine.state + == h2.connection.ConnectionState.CLOSED + ) + ) + + def has_expired(self) -> bool: + now = time.monotonic() + return self._expire_at is not None and now > self._expire_at + + def is_idle(self) -> bool: + return self._state == HTTPConnectionState.IDLE + + def is_closed(self) -> bool: + return self._state == HTTPConnectionState.CLOSED + + def info(self) -> str: + origin = str(self._origin) + return ( + f"{origin!r}, HTTP/2, {self._state.name}, " + f"Request Count: {self._request_count}" + ) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + origin = str(self._origin) + return ( + f"<{class_name} [{origin!r}, {self._state.name}, " + f"Request Count: {self._request_count}]>" + ) + + # These context managers are not used in the standard flow, but are + # useful for testing or working with connection instances directly. + + def __enter__(self) -> HTTP2Connection: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + self.close() + + +class HTTP2ConnectionByteStream: + def __init__( + self, connection: HTTP2Connection, request: Request, stream_id: int + ) -> None: + self._connection = connection + self._request = request + self._stream_id = stream_id + self._closed = False + + def __iter__(self) -> typing.Iterator[bytes]: + kwargs = {"request": self._request, "stream_id": self._stream_id} + try: + with Trace("receive_response_body", logger, self._request, kwargs): + for chunk in self._connection._receive_response_body( + request=self._request, stream_id=self._stream_id + ): + yield chunk + except BaseException as exc: + # If we get an exception while streaming the response, + # we want to close the response (and possibly the connection) + # before raising that exception. + with ShieldCancellation(): + self.close() + raise exc + + def close(self) -> None: + if not self._closed: + self._closed = True + kwargs = {"stream_id": self._stream_id} + with Trace("response_closed", logger, self._request, kwargs): + self._connection._response_closed(stream_id=self._stream_id) diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/http_proxy.py b/venv/lib/python3.12/site-packages/httpcore/_sync/http_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..ecca88f7dc93b78f2aa26f16cf29d17a8a83ae27 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_sync/http_proxy.py @@ -0,0 +1,367 @@ +from __future__ import annotations + +import base64 +import logging +import ssl +import typing + +from .._backends.base import SOCKET_OPTION, NetworkBackend +from .._exceptions import ProxyError +from .._models import ( + URL, + Origin, + Request, + Response, + enforce_bytes, + enforce_headers, + enforce_url, +) +from .._ssl import default_ssl_context +from .._synchronization import Lock +from .._trace import Trace +from .connection import HTTPConnection +from .connection_pool import ConnectionPool +from .http11 import HTTP11Connection +from .interfaces import ConnectionInterface + +ByteOrStr = typing.Union[bytes, str] +HeadersAsSequence = typing.Sequence[typing.Tuple[ByteOrStr, ByteOrStr]] +HeadersAsMapping = typing.Mapping[ByteOrStr, ByteOrStr] + + +logger = logging.getLogger("httpcore.proxy") + + +def merge_headers( + default_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, + override_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, +) -> list[tuple[bytes, bytes]]: + """ + Append default_headers and override_headers, de-duplicating if a key exists + in both cases. + """ + default_headers = [] if default_headers is None else list(default_headers) + override_headers = [] if override_headers is None else list(override_headers) + has_override = set(key.lower() for key, value in override_headers) + default_headers = [ + (key, value) + for key, value in default_headers + if key.lower() not in has_override + ] + return default_headers + override_headers + + +class HTTPProxy(ConnectionPool): # pragma: nocover + """ + A connection pool that sends requests via an HTTP proxy. + """ + + def __init__( + self, + proxy_url: URL | bytes | str, + proxy_auth: tuple[bytes | str, bytes | str] | None = None, + proxy_headers: HeadersAsMapping | HeadersAsSequence | None = None, + ssl_context: ssl.SSLContext | None = None, + proxy_ssl_context: ssl.SSLContext | None = None, + max_connections: int | None = 10, + max_keepalive_connections: int | None = None, + keepalive_expiry: float | None = None, + http1: bool = True, + http2: bool = False, + retries: int = 0, + local_address: str | None = None, + uds: str | None = None, + network_backend: NetworkBackend | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + """ + A connection pool for making HTTP requests. + + Parameters: + proxy_url: The URL to use when connecting to the proxy server. + For example `"http://127.0.0.1:8080/"`. + proxy_auth: Any proxy authentication as a two-tuple of + (username, password). May be either bytes or ascii-only str. + proxy_headers: Any HTTP headers to use for the proxy requests. + For example `{"Proxy-Authorization": "Basic :"}`. + ssl_context: An SSL context to use for verifying connections. + If not specified, the default `httpcore.default_ssl_context()` + will be used. + proxy_ssl_context: The same as `ssl_context`, but for a proxy server rather than a remote origin. + max_connections: The maximum number of concurrent HTTP connections that + the pool should allow. Any attempt to send a request on a pool that + would exceed this amount will block until a connection is available. + max_keepalive_connections: The maximum number of idle HTTP connections + that will be maintained in the pool. + keepalive_expiry: The duration in seconds that an idle HTTP connection + may be maintained for before being expired from the pool. + http1: A boolean indicating if HTTP/1.1 requests should be supported + by the connection pool. Defaults to True. + http2: A boolean indicating if HTTP/2 requests should be supported by + the connection pool. Defaults to False. + retries: The maximum number of retries when trying to establish + a connection. + local_address: Local address to connect from. Can also be used to + connect using a particular address family. Using + `local_address="0.0.0.0"` will connect using an `AF_INET` address + (IPv4), while using `local_address="::"` will connect using an + `AF_INET6` address (IPv6). + uds: Path to a Unix Domain Socket to use instead of TCP sockets. + network_backend: A backend instance to use for handling network I/O. + """ + super().__init__( + ssl_context=ssl_context, + max_connections=max_connections, + max_keepalive_connections=max_keepalive_connections, + keepalive_expiry=keepalive_expiry, + http1=http1, + http2=http2, + network_backend=network_backend, + retries=retries, + local_address=local_address, + uds=uds, + socket_options=socket_options, + ) + + self._proxy_url = enforce_url(proxy_url, name="proxy_url") + if ( + self._proxy_url.scheme == b"http" and proxy_ssl_context is not None + ): # pragma: no cover + raise RuntimeError( + "The `proxy_ssl_context` argument is not allowed for the http scheme" + ) + + self._ssl_context = ssl_context + self._proxy_ssl_context = proxy_ssl_context + self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") + if proxy_auth is not None: + username = enforce_bytes(proxy_auth[0], name="proxy_auth") + password = enforce_bytes(proxy_auth[1], name="proxy_auth") + userpass = username + b":" + password + authorization = b"Basic " + base64.b64encode(userpass) + self._proxy_headers = [ + (b"Proxy-Authorization", authorization) + ] + self._proxy_headers + + def create_connection(self, origin: Origin) -> ConnectionInterface: + if origin.scheme == b"http": + return ForwardHTTPConnection( + proxy_origin=self._proxy_url.origin, + proxy_headers=self._proxy_headers, + remote_origin=origin, + keepalive_expiry=self._keepalive_expiry, + network_backend=self._network_backend, + proxy_ssl_context=self._proxy_ssl_context, + ) + return TunnelHTTPConnection( + proxy_origin=self._proxy_url.origin, + proxy_headers=self._proxy_headers, + remote_origin=origin, + ssl_context=self._ssl_context, + proxy_ssl_context=self._proxy_ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + + +class ForwardHTTPConnection(ConnectionInterface): + def __init__( + self, + proxy_origin: Origin, + remote_origin: Origin, + proxy_headers: HeadersAsMapping | HeadersAsSequence | None = None, + keepalive_expiry: float | None = None, + network_backend: NetworkBackend | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + proxy_ssl_context: ssl.SSLContext | None = None, + ) -> None: + self._connection = HTTPConnection( + origin=proxy_origin, + keepalive_expiry=keepalive_expiry, + network_backend=network_backend, + socket_options=socket_options, + ssl_context=proxy_ssl_context, + ) + self._proxy_origin = proxy_origin + self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") + self._remote_origin = remote_origin + + def handle_request(self, request: Request) -> Response: + headers = merge_headers(self._proxy_headers, request.headers) + url = URL( + scheme=self._proxy_origin.scheme, + host=self._proxy_origin.host, + port=self._proxy_origin.port, + target=bytes(request.url), + ) + proxy_request = Request( + method=request.method, + url=url, + headers=headers, + content=request.stream, + extensions=request.extensions, + ) + return self._connection.handle_request(proxy_request) + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._remote_origin + + def close(self) -> None: + self._connection.close() + + def info(self) -> str: + return self._connection.info() + + def is_available(self) -> bool: + return self._connection.is_available() + + def has_expired(self) -> bool: + return self._connection.has_expired() + + def is_idle(self) -> bool: + return self._connection.is_idle() + + def is_closed(self) -> bool: + return self._connection.is_closed() + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{self.info()}]>" + + +class TunnelHTTPConnection(ConnectionInterface): + def __init__( + self, + proxy_origin: Origin, + remote_origin: Origin, + ssl_context: ssl.SSLContext | None = None, + proxy_ssl_context: ssl.SSLContext | None = None, + proxy_headers: typing.Sequence[tuple[bytes, bytes]] | None = None, + keepalive_expiry: float | None = None, + http1: bool = True, + http2: bool = False, + network_backend: NetworkBackend | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + self._connection: ConnectionInterface = HTTPConnection( + origin=proxy_origin, + keepalive_expiry=keepalive_expiry, + network_backend=network_backend, + socket_options=socket_options, + ssl_context=proxy_ssl_context, + ) + self._proxy_origin = proxy_origin + self._remote_origin = remote_origin + self._ssl_context = ssl_context + self._proxy_ssl_context = proxy_ssl_context + self._proxy_headers = enforce_headers(proxy_headers, name="proxy_headers") + self._keepalive_expiry = keepalive_expiry + self._http1 = http1 + self._http2 = http2 + self._connect_lock = Lock() + self._connected = False + + def handle_request(self, request: Request) -> Response: + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("connect", None) + + with self._connect_lock: + if not self._connected: + target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) + + connect_url = URL( + scheme=self._proxy_origin.scheme, + host=self._proxy_origin.host, + port=self._proxy_origin.port, + target=target, + ) + connect_headers = merge_headers( + [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers + ) + connect_request = Request( + method=b"CONNECT", + url=connect_url, + headers=connect_headers, + extensions=request.extensions, + ) + connect_response = self._connection.handle_request( + connect_request + ) + + if connect_response.status < 200 or connect_response.status > 299: + reason_bytes = connect_response.extensions.get("reason_phrase", b"") + reason_str = reason_bytes.decode("ascii", errors="ignore") + msg = "%d %s" % (connect_response.status, reason_str) + self._connection.close() + raise ProxyError(msg) + + stream = connect_response.extensions["network_stream"] + + # Upgrade the stream to SSL + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + with Trace("start_tls", logger, request, kwargs) as trace: + stream = stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) + + # Create the HTTP/1.1 or HTTP/2 connection + if http2_negotiated or (self._http2 and not self._http1): + from .http2 import HTTP2Connection + + self._connection = HTTP2Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + else: + self._connection = HTTP11Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + + self._connected = True + return self._connection.handle_request(request) + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._remote_origin + + def close(self) -> None: + self._connection.close() + + def info(self) -> str: + return self._connection.info() + + def is_available(self) -> bool: + return self._connection.is_available() + + def has_expired(self) -> bool: + return self._connection.has_expired() + + def is_idle(self) -> bool: + return self._connection.is_idle() + + def is_closed(self) -> bool: + return self._connection.is_closed() + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{self.info()}]>" diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/interfaces.py b/venv/lib/python3.12/site-packages/httpcore/_sync/interfaces.py new file mode 100644 index 0000000000000000000000000000000000000000..e673d4cc1b1dd7e7ecdbde91fd6ada386c3de03f --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_sync/interfaces.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import contextlib +import typing + +from .._models import ( + URL, + Extensions, + HeaderTypes, + Origin, + Request, + Response, + enforce_bytes, + enforce_headers, + enforce_url, + include_request_headers, +) + + +class RequestInterface: + def request( + self, + method: bytes | str, + url: URL | bytes | str, + *, + headers: HeaderTypes = None, + content: bytes | typing.Iterator[bytes] | None = None, + extensions: Extensions | None = None, + ) -> Response: + # Strict type checking on our parameters. + method = enforce_bytes(method, name="method") + url = enforce_url(url, name="url") + headers = enforce_headers(headers, name="headers") + + # Include Host header, and optionally Content-Length or Transfer-Encoding. + headers = include_request_headers(headers, url=url, content=content) + + request = Request( + method=method, + url=url, + headers=headers, + content=content, + extensions=extensions, + ) + response = self.handle_request(request) + try: + response.read() + finally: + response.close() + return response + + @contextlib.contextmanager + def stream( + self, + method: bytes | str, + url: URL | bytes | str, + *, + headers: HeaderTypes = None, + content: bytes | typing.Iterator[bytes] | None = None, + extensions: Extensions | None = None, + ) -> typing.Iterator[Response]: + # Strict type checking on our parameters. + method = enforce_bytes(method, name="method") + url = enforce_url(url, name="url") + headers = enforce_headers(headers, name="headers") + + # Include Host header, and optionally Content-Length or Transfer-Encoding. + headers = include_request_headers(headers, url=url, content=content) + + request = Request( + method=method, + url=url, + headers=headers, + content=content, + extensions=extensions, + ) + response = self.handle_request(request) + try: + yield response + finally: + response.close() + + def handle_request(self, request: Request) -> Response: + raise NotImplementedError() # pragma: nocover + + +class ConnectionInterface(RequestInterface): + def close(self) -> None: + raise NotImplementedError() # pragma: nocover + + def info(self) -> str: + raise NotImplementedError() # pragma: nocover + + def can_handle_request(self, origin: Origin) -> bool: + raise NotImplementedError() # pragma: nocover + + def is_available(self) -> bool: + """ + Return `True` if the connection is currently able to accept an + outgoing request. + + An HTTP/1.1 connection will only be available if it is currently idle. + + An HTTP/2 connection will be available so long as the stream ID space is + not yet exhausted, and the connection is not in an error state. + + While the connection is being established we may not yet know if it is going + to result in an HTTP/1.1 or HTTP/2 connection. The connection should be + treated as being available, but might ultimately raise `NewConnectionRequired` + required exceptions if multiple requests are attempted over a connection + that ends up being established as HTTP/1.1. + """ + raise NotImplementedError() # pragma: nocover + + def has_expired(self) -> bool: + """ + Return `True` if the connection is in a state where it should be closed. + + This either means that the connection is idle and it has passed the + expiry time on its keep-alive, or that server has sent an EOF. + """ + raise NotImplementedError() # pragma: nocover + + def is_idle(self) -> bool: + """ + Return `True` if the connection is currently idle. + """ + raise NotImplementedError() # pragma: nocover + + def is_closed(self) -> bool: + """ + Return `True` if the connection has been closed. + + Used when a response is closed to determine if the connection may be + returned to the connection pool or not. + """ + raise NotImplementedError() # pragma: nocover diff --git a/venv/lib/python3.12/site-packages/httpcore/_sync/socks_proxy.py b/venv/lib/python3.12/site-packages/httpcore/_sync/socks_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..0ca96ddfb580b19413797f41e79f7abcecdd9d79 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_sync/socks_proxy.py @@ -0,0 +1,341 @@ +from __future__ import annotations + +import logging +import ssl + +import socksio + +from .._backends.sync import SyncBackend +from .._backends.base import NetworkBackend, NetworkStream +from .._exceptions import ConnectionNotAvailable, ProxyError +from .._models import URL, Origin, Request, Response, enforce_bytes, enforce_url +from .._ssl import default_ssl_context +from .._synchronization import Lock +from .._trace import Trace +from .connection_pool import ConnectionPool +from .http11 import HTTP11Connection +from .interfaces import ConnectionInterface + +logger = logging.getLogger("httpcore.socks") + + +AUTH_METHODS = { + b"\x00": "NO AUTHENTICATION REQUIRED", + b"\x01": "GSSAPI", + b"\x02": "USERNAME/PASSWORD", + b"\xff": "NO ACCEPTABLE METHODS", +} + +REPLY_CODES = { + b"\x00": "Succeeded", + b"\x01": "General SOCKS server failure", + b"\x02": "Connection not allowed by ruleset", + b"\x03": "Network unreachable", + b"\x04": "Host unreachable", + b"\x05": "Connection refused", + b"\x06": "TTL expired", + b"\x07": "Command not supported", + b"\x08": "Address type not supported", +} + + +def _init_socks5_connection( + stream: NetworkStream, + *, + host: bytes, + port: int, + auth: tuple[bytes, bytes] | None = None, +) -> None: + conn = socksio.socks5.SOCKS5Connection() + + # Auth method request + auth_method = ( + socksio.socks5.SOCKS5AuthMethod.NO_AUTH_REQUIRED + if auth is None + else socksio.socks5.SOCKS5AuthMethod.USERNAME_PASSWORD + ) + conn.send(socksio.socks5.SOCKS5AuthMethodsRequest([auth_method])) + outgoing_bytes = conn.data_to_send() + stream.write(outgoing_bytes) + + # Auth method response + incoming_bytes = stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socksio.socks5.SOCKS5AuthReply) + if response.method != auth_method: + requested = AUTH_METHODS.get(auth_method, "UNKNOWN") + responded = AUTH_METHODS.get(response.method, "UNKNOWN") + raise ProxyError( + f"Requested {requested} from proxy server, but got {responded}." + ) + + if response.method == socksio.socks5.SOCKS5AuthMethod.USERNAME_PASSWORD: + # Username/password request + assert auth is not None + username, password = auth + conn.send(socksio.socks5.SOCKS5UsernamePasswordRequest(username, password)) + outgoing_bytes = conn.data_to_send() + stream.write(outgoing_bytes) + + # Username/password response + incoming_bytes = stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socksio.socks5.SOCKS5UsernamePasswordReply) + if not response.success: + raise ProxyError("Invalid username/password") + + # Connect request + conn.send( + socksio.socks5.SOCKS5CommandRequest.from_address( + socksio.socks5.SOCKS5Command.CONNECT, (host, port) + ) + ) + outgoing_bytes = conn.data_to_send() + stream.write(outgoing_bytes) + + # Connect response + incoming_bytes = stream.read(max_bytes=4096) + response = conn.receive_data(incoming_bytes) + assert isinstance(response, socksio.socks5.SOCKS5Reply) + if response.reply_code != socksio.socks5.SOCKS5ReplyCode.SUCCEEDED: + reply_code = REPLY_CODES.get(response.reply_code, "UNKOWN") + raise ProxyError(f"Proxy Server could not connect: {reply_code}.") + + +class SOCKSProxy(ConnectionPool): # pragma: nocover + """ + A connection pool that sends requests via an HTTP proxy. + """ + + def __init__( + self, + proxy_url: URL | bytes | str, + proxy_auth: tuple[bytes | str, bytes | str] | None = None, + ssl_context: ssl.SSLContext | None = None, + max_connections: int | None = 10, + max_keepalive_connections: int | None = None, + keepalive_expiry: float | None = None, + http1: bool = True, + http2: bool = False, + retries: int = 0, + network_backend: NetworkBackend | None = None, + ) -> None: + """ + A connection pool for making HTTP requests. + + Parameters: + proxy_url: The URL to use when connecting to the proxy server. + For example `"http://127.0.0.1:8080/"`. + ssl_context: An SSL context to use for verifying connections. + If not specified, the default `httpcore.default_ssl_context()` + will be used. + max_connections: The maximum number of concurrent HTTP connections that + the pool should allow. Any attempt to send a request on a pool that + would exceed this amount will block until a connection is available. + max_keepalive_connections: The maximum number of idle HTTP connections + that will be maintained in the pool. + keepalive_expiry: The duration in seconds that an idle HTTP connection + may be maintained for before being expired from the pool. + http1: A boolean indicating if HTTP/1.1 requests should be supported + by the connection pool. Defaults to True. + http2: A boolean indicating if HTTP/2 requests should be supported by + the connection pool. Defaults to False. + retries: The maximum number of retries when trying to establish + a connection. + local_address: Local address to connect from. Can also be used to + connect using a particular address family. Using + `local_address="0.0.0.0"` will connect using an `AF_INET` address + (IPv4), while using `local_address="::"` will connect using an + `AF_INET6` address (IPv6). + uds: Path to a Unix Domain Socket to use instead of TCP sockets. + network_backend: A backend instance to use for handling network I/O. + """ + super().__init__( + ssl_context=ssl_context, + max_connections=max_connections, + max_keepalive_connections=max_keepalive_connections, + keepalive_expiry=keepalive_expiry, + http1=http1, + http2=http2, + network_backend=network_backend, + retries=retries, + ) + self._ssl_context = ssl_context + self._proxy_url = enforce_url(proxy_url, name="proxy_url") + if proxy_auth is not None: + username, password = proxy_auth + username_bytes = enforce_bytes(username, name="proxy_auth") + password_bytes = enforce_bytes(password, name="proxy_auth") + self._proxy_auth: tuple[bytes, bytes] | None = ( + username_bytes, + password_bytes, + ) + else: + self._proxy_auth = None + + def create_connection(self, origin: Origin) -> ConnectionInterface: + return Socks5Connection( + proxy_origin=self._proxy_url.origin, + remote_origin=origin, + proxy_auth=self._proxy_auth, + ssl_context=self._ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + + +class Socks5Connection(ConnectionInterface): + def __init__( + self, + proxy_origin: Origin, + remote_origin: Origin, + proxy_auth: tuple[bytes, bytes] | None = None, + ssl_context: ssl.SSLContext | None = None, + keepalive_expiry: float | None = None, + http1: bool = True, + http2: bool = False, + network_backend: NetworkBackend | None = None, + ) -> None: + self._proxy_origin = proxy_origin + self._remote_origin = remote_origin + self._proxy_auth = proxy_auth + self._ssl_context = ssl_context + self._keepalive_expiry = keepalive_expiry + self._http1 = http1 + self._http2 = http2 + + self._network_backend: NetworkBackend = ( + SyncBackend() if network_backend is None else network_backend + ) + self._connect_lock = Lock() + self._connection: ConnectionInterface | None = None + self._connect_failed = False + + def handle_request(self, request: Request) -> Response: + timeouts = request.extensions.get("timeout", {}) + sni_hostname = request.extensions.get("sni_hostname", None) + timeout = timeouts.get("connect", None) + + with self._connect_lock: + if self._connection is None: + try: + # Connect to the proxy + kwargs = { + "host": self._proxy_origin.host.decode("ascii"), + "port": self._proxy_origin.port, + "timeout": timeout, + } + with Trace("connect_tcp", logger, request, kwargs) as trace: + stream = self._network_backend.connect_tcp(**kwargs) + trace.return_value = stream + + # Connect to the remote host using socks5 + kwargs = { + "stream": stream, + "host": self._remote_origin.host.decode("ascii"), + "port": self._remote_origin.port, + "auth": self._proxy_auth, + } + with Trace( + "setup_socks5_connection", logger, request, kwargs + ) as trace: + _init_socks5_connection(**kwargs) + trace.return_value = stream + + # Upgrade the stream to SSL + if self._remote_origin.scheme == b"https": + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ( + ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ) + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": sni_hostname + or self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + with Trace("start_tls", logger, request, kwargs) as trace: + stream = stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) + + # Create the HTTP/1.1 or HTTP/2 connection + if http2_negotiated or ( + self._http2 and not self._http1 + ): # pragma: nocover + from .http2 import HTTP2Connection + + self._connection = HTTP2Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + else: + self._connection = HTTP11Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + except Exception as exc: + self._connect_failed = True + raise exc + elif not self._connection.is_available(): # pragma: nocover + raise ConnectionNotAvailable() + + return self._connection.handle_request(request) + + def can_handle_request(self, origin: Origin) -> bool: + return origin == self._remote_origin + + def close(self) -> None: + if self._connection is not None: + self._connection.close() + + def is_available(self) -> bool: + if self._connection is None: # pragma: nocover + # If HTTP/2 support is enabled, and the resulting connection could + # end up as HTTP/2 then we should indicate the connection as being + # available to service multiple requests. + return ( + self._http2 + and (self._remote_origin.scheme == b"https" or not self._http1) + and not self._connect_failed + ) + return self._connection.is_available() + + def has_expired(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.has_expired() + + def is_idle(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.is_idle() + + def is_closed(self) -> bool: + if self._connection is None: # pragma: nocover + return self._connect_failed + return self._connection.is_closed() + + def info(self) -> str: + if self._connection is None: # pragma: nocover + return "CONNECTION FAILED" if self._connect_failed else "CONNECTING" + return self._connection.info() + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} [{self.info()}]>" diff --git a/venv/lib/python3.12/site-packages/httpcore/_synchronization.py b/venv/lib/python3.12/site-packages/httpcore/_synchronization.py new file mode 100644 index 0000000000000000000000000000000000000000..2ecc9e9c363e2f16c4f934cf41cf871826d6a495 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_synchronization.py @@ -0,0 +1,318 @@ +from __future__ import annotations + +import threading +import types + +from ._exceptions import ExceptionMapping, PoolTimeout, map_exceptions + +# Our async synchronization primatives use either 'anyio' or 'trio' depending +# on if they're running under asyncio or trio. + +try: + import trio +except (ImportError, NotImplementedError): # pragma: nocover + trio = None # type: ignore + +try: + import anyio +except ImportError: # pragma: nocover + anyio = None # type: ignore + + +def current_async_library() -> str: + # Determine if we're running under trio or asyncio. + # See https://sniffio.readthedocs.io/en/latest/ + try: + import sniffio + except ImportError: # pragma: nocover + environment = "asyncio" + else: + environment = sniffio.current_async_library() + + if environment not in ("asyncio", "trio"): # pragma: nocover + raise RuntimeError("Running under an unsupported async environment.") + + if environment == "asyncio" and anyio is None: # pragma: nocover + raise RuntimeError( + "Running with asyncio requires installation of 'httpcore[asyncio]'." + ) + + if environment == "trio" and trio is None: # pragma: nocover + raise RuntimeError( + "Running with trio requires installation of 'httpcore[trio]'." + ) + + return environment + + +class AsyncLock: + """ + This is a standard lock. + + In the sync case `Lock` provides thread locking. + In the async case `AsyncLock` provides async locking. + """ + + def __init__(self) -> None: + self._backend = "" + + def setup(self) -> None: + """ + Detect if we're running under 'asyncio' or 'trio' and create + a lock with the correct implementation. + """ + self._backend = current_async_library() + if self._backend == "trio": + self._trio_lock = trio.Lock() + elif self._backend == "asyncio": + self._anyio_lock = anyio.Lock() + + async def __aenter__(self) -> AsyncLock: + if not self._backend: + self.setup() + + if self._backend == "trio": + await self._trio_lock.acquire() + elif self._backend == "asyncio": + await self._anyio_lock.acquire() + + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + if self._backend == "trio": + self._trio_lock.release() + elif self._backend == "asyncio": + self._anyio_lock.release() + + +class AsyncThreadLock: + """ + This is a threading-only lock for no-I/O contexts. + + In the sync case `ThreadLock` provides thread locking. + In the async case `AsyncThreadLock` is a no-op. + """ + + def __enter__(self) -> AsyncThreadLock: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + pass + + +class AsyncEvent: + def __init__(self) -> None: + self._backend = "" + + def setup(self) -> None: + """ + Detect if we're running under 'asyncio' or 'trio' and create + a lock with the correct implementation. + """ + self._backend = current_async_library() + if self._backend == "trio": + self._trio_event = trio.Event() + elif self._backend == "asyncio": + self._anyio_event = anyio.Event() + + def set(self) -> None: + if not self._backend: + self.setup() + + if self._backend == "trio": + self._trio_event.set() + elif self._backend == "asyncio": + self._anyio_event.set() + + async def wait(self, timeout: float | None = None) -> None: + if not self._backend: + self.setup() + + if self._backend == "trio": + trio_exc_map: ExceptionMapping = {trio.TooSlowError: PoolTimeout} + timeout_or_inf = float("inf") if timeout is None else timeout + with map_exceptions(trio_exc_map): + with trio.fail_after(timeout_or_inf): + await self._trio_event.wait() + elif self._backend == "asyncio": + anyio_exc_map: ExceptionMapping = {TimeoutError: PoolTimeout} + with map_exceptions(anyio_exc_map): + with anyio.fail_after(timeout): + await self._anyio_event.wait() + + +class AsyncSemaphore: + def __init__(self, bound: int) -> None: + self._bound = bound + self._backend = "" + + def setup(self) -> None: + """ + Detect if we're running under 'asyncio' or 'trio' and create + a semaphore with the correct implementation. + """ + self._backend = current_async_library() + if self._backend == "trio": + self._trio_semaphore = trio.Semaphore( + initial_value=self._bound, max_value=self._bound + ) + elif self._backend == "asyncio": + self._anyio_semaphore = anyio.Semaphore( + initial_value=self._bound, max_value=self._bound + ) + + async def acquire(self) -> None: + if not self._backend: + self.setup() + + if self._backend == "trio": + await self._trio_semaphore.acquire() + elif self._backend == "asyncio": + await self._anyio_semaphore.acquire() + + async def release(self) -> None: + if self._backend == "trio": + self._trio_semaphore.release() + elif self._backend == "asyncio": + self._anyio_semaphore.release() + + +class AsyncShieldCancellation: + # For certain portions of our codebase where we're dealing with + # closing connections during exception handling we want to shield + # the operation from being cancelled. + # + # with AsyncShieldCancellation(): + # ... # clean-up operations, shielded from cancellation. + + def __init__(self) -> None: + """ + Detect if we're running under 'asyncio' or 'trio' and create + a shielded scope with the correct implementation. + """ + self._backend = current_async_library() + + if self._backend == "trio": + self._trio_shield = trio.CancelScope(shield=True) + elif self._backend == "asyncio": + self._anyio_shield = anyio.CancelScope(shield=True) + + def __enter__(self) -> AsyncShieldCancellation: + if self._backend == "trio": + self._trio_shield.__enter__() + elif self._backend == "asyncio": + self._anyio_shield.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + if self._backend == "trio": + self._trio_shield.__exit__(exc_type, exc_value, traceback) + elif self._backend == "asyncio": + self._anyio_shield.__exit__(exc_type, exc_value, traceback) + + +# Our thread-based synchronization primitives... + + +class Lock: + """ + This is a standard lock. + + In the sync case `Lock` provides thread locking. + In the async case `AsyncLock` provides async locking. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + + def __enter__(self) -> Lock: + self._lock.acquire() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + self._lock.release() + + +class ThreadLock: + """ + This is a threading-only lock for no-I/O contexts. + + In the sync case `ThreadLock` provides thread locking. + In the async case `AsyncThreadLock` is a no-op. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + + def __enter__(self) -> ThreadLock: + self._lock.acquire() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + self._lock.release() + + +class Event: + def __init__(self) -> None: + self._event = threading.Event() + + def set(self) -> None: + self._event.set() + + def wait(self, timeout: float | None = None) -> None: + if timeout == float("inf"): # pragma: no cover + timeout = None + if not self._event.wait(timeout=timeout): + raise PoolTimeout() # pragma: nocover + + +class Semaphore: + def __init__(self, bound: int) -> None: + self._semaphore = threading.Semaphore(value=bound) + + def acquire(self) -> None: + self._semaphore.acquire() + + def release(self) -> None: + self._semaphore.release() + + +class ShieldCancellation: + # Thread-synchronous codebases don't support cancellation semantics. + # We have this class because we need to mirror the async and sync + # cases within our package, but it's just a no-op. + def __enter__(self) -> ShieldCancellation: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + pass diff --git a/venv/lib/python3.12/site-packages/httpcore/_trace.py b/venv/lib/python3.12/site-packages/httpcore/_trace.py new file mode 100644 index 0000000000000000000000000000000000000000..5f1cd7c47829ce17dbcf651ab56b4ffdce04a485 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_trace.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import inspect +import logging +import types +import typing + +from ._models import Request + + +class Trace: + def __init__( + self, + name: str, + logger: logging.Logger, + request: Request | None = None, + kwargs: dict[str, typing.Any] | None = None, + ) -> None: + self.name = name + self.logger = logger + self.trace_extension = ( + None if request is None else request.extensions.get("trace") + ) + self.debug = self.logger.isEnabledFor(logging.DEBUG) + self.kwargs = kwargs or {} + self.return_value: typing.Any = None + self.should_trace = self.debug or self.trace_extension is not None + self.prefix = self.logger.name.split(".")[-1] + + def trace(self, name: str, info: dict[str, typing.Any]) -> None: + if self.trace_extension is not None: + prefix_and_name = f"{self.prefix}.{name}" + ret = self.trace_extension(prefix_and_name, info) + if inspect.iscoroutine(ret): # pragma: no cover + raise TypeError( + "If you are using a synchronous interface, " + "the callback of the `trace` extension should " + "be a normal function instead of an asynchronous function." + ) + + if self.debug: + if not info or "return_value" in info and info["return_value"] is None: + message = name + else: + args = " ".join([f"{key}={value!r}" for key, value in info.items()]) + message = f"{name} {args}" + self.logger.debug(message) + + def __enter__(self) -> Trace: + if self.should_trace: + info = self.kwargs + self.trace(f"{self.name}.started", info) + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + if self.should_trace: + if exc_value is None: + info = {"return_value": self.return_value} + self.trace(f"{self.name}.complete", info) + else: + info = {"exception": exc_value} + self.trace(f"{self.name}.failed", info) + + async def atrace(self, name: str, info: dict[str, typing.Any]) -> None: + if self.trace_extension is not None: + prefix_and_name = f"{self.prefix}.{name}" + coro = self.trace_extension(prefix_and_name, info) + if not inspect.iscoroutine(coro): # pragma: no cover + raise TypeError( + "If you're using an asynchronous interface, " + "the callback of the `trace` extension should " + "be an asynchronous function rather than a normal function." + ) + await coro + + if self.debug: + if not info or "return_value" in info and info["return_value"] is None: + message = name + else: + args = " ".join([f"{key}={value!r}" for key, value in info.items()]) + message = f"{name} {args}" + self.logger.debug(message) + + async def __aenter__(self) -> Trace: + if self.should_trace: + info = self.kwargs + await self.atrace(f"{self.name}.started", info) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + if self.should_trace: + if exc_value is None: + info = {"return_value": self.return_value} + await self.atrace(f"{self.name}.complete", info) + else: + info = {"exception": exc_value} + await self.atrace(f"{self.name}.failed", info) diff --git a/venv/lib/python3.12/site-packages/httpcore/_utils.py b/venv/lib/python3.12/site-packages/httpcore/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c44ff93cb2f572afc6e679308024b744b65c3b0a --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpcore/_utils.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import select +import socket +import sys + + +def is_socket_readable(sock: socket.socket | None) -> bool: + """ + Return whether a socket, as identifed by its file descriptor, is readable. + "A socket is readable" means that the read buffer isn't empty, i.e. that calling + .recv() on it would immediately return some data. + """ + # NOTE: we want check for readability without actually attempting to read, because + # we don't want to block forever if it's not readable. + + # In the case that the socket no longer exists, or cannot return a file + # descriptor, we treat it as being readable, as if it the next read operation + # on it is ready to return the terminating `b""`. + sock_fd = None if sock is None else sock.fileno() + if sock_fd is None or sock_fd < 0: # pragma: nocover + return True + + # The implementation below was stolen from: + # https://github.com/python-trio/trio/blob/20ee2b1b7376db637435d80e266212a35837ddcc/trio/_socket.py#L471-L478 + # See also: https://github.com/encode/httpcore/pull/193#issuecomment-703129316 + + # Use select.select on Windows, and when poll is unavailable and select.poll + # everywhere else. (E.g. When eventlet is in use. See #327) + if ( + sys.platform == "win32" or getattr(select, "poll", None) is None + ): # pragma: nocover + rready, _, _ = select.select([sock_fd], [], [], 0) + return bool(rready) + p = select.poll() + p.register(sock_fd, select.POLLIN) + return bool(p.poll(0)) diff --git a/venv/lib/python3.12/site-packages/httpcore/py.typed b/venv/lib/python3.12/site-packages/httpcore/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/INSTALLER b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/METADATA b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..b0d2b196385e98259971519793447c1fd7a9a643 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/METADATA @@ -0,0 +1,203 @@ +Metadata-Version: 2.3 +Name: httpx +Version: 0.28.1 +Summary: The next generation HTTP client. +Project-URL: Changelog, https://github.com/encode/httpx/blob/master/CHANGELOG.md +Project-URL: Documentation, https://www.python-httpx.org +Project-URL: Homepage, https://github.com/encode/httpx +Project-URL: Source, https://github.com/encode/httpx +Author-email: Tom Christie +License: BSD-3-Clause +Classifier: Development Status :: 4 - Beta +Classifier: Environment :: Web Environment +Classifier: Framework :: AsyncIO +Classifier: Framework :: Trio +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Topic :: Internet :: WWW/HTTP +Requires-Python: >=3.8 +Requires-Dist: anyio +Requires-Dist: certifi +Requires-Dist: httpcore==1.* +Requires-Dist: idna +Provides-Extra: brotli +Requires-Dist: brotli; (platform_python_implementation == 'CPython') and extra == 'brotli' +Requires-Dist: brotlicffi; (platform_python_implementation != 'CPython') and extra == 'brotli' +Provides-Extra: cli +Requires-Dist: click==8.*; extra == 'cli' +Requires-Dist: pygments==2.*; extra == 'cli' +Requires-Dist: rich<14,>=10; extra == 'cli' +Provides-Extra: http2 +Requires-Dist: h2<5,>=3; extra == 'http2' +Provides-Extra: socks +Requires-Dist: socksio==1.*; extra == 'socks' +Provides-Extra: zstd +Requires-Dist: zstandard>=0.18.0; extra == 'zstd' +Description-Content-Type: text/markdown + +

+ HTTPX +

+ +

HTTPX - A next-generation HTTP client for Python.

+ +

+ + Test Suite + + + Package version + +

+ +HTTPX is a fully featured HTTP client library for Python 3. It includes **an integrated command line client**, has support for both **HTTP/1.1 and HTTP/2**, and provides both **sync and async APIs**. + +--- + +Install HTTPX using pip: + +```shell +$ pip install httpx +``` + +Now, let's get started: + +```pycon +>>> import httpx +>>> r = httpx.get('https://www.example.org/') +>>> r + +>>> r.status_code +200 +>>> r.headers['content-type'] +'text/html; charset=UTF-8' +>>> r.text +'\n\n\nExample Domain...' +``` + +Or, using the command-line client. + +```shell +$ pip install 'httpx[cli]' # The command line client is an optional dependency. +``` + +Which now allows us to use HTTPX directly from the command-line... + +

+ httpx --help +

+ +Sending a request... + +

+ httpx http://httpbin.org/json +

+ +## Features + +HTTPX builds on the well-established usability of `requests`, and gives you: + +* A broadly [requests-compatible API](https://www.python-httpx.org/compatibility/). +* An integrated command-line client. +* HTTP/1.1 [and HTTP/2 support](https://www.python-httpx.org/http2/). +* Standard synchronous interface, but with [async support if you need it](https://www.python-httpx.org/async/). +* Ability to make requests directly to [WSGI applications](https://www.python-httpx.org/advanced/transports/#wsgi-transport) or [ASGI applications](https://www.python-httpx.org/advanced/transports/#asgi-transport). +* Strict timeouts everywhere. +* Fully type annotated. +* 100% test coverage. + +Plus all the standard features of `requests`... + +* International Domains and URLs +* Keep-Alive & Connection Pooling +* Sessions with Cookie Persistence +* Browser-style SSL Verification +* Basic/Digest Authentication +* Elegant Key/Value Cookies +* Automatic Decompression +* Automatic Content Decoding +* Unicode Response Bodies +* Multipart File Uploads +* HTTP(S) Proxy Support +* Connection Timeouts +* Streaming Downloads +* .netrc Support +* Chunked Requests + +## Installation + +Install with pip: + +```shell +$ pip install httpx +``` + +Or, to include the optional HTTP/2 support, use: + +```shell +$ pip install httpx[http2] +``` + +HTTPX requires Python 3.8+. + +## Documentation + +Project documentation is available at [https://www.python-httpx.org/](https://www.python-httpx.org/). + +For a run-through of all the basics, head over to the [QuickStart](https://www.python-httpx.org/quickstart/). + +For more advanced topics, see the [Advanced Usage](https://www.python-httpx.org/advanced/) section, the [async support](https://www.python-httpx.org/async/) section, or the [HTTP/2](https://www.python-httpx.org/http2/) section. + +The [Developer Interface](https://www.python-httpx.org/api/) provides a comprehensive API reference. + +To find out about tools that integrate with HTTPX, see [Third Party Packages](https://www.python-httpx.org/third_party_packages/). + +## Contribute + +If you want to contribute with HTTPX check out the [Contributing Guide](https://www.python-httpx.org/contributing/) to learn how to start. + +## Dependencies + +The HTTPX project relies on these excellent libraries: + +* `httpcore` - The underlying transport implementation for `httpx`. + * `h11` - HTTP/1.1 support. +* `certifi` - SSL certificates. +* `idna` - Internationalized domain name support. +* `sniffio` - Async library autodetection. + +As well as these optional installs: + +* `h2` - HTTP/2 support. *(Optional, with `httpx[http2]`)* +* `socksio` - SOCKS proxy support. *(Optional, with `httpx[socks]`)* +* `rich` - Rich terminal support. *(Optional, with `httpx[cli]`)* +* `click` - Command line client support. *(Optional, with `httpx[cli]`)* +* `brotli` or `brotlicffi` - Decoding for "brotli" compressed responses. *(Optional, with `httpx[brotli]`)* +* `zstandard` - Decoding for "zstd" compressed responses. *(Optional, with `httpx[zstd]`)* + +A huge amount of credit is due to `requests` for the API layout that +much of this work follows, as well as to `urllib3` for plenty of design +inspiration around the lower-level networking details. + +--- + +

HTTPX is BSD licensed code.
Designed & crafted with care.
— 🦋 —

+ +## Release Information + +### Fixed + +* Reintroduced supposedly-private `URLTypes` shortcut. (#2673) + + +--- + +[Full changelog](https://github.com/encode/httpx/blob/master/CHANGELOG.md) diff --git a/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/RECORD b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..9a21668cd903de8dbb2800c8664fca0b05fe2be4 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/RECORD @@ -0,0 +1,54 @@ +../../../bin/httpx,sha256=MnZC8HUjBL-ubtCy37H8jXXP7LM_acAi9t3VeBqtRgM,223 +httpx-0.28.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +httpx-0.28.1.dist-info/METADATA,sha256=_rubD48-gNV8gZnDBPNcQzboWB0dGNeYPJJ2a4J5OyU,7052 +httpx-0.28.1.dist-info/RECORD,, +httpx-0.28.1.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87 +httpx-0.28.1.dist-info/entry_points.txt,sha256=2lVkdQmxLA1pNMgSN2eV89o90HCZezhmNwsy6ryKDSA,37 +httpx-0.28.1.dist-info/licenses/LICENSE.md,sha256=TsWdVE8StfU5o6cW_TIaxYzNgDC0ZSIfLIgCAM3yjY0,1508 +httpx/__init__.py,sha256=CsaZe6yZj0rHg6322AWKWHGTMVr9txgEfD5P3_Rrz60,2171 +httpx/__pycache__/__init__.cpython-312.pyc,, +httpx/__pycache__/__version__.cpython-312.pyc,, +httpx/__pycache__/_api.cpython-312.pyc,, +httpx/__pycache__/_auth.cpython-312.pyc,, +httpx/__pycache__/_client.cpython-312.pyc,, +httpx/__pycache__/_config.cpython-312.pyc,, +httpx/__pycache__/_content.cpython-312.pyc,, +httpx/__pycache__/_decoders.cpython-312.pyc,, +httpx/__pycache__/_exceptions.cpython-312.pyc,, +httpx/__pycache__/_main.cpython-312.pyc,, +httpx/__pycache__/_models.cpython-312.pyc,, +httpx/__pycache__/_multipart.cpython-312.pyc,, +httpx/__pycache__/_status_codes.cpython-312.pyc,, +httpx/__pycache__/_types.cpython-312.pyc,, +httpx/__pycache__/_urlparse.cpython-312.pyc,, +httpx/__pycache__/_urls.cpython-312.pyc,, +httpx/__pycache__/_utils.cpython-312.pyc,, +httpx/__version__.py,sha256=LoUyYeOXTieGzuP_64UL0wxdtxjuu_QbOvE7NOg-IqU,108 +httpx/_api.py,sha256=r_Zgs4jIpcPJLqK5dbbSayqo_iVMKFaxZCd-oOHxLEs,11743 +httpx/_auth.py,sha256=Yr3QwaUSK17rGYx-7j-FdicFIzz4Y9FFV-1F4-7RXX4,11891 +httpx/_client.py,sha256=xD-UG67-WMkeltAAOeGGj-cZ2RRTAm19sWRxlFY7_40,65714 +httpx/_config.py,sha256=pPp2U-wicfcKsF-KYRE1LYdt3e6ERGeIoXZ8Gjo3LWc,8547 +httpx/_content.py,sha256=LGGzrJTR3OvN4Mb1GVVNLXkXJH-6oKlwAttO9p5w_yg,8161 +httpx/_decoders.py,sha256=p0dX8I0NEHexs3UGp4SsZutiMhsXrrWl6-GnqVb0iKM,12041 +httpx/_exceptions.py,sha256=bxW7fxzgVMAdNTbwT0Vnq04gJDW1_gI_GFiQPuMyjL0,8527 +httpx/_main.py,sha256=Cg9GMabiTT_swaDfUgIRitSwxLRMSwUDOm7LdSGqlA4,15626 +httpx/_models.py,sha256=4__Guyv1gLxuZChwim8kfQNiIOcJ9acreFOSurvZfms,44700 +httpx/_multipart.py,sha256=KOHEZZl6oohg9mPaKyyu345qq1rJLg35TUG3YAzXB3Y,9843 +httpx/_status_codes.py,sha256=DYn-2ufBgMeXy5s8x3_TB7wjAuAAMewTakPrm5rXEsc,5639 +httpx/_transports/__init__.py,sha256=GbUoBSAOp7z-l-9j5YhMhR3DMIcn6FVLhj072O3Nnno,275 +httpx/_transports/__pycache__/__init__.cpython-312.pyc,, +httpx/_transports/__pycache__/asgi.cpython-312.pyc,, +httpx/_transports/__pycache__/base.cpython-312.pyc,, +httpx/_transports/__pycache__/default.cpython-312.pyc,, +httpx/_transports/__pycache__/mock.cpython-312.pyc,, +httpx/_transports/__pycache__/wsgi.cpython-312.pyc,, +httpx/_transports/asgi.py,sha256=HRfiDYMPt4wQH2gFgHZg4c-i3sblo6bL5GTqcET-xz8,5501 +httpx/_transports/base.py,sha256=kZS_VMbViYfF570pogUCJ1bulz-ybfL51Pqs9yktebU,2523 +httpx/_transports/default.py,sha256=AzeaRUyVwCccTyyNJexDf0n1dFfzzydpdIQgvw7PLnk,13983 +httpx/_transports/mock.py,sha256=PTo0d567RITXxGrki6kN7_67wwAxfwiMDcuXJiZCjEo,1232 +httpx/_transports/wsgi.py,sha256=NcPX3Xap_EwCFZWO_OaSyQNuInCYx1QMNbO8GAei6jY,4825 +httpx/_types.py,sha256=Jyh41GQq7AOev8IOWKDAg7zCbvHAfufmW5g_PiTtErY,2965 +httpx/_urlparse.py,sha256=ZAmH47ONfkxrrj-PPYhGeiHjb6AjKCS-ANWIN4OL_KY,18546 +httpx/_urls.py,sha256=dX99VR1DSOHpgo9Aq7PzYO4FKdxqKjwyNp8grf8dHN0,21550 +httpx/_utils.py,sha256=_TVeqAKvxJkKHdz7dFeb4s0LZqQXgeFkXSgfiHBK_1o,8285 +httpx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/WHEEL b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..21aaa72961a8af71c17d2cb3b76d5f7f567100e4 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.26.3 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/entry_points.txt b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..8ae96007f7d725813fd02dc1d06d3834ee1939e4 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +httpx = httpx:main diff --git a/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/licenses/LICENSE.md b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/licenses/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..ab79d16a3f4c6c894c028d1f7431811e8711b42b --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx-0.28.1.dist-info/licenses/LICENSE.md @@ -0,0 +1,12 @@ +Copyright © 2019, [Encode OSS Ltd](https://www.encode.io/). +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/venv/lib/python3.12/site-packages/httpx/__init__.py b/venv/lib/python3.12/site-packages/httpx/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e9addde071f81758baf350c4ab6bde2556340131 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/__init__.py @@ -0,0 +1,105 @@ +from .__version__ import __description__, __title__, __version__ +from ._api import * +from ._auth import * +from ._client import * +from ._config import * +from ._content import * +from ._exceptions import * +from ._models import * +from ._status_codes import * +from ._transports import * +from ._types import * +from ._urls import * + +try: + from ._main import main +except ImportError: # pragma: no cover + + def main() -> None: # type: ignore + import sys + + print( + "The httpx command line client could not run because the required " + "dependencies were not installed.\nMake sure you've installed " + "everything with: pip install 'httpx[cli]'" + ) + sys.exit(1) + + +__all__ = [ + "__description__", + "__title__", + "__version__", + "ASGITransport", + "AsyncBaseTransport", + "AsyncByteStream", + "AsyncClient", + "AsyncHTTPTransport", + "Auth", + "BaseTransport", + "BasicAuth", + "ByteStream", + "Client", + "CloseError", + "codes", + "ConnectError", + "ConnectTimeout", + "CookieConflict", + "Cookies", + "create_ssl_context", + "DecodingError", + "delete", + "DigestAuth", + "get", + "head", + "Headers", + "HTTPError", + "HTTPStatusError", + "HTTPTransport", + "InvalidURL", + "Limits", + "LocalProtocolError", + "main", + "MockTransport", + "NetRCAuth", + "NetworkError", + "options", + "patch", + "PoolTimeout", + "post", + "ProtocolError", + "Proxy", + "ProxyError", + "put", + "QueryParams", + "ReadError", + "ReadTimeout", + "RemoteProtocolError", + "request", + "Request", + "RequestError", + "RequestNotRead", + "Response", + "ResponseNotRead", + "stream", + "StreamClosed", + "StreamConsumed", + "StreamError", + "SyncByteStream", + "Timeout", + "TimeoutException", + "TooManyRedirects", + "TransportError", + "UnsupportedProtocol", + "URL", + "USE_CLIENT_DEFAULT", + "WriteError", + "WriteTimeout", + "WSGITransport", +] + + +__locals = locals() +for __name in __all__: + if not __name.startswith("__"): + setattr(__locals[__name], "__module__", "httpx") # noqa diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..851c78044afd47f690550675da2b89001b421d6b Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/__version__.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/__version__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bba71fb6249a2ac2e02c65b597853be1662ced9b Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/__version__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_api.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..72197fe4c8e026db78992eff68190b4fbb370248 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_api.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_auth.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_auth.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c91992771cf711fbea8b1737607db837d939151 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_auth.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_client.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..608e83a3f14d8448f30aaed1e9d06d8b663de44a Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_client.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_config.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0442448388c35efc2294779899ed38ad4d3f1e18 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_config.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_content.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_content.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..afc234c4022205aa7df831ce6c2bf6f95b4f097f Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_content.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_decoders.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_decoders.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cfa8e30fdd923abe3070828b6dac18e5fbbc3686 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_decoders.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_exceptions.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ce32a873d16c0934fe34f540d48dd78c98b50c6 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_exceptions.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_main.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_main.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..760e56e371e0f837ceda73a3ec6594a7eb888fbb Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_main.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_models.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..28e8edf525e1e431748532bb288a5febab2e5b58 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_models.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_multipart.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_multipart.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c123c56119007d7fa60985942f5b6eb02dd27b2 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_multipart.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_status_codes.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_status_codes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d4cd5a0810fe77be5a3d208b2f416c5e22421b51 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_status_codes.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_types.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6a139b94dd449885414c8333a9ff785632b072da Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_types.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_urlparse.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_urlparse.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2046c9d659a3002b5b676bf60f0cf0423b058a6d Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_urlparse.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_urls.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_urls.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..513f5b5af124056c9d3c71873039bc63ada6ddfa Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_urls.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__pycache__/_utils.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/__pycache__/_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6345c62878d6951124b7320fbe51a027efa0cd2 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/__pycache__/_utils.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/__version__.py b/venv/lib/python3.12/site-packages/httpx/__version__.py new file mode 100644 index 0000000000000000000000000000000000000000..801bfacf671017cfbebf1ac26ec385daa02ed260 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/__version__.py @@ -0,0 +1,3 @@ +__title__ = "httpx" +__description__ = "A next generation HTTP client, for Python 3." +__version__ = "0.28.1" diff --git a/venv/lib/python3.12/site-packages/httpx/_api.py b/venv/lib/python3.12/site-packages/httpx/_api.py new file mode 100644 index 0000000000000000000000000000000000000000..c3cda1ecda8629edbdca2e3bc04bc51dba5e1430 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_api.py @@ -0,0 +1,438 @@ +from __future__ import annotations + +import typing +from contextlib import contextmanager + +from ._client import Client +from ._config import DEFAULT_TIMEOUT_CONFIG +from ._models import Response +from ._types import ( + AuthTypes, + CookieTypes, + HeaderTypes, + ProxyTypes, + QueryParamTypes, + RequestContent, + RequestData, + RequestFiles, + TimeoutTypes, +) +from ._urls import URL + +if typing.TYPE_CHECKING: + import ssl # pragma: no cover + + +__all__ = [ + "delete", + "get", + "head", + "options", + "patch", + "post", + "put", + "request", + "stream", +] + + +def request( + method: str, + url: URL | str, + *, + params: QueryParamTypes | None = None, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + trust_env: bool = True, +) -> Response: + """ + Sends an HTTP request. + + **Parameters:** + + * **method** - HTTP method for the new `Request` object: `GET`, `OPTIONS`, + `HEAD`, `POST`, `PUT`, `PATCH`, or `DELETE`. + * **url** - URL for the new `Request` object. + * **params** - *(optional)* Query parameters to include in the URL, as a + string, dictionary, or sequence of two-tuples. + * **content** - *(optional)* Binary content to include in the body of the + request, as bytes or a byte iterator. + * **data** - *(optional)* Form data to include in the body of the request, + as a dictionary. + * **files** - *(optional)* A dictionary of upload files to include in the + body of the request. + * **json** - *(optional)* A JSON serializable object to include in the body + of the request. + * **headers** - *(optional)* Dictionary of HTTP headers to include in the + request. + * **cookies** - *(optional)* Dictionary of Cookie items to include in the + request. + * **auth** - *(optional)* An authentication class to use when sending the + request. + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. + * **timeout** - *(optional)* The timeout configuration to use when sending + the request. + * **follow_redirects** - *(optional)* Enables or disables HTTP redirects. + * **verify** - *(optional)* Either `True` to use an SSL context with the + default CA bundle, `False` to disable verification, or an instance of + `ssl.SSLContext` to use a custom context. + * **trust_env** - *(optional)* Enables or disables usage of environment + variables for configuration. + + **Returns:** `Response` + + Usage: + + ``` + >>> import httpx + >>> response = httpx.request('GET', 'https://httpbin.org/get') + >>> response + + ``` + """ + with Client( + cookies=cookies, + proxy=proxy, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) as client: + return client.request( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + auth=auth, + follow_redirects=follow_redirects, + ) + + +@contextmanager +def stream( + method: str, + url: URL | str, + *, + params: QueryParamTypes | None = None, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + trust_env: bool = True, +) -> typing.Iterator[Response]: + """ + Alternative to `httpx.request()` that streams the response body + instead of loading it into memory at once. + + **Parameters**: See `httpx.request`. + + See also: [Streaming Responses][0] + + [0]: /quickstart#streaming-responses + """ + with Client( + cookies=cookies, + proxy=proxy, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) as client: + with client.stream( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + auth=auth, + follow_redirects=follow_redirects, + ) as response: + yield response + + +def get( + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends a `GET` request. + + **Parameters**: See `httpx.request`. + + Note that the `data`, `files`, `json` and `content` parameters are not available + on this function, as `GET` requests should not include a request body. + """ + return request( + "GET", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def options( + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends an `OPTIONS` request. + + **Parameters**: See `httpx.request`. + + Note that the `data`, `files`, `json` and `content` parameters are not available + on this function, as `OPTIONS` requests should not include a request body. + """ + return request( + "OPTIONS", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def head( + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends a `HEAD` request. + + **Parameters**: See `httpx.request`. + + Note that the `data`, `files`, `json` and `content` parameters are not available + on this function, as `HEAD` requests should not include a request body. + """ + return request( + "HEAD", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def post( + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends a `POST` request. + + **Parameters**: See `httpx.request`. + """ + return request( + "POST", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def put( + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends a `PUT` request. + + **Parameters**: See `httpx.request`. + """ + return request( + "PUT", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def patch( + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + verify: ssl.SSLContext | str | bool = True, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + trust_env: bool = True, +) -> Response: + """ + Sends a `PATCH` request. + + **Parameters**: See `httpx.request`. + """ + return request( + "PATCH", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) + + +def delete( + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | None = None, + proxy: ProxyTypes | None = None, + follow_redirects: bool = False, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + verify: ssl.SSLContext | str | bool = True, + trust_env: bool = True, +) -> Response: + """ + Sends a `DELETE` request. + + **Parameters**: See `httpx.request`. + + Note that the `data`, `files`, `json` and `content` parameters are not available + on this function, as `DELETE` requests should not include a request body. + """ + return request( + "DELETE", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + proxy=proxy, + follow_redirects=follow_redirects, + verify=verify, + timeout=timeout, + trust_env=trust_env, + ) diff --git a/venv/lib/python3.12/site-packages/httpx/_auth.py b/venv/lib/python3.12/site-packages/httpx/_auth.py new file mode 100644 index 0000000000000000000000000000000000000000..b03971ab4b311d60790dc22ca24d9966426ec0a4 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_auth.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +import hashlib +import os +import re +import time +import typing +from base64 import b64encode +from urllib.request import parse_http_list + +from ._exceptions import ProtocolError +from ._models import Cookies, Request, Response +from ._utils import to_bytes, to_str, unquote + +if typing.TYPE_CHECKING: # pragma: no cover + from hashlib import _Hash + + +__all__ = ["Auth", "BasicAuth", "DigestAuth", "NetRCAuth"] + + +class Auth: + """ + Base class for all authentication schemes. + + To implement a custom authentication scheme, subclass `Auth` and override + the `.auth_flow()` method. + + If the authentication scheme does I/O such as disk access or network calls, or uses + synchronization primitives such as locks, you should override `.sync_auth_flow()` + and/or `.async_auth_flow()` instead of `.auth_flow()` to provide specialized + implementations that will be used by `Client` and `AsyncClient` respectively. + """ + + requires_request_body = False + requires_response_body = False + + def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + """ + Execute the authentication flow. + + To dispatch a request, `yield` it: + + ``` + yield request + ``` + + The client will `.send()` the response back into the flow generator. You can + access it like so: + + ``` + response = yield request + ``` + + A `return` (or reaching the end of the generator) will result in the + client returning the last response obtained from the server. + + You can dispatch as many requests as is necessary. + """ + yield request + + def sync_auth_flow( + self, request: Request + ) -> typing.Generator[Request, Response, None]: + """ + Execute the authentication flow synchronously. + + By default, this defers to `.auth_flow()`. You should override this method + when the authentication scheme does I/O and/or uses concurrency primitives. + """ + if self.requires_request_body: + request.read() + + flow = self.auth_flow(request) + request = next(flow) + + while True: + response = yield request + if self.requires_response_body: + response.read() + + try: + request = flow.send(response) + except StopIteration: + break + + async def async_auth_flow( + self, request: Request + ) -> typing.AsyncGenerator[Request, Response]: + """ + Execute the authentication flow asynchronously. + + By default, this defers to `.auth_flow()`. You should override this method + when the authentication scheme does I/O and/or uses concurrency primitives. + """ + if self.requires_request_body: + await request.aread() + + flow = self.auth_flow(request) + request = next(flow) + + while True: + response = yield request + if self.requires_response_body: + await response.aread() + + try: + request = flow.send(response) + except StopIteration: + break + + +class FunctionAuth(Auth): + """ + Allows the 'auth' argument to be passed as a simple callable function, + that takes the request, and returns a new, modified request. + """ + + def __init__(self, func: typing.Callable[[Request], Request]) -> None: + self._func = func + + def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + yield self._func(request) + + +class BasicAuth(Auth): + """ + Allows the 'auth' argument to be passed as a (username, password) pair, + and uses HTTP Basic authentication. + """ + + def __init__(self, username: str | bytes, password: str | bytes) -> None: + self._auth_header = self._build_auth_header(username, password) + + def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + request.headers["Authorization"] = self._auth_header + yield request + + def _build_auth_header(self, username: str | bytes, password: str | bytes) -> str: + userpass = b":".join((to_bytes(username), to_bytes(password))) + token = b64encode(userpass).decode() + return f"Basic {token}" + + +class NetRCAuth(Auth): + """ + Use a 'netrc' file to lookup basic auth credentials based on the url host. + """ + + def __init__(self, file: str | None = None) -> None: + # Lazily import 'netrc'. + # There's no need for us to load this module unless 'NetRCAuth' is being used. + import netrc + + self._netrc_info = netrc.netrc(file) + + def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + auth_info = self._netrc_info.authenticators(request.url.host) + if auth_info is None or not auth_info[2]: + # The netrc file did not have authentication credentials for this host. + yield request + else: + # Build a basic auth header with credentials from the netrc file. + request.headers["Authorization"] = self._build_auth_header( + username=auth_info[0], password=auth_info[2] + ) + yield request + + def _build_auth_header(self, username: str | bytes, password: str | bytes) -> str: + userpass = b":".join((to_bytes(username), to_bytes(password))) + token = b64encode(userpass).decode() + return f"Basic {token}" + + +class DigestAuth(Auth): + _ALGORITHM_TO_HASH_FUNCTION: dict[str, typing.Callable[[bytes], _Hash]] = { + "MD5": hashlib.md5, + "MD5-SESS": hashlib.md5, + "SHA": hashlib.sha1, + "SHA-SESS": hashlib.sha1, + "SHA-256": hashlib.sha256, + "SHA-256-SESS": hashlib.sha256, + "SHA-512": hashlib.sha512, + "SHA-512-SESS": hashlib.sha512, + } + + def __init__(self, username: str | bytes, password: str | bytes) -> None: + self._username = to_bytes(username) + self._password = to_bytes(password) + self._last_challenge: _DigestAuthChallenge | None = None + self._nonce_count = 1 + + def auth_flow(self, request: Request) -> typing.Generator[Request, Response, None]: + if self._last_challenge: + request.headers["Authorization"] = self._build_auth_header( + request, self._last_challenge + ) + + response = yield request + + if response.status_code != 401 or "www-authenticate" not in response.headers: + # If the response is not a 401 then we don't + # need to build an authenticated request. + return + + for auth_header in response.headers.get_list("www-authenticate"): + if auth_header.lower().startswith("digest "): + break + else: + # If the response does not include a 'WWW-Authenticate: Digest ...' + # header, then we don't need to build an authenticated request. + return + + self._last_challenge = self._parse_challenge(request, response, auth_header) + self._nonce_count = 1 + + request.headers["Authorization"] = self._build_auth_header( + request, self._last_challenge + ) + if response.cookies: + Cookies(response.cookies).set_cookie_header(request=request) + yield request + + def _parse_challenge( + self, request: Request, response: Response, auth_header: str + ) -> _DigestAuthChallenge: + """ + Returns a challenge from a Digest WWW-Authenticate header. + These take the form of: + `Digest realm="realm@host.com",qop="auth,auth-int",nonce="abc",opaque="xyz"` + """ + scheme, _, fields = auth_header.partition(" ") + + # This method should only ever have been called with a Digest auth header. + assert scheme.lower() == "digest" + + header_dict: dict[str, str] = {} + for field in parse_http_list(fields): + key, value = field.strip().split("=", 1) + header_dict[key] = unquote(value) + + try: + realm = header_dict["realm"].encode() + nonce = header_dict["nonce"].encode() + algorithm = header_dict.get("algorithm", "MD5") + opaque = header_dict["opaque"].encode() if "opaque" in header_dict else None + qop = header_dict["qop"].encode() if "qop" in header_dict else None + return _DigestAuthChallenge( + realm=realm, nonce=nonce, algorithm=algorithm, opaque=opaque, qop=qop + ) + except KeyError as exc: + message = "Malformed Digest WWW-Authenticate header" + raise ProtocolError(message, request=request) from exc + + def _build_auth_header( + self, request: Request, challenge: _DigestAuthChallenge + ) -> str: + hash_func = self._ALGORITHM_TO_HASH_FUNCTION[challenge.algorithm.upper()] + + def digest(data: bytes) -> bytes: + return hash_func(data).hexdigest().encode() + + A1 = b":".join((self._username, challenge.realm, self._password)) + + path = request.url.raw_path + A2 = b":".join((request.method.encode(), path)) + # TODO: implement auth-int + HA2 = digest(A2) + + nc_value = b"%08x" % self._nonce_count + cnonce = self._get_client_nonce(self._nonce_count, challenge.nonce) + self._nonce_count += 1 + + HA1 = digest(A1) + if challenge.algorithm.lower().endswith("-sess"): + HA1 = digest(b":".join((HA1, challenge.nonce, cnonce))) + + qop = self._resolve_qop(challenge.qop, request=request) + if qop is None: + # Following RFC 2069 + digest_data = [HA1, challenge.nonce, HA2] + else: + # Following RFC 2617/7616 + digest_data = [HA1, challenge.nonce, nc_value, cnonce, qop, HA2] + + format_args = { + "username": self._username, + "realm": challenge.realm, + "nonce": challenge.nonce, + "uri": path, + "response": digest(b":".join(digest_data)), + "algorithm": challenge.algorithm.encode(), + } + if challenge.opaque: + format_args["opaque"] = challenge.opaque + if qop: + format_args["qop"] = b"auth" + format_args["nc"] = nc_value + format_args["cnonce"] = cnonce + + return "Digest " + self._get_header_value(format_args) + + def _get_client_nonce(self, nonce_count: int, nonce: bytes) -> bytes: + s = str(nonce_count).encode() + s += nonce + s += time.ctime().encode() + s += os.urandom(8) + + return hashlib.sha1(s).hexdigest()[:16].encode() + + def _get_header_value(self, header_fields: dict[str, bytes]) -> str: + NON_QUOTED_FIELDS = ("algorithm", "qop", "nc") + QUOTED_TEMPLATE = '{}="{}"' + NON_QUOTED_TEMPLATE = "{}={}" + + header_value = "" + for i, (field, value) in enumerate(header_fields.items()): + if i > 0: + header_value += ", " + template = ( + QUOTED_TEMPLATE + if field not in NON_QUOTED_FIELDS + else NON_QUOTED_TEMPLATE + ) + header_value += template.format(field, to_str(value)) + + return header_value + + def _resolve_qop(self, qop: bytes | None, request: Request) -> bytes | None: + if qop is None: + return None + qops = re.split(b", ?", qop) + if b"auth" in qops: + return b"auth" + + if qops == [b"auth-int"]: + raise NotImplementedError("Digest auth-int support is not yet implemented") + + message = f'Unexpected qop value "{qop!r}" in digest auth' + raise ProtocolError(message, request=request) + + +class _DigestAuthChallenge(typing.NamedTuple): + realm: bytes + nonce: bytes + algorithm: str + opaque: bytes | None + qop: bytes | None diff --git a/venv/lib/python3.12/site-packages/httpx/_client.py b/venv/lib/python3.12/site-packages/httpx/_client.py new file mode 100644 index 0000000000000000000000000000000000000000..2249231f8c3b912c731ff160344d3672e2f11738 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_client.py @@ -0,0 +1,2019 @@ +from __future__ import annotations + +import datetime +import enum +import logging +import time +import typing +import warnings +from contextlib import asynccontextmanager, contextmanager +from types import TracebackType + +from .__version__ import __version__ +from ._auth import Auth, BasicAuth, FunctionAuth +from ._config import ( + DEFAULT_LIMITS, + DEFAULT_MAX_REDIRECTS, + DEFAULT_TIMEOUT_CONFIG, + Limits, + Proxy, + Timeout, +) +from ._decoders import SUPPORTED_DECODERS +from ._exceptions import ( + InvalidURL, + RemoteProtocolError, + TooManyRedirects, + request_context, +) +from ._models import Cookies, Headers, Request, Response +from ._status_codes import codes +from ._transports.base import AsyncBaseTransport, BaseTransport +from ._transports.default import AsyncHTTPTransport, HTTPTransport +from ._types import ( + AsyncByteStream, + AuthTypes, + CertTypes, + CookieTypes, + HeaderTypes, + ProxyTypes, + QueryParamTypes, + RequestContent, + RequestData, + RequestExtensions, + RequestFiles, + SyncByteStream, + TimeoutTypes, +) +from ._urls import URL, QueryParams +from ._utils import URLPattern, get_environment_proxies + +if typing.TYPE_CHECKING: + import ssl # pragma: no cover + +__all__ = ["USE_CLIENT_DEFAULT", "AsyncClient", "Client"] + +# The type annotation for @classmethod and context managers here follows PEP 484 +# https://www.python.org/dev/peps/pep-0484/#annotating-instance-and-class-methods +T = typing.TypeVar("T", bound="Client") +U = typing.TypeVar("U", bound="AsyncClient") + + +def _is_https_redirect(url: URL, location: URL) -> bool: + """ + Return 'True' if 'location' is a HTTPS upgrade of 'url' + """ + if url.host != location.host: + return False + + return ( + url.scheme == "http" + and _port_or_default(url) == 80 + and location.scheme == "https" + and _port_or_default(location) == 443 + ) + + +def _port_or_default(url: URL) -> int | None: + if url.port is not None: + return url.port + return {"http": 80, "https": 443}.get(url.scheme) + + +def _same_origin(url: URL, other: URL) -> bool: + """ + Return 'True' if the given URLs share the same origin. + """ + return ( + url.scheme == other.scheme + and url.host == other.host + and _port_or_default(url) == _port_or_default(other) + ) + + +class UseClientDefault: + """ + For some parameters such as `auth=...` and `timeout=...` we need to be able + to indicate the default "unset" state, in a way that is distinctly different + to using `None`. + + The default "unset" state indicates that whatever default is set on the + client should be used. This is different to setting `None`, which + explicitly disables the parameter, possibly overriding a client default. + + For example we use `timeout=USE_CLIENT_DEFAULT` in the `request()` signature. + Omitting the `timeout` parameter will send a request using whatever default + timeout has been configured on the client. Including `timeout=None` will + ensure no timeout is used. + + Note that user code shouldn't need to use the `USE_CLIENT_DEFAULT` constant, + but it is used internally when a parameter is not included. + """ + + +USE_CLIENT_DEFAULT = UseClientDefault() + + +logger = logging.getLogger("httpx") + +USER_AGENT = f"python-httpx/{__version__}" +ACCEPT_ENCODING = ", ".join( + [key for key in SUPPORTED_DECODERS.keys() if key != "identity"] +) + + +class ClientState(enum.Enum): + # UNOPENED: + # The client has been instantiated, but has not been used to send a request, + # or been opened by entering the context of a `with` block. + UNOPENED = 1 + # OPENED: + # The client has either sent a request, or is within a `with` block. + OPENED = 2 + # CLOSED: + # The client has either exited the `with` block, or `close()` has + # been called explicitly. + CLOSED = 3 + + +class BoundSyncStream(SyncByteStream): + """ + A byte stream that is bound to a given response instance, and that + ensures the `response.elapsed` is set once the response is closed. + """ + + def __init__( + self, stream: SyncByteStream, response: Response, start: float + ) -> None: + self._stream = stream + self._response = response + self._start = start + + def __iter__(self) -> typing.Iterator[bytes]: + for chunk in self._stream: + yield chunk + + def close(self) -> None: + elapsed = time.perf_counter() - self._start + self._response.elapsed = datetime.timedelta(seconds=elapsed) + self._stream.close() + + +class BoundAsyncStream(AsyncByteStream): + """ + An async byte stream that is bound to a given response instance, and that + ensures the `response.elapsed` is set once the response is closed. + """ + + def __init__( + self, stream: AsyncByteStream, response: Response, start: float + ) -> None: + self._stream = stream + self._response = response + self._start = start + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + async for chunk in self._stream: + yield chunk + + async def aclose(self) -> None: + elapsed = time.perf_counter() - self._start + self._response.elapsed = datetime.timedelta(seconds=elapsed) + await self._stream.aclose() + + +EventHook = typing.Callable[..., typing.Any] + + +class BaseClient: + def __init__( + self, + *, + auth: AuthTypes | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + follow_redirects: bool = False, + max_redirects: int = DEFAULT_MAX_REDIRECTS, + event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, + base_url: URL | str = "", + trust_env: bool = True, + default_encoding: str | typing.Callable[[bytes], str] = "utf-8", + ) -> None: + event_hooks = {} if event_hooks is None else event_hooks + + self._base_url = self._enforce_trailing_slash(URL(base_url)) + + self._auth = self._build_auth(auth) + self._params = QueryParams(params) + self.headers = Headers(headers) + self._cookies = Cookies(cookies) + self._timeout = Timeout(timeout) + self.follow_redirects = follow_redirects + self.max_redirects = max_redirects + self._event_hooks = { + "request": list(event_hooks.get("request", [])), + "response": list(event_hooks.get("response", [])), + } + self._trust_env = trust_env + self._default_encoding = default_encoding + self._state = ClientState.UNOPENED + + @property + def is_closed(self) -> bool: + """ + Check if the client being closed + """ + return self._state == ClientState.CLOSED + + @property + def trust_env(self) -> bool: + return self._trust_env + + def _enforce_trailing_slash(self, url: URL) -> URL: + if url.raw_path.endswith(b"/"): + return url + return url.copy_with(raw_path=url.raw_path + b"/") + + def _get_proxy_map( + self, proxy: ProxyTypes | None, allow_env_proxies: bool + ) -> dict[str, Proxy | None]: + if proxy is None: + if allow_env_proxies: + return { + key: None if url is None else Proxy(url=url) + for key, url in get_environment_proxies().items() + } + return {} + else: + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + return {"all://": proxy} + + @property + def timeout(self) -> Timeout: + return self._timeout + + @timeout.setter + def timeout(self, timeout: TimeoutTypes) -> None: + self._timeout = Timeout(timeout) + + @property + def event_hooks(self) -> dict[str, list[EventHook]]: + return self._event_hooks + + @event_hooks.setter + def event_hooks(self, event_hooks: dict[str, list[EventHook]]) -> None: + self._event_hooks = { + "request": list(event_hooks.get("request", [])), + "response": list(event_hooks.get("response", [])), + } + + @property + def auth(self) -> Auth | None: + """ + Authentication class used when none is passed at the request-level. + + See also [Authentication][0]. + + [0]: /quickstart/#authentication + """ + return self._auth + + @auth.setter + def auth(self, auth: AuthTypes) -> None: + self._auth = self._build_auth(auth) + + @property + def base_url(self) -> URL: + """ + Base URL to use when sending requests with relative URLs. + """ + return self._base_url + + @base_url.setter + def base_url(self, url: URL | str) -> None: + self._base_url = self._enforce_trailing_slash(URL(url)) + + @property + def headers(self) -> Headers: + """ + HTTP headers to include when sending requests. + """ + return self._headers + + @headers.setter + def headers(self, headers: HeaderTypes) -> None: + client_headers = Headers( + { + b"Accept": b"*/*", + b"Accept-Encoding": ACCEPT_ENCODING.encode("ascii"), + b"Connection": b"keep-alive", + b"User-Agent": USER_AGENT.encode("ascii"), + } + ) + client_headers.update(headers) + self._headers = client_headers + + @property + def cookies(self) -> Cookies: + """ + Cookie values to include when sending requests. + """ + return self._cookies + + @cookies.setter + def cookies(self, cookies: CookieTypes) -> None: + self._cookies = Cookies(cookies) + + @property + def params(self) -> QueryParams: + """ + Query parameters to include in the URL when sending requests. + """ + return self._params + + @params.setter + def params(self, params: QueryParamTypes) -> None: + self._params = QueryParams(params) + + def build_request( + self, + method: str, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Request: + """ + Build and return a request instance. + + * The `params`, `headers` and `cookies` arguments + are merged with any values set on the client. + * The `url` argument is merged with any `base_url` set on the client. + + See also: [Request instances][0] + + [0]: /advanced/clients/#request-instances + """ + url = self._merge_url(url) + headers = self._merge_headers(headers) + cookies = self._merge_cookies(cookies) + params = self._merge_queryparams(params) + extensions = {} if extensions is None else extensions + if "timeout" not in extensions: + timeout = ( + self.timeout + if isinstance(timeout, UseClientDefault) + else Timeout(timeout) + ) + extensions = dict(**extensions, timeout=timeout.as_dict()) + return Request( + method, + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + extensions=extensions, + ) + + def _merge_url(self, url: URL | str) -> URL: + """ + Merge a URL argument together with any 'base_url' on the client, + to create the URL used for the outgoing request. + """ + merge_url = URL(url) + if merge_url.is_relative_url: + # To merge URLs we always append to the base URL. To get this + # behaviour correct we always ensure the base URL ends in a '/' + # separator, and strip any leading '/' from the merge URL. + # + # So, eg... + # + # >>> client = Client(base_url="https://www.example.com/subpath") + # >>> client.base_url + # URL('https://www.example.com/subpath/') + # >>> client.build_request("GET", "/path").url + # URL('https://www.example.com/subpath/path') + merge_raw_path = self.base_url.raw_path + merge_url.raw_path.lstrip(b"/") + return self.base_url.copy_with(raw_path=merge_raw_path) + return merge_url + + def _merge_cookies(self, cookies: CookieTypes | None = None) -> CookieTypes | None: + """ + Merge a cookies argument together with any cookies on the client, + to create the cookies used for the outgoing request. + """ + if cookies or self.cookies: + merged_cookies = Cookies(self.cookies) + merged_cookies.update(cookies) + return merged_cookies + return cookies + + def _merge_headers(self, headers: HeaderTypes | None = None) -> HeaderTypes | None: + """ + Merge a headers argument together with any headers on the client, + to create the headers used for the outgoing request. + """ + merged_headers = Headers(self.headers) + merged_headers.update(headers) + return merged_headers + + def _merge_queryparams( + self, params: QueryParamTypes | None = None + ) -> QueryParamTypes | None: + """ + Merge a queryparams argument together with any queryparams on the client, + to create the queryparams used for the outgoing request. + """ + if params or self.params: + merged_queryparams = QueryParams(self.params) + return merged_queryparams.merge(params) + return params + + def _build_auth(self, auth: AuthTypes | None) -> Auth | None: + if auth is None: + return None + elif isinstance(auth, tuple): + return BasicAuth(username=auth[0], password=auth[1]) + elif isinstance(auth, Auth): + return auth + elif callable(auth): + return FunctionAuth(func=auth) + else: + raise TypeError(f'Invalid "auth" argument: {auth!r}') + + def _build_request_auth( + self, + request: Request, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + ) -> Auth: + auth = ( + self._auth if isinstance(auth, UseClientDefault) else self._build_auth(auth) + ) + + if auth is not None: + return auth + + username, password = request.url.username, request.url.password + if username or password: + return BasicAuth(username=username, password=password) + + return Auth() + + def _build_redirect_request(self, request: Request, response: Response) -> Request: + """ + Given a request and a redirect response, return a new request that + should be used to effect the redirect. + """ + method = self._redirect_method(request, response) + url = self._redirect_url(request, response) + headers = self._redirect_headers(request, url, method) + stream = self._redirect_stream(request, method) + cookies = Cookies(self.cookies) + return Request( + method=method, + url=url, + headers=headers, + cookies=cookies, + stream=stream, + extensions=request.extensions, + ) + + def _redirect_method(self, request: Request, response: Response) -> str: + """ + When being redirected we may want to change the method of the request + based on certain specs or browser behavior. + """ + method = request.method + + # https://tools.ietf.org/html/rfc7231#section-6.4.4 + if response.status_code == codes.SEE_OTHER and method != "HEAD": + method = "GET" + + # Do what the browsers do, despite standards... + # Turn 302s into GETs. + if response.status_code == codes.FOUND and method != "HEAD": + method = "GET" + + # If a POST is responded to with a 301, turn it into a GET. + # This bizarre behaviour is explained in 'requests' issue 1704. + if response.status_code == codes.MOVED_PERMANENTLY and method == "POST": + method = "GET" + + return method + + def _redirect_url(self, request: Request, response: Response) -> URL: + """ + Return the URL for the redirect to follow. + """ + location = response.headers["Location"] + + try: + url = URL(location) + except InvalidURL as exc: + raise RemoteProtocolError( + f"Invalid URL in location header: {exc}.", request=request + ) from None + + # Handle malformed 'Location' headers that are "absolute" form, have no host. + # See: https://github.com/encode/httpx/issues/771 + if url.scheme and not url.host: + url = url.copy_with(host=request.url.host) + + # Facilitate relative 'Location' headers, as allowed by RFC 7231. + # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') + if url.is_relative_url: + url = request.url.join(url) + + # Attach previous fragment if needed (RFC 7231 7.1.2) + if request.url.fragment and not url.fragment: + url = url.copy_with(fragment=request.url.fragment) + + return url + + def _redirect_headers(self, request: Request, url: URL, method: str) -> Headers: + """ + Return the headers that should be used for the redirect request. + """ + headers = Headers(request.headers) + + if not _same_origin(url, request.url): + if not _is_https_redirect(request.url, url): + # Strip Authorization headers when responses are redirected + # away from the origin. (Except for direct HTTP to HTTPS redirects.) + headers.pop("Authorization", None) + + # Update the Host header. + headers["Host"] = url.netloc.decode("ascii") + + if method != request.method and method == "GET": + # If we've switch to a 'GET' request, then strip any headers which + # are only relevant to the request body. + headers.pop("Content-Length", None) + headers.pop("Transfer-Encoding", None) + + # We should use the client cookie store to determine any cookie header, + # rather than whatever was on the original outgoing request. + headers.pop("Cookie", None) + + return headers + + def _redirect_stream( + self, request: Request, method: str + ) -> SyncByteStream | AsyncByteStream | None: + """ + Return the body that should be used for the redirect request. + """ + if method != request.method and method == "GET": + return None + + return request.stream + + def _set_timeout(self, request: Request) -> None: + if "timeout" not in request.extensions: + timeout = ( + self.timeout + if isinstance(self.timeout, UseClientDefault) + else Timeout(self.timeout) + ) + request.extensions = dict(**request.extensions, timeout=timeout.as_dict()) + + +class Client(BaseClient): + """ + An HTTP client, with connection pooling, HTTP/2, redirects, cookie persistence, etc. + + It can be shared between threads. + + Usage: + + ```python + >>> client = httpx.Client() + >>> response = client.get('https://example.org') + ``` + + **Parameters:** + + * **auth** - *(optional)* An authentication class to use when sending + requests. + * **params** - *(optional)* Query parameters to include in request URLs, as + a string, dictionary, or sequence of two-tuples. + * **headers** - *(optional)* Dictionary of HTTP headers to include when + sending requests. + * **cookies** - *(optional)* Dictionary of Cookie items to include when + sending requests. + * **verify** - *(optional)* Either `True` to use an SSL context with the + default CA bundle, `False` to disable verification, or an instance of + `ssl.SSLContext` to use a custom context. + * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be + enabled. Defaults to `False`. + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. + * **timeout** - *(optional)* The timeout configuration to use when sending + requests. + * **limits** - *(optional)* The limits configuration to use. + * **max_redirects** - *(optional)* The maximum number of redirect responses + that should be followed. + * **base_url** - *(optional)* A URL to use as the base when building + request URLs. + * **transport** - *(optional)* A transport class to use for sending requests + over the network. + * **trust_env** - *(optional)* Enables or disables usage of environment + variables for configuration. + * **default_encoding** - *(optional)* The default encoding to use for decoding + response text, if no charset information is included in a response Content-Type + header. Set to a callable for automatic character set detection. Default: "utf-8". + """ + + def __init__( + self, + *, + auth: AuthTypes | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + proxy: ProxyTypes | None = None, + mounts: None | (typing.Mapping[str, BaseTransport | None]) = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + follow_redirects: bool = False, + limits: Limits = DEFAULT_LIMITS, + max_redirects: int = DEFAULT_MAX_REDIRECTS, + event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, + base_url: URL | str = "", + transport: BaseTransport | None = None, + default_encoding: str | typing.Callable[[bytes], str] = "utf-8", + ) -> None: + super().__init__( + auth=auth, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + follow_redirects=follow_redirects, + max_redirects=max_redirects, + event_hooks=event_hooks, + base_url=base_url, + trust_env=trust_env, + default_encoding=default_encoding, + ) + + if http2: + try: + import h2 # noqa + except ImportError: # pragma: no cover + raise ImportError( + "Using http2=True, but the 'h2' package is not installed. " + "Make sure to install httpx using `pip install httpx[http2]`." + ) from None + + allow_env_proxies = trust_env and transport is None + proxy_map = self._get_proxy_map(proxy, allow_env_proxies) + + self._transport = self._init_transport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + transport=transport, + ) + self._mounts: dict[URLPattern, BaseTransport | None] = { + URLPattern(key): None + if proxy is None + else self._init_proxy_transport( + proxy, + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + ) + for key, proxy in proxy_map.items() + } + if mounts is not None: + self._mounts.update( + {URLPattern(key): transport for key, transport in mounts.items()} + ) + + self._mounts = dict(sorted(self._mounts.items())) + + def _init_transport( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + transport: BaseTransport | None = None, + ) -> BaseTransport: + if transport is not None: + return transport + + return HTTPTransport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + ) + + def _init_proxy_transport( + self, + proxy: Proxy, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + ) -> BaseTransport: + return HTTPTransport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + proxy=proxy, + ) + + def _transport_for_url(self, url: URL) -> BaseTransport: + """ + Returns the transport instance that should be used for a given URL. + This will either be the standard connection pool, or a proxy. + """ + for pattern, transport in self._mounts.items(): + if pattern.matches(url): + return self._transport if transport is None else transport + + return self._transport + + def request( + self, + method: str, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Build and send a request. + + Equivalent to: + + ```python + request = client.build_request(...) + response = client.send(request, ...) + ``` + + See `Client.build_request()`, `Client.send()` and + [Merging of configuration][0] for how the various parameters + are merged with client-level configuration. + + [0]: /advanced/clients/#merging-of-configuration + """ + if cookies is not None: + message = ( + "Setting per-request cookies=<...> is being deprecated, because " + "the expected behaviour on cookie persistence is ambiguous. Set " + "cookies directly on the client instance instead." + ) + warnings.warn(message, DeprecationWarning, stacklevel=2) + + request = self.build_request( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + extensions=extensions, + ) + return self.send(request, auth=auth, follow_redirects=follow_redirects) + + @contextmanager + def stream( + self, + method: str, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> typing.Iterator[Response]: + """ + Alternative to `httpx.request()` that streams the response body + instead of loading it into memory at once. + + **Parameters**: See `httpx.request`. + + See also: [Streaming Responses][0] + + [0]: /quickstart#streaming-responses + """ + request = self.build_request( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + extensions=extensions, + ) + response = self.send( + request=request, + auth=auth, + follow_redirects=follow_redirects, + stream=True, + ) + try: + yield response + finally: + response.close() + + def send( + self, + request: Request, + *, + stream: bool = False, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + ) -> Response: + """ + Send a request. + + The request is sent as-is, unmodified. + + Typically you'll want to build one with `Client.build_request()` + so that any client-level configuration is merged into the request, + but passing an explicit `httpx.Request()` is supported as well. + + See also: [Request instances][0] + + [0]: /advanced/clients/#request-instances + """ + if self._state == ClientState.CLOSED: + raise RuntimeError("Cannot send a request, as the client has been closed.") + + self._state = ClientState.OPENED + follow_redirects = ( + self.follow_redirects + if isinstance(follow_redirects, UseClientDefault) + else follow_redirects + ) + + self._set_timeout(request) + + auth = self._build_request_auth(request, auth) + + response = self._send_handling_auth( + request, + auth=auth, + follow_redirects=follow_redirects, + history=[], + ) + try: + if not stream: + response.read() + + return response + + except BaseException as exc: + response.close() + raise exc + + def _send_handling_auth( + self, + request: Request, + auth: Auth, + follow_redirects: bool, + history: list[Response], + ) -> Response: + auth_flow = auth.sync_auth_flow(request) + try: + request = next(auth_flow) + + while True: + response = self._send_handling_redirects( + request, + follow_redirects=follow_redirects, + history=history, + ) + try: + try: + next_request = auth_flow.send(response) + except StopIteration: + return response + + response.history = list(history) + response.read() + request = next_request + history.append(response) + + except BaseException as exc: + response.close() + raise exc + finally: + auth_flow.close() + + def _send_handling_redirects( + self, + request: Request, + follow_redirects: bool, + history: list[Response], + ) -> Response: + while True: + if len(history) > self.max_redirects: + raise TooManyRedirects( + "Exceeded maximum allowed redirects.", request=request + ) + + for hook in self._event_hooks["request"]: + hook(request) + + response = self._send_single_request(request) + try: + for hook in self._event_hooks["response"]: + hook(response) + response.history = list(history) + + if not response.has_redirect_location: + return response + + request = self._build_redirect_request(request, response) + history = history + [response] + + if follow_redirects: + response.read() + else: + response.next_request = request + return response + + except BaseException as exc: + response.close() + raise exc + + def _send_single_request(self, request: Request) -> Response: + """ + Sends a single request, without handling any redirections. + """ + transport = self._transport_for_url(request.url) + start = time.perf_counter() + + if not isinstance(request.stream, SyncByteStream): + raise RuntimeError( + "Attempted to send an async request with a sync Client instance." + ) + + with request_context(request=request): + response = transport.handle_request(request) + + assert isinstance(response.stream, SyncByteStream) + + response.request = request + response.stream = BoundSyncStream( + response.stream, response=response, start=start + ) + self.cookies.extract_cookies(response) + response.default_encoding = self._default_encoding + + logger.info( + 'HTTP Request: %s %s "%s %d %s"', + request.method, + request.url, + response.http_version, + response.status_code, + response.reason_phrase, + ) + + return response + + def get( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `GET` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "GET", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def options( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send an `OPTIONS` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "OPTIONS", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def head( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `HEAD` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "HEAD", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def post( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `POST` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "POST", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def put( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `PUT` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "PUT", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def patch( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `PATCH` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "PATCH", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def delete( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `DELETE` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "DELETE", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + def close(self) -> None: + """ + Close transport and proxies. + """ + if self._state != ClientState.CLOSED: + self._state = ClientState.CLOSED + + self._transport.close() + for transport in self._mounts.values(): + if transport is not None: + transport.close() + + def __enter__(self: T) -> T: + if self._state != ClientState.UNOPENED: + msg = { + ClientState.OPENED: "Cannot open a client instance more than once.", + ClientState.CLOSED: ( + "Cannot reopen a client instance, once it has been closed." + ), + }[self._state] + raise RuntimeError(msg) + + self._state = ClientState.OPENED + + self._transport.__enter__() + for transport in self._mounts.values(): + if transport is not None: + transport.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + self._state = ClientState.CLOSED + + self._transport.__exit__(exc_type, exc_value, traceback) + for transport in self._mounts.values(): + if transport is not None: + transport.__exit__(exc_type, exc_value, traceback) + + +class AsyncClient(BaseClient): + """ + An asynchronous HTTP client, with connection pooling, HTTP/2, redirects, + cookie persistence, etc. + + It can be shared between tasks. + + Usage: + + ```python + >>> async with httpx.AsyncClient() as client: + >>> response = await client.get('https://example.org') + ``` + + **Parameters:** + + * **auth** - *(optional)* An authentication class to use when sending + requests. + * **params** - *(optional)* Query parameters to include in request URLs, as + a string, dictionary, or sequence of two-tuples. + * **headers** - *(optional)* Dictionary of HTTP headers to include when + sending requests. + * **cookies** - *(optional)* Dictionary of Cookie items to include when + sending requests. + * **verify** - *(optional)* Either `True` to use an SSL context with the + default CA bundle, `False` to disable verification, or an instance of + `ssl.SSLContext` to use a custom context. + * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be + enabled. Defaults to `False`. + * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. + * **timeout** - *(optional)* The timeout configuration to use when sending + requests. + * **limits** - *(optional)* The limits configuration to use. + * **max_redirects** - *(optional)* The maximum number of redirect responses + that should be followed. + * **base_url** - *(optional)* A URL to use as the base when building + request URLs. + * **transport** - *(optional)* A transport class to use for sending requests + over the network. + * **trust_env** - *(optional)* Enables or disables usage of environment + variables for configuration. + * **default_encoding** - *(optional)* The default encoding to use for decoding + response text, if no charset information is included in a response Content-Type + header. Set to a callable for automatic character set detection. Default: "utf-8". + """ + + def __init__( + self, + *, + auth: AuthTypes | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + http1: bool = True, + http2: bool = False, + proxy: ProxyTypes | None = None, + mounts: None | (typing.Mapping[str, AsyncBaseTransport | None]) = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + follow_redirects: bool = False, + limits: Limits = DEFAULT_LIMITS, + max_redirects: int = DEFAULT_MAX_REDIRECTS, + event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, + base_url: URL | str = "", + transport: AsyncBaseTransport | None = None, + trust_env: bool = True, + default_encoding: str | typing.Callable[[bytes], str] = "utf-8", + ) -> None: + super().__init__( + auth=auth, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + follow_redirects=follow_redirects, + max_redirects=max_redirects, + event_hooks=event_hooks, + base_url=base_url, + trust_env=trust_env, + default_encoding=default_encoding, + ) + + if http2: + try: + import h2 # noqa + except ImportError: # pragma: no cover + raise ImportError( + "Using http2=True, but the 'h2' package is not installed. " + "Make sure to install httpx using `pip install httpx[http2]`." + ) from None + + allow_env_proxies = trust_env and transport is None + proxy_map = self._get_proxy_map(proxy, allow_env_proxies) + + self._transport = self._init_transport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + transport=transport, + ) + + self._mounts: dict[URLPattern, AsyncBaseTransport | None] = { + URLPattern(key): None + if proxy is None + else self._init_proxy_transport( + proxy, + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + ) + for key, proxy in proxy_map.items() + } + if mounts is not None: + self._mounts.update( + {URLPattern(key): transport for key, transport in mounts.items()} + ) + self._mounts = dict(sorted(self._mounts.items())) + + def _init_transport( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + transport: AsyncBaseTransport | None = None, + ) -> AsyncBaseTransport: + if transport is not None: + return transport + + return AsyncHTTPTransport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + ) + + def _init_proxy_transport( + self, + proxy: Proxy, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + ) -> AsyncBaseTransport: + return AsyncHTTPTransport( + verify=verify, + cert=cert, + trust_env=trust_env, + http1=http1, + http2=http2, + limits=limits, + proxy=proxy, + ) + + def _transport_for_url(self, url: URL) -> AsyncBaseTransport: + """ + Returns the transport instance that should be used for a given URL. + This will either be the standard connection pool, or a proxy. + """ + for pattern, transport in self._mounts.items(): + if pattern.matches(url): + return self._transport if transport is None else transport + + return self._transport + + async def request( + self, + method: str, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Build and send a request. + + Equivalent to: + + ```python + request = client.build_request(...) + response = await client.send(request, ...) + ``` + + See `AsyncClient.build_request()`, `AsyncClient.send()` + and [Merging of configuration][0] for how the various parameters + are merged with client-level configuration. + + [0]: /advanced/clients/#merging-of-configuration + """ + + if cookies is not None: # pragma: no cover + message = ( + "Setting per-request cookies=<...> is being deprecated, because " + "the expected behaviour on cookie persistence is ambiguous. Set " + "cookies directly on the client instance instead." + ) + warnings.warn(message, DeprecationWarning, stacklevel=2) + + request = self.build_request( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + extensions=extensions, + ) + return await self.send(request, auth=auth, follow_redirects=follow_redirects) + + @asynccontextmanager + async def stream( + self, + method: str, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> typing.AsyncIterator[Response]: + """ + Alternative to `httpx.request()` that streams the response body + instead of loading it into memory at once. + + **Parameters**: See `httpx.request`. + + See also: [Streaming Responses][0] + + [0]: /quickstart#streaming-responses + """ + request = self.build_request( + method=method, + url=url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + extensions=extensions, + ) + response = await self.send( + request=request, + auth=auth, + follow_redirects=follow_redirects, + stream=True, + ) + try: + yield response + finally: + await response.aclose() + + async def send( + self, + request: Request, + *, + stream: bool = False, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + ) -> Response: + """ + Send a request. + + The request is sent as-is, unmodified. + + Typically you'll want to build one with `AsyncClient.build_request()` + so that any client-level configuration is merged into the request, + but passing an explicit `httpx.Request()` is supported as well. + + See also: [Request instances][0] + + [0]: /advanced/clients/#request-instances + """ + if self._state == ClientState.CLOSED: + raise RuntimeError("Cannot send a request, as the client has been closed.") + + self._state = ClientState.OPENED + follow_redirects = ( + self.follow_redirects + if isinstance(follow_redirects, UseClientDefault) + else follow_redirects + ) + + self._set_timeout(request) + + auth = self._build_request_auth(request, auth) + + response = await self._send_handling_auth( + request, + auth=auth, + follow_redirects=follow_redirects, + history=[], + ) + try: + if not stream: + await response.aread() + + return response + + except BaseException as exc: + await response.aclose() + raise exc + + async def _send_handling_auth( + self, + request: Request, + auth: Auth, + follow_redirects: bool, + history: list[Response], + ) -> Response: + auth_flow = auth.async_auth_flow(request) + try: + request = await auth_flow.__anext__() + + while True: + response = await self._send_handling_redirects( + request, + follow_redirects=follow_redirects, + history=history, + ) + try: + try: + next_request = await auth_flow.asend(response) + except StopAsyncIteration: + return response + + response.history = list(history) + await response.aread() + request = next_request + history.append(response) + + except BaseException as exc: + await response.aclose() + raise exc + finally: + await auth_flow.aclose() + + async def _send_handling_redirects( + self, + request: Request, + follow_redirects: bool, + history: list[Response], + ) -> Response: + while True: + if len(history) > self.max_redirects: + raise TooManyRedirects( + "Exceeded maximum allowed redirects.", request=request + ) + + for hook in self._event_hooks["request"]: + await hook(request) + + response = await self._send_single_request(request) + try: + for hook in self._event_hooks["response"]: + await hook(response) + + response.history = list(history) + + if not response.has_redirect_location: + return response + + request = self._build_redirect_request(request, response) + history = history + [response] + + if follow_redirects: + await response.aread() + else: + response.next_request = request + return response + + except BaseException as exc: + await response.aclose() + raise exc + + async def _send_single_request(self, request: Request) -> Response: + """ + Sends a single request, without handling any redirections. + """ + transport = self._transport_for_url(request.url) + start = time.perf_counter() + + if not isinstance(request.stream, AsyncByteStream): + raise RuntimeError( + "Attempted to send an sync request with an AsyncClient instance." + ) + + with request_context(request=request): + response = await transport.handle_async_request(request) + + assert isinstance(response.stream, AsyncByteStream) + response.request = request + response.stream = BoundAsyncStream( + response.stream, response=response, start=start + ) + self.cookies.extract_cookies(response) + response.default_encoding = self._default_encoding + + logger.info( + 'HTTP Request: %s %s "%s %d %s"', + request.method, + request.url, + response.http_version, + response.status_code, + response.reason_phrase, + ) + + return response + + async def get( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `GET` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "GET", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def options( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send an `OPTIONS` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "OPTIONS", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def head( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `HEAD` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "HEAD", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def post( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `POST` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "POST", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def put( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `PUT` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "PUT", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def patch( + self, + url: URL | str, + *, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `PATCH` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "PATCH", + url, + content=content, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def delete( + self, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT, + follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT, + timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT, + extensions: RequestExtensions | None = None, + ) -> Response: + """ + Send a `DELETE` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "DELETE", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + follow_redirects=follow_redirects, + timeout=timeout, + extensions=extensions, + ) + + async def aclose(self) -> None: + """ + Close transport and proxies. + """ + if self._state != ClientState.CLOSED: + self._state = ClientState.CLOSED + + await self._transport.aclose() + for proxy in self._mounts.values(): + if proxy is not None: + await proxy.aclose() + + async def __aenter__(self: U) -> U: + if self._state != ClientState.UNOPENED: + msg = { + ClientState.OPENED: "Cannot open a client instance more than once.", + ClientState.CLOSED: ( + "Cannot reopen a client instance, once it has been closed." + ), + }[self._state] + raise RuntimeError(msg) + + self._state = ClientState.OPENED + + await self._transport.__aenter__() + for proxy in self._mounts.values(): + if proxy is not None: + await proxy.__aenter__() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + self._state = ClientState.CLOSED + + await self._transport.__aexit__(exc_type, exc_value, traceback) + for proxy in self._mounts.values(): + if proxy is not None: + await proxy.__aexit__(exc_type, exc_value, traceback) diff --git a/venv/lib/python3.12/site-packages/httpx/_config.py b/venv/lib/python3.12/site-packages/httpx/_config.py new file mode 100644 index 0000000000000000000000000000000000000000..467a6c90ae269babe3af7963d9d7c78b9f012268 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_config.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import os +import typing + +from ._models import Headers +from ._types import CertTypes, HeaderTypes, TimeoutTypes +from ._urls import URL + +if typing.TYPE_CHECKING: + import ssl # pragma: no cover + +__all__ = ["Limits", "Proxy", "Timeout", "create_ssl_context"] + + +class UnsetType: + pass # pragma: no cover + + +UNSET = UnsetType() + + +def create_ssl_context( + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, +) -> ssl.SSLContext: + import ssl + import warnings + + import certifi + + if verify is True: + if trust_env and os.environ.get("SSL_CERT_FILE"): # pragma: nocover + ctx = ssl.create_default_context(cafile=os.environ["SSL_CERT_FILE"]) + elif trust_env and os.environ.get("SSL_CERT_DIR"): # pragma: nocover + ctx = ssl.create_default_context(capath=os.environ["SSL_CERT_DIR"]) + else: + # Default case... + ctx = ssl.create_default_context(cafile=certifi.where()) + elif verify is False: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + elif isinstance(verify, str): # pragma: nocover + message = ( + "`verify=` is deprecated. " + "Use `verify=ssl.create_default_context(cafile=...)` " + "or `verify=ssl.create_default_context(capath=...)` instead." + ) + warnings.warn(message, DeprecationWarning) + if os.path.isdir(verify): + return ssl.create_default_context(capath=verify) + return ssl.create_default_context(cafile=verify) + else: + ctx = verify + + if cert: # pragma: nocover + message = ( + "`cert=...` is deprecated. Use `verify=` instead," + "with `.load_cert_chain()` to configure the certificate chain." + ) + warnings.warn(message, DeprecationWarning) + if isinstance(cert, str): + ctx.load_cert_chain(cert) + else: + ctx.load_cert_chain(*cert) + + return ctx + + +class Timeout: + """ + Timeout configuration. + + **Usage**: + + Timeout(None) # No timeouts. + Timeout(5.0) # 5s timeout on all operations. + Timeout(None, connect=5.0) # 5s timeout on connect, no other timeouts. + Timeout(5.0, connect=10.0) # 10s timeout on connect. 5s timeout elsewhere. + Timeout(5.0, pool=None) # No timeout on acquiring connection from pool. + # 5s timeout elsewhere. + """ + + def __init__( + self, + timeout: TimeoutTypes | UnsetType = UNSET, + *, + connect: None | float | UnsetType = UNSET, + read: None | float | UnsetType = UNSET, + write: None | float | UnsetType = UNSET, + pool: None | float | UnsetType = UNSET, + ) -> None: + if isinstance(timeout, Timeout): + # Passed as a single explicit Timeout. + assert connect is UNSET + assert read is UNSET + assert write is UNSET + assert pool is UNSET + self.connect = timeout.connect # type: typing.Optional[float] + self.read = timeout.read # type: typing.Optional[float] + self.write = timeout.write # type: typing.Optional[float] + self.pool = timeout.pool # type: typing.Optional[float] + elif isinstance(timeout, tuple): + # Passed as a tuple. + self.connect = timeout[0] + self.read = timeout[1] + self.write = None if len(timeout) < 3 else timeout[2] + self.pool = None if len(timeout) < 4 else timeout[3] + elif not ( + isinstance(connect, UnsetType) + or isinstance(read, UnsetType) + or isinstance(write, UnsetType) + or isinstance(pool, UnsetType) + ): + self.connect = connect + self.read = read + self.write = write + self.pool = pool + else: + if isinstance(timeout, UnsetType): + raise ValueError( + "httpx.Timeout must either include a default, or set all " + "four parameters explicitly." + ) + self.connect = timeout if isinstance(connect, UnsetType) else connect + self.read = timeout if isinstance(read, UnsetType) else read + self.write = timeout if isinstance(write, UnsetType) else write + self.pool = timeout if isinstance(pool, UnsetType) else pool + + def as_dict(self) -> dict[str, float | None]: + return { + "connect": self.connect, + "read": self.read, + "write": self.write, + "pool": self.pool, + } + + def __eq__(self, other: typing.Any) -> bool: + return ( + isinstance(other, self.__class__) + and self.connect == other.connect + and self.read == other.read + and self.write == other.write + and self.pool == other.pool + ) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + if len({self.connect, self.read, self.write, self.pool}) == 1: + return f"{class_name}(timeout={self.connect})" + return ( + f"{class_name}(connect={self.connect}, " + f"read={self.read}, write={self.write}, pool={self.pool})" + ) + + +class Limits: + """ + Configuration for limits to various client behaviors. + + **Parameters:** + + * **max_connections** - The maximum number of concurrent connections that may be + established. + * **max_keepalive_connections** - Allow the connection pool to maintain + keep-alive connections below this point. Should be less than or equal + to `max_connections`. + * **keepalive_expiry** - Time limit on idle keep-alive connections in seconds. + """ + + def __init__( + self, + *, + max_connections: int | None = None, + max_keepalive_connections: int | None = None, + keepalive_expiry: float | None = 5.0, + ) -> None: + self.max_connections = max_connections + self.max_keepalive_connections = max_keepalive_connections + self.keepalive_expiry = keepalive_expiry + + def __eq__(self, other: typing.Any) -> bool: + return ( + isinstance(other, self.__class__) + and self.max_connections == other.max_connections + and self.max_keepalive_connections == other.max_keepalive_connections + and self.keepalive_expiry == other.keepalive_expiry + ) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + return ( + f"{class_name}(max_connections={self.max_connections}, " + f"max_keepalive_connections={self.max_keepalive_connections}, " + f"keepalive_expiry={self.keepalive_expiry})" + ) + + +class Proxy: + def __init__( + self, + url: URL | str, + *, + ssl_context: ssl.SSLContext | None = None, + auth: tuple[str, str] | None = None, + headers: HeaderTypes | None = None, + ) -> None: + url = URL(url) + headers = Headers(headers) + + if url.scheme not in ("http", "https", "socks5", "socks5h"): + raise ValueError(f"Unknown scheme for proxy URL {url!r}") + + if url.username or url.password: + # Remove any auth credentials from the URL. + auth = (url.username, url.password) + url = url.copy_with(username=None, password=None) + + self.url = url + self.auth = auth + self.headers = headers + self.ssl_context = ssl_context + + @property + def raw_auth(self) -> tuple[bytes, bytes] | None: + # The proxy authentication as raw bytes. + return ( + None + if self.auth is None + else (self.auth[0].encode("utf-8"), self.auth[1].encode("utf-8")) + ) + + def __repr__(self) -> str: + # The authentication is represented with the password component masked. + auth = (self.auth[0], "********") if self.auth else None + + # Build a nice concise representation. + url_str = f"{str(self.url)!r}" + auth_str = f", auth={auth!r}" if auth else "" + headers_str = f", headers={dict(self.headers)!r}" if self.headers else "" + return f"Proxy({url_str}{auth_str}{headers_str})" + + +DEFAULT_TIMEOUT_CONFIG = Timeout(timeout=5.0) +DEFAULT_LIMITS = Limits(max_connections=100, max_keepalive_connections=20) +DEFAULT_MAX_REDIRECTS = 20 diff --git a/venv/lib/python3.12/site-packages/httpx/_content.py b/venv/lib/python3.12/site-packages/httpx/_content.py new file mode 100644 index 0000000000000000000000000000000000000000..6f479a0885f723b7395843d41164a87041820776 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_content.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +import inspect +import warnings +from json import dumps as json_dumps +from typing import ( + Any, + AsyncIterable, + AsyncIterator, + Iterable, + Iterator, + Mapping, +) +from urllib.parse import urlencode + +from ._exceptions import StreamClosed, StreamConsumed +from ._multipart import MultipartStream +from ._types import ( + AsyncByteStream, + RequestContent, + RequestData, + RequestFiles, + ResponseContent, + SyncByteStream, +) +from ._utils import peek_filelike_length, primitive_value_to_str + +__all__ = ["ByteStream"] + + +class ByteStream(AsyncByteStream, SyncByteStream): + def __init__(self, stream: bytes) -> None: + self._stream = stream + + def __iter__(self) -> Iterator[bytes]: + yield self._stream + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield self._stream + + +class IteratorByteStream(SyncByteStream): + CHUNK_SIZE = 65_536 + + def __init__(self, stream: Iterable[bytes]) -> None: + self._stream = stream + self._is_stream_consumed = False + self._is_generator = inspect.isgenerator(stream) + + def __iter__(self) -> Iterator[bytes]: + if self._is_stream_consumed and self._is_generator: + raise StreamConsumed() + + self._is_stream_consumed = True + if hasattr(self._stream, "read"): + # File-like interfaces should use 'read' directly. + chunk = self._stream.read(self.CHUNK_SIZE) + while chunk: + yield chunk + chunk = self._stream.read(self.CHUNK_SIZE) + else: + # Otherwise iterate. + for part in self._stream: + yield part + + +class AsyncIteratorByteStream(AsyncByteStream): + CHUNK_SIZE = 65_536 + + def __init__(self, stream: AsyncIterable[bytes]) -> None: + self._stream = stream + self._is_stream_consumed = False + self._is_generator = inspect.isasyncgen(stream) + + async def __aiter__(self) -> AsyncIterator[bytes]: + if self._is_stream_consumed and self._is_generator: + raise StreamConsumed() + + self._is_stream_consumed = True + if hasattr(self._stream, "aread"): + # File-like interfaces should use 'aread' directly. + chunk = await self._stream.aread(self.CHUNK_SIZE) + while chunk: + yield chunk + chunk = await self._stream.aread(self.CHUNK_SIZE) + else: + # Otherwise iterate. + async for part in self._stream: + yield part + + +class UnattachedStream(AsyncByteStream, SyncByteStream): + """ + If a request or response is serialized using pickle, then it is no longer + attached to a stream for I/O purposes. Any stream operations should result + in `httpx.StreamClosed`. + """ + + def __iter__(self) -> Iterator[bytes]: + raise StreamClosed() + + async def __aiter__(self) -> AsyncIterator[bytes]: + raise StreamClosed() + yield b"" # pragma: no cover + + +def encode_content( + content: str | bytes | Iterable[bytes] | AsyncIterable[bytes], +) -> tuple[dict[str, str], SyncByteStream | AsyncByteStream]: + if isinstance(content, (bytes, str)): + body = content.encode("utf-8") if isinstance(content, str) else content + content_length = len(body) + headers = {"Content-Length": str(content_length)} if body else {} + return headers, ByteStream(body) + + elif isinstance(content, Iterable) and not isinstance(content, dict): + # `not isinstance(content, dict)` is a bit oddly specific, but it + # catches a case that's easy for users to make in error, and would + # otherwise pass through here, like any other bytes-iterable, + # because `dict` happens to be iterable. See issue #2491. + content_length_or_none = peek_filelike_length(content) + + if content_length_or_none is None: + headers = {"Transfer-Encoding": "chunked"} + else: + headers = {"Content-Length": str(content_length_or_none)} + return headers, IteratorByteStream(content) # type: ignore + + elif isinstance(content, AsyncIterable): + headers = {"Transfer-Encoding": "chunked"} + return headers, AsyncIteratorByteStream(content) + + raise TypeError(f"Unexpected type for 'content', {type(content)!r}") + + +def encode_urlencoded_data( + data: RequestData, +) -> tuple[dict[str, str], ByteStream]: + plain_data = [] + for key, value in data.items(): + if isinstance(value, (list, tuple)): + plain_data.extend([(key, primitive_value_to_str(item)) for item in value]) + else: + plain_data.append((key, primitive_value_to_str(value))) + body = urlencode(plain_data, doseq=True).encode("utf-8") + content_length = str(len(body)) + content_type = "application/x-www-form-urlencoded" + headers = {"Content-Length": content_length, "Content-Type": content_type} + return headers, ByteStream(body) + + +def encode_multipart_data( + data: RequestData, files: RequestFiles, boundary: bytes | None +) -> tuple[dict[str, str], MultipartStream]: + multipart = MultipartStream(data=data, files=files, boundary=boundary) + headers = multipart.get_headers() + return headers, multipart + + +def encode_text(text: str) -> tuple[dict[str, str], ByteStream]: + body = text.encode("utf-8") + content_length = str(len(body)) + content_type = "text/plain; charset=utf-8" + headers = {"Content-Length": content_length, "Content-Type": content_type} + return headers, ByteStream(body) + + +def encode_html(html: str) -> tuple[dict[str, str], ByteStream]: + body = html.encode("utf-8") + content_length = str(len(body)) + content_type = "text/html; charset=utf-8" + headers = {"Content-Length": content_length, "Content-Type": content_type} + return headers, ByteStream(body) + + +def encode_json(json: Any) -> tuple[dict[str, str], ByteStream]: + body = json_dumps( + json, ensure_ascii=False, separators=(",", ":"), allow_nan=False + ).encode("utf-8") + content_length = str(len(body)) + content_type = "application/json" + headers = {"Content-Length": content_length, "Content-Type": content_type} + return headers, ByteStream(body) + + +def encode_request( + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: Any | None = None, + boundary: bytes | None = None, +) -> tuple[dict[str, str], SyncByteStream | AsyncByteStream]: + """ + Handles encoding the given `content`, `data`, `files`, and `json`, + returning a two-tuple of (, ). + """ + if data is not None and not isinstance(data, Mapping): + # We prefer to separate `content=` + # for raw request content, and `data=
` for url encoded or + # multipart form content. + # + # However for compat with requests, we *do* still support + # `data=` usages. We deal with that case here, treating it + # as if `content=<...>` had been supplied instead. + message = "Use 'content=<...>' to upload raw bytes/text content." + warnings.warn(message, DeprecationWarning, stacklevel=2) + return encode_content(data) + + if content is not None: + return encode_content(content) + elif files: + return encode_multipart_data(data or {}, files, boundary) + elif data: + return encode_urlencoded_data(data) + elif json is not None: + return encode_json(json) + + return {}, ByteStream(b"") + + +def encode_response( + content: ResponseContent | None = None, + text: str | None = None, + html: str | None = None, + json: Any | None = None, +) -> tuple[dict[str, str], SyncByteStream | AsyncByteStream]: + """ + Handles encoding the given `content`, returning a two-tuple of + (, ). + """ + if content is not None: + return encode_content(content) + elif text is not None: + return encode_text(text) + elif html is not None: + return encode_html(html) + elif json is not None: + return encode_json(json) + + return {}, ByteStream(b"") diff --git a/venv/lib/python3.12/site-packages/httpx/_decoders.py b/venv/lib/python3.12/site-packages/httpx/_decoders.py new file mode 100644 index 0000000000000000000000000000000000000000..899dfada878e1181fca6d3c75a79526a076abb9e --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_decoders.py @@ -0,0 +1,393 @@ +""" +Handlers for Content-Encoding. + +See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding +""" + +from __future__ import annotations + +import codecs +import io +import typing +import zlib + +from ._exceptions import DecodingError + +# Brotli support is optional +try: + # The C bindings in `brotli` are recommended for CPython. + import brotli +except ImportError: # pragma: no cover + try: + # The CFFI bindings in `brotlicffi` are recommended for PyPy + # and other environments. + import brotlicffi as brotli + except ImportError: + brotli = None + + +# Zstandard support is optional +try: + import zstandard +except ImportError: # pragma: no cover + zstandard = None # type: ignore + + +class ContentDecoder: + def decode(self, data: bytes) -> bytes: + raise NotImplementedError() # pragma: no cover + + def flush(self) -> bytes: + raise NotImplementedError() # pragma: no cover + + +class IdentityDecoder(ContentDecoder): + """ + Handle unencoded data. + """ + + def decode(self, data: bytes) -> bytes: + return data + + def flush(self) -> bytes: + return b"" + + +class DeflateDecoder(ContentDecoder): + """ + Handle 'deflate' decoding. + + See: https://stackoverflow.com/questions/1838699 + """ + + def __init__(self) -> None: + self.first_attempt = True + self.decompressor = zlib.decompressobj() + + def decode(self, data: bytes) -> bytes: + was_first_attempt = self.first_attempt + self.first_attempt = False + try: + return self.decompressor.decompress(data) + except zlib.error as exc: + if was_first_attempt: + self.decompressor = zlib.decompressobj(-zlib.MAX_WBITS) + return self.decode(data) + raise DecodingError(str(exc)) from exc + + def flush(self) -> bytes: + try: + return self.decompressor.flush() + except zlib.error as exc: # pragma: no cover + raise DecodingError(str(exc)) from exc + + +class GZipDecoder(ContentDecoder): + """ + Handle 'gzip' decoding. + + See: https://stackoverflow.com/questions/1838699 + """ + + def __init__(self) -> None: + self.decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16) + + def decode(self, data: bytes) -> bytes: + try: + return self.decompressor.decompress(data) + except zlib.error as exc: + raise DecodingError(str(exc)) from exc + + def flush(self) -> bytes: + try: + return self.decompressor.flush() + except zlib.error as exc: # pragma: no cover + raise DecodingError(str(exc)) from exc + + +class BrotliDecoder(ContentDecoder): + """ + Handle 'brotli' decoding. + + Requires `pip install brotlipy`. See: https://brotlipy.readthedocs.io/ + or `pip install brotli`. See https://github.com/google/brotli + Supports both 'brotlipy' and 'Brotli' packages since they share an import + name. The top branches are for 'brotlipy' and bottom branches for 'Brotli' + """ + + def __init__(self) -> None: + if brotli is None: # pragma: no cover + raise ImportError( + "Using 'BrotliDecoder', but neither of the 'brotlicffi' or 'brotli' " + "packages have been installed. " + "Make sure to install httpx using `pip install httpx[brotli]`." + ) from None + + self.decompressor = brotli.Decompressor() + self.seen_data = False + self._decompress: typing.Callable[[bytes], bytes] + if hasattr(self.decompressor, "decompress"): + # The 'brotlicffi' package. + self._decompress = self.decompressor.decompress # pragma: no cover + else: + # The 'brotli' package. + self._decompress = self.decompressor.process # pragma: no cover + + def decode(self, data: bytes) -> bytes: + if not data: + return b"" + self.seen_data = True + try: + return self._decompress(data) + except brotli.error as exc: + raise DecodingError(str(exc)) from exc + + def flush(self) -> bytes: + if not self.seen_data: + return b"" + try: + if hasattr(self.decompressor, "finish"): + # Only available in the 'brotlicffi' package. + + # As the decompressor decompresses eagerly, this + # will never actually emit any data. However, it will potentially throw + # errors if a truncated or damaged data stream has been used. + self.decompressor.finish() # pragma: no cover + return b"" + except brotli.error as exc: # pragma: no cover + raise DecodingError(str(exc)) from exc + + +class ZStandardDecoder(ContentDecoder): + """ + Handle 'zstd' RFC 8878 decoding. + + Requires `pip install zstandard`. + Can be installed as a dependency of httpx using `pip install httpx[zstd]`. + """ + + # inspired by the ZstdDecoder implementation in urllib3 + def __init__(self) -> None: + if zstandard is None: # pragma: no cover + raise ImportError( + "Using 'ZStandardDecoder', ..." + "Make sure to install httpx using `pip install httpx[zstd]`." + ) from None + + self.decompressor = zstandard.ZstdDecompressor().decompressobj() + self.seen_data = False + + def decode(self, data: bytes) -> bytes: + assert zstandard is not None + self.seen_data = True + output = io.BytesIO() + try: + output.write(self.decompressor.decompress(data)) + while self.decompressor.eof and self.decompressor.unused_data: + unused_data = self.decompressor.unused_data + self.decompressor = zstandard.ZstdDecompressor().decompressobj() + output.write(self.decompressor.decompress(unused_data)) + except zstandard.ZstdError as exc: + raise DecodingError(str(exc)) from exc + return output.getvalue() + + def flush(self) -> bytes: + if not self.seen_data: + return b"" + ret = self.decompressor.flush() # note: this is a no-op + if not self.decompressor.eof: + raise DecodingError("Zstandard data is incomplete") # pragma: no cover + return bytes(ret) + + +class MultiDecoder(ContentDecoder): + """ + Handle the case where multiple encodings have been applied. + """ + + def __init__(self, children: typing.Sequence[ContentDecoder]) -> None: + """ + 'children' should be a sequence of decoders in the order in which + each was applied. + """ + # Note that we reverse the order for decoding. + self.children = list(reversed(children)) + + def decode(self, data: bytes) -> bytes: + for child in self.children: + data = child.decode(data) + return data + + def flush(self) -> bytes: + data = b"" + for child in self.children: + data = child.decode(data) + child.flush() + return data + + +class ByteChunker: + """ + Handles returning byte content in fixed-size chunks. + """ + + def __init__(self, chunk_size: int | None = None) -> None: + self._buffer = io.BytesIO() + self._chunk_size = chunk_size + + def decode(self, content: bytes) -> list[bytes]: + if self._chunk_size is None: + return [content] if content else [] + + self._buffer.write(content) + if self._buffer.tell() >= self._chunk_size: + value = self._buffer.getvalue() + chunks = [ + value[i : i + self._chunk_size] + for i in range(0, len(value), self._chunk_size) + ] + if len(chunks[-1]) == self._chunk_size: + self._buffer.seek(0) + self._buffer.truncate() + return chunks + else: + self._buffer.seek(0) + self._buffer.write(chunks[-1]) + self._buffer.truncate() + return chunks[:-1] + else: + return [] + + def flush(self) -> list[bytes]: + value = self._buffer.getvalue() + self._buffer.seek(0) + self._buffer.truncate() + return [value] if value else [] + + +class TextChunker: + """ + Handles returning text content in fixed-size chunks. + """ + + def __init__(self, chunk_size: int | None = None) -> None: + self._buffer = io.StringIO() + self._chunk_size = chunk_size + + def decode(self, content: str) -> list[str]: + if self._chunk_size is None: + return [content] if content else [] + + self._buffer.write(content) + if self._buffer.tell() >= self._chunk_size: + value = self._buffer.getvalue() + chunks = [ + value[i : i + self._chunk_size] + for i in range(0, len(value), self._chunk_size) + ] + if len(chunks[-1]) == self._chunk_size: + self._buffer.seek(0) + self._buffer.truncate() + return chunks + else: + self._buffer.seek(0) + self._buffer.write(chunks[-1]) + self._buffer.truncate() + return chunks[:-1] + else: + return [] + + def flush(self) -> list[str]: + value = self._buffer.getvalue() + self._buffer.seek(0) + self._buffer.truncate() + return [value] if value else [] + + +class TextDecoder: + """ + Handles incrementally decoding bytes into text + """ + + def __init__(self, encoding: str = "utf-8") -> None: + self.decoder = codecs.getincrementaldecoder(encoding)(errors="replace") + + def decode(self, data: bytes) -> str: + return self.decoder.decode(data) + + def flush(self) -> str: + return self.decoder.decode(b"", True) + + +class LineDecoder: + """ + Handles incrementally reading lines from text. + + Has the same behaviour as the stdllib splitlines, + but handling the input iteratively. + """ + + def __init__(self) -> None: + self.buffer: list[str] = [] + self.trailing_cr: bool = False + + def decode(self, text: str) -> list[str]: + # See https://docs.python.org/3/library/stdtypes.html#str.splitlines + NEWLINE_CHARS = "\n\r\x0b\x0c\x1c\x1d\x1e\x85\u2028\u2029" + + # We always push a trailing `\r` into the next decode iteration. + if self.trailing_cr: + text = "\r" + text + self.trailing_cr = False + if text.endswith("\r"): + self.trailing_cr = True + text = text[:-1] + + if not text: + # NOTE: the edge case input of empty text doesn't occur in practice, + # because other httpx internals filter out this value + return [] # pragma: no cover + + trailing_newline = text[-1] in NEWLINE_CHARS + lines = text.splitlines() + + if len(lines) == 1 and not trailing_newline: + # No new lines, buffer the input and continue. + self.buffer.append(lines[0]) + return [] + + if self.buffer: + # Include any existing buffer in the first portion of the + # splitlines result. + lines = ["".join(self.buffer) + lines[0]] + lines[1:] + self.buffer = [] + + if not trailing_newline: + # If the last segment of splitlines is not newline terminated, + # then drop it from our output and start a new buffer. + self.buffer = [lines.pop()] + + return lines + + def flush(self) -> list[str]: + if not self.buffer and not self.trailing_cr: + return [] + + lines = ["".join(self.buffer)] + self.buffer = [] + self.trailing_cr = False + return lines + + +SUPPORTED_DECODERS = { + "identity": IdentityDecoder, + "gzip": GZipDecoder, + "deflate": DeflateDecoder, + "br": BrotliDecoder, + "zstd": ZStandardDecoder, +} + + +if brotli is None: + SUPPORTED_DECODERS.pop("br") # pragma: no cover +if zstandard is None: + SUPPORTED_DECODERS.pop("zstd") # pragma: no cover diff --git a/venv/lib/python3.12/site-packages/httpx/_exceptions.py b/venv/lib/python3.12/site-packages/httpx/_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..77f45a6d3986d15626fc8a5fd459d6a3e0fbe466 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_exceptions.py @@ -0,0 +1,379 @@ +""" +Our exception hierarchy: + +* HTTPError + x RequestError + + TransportError + - TimeoutException + · ConnectTimeout + · ReadTimeout + · WriteTimeout + · PoolTimeout + - NetworkError + · ConnectError + · ReadError + · WriteError + · CloseError + - ProtocolError + · LocalProtocolError + · RemoteProtocolError + - ProxyError + - UnsupportedProtocol + + DecodingError + + TooManyRedirects + x HTTPStatusError +* InvalidURL +* CookieConflict +* StreamError + x StreamConsumed + x StreamClosed + x ResponseNotRead + x RequestNotRead +""" + +from __future__ import annotations + +import contextlib +import typing + +if typing.TYPE_CHECKING: + from ._models import Request, Response # pragma: no cover + +__all__ = [ + "CloseError", + "ConnectError", + "ConnectTimeout", + "CookieConflict", + "DecodingError", + "HTTPError", + "HTTPStatusError", + "InvalidURL", + "LocalProtocolError", + "NetworkError", + "PoolTimeout", + "ProtocolError", + "ProxyError", + "ReadError", + "ReadTimeout", + "RemoteProtocolError", + "RequestError", + "RequestNotRead", + "ResponseNotRead", + "StreamClosed", + "StreamConsumed", + "StreamError", + "TimeoutException", + "TooManyRedirects", + "TransportError", + "UnsupportedProtocol", + "WriteError", + "WriteTimeout", +] + + +class HTTPError(Exception): + """ + Base class for `RequestError` and `HTTPStatusError`. + + Useful for `try...except` blocks when issuing a request, + and then calling `.raise_for_status()`. + + For example: + + ``` + try: + response = httpx.get("https://www.example.com") + response.raise_for_status() + except httpx.HTTPError as exc: + print(f"HTTP Exception for {exc.request.url} - {exc}") + ``` + """ + + def __init__(self, message: str) -> None: + super().__init__(message) + self._request: Request | None = None + + @property + def request(self) -> Request: + if self._request is None: + raise RuntimeError("The .request property has not been set.") + return self._request + + @request.setter + def request(self, request: Request) -> None: + self._request = request + + +class RequestError(HTTPError): + """ + Base class for all exceptions that may occur when issuing a `.request()`. + """ + + def __init__(self, message: str, *, request: Request | None = None) -> None: + super().__init__(message) + # At the point an exception is raised we won't typically have a request + # instance to associate it with. + # + # The 'request_context' context manager is used within the Client and + # Response methods in order to ensure that any raised exceptions + # have a `.request` property set on them. + self._request = request + + +class TransportError(RequestError): + """ + Base class for all exceptions that occur at the level of the Transport API. + """ + + +# Timeout exceptions... + + +class TimeoutException(TransportError): + """ + The base class for timeout errors. + + An operation has timed out. + """ + + +class ConnectTimeout(TimeoutException): + """ + Timed out while connecting to the host. + """ + + +class ReadTimeout(TimeoutException): + """ + Timed out while receiving data from the host. + """ + + +class WriteTimeout(TimeoutException): + """ + Timed out while sending data to the host. + """ + + +class PoolTimeout(TimeoutException): + """ + Timed out waiting to acquire a connection from the pool. + """ + + +# Core networking exceptions... + + +class NetworkError(TransportError): + """ + The base class for network-related errors. + + An error occurred while interacting with the network. + """ + + +class ReadError(NetworkError): + """ + Failed to receive data from the network. + """ + + +class WriteError(NetworkError): + """ + Failed to send data through the network. + """ + + +class ConnectError(NetworkError): + """ + Failed to establish a connection. + """ + + +class CloseError(NetworkError): + """ + Failed to close a connection. + """ + + +# Other transport exceptions... + + +class ProxyError(TransportError): + """ + An error occurred while establishing a proxy connection. + """ + + +class UnsupportedProtocol(TransportError): + """ + Attempted to make a request to an unsupported protocol. + + For example issuing a request to `ftp://www.example.com`. + """ + + +class ProtocolError(TransportError): + """ + The protocol was violated. + """ + + +class LocalProtocolError(ProtocolError): + """ + A protocol was violated by the client. + + For example if the user instantiated a `Request` instance explicitly, + failed to include the mandatory `Host:` header, and then issued it directly + using `client.send()`. + """ + + +class RemoteProtocolError(ProtocolError): + """ + The protocol was violated by the server. + + For example, returning malformed HTTP. + """ + + +# Other request exceptions... + + +class DecodingError(RequestError): + """ + Decoding of the response failed, due to a malformed encoding. + """ + + +class TooManyRedirects(RequestError): + """ + Too many redirects. + """ + + +# Client errors + + +class HTTPStatusError(HTTPError): + """ + The response had an error HTTP status of 4xx or 5xx. + + May be raised when calling `response.raise_for_status()` + """ + + def __init__(self, message: str, *, request: Request, response: Response) -> None: + super().__init__(message) + self.request = request + self.response = response + + +class InvalidURL(Exception): + """ + URL is improperly formed or cannot be parsed. + """ + + def __init__(self, message: str) -> None: + super().__init__(message) + + +class CookieConflict(Exception): + """ + Attempted to lookup a cookie by name, but multiple cookies existed. + + Can occur when calling `response.cookies.get(...)`. + """ + + def __init__(self, message: str) -> None: + super().__init__(message) + + +# Stream exceptions... + +# These may occur as the result of a programming error, by accessing +# the request/response stream in an invalid manner. + + +class StreamError(RuntimeError): + """ + The base class for stream exceptions. + + The developer made an error in accessing the request stream in + an invalid way. + """ + + def __init__(self, message: str) -> None: + super().__init__(message) + + +class StreamConsumed(StreamError): + """ + Attempted to read or stream content, but the content has already + been streamed. + """ + + def __init__(self) -> None: + message = ( + "Attempted to read or stream some content, but the content has " + "already been streamed. For requests, this could be due to passing " + "a generator as request content, and then receiving a redirect " + "response or a secondary request as part of an authentication flow." + "For responses, this could be due to attempting to stream the response " + "content more than once." + ) + super().__init__(message) + + +class StreamClosed(StreamError): + """ + Attempted to read or stream response content, but the request has been + closed. + """ + + def __init__(self) -> None: + message = ( + "Attempted to read or stream content, but the stream has " "been closed." + ) + super().__init__(message) + + +class ResponseNotRead(StreamError): + """ + Attempted to access streaming response content, without having called `read()`. + """ + + def __init__(self) -> None: + message = ( + "Attempted to access streaming response content," + " without having called `read()`." + ) + super().__init__(message) + + +class RequestNotRead(StreamError): + """ + Attempted to access streaming request content, without having called `read()`. + """ + + def __init__(self) -> None: + message = ( + "Attempted to access streaming request content," + " without having called `read()`." + ) + super().__init__(message) + + +@contextlib.contextmanager +def request_context( + request: Request | None = None, +) -> typing.Iterator[None]: + """ + A context manager that can be used to attach the given request context + to any `RequestError` exceptions that are raised within the block. + """ + try: + yield + except RequestError as exc: + if request is not None: + exc.request = request + raise exc diff --git a/venv/lib/python3.12/site-packages/httpx/_main.py b/venv/lib/python3.12/site-packages/httpx/_main.py new file mode 100644 index 0000000000000000000000000000000000000000..cffa4bb7db0f930f4db56653a061c4d7400ba4e6 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_main.py @@ -0,0 +1,506 @@ +from __future__ import annotations + +import functools +import json +import sys +import typing + +import click +import pygments.lexers +import pygments.util +import rich.console +import rich.markup +import rich.progress +import rich.syntax +import rich.table + +from ._client import Client +from ._exceptions import RequestError +from ._models import Response +from ._status_codes import codes + +if typing.TYPE_CHECKING: + import httpcore # pragma: no cover + + +def print_help() -> None: + console = rich.console.Console() + + console.print("[bold]HTTPX :butterfly:", justify="center") + console.print() + console.print("A next generation HTTP client.", justify="center") + console.print() + console.print( + "Usage: [bold]httpx[/bold] [cyan] [OPTIONS][/cyan] ", justify="left" + ) + console.print() + + table = rich.table.Table.grid(padding=1, pad_edge=True) + table.add_column("Parameter", no_wrap=True, justify="left", style="bold") + table.add_column("Description") + table.add_row( + "-m, --method [cyan]METHOD", + "Request method, such as GET, POST, PUT, PATCH, DELETE, OPTIONS, HEAD.\n" + "[Default: GET, or POST if a request body is included]", + ) + table.add_row( + "-p, --params [cyan] ...", + "Query parameters to include in the request URL.", + ) + table.add_row( + "-c, --content [cyan]TEXT", "Byte content to include in the request body." + ) + table.add_row( + "-d, --data [cyan] ...", "Form data to include in the request body." + ) + table.add_row( + "-f, --files [cyan] ...", + "Form files to include in the request body.", + ) + table.add_row("-j, --json [cyan]TEXT", "JSON data to include in the request body.") + table.add_row( + "-h, --headers [cyan] ...", + "Include additional HTTP headers in the request.", + ) + table.add_row( + "--cookies [cyan] ...", "Cookies to include in the request." + ) + table.add_row( + "--auth [cyan]", + "Username and password to include in the request. Specify '-' for the password" + " to use a password prompt. Note that using --verbose/-v will expose" + " the Authorization header, including the password encoding" + " in a trivially reversible format.", + ) + + table.add_row( + "--proxy [cyan]URL", + "Send the request via a proxy. Should be the URL giving the proxy address.", + ) + + table.add_row( + "--timeout [cyan]FLOAT", + "Timeout value to use for network operations, such as establishing the" + " connection, reading some data, etc... [Default: 5.0]", + ) + + table.add_row("--follow-redirects", "Automatically follow redirects.") + table.add_row("--no-verify", "Disable SSL verification.") + table.add_row( + "--http2", "Send the request using HTTP/2, if the remote server supports it." + ) + + table.add_row( + "--download [cyan]FILE", + "Save the response content as a file, rather than displaying it.", + ) + + table.add_row("-v, --verbose", "Verbose output. Show request as well as response.") + table.add_row("--help", "Show this message and exit.") + console.print(table) + + +def get_lexer_for_response(response: Response) -> str: + content_type = response.headers.get("Content-Type") + if content_type is not None: + mime_type, _, _ = content_type.partition(";") + try: + return typing.cast( + str, pygments.lexers.get_lexer_for_mimetype(mime_type.strip()).name + ) + except pygments.util.ClassNotFound: # pragma: no cover + pass + return "" # pragma: no cover + + +def format_request_headers(request: httpcore.Request, http2: bool = False) -> str: + version = "HTTP/2" if http2 else "HTTP/1.1" + headers = [ + (name.lower() if http2 else name, value) for name, value in request.headers + ] + method = request.method.decode("ascii") + target = request.url.target.decode("ascii") + lines = [f"{method} {target} {version}"] + [ + f"{name.decode('ascii')}: {value.decode('ascii')}" for name, value in headers + ] + return "\n".join(lines) + + +def format_response_headers( + http_version: bytes, + status: int, + reason_phrase: bytes | None, + headers: list[tuple[bytes, bytes]], +) -> str: + version = http_version.decode("ascii") + reason = ( + codes.get_reason_phrase(status) + if reason_phrase is None + else reason_phrase.decode("ascii") + ) + lines = [f"{version} {status} {reason}"] + [ + f"{name.decode('ascii')}: {value.decode('ascii')}" for name, value in headers + ] + return "\n".join(lines) + + +def print_request_headers(request: httpcore.Request, http2: bool = False) -> None: + console = rich.console.Console() + http_text = format_request_headers(request, http2=http2) + syntax = rich.syntax.Syntax(http_text, "http", theme="ansi_dark", word_wrap=True) + console.print(syntax) + syntax = rich.syntax.Syntax("", "http", theme="ansi_dark", word_wrap=True) + console.print(syntax) + + +def print_response_headers( + http_version: bytes, + status: int, + reason_phrase: bytes | None, + headers: list[tuple[bytes, bytes]], +) -> None: + console = rich.console.Console() + http_text = format_response_headers(http_version, status, reason_phrase, headers) + syntax = rich.syntax.Syntax(http_text, "http", theme="ansi_dark", word_wrap=True) + console.print(syntax) + syntax = rich.syntax.Syntax("", "http", theme="ansi_dark", word_wrap=True) + console.print(syntax) + + +def print_response(response: Response) -> None: + console = rich.console.Console() + lexer_name = get_lexer_for_response(response) + if lexer_name: + if lexer_name.lower() == "json": + try: + data = response.json() + text = json.dumps(data, indent=4) + except ValueError: # pragma: no cover + text = response.text + else: + text = response.text + + syntax = rich.syntax.Syntax(text, lexer_name, theme="ansi_dark", word_wrap=True) + console.print(syntax) + else: + console.print(f"<{len(response.content)} bytes of binary data>") + + +_PCTRTT = typing.Tuple[typing.Tuple[str, str], ...] +_PCTRTTT = typing.Tuple[_PCTRTT, ...] +_PeerCertRetDictType = typing.Dict[str, typing.Union[str, _PCTRTTT, _PCTRTT]] + + +def format_certificate(cert: _PeerCertRetDictType) -> str: # pragma: no cover + lines = [] + for key, value in cert.items(): + if isinstance(value, (list, tuple)): + lines.append(f"* {key}:") + for item in value: + if key in ("subject", "issuer"): + for sub_item in item: + lines.append(f"* {sub_item[0]}: {sub_item[1]!r}") + elif isinstance(item, tuple) and len(item) == 2: + lines.append(f"* {item[0]}: {item[1]!r}") + else: + lines.append(f"* {item!r}") + else: + lines.append(f"* {key}: {value!r}") + return "\n".join(lines) + + +def trace( + name: str, info: typing.Mapping[str, typing.Any], verbose: bool = False +) -> None: + console = rich.console.Console() + if name == "connection.connect_tcp.started" and verbose: + host = info["host"] + console.print(f"* Connecting to {host!r}") + elif name == "connection.connect_tcp.complete" and verbose: + stream = info["return_value"] + server_addr = stream.get_extra_info("server_addr") + console.print(f"* Connected to {server_addr[0]!r} on port {server_addr[1]}") + elif name == "connection.start_tls.complete" and verbose: # pragma: no cover + stream = info["return_value"] + ssl_object = stream.get_extra_info("ssl_object") + version = ssl_object.version() + cipher = ssl_object.cipher() + server_cert = ssl_object.getpeercert() + alpn = ssl_object.selected_alpn_protocol() + console.print(f"* SSL established using {version!r} / {cipher[0]!r}") + console.print(f"* Selected ALPN protocol: {alpn!r}") + if server_cert: + console.print("* Server certificate:") + console.print(format_certificate(server_cert)) + elif name == "http11.send_request_headers.started" and verbose: + request = info["request"] + print_request_headers(request, http2=False) + elif name == "http2.send_request_headers.started" and verbose: # pragma: no cover + request = info["request"] + print_request_headers(request, http2=True) + elif name == "http11.receive_response_headers.complete": + http_version, status, reason_phrase, headers = info["return_value"] + print_response_headers(http_version, status, reason_phrase, headers) + elif name == "http2.receive_response_headers.complete": # pragma: no cover + status, headers = info["return_value"] + http_version = b"HTTP/2" + reason_phrase = None + print_response_headers(http_version, status, reason_phrase, headers) + + +def download_response(response: Response, download: typing.BinaryIO) -> None: + console = rich.console.Console() + console.print() + content_length = response.headers.get("Content-Length") + with rich.progress.Progress( + "[progress.description]{task.description}", + "[progress.percentage]{task.percentage:>3.0f}%", + rich.progress.BarColumn(bar_width=None), + rich.progress.DownloadColumn(), + rich.progress.TransferSpeedColumn(), + ) as progress: + description = f"Downloading [bold]{rich.markup.escape(download.name)}" + download_task = progress.add_task( + description, + total=int(content_length or 0), + start=content_length is not None, + ) + for chunk in response.iter_bytes(): + download.write(chunk) + progress.update(download_task, completed=response.num_bytes_downloaded) + + +def validate_json( + ctx: click.Context, + param: click.Option | click.Parameter, + value: typing.Any, +) -> typing.Any: + if value is None: + return None + + try: + return json.loads(value) + except json.JSONDecodeError: # pragma: no cover + raise click.BadParameter("Not valid JSON") + + +def validate_auth( + ctx: click.Context, + param: click.Option | click.Parameter, + value: typing.Any, +) -> typing.Any: + if value == (None, None): + return None + + username, password = value + if password == "-": # pragma: no cover + password = click.prompt("Password", hide_input=True) + return (username, password) + + +def handle_help( + ctx: click.Context, + param: click.Option | click.Parameter, + value: typing.Any, +) -> None: + if not value or ctx.resilient_parsing: + return + + print_help() + ctx.exit() + + +@click.command(add_help_option=False) +@click.argument("url", type=str) +@click.option( + "--method", + "-m", + "method", + type=str, + help=( + "Request method, such as GET, POST, PUT, PATCH, DELETE, OPTIONS, HEAD. " + "[Default: GET, or POST if a request body is included]" + ), +) +@click.option( + "--params", + "-p", + "params", + type=(str, str), + multiple=True, + help="Query parameters to include in the request URL.", +) +@click.option( + "--content", + "-c", + "content", + type=str, + help="Byte content to include in the request body.", +) +@click.option( + "--data", + "-d", + "data", + type=(str, str), + multiple=True, + help="Form data to include in the request body.", +) +@click.option( + "--files", + "-f", + "files", + type=(str, click.File(mode="rb")), + multiple=True, + help="Form files to include in the request body.", +) +@click.option( + "--json", + "-j", + "json", + type=str, + callback=validate_json, + help="JSON data to include in the request body.", +) +@click.option( + "--headers", + "-h", + "headers", + type=(str, str), + multiple=True, + help="Include additional HTTP headers in the request.", +) +@click.option( + "--cookies", + "cookies", + type=(str, str), + multiple=True, + help="Cookies to include in the request.", +) +@click.option( + "--auth", + "auth", + type=(str, str), + default=(None, None), + callback=validate_auth, + help=( + "Username and password to include in the request. " + "Specify '-' for the password to use a password prompt. " + "Note that using --verbose/-v will expose the Authorization header, " + "including the password encoding in a trivially reversible format." + ), +) +@click.option( + "--proxy", + "proxy", + type=str, + default=None, + help="Send the request via a proxy. Should be the URL giving the proxy address.", +) +@click.option( + "--timeout", + "timeout", + type=float, + default=5.0, + help=( + "Timeout value to use for network operations, such as establishing the " + "connection, reading some data, etc... [Default: 5.0]" + ), +) +@click.option( + "--follow-redirects", + "follow_redirects", + is_flag=True, + default=False, + help="Automatically follow redirects.", +) +@click.option( + "--no-verify", + "verify", + is_flag=True, + default=True, + help="Disable SSL verification.", +) +@click.option( + "--http2", + "http2", + type=bool, + is_flag=True, + default=False, + help="Send the request using HTTP/2, if the remote server supports it.", +) +@click.option( + "--download", + type=click.File("wb"), + help="Save the response content as a file, rather than displaying it.", +) +@click.option( + "--verbose", + "-v", + type=bool, + is_flag=True, + default=False, + help="Verbose. Show request as well as response.", +) +@click.option( + "--help", + is_flag=True, + is_eager=True, + expose_value=False, + callback=handle_help, + help="Show this message and exit.", +) +def main( + url: str, + method: str, + params: list[tuple[str, str]], + content: str, + data: list[tuple[str, str]], + files: list[tuple[str, click.File]], + json: str, + headers: list[tuple[str, str]], + cookies: list[tuple[str, str]], + auth: tuple[str, str] | None, + proxy: str, + timeout: float, + follow_redirects: bool, + verify: bool, + http2: bool, + download: typing.BinaryIO | None, + verbose: bool, +) -> None: + """ + An HTTP command line client. + Sends a request and displays the response. + """ + if not method: + method = "POST" if content or data or files or json else "GET" + + try: + with Client(proxy=proxy, timeout=timeout, http2=http2, verify=verify) as client: + with client.stream( + method, + url, + params=list(params), + content=content, + data=dict(data), + files=files, # type: ignore + json=json, + headers=headers, + cookies=dict(cookies), + auth=auth, + follow_redirects=follow_redirects, + extensions={"trace": functools.partial(trace, verbose=verbose)}, + ) as response: + if download is not None: + download_response(response, download) + else: + response.read() + if response.content: + print_response(response) + + except RequestError as exc: + console = rich.console.Console() + console.print(f"[red]{type(exc).__name__}[/red]: {exc}") + sys.exit(1) + + sys.exit(0 if response.is_success else 1) diff --git a/venv/lib/python3.12/site-packages/httpx/_models.py b/venv/lib/python3.12/site-packages/httpx/_models.py new file mode 100644 index 0000000000000000000000000000000000000000..67d74bf86bfc80e22d9a4a3153572845accd9039 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_models.py @@ -0,0 +1,1277 @@ +from __future__ import annotations + +import codecs +import datetime +import email.message +import json as jsonlib +import re +import typing +import urllib.request +from collections.abc import Mapping +from http.cookiejar import Cookie, CookieJar + +from ._content import ByteStream, UnattachedStream, encode_request, encode_response +from ._decoders import ( + SUPPORTED_DECODERS, + ByteChunker, + ContentDecoder, + IdentityDecoder, + LineDecoder, + MultiDecoder, + TextChunker, + TextDecoder, +) +from ._exceptions import ( + CookieConflict, + HTTPStatusError, + RequestNotRead, + ResponseNotRead, + StreamClosed, + StreamConsumed, + request_context, +) +from ._multipart import get_multipart_boundary_from_content_type +from ._status_codes import codes +from ._types import ( + AsyncByteStream, + CookieTypes, + HeaderTypes, + QueryParamTypes, + RequestContent, + RequestData, + RequestExtensions, + RequestFiles, + ResponseContent, + ResponseExtensions, + SyncByteStream, +) +from ._urls import URL +from ._utils import to_bytes_or_str, to_str + +__all__ = ["Cookies", "Headers", "Request", "Response"] + +SENSITIVE_HEADERS = {"authorization", "proxy-authorization"} + + +def _is_known_encoding(encoding: str) -> bool: + """ + Return `True` if `encoding` is a known codec. + """ + try: + codecs.lookup(encoding) + except LookupError: + return False + return True + + +def _normalize_header_key(key: str | bytes, encoding: str | None = None) -> bytes: + """ + Coerce str/bytes into a strictly byte-wise HTTP header key. + """ + return key if isinstance(key, bytes) else key.encode(encoding or "ascii") + + +def _normalize_header_value(value: str | bytes, encoding: str | None = None) -> bytes: + """ + Coerce str/bytes into a strictly byte-wise HTTP header value. + """ + if isinstance(value, bytes): + return value + if not isinstance(value, str): + raise TypeError(f"Header value must be str or bytes, not {type(value)}") + return value.encode(encoding or "ascii") + + +def _parse_content_type_charset(content_type: str) -> str | None: + # We used to use `cgi.parse_header()` here, but `cgi` became a dead battery. + # See: https://peps.python.org/pep-0594/#cgi + msg = email.message.Message() + msg["content-type"] = content_type + return msg.get_content_charset(failobj=None) + + +def _parse_header_links(value: str) -> list[dict[str, str]]: + """ + Returns a list of parsed link headers, for more info see: + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link + The generic syntax of those is: + Link: < uri-reference >; param1=value1; param2="value2" + So for instance: + Link; '; type="image/jpeg",;' + would return + [ + {"url": "http:/.../front.jpeg", "type": "image/jpeg"}, + {"url": "http://.../back.jpeg"}, + ] + :param value: HTTP Link entity-header field + :return: list of parsed link headers + """ + links: list[dict[str, str]] = [] + replace_chars = " '\"" + value = value.strip(replace_chars) + if not value: + return links + for val in re.split(", *<", value): + try: + url, params = val.split(";", 1) + except ValueError: + url, params = val, "" + link = {"url": url.strip("<> '\"")} + for param in params.split(";"): + try: + key, value = param.split("=") + except ValueError: + break + link[key.strip(replace_chars)] = value.strip(replace_chars) + links.append(link) + return links + + +def _obfuscate_sensitive_headers( + items: typing.Iterable[tuple[typing.AnyStr, typing.AnyStr]], +) -> typing.Iterator[tuple[typing.AnyStr, typing.AnyStr]]: + for k, v in items: + if to_str(k.lower()) in SENSITIVE_HEADERS: + v = to_bytes_or_str("[secure]", match_type_of=v) + yield k, v + + +class Headers(typing.MutableMapping[str, str]): + """ + HTTP headers, as a case-insensitive multi-dict. + """ + + def __init__( + self, + headers: HeaderTypes | None = None, + encoding: str | None = None, + ) -> None: + self._list = [] # type: typing.List[typing.Tuple[bytes, bytes, bytes]] + + if isinstance(headers, Headers): + self._list = list(headers._list) + elif isinstance(headers, Mapping): + for k, v in headers.items(): + bytes_key = _normalize_header_key(k, encoding) + bytes_value = _normalize_header_value(v, encoding) + self._list.append((bytes_key, bytes_key.lower(), bytes_value)) + elif headers is not None: + for k, v in headers: + bytes_key = _normalize_header_key(k, encoding) + bytes_value = _normalize_header_value(v, encoding) + self._list.append((bytes_key, bytes_key.lower(), bytes_value)) + + self._encoding = encoding + + @property + def encoding(self) -> str: + """ + Header encoding is mandated as ascii, but we allow fallbacks to utf-8 + or iso-8859-1. + """ + if self._encoding is None: + for encoding in ["ascii", "utf-8"]: + for key, value in self.raw: + try: + key.decode(encoding) + value.decode(encoding) + except UnicodeDecodeError: + break + else: + # The else block runs if 'break' did not occur, meaning + # all values fitted the encoding. + self._encoding = encoding + break + else: + # The ISO-8859-1 encoding covers all 256 code points in a byte, + # so will never raise decode errors. + self._encoding = "iso-8859-1" + return self._encoding + + @encoding.setter + def encoding(self, value: str) -> None: + self._encoding = value + + @property + def raw(self) -> list[tuple[bytes, bytes]]: + """ + Returns a list of the raw header items, as byte pairs. + """ + return [(raw_key, value) for raw_key, _, value in self._list] + + def keys(self) -> typing.KeysView[str]: + return {key.decode(self.encoding): None for _, key, value in self._list}.keys() + + def values(self) -> typing.ValuesView[str]: + values_dict: dict[str, str] = {} + for _, key, value in self._list: + str_key = key.decode(self.encoding) + str_value = value.decode(self.encoding) + if str_key in values_dict: + values_dict[str_key] += f", {str_value}" + else: + values_dict[str_key] = str_value + return values_dict.values() + + def items(self) -> typing.ItemsView[str, str]: + """ + Return `(key, value)` items of headers. Concatenate headers + into a single comma separated value when a key occurs multiple times. + """ + values_dict: dict[str, str] = {} + for _, key, value in self._list: + str_key = key.decode(self.encoding) + str_value = value.decode(self.encoding) + if str_key in values_dict: + values_dict[str_key] += f", {str_value}" + else: + values_dict[str_key] = str_value + return values_dict.items() + + def multi_items(self) -> list[tuple[str, str]]: + """ + Return a list of `(key, value)` pairs of headers. Allow multiple + occurrences of the same key without concatenating into a single + comma separated value. + """ + return [ + (key.decode(self.encoding), value.decode(self.encoding)) + for _, key, value in self._list + ] + + def get(self, key: str, default: typing.Any = None) -> typing.Any: + """ + Return a header value. If multiple occurrences of the header occur + then concatenate them together with commas. + """ + try: + return self[key] + except KeyError: + return default + + def get_list(self, key: str, split_commas: bool = False) -> list[str]: + """ + Return a list of all header values for a given key. + If `split_commas=True` is passed, then any comma separated header + values are split into multiple return strings. + """ + get_header_key = key.lower().encode(self.encoding) + + values = [ + item_value.decode(self.encoding) + for _, item_key, item_value in self._list + if item_key.lower() == get_header_key + ] + + if not split_commas: + return values + + split_values = [] + for value in values: + split_values.extend([item.strip() for item in value.split(",")]) + return split_values + + def update(self, headers: HeaderTypes | None = None) -> None: # type: ignore + headers = Headers(headers) + for key in headers.keys(): + if key in self: + self.pop(key) + self._list.extend(headers._list) + + def copy(self) -> Headers: + return Headers(self, encoding=self.encoding) + + def __getitem__(self, key: str) -> str: + """ + Return a single header value. + + If there are multiple headers with the same key, then we concatenate + them with commas. See: https://tools.ietf.org/html/rfc7230#section-3.2.2 + """ + normalized_key = key.lower().encode(self.encoding) + + items = [ + header_value.decode(self.encoding) + for _, header_key, header_value in self._list + if header_key == normalized_key + ] + + if items: + return ", ".join(items) + + raise KeyError(key) + + def __setitem__(self, key: str, value: str) -> None: + """ + Set the header `key` to `value`, removing any duplicate entries. + Retains insertion order. + """ + set_key = key.encode(self._encoding or "utf-8") + set_value = value.encode(self._encoding or "utf-8") + lookup_key = set_key.lower() + + found_indexes = [ + idx + for idx, (_, item_key, _) in enumerate(self._list) + if item_key == lookup_key + ] + + for idx in reversed(found_indexes[1:]): + del self._list[idx] + + if found_indexes: + idx = found_indexes[0] + self._list[idx] = (set_key, lookup_key, set_value) + else: + self._list.append((set_key, lookup_key, set_value)) + + def __delitem__(self, key: str) -> None: + """ + Remove the header `key`. + """ + del_key = key.lower().encode(self.encoding) + + pop_indexes = [ + idx + for idx, (_, item_key, _) in enumerate(self._list) + if item_key.lower() == del_key + ] + + if not pop_indexes: + raise KeyError(key) + + for idx in reversed(pop_indexes): + del self._list[idx] + + def __contains__(self, key: typing.Any) -> bool: + header_key = key.lower().encode(self.encoding) + return header_key in [key for _, key, _ in self._list] + + def __iter__(self) -> typing.Iterator[typing.Any]: + return iter(self.keys()) + + def __len__(self) -> int: + return len(self._list) + + def __eq__(self, other: typing.Any) -> bool: + try: + other_headers = Headers(other) + except ValueError: + return False + + self_list = [(key, value) for _, key, value in self._list] + other_list = [(key, value) for _, key, value in other_headers._list] + return sorted(self_list) == sorted(other_list) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + + encoding_str = "" + if self.encoding != "ascii": + encoding_str = f", encoding={self.encoding!r}" + + as_list = list(_obfuscate_sensitive_headers(self.multi_items())) + as_dict = dict(as_list) + + no_duplicate_keys = len(as_dict) == len(as_list) + if no_duplicate_keys: + return f"{class_name}({as_dict!r}{encoding_str})" + return f"{class_name}({as_list!r}{encoding_str})" + + +class Request: + def __init__( + self, + method: str, + url: URL | str, + *, + params: QueryParamTypes | None = None, + headers: HeaderTypes | None = None, + cookies: CookieTypes | None = None, + content: RequestContent | None = None, + data: RequestData | None = None, + files: RequestFiles | None = None, + json: typing.Any | None = None, + stream: SyncByteStream | AsyncByteStream | None = None, + extensions: RequestExtensions | None = None, + ) -> None: + self.method = method.upper() + self.url = URL(url) if params is None else URL(url, params=params) + self.headers = Headers(headers) + self.extensions = {} if extensions is None else dict(extensions) + + if cookies: + Cookies(cookies).set_cookie_header(self) + + if stream is None: + content_type: str | None = self.headers.get("content-type") + headers, stream = encode_request( + content=content, + data=data, + files=files, + json=json, + boundary=get_multipart_boundary_from_content_type( + content_type=content_type.encode(self.headers.encoding) + if content_type + else None + ), + ) + self._prepare(headers) + self.stream = stream + # Load the request body, except for streaming content. + if isinstance(stream, ByteStream): + self.read() + else: + # There's an important distinction between `Request(content=...)`, + # and `Request(stream=...)`. + # + # Using `content=...` implies automatically populated `Host` and content + # headers, of either `Content-Length: ...` or `Transfer-Encoding: chunked`. + # + # Using `stream=...` will not automatically include *any* + # auto-populated headers. + # + # As an end-user you don't really need `stream=...`. It's only + # useful when: + # + # * Preserving the request stream when copying requests, eg for redirects. + # * Creating request instances on the *server-side* of the transport API. + self.stream = stream + + def _prepare(self, default_headers: dict[str, str]) -> None: + for key, value in default_headers.items(): + # Ignore Transfer-Encoding if the Content-Length has been set explicitly. + if key.lower() == "transfer-encoding" and "Content-Length" in self.headers: + continue + self.headers.setdefault(key, value) + + auto_headers: list[tuple[bytes, bytes]] = [] + + has_host = "Host" in self.headers + has_content_length = ( + "Content-Length" in self.headers or "Transfer-Encoding" in self.headers + ) + + if not has_host and self.url.host: + auto_headers.append((b"Host", self.url.netloc)) + if not has_content_length and self.method in ("POST", "PUT", "PATCH"): + auto_headers.append((b"Content-Length", b"0")) + + self.headers = Headers(auto_headers + self.headers.raw) + + @property + def content(self) -> bytes: + if not hasattr(self, "_content"): + raise RequestNotRead() + return self._content + + def read(self) -> bytes: + """ + Read and return the request content. + """ + if not hasattr(self, "_content"): + assert isinstance(self.stream, typing.Iterable) + self._content = b"".join(self.stream) + if not isinstance(self.stream, ByteStream): + # If a streaming request has been read entirely into memory, then + # we can replace the stream with a raw bytes implementation, + # to ensure that any non-replayable streams can still be used. + self.stream = ByteStream(self._content) + return self._content + + async def aread(self) -> bytes: + """ + Read and return the request content. + """ + if not hasattr(self, "_content"): + assert isinstance(self.stream, typing.AsyncIterable) + self._content = b"".join([part async for part in self.stream]) + if not isinstance(self.stream, ByteStream): + # If a streaming request has been read entirely into memory, then + # we can replace the stream with a raw bytes implementation, + # to ensure that any non-replayable streams can still be used. + self.stream = ByteStream(self._content) + return self._content + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + url = str(self.url) + return f"<{class_name}({self.method!r}, {url!r})>" + + def __getstate__(self) -> dict[str, typing.Any]: + return { + name: value + for name, value in self.__dict__.items() + if name not in ["extensions", "stream"] + } + + def __setstate__(self, state: dict[str, typing.Any]) -> None: + for name, value in state.items(): + setattr(self, name, value) + self.extensions = {} + self.stream = UnattachedStream() + + +class Response: + def __init__( + self, + status_code: int, + *, + headers: HeaderTypes | None = None, + content: ResponseContent | None = None, + text: str | None = None, + html: str | None = None, + json: typing.Any = None, + stream: SyncByteStream | AsyncByteStream | None = None, + request: Request | None = None, + extensions: ResponseExtensions | None = None, + history: list[Response] | None = None, + default_encoding: str | typing.Callable[[bytes], str] = "utf-8", + ) -> None: + self.status_code = status_code + self.headers = Headers(headers) + + self._request: Request | None = request + + # When follow_redirects=False and a redirect is received, + # the client will set `response.next_request`. + self.next_request: Request | None = None + + self.extensions = {} if extensions is None else dict(extensions) + self.history = [] if history is None else list(history) + + self.is_closed = False + self.is_stream_consumed = False + + self.default_encoding = default_encoding + + if stream is None: + headers, stream = encode_response(content, text, html, json) + self._prepare(headers) + self.stream = stream + if isinstance(stream, ByteStream): + # Load the response body, except for streaming content. + self.read() + else: + # There's an important distinction between `Response(content=...)`, + # and `Response(stream=...)`. + # + # Using `content=...` implies automatically populated content headers, + # of either `Content-Length: ...` or `Transfer-Encoding: chunked`. + # + # Using `stream=...` will not automatically include any content headers. + # + # As an end-user you don't really need `stream=...`. It's only + # useful when creating response instances having received a stream + # from the transport API. + self.stream = stream + + self._num_bytes_downloaded = 0 + + def _prepare(self, default_headers: dict[str, str]) -> None: + for key, value in default_headers.items(): + # Ignore Transfer-Encoding if the Content-Length has been set explicitly. + if key.lower() == "transfer-encoding" and "content-length" in self.headers: + continue + self.headers.setdefault(key, value) + + @property + def elapsed(self) -> datetime.timedelta: + """ + Returns the time taken for the complete request/response + cycle to complete. + """ + if not hasattr(self, "_elapsed"): + raise RuntimeError( + "'.elapsed' may only be accessed after the response " + "has been read or closed." + ) + return self._elapsed + + @elapsed.setter + def elapsed(self, elapsed: datetime.timedelta) -> None: + self._elapsed = elapsed + + @property + def request(self) -> Request: + """ + Returns the request instance associated to the current response. + """ + if self._request is None: + raise RuntimeError( + "The request instance has not been set on this response." + ) + return self._request + + @request.setter + def request(self, value: Request) -> None: + self._request = value + + @property + def http_version(self) -> str: + try: + http_version: bytes = self.extensions["http_version"] + except KeyError: + return "HTTP/1.1" + else: + return http_version.decode("ascii", errors="ignore") + + @property + def reason_phrase(self) -> str: + try: + reason_phrase: bytes = self.extensions["reason_phrase"] + except KeyError: + return codes.get_reason_phrase(self.status_code) + else: + return reason_phrase.decode("ascii", errors="ignore") + + @property + def url(self) -> URL: + """ + Returns the URL for which the request was made. + """ + return self.request.url + + @property + def content(self) -> bytes: + if not hasattr(self, "_content"): + raise ResponseNotRead() + return self._content + + @property + def text(self) -> str: + if not hasattr(self, "_text"): + content = self.content + if not content: + self._text = "" + else: + decoder = TextDecoder(encoding=self.encoding or "utf-8") + self._text = "".join([decoder.decode(self.content), decoder.flush()]) + return self._text + + @property + def encoding(self) -> str | None: + """ + Return an encoding to use for decoding the byte content into text. + The priority for determining this is given by... + + * `.encoding = <>` has been set explicitly. + * The encoding as specified by the charset parameter in the Content-Type header. + * The encoding as determined by `default_encoding`, which may either be + a string like "utf-8" indicating the encoding to use, or may be a callable + which enables charset autodetection. + """ + if not hasattr(self, "_encoding"): + encoding = self.charset_encoding + if encoding is None or not _is_known_encoding(encoding): + if isinstance(self.default_encoding, str): + encoding = self.default_encoding + elif hasattr(self, "_content"): + encoding = self.default_encoding(self._content) + self._encoding = encoding or "utf-8" + return self._encoding + + @encoding.setter + def encoding(self, value: str) -> None: + """ + Set the encoding to use for decoding the byte content into text. + + If the `text` attribute has been accessed, attempting to set the + encoding will throw a ValueError. + """ + if hasattr(self, "_text"): + raise ValueError( + "Setting encoding after `text` has been accessed is not allowed." + ) + self._encoding = value + + @property + def charset_encoding(self) -> str | None: + """ + Return the encoding, as specified by the Content-Type header. + """ + content_type = self.headers.get("Content-Type") + if content_type is None: + return None + + return _parse_content_type_charset(content_type) + + def _get_content_decoder(self) -> ContentDecoder: + """ + Returns a decoder instance which can be used to decode the raw byte + content, depending on the Content-Encoding used in the response. + """ + if not hasattr(self, "_decoder"): + decoders: list[ContentDecoder] = [] + values = self.headers.get_list("content-encoding", split_commas=True) + for value in values: + value = value.strip().lower() + try: + decoder_cls = SUPPORTED_DECODERS[value] + decoders.append(decoder_cls()) + except KeyError: + continue + + if len(decoders) == 1: + self._decoder = decoders[0] + elif len(decoders) > 1: + self._decoder = MultiDecoder(children=decoders) + else: + self._decoder = IdentityDecoder() + + return self._decoder + + @property + def is_informational(self) -> bool: + """ + A property which is `True` for 1xx status codes, `False` otherwise. + """ + return codes.is_informational(self.status_code) + + @property + def is_success(self) -> bool: + """ + A property which is `True` for 2xx status codes, `False` otherwise. + """ + return codes.is_success(self.status_code) + + @property + def is_redirect(self) -> bool: + """ + A property which is `True` for 3xx status codes, `False` otherwise. + + Note that not all responses with a 3xx status code indicate a URL redirect. + + Use `response.has_redirect_location` to determine responses with a properly + formed URL redirection. + """ + return codes.is_redirect(self.status_code) + + @property + def is_client_error(self) -> bool: + """ + A property which is `True` for 4xx status codes, `False` otherwise. + """ + return codes.is_client_error(self.status_code) + + @property + def is_server_error(self) -> bool: + """ + A property which is `True` for 5xx status codes, `False` otherwise. + """ + return codes.is_server_error(self.status_code) + + @property + def is_error(self) -> bool: + """ + A property which is `True` for 4xx and 5xx status codes, `False` otherwise. + """ + return codes.is_error(self.status_code) + + @property + def has_redirect_location(self) -> bool: + """ + Returns True for 3xx responses with a properly formed URL redirection, + `False` otherwise. + """ + return ( + self.status_code + in ( + # 301 (Cacheable redirect. Method may change to GET.) + codes.MOVED_PERMANENTLY, + # 302 (Uncacheable redirect. Method may change to GET.) + codes.FOUND, + # 303 (Client should make a GET or HEAD request.) + codes.SEE_OTHER, + # 307 (Equiv. 302, but retain method) + codes.TEMPORARY_REDIRECT, + # 308 (Equiv. 301, but retain method) + codes.PERMANENT_REDIRECT, + ) + and "Location" in self.headers + ) + + def raise_for_status(self) -> Response: + """ + Raise the `HTTPStatusError` if one occurred. + """ + request = self._request + if request is None: + raise RuntimeError( + "Cannot call `raise_for_status` as the request " + "instance has not been set on this response." + ) + + if self.is_success: + return self + + if self.has_redirect_location: + message = ( + "{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n" + "Redirect location: '{0.headers[location]}'\n" + "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/{0.status_code}" + ) + else: + message = ( + "{error_type} '{0.status_code} {0.reason_phrase}' for url '{0.url}'\n" + "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/{0.status_code}" + ) + + status_class = self.status_code // 100 + error_types = { + 1: "Informational response", + 3: "Redirect response", + 4: "Client error", + 5: "Server error", + } + error_type = error_types.get(status_class, "Invalid status code") + message = message.format(self, error_type=error_type) + raise HTTPStatusError(message, request=request, response=self) + + def json(self, **kwargs: typing.Any) -> typing.Any: + return jsonlib.loads(self.content, **kwargs) + + @property + def cookies(self) -> Cookies: + if not hasattr(self, "_cookies"): + self._cookies = Cookies() + self._cookies.extract_cookies(self) + return self._cookies + + @property + def links(self) -> dict[str | None, dict[str, str]]: + """ + Returns the parsed header links of the response, if any + """ + header = self.headers.get("link") + if header is None: + return {} + + return { + (link.get("rel") or link.get("url")): link + for link in _parse_header_links(header) + } + + @property + def num_bytes_downloaded(self) -> int: + return self._num_bytes_downloaded + + def __repr__(self) -> str: + return f"" + + def __getstate__(self) -> dict[str, typing.Any]: + return { + name: value + for name, value in self.__dict__.items() + if name not in ["extensions", "stream", "is_closed", "_decoder"] + } + + def __setstate__(self, state: dict[str, typing.Any]) -> None: + for name, value in state.items(): + setattr(self, name, value) + self.is_closed = True + self.extensions = {} + self.stream = UnattachedStream() + + def read(self) -> bytes: + """ + Read and return the response content. + """ + if not hasattr(self, "_content"): + self._content = b"".join(self.iter_bytes()) + return self._content + + def iter_bytes(self, chunk_size: int | None = None) -> typing.Iterator[bytes]: + """ + A byte-iterator over the decoded response content. + This allows us to handle gzip, deflate, brotli, and zstd encoded responses. + """ + if hasattr(self, "_content"): + chunk_size = len(self._content) if chunk_size is None else chunk_size + for i in range(0, len(self._content), max(chunk_size, 1)): + yield self._content[i : i + chunk_size] + else: + decoder = self._get_content_decoder() + chunker = ByteChunker(chunk_size=chunk_size) + with request_context(request=self._request): + for raw_bytes in self.iter_raw(): + decoded = decoder.decode(raw_bytes) + for chunk in chunker.decode(decoded): + yield chunk + decoded = decoder.flush() + for chunk in chunker.decode(decoded): + yield chunk # pragma: no cover + for chunk in chunker.flush(): + yield chunk + + def iter_text(self, chunk_size: int | None = None) -> typing.Iterator[str]: + """ + A str-iterator over the decoded response content + that handles both gzip, deflate, etc but also detects the content's + string encoding. + """ + decoder = TextDecoder(encoding=self.encoding or "utf-8") + chunker = TextChunker(chunk_size=chunk_size) + with request_context(request=self._request): + for byte_content in self.iter_bytes(): + text_content = decoder.decode(byte_content) + for chunk in chunker.decode(text_content): + yield chunk + text_content = decoder.flush() + for chunk in chunker.decode(text_content): + yield chunk # pragma: no cover + for chunk in chunker.flush(): + yield chunk + + def iter_lines(self) -> typing.Iterator[str]: + decoder = LineDecoder() + with request_context(request=self._request): + for text in self.iter_text(): + for line in decoder.decode(text): + yield line + for line in decoder.flush(): + yield line + + def iter_raw(self, chunk_size: int | None = None) -> typing.Iterator[bytes]: + """ + A byte-iterator over the raw response content. + """ + if self.is_stream_consumed: + raise StreamConsumed() + if self.is_closed: + raise StreamClosed() + if not isinstance(self.stream, SyncByteStream): + raise RuntimeError("Attempted to call a sync iterator on an async stream.") + + self.is_stream_consumed = True + self._num_bytes_downloaded = 0 + chunker = ByteChunker(chunk_size=chunk_size) + + with request_context(request=self._request): + for raw_stream_bytes in self.stream: + self._num_bytes_downloaded += len(raw_stream_bytes) + for chunk in chunker.decode(raw_stream_bytes): + yield chunk + + for chunk in chunker.flush(): + yield chunk + + self.close() + + def close(self) -> None: + """ + Close the response and release the connection. + Automatically called if the response body is read to completion. + """ + if not isinstance(self.stream, SyncByteStream): + raise RuntimeError("Attempted to call an sync close on an async stream.") + + if not self.is_closed: + self.is_closed = True + with request_context(request=self._request): + self.stream.close() + + async def aread(self) -> bytes: + """ + Read and return the response content. + """ + if not hasattr(self, "_content"): + self._content = b"".join([part async for part in self.aiter_bytes()]) + return self._content + + async def aiter_bytes( + self, chunk_size: int | None = None + ) -> typing.AsyncIterator[bytes]: + """ + A byte-iterator over the decoded response content. + This allows us to handle gzip, deflate, brotli, and zstd encoded responses. + """ + if hasattr(self, "_content"): + chunk_size = len(self._content) if chunk_size is None else chunk_size + for i in range(0, len(self._content), max(chunk_size, 1)): + yield self._content[i : i + chunk_size] + else: + decoder = self._get_content_decoder() + chunker = ByteChunker(chunk_size=chunk_size) + with request_context(request=self._request): + async for raw_bytes in self.aiter_raw(): + decoded = decoder.decode(raw_bytes) + for chunk in chunker.decode(decoded): + yield chunk + decoded = decoder.flush() + for chunk in chunker.decode(decoded): + yield chunk # pragma: no cover + for chunk in chunker.flush(): + yield chunk + + async def aiter_text( + self, chunk_size: int | None = None + ) -> typing.AsyncIterator[str]: + """ + A str-iterator over the decoded response content + that handles both gzip, deflate, etc but also detects the content's + string encoding. + """ + decoder = TextDecoder(encoding=self.encoding or "utf-8") + chunker = TextChunker(chunk_size=chunk_size) + with request_context(request=self._request): + async for byte_content in self.aiter_bytes(): + text_content = decoder.decode(byte_content) + for chunk in chunker.decode(text_content): + yield chunk + text_content = decoder.flush() + for chunk in chunker.decode(text_content): + yield chunk # pragma: no cover + for chunk in chunker.flush(): + yield chunk + + async def aiter_lines(self) -> typing.AsyncIterator[str]: + decoder = LineDecoder() + with request_context(request=self._request): + async for text in self.aiter_text(): + for line in decoder.decode(text): + yield line + for line in decoder.flush(): + yield line + + async def aiter_raw( + self, chunk_size: int | None = None + ) -> typing.AsyncIterator[bytes]: + """ + A byte-iterator over the raw response content. + """ + if self.is_stream_consumed: + raise StreamConsumed() + if self.is_closed: + raise StreamClosed() + if not isinstance(self.stream, AsyncByteStream): + raise RuntimeError("Attempted to call an async iterator on an sync stream.") + + self.is_stream_consumed = True + self._num_bytes_downloaded = 0 + chunker = ByteChunker(chunk_size=chunk_size) + + with request_context(request=self._request): + async for raw_stream_bytes in self.stream: + self._num_bytes_downloaded += len(raw_stream_bytes) + for chunk in chunker.decode(raw_stream_bytes): + yield chunk + + for chunk in chunker.flush(): + yield chunk + + await self.aclose() + + async def aclose(self) -> None: + """ + Close the response and release the connection. + Automatically called if the response body is read to completion. + """ + if not isinstance(self.stream, AsyncByteStream): + raise RuntimeError("Attempted to call an async close on an sync stream.") + + if not self.is_closed: + self.is_closed = True + with request_context(request=self._request): + await self.stream.aclose() + + +class Cookies(typing.MutableMapping[str, str]): + """ + HTTP Cookies, as a mutable mapping. + """ + + def __init__(self, cookies: CookieTypes | None = None) -> None: + if cookies is None or isinstance(cookies, dict): + self.jar = CookieJar() + if isinstance(cookies, dict): + for key, value in cookies.items(): + self.set(key, value) + elif isinstance(cookies, list): + self.jar = CookieJar() + for key, value in cookies: + self.set(key, value) + elif isinstance(cookies, Cookies): + self.jar = CookieJar() + for cookie in cookies.jar: + self.jar.set_cookie(cookie) + else: + self.jar = cookies + + def extract_cookies(self, response: Response) -> None: + """ + Loads any cookies based on the response `Set-Cookie` headers. + """ + urllib_response = self._CookieCompatResponse(response) + urllib_request = self._CookieCompatRequest(response.request) + + self.jar.extract_cookies(urllib_response, urllib_request) # type: ignore + + def set_cookie_header(self, request: Request) -> None: + """ + Sets an appropriate 'Cookie:' HTTP header on the `Request`. + """ + urllib_request = self._CookieCompatRequest(request) + self.jar.add_cookie_header(urllib_request) + + def set(self, name: str, value: str, domain: str = "", path: str = "/") -> None: + """ + Set a cookie value by name. May optionally include domain and path. + """ + kwargs = { + "version": 0, + "name": name, + "value": value, + "port": None, + "port_specified": False, + "domain": domain, + "domain_specified": bool(domain), + "domain_initial_dot": domain.startswith("."), + "path": path, + "path_specified": bool(path), + "secure": False, + "expires": None, + "discard": True, + "comment": None, + "comment_url": None, + "rest": {"HttpOnly": None}, + "rfc2109": False, + } + cookie = Cookie(**kwargs) # type: ignore + self.jar.set_cookie(cookie) + + def get( # type: ignore + self, + name: str, + default: str | None = None, + domain: str | None = None, + path: str | None = None, + ) -> str | None: + """ + Get a cookie by name. May optionally include domain and path + in order to specify exactly which cookie to retrieve. + """ + value = None + for cookie in self.jar: + if cookie.name == name: + if domain is None or cookie.domain == domain: + if path is None or cookie.path == path: + if value is not None: + message = f"Multiple cookies exist with name={name}" + raise CookieConflict(message) + value = cookie.value + + if value is None: + return default + return value + + def delete( + self, + name: str, + domain: str | None = None, + path: str | None = None, + ) -> None: + """ + Delete a cookie by name. May optionally include domain and path + in order to specify exactly which cookie to delete. + """ + if domain is not None and path is not None: + return self.jar.clear(domain, path, name) + + remove = [ + cookie + for cookie in self.jar + if cookie.name == name + and (domain is None or cookie.domain == domain) + and (path is None or cookie.path == path) + ] + + for cookie in remove: + self.jar.clear(cookie.domain, cookie.path, cookie.name) + + def clear(self, domain: str | None = None, path: str | None = None) -> None: + """ + Delete all cookies. Optionally include a domain and path in + order to only delete a subset of all the cookies. + """ + args = [] + if domain is not None: + args.append(domain) + if path is not None: + assert domain is not None + args.append(path) + self.jar.clear(*args) + + def update(self, cookies: CookieTypes | None = None) -> None: # type: ignore + cookies = Cookies(cookies) + for cookie in cookies.jar: + self.jar.set_cookie(cookie) + + def __setitem__(self, name: str, value: str) -> None: + return self.set(name, value) + + def __getitem__(self, name: str) -> str: + value = self.get(name) + if value is None: + raise KeyError(name) + return value + + def __delitem__(self, name: str) -> None: + return self.delete(name) + + def __len__(self) -> int: + return len(self.jar) + + def __iter__(self) -> typing.Iterator[str]: + return (cookie.name for cookie in self.jar) + + def __bool__(self) -> bool: + for _ in self.jar: + return True + return False + + def __repr__(self) -> str: + cookies_repr = ", ".join( + [ + f"" + for cookie in self.jar + ] + ) + + return f"" + + class _CookieCompatRequest(urllib.request.Request): + """ + Wraps a `Request` instance up in a compatibility interface suitable + for use with `CookieJar` operations. + """ + + def __init__(self, request: Request) -> None: + super().__init__( + url=str(request.url), + headers=dict(request.headers), + method=request.method, + ) + self.request = request + + def add_unredirected_header(self, key: str, value: str) -> None: + super().add_unredirected_header(key, value) + self.request.headers[key] = value + + class _CookieCompatResponse: + """ + Wraps a `Request` instance up in a compatibility interface suitable + for use with `CookieJar` operations. + """ + + def __init__(self, response: Response) -> None: + self.response = response + + def info(self) -> email.message.Message: + info = email.message.Message() + for key, value in self.response.headers.multi_items(): + # Note that setting `info[key]` here is an "append" operation, + # not a "replace" operation. + # https://docs.python.org/3/library/email.compat32-message.html#email.message.Message.__setitem__ + info[key] = value + return info diff --git a/venv/lib/python3.12/site-packages/httpx/_multipart.py b/venv/lib/python3.12/site-packages/httpx/_multipart.py new file mode 100644 index 0000000000000000000000000000000000000000..b4761af9b2cf384de5189269927d781a700dbe46 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_multipart.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +import io +import mimetypes +import os +import re +import typing +from pathlib import Path + +from ._types import ( + AsyncByteStream, + FileContent, + FileTypes, + RequestData, + RequestFiles, + SyncByteStream, +) +from ._utils import ( + peek_filelike_length, + primitive_value_to_str, + to_bytes, +) + +_HTML5_FORM_ENCODING_REPLACEMENTS = {'"': "%22", "\\": "\\\\"} +_HTML5_FORM_ENCODING_REPLACEMENTS.update( + {chr(c): "%{:02X}".format(c) for c in range(0x1F + 1) if c != 0x1B} +) +_HTML5_FORM_ENCODING_RE = re.compile( + r"|".join([re.escape(c) for c in _HTML5_FORM_ENCODING_REPLACEMENTS.keys()]) +) + + +def _format_form_param(name: str, value: str) -> bytes: + """ + Encode a name/value pair within a multipart form. + """ + + def replacer(match: typing.Match[str]) -> str: + return _HTML5_FORM_ENCODING_REPLACEMENTS[match.group(0)] + + value = _HTML5_FORM_ENCODING_RE.sub(replacer, value) + return f'{name}="{value}"'.encode() + + +def _guess_content_type(filename: str | None) -> str | None: + """ + Guesses the mimetype based on a filename. Defaults to `application/octet-stream`. + + Returns `None` if `filename` is `None` or empty. + """ + if filename: + return mimetypes.guess_type(filename)[0] or "application/octet-stream" + return None + + +def get_multipart_boundary_from_content_type( + content_type: bytes | None, +) -> bytes | None: + if not content_type or not content_type.startswith(b"multipart/form-data"): + return None + # parse boundary according to + # https://www.rfc-editor.org/rfc/rfc2046#section-5.1.1 + if b";" in content_type: + for section in content_type.split(b";"): + if section.strip().lower().startswith(b"boundary="): + return section.strip()[len(b"boundary=") :].strip(b'"') + return None + + +class DataField: + """ + A single form field item, within a multipart form field. + """ + + def __init__(self, name: str, value: str | bytes | int | float | None) -> None: + if not isinstance(name, str): + raise TypeError( + f"Invalid type for name. Expected str, got {type(name)}: {name!r}" + ) + if value is not None and not isinstance(value, (str, bytes, int, float)): + raise TypeError( + "Invalid type for value. Expected primitive type," + f" got {type(value)}: {value!r}" + ) + self.name = name + self.value: str | bytes = ( + value if isinstance(value, bytes) else primitive_value_to_str(value) + ) + + def render_headers(self) -> bytes: + if not hasattr(self, "_headers"): + name = _format_form_param("name", self.name) + self._headers = b"".join( + [b"Content-Disposition: form-data; ", name, b"\r\n\r\n"] + ) + + return self._headers + + def render_data(self) -> bytes: + if not hasattr(self, "_data"): + self._data = to_bytes(self.value) + + return self._data + + def get_length(self) -> int: + headers = self.render_headers() + data = self.render_data() + return len(headers) + len(data) + + def render(self) -> typing.Iterator[bytes]: + yield self.render_headers() + yield self.render_data() + + +class FileField: + """ + A single file field item, within a multipart form field. + """ + + CHUNK_SIZE = 64 * 1024 + + def __init__(self, name: str, value: FileTypes) -> None: + self.name = name + + fileobj: FileContent + + headers: dict[str, str] = {} + content_type: str | None = None + + # This large tuple based API largely mirror's requests' API + # It would be good to think of better APIs for this that we could + # include in httpx 2.0 since variable length tuples(especially of 4 elements) + # are quite unwieldly + if isinstance(value, tuple): + if len(value) == 2: + # neither the 3rd parameter (content_type) nor the 4th (headers) + # was included + filename, fileobj = value + elif len(value) == 3: + filename, fileobj, content_type = value + else: + # all 4 parameters included + filename, fileobj, content_type, headers = value # type: ignore + else: + filename = Path(str(getattr(value, "name", "upload"))).name + fileobj = value + + if content_type is None: + content_type = _guess_content_type(filename) + + has_content_type_header = any("content-type" in key.lower() for key in headers) + if content_type is not None and not has_content_type_header: + # note that unlike requests, we ignore the content_type provided in the 3rd + # tuple element if it is also included in the headers requests does + # the opposite (it overwrites the headerwith the 3rd tuple element) + headers["Content-Type"] = content_type + + if isinstance(fileobj, io.StringIO): + raise TypeError( + "Multipart file uploads require 'io.BytesIO', not 'io.StringIO'." + ) + if isinstance(fileobj, io.TextIOBase): + raise TypeError( + "Multipart file uploads must be opened in binary mode, not text mode." + ) + + self.filename = filename + self.file = fileobj + self.headers = headers + + def get_length(self) -> int | None: + headers = self.render_headers() + + if isinstance(self.file, (str, bytes)): + return len(headers) + len(to_bytes(self.file)) + + file_length = peek_filelike_length(self.file) + + # If we can't determine the filesize without reading it into memory, + # then return `None` here, to indicate an unknown file length. + if file_length is None: + return None + + return len(headers) + file_length + + def render_headers(self) -> bytes: + if not hasattr(self, "_headers"): + parts = [ + b"Content-Disposition: form-data; ", + _format_form_param("name", self.name), + ] + if self.filename: + filename = _format_form_param("filename", self.filename) + parts.extend([b"; ", filename]) + for header_name, header_value in self.headers.items(): + key, val = f"\r\n{header_name}: ".encode(), header_value.encode() + parts.extend([key, val]) + parts.append(b"\r\n\r\n") + self._headers = b"".join(parts) + + return self._headers + + def render_data(self) -> typing.Iterator[bytes]: + if isinstance(self.file, (str, bytes)): + yield to_bytes(self.file) + return + + if hasattr(self.file, "seek"): + try: + self.file.seek(0) + except io.UnsupportedOperation: + pass + + chunk = self.file.read(self.CHUNK_SIZE) + while chunk: + yield to_bytes(chunk) + chunk = self.file.read(self.CHUNK_SIZE) + + def render(self) -> typing.Iterator[bytes]: + yield self.render_headers() + yield from self.render_data() + + +class MultipartStream(SyncByteStream, AsyncByteStream): + """ + Request content as streaming multipart encoded form data. + """ + + def __init__( + self, + data: RequestData, + files: RequestFiles, + boundary: bytes | None = None, + ) -> None: + if boundary is None: + boundary = os.urandom(16).hex().encode("ascii") + + self.boundary = boundary + self.content_type = "multipart/form-data; boundary=%s" % boundary.decode( + "ascii" + ) + self.fields = list(self._iter_fields(data, files)) + + def _iter_fields( + self, data: RequestData, files: RequestFiles + ) -> typing.Iterator[FileField | DataField]: + for name, value in data.items(): + if isinstance(value, (tuple, list)): + for item in value: + yield DataField(name=name, value=item) + else: + yield DataField(name=name, value=value) + + file_items = files.items() if isinstance(files, typing.Mapping) else files + for name, value in file_items: + yield FileField(name=name, value=value) + + def iter_chunks(self) -> typing.Iterator[bytes]: + for field in self.fields: + yield b"--%s\r\n" % self.boundary + yield from field.render() + yield b"\r\n" + yield b"--%s--\r\n" % self.boundary + + def get_content_length(self) -> int | None: + """ + Return the length of the multipart encoded content, or `None` if + any of the files have a length that cannot be determined upfront. + """ + boundary_length = len(self.boundary) + length = 0 + + for field in self.fields: + field_length = field.get_length() + if field_length is None: + return None + + length += 2 + boundary_length + 2 # b"--{boundary}\r\n" + length += field_length + length += 2 # b"\r\n" + + length += 2 + boundary_length + 4 # b"--{boundary}--\r\n" + return length + + # Content stream interface. + + def get_headers(self) -> dict[str, str]: + content_length = self.get_content_length() + content_type = self.content_type + if content_length is None: + return {"Transfer-Encoding": "chunked", "Content-Type": content_type} + return {"Content-Length": str(content_length), "Content-Type": content_type} + + def __iter__(self) -> typing.Iterator[bytes]: + for chunk in self.iter_chunks(): + yield chunk + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + for chunk in self.iter_chunks(): + yield chunk diff --git a/venv/lib/python3.12/site-packages/httpx/_status_codes.py b/venv/lib/python3.12/site-packages/httpx/_status_codes.py new file mode 100644 index 0000000000000000000000000000000000000000..133a6231a5b53fd2f073799ca1bd07c50abe40ae --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_status_codes.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from enum import IntEnum + +__all__ = ["codes"] + + +class codes(IntEnum): + """HTTP status codes and reason phrases + + Status codes from the following RFCs are all observed: + + * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 + * RFC 6585: Additional HTTP Status Codes + * RFC 3229: Delta encoding in HTTP + * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 + * RFC 5842: Binding Extensions to WebDAV + * RFC 7238: Permanent Redirect + * RFC 2295: Transparent Content Negotiation in HTTP + * RFC 2774: An HTTP Extension Framework + * RFC 7540: Hypertext Transfer Protocol Version 2 (HTTP/2) + * RFC 2324: Hyper Text Coffee Pot Control Protocol (HTCPCP/1.0) + * RFC 7725: An HTTP Status Code to Report Legal Obstacles + * RFC 8297: An HTTP Status Code for Indicating Hints + * RFC 8470: Using Early Data in HTTP + """ + + def __new__(cls, value: int, phrase: str = "") -> codes: + obj = int.__new__(cls, value) + obj._value_ = value + + obj.phrase = phrase # type: ignore[attr-defined] + return obj + + def __str__(self) -> str: + return str(self.value) + + @classmethod + def get_reason_phrase(cls, value: int) -> str: + try: + return codes(value).phrase # type: ignore + except ValueError: + return "" + + @classmethod + def is_informational(cls, value: int) -> bool: + """ + Returns `True` for 1xx status codes, `False` otherwise. + """ + return 100 <= value <= 199 + + @classmethod + def is_success(cls, value: int) -> bool: + """ + Returns `True` for 2xx status codes, `False` otherwise. + """ + return 200 <= value <= 299 + + @classmethod + def is_redirect(cls, value: int) -> bool: + """ + Returns `True` for 3xx status codes, `False` otherwise. + """ + return 300 <= value <= 399 + + @classmethod + def is_client_error(cls, value: int) -> bool: + """ + Returns `True` for 4xx status codes, `False` otherwise. + """ + return 400 <= value <= 499 + + @classmethod + def is_server_error(cls, value: int) -> bool: + """ + Returns `True` for 5xx status codes, `False` otherwise. + """ + return 500 <= value <= 599 + + @classmethod + def is_error(cls, value: int) -> bool: + """ + Returns `True` for 4xx or 5xx status codes, `False` otherwise. + """ + return 400 <= value <= 599 + + # informational + CONTINUE = 100, "Continue" + SWITCHING_PROTOCOLS = 101, "Switching Protocols" + PROCESSING = 102, "Processing" + EARLY_HINTS = 103, "Early Hints" + + # success + OK = 200, "OK" + CREATED = 201, "Created" + ACCEPTED = 202, "Accepted" + NON_AUTHORITATIVE_INFORMATION = 203, "Non-Authoritative Information" + NO_CONTENT = 204, "No Content" + RESET_CONTENT = 205, "Reset Content" + PARTIAL_CONTENT = 206, "Partial Content" + MULTI_STATUS = 207, "Multi-Status" + ALREADY_REPORTED = 208, "Already Reported" + IM_USED = 226, "IM Used" + + # redirection + MULTIPLE_CHOICES = 300, "Multiple Choices" + MOVED_PERMANENTLY = 301, "Moved Permanently" + FOUND = 302, "Found" + SEE_OTHER = 303, "See Other" + NOT_MODIFIED = 304, "Not Modified" + USE_PROXY = 305, "Use Proxy" + TEMPORARY_REDIRECT = 307, "Temporary Redirect" + PERMANENT_REDIRECT = 308, "Permanent Redirect" + + # client error + BAD_REQUEST = 400, "Bad Request" + UNAUTHORIZED = 401, "Unauthorized" + PAYMENT_REQUIRED = 402, "Payment Required" + FORBIDDEN = 403, "Forbidden" + NOT_FOUND = 404, "Not Found" + METHOD_NOT_ALLOWED = 405, "Method Not Allowed" + NOT_ACCEPTABLE = 406, "Not Acceptable" + PROXY_AUTHENTICATION_REQUIRED = 407, "Proxy Authentication Required" + REQUEST_TIMEOUT = 408, "Request Timeout" + CONFLICT = 409, "Conflict" + GONE = 410, "Gone" + LENGTH_REQUIRED = 411, "Length Required" + PRECONDITION_FAILED = 412, "Precondition Failed" + REQUEST_ENTITY_TOO_LARGE = 413, "Request Entity Too Large" + REQUEST_URI_TOO_LONG = 414, "Request-URI Too Long" + UNSUPPORTED_MEDIA_TYPE = 415, "Unsupported Media Type" + REQUESTED_RANGE_NOT_SATISFIABLE = 416, "Requested Range Not Satisfiable" + EXPECTATION_FAILED = 417, "Expectation Failed" + IM_A_TEAPOT = 418, "I'm a teapot" + MISDIRECTED_REQUEST = 421, "Misdirected Request" + UNPROCESSABLE_ENTITY = 422, "Unprocessable Entity" + LOCKED = 423, "Locked" + FAILED_DEPENDENCY = 424, "Failed Dependency" + TOO_EARLY = 425, "Too Early" + UPGRADE_REQUIRED = 426, "Upgrade Required" + PRECONDITION_REQUIRED = 428, "Precondition Required" + TOO_MANY_REQUESTS = 429, "Too Many Requests" + REQUEST_HEADER_FIELDS_TOO_LARGE = 431, "Request Header Fields Too Large" + UNAVAILABLE_FOR_LEGAL_REASONS = 451, "Unavailable For Legal Reasons" + + # server errors + INTERNAL_SERVER_ERROR = 500, "Internal Server Error" + NOT_IMPLEMENTED = 501, "Not Implemented" + BAD_GATEWAY = 502, "Bad Gateway" + SERVICE_UNAVAILABLE = 503, "Service Unavailable" + GATEWAY_TIMEOUT = 504, "Gateway Timeout" + HTTP_VERSION_NOT_SUPPORTED = 505, "HTTP Version Not Supported" + VARIANT_ALSO_NEGOTIATES = 506, "Variant Also Negotiates" + INSUFFICIENT_STORAGE = 507, "Insufficient Storage" + LOOP_DETECTED = 508, "Loop Detected" + NOT_EXTENDED = 510, "Not Extended" + NETWORK_AUTHENTICATION_REQUIRED = 511, "Network Authentication Required" + + +# Include lower-case styles for `requests` compatibility. +for code in codes: + setattr(codes, code._name_.lower(), int(code)) diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/__init__.py b/venv/lib/python3.12/site-packages/httpx/_transports/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7a321053b29bcd48698cf2bd74a1d19c8556aefb --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_transports/__init__.py @@ -0,0 +1,15 @@ +from .asgi import * +from .base import * +from .default import * +from .mock import * +from .wsgi import * + +__all__ = [ + "ASGITransport", + "AsyncBaseTransport", + "BaseTransport", + "AsyncHTTPTransport", + "HTTPTransport", + "MockTransport", + "WSGITransport", +] diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5a0f58267100f0a051e361bea1e4eba00dccb44 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/asgi.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/asgi.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c006a23c6e2b65f778493545de4cc338cb176538 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/asgi.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/base.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/base.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..515d834e0aecd23db4af470c5273b51fd1b50677 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/base.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/default.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/default.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aed84761c5f748b4588d49ce71f5689d6ec75bc3 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/default.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/mock.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/mock.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..68c11ef2500b300ce1794548b595a16a0d1f3810 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/mock.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/wsgi.cpython-312.pyc b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/wsgi.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d662673bfe2f3a53e989349e1fafa20642bbaf99 Binary files /dev/null and b/venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/wsgi.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/asgi.py b/venv/lib/python3.12/site-packages/httpx/_transports/asgi.py new file mode 100644 index 0000000000000000000000000000000000000000..2bc4efae0e1b14620f75f712eb15ecf500d14eef --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_transports/asgi.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import typing + +from .._models import Request, Response +from .._types import AsyncByteStream +from .base import AsyncBaseTransport + +if typing.TYPE_CHECKING: # pragma: no cover + import asyncio + + import trio + + Event = typing.Union[asyncio.Event, trio.Event] + + +_Message = typing.MutableMapping[str, typing.Any] +_Receive = typing.Callable[[], typing.Awaitable[_Message]] +_Send = typing.Callable[ + [typing.MutableMapping[str, typing.Any]], typing.Awaitable[None] +] +_ASGIApp = typing.Callable[ + [typing.MutableMapping[str, typing.Any], _Receive, _Send], typing.Awaitable[None] +] + +__all__ = ["ASGITransport"] + + +def is_running_trio() -> bool: + try: + # sniffio is a dependency of trio. + + # See https://github.com/python-trio/trio/issues/2802 + import sniffio + + if sniffio.current_async_library() == "trio": + return True + except ImportError: # pragma: nocover + pass + + return False + + +def create_event() -> Event: + if is_running_trio(): + import trio + + return trio.Event() + + import asyncio + + return asyncio.Event() + + +class ASGIResponseStream(AsyncByteStream): + def __init__(self, body: list[bytes]) -> None: + self._body = body + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + yield b"".join(self._body) + + +class ASGITransport(AsyncBaseTransport): + """ + A custom AsyncTransport that handles sending requests directly to an ASGI app. + + ```python + transport = httpx.ASGITransport( + app=app, + root_path="/submount", + client=("1.2.3.4", 123) + ) + client = httpx.AsyncClient(transport=transport) + ``` + + Arguments: + + * `app` - The ASGI application. + * `raise_app_exceptions` - Boolean indicating if exceptions in the application + should be raised. Default to `True`. Can be set to `False` for use cases + such as testing the content of a client 500 response. + * `root_path` - The root path on which the ASGI application should be mounted. + * `client` - A two-tuple indicating the client IP and port of incoming requests. + ``` + """ + + def __init__( + self, + app: _ASGIApp, + raise_app_exceptions: bool = True, + root_path: str = "", + client: tuple[str, int] = ("127.0.0.1", 123), + ) -> None: + self.app = app + self.raise_app_exceptions = raise_app_exceptions + self.root_path = root_path + self.client = client + + async def handle_async_request( + self, + request: Request, + ) -> Response: + assert isinstance(request.stream, AsyncByteStream) + + # ASGI scope. + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": request.method, + "headers": [(k.lower(), v) for (k, v) in request.headers.raw], + "scheme": request.url.scheme, + "path": request.url.path, + "raw_path": request.url.raw_path.split(b"?")[0], + "query_string": request.url.query, + "server": (request.url.host, request.url.port), + "client": self.client, + "root_path": self.root_path, + } + + # Request. + request_body_chunks = request.stream.__aiter__() + request_complete = False + + # Response. + status_code = None + response_headers = None + body_parts = [] + response_started = False + response_complete = create_event() + + # ASGI callables. + + async def receive() -> dict[str, typing.Any]: + nonlocal request_complete + + if request_complete: + await response_complete.wait() + return {"type": "http.disconnect"} + + try: + body = await request_body_chunks.__anext__() + except StopAsyncIteration: + request_complete = True + return {"type": "http.request", "body": b"", "more_body": False} + return {"type": "http.request", "body": body, "more_body": True} + + async def send(message: typing.MutableMapping[str, typing.Any]) -> None: + nonlocal status_code, response_headers, response_started + + if message["type"] == "http.response.start": + assert not response_started + + status_code = message["status"] + response_headers = message.get("headers", []) + response_started = True + + elif message["type"] == "http.response.body": + assert not response_complete.is_set() + body = message.get("body", b"") + more_body = message.get("more_body", False) + + if body and request.method != "HEAD": + body_parts.append(body) + + if not more_body: + response_complete.set() + + try: + await self.app(scope, receive, send) + except Exception: # noqa: PIE-786 + if self.raise_app_exceptions: + raise + + response_complete.set() + if status_code is None: + status_code = 500 + if response_headers is None: + response_headers = {} + + assert response_complete.is_set() + assert status_code is not None + assert response_headers is not None + + stream = ASGIResponseStream(body_parts) + + return Response(status_code, headers=response_headers, stream=stream) diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/base.py b/venv/lib/python3.12/site-packages/httpx/_transports/base.py new file mode 100644 index 0000000000000000000000000000000000000000..66fd99d702480b555c06694fe14715ea6df3dfc3 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_transports/base.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import typing +from types import TracebackType + +from .._models import Request, Response + +T = typing.TypeVar("T", bound="BaseTransport") +A = typing.TypeVar("A", bound="AsyncBaseTransport") + +__all__ = ["AsyncBaseTransport", "BaseTransport"] + + +class BaseTransport: + def __enter__(self: T) -> T: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + self.close() + + def handle_request(self, request: Request) -> Response: + """ + Send a single HTTP request and return a response. + + Developers shouldn't typically ever need to call into this API directly, + since the Client class provides all the higher level user-facing API + niceties. + + In order to properly release any network resources, the response + stream should *either* be consumed immediately, with a call to + `response.stream.read()`, or else the `handle_request` call should + be followed with a try/finally block to ensuring the stream is + always closed. + + Example usage: + + with httpx.HTTPTransport() as transport: + req = httpx.Request( + method=b"GET", + url=(b"https", b"www.example.com", 443, b"/"), + headers=[(b"Host", b"www.example.com")], + ) + resp = transport.handle_request(req) + body = resp.stream.read() + print(resp.status_code, resp.headers, body) + + + Takes a `Request` instance as the only argument. + + Returns a `Response` instance. + """ + raise NotImplementedError( + "The 'handle_request' method must be implemented." + ) # pragma: no cover + + def close(self) -> None: + pass + + +class AsyncBaseTransport: + async def __aenter__(self: A) -> A: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + await self.aclose() + + async def handle_async_request( + self, + request: Request, + ) -> Response: + raise NotImplementedError( + "The 'handle_async_request' method must be implemented." + ) # pragma: no cover + + async def aclose(self) -> None: + pass diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/default.py b/venv/lib/python3.12/site-packages/httpx/_transports/default.py new file mode 100644 index 0000000000000000000000000000000000000000..d5aa05ff234fd3fbf4fee88c4a7d3e3c151a538f --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_transports/default.py @@ -0,0 +1,406 @@ +""" +Custom transports, with nicely configured defaults. + +The following additional keyword arguments are currently supported by httpcore... + +* uds: str +* local_address: str +* retries: int + +Example usages... + +# Disable HTTP/2 on a single specific domain. +mounts = { + "all://": httpx.HTTPTransport(http2=True), + "all://*example.org": httpx.HTTPTransport() +} + +# Using advanced httpcore configuration, with connection retries. +transport = httpx.HTTPTransport(retries=1) +client = httpx.Client(transport=transport) + +# Using advanced httpcore configuration, with unix domain sockets. +transport = httpx.HTTPTransport(uds="socket.uds") +client = httpx.Client(transport=transport) +""" + +from __future__ import annotations + +import contextlib +import typing +from types import TracebackType + +if typing.TYPE_CHECKING: + import ssl # pragma: no cover + + import httpx # pragma: no cover + +from .._config import DEFAULT_LIMITS, Limits, Proxy, create_ssl_context +from .._exceptions import ( + ConnectError, + ConnectTimeout, + LocalProtocolError, + NetworkError, + PoolTimeout, + ProtocolError, + ProxyError, + ReadError, + ReadTimeout, + RemoteProtocolError, + TimeoutException, + UnsupportedProtocol, + WriteError, + WriteTimeout, +) +from .._models import Request, Response +from .._types import AsyncByteStream, CertTypes, ProxyTypes, SyncByteStream +from .._urls import URL +from .base import AsyncBaseTransport, BaseTransport + +T = typing.TypeVar("T", bound="HTTPTransport") +A = typing.TypeVar("A", bound="AsyncHTTPTransport") + +SOCKET_OPTION = typing.Union[ + typing.Tuple[int, int, int], + typing.Tuple[int, int, typing.Union[bytes, bytearray]], + typing.Tuple[int, int, None, int], +] + +__all__ = ["AsyncHTTPTransport", "HTTPTransport"] + +HTTPCORE_EXC_MAP: dict[type[Exception], type[httpx.HTTPError]] = {} + + +def _load_httpcore_exceptions() -> dict[type[Exception], type[httpx.HTTPError]]: + import httpcore + + return { + httpcore.TimeoutException: TimeoutException, + httpcore.ConnectTimeout: ConnectTimeout, + httpcore.ReadTimeout: ReadTimeout, + httpcore.WriteTimeout: WriteTimeout, + httpcore.PoolTimeout: PoolTimeout, + httpcore.NetworkError: NetworkError, + httpcore.ConnectError: ConnectError, + httpcore.ReadError: ReadError, + httpcore.WriteError: WriteError, + httpcore.ProxyError: ProxyError, + httpcore.UnsupportedProtocol: UnsupportedProtocol, + httpcore.ProtocolError: ProtocolError, + httpcore.LocalProtocolError: LocalProtocolError, + httpcore.RemoteProtocolError: RemoteProtocolError, + } + + +@contextlib.contextmanager +def map_httpcore_exceptions() -> typing.Iterator[None]: + global HTTPCORE_EXC_MAP + if len(HTTPCORE_EXC_MAP) == 0: + HTTPCORE_EXC_MAP = _load_httpcore_exceptions() + try: + yield + except Exception as exc: + mapped_exc = None + + for from_exc, to_exc in HTTPCORE_EXC_MAP.items(): + if not isinstance(exc, from_exc): + continue + # We want to map to the most specific exception we can find. + # Eg if `exc` is an `httpcore.ReadTimeout`, we want to map to + # `httpx.ReadTimeout`, not just `httpx.TimeoutException`. + if mapped_exc is None or issubclass(to_exc, mapped_exc): + mapped_exc = to_exc + + if mapped_exc is None: # pragma: no cover + raise + + message = str(exc) + raise mapped_exc(message) from exc + + +class ResponseStream(SyncByteStream): + def __init__(self, httpcore_stream: typing.Iterable[bytes]) -> None: + self._httpcore_stream = httpcore_stream + + def __iter__(self) -> typing.Iterator[bytes]: + with map_httpcore_exceptions(): + for part in self._httpcore_stream: + yield part + + def close(self) -> None: + if hasattr(self._httpcore_stream, "close"): + self._httpcore_stream.close() + + +class HTTPTransport(BaseTransport): + def __init__( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + proxy: ProxyTypes | None = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + import httpcore + + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + if proxy is None: + self._pool = httpcore.ConnectionPool( + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + uds=uds, + local_address=local_address, + retries=retries, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("http", "https"): + self._pool = httpcore.HTTPProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + ssl_context=ssl_context, + proxy_ssl_context=proxy.ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("socks5", "socks5h"): + try: + import socksio # noqa + except ImportError: # pragma: no cover + raise ImportError( + "Using SOCKS proxy, but the 'socksio' package is not installed. " + "Make sure to install httpx using `pip install httpx[socks]`." + ) from None + + self._pool = httpcore.SOCKSProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + ) + else: # pragma: no cover + raise ValueError( + "Proxy protocol must be either 'http', 'https', 'socks5', or 'socks5h'," + f" but got {proxy.url.scheme!r}." + ) + + def __enter__(self: T) -> T: # Use generics for subclass support. + self._pool.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + with map_httpcore_exceptions(): + self._pool.__exit__(exc_type, exc_value, traceback) + + def handle_request( + self, + request: Request, + ) -> Response: + assert isinstance(request.stream, SyncByteStream) + import httpcore + + req = httpcore.Request( + method=request.method, + url=httpcore.URL( + scheme=request.url.raw_scheme, + host=request.url.raw_host, + port=request.url.port, + target=request.url.raw_path, + ), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + with map_httpcore_exceptions(): + resp = self._pool.handle_request(req) + + assert isinstance(resp.stream, typing.Iterable) + + return Response( + status_code=resp.status, + headers=resp.headers, + stream=ResponseStream(resp.stream), + extensions=resp.extensions, + ) + + def close(self) -> None: + self._pool.close() + + +class AsyncResponseStream(AsyncByteStream): + def __init__(self, httpcore_stream: typing.AsyncIterable[bytes]) -> None: + self._httpcore_stream = httpcore_stream + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + with map_httpcore_exceptions(): + async for part in self._httpcore_stream: + yield part + + async def aclose(self) -> None: + if hasattr(self._httpcore_stream, "aclose"): + await self._httpcore_stream.aclose() + + +class AsyncHTTPTransport(AsyncBaseTransport): + def __init__( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + proxy: ProxyTypes | None = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + import httpcore + + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + if proxy is None: + self._pool = httpcore.AsyncConnectionPool( + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + uds=uds, + local_address=local_address, + retries=retries, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("http", "https"): + self._pool = httpcore.AsyncHTTPProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + proxy_ssl_context=proxy.ssl_context, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("socks5", "socks5h"): + try: + import socksio # noqa + except ImportError: # pragma: no cover + raise ImportError( + "Using SOCKS proxy, but the 'socksio' package is not installed. " + "Make sure to install httpx using `pip install httpx[socks]`." + ) from None + + self._pool = httpcore.AsyncSOCKSProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + ) + else: # pragma: no cover + raise ValueError( + "Proxy protocol must be either 'http', 'https', 'socks5', or 'socks5h'," + " but got {proxy.url.scheme!r}." + ) + + async def __aenter__(self: A) -> A: # Use generics for subclass support. + await self._pool.__aenter__() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + with map_httpcore_exceptions(): + await self._pool.__aexit__(exc_type, exc_value, traceback) + + async def handle_async_request( + self, + request: Request, + ) -> Response: + assert isinstance(request.stream, AsyncByteStream) + import httpcore + + req = httpcore.Request( + method=request.method, + url=httpcore.URL( + scheme=request.url.raw_scheme, + host=request.url.raw_host, + port=request.url.port, + target=request.url.raw_path, + ), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + with map_httpcore_exceptions(): + resp = await self._pool.handle_async_request(req) + + assert isinstance(resp.stream, typing.AsyncIterable) + + return Response( + status_code=resp.status, + headers=resp.headers, + stream=AsyncResponseStream(resp.stream), + extensions=resp.extensions, + ) + + async def aclose(self) -> None: + await self._pool.aclose() diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/mock.py b/venv/lib/python3.12/site-packages/httpx/_transports/mock.py new file mode 100644 index 0000000000000000000000000000000000000000..8c418f59e06cae43abdbb626ec21cafc7e8c6277 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_transports/mock.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import typing + +from .._models import Request, Response +from .base import AsyncBaseTransport, BaseTransport + +SyncHandler = typing.Callable[[Request], Response] +AsyncHandler = typing.Callable[[Request], typing.Coroutine[None, None, Response]] + + +__all__ = ["MockTransport"] + + +class MockTransport(AsyncBaseTransport, BaseTransport): + def __init__(self, handler: SyncHandler | AsyncHandler) -> None: + self.handler = handler + + def handle_request( + self, + request: Request, + ) -> Response: + request.read() + response = self.handler(request) + if not isinstance(response, Response): # pragma: no cover + raise TypeError("Cannot use an async handler in a sync Client") + return response + + async def handle_async_request( + self, + request: Request, + ) -> Response: + await request.aread() + response = self.handler(request) + + # Allow handler to *optionally* be an `async` function. + # If it is, then the `response` variable need to be awaited to actually + # return the result. + + if not isinstance(response, Response): + response = await response + + return response diff --git a/venv/lib/python3.12/site-packages/httpx/_transports/wsgi.py b/venv/lib/python3.12/site-packages/httpx/_transports/wsgi.py new file mode 100644 index 0000000000000000000000000000000000000000..8592ffe017a87367cc7578184540096a9682908d --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_transports/wsgi.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import io +import itertools +import sys +import typing + +from .._models import Request, Response +from .._types import SyncByteStream +from .base import BaseTransport + +if typing.TYPE_CHECKING: + from _typeshed import OptExcInfo # pragma: no cover + from _typeshed.wsgi import WSGIApplication # pragma: no cover + +_T = typing.TypeVar("_T") + + +__all__ = ["WSGITransport"] + + +def _skip_leading_empty_chunks(body: typing.Iterable[_T]) -> typing.Iterable[_T]: + body = iter(body) + for chunk in body: + if chunk: + return itertools.chain([chunk], body) + return [] + + +class WSGIByteStream(SyncByteStream): + def __init__(self, result: typing.Iterable[bytes]) -> None: + self._close = getattr(result, "close", None) + self._result = _skip_leading_empty_chunks(result) + + def __iter__(self) -> typing.Iterator[bytes]: + for part in self._result: + yield part + + def close(self) -> None: + if self._close is not None: + self._close() + + +class WSGITransport(BaseTransport): + """ + A custom transport that handles sending requests directly to an WSGI app. + The simplest way to use this functionality is to use the `app` argument. + + ``` + client = httpx.Client(app=app) + ``` + + Alternatively, you can setup the transport instance explicitly. + This allows you to include any additional configuration arguments specific + to the WSGITransport class: + + ``` + transport = httpx.WSGITransport( + app=app, + script_name="/submount", + remote_addr="1.2.3.4" + ) + client = httpx.Client(transport=transport) + ``` + + Arguments: + + * `app` - The WSGI application. + * `raise_app_exceptions` - Boolean indicating if exceptions in the application + should be raised. Default to `True`. Can be set to `False` for use cases + such as testing the content of a client 500 response. + * `script_name` - The root path on which the WSGI application should be mounted. + * `remote_addr` - A string indicating the client IP of incoming requests. + ``` + """ + + def __init__( + self, + app: WSGIApplication, + raise_app_exceptions: bool = True, + script_name: str = "", + remote_addr: str = "127.0.0.1", + wsgi_errors: typing.TextIO | None = None, + ) -> None: + self.app = app + self.raise_app_exceptions = raise_app_exceptions + self.script_name = script_name + self.remote_addr = remote_addr + self.wsgi_errors = wsgi_errors + + def handle_request(self, request: Request) -> Response: + request.read() + wsgi_input = io.BytesIO(request.content) + + port = request.url.port or {"http": 80, "https": 443}[request.url.scheme] + environ = { + "wsgi.version": (1, 0), + "wsgi.url_scheme": request.url.scheme, + "wsgi.input": wsgi_input, + "wsgi.errors": self.wsgi_errors or sys.stderr, + "wsgi.multithread": True, + "wsgi.multiprocess": False, + "wsgi.run_once": False, + "REQUEST_METHOD": request.method, + "SCRIPT_NAME": self.script_name, + "PATH_INFO": request.url.path, + "QUERY_STRING": request.url.query.decode("ascii"), + "SERVER_NAME": request.url.host, + "SERVER_PORT": str(port), + "SERVER_PROTOCOL": "HTTP/1.1", + "REMOTE_ADDR": self.remote_addr, + } + for header_key, header_value in request.headers.raw: + key = header_key.decode("ascii").upper().replace("-", "_") + if key not in ("CONTENT_TYPE", "CONTENT_LENGTH"): + key = "HTTP_" + key + environ[key] = header_value.decode("ascii") + + seen_status = None + seen_response_headers = None + seen_exc_info = None + + def start_response( + status: str, + response_headers: list[tuple[str, str]], + exc_info: OptExcInfo | None = None, + ) -> typing.Callable[[bytes], typing.Any]: + nonlocal seen_status, seen_response_headers, seen_exc_info + seen_status = status + seen_response_headers = response_headers + seen_exc_info = exc_info + return lambda _: None + + result = self.app(environ, start_response) + + stream = WSGIByteStream(result) + + assert seen_status is not None + assert seen_response_headers is not None + if seen_exc_info and seen_exc_info[0] and self.raise_app_exceptions: + raise seen_exc_info[1] + + status_code = int(seen_status.split()[0]) + headers = [ + (key.encode("ascii"), value.encode("ascii")) + for key, value in seen_response_headers + ] + + return Response(status_code, headers=headers, stream=stream) diff --git a/venv/lib/python3.12/site-packages/httpx/_types.py b/venv/lib/python3.12/site-packages/httpx/_types.py new file mode 100644 index 0000000000000000000000000000000000000000..704dfdffc8ba61eb913fa918072381e410b23c00 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_types.py @@ -0,0 +1,114 @@ +""" +Type definitions for type checking purposes. +""" + +from http.cookiejar import CookieJar +from typing import ( + IO, + TYPE_CHECKING, + Any, + AsyncIterable, + AsyncIterator, + Callable, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Sequence, + Tuple, + Union, +) + +if TYPE_CHECKING: # pragma: no cover + from ._auth import Auth # noqa: F401 + from ._config import Proxy, Timeout # noqa: F401 + from ._models import Cookies, Headers, Request # noqa: F401 + from ._urls import URL, QueryParams # noqa: F401 + + +PrimitiveData = Optional[Union[str, int, float, bool]] + +URLTypes = Union["URL", str] + +QueryParamTypes = Union[ + "QueryParams", + Mapping[str, Union[PrimitiveData, Sequence[PrimitiveData]]], + List[Tuple[str, PrimitiveData]], + Tuple[Tuple[str, PrimitiveData], ...], + str, + bytes, +] + +HeaderTypes = Union[ + "Headers", + Mapping[str, str], + Mapping[bytes, bytes], + Sequence[Tuple[str, str]], + Sequence[Tuple[bytes, bytes]], +] + +CookieTypes = Union["Cookies", CookieJar, Dict[str, str], List[Tuple[str, str]]] + +TimeoutTypes = Union[ + Optional[float], + Tuple[Optional[float], Optional[float], Optional[float], Optional[float]], + "Timeout", +] +ProxyTypes = Union["URL", str, "Proxy"] +CertTypes = Union[str, Tuple[str, str], Tuple[str, str, str]] + +AuthTypes = Union[ + Tuple[Union[str, bytes], Union[str, bytes]], + Callable[["Request"], "Request"], + "Auth", +] + +RequestContent = Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] +ResponseContent = Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] +ResponseExtensions = Mapping[str, Any] + +RequestData = Mapping[str, Any] + +FileContent = Union[IO[bytes], bytes, str] +FileTypes = Union[ + # file (or bytes) + FileContent, + # (filename, file (or bytes)) + Tuple[Optional[str], FileContent], + # (filename, file (or bytes), content_type) + Tuple[Optional[str], FileContent, Optional[str]], + # (filename, file (or bytes), content_type, headers) + Tuple[Optional[str], FileContent, Optional[str], Mapping[str, str]], +] +RequestFiles = Union[Mapping[str, FileTypes], Sequence[Tuple[str, FileTypes]]] + +RequestExtensions = Mapping[str, Any] + +__all__ = ["AsyncByteStream", "SyncByteStream"] + + +class SyncByteStream: + def __iter__(self) -> Iterator[bytes]: + raise NotImplementedError( + "The '__iter__' method must be implemented." + ) # pragma: no cover + yield b"" # pragma: no cover + + def close(self) -> None: + """ + Subclasses can override this method to release any network resources + after a request/response cycle is complete. + """ + + +class AsyncByteStream: + async def __aiter__(self) -> AsyncIterator[bytes]: + raise NotImplementedError( + "The '__aiter__' method must be implemented." + ) # pragma: no cover + yield b"" # pragma: no cover + + async def aclose(self) -> None: + pass diff --git a/venv/lib/python3.12/site-packages/httpx/_urlparse.py b/venv/lib/python3.12/site-packages/httpx/_urlparse.py new file mode 100644 index 0000000000000000000000000000000000000000..bf190fd560ee4fc8a11af371a15fc5f1dc284d34 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_urlparse.py @@ -0,0 +1,527 @@ +""" +An implementation of `urlparse` that provides URL validation and normalization +as described by RFC3986. + +We rely on this implementation rather than the one in Python's stdlib, because: + +* It provides more complete URL validation. +* It properly differentiates between an empty querystring and an absent querystring, + to distinguish URLs with a trailing '?'. +* It handles scheme, hostname, port, and path normalization. +* It supports IDNA hostnames, normalizing them to their encoded form. +* The API supports passing individual components, as well as the complete URL string. + +Previously we relied on the excellent `rfc3986` package to handle URL parsing and +validation, but this module provides a simpler alternative, with less indirection +required. +""" + +from __future__ import annotations + +import ipaddress +import re +import typing + +import idna + +from ._exceptions import InvalidURL + +MAX_URL_LENGTH = 65536 + +# https://datatracker.ietf.org/doc/html/rfc3986.html#section-2.3 +UNRESERVED_CHARACTERS = ( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" +) +SUB_DELIMS = "!$&'()*+,;=" + +PERCENT_ENCODED_REGEX = re.compile("%[A-Fa-f0-9]{2}") + +# https://url.spec.whatwg.org/#percent-encoded-bytes + +# The fragment percent-encode set is the C0 control percent-encode set +# and U+0020 SPACE, U+0022 ("), U+003C (<), U+003E (>), and U+0060 (`). +FRAG_SAFE = "".join( + [chr(i) for i in range(0x20, 0x7F) if i not in (0x20, 0x22, 0x3C, 0x3E, 0x60)] +) + +# The query percent-encode set is the C0 control percent-encode set +# and U+0020 SPACE, U+0022 ("), U+0023 (#), U+003C (<), and U+003E (>). +QUERY_SAFE = "".join( + [chr(i) for i in range(0x20, 0x7F) if i not in (0x20, 0x22, 0x23, 0x3C, 0x3E)] +) + +# The path percent-encode set is the query percent-encode set +# and U+003F (?), U+0060 (`), U+007B ({), and U+007D (}). +PATH_SAFE = "".join( + [ + chr(i) + for i in range(0x20, 0x7F) + if i not in (0x20, 0x22, 0x23, 0x3C, 0x3E) + (0x3F, 0x60, 0x7B, 0x7D) + ] +) + +# The userinfo percent-encode set is the path percent-encode set +# and U+002F (/), U+003A (:), U+003B (;), U+003D (=), U+0040 (@), +# U+005B ([) to U+005E (^), inclusive, and U+007C (|). +USERNAME_SAFE = "".join( + [ + chr(i) + for i in range(0x20, 0x7F) + if i + not in (0x20, 0x22, 0x23, 0x3C, 0x3E) + + (0x3F, 0x60, 0x7B, 0x7D) + + (0x2F, 0x3A, 0x3B, 0x3D, 0x40, 0x5B, 0x5C, 0x5D, 0x5E, 0x7C) + ] +) +PASSWORD_SAFE = "".join( + [ + chr(i) + for i in range(0x20, 0x7F) + if i + not in (0x20, 0x22, 0x23, 0x3C, 0x3E) + + (0x3F, 0x60, 0x7B, 0x7D) + + (0x2F, 0x3A, 0x3B, 0x3D, 0x40, 0x5B, 0x5C, 0x5D, 0x5E, 0x7C) + ] +) +# Note... The terminology 'userinfo' percent-encode set in the WHATWG document +# is used for the username and password quoting. For the joint userinfo component +# we remove U+003A (:) from the safe set. +USERINFO_SAFE = "".join( + [ + chr(i) + for i in range(0x20, 0x7F) + if i + not in (0x20, 0x22, 0x23, 0x3C, 0x3E) + + (0x3F, 0x60, 0x7B, 0x7D) + + (0x2F, 0x3B, 0x3D, 0x40, 0x5B, 0x5C, 0x5D, 0x5E, 0x7C) + ] +) + + +# {scheme}: (optional) +# //{authority} (optional) +# {path} +# ?{query} (optional) +# #{fragment} (optional) +URL_REGEX = re.compile( + ( + r"(?:(?P{scheme}):)?" + r"(?://(?P{authority}))?" + r"(?P{path})" + r"(?:\?(?P{query}))?" + r"(?:#(?P{fragment}))?" + ).format( + scheme="([a-zA-Z][a-zA-Z0-9+.-]*)?", + authority="[^/?#]*", + path="[^?#]*", + query="[^#]*", + fragment=".*", + ) +) + +# {userinfo}@ (optional) +# {host} +# :{port} (optional) +AUTHORITY_REGEX = re.compile( + ( + r"(?:(?P{userinfo})@)?" r"(?P{host})" r":?(?P{port})?" + ).format( + userinfo=".*", # Any character sequence. + host="(\\[.*\\]|[^:@]*)", # Either any character sequence excluding ':' or '@', + # or an IPv6 address enclosed within square brackets. + port=".*", # Any character sequence. + ) +) + + +# If we call urlparse with an individual component, then we need to regex +# validate that component individually. +# Note that we're duplicating the same strings as above. Shock! Horror!! +COMPONENT_REGEX = { + "scheme": re.compile("([a-zA-Z][a-zA-Z0-9+.-]*)?"), + "authority": re.compile("[^/?#]*"), + "path": re.compile("[^?#]*"), + "query": re.compile("[^#]*"), + "fragment": re.compile(".*"), + "userinfo": re.compile("[^@]*"), + "host": re.compile("(\\[.*\\]|[^:]*)"), + "port": re.compile(".*"), +} + + +# We use these simple regexs as a first pass before handing off to +# the stdlib 'ipaddress' module for IP address validation. +IPv4_STYLE_HOSTNAME = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$") +IPv6_STYLE_HOSTNAME = re.compile(r"^\[.*\]$") + + +class ParseResult(typing.NamedTuple): + scheme: str + userinfo: str + host: str + port: int | None + path: str + query: str | None + fragment: str | None + + @property + def authority(self) -> str: + return "".join( + [ + f"{self.userinfo}@" if self.userinfo else "", + f"[{self.host}]" if ":" in self.host else self.host, + f":{self.port}" if self.port is not None else "", + ] + ) + + @property + def netloc(self) -> str: + return "".join( + [ + f"[{self.host}]" if ":" in self.host else self.host, + f":{self.port}" if self.port is not None else "", + ] + ) + + def copy_with(self, **kwargs: str | None) -> ParseResult: + if not kwargs: + return self + + defaults = { + "scheme": self.scheme, + "authority": self.authority, + "path": self.path, + "query": self.query, + "fragment": self.fragment, + } + defaults.update(kwargs) + return urlparse("", **defaults) + + def __str__(self) -> str: + authority = self.authority + return "".join( + [ + f"{self.scheme}:" if self.scheme else "", + f"//{authority}" if authority else "", + self.path, + f"?{self.query}" if self.query is not None else "", + f"#{self.fragment}" if self.fragment is not None else "", + ] + ) + + +def urlparse(url: str = "", **kwargs: str | None) -> ParseResult: + # Initial basic checks on allowable URLs. + # --------------------------------------- + + # Hard limit the maximum allowable URL length. + if len(url) > MAX_URL_LENGTH: + raise InvalidURL("URL too long") + + # If a URL includes any ASCII control characters including \t, \r, \n, + # then treat it as invalid. + if any(char.isascii() and not char.isprintable() for char in url): + char = next(char for char in url if char.isascii() and not char.isprintable()) + idx = url.find(char) + error = ( + f"Invalid non-printable ASCII character in URL, {char!r} at position {idx}." + ) + raise InvalidURL(error) + + # Some keyword arguments require special handling. + # ------------------------------------------------ + + # Coerce "port" to a string, if it is provided as an integer. + if "port" in kwargs: + port = kwargs["port"] + kwargs["port"] = str(port) if isinstance(port, int) else port + + # Replace "netloc" with "host and "port". + if "netloc" in kwargs: + netloc = kwargs.pop("netloc") or "" + kwargs["host"], _, kwargs["port"] = netloc.partition(":") + + # Replace "username" and/or "password" with "userinfo". + if "username" in kwargs or "password" in kwargs: + username = quote(kwargs.pop("username", "") or "", safe=USERNAME_SAFE) + password = quote(kwargs.pop("password", "") or "", safe=PASSWORD_SAFE) + kwargs["userinfo"] = f"{username}:{password}" if password else username + + # Replace "raw_path" with "path" and "query". + if "raw_path" in kwargs: + raw_path = kwargs.pop("raw_path") or "" + kwargs["path"], seperator, kwargs["query"] = raw_path.partition("?") + if not seperator: + kwargs["query"] = None + + # Ensure that IPv6 "host" addresses are always escaped with "[...]". + if "host" in kwargs: + host = kwargs.get("host") or "" + if ":" in host and not (host.startswith("[") and host.endswith("]")): + kwargs["host"] = f"[{host}]" + + # If any keyword arguments are provided, ensure they are valid. + # ------------------------------------------------------------- + + for key, value in kwargs.items(): + if value is not None: + if len(value) > MAX_URL_LENGTH: + raise InvalidURL(f"URL component '{key}' too long") + + # If a component includes any ASCII control characters including \t, \r, \n, + # then treat it as invalid. + if any(char.isascii() and not char.isprintable() for char in value): + char = next( + char for char in value if char.isascii() and not char.isprintable() + ) + idx = value.find(char) + error = ( + f"Invalid non-printable ASCII character in URL {key} component, " + f"{char!r} at position {idx}." + ) + raise InvalidURL(error) + + # Ensure that keyword arguments match as a valid regex. + if not COMPONENT_REGEX[key].fullmatch(value): + raise InvalidURL(f"Invalid URL component '{key}'") + + # The URL_REGEX will always match, but may have empty components. + url_match = URL_REGEX.match(url) + assert url_match is not None + url_dict = url_match.groupdict() + + # * 'scheme', 'authority', and 'path' may be empty strings. + # * 'query' may be 'None', indicating no trailing "?" portion. + # Any string including the empty string, indicates a trailing "?". + # * 'fragment' may be 'None', indicating no trailing "#" portion. + # Any string including the empty string, indicates a trailing "#". + scheme = kwargs.get("scheme", url_dict["scheme"]) or "" + authority = kwargs.get("authority", url_dict["authority"]) or "" + path = kwargs.get("path", url_dict["path"]) or "" + query = kwargs.get("query", url_dict["query"]) + frag = kwargs.get("fragment", url_dict["fragment"]) + + # The AUTHORITY_REGEX will always match, but may have empty components. + authority_match = AUTHORITY_REGEX.match(authority) + assert authority_match is not None + authority_dict = authority_match.groupdict() + + # * 'userinfo' and 'host' may be empty strings. + # * 'port' may be 'None'. + userinfo = kwargs.get("userinfo", authority_dict["userinfo"]) or "" + host = kwargs.get("host", authority_dict["host"]) or "" + port = kwargs.get("port", authority_dict["port"]) + + # Normalize and validate each component. + # We end up with a parsed representation of the URL, + # with components that are plain ASCII bytestrings. + parsed_scheme: str = scheme.lower() + parsed_userinfo: str = quote(userinfo, safe=USERINFO_SAFE) + parsed_host: str = encode_host(host) + parsed_port: int | None = normalize_port(port, scheme) + + has_scheme = parsed_scheme != "" + has_authority = ( + parsed_userinfo != "" or parsed_host != "" or parsed_port is not None + ) + validate_path(path, has_scheme=has_scheme, has_authority=has_authority) + if has_scheme or has_authority: + path = normalize_path(path) + + parsed_path: str = quote(path, safe=PATH_SAFE) + parsed_query: str | None = None if query is None else quote(query, safe=QUERY_SAFE) + parsed_frag: str | None = None if frag is None else quote(frag, safe=FRAG_SAFE) + + # The parsed ASCII bytestrings are our canonical form. + # All properties of the URL are derived from these. + return ParseResult( + parsed_scheme, + parsed_userinfo, + parsed_host, + parsed_port, + parsed_path, + parsed_query, + parsed_frag, + ) + + +def encode_host(host: str) -> str: + if not host: + return "" + + elif IPv4_STYLE_HOSTNAME.match(host): + # Validate IPv4 hostnames like #.#.#.# + # + # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2 + # + # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet + try: + ipaddress.IPv4Address(host) + except ipaddress.AddressValueError: + raise InvalidURL(f"Invalid IPv4 address: {host!r}") + return host + + elif IPv6_STYLE_HOSTNAME.match(host): + # Validate IPv6 hostnames like [...] + # + # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2 + # + # "A host identified by an Internet Protocol literal address, version 6 + # [RFC3513] or later, is distinguished by enclosing the IP literal + # within square brackets ("[" and "]"). This is the only place where + # square bracket characters are allowed in the URI syntax." + try: + ipaddress.IPv6Address(host[1:-1]) + except ipaddress.AddressValueError: + raise InvalidURL(f"Invalid IPv6 address: {host!r}") + return host[1:-1] + + elif host.isascii(): + # Regular ASCII hostnames + # + # From https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2.2 + # + # reg-name = *( unreserved / pct-encoded / sub-delims ) + WHATWG_SAFE = '"`{}%|\\' + return quote(host.lower(), safe=SUB_DELIMS + WHATWG_SAFE) + + # IDNA hostnames + try: + return idna.encode(host.lower()).decode("ascii") + except idna.IDNAError: + raise InvalidURL(f"Invalid IDNA hostname: {host!r}") + + +def normalize_port(port: str | int | None, scheme: str) -> int | None: + # From https://tools.ietf.org/html/rfc3986#section-3.2.3 + # + # "A scheme may define a default port. For example, the "http" scheme + # defines a default port of "80", corresponding to its reserved TCP + # port number. The type of port designated by the port number (e.g., + # TCP, UDP, SCTP) is defined by the URI scheme. URI producers and + # normalizers should omit the port component and its ":" delimiter if + # port is empty or if its value would be the same as that of the + # scheme's default." + if port is None or port == "": + return None + + try: + port_as_int = int(port) + except ValueError: + raise InvalidURL(f"Invalid port: {port!r}") + + # See https://url.spec.whatwg.org/#url-miscellaneous + default_port = {"ftp": 21, "http": 80, "https": 443, "ws": 80, "wss": 443}.get( + scheme + ) + if port_as_int == default_port: + return None + return port_as_int + + +def validate_path(path: str, has_scheme: bool, has_authority: bool) -> None: + """ + Path validation rules that depend on if the URL contains + a scheme or authority component. + + See https://datatracker.ietf.org/doc/html/rfc3986.html#section-3.3 + """ + if has_authority: + # If a URI contains an authority component, then the path component + # must either be empty or begin with a slash ("/") character." + if path and not path.startswith("/"): + raise InvalidURL("For absolute URLs, path must be empty or begin with '/'") + + if not has_scheme and not has_authority: + # If a URI does not contain an authority component, then the path cannot begin + # with two slash characters ("//"). + if path.startswith("//"): + raise InvalidURL("Relative URLs cannot have a path starting with '//'") + + # In addition, a URI reference (Section 4.1) may be a relative-path reference, + # in which case the first path segment cannot contain a colon (":") character. + if path.startswith(":"): + raise InvalidURL("Relative URLs cannot have a path starting with ':'") + + +def normalize_path(path: str) -> str: + """ + Drop "." and ".." segments from a URL path. + + For example: + + normalize_path("/path/./to/somewhere/..") == "/path/to" + """ + # Fast return when no '.' characters in the path. + if "." not in path: + return path + + components = path.split("/") + + # Fast return when no '.' or '..' components in the path. + if "." not in components and ".." not in components: + return path + + # https://datatracker.ietf.org/doc/html/rfc3986#section-5.2.4 + output: list[str] = [] + for component in components: + if component == ".": + pass + elif component == "..": + if output and output != [""]: + output.pop() + else: + output.append(component) + return "/".join(output) + + +def PERCENT(string: str) -> str: + return "".join([f"%{byte:02X}" for byte in string.encode("utf-8")]) + + +def percent_encoded(string: str, safe: str) -> str: + """ + Use percent-encoding to quote a string. + """ + NON_ESCAPED_CHARS = UNRESERVED_CHARACTERS + safe + + # Fast path for strings that don't need escaping. + if not string.rstrip(NON_ESCAPED_CHARS): + return string + + return "".join( + [char if char in NON_ESCAPED_CHARS else PERCENT(char) for char in string] + ) + + +def quote(string: str, safe: str) -> str: + """ + Use percent-encoding to quote a string, omitting existing '%xx' escape sequences. + + See: https://www.rfc-editor.org/rfc/rfc3986#section-2.1 + + * `string`: The string to be percent-escaped. + * `safe`: A string containing characters that may be treated as safe, and do not + need to be escaped. Unreserved characters are always treated as safe. + See: https://www.rfc-editor.org/rfc/rfc3986#section-2.3 + """ + parts = [] + current_position = 0 + for match in re.finditer(PERCENT_ENCODED_REGEX, string): + start_position, end_position = match.start(), match.end() + matched_text = match.group(0) + # Add any text up to the '%xx' escape sequence. + if start_position != current_position: + leading_text = string[current_position:start_position] + parts.append(percent_encoded(leading_text, safe=safe)) + + # Add the '%xx' escape sequence. + parts.append(matched_text) + current_position = end_position + + # Add any text after the final '%xx' escape sequence. + if current_position != len(string): + trailing_text = string[current_position:] + parts.append(percent_encoded(trailing_text, safe=safe)) + + return "".join(parts) diff --git a/venv/lib/python3.12/site-packages/httpx/_urls.py b/venv/lib/python3.12/site-packages/httpx/_urls.py new file mode 100644 index 0000000000000000000000000000000000000000..147a8fa333acaf31618d37ba2896e3a5bf5e4d02 --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_urls.py @@ -0,0 +1,641 @@ +from __future__ import annotations + +import typing +from urllib.parse import parse_qs, unquote, urlencode + +import idna + +from ._types import QueryParamTypes +from ._urlparse import urlparse +from ._utils import primitive_value_to_str + +__all__ = ["URL", "QueryParams"] + + +class URL: + """ + url = httpx.URL("HTTPS://jo%40email.com:a%20secret@müller.de:1234/pa%20th?search=ab#anchorlink") + + assert url.scheme == "https" + assert url.username == "jo@email.com" + assert url.password == "a secret" + assert url.userinfo == b"jo%40email.com:a%20secret" + assert url.host == "müller.de" + assert url.raw_host == b"xn--mller-kva.de" + assert url.port == 1234 + assert url.netloc == b"xn--mller-kva.de:1234" + assert url.path == "/pa th" + assert url.query == b"?search=ab" + assert url.raw_path == b"/pa%20th?search=ab" + assert url.fragment == "anchorlink" + + The components of a URL are broken down like this: + + https://jo%40email.com:a%20secret@müller.de:1234/pa%20th?search=ab#anchorlink + [scheme] [ username ] [password] [ host ][port][ path ] [ query ] [fragment] + [ userinfo ] [ netloc ][ raw_path ] + + Note that: + + * `url.scheme` is normalized to always be lowercased. + + * `url.host` is normalized to always be lowercased. Internationalized domain + names are represented in unicode, without IDNA encoding applied. For instance: + + url = httpx.URL("http://中国.icom.museum") + assert url.host == "中国.icom.museum" + url = httpx.URL("http://xn--fiqs8s.icom.museum") + assert url.host == "中国.icom.museum" + + * `url.raw_host` is normalized to always be lowercased, and is IDNA encoded. + + url = httpx.URL("http://中国.icom.museum") + assert url.raw_host == b"xn--fiqs8s.icom.museum" + url = httpx.URL("http://xn--fiqs8s.icom.museum") + assert url.raw_host == b"xn--fiqs8s.icom.museum" + + * `url.port` is either None or an integer. URLs that include the default port for + "http", "https", "ws", "wss", and "ftp" schemes have their port + normalized to `None`. + + assert httpx.URL("http://example.com") == httpx.URL("http://example.com:80") + assert httpx.URL("http://example.com").port is None + assert httpx.URL("http://example.com:80").port is None + + * `url.userinfo` is raw bytes, without URL escaping. Usually you'll want to work + with `url.username` and `url.password` instead, which handle the URL escaping. + + * `url.raw_path` is raw bytes of both the path and query, without URL escaping. + This portion is used as the target when constructing HTTP requests. Usually you'll + want to work with `url.path` instead. + + * `url.query` is raw bytes, without URL escaping. A URL query string portion can + only be properly URL escaped when decoding the parameter names and values + themselves. + """ + + def __init__(self, url: URL | str = "", **kwargs: typing.Any) -> None: + if kwargs: + allowed = { + "scheme": str, + "username": str, + "password": str, + "userinfo": bytes, + "host": str, + "port": int, + "netloc": bytes, + "path": str, + "query": bytes, + "raw_path": bytes, + "fragment": str, + "params": object, + } + + # Perform type checking for all supported keyword arguments. + for key, value in kwargs.items(): + if key not in allowed: + message = f"{key!r} is an invalid keyword argument for URL()" + raise TypeError(message) + if value is not None and not isinstance(value, allowed[key]): + expected = allowed[key].__name__ + seen = type(value).__name__ + message = f"Argument {key!r} must be {expected} but got {seen}" + raise TypeError(message) + if isinstance(value, bytes): + kwargs[key] = value.decode("ascii") + + if "params" in kwargs: + # Replace any "params" keyword with the raw "query" instead. + # + # Ensure that empty params use `kwargs["query"] = None` rather + # than `kwargs["query"] = ""`, so that generated URLs do not + # include an empty trailing "?". + params = kwargs.pop("params") + kwargs["query"] = None if not params else str(QueryParams(params)) + + if isinstance(url, str): + self._uri_reference = urlparse(url, **kwargs) + elif isinstance(url, URL): + self._uri_reference = url._uri_reference.copy_with(**kwargs) + else: + raise TypeError( + "Invalid type for url. Expected str or httpx.URL," + f" got {type(url)}: {url!r}" + ) + + @property + def scheme(self) -> str: + """ + The URL scheme, such as "http", "https". + Always normalised to lowercase. + """ + return self._uri_reference.scheme + + @property + def raw_scheme(self) -> bytes: + """ + The raw bytes representation of the URL scheme, such as b"http", b"https". + Always normalised to lowercase. + """ + return self._uri_reference.scheme.encode("ascii") + + @property + def userinfo(self) -> bytes: + """ + The URL userinfo as a raw bytestring. + For example: b"jo%40email.com:a%20secret". + """ + return self._uri_reference.userinfo.encode("ascii") + + @property + def username(self) -> str: + """ + The URL username as a string, with URL decoding applied. + For example: "jo@email.com" + """ + userinfo = self._uri_reference.userinfo + return unquote(userinfo.partition(":")[0]) + + @property + def password(self) -> str: + """ + The URL password as a string, with URL decoding applied. + For example: "a secret" + """ + userinfo = self._uri_reference.userinfo + return unquote(userinfo.partition(":")[2]) + + @property + def host(self) -> str: + """ + The URL host as a string. + Always normalized to lowercase, with IDNA hosts decoded into unicode. + + Examples: + + url = httpx.URL("http://www.EXAMPLE.org") + assert url.host == "www.example.org" + + url = httpx.URL("http://中国.icom.museum") + assert url.host == "中国.icom.museum" + + url = httpx.URL("http://xn--fiqs8s.icom.museum") + assert url.host == "中国.icom.museum" + + url = httpx.URL("https://[::ffff:192.168.0.1]") + assert url.host == "::ffff:192.168.0.1" + """ + host: str = self._uri_reference.host + + if host.startswith("xn--"): + host = idna.decode(host) + + return host + + @property + def raw_host(self) -> bytes: + """ + The raw bytes representation of the URL host. + Always normalized to lowercase, and IDNA encoded. + + Examples: + + url = httpx.URL("http://www.EXAMPLE.org") + assert url.raw_host == b"www.example.org" + + url = httpx.URL("http://中国.icom.museum") + assert url.raw_host == b"xn--fiqs8s.icom.museum" + + url = httpx.URL("http://xn--fiqs8s.icom.museum") + assert url.raw_host == b"xn--fiqs8s.icom.museum" + + url = httpx.URL("https://[::ffff:192.168.0.1]") + assert url.raw_host == b"::ffff:192.168.0.1" + """ + return self._uri_reference.host.encode("ascii") + + @property + def port(self) -> int | None: + """ + The URL port as an integer. + + Note that the URL class performs port normalization as per the WHATWG spec. + Default ports for "http", "https", "ws", "wss", and "ftp" schemes are always + treated as `None`. + + For example: + + assert httpx.URL("http://www.example.com") == httpx.URL("http://www.example.com:80") + assert httpx.URL("http://www.example.com:80").port is None + """ + return self._uri_reference.port + + @property + def netloc(self) -> bytes: + """ + Either `` or `:` as bytes. + Always normalized to lowercase, and IDNA encoded. + + This property may be used for generating the value of a request + "Host" header. + """ + return self._uri_reference.netloc.encode("ascii") + + @property + def path(self) -> str: + """ + The URL path as a string. Excluding the query string, and URL decoded. + + For example: + + url = httpx.URL("https://example.com/pa%20th") + assert url.path == "/pa th" + """ + path = self._uri_reference.path or "/" + return unquote(path) + + @property + def query(self) -> bytes: + """ + The URL query string, as raw bytes, excluding the leading b"?". + + This is necessarily a bytewise interface, because we cannot + perform URL decoding of this representation until we've parsed + the keys and values into a QueryParams instance. + + For example: + + url = httpx.URL("https://example.com/?filter=some%20search%20terms") + assert url.query == b"filter=some%20search%20terms" + """ + query = self._uri_reference.query or "" + return query.encode("ascii") + + @property + def params(self) -> QueryParams: + """ + The URL query parameters, neatly parsed and packaged into an immutable + multidict representation. + """ + return QueryParams(self._uri_reference.query) + + @property + def raw_path(self) -> bytes: + """ + The complete URL path and query string as raw bytes. + Used as the target when constructing HTTP requests. + + For example: + + GET /users?search=some%20text HTTP/1.1 + Host: www.example.org + Connection: close + """ + path = self._uri_reference.path or "/" + if self._uri_reference.query is not None: + path += "?" + self._uri_reference.query + return path.encode("ascii") + + @property + def fragment(self) -> str: + """ + The URL fragments, as used in HTML anchors. + As a string, without the leading '#'. + """ + return unquote(self._uri_reference.fragment or "") + + @property + def is_absolute_url(self) -> bool: + """ + Return `True` for absolute URLs such as 'http://example.com/path', + and `False` for relative URLs such as '/path'. + """ + # We don't use `.is_absolute` from `rfc3986` because it treats + # URLs with a fragment portion as not absolute. + # What we actually care about is if the URL provides + # a scheme and hostname to which connections should be made. + return bool(self._uri_reference.scheme and self._uri_reference.host) + + @property + def is_relative_url(self) -> bool: + """ + Return `False` for absolute URLs such as 'http://example.com/path', + and `True` for relative URLs such as '/path'. + """ + return not self.is_absolute_url + + def copy_with(self, **kwargs: typing.Any) -> URL: + """ + Copy this URL, returning a new URL with some components altered. + Accepts the same set of parameters as the components that are made + available via properties on the `URL` class. + + For example: + + url = httpx.URL("https://www.example.com").copy_with( + username="jo@gmail.com", password="a secret" + ) + assert url == "https://jo%40email.com:a%20secret@www.example.com" + """ + return URL(self, **kwargs) + + def copy_set_param(self, key: str, value: typing.Any = None) -> URL: + return self.copy_with(params=self.params.set(key, value)) + + def copy_add_param(self, key: str, value: typing.Any = None) -> URL: + return self.copy_with(params=self.params.add(key, value)) + + def copy_remove_param(self, key: str) -> URL: + return self.copy_with(params=self.params.remove(key)) + + def copy_merge_params(self, params: QueryParamTypes) -> URL: + return self.copy_with(params=self.params.merge(params)) + + def join(self, url: URL | str) -> URL: + """ + Return an absolute URL, using this URL as the base. + + Eg. + + url = httpx.URL("https://www.example.com/test") + url = url.join("/new/path") + assert url == "https://www.example.com/new/path" + """ + from urllib.parse import urljoin + + return URL(urljoin(str(self), str(URL(url)))) + + def __hash__(self) -> int: + return hash(str(self)) + + def __eq__(self, other: typing.Any) -> bool: + return isinstance(other, (URL, str)) and str(self) == str(URL(other)) + + def __str__(self) -> str: + return str(self._uri_reference) + + def __repr__(self) -> str: + scheme, userinfo, host, port, path, query, fragment = self._uri_reference + + if ":" in userinfo: + # Mask any password component. + userinfo = f'{userinfo.split(":")[0]}:[secure]' + + authority = "".join( + [ + f"{userinfo}@" if userinfo else "", + f"[{host}]" if ":" in host else host, + f":{port}" if port is not None else "", + ] + ) + url = "".join( + [ + f"{self.scheme}:" if scheme else "", + f"//{authority}" if authority else "", + path, + f"?{query}" if query is not None else "", + f"#{fragment}" if fragment is not None else "", + ] + ) + + return f"{self.__class__.__name__}({url!r})" + + @property + def raw(self) -> tuple[bytes, bytes, int, bytes]: # pragma: nocover + import collections + import warnings + + warnings.warn("URL.raw is deprecated.") + RawURL = collections.namedtuple( + "RawURL", ["raw_scheme", "raw_host", "port", "raw_path"] + ) + return RawURL( + raw_scheme=self.raw_scheme, + raw_host=self.raw_host, + port=self.port, + raw_path=self.raw_path, + ) + + +class QueryParams(typing.Mapping[str, str]): + """ + URL query parameters, as a multi-dict. + """ + + def __init__(self, *args: QueryParamTypes | None, **kwargs: typing.Any) -> None: + assert len(args) < 2, "Too many arguments." + assert not (args and kwargs), "Cannot mix named and unnamed arguments." + + value = args[0] if args else kwargs + + if value is None or isinstance(value, (str, bytes)): + value = value.decode("ascii") if isinstance(value, bytes) else value + self._dict = parse_qs(value, keep_blank_values=True) + elif isinstance(value, QueryParams): + self._dict = {k: list(v) for k, v in value._dict.items()} + else: + dict_value: dict[typing.Any, list[typing.Any]] = {} + if isinstance(value, (list, tuple)): + # Convert list inputs like: + # [("a", "123"), ("a", "456"), ("b", "789")] + # To a dict representation, like: + # {"a": ["123", "456"], "b": ["789"]} + for item in value: + dict_value.setdefault(item[0], []).append(item[1]) + else: + # Convert dict inputs like: + # {"a": "123", "b": ["456", "789"]} + # To dict inputs where values are always lists, like: + # {"a": ["123"], "b": ["456", "789"]} + dict_value = { + k: list(v) if isinstance(v, (list, tuple)) else [v] + for k, v in value.items() + } + + # Ensure that keys and values are neatly coerced to strings. + # We coerce values `True` and `False` to JSON-like "true" and "false" + # representations, and coerce `None` values to the empty string. + self._dict = { + str(k): [primitive_value_to_str(item) for item in v] + for k, v in dict_value.items() + } + + def keys(self) -> typing.KeysView[str]: + """ + Return all the keys in the query params. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert list(q.keys()) == ["a", "b"] + """ + return self._dict.keys() + + def values(self) -> typing.ValuesView[str]: + """ + Return all the values in the query params. If a key occurs more than once + only the first item for that key is returned. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert list(q.values()) == ["123", "789"] + """ + return {k: v[0] for k, v in self._dict.items()}.values() + + def items(self) -> typing.ItemsView[str, str]: + """ + Return all items in the query params. If a key occurs more than once + only the first item for that key is returned. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert list(q.items()) == [("a", "123"), ("b", "789")] + """ + return {k: v[0] for k, v in self._dict.items()}.items() + + def multi_items(self) -> list[tuple[str, str]]: + """ + Return all items in the query params. Allow duplicate keys to occur. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert list(q.multi_items()) == [("a", "123"), ("a", "456"), ("b", "789")] + """ + multi_items: list[tuple[str, str]] = [] + for k, v in self._dict.items(): + multi_items.extend([(k, i) for i in v]) + return multi_items + + def get(self, key: typing.Any, default: typing.Any = None) -> typing.Any: + """ + Get a value from the query param for a given key. If the key occurs + more than once, then only the first value is returned. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert q.get("a") == "123" + """ + if key in self._dict: + return self._dict[str(key)][0] + return default + + def get_list(self, key: str) -> list[str]: + """ + Get all values from the query param for a given key. + + Usage: + + q = httpx.QueryParams("a=123&a=456&b=789") + assert q.get_list("a") == ["123", "456"] + """ + return list(self._dict.get(str(key), [])) + + def set(self, key: str, value: typing.Any = None) -> QueryParams: + """ + Return a new QueryParams instance, setting the value of a key. + + Usage: + + q = httpx.QueryParams("a=123") + q = q.set("a", "456") + assert q == httpx.QueryParams("a=456") + """ + q = QueryParams() + q._dict = dict(self._dict) + q._dict[str(key)] = [primitive_value_to_str(value)] + return q + + def add(self, key: str, value: typing.Any = None) -> QueryParams: + """ + Return a new QueryParams instance, setting or appending the value of a key. + + Usage: + + q = httpx.QueryParams("a=123") + q = q.add("a", "456") + assert q == httpx.QueryParams("a=123&a=456") + """ + q = QueryParams() + q._dict = dict(self._dict) + q._dict[str(key)] = q.get_list(key) + [primitive_value_to_str(value)] + return q + + def remove(self, key: str) -> QueryParams: + """ + Return a new QueryParams instance, removing the value of a key. + + Usage: + + q = httpx.QueryParams("a=123") + q = q.remove("a") + assert q == httpx.QueryParams("") + """ + q = QueryParams() + q._dict = dict(self._dict) + q._dict.pop(str(key), None) + return q + + def merge(self, params: QueryParamTypes | None = None) -> QueryParams: + """ + Return a new QueryParams instance, updated with. + + Usage: + + q = httpx.QueryParams("a=123") + q = q.merge({"b": "456"}) + assert q == httpx.QueryParams("a=123&b=456") + + q = httpx.QueryParams("a=123") + q = q.merge({"a": "456", "b": "789"}) + assert q == httpx.QueryParams("a=456&b=789") + """ + q = QueryParams(params) + q._dict = {**self._dict, **q._dict} + return q + + def __getitem__(self, key: typing.Any) -> str: + return self._dict[key][0] + + def __contains__(self, key: typing.Any) -> bool: + return key in self._dict + + def __iter__(self) -> typing.Iterator[typing.Any]: + return iter(self.keys()) + + def __len__(self) -> int: + return len(self._dict) + + def __bool__(self) -> bool: + return bool(self._dict) + + def __hash__(self) -> int: + return hash(str(self)) + + def __eq__(self, other: typing.Any) -> bool: + if not isinstance(other, self.__class__): + return False + return sorted(self.multi_items()) == sorted(other.multi_items()) + + def __str__(self) -> str: + return urlencode(self.multi_items()) + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + query_string = str(self) + return f"{class_name}({query_string!r})" + + def update(self, params: QueryParamTypes | None = None) -> None: + raise RuntimeError( + "QueryParams are immutable since 0.18.0. " + "Use `q = q.merge(...)` to create an updated copy." + ) + + def __setitem__(self, key: str, value: str) -> None: + raise RuntimeError( + "QueryParams are immutable since 0.18.0. " + "Use `q = q.set(key, value)` to create an updated copy." + ) diff --git a/venv/lib/python3.12/site-packages/httpx/_utils.py b/venv/lib/python3.12/site-packages/httpx/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7fe827da4d071b32ea6da44328629699d6fc88ce --- /dev/null +++ b/venv/lib/python3.12/site-packages/httpx/_utils.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import ipaddress +import os +import re +import typing +from urllib.request import getproxies + +from ._types import PrimitiveData + +if typing.TYPE_CHECKING: # pragma: no cover + from ._urls import URL + + +def primitive_value_to_str(value: PrimitiveData) -> str: + """ + Coerce a primitive data type into a string value. + + Note that we prefer JSON-style 'true'/'false' for boolean values here. + """ + if value is True: + return "true" + elif value is False: + return "false" + elif value is None: + return "" + return str(value) + + +def get_environment_proxies() -> dict[str, str | None]: + """Gets proxy information from the environment""" + + # urllib.request.getproxies() falls back on System + # Registry and Config for proxies on Windows and macOS. + # We don't want to propagate non-HTTP proxies into + # our configuration such as 'TRAVIS_APT_PROXY'. + proxy_info = getproxies() + mounts: dict[str, str | None] = {} + + for scheme in ("http", "https", "all"): + if proxy_info.get(scheme): + hostname = proxy_info[scheme] + mounts[f"{scheme}://"] = ( + hostname if "://" in hostname else f"http://{hostname}" + ) + + no_proxy_hosts = [host.strip() for host in proxy_info.get("no", "").split(",")] + for hostname in no_proxy_hosts: + # See https://curl.haxx.se/libcurl/c/CURLOPT_NOPROXY.html for details + # on how names in `NO_PROXY` are handled. + if hostname == "*": + # If NO_PROXY=* is used or if "*" occurs as any one of the comma + # separated hostnames, then we should just bypass any information + # from HTTP_PROXY, HTTPS_PROXY, ALL_PROXY, and always ignore + # proxies. + return {} + elif hostname: + # NO_PROXY=.google.com is marked as "all://*.google.com, + # which disables "www.google.com" but not "google.com" + # NO_PROXY=google.com is marked as "all://*google.com, + # which disables "www.google.com" and "google.com". + # (But not "wwwgoogle.com") + # NO_PROXY can include domains, IPv6, IPv4 addresses and "localhost" + # NO_PROXY=example.com,::1,localhost,192.168.0.0/16 + if "://" in hostname: + mounts[hostname] = None + elif is_ipv4_hostname(hostname): + mounts[f"all://{hostname}"] = None + elif is_ipv6_hostname(hostname): + mounts[f"all://[{hostname}]"] = None + elif hostname.lower() == "localhost": + mounts[f"all://{hostname}"] = None + else: + mounts[f"all://*{hostname}"] = None + + return mounts + + +def to_bytes(value: str | bytes, encoding: str = "utf-8") -> bytes: + return value.encode(encoding) if isinstance(value, str) else value + + +def to_str(value: str | bytes, encoding: str = "utf-8") -> str: + return value if isinstance(value, str) else value.decode(encoding) + + +def to_bytes_or_str(value: str, match_type_of: typing.AnyStr) -> typing.AnyStr: + return value if isinstance(match_type_of, str) else value.encode() + + +def unquote(value: str) -> str: + return value[1:-1] if value[0] == value[-1] == '"' else value + + +def peek_filelike_length(stream: typing.Any) -> int | None: + """ + Given a file-like stream object, return its length in number of bytes + without reading it into memory. + """ + try: + # Is it an actual file? + fd = stream.fileno() + # Yup, seems to be an actual file. + length = os.fstat(fd).st_size + except (AttributeError, OSError): + # No... Maybe it's something that supports random access, like `io.BytesIO`? + try: + # Assuming so, go to end of stream to figure out its length, + # then put it back in place. + offset = stream.tell() + length = stream.seek(0, os.SEEK_END) + stream.seek(offset) + except (AttributeError, OSError): + # Not even that? Sorry, we're doomed... + return None + + return length + + +class URLPattern: + """ + A utility class currently used for making lookups against proxy keys... + + # Wildcard matching... + >>> pattern = URLPattern("all://") + >>> pattern.matches(httpx.URL("http://example.com")) + True + + # Witch scheme matching... + >>> pattern = URLPattern("https://") + >>> pattern.matches(httpx.URL("https://example.com")) + True + >>> pattern.matches(httpx.URL("http://example.com")) + False + + # With domain matching... + >>> pattern = URLPattern("https://example.com") + >>> pattern.matches(httpx.URL("https://example.com")) + True + >>> pattern.matches(httpx.URL("http://example.com")) + False + >>> pattern.matches(httpx.URL("https://other.com")) + False + + # Wildcard scheme, with domain matching... + >>> pattern = URLPattern("all://example.com") + >>> pattern.matches(httpx.URL("https://example.com")) + True + >>> pattern.matches(httpx.URL("http://example.com")) + True + >>> pattern.matches(httpx.URL("https://other.com")) + False + + # With port matching... + >>> pattern = URLPattern("https://example.com:1234") + >>> pattern.matches(httpx.URL("https://example.com:1234")) + True + >>> pattern.matches(httpx.URL("https://example.com")) + False + """ + + def __init__(self, pattern: str) -> None: + from ._urls import URL + + if pattern and ":" not in pattern: + raise ValueError( + f"Proxy keys should use proper URL forms rather " + f"than plain scheme strings. " + f'Instead of "{pattern}", use "{pattern}://"' + ) + + url = URL(pattern) + self.pattern = pattern + self.scheme = "" if url.scheme == "all" else url.scheme + self.host = "" if url.host == "*" else url.host + self.port = url.port + if not url.host or url.host == "*": + self.host_regex: typing.Pattern[str] | None = None + elif url.host.startswith("*."): + # *.example.com should match "www.example.com", but not "example.com" + domain = re.escape(url.host[2:]) + self.host_regex = re.compile(f"^.+\\.{domain}$") + elif url.host.startswith("*"): + # *example.com should match "www.example.com" and "example.com" + domain = re.escape(url.host[1:]) + self.host_regex = re.compile(f"^(.+\\.)?{domain}$") + else: + # example.com should match "example.com" but not "www.example.com" + domain = re.escape(url.host) + self.host_regex = re.compile(f"^{domain}$") + + def matches(self, other: URL) -> bool: + if self.scheme and self.scheme != other.scheme: + return False + if ( + self.host + and self.host_regex is not None + and not self.host_regex.match(other.host) + ): + return False + if self.port is not None and self.port != other.port: + return False + return True + + @property + def priority(self) -> tuple[int, int, int]: + """ + The priority allows URLPattern instances to be sortable, so that + we can match from most specific to least specific. + """ + # URLs with a port should take priority over URLs without a port. + port_priority = 0 if self.port is not None else 1 + # Longer hostnames should match first. + host_priority = -len(self.host) + # Longer schemes should match first. + scheme_priority = -len(self.scheme) + return (port_priority, host_priority, scheme_priority) + + def __hash__(self) -> int: + return hash(self.pattern) + + def __lt__(self, other: URLPattern) -> bool: + return self.priority < other.priority + + def __eq__(self, other: typing.Any) -> bool: + return isinstance(other, URLPattern) and self.pattern == other.pattern + + +def is_ipv4_hostname(hostname: str) -> bool: + try: + ipaddress.IPv4Address(hostname.split("/")[0]) + except Exception: + return False + return True + + +def is_ipv6_hostname(hostname: str) -> bool: + try: + ipaddress.IPv6Address(hostname.split("/")[0]) + except Exception: + return False + return True diff --git a/venv/lib/python3.12/site-packages/httpx/py.typed b/venv/lib/python3.12/site-packages/httpx/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__init__.py b/venv/lib/python3.12/site-packages/huggingface_hub/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..051ff4cf6353fb26e1ca2bf24545e7b9ac461ce0 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/__init__.py @@ -0,0 +1,1804 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# *********** +# `huggingface_hub` init has 2 modes: +# - Normal usage: +# If imported to use it, all modules and functions are lazy-loaded. This means +# they exist at top level in module but are imported only the first time they are +# used. This way, `from huggingface_hub import something` will import `something` +# quickly without the hassle of importing all the features from `huggingface_hub`. +# - Static check: +# If statically analyzed, all modules and functions are loaded normally. This way +# static typing check works properly as well as autocomplete in text editors and +# IDEs. +# +# The static model imports are done inside the `if TYPE_CHECKING:` statement at +# the bottom of this file. Since module/functions imports are duplicated, it is +# mandatory to make sure to add them twice when adding one. This is checked in the +# `make quality` command. +# +# To update the static imports, please run the following command and commit the changes. +# ``` +# # Use script +# python utils/check_static_imports.py --update-file +# +# # Or run style on codebase +# make style +# ``` +# +# *********** +# Lazy loader vendored from https://github.com/scientific-python/lazy_loader +import importlib +import os +import sys +from typing import TYPE_CHECKING + + +__version__ = "1.25.1" + +# Alphabetical order of definitions is ensured in tests +# WARNING: any comment added in this dictionary definition will be lost when +# re-generating the file ! +_SUBMOD_ATTRS = { + "_buckets": [ + "BucketFile", + "BucketFileMetadata", + "BucketFolder", + "BucketInfo", + "BucketUrl", + "SyncOperation", + "SyncPlan", + ], + "_commit_scheduler": [ + "CommitScheduler", + ], + "_eval_results": [ + "EvalResultEntry", + "eval_result_entries_to_yaml", + "parse_eval_result_entries", + ], + "_inference_endpoints": [ + "InferenceEndpoint", + "InferenceEndpointError", + "InferenceEndpointStatus", + "InferenceEndpointTimeoutError", + "InferenceEndpointType", + ], + "_jobs_api": [ + "JobAccelerator", + "JobDurations", + "JobHardware", + "JobHardwareInfo", + "JobInfo", + "JobInitiator", + "JobOwner", + "JobStage", + "JobStatus", + ], + "_login": [ + "auth_list", + "auth_switch", + "interpreter_login", + "login", + "logout", + "notebook_login", + ], + "_oauth": [ + "OAuthInfo", + "OAuthOrgInfo", + "OAuthUserInfo", + "attach_huggingface_oauth", + "parse_huggingface_oauth", + ], + "_sandbox": [ + "Sandbox", + "SandboxCommandResult", + "SandboxPool", + "SandboxProcess", + ], + "_snapshot_download": [ + "get_cached_repo_tree", + "snapshot_download", + ], + "_space_api": [ + "SpaceHardware", + "SpaceRuntime", + "SpaceSecret", + "SpaceStage", + "SpaceStorage", + "SpaceVariable", + "Volume", + ], + "_tensorboard_logger": [ + "HFSummaryWriter", + ], + "_webhooks_payload": [ + "WebhookPayload", + "WebhookPayloadComment", + "WebhookPayloadDiscussion", + "WebhookPayloadDiscussionChanges", + "WebhookPayloadEvent", + "WebhookPayloadMovedTo", + "WebhookPayloadRepo", + "WebhookPayloadUrl", + "WebhookPayloadWebhook", + ], + "_webhooks_server": [ + "WebhooksServer", + "webhook_endpoint", + ], + "cli._cli_utils": [ + "check_cli_update", + "typer_factory", + ], + "community": [ + "Discussion", + "DiscussionComment", + "DiscussionCommit", + "DiscussionEvent", + "DiscussionStatusChange", + "DiscussionTitleChange", + "DiscussionWithDetails", + ], + "constants": [ + "CONFIG_NAME", + "FLAX_WEIGHTS_NAME", + "HUGGINGFACE_CO_URL_HOME", + "HUGGINGFACE_CO_URL_TEMPLATE", + "PYTORCH_WEIGHTS_NAME", + "REPO_TYPE_DATASET", + "REPO_TYPE_MODEL", + "REPO_TYPE_SPACE", + "TF2_WEIGHTS_NAME", + "TF_WEIGHTS_NAME", + "is_offline_mode", + ], + "fastai_utils": [ + "_save_pretrained_fastai", + "from_pretrained_fastai", + "push_to_hub_fastai", + ], + "file_download": [ + "DryRunFileInfo", + "HfFileMetadata", + "_CACHED_NO_EXIST", + "get_hf_file_metadata", + "hf_hub_download", + "hf_hub_url", + "try_to_load_from_cache", + ], + "hf_api": [ + "Collection", + "CollectionItem", + "CommitInfo", + "CommitOperation", + "CommitOperationAdd", + "CommitOperationCopy", + "CommitOperationDelete", + "DatasetInfo", + "DatasetLeaderboardEntry", + "GitCommitInfo", + "GitRefInfo", + "GitRefs", + "HfApi", + "KernelInfo", + "ModelInfo", + "Organization", + "RepoFile", + "RepoFolder", + "RepoStorageInfo", + "RepoUrl", + "SpaceInfo", + "SpaceSearchResult", + "SpaceTemplate", + "User", + "UserLikes", + "WebhookInfo", + "WebhookWatchedItem", + "accept_access_request", + "add_collection_item", + "add_space_secret", + "add_space_variable", + "auth_check", + "batch_bucket_files", + "bucket_info", + "cancel_access_request", + "cancel_job", + "change_discussion_status", + "comment_discussion", + "copy_files", + "create_branch", + "create_bucket", + "create_collection", + "create_commit", + "create_discussion", + "create_inference_endpoint", + "create_inference_endpoint_from_catalog", + "create_pull_request", + "create_repo", + "create_scheduled_job", + "create_scheduled_uv_job", + "create_tag", + "create_webhook", + "dataset_info", + "delete_branch", + "delete_bucket", + "delete_collection", + "delete_collection_item", + "delete_file", + "delete_folder", + "delete_inference_endpoint", + "delete_repo", + "delete_scheduled_job", + "delete_space_secret", + "delete_space_storage", + "delete_space_variable", + "delete_space_volumes", + "delete_tag", + "delete_webhook", + "disable_space_dev_mode", + "disable_webhook", + "download_bucket_files", + "duplicate_repo", + "duplicate_space", + "edit_discussion_comment", + "enable_space_dev_mode", + "enable_webhook", + "fetch_job_logs", + "fetch_job_metrics", + "fetch_space_logs", + "file_exists", + "get_bucket_file_metadata", + "get_bucket_paths_info", + "get_collection", + "get_dataset_leaderboard", + "get_dataset_tags", + "get_discussion_details", + "get_full_repo_name", + "get_inference_endpoint", + "get_local_safetensors_metadata", + "get_model_tags", + "get_organization_overview", + "get_paths_info", + "get_repo_discussions", + "get_safetensors_metadata", + "get_space_runtime", + "get_space_secrets", + "get_space_variables", + "get_user_overview", + "get_webhook", + "grant_access", + "inspect_job", + "inspect_scheduled_job", + "kernel_info", + "list_accepted_access_requests", + "list_bucket_tree", + "list_buckets", + "list_collections", + "list_daily_papers", + "list_dataset_parquet_files", + "list_datasets", + "list_inference_catalog", + "list_inference_endpoints", + "list_jobs", + "list_jobs_hardware", + "list_lfs_files", + "list_liked_repos", + "list_models", + "list_organization_followers", + "list_organization_members", + "list_papers", + "list_pending_access_requests", + "list_rejected_access_requests", + "list_repo_commits", + "list_repo_files", + "list_repo_likers", + "list_repo_refs", + "list_repo_tree", + "list_space_templates", + "list_spaces", + "list_spaces_hardware", + "list_user_followers", + "list_user_following", + "list_user_repos", + "list_webhooks", + "merge_pull_request", + "model_info", + "move_bucket", + "move_repo", + "paper_info", + "parse_local_safetensors_file_metadata", + "parse_safetensors_file_metadata", + "pause_inference_endpoint", + "pause_space", + "permanently_delete_lfs_files", + "preupload_lfs_files", + "read_paper", + "reject_access_request", + "rename_discussion", + "repo_exists", + "repo_info", + "repo_type_and_id_from_hf_id", + "request_space_hardware", + "request_space_storage", + "restart_space", + "resume_inference_endpoint", + "resume_scheduled_job", + "revision_exists", + "run_as_future", + "run_job", + "run_uv_job", + "scale_to_zero_inference_endpoint", + "search_spaces", + "set_space_sleep_time", + "set_space_volumes", + "space_info", + "super_squash_history", + "suspend_scheduled_job", + "sync_bucket", + "sync_job_volume", + "trigger_scheduled_job", + "unlike", + "update_collection_item", + "update_collection_metadata", + "update_inference_endpoint", + "update_job_labels", + "update_repo_settings", + "update_scheduled_job_labels", + "update_webhook", + "upload_file", + "upload_folder", + "upload_large_folder", + "verify_repo_checksums", + "wait_for_job", + "wait_for_space", + "whoami", + ], + "hf_file_system": [ + "HfFileSystem", + "HfFileSystemFile", + "HfFileSystemResolvedPath", + "HfFileSystemStreamFile", + "hffs", + ], + "hub_mixin": [ + "ModelHubMixin", + "PyTorchModelHubMixin", + ], + "inference._client": [ + "InferenceClient", + "InferenceTimeoutError", + ], + "inference._generated._async_client": [ + "AsyncInferenceClient", + ], + "inference._generated.types": [ + "AudioClassificationInput", + "AudioClassificationOutputElement", + "AudioClassificationOutputTransform", + "AudioClassificationParameters", + "AudioToAudioInput", + "AudioToAudioOutputElement", + "AutomaticSpeechRecognitionEarlyStoppingEnum", + "AutomaticSpeechRecognitionGenerationParameters", + "AutomaticSpeechRecognitionInput", + "AutomaticSpeechRecognitionOutput", + "AutomaticSpeechRecognitionOutputChunk", + "AutomaticSpeechRecognitionParameters", + "ChatCompletionInput", + "ChatCompletionInputFunctionDefinition", + "ChatCompletionInputFunctionName", + "ChatCompletionInputGrammarType", + "ChatCompletionInputJSONSchema", + "ChatCompletionInputMessage", + "ChatCompletionInputMessageChunk", + "ChatCompletionInputMessageChunkType", + "ChatCompletionInputResponseFormatJSONObject", + "ChatCompletionInputResponseFormatJSONSchema", + "ChatCompletionInputResponseFormatText", + "ChatCompletionInputStreamOptions", + "ChatCompletionInputTool", + "ChatCompletionInputToolCall", + "ChatCompletionInputToolChoiceClass", + "ChatCompletionInputToolChoiceEnum", + "ChatCompletionInputURL", + "ChatCompletionOutput", + "ChatCompletionOutputComplete", + "ChatCompletionOutputFunctionDefinition", + "ChatCompletionOutputLogprob", + "ChatCompletionOutputLogprobs", + "ChatCompletionOutputMessage", + "ChatCompletionOutputToolCall", + "ChatCompletionOutputTopLogprob", + "ChatCompletionOutputUsage", + "ChatCompletionStreamOutput", + "ChatCompletionStreamOutputChoice", + "ChatCompletionStreamOutputDelta", + "ChatCompletionStreamOutputDeltaToolCall", + "ChatCompletionStreamOutputFunction", + "ChatCompletionStreamOutputLogprob", + "ChatCompletionStreamOutputLogprobs", + "ChatCompletionStreamOutputTopLogprob", + "ChatCompletionStreamOutputUsage", + "DepthEstimationInput", + "DepthEstimationOutput", + "DocumentQuestionAnsweringInput", + "DocumentQuestionAnsweringInputData", + "DocumentQuestionAnsweringOutputElement", + "DocumentQuestionAnsweringParameters", + "FeatureExtractionInput", + "FeatureExtractionInputTruncationDirection", + "FillMaskInput", + "FillMaskOutputElement", + "FillMaskParameters", + "ImageClassificationInput", + "ImageClassificationOutputElement", + "ImageClassificationOutputTransform", + "ImageClassificationParameters", + "ImageSegmentationInput", + "ImageSegmentationOutputElement", + "ImageSegmentationParameters", + "ImageSegmentationSubtask", + "ImageTextToImageInput", + "ImageTextToImageOutput", + "ImageTextToImageParameters", + "ImageTextToImageTargetSize", + "ImageTextToVideoInput", + "ImageTextToVideoOutput", + "ImageTextToVideoParameters", + "ImageTextToVideoTargetSize", + "ImageToImageInput", + "ImageToImageOutput", + "ImageToImageParameters", + "ImageToImageTargetSize", + "ImageToTextEarlyStoppingEnum", + "ImageToTextGenerationParameters", + "ImageToTextInput", + "ImageToTextOutput", + "ImageToTextParameters", + "ImageToVideoInput", + "ImageToVideoOutput", + "ImageToVideoParameters", + "ImageToVideoTargetSize", + "ObjectDetectionBoundingBox", + "ObjectDetectionInput", + "ObjectDetectionOutputElement", + "ObjectDetectionParameters", + "Padding", + "QuestionAnsweringInput", + "QuestionAnsweringInputData", + "QuestionAnsweringOutputElement", + "QuestionAnsweringParameters", + "SentenceSimilarityInput", + "SentenceSimilarityInputData", + "SummarizationInput", + "SummarizationOutput", + "SummarizationParameters", + "SummarizationTruncationStrategy", + "TableQuestionAnsweringInput", + "TableQuestionAnsweringInputData", + "TableQuestionAnsweringOutputElement", + "TableQuestionAnsweringParameters", + "Text2TextGenerationInput", + "Text2TextGenerationOutput", + "Text2TextGenerationParameters", + "Text2TextGenerationTruncationStrategy", + "TextClassificationInput", + "TextClassificationOutputElement", + "TextClassificationOutputTransform", + "TextClassificationParameters", + "TextGenerationInput", + "TextGenerationInputGenerateParameters", + "TextGenerationInputGrammarType", + "TextGenerationOutput", + "TextGenerationOutputBestOfSequence", + "TextGenerationOutputDetails", + "TextGenerationOutputFinishReason", + "TextGenerationOutputPrefillToken", + "TextGenerationOutputToken", + "TextGenerationStreamOutput", + "TextGenerationStreamOutputStreamDetails", + "TextGenerationStreamOutputToken", + "TextToAudioEarlyStoppingEnum", + "TextToAudioGenerationParameters", + "TextToAudioInput", + "TextToAudioOutput", + "TextToAudioParameters", + "TextToImageInput", + "TextToImageOutput", + "TextToImageParameters", + "TextToSpeechEarlyStoppingEnum", + "TextToSpeechGenerationParameters", + "TextToSpeechInput", + "TextToSpeechOutput", + "TextToSpeechParameters", + "TextToVideoInput", + "TextToVideoOutput", + "TextToVideoParameters", + "TokenClassificationAggregationStrategy", + "TokenClassificationInput", + "TokenClassificationOutputElement", + "TokenClassificationParameters", + "TranslationInput", + "TranslationOutput", + "TranslationParameters", + "TranslationTruncationStrategy", + "TypeEnum", + "VideoClassificationInput", + "VideoClassificationOutputElement", + "VideoClassificationOutputTransform", + "VideoClassificationParameters", + "VisualQuestionAnsweringInput", + "VisualQuestionAnsweringInputData", + "VisualQuestionAnsweringOutputElement", + "VisualQuestionAnsweringParameters", + "ZeroShotClassificationInput", + "ZeroShotClassificationOutputElement", + "ZeroShotClassificationParameters", + "ZeroShotImageClassificationInput", + "ZeroShotImageClassificationOutputElement", + "ZeroShotImageClassificationParameters", + "ZeroShotObjectDetectionBoundingBox", + "ZeroShotObjectDetectionInput", + "ZeroShotObjectDetectionOutputElement", + "ZeroShotObjectDetectionParameters", + ], + "inference._mcp.agent": [ + "Agent", + ], + "inference._mcp.mcp_client": [ + "MCPClient", + ], + "repocard": [ + "DatasetCard", + "ModelCard", + "RepoCard", + "SpaceCard", + "metadata_eval_result", + "metadata_load", + "metadata_save", + "metadata_update", + ], + "repocard_data": [ + "CardData", + "DatasetCardData", + "EvalResult", + "ModelCardData", + "SpaceCardData", + ], + "serialization": [ + "StateDictSplit", + "get_torch_storage_id", + "get_torch_storage_size", + "load_state_dict_from_file", + "load_torch_model", + "save_torch_model", + "save_torch_state_dict", + "split_state_dict_into_shards_factory", + "split_torch_state_dict_into_shards", + ], + "serialization._dduf": [ + "DDUFEntry", + "export_entries_as_dduf", + "export_folder_as_dduf", + "read_dduf_file", + ], + "utils": [ + "ASYNC_CLIENT_FACTORY_T", + "CLIENT_FACTORY_T", + "CacheNotFound", + "CachedFileInfo", + "CachedIncompleteFileInfo", + "CachedRepoInfo", + "CachedRevisionInfo", + "CorruptedCacheException", + "DeleteCacheStrategy", + "HFCacheInfo", + "HfUri", + "cached_assets_path", + "close_session", + "dump_environment_info", + "get_async_session", + "get_session", + "get_token", + "hf_raise_for_status", + "logging", + "parse_hf_uri", + "scan_cache_dir", + "set_async_client_factory", + "set_client_factory", + ], +} + +# WARNING: __all__ is generated automatically, Any manual edit will be lost when re-generating this file ! +# +# To update the static imports, please run the following command and commit the changes. +# ``` +# # Use script +# python utils/check_all_variable.py --update +# +# # Or run style on codebase +# make style +# ``` + +__all__ = [ + "ASYNC_CLIENT_FACTORY_T", + "Agent", + "AsyncInferenceClient", + "AudioClassificationInput", + "AudioClassificationOutputElement", + "AudioClassificationOutputTransform", + "AudioClassificationParameters", + "AudioToAudioInput", + "AudioToAudioOutputElement", + "AutomaticSpeechRecognitionEarlyStoppingEnum", + "AutomaticSpeechRecognitionGenerationParameters", + "AutomaticSpeechRecognitionInput", + "AutomaticSpeechRecognitionOutput", + "AutomaticSpeechRecognitionOutputChunk", + "AutomaticSpeechRecognitionParameters", + "BucketFile", + "BucketFileMetadata", + "BucketFolder", + "BucketInfo", + "BucketUrl", + "CLIENT_FACTORY_T", + "CONFIG_NAME", + "CacheNotFound", + "CachedFileInfo", + "CachedIncompleteFileInfo", + "CachedRepoInfo", + "CachedRevisionInfo", + "CardData", + "ChatCompletionInput", + "ChatCompletionInputFunctionDefinition", + "ChatCompletionInputFunctionName", + "ChatCompletionInputGrammarType", + "ChatCompletionInputJSONSchema", + "ChatCompletionInputMessage", + "ChatCompletionInputMessageChunk", + "ChatCompletionInputMessageChunkType", + "ChatCompletionInputResponseFormatJSONObject", + "ChatCompletionInputResponseFormatJSONSchema", + "ChatCompletionInputResponseFormatText", + "ChatCompletionInputStreamOptions", + "ChatCompletionInputTool", + "ChatCompletionInputToolCall", + "ChatCompletionInputToolChoiceClass", + "ChatCompletionInputToolChoiceEnum", + "ChatCompletionInputURL", + "ChatCompletionOutput", + "ChatCompletionOutputComplete", + "ChatCompletionOutputFunctionDefinition", + "ChatCompletionOutputLogprob", + "ChatCompletionOutputLogprobs", + "ChatCompletionOutputMessage", + "ChatCompletionOutputToolCall", + "ChatCompletionOutputTopLogprob", + "ChatCompletionOutputUsage", + "ChatCompletionStreamOutput", + "ChatCompletionStreamOutputChoice", + "ChatCompletionStreamOutputDelta", + "ChatCompletionStreamOutputDeltaToolCall", + "ChatCompletionStreamOutputFunction", + "ChatCompletionStreamOutputLogprob", + "ChatCompletionStreamOutputLogprobs", + "ChatCompletionStreamOutputTopLogprob", + "ChatCompletionStreamOutputUsage", + "Collection", + "CollectionItem", + "CommitInfo", + "CommitOperation", + "CommitOperationAdd", + "CommitOperationCopy", + "CommitOperationDelete", + "CommitScheduler", + "CorruptedCacheException", + "DDUFEntry", + "DatasetCard", + "DatasetCardData", + "DatasetInfo", + "DatasetLeaderboardEntry", + "DeleteCacheStrategy", + "DepthEstimationInput", + "DepthEstimationOutput", + "Discussion", + "DiscussionComment", + "DiscussionCommit", + "DiscussionEvent", + "DiscussionStatusChange", + "DiscussionTitleChange", + "DiscussionWithDetails", + "DocumentQuestionAnsweringInput", + "DocumentQuestionAnsweringInputData", + "DocumentQuestionAnsweringOutputElement", + "DocumentQuestionAnsweringParameters", + "DryRunFileInfo", + "EvalResult", + "EvalResultEntry", + "FLAX_WEIGHTS_NAME", + "FeatureExtractionInput", + "FeatureExtractionInputTruncationDirection", + "FillMaskInput", + "FillMaskOutputElement", + "FillMaskParameters", + "GitCommitInfo", + "GitRefInfo", + "GitRefs", + "HFCacheInfo", + "HFSummaryWriter", + "HUGGINGFACE_CO_URL_HOME", + "HUGGINGFACE_CO_URL_TEMPLATE", + "HfApi", + "HfFileMetadata", + "HfFileSystem", + "HfFileSystemFile", + "HfFileSystemResolvedPath", + "HfFileSystemStreamFile", + "HfUri", + "ImageClassificationInput", + "ImageClassificationOutputElement", + "ImageClassificationOutputTransform", + "ImageClassificationParameters", + "ImageSegmentationInput", + "ImageSegmentationOutputElement", + "ImageSegmentationParameters", + "ImageSegmentationSubtask", + "ImageTextToImageInput", + "ImageTextToImageOutput", + "ImageTextToImageParameters", + "ImageTextToImageTargetSize", + "ImageTextToVideoInput", + "ImageTextToVideoOutput", + "ImageTextToVideoParameters", + "ImageTextToVideoTargetSize", + "ImageToImageInput", + "ImageToImageOutput", + "ImageToImageParameters", + "ImageToImageTargetSize", + "ImageToTextEarlyStoppingEnum", + "ImageToTextGenerationParameters", + "ImageToTextInput", + "ImageToTextOutput", + "ImageToTextParameters", + "ImageToVideoInput", + "ImageToVideoOutput", + "ImageToVideoParameters", + "ImageToVideoTargetSize", + "InferenceClient", + "InferenceEndpoint", + "InferenceEndpointError", + "InferenceEndpointStatus", + "InferenceEndpointTimeoutError", + "InferenceEndpointType", + "InferenceTimeoutError", + "JobAccelerator", + "JobDurations", + "JobHardware", + "JobHardwareInfo", + "JobInfo", + "JobInitiator", + "JobOwner", + "JobStage", + "JobStatus", + "KernelInfo", + "MCPClient", + "ModelCard", + "ModelCardData", + "ModelHubMixin", + "ModelInfo", + "OAuthInfo", + "OAuthOrgInfo", + "OAuthUserInfo", + "ObjectDetectionBoundingBox", + "ObjectDetectionInput", + "ObjectDetectionOutputElement", + "ObjectDetectionParameters", + "Organization", + "PYTORCH_WEIGHTS_NAME", + "Padding", + "PyTorchModelHubMixin", + "QuestionAnsweringInput", + "QuestionAnsweringInputData", + "QuestionAnsweringOutputElement", + "QuestionAnsweringParameters", + "REPO_TYPE_DATASET", + "REPO_TYPE_MODEL", + "REPO_TYPE_SPACE", + "RepoCard", + "RepoFile", + "RepoFolder", + "RepoStorageInfo", + "RepoUrl", + "Sandbox", + "SandboxCommandResult", + "SandboxPool", + "SandboxProcess", + "SentenceSimilarityInput", + "SentenceSimilarityInputData", + "SpaceCard", + "SpaceCardData", + "SpaceHardware", + "SpaceInfo", + "SpaceRuntime", + "SpaceSearchResult", + "SpaceSecret", + "SpaceStage", + "SpaceStorage", + "SpaceTemplate", + "SpaceVariable", + "StateDictSplit", + "SummarizationInput", + "SummarizationOutput", + "SummarizationParameters", + "SummarizationTruncationStrategy", + "SyncOperation", + "SyncPlan", + "TF2_WEIGHTS_NAME", + "TF_WEIGHTS_NAME", + "TableQuestionAnsweringInput", + "TableQuestionAnsweringInputData", + "TableQuestionAnsweringOutputElement", + "TableQuestionAnsweringParameters", + "Text2TextGenerationInput", + "Text2TextGenerationOutput", + "Text2TextGenerationParameters", + "Text2TextGenerationTruncationStrategy", + "TextClassificationInput", + "TextClassificationOutputElement", + "TextClassificationOutputTransform", + "TextClassificationParameters", + "TextGenerationInput", + "TextGenerationInputGenerateParameters", + "TextGenerationInputGrammarType", + "TextGenerationOutput", + "TextGenerationOutputBestOfSequence", + "TextGenerationOutputDetails", + "TextGenerationOutputFinishReason", + "TextGenerationOutputPrefillToken", + "TextGenerationOutputToken", + "TextGenerationStreamOutput", + "TextGenerationStreamOutputStreamDetails", + "TextGenerationStreamOutputToken", + "TextToAudioEarlyStoppingEnum", + "TextToAudioGenerationParameters", + "TextToAudioInput", + "TextToAudioOutput", + "TextToAudioParameters", + "TextToImageInput", + "TextToImageOutput", + "TextToImageParameters", + "TextToSpeechEarlyStoppingEnum", + "TextToSpeechGenerationParameters", + "TextToSpeechInput", + "TextToSpeechOutput", + "TextToSpeechParameters", + "TextToVideoInput", + "TextToVideoOutput", + "TextToVideoParameters", + "TokenClassificationAggregationStrategy", + "TokenClassificationInput", + "TokenClassificationOutputElement", + "TokenClassificationParameters", + "TranslationInput", + "TranslationOutput", + "TranslationParameters", + "TranslationTruncationStrategy", + "TypeEnum", + "User", + "UserLikes", + "VideoClassificationInput", + "VideoClassificationOutputElement", + "VideoClassificationOutputTransform", + "VideoClassificationParameters", + "VisualQuestionAnsweringInput", + "VisualQuestionAnsweringInputData", + "VisualQuestionAnsweringOutputElement", + "VisualQuestionAnsweringParameters", + "Volume", + "WebhookInfo", + "WebhookPayload", + "WebhookPayloadComment", + "WebhookPayloadDiscussion", + "WebhookPayloadDiscussionChanges", + "WebhookPayloadEvent", + "WebhookPayloadMovedTo", + "WebhookPayloadRepo", + "WebhookPayloadUrl", + "WebhookPayloadWebhook", + "WebhookWatchedItem", + "WebhooksServer", + "ZeroShotClassificationInput", + "ZeroShotClassificationOutputElement", + "ZeroShotClassificationParameters", + "ZeroShotImageClassificationInput", + "ZeroShotImageClassificationOutputElement", + "ZeroShotImageClassificationParameters", + "ZeroShotObjectDetectionBoundingBox", + "ZeroShotObjectDetectionInput", + "ZeroShotObjectDetectionOutputElement", + "ZeroShotObjectDetectionParameters", + "_CACHED_NO_EXIST", + "_save_pretrained_fastai", + "accept_access_request", + "add_collection_item", + "add_space_secret", + "add_space_variable", + "attach_huggingface_oauth", + "auth_check", + "auth_list", + "auth_switch", + "batch_bucket_files", + "bucket_info", + "cached_assets_path", + "cancel_access_request", + "cancel_job", + "change_discussion_status", + "check_cli_update", + "close_session", + "comment_discussion", + "copy_files", + "create_branch", + "create_bucket", + "create_collection", + "create_commit", + "create_discussion", + "create_inference_endpoint", + "create_inference_endpoint_from_catalog", + "create_pull_request", + "create_repo", + "create_scheduled_job", + "create_scheduled_uv_job", + "create_tag", + "create_webhook", + "dataset_info", + "delete_branch", + "delete_bucket", + "delete_collection", + "delete_collection_item", + "delete_file", + "delete_folder", + "delete_inference_endpoint", + "delete_repo", + "delete_scheduled_job", + "delete_space_secret", + "delete_space_storage", + "delete_space_variable", + "delete_space_volumes", + "delete_tag", + "delete_webhook", + "disable_space_dev_mode", + "disable_webhook", + "download_bucket_files", + "dump_environment_info", + "duplicate_repo", + "duplicate_space", + "edit_discussion_comment", + "enable_space_dev_mode", + "enable_webhook", + "eval_result_entries_to_yaml", + "export_entries_as_dduf", + "export_folder_as_dduf", + "fetch_job_logs", + "fetch_job_metrics", + "fetch_space_logs", + "file_exists", + "from_pretrained_fastai", + "get_async_session", + "get_bucket_file_metadata", + "get_bucket_paths_info", + "get_cached_repo_tree", + "get_collection", + "get_dataset_leaderboard", + "get_dataset_tags", + "get_discussion_details", + "get_full_repo_name", + "get_hf_file_metadata", + "get_inference_endpoint", + "get_local_safetensors_metadata", + "get_model_tags", + "get_organization_overview", + "get_paths_info", + "get_repo_discussions", + "get_safetensors_metadata", + "get_session", + "get_space_runtime", + "get_space_secrets", + "get_space_variables", + "get_token", + "get_torch_storage_id", + "get_torch_storage_size", + "get_user_overview", + "get_webhook", + "grant_access", + "hf_hub_download", + "hf_hub_url", + "hf_raise_for_status", + "hffs", + "inspect_job", + "inspect_scheduled_job", + "interpreter_login", + "is_offline_mode", + "kernel_info", + "list_accepted_access_requests", + "list_bucket_tree", + "list_buckets", + "list_collections", + "list_daily_papers", + "list_dataset_parquet_files", + "list_datasets", + "list_inference_catalog", + "list_inference_endpoints", + "list_jobs", + "list_jobs_hardware", + "list_lfs_files", + "list_liked_repos", + "list_models", + "list_organization_followers", + "list_organization_members", + "list_papers", + "list_pending_access_requests", + "list_rejected_access_requests", + "list_repo_commits", + "list_repo_files", + "list_repo_likers", + "list_repo_refs", + "list_repo_tree", + "list_space_templates", + "list_spaces", + "list_spaces_hardware", + "list_user_followers", + "list_user_following", + "list_user_repos", + "list_webhooks", + "load_state_dict_from_file", + "load_torch_model", + "logging", + "login", + "logout", + "merge_pull_request", + "metadata_eval_result", + "metadata_load", + "metadata_save", + "metadata_update", + "model_info", + "move_bucket", + "move_repo", + "notebook_login", + "paper_info", + "parse_eval_result_entries", + "parse_hf_uri", + "parse_huggingface_oauth", + "parse_local_safetensors_file_metadata", + "parse_safetensors_file_metadata", + "pause_inference_endpoint", + "pause_space", + "permanently_delete_lfs_files", + "preupload_lfs_files", + "push_to_hub_fastai", + "read_dduf_file", + "read_paper", + "reject_access_request", + "rename_discussion", + "repo_exists", + "repo_info", + "repo_type_and_id_from_hf_id", + "request_space_hardware", + "request_space_storage", + "restart_space", + "resume_inference_endpoint", + "resume_scheduled_job", + "revision_exists", + "run_as_future", + "run_job", + "run_uv_job", + "save_torch_model", + "save_torch_state_dict", + "scale_to_zero_inference_endpoint", + "scan_cache_dir", + "search_spaces", + "set_async_client_factory", + "set_client_factory", + "set_space_sleep_time", + "set_space_volumes", + "snapshot_download", + "space_info", + "split_state_dict_into_shards_factory", + "split_torch_state_dict_into_shards", + "super_squash_history", + "suspend_scheduled_job", + "sync_bucket", + "sync_job_volume", + "trigger_scheduled_job", + "try_to_load_from_cache", + "typer_factory", + "unlike", + "update_collection_item", + "update_collection_metadata", + "update_inference_endpoint", + "update_job_labels", + "update_repo_settings", + "update_scheduled_job_labels", + "update_webhook", + "upload_file", + "upload_folder", + "upload_large_folder", + "verify_repo_checksums", + "wait_for_job", + "wait_for_space", + "webhook_endpoint", + "whoami", +] + + +def _attach(package_name, submodules=None, submod_attrs=None): + """Attach lazily loaded submodules, functions, or other attributes. + + Typically, modules import submodules and attributes as follows: + + ```py + import mysubmodule + import anothersubmodule + + from .foo import someattr + ``` + + The idea is to replace a package's `__getattr__`, `__dir__`, such that all imports + work exactly the way they would with normal imports, except that the import occurs + upon first use. + + The typical way to call this function, replacing the above imports, is: + + ```python + __getattr__, __dir__ = lazy.attach( + __name__, + ['mysubmodule', 'anothersubmodule'], + {'foo': ['someattr']} + ) + ``` + This functionality requires Python 3.7 or higher. + + Args: + package_name (`str`): + Typically use `__name__`. + submodules (`set`): + List of submodules to attach. + submod_attrs (`dict`): + Dictionary of submodule -> list of attributes / functions. + These attributes are imported as they are used. + + Returns: + __getattr__, __dir__, __all__ + + """ + if submod_attrs is None: + submod_attrs = {} + + if submodules is None: + submodules = set() + else: + submodules = set(submodules) + + attr_to_modules = {attr: mod for mod, attrs in submod_attrs.items() for attr in attrs} + + def __getattr__(name): + if name in submodules: + try: + return importlib.import_module(f"{package_name}.{name}") + except Exception as e: + print(f"Error importing {package_name}.{name}: {e}") + raise + elif name in attr_to_modules: + submod_path = f"{package_name}.{attr_to_modules[name]}" + try: + submod = importlib.import_module(submod_path) + except Exception as e: + print(f"Error importing {submod_path}: {e}") + raise + attr = getattr(submod, name) + + # If the attribute lives in a file (module) with the same + # name as the attribute, ensure that the attribute and *not* + # the module is accessible on the package. + if name == attr_to_modules[name]: + pkg = sys.modules[package_name] + pkg.__dict__[name] = attr + + return attr + else: + raise AttributeError(f"No {package_name} attribute {name}") + + def __dir__(): + return __all__ + + return __getattr__, __dir__ + + +__getattr__, __dir__ = _attach(__name__, submodules=[], submod_attrs=_SUBMOD_ATTRS) + +if os.environ.get("EAGER_IMPORT", ""): + for attr in __all__: + __getattr__(attr) + +# WARNING: any content below this statement is generated automatically. Any manual edit +# will be lost when re-generating this file ! +# +# To update the static imports, please run the following command and commit the changes. +# ``` +# # Use script +# python utils/check_static_imports.py --update +# +# # Or run style on codebase +# make style +# ``` +if TYPE_CHECKING: # pragma: no cover + from ._buckets import ( + BucketFile, # noqa: F401 + BucketFileMetadata, # noqa: F401 + BucketFolder, # noqa: F401 + BucketInfo, # noqa: F401 + BucketUrl, # noqa: F401 + SyncOperation, # noqa: F401 + SyncPlan, # noqa: F401 + ) + from ._commit_scheduler import CommitScheduler # noqa: F401 + from ._eval_results import ( + EvalResultEntry, # noqa: F401 + eval_result_entries_to_yaml, # noqa: F401 + parse_eval_result_entries, # noqa: F401 + ) + from ._inference_endpoints import ( + InferenceEndpoint, # noqa: F401 + InferenceEndpointError, # noqa: F401 + InferenceEndpointStatus, # noqa: F401 + InferenceEndpointTimeoutError, # noqa: F401 + InferenceEndpointType, # noqa: F401 + ) + from ._jobs_api import ( + JobAccelerator, # noqa: F401 + JobDurations, # noqa: F401 + JobHardware, # noqa: F401 + JobHardwareInfo, # noqa: F401 + JobInfo, # noqa: F401 + JobInitiator, # noqa: F401 + JobOwner, # noqa: F401 + JobStage, # noqa: F401 + JobStatus, # noqa: F401 + ) + from ._login import ( + auth_list, # noqa: F401 + auth_switch, # noqa: F401 + interpreter_login, # noqa: F401 + login, # noqa: F401 + logout, # noqa: F401 + notebook_login, # noqa: F401 + ) + from ._oauth import ( + OAuthInfo, # noqa: F401 + OAuthOrgInfo, # noqa: F401 + OAuthUserInfo, # noqa: F401 + attach_huggingface_oauth, # noqa: F401 + parse_huggingface_oauth, # noqa: F401 + ) + from ._sandbox import ( + Sandbox, # noqa: F401 + SandboxCommandResult, # noqa: F401 + SandboxPool, # noqa: F401 + SandboxProcess, # noqa: F401 + ) + from ._snapshot_download import ( + get_cached_repo_tree, # noqa: F401 + snapshot_download, # noqa: F401 + ) + from ._space_api import ( + SpaceHardware, # noqa: F401 + SpaceRuntime, # noqa: F401 + SpaceSecret, # noqa: F401 + SpaceStage, # noqa: F401 + SpaceStorage, # noqa: F401 + SpaceVariable, # noqa: F401 + Volume, # noqa: F401 + ) + from ._tensorboard_logger import HFSummaryWriter # noqa: F401 + from ._webhooks_payload import ( + WebhookPayload, # noqa: F401 + WebhookPayloadComment, # noqa: F401 + WebhookPayloadDiscussion, # noqa: F401 + WebhookPayloadDiscussionChanges, # noqa: F401 + WebhookPayloadEvent, # noqa: F401 + WebhookPayloadMovedTo, # noqa: F401 + WebhookPayloadRepo, # noqa: F401 + WebhookPayloadUrl, # noqa: F401 + WebhookPayloadWebhook, # noqa: F401 + ) + from ._webhooks_server import ( + WebhooksServer, # noqa: F401 + webhook_endpoint, # noqa: F401 + ) + from .cli._cli_utils import ( + check_cli_update, # noqa: F401 + typer_factory, # noqa: F401 + ) + from .community import ( + Discussion, # noqa: F401 + DiscussionComment, # noqa: F401 + DiscussionCommit, # noqa: F401 + DiscussionEvent, # noqa: F401 + DiscussionStatusChange, # noqa: F401 + DiscussionTitleChange, # noqa: F401 + DiscussionWithDetails, # noqa: F401 + ) + from .constants import ( + CONFIG_NAME, # noqa: F401 + FLAX_WEIGHTS_NAME, # noqa: F401 + HUGGINGFACE_CO_URL_HOME, # noqa: F401 + HUGGINGFACE_CO_URL_TEMPLATE, # noqa: F401 + PYTORCH_WEIGHTS_NAME, # noqa: F401 + REPO_TYPE_DATASET, # noqa: F401 + REPO_TYPE_MODEL, # noqa: F401 + REPO_TYPE_SPACE, # noqa: F401 + TF2_WEIGHTS_NAME, # noqa: F401 + TF_WEIGHTS_NAME, # noqa: F401 + is_offline_mode, # noqa: F401 + ) + from .fastai_utils import ( + _save_pretrained_fastai, # noqa: F401 + from_pretrained_fastai, # noqa: F401 + push_to_hub_fastai, # noqa: F401 + ) + from .file_download import ( + _CACHED_NO_EXIST, # noqa: F401 + DryRunFileInfo, # noqa: F401 + HfFileMetadata, # noqa: F401 + get_hf_file_metadata, # noqa: F401 + hf_hub_download, # noqa: F401 + hf_hub_url, # noqa: F401 + try_to_load_from_cache, # noqa: F401 + ) + from .hf_api import ( + Collection, # noqa: F401 + CollectionItem, # noqa: F401 + CommitInfo, # noqa: F401 + CommitOperation, # noqa: F401 + CommitOperationAdd, # noqa: F401 + CommitOperationCopy, # noqa: F401 + CommitOperationDelete, # noqa: F401 + DatasetInfo, # noqa: F401 + DatasetLeaderboardEntry, # noqa: F401 + GitCommitInfo, # noqa: F401 + GitRefInfo, # noqa: F401 + GitRefs, # noqa: F401 + HfApi, # noqa: F401 + KernelInfo, # noqa: F401 + ModelInfo, # noqa: F401 + Organization, # noqa: F401 + RepoFile, # noqa: F401 + RepoFolder, # noqa: F401 + RepoStorageInfo, # noqa: F401 + RepoUrl, # noqa: F401 + SpaceInfo, # noqa: F401 + SpaceSearchResult, # noqa: F401 + SpaceTemplate, # noqa: F401 + User, # noqa: F401 + UserLikes, # noqa: F401 + WebhookInfo, # noqa: F401 + WebhookWatchedItem, # noqa: F401 + accept_access_request, # noqa: F401 + add_collection_item, # noqa: F401 + add_space_secret, # noqa: F401 + add_space_variable, # noqa: F401 + auth_check, # noqa: F401 + batch_bucket_files, # noqa: F401 + bucket_info, # noqa: F401 + cancel_access_request, # noqa: F401 + cancel_job, # noqa: F401 + change_discussion_status, # noqa: F401 + comment_discussion, # noqa: F401 + copy_files, # noqa: F401 + create_branch, # noqa: F401 + create_bucket, # noqa: F401 + create_collection, # noqa: F401 + create_commit, # noqa: F401 + create_discussion, # noqa: F401 + create_inference_endpoint, # noqa: F401 + create_inference_endpoint_from_catalog, # noqa: F401 + create_pull_request, # noqa: F401 + create_repo, # noqa: F401 + create_scheduled_job, # noqa: F401 + create_scheduled_uv_job, # noqa: F401 + create_tag, # noqa: F401 + create_webhook, # noqa: F401 + dataset_info, # noqa: F401 + delete_branch, # noqa: F401 + delete_bucket, # noqa: F401 + delete_collection, # noqa: F401 + delete_collection_item, # noqa: F401 + delete_file, # noqa: F401 + delete_folder, # noqa: F401 + delete_inference_endpoint, # noqa: F401 + delete_repo, # noqa: F401 + delete_scheduled_job, # noqa: F401 + delete_space_secret, # noqa: F401 + delete_space_storage, # noqa: F401 + delete_space_variable, # noqa: F401 + delete_space_volumes, # noqa: F401 + delete_tag, # noqa: F401 + delete_webhook, # noqa: F401 + disable_space_dev_mode, # noqa: F401 + disable_webhook, # noqa: F401 + download_bucket_files, # noqa: F401 + duplicate_repo, # noqa: F401 + duplicate_space, # noqa: F401 + edit_discussion_comment, # noqa: F401 + enable_space_dev_mode, # noqa: F401 + enable_webhook, # noqa: F401 + fetch_job_logs, # noqa: F401 + fetch_job_metrics, # noqa: F401 + fetch_space_logs, # noqa: F401 + file_exists, # noqa: F401 + get_bucket_file_metadata, # noqa: F401 + get_bucket_paths_info, # noqa: F401 + get_collection, # noqa: F401 + get_dataset_leaderboard, # noqa: F401 + get_dataset_tags, # noqa: F401 + get_discussion_details, # noqa: F401 + get_full_repo_name, # noqa: F401 + get_inference_endpoint, # noqa: F401 + get_local_safetensors_metadata, # noqa: F401 + get_model_tags, # noqa: F401 + get_organization_overview, # noqa: F401 + get_paths_info, # noqa: F401 + get_repo_discussions, # noqa: F401 + get_safetensors_metadata, # noqa: F401 + get_space_runtime, # noqa: F401 + get_space_secrets, # noqa: F401 + get_space_variables, # noqa: F401 + get_user_overview, # noqa: F401 + get_webhook, # noqa: F401 + grant_access, # noqa: F401 + inspect_job, # noqa: F401 + inspect_scheduled_job, # noqa: F401 + kernel_info, # noqa: F401 + list_accepted_access_requests, # noqa: F401 + list_bucket_tree, # noqa: F401 + list_buckets, # noqa: F401 + list_collections, # noqa: F401 + list_daily_papers, # noqa: F401 + list_dataset_parquet_files, # noqa: F401 + list_datasets, # noqa: F401 + list_inference_catalog, # noqa: F401 + list_inference_endpoints, # noqa: F401 + list_jobs, # noqa: F401 + list_jobs_hardware, # noqa: F401 + list_lfs_files, # noqa: F401 + list_liked_repos, # noqa: F401 + list_models, # noqa: F401 + list_organization_followers, # noqa: F401 + list_organization_members, # noqa: F401 + list_papers, # noqa: F401 + list_pending_access_requests, # noqa: F401 + list_rejected_access_requests, # noqa: F401 + list_repo_commits, # noqa: F401 + list_repo_files, # noqa: F401 + list_repo_likers, # noqa: F401 + list_repo_refs, # noqa: F401 + list_repo_tree, # noqa: F401 + list_space_templates, # noqa: F401 + list_spaces, # noqa: F401 + list_spaces_hardware, # noqa: F401 + list_user_followers, # noqa: F401 + list_user_following, # noqa: F401 + list_user_repos, # noqa: F401 + list_webhooks, # noqa: F401 + merge_pull_request, # noqa: F401 + model_info, # noqa: F401 + move_bucket, # noqa: F401 + move_repo, # noqa: F401 + paper_info, # noqa: F401 + parse_local_safetensors_file_metadata, # noqa: F401 + parse_safetensors_file_metadata, # noqa: F401 + pause_inference_endpoint, # noqa: F401 + pause_space, # noqa: F401 + permanently_delete_lfs_files, # noqa: F401 + preupload_lfs_files, # noqa: F401 + read_paper, # noqa: F401 + reject_access_request, # noqa: F401 + rename_discussion, # noqa: F401 + repo_exists, # noqa: F401 + repo_info, # noqa: F401 + repo_type_and_id_from_hf_id, # noqa: F401 + request_space_hardware, # noqa: F401 + request_space_storage, # noqa: F401 + restart_space, # noqa: F401 + resume_inference_endpoint, # noqa: F401 + resume_scheduled_job, # noqa: F401 + revision_exists, # noqa: F401 + run_as_future, # noqa: F401 + run_job, # noqa: F401 + run_uv_job, # noqa: F401 + scale_to_zero_inference_endpoint, # noqa: F401 + search_spaces, # noqa: F401 + set_space_sleep_time, # noqa: F401 + set_space_volumes, # noqa: F401 + space_info, # noqa: F401 + super_squash_history, # noqa: F401 + suspend_scheduled_job, # noqa: F401 + sync_bucket, # noqa: F401 + sync_job_volume, # noqa: F401 + trigger_scheduled_job, # noqa: F401 + unlike, # noqa: F401 + update_collection_item, # noqa: F401 + update_collection_metadata, # noqa: F401 + update_inference_endpoint, # noqa: F401 + update_job_labels, # noqa: F401 + update_repo_settings, # noqa: F401 + update_scheduled_job_labels, # noqa: F401 + update_webhook, # noqa: F401 + upload_file, # noqa: F401 + upload_folder, # noqa: F401 + upload_large_folder, # noqa: F401 + verify_repo_checksums, # noqa: F401 + wait_for_job, # noqa: F401 + wait_for_space, # noqa: F401 + whoami, # noqa: F401 + ) + from .hf_file_system import ( + HfFileSystem, # noqa: F401 + HfFileSystemFile, # noqa: F401 + HfFileSystemResolvedPath, # noqa: F401 + HfFileSystemStreamFile, # noqa: F401 + hffs, # noqa: F401 + ) + from .hub_mixin import ( + ModelHubMixin, # noqa: F401 + PyTorchModelHubMixin, # noqa: F401 + ) + from .inference._client import ( + InferenceClient, # noqa: F401 + InferenceTimeoutError, # noqa: F401 + ) + from .inference._generated._async_client import AsyncInferenceClient # noqa: F401 + from .inference._generated.types import ( + AudioClassificationInput, # noqa: F401 + AudioClassificationOutputElement, # noqa: F401 + AudioClassificationOutputTransform, # noqa: F401 + AudioClassificationParameters, # noqa: F401 + AudioToAudioInput, # noqa: F401 + AudioToAudioOutputElement, # noqa: F401 + AutomaticSpeechRecognitionEarlyStoppingEnum, # noqa: F401 + AutomaticSpeechRecognitionGenerationParameters, # noqa: F401 + AutomaticSpeechRecognitionInput, # noqa: F401 + AutomaticSpeechRecognitionOutput, # noqa: F401 + AutomaticSpeechRecognitionOutputChunk, # noqa: F401 + AutomaticSpeechRecognitionParameters, # noqa: F401 + ChatCompletionInput, # noqa: F401 + ChatCompletionInputFunctionDefinition, # noqa: F401 + ChatCompletionInputFunctionName, # noqa: F401 + ChatCompletionInputGrammarType, # noqa: F401 + ChatCompletionInputJSONSchema, # noqa: F401 + ChatCompletionInputMessage, # noqa: F401 + ChatCompletionInputMessageChunk, # noqa: F401 + ChatCompletionInputMessageChunkType, # noqa: F401 + ChatCompletionInputResponseFormatJSONObject, # noqa: F401 + ChatCompletionInputResponseFormatJSONSchema, # noqa: F401 + ChatCompletionInputResponseFormatText, # noqa: F401 + ChatCompletionInputStreamOptions, # noqa: F401 + ChatCompletionInputTool, # noqa: F401 + ChatCompletionInputToolCall, # noqa: F401 + ChatCompletionInputToolChoiceClass, # noqa: F401 + ChatCompletionInputToolChoiceEnum, # noqa: F401 + ChatCompletionInputURL, # noqa: F401 + ChatCompletionOutput, # noqa: F401 + ChatCompletionOutputComplete, # noqa: F401 + ChatCompletionOutputFunctionDefinition, # noqa: F401 + ChatCompletionOutputLogprob, # noqa: F401 + ChatCompletionOutputLogprobs, # noqa: F401 + ChatCompletionOutputMessage, # noqa: F401 + ChatCompletionOutputToolCall, # noqa: F401 + ChatCompletionOutputTopLogprob, # noqa: F401 + ChatCompletionOutputUsage, # noqa: F401 + ChatCompletionStreamOutput, # noqa: F401 + ChatCompletionStreamOutputChoice, # noqa: F401 + ChatCompletionStreamOutputDelta, # noqa: F401 + ChatCompletionStreamOutputDeltaToolCall, # noqa: F401 + ChatCompletionStreamOutputFunction, # noqa: F401 + ChatCompletionStreamOutputLogprob, # noqa: F401 + ChatCompletionStreamOutputLogprobs, # noqa: F401 + ChatCompletionStreamOutputTopLogprob, # noqa: F401 + ChatCompletionStreamOutputUsage, # noqa: F401 + DepthEstimationInput, # noqa: F401 + DepthEstimationOutput, # noqa: F401 + DocumentQuestionAnsweringInput, # noqa: F401 + DocumentQuestionAnsweringInputData, # noqa: F401 + DocumentQuestionAnsweringOutputElement, # noqa: F401 + DocumentQuestionAnsweringParameters, # noqa: F401 + FeatureExtractionInput, # noqa: F401 + FeatureExtractionInputTruncationDirection, # noqa: F401 + FillMaskInput, # noqa: F401 + FillMaskOutputElement, # noqa: F401 + FillMaskParameters, # noqa: F401 + ImageClassificationInput, # noqa: F401 + ImageClassificationOutputElement, # noqa: F401 + ImageClassificationOutputTransform, # noqa: F401 + ImageClassificationParameters, # noqa: F401 + ImageSegmentationInput, # noqa: F401 + ImageSegmentationOutputElement, # noqa: F401 + ImageSegmentationParameters, # noqa: F401 + ImageSegmentationSubtask, # noqa: F401 + ImageTextToImageInput, # noqa: F401 + ImageTextToImageOutput, # noqa: F401 + ImageTextToImageParameters, # noqa: F401 + ImageTextToImageTargetSize, # noqa: F401 + ImageTextToVideoInput, # noqa: F401 + ImageTextToVideoOutput, # noqa: F401 + ImageTextToVideoParameters, # noqa: F401 + ImageTextToVideoTargetSize, # noqa: F401 + ImageToImageInput, # noqa: F401 + ImageToImageOutput, # noqa: F401 + ImageToImageParameters, # noqa: F401 + ImageToImageTargetSize, # noqa: F401 + ImageToTextEarlyStoppingEnum, # noqa: F401 + ImageToTextGenerationParameters, # noqa: F401 + ImageToTextInput, # noqa: F401 + ImageToTextOutput, # noqa: F401 + ImageToTextParameters, # noqa: F401 + ImageToVideoInput, # noqa: F401 + ImageToVideoOutput, # noqa: F401 + ImageToVideoParameters, # noqa: F401 + ImageToVideoTargetSize, # noqa: F401 + ObjectDetectionBoundingBox, # noqa: F401 + ObjectDetectionInput, # noqa: F401 + ObjectDetectionOutputElement, # noqa: F401 + ObjectDetectionParameters, # noqa: F401 + Padding, # noqa: F401 + QuestionAnsweringInput, # noqa: F401 + QuestionAnsweringInputData, # noqa: F401 + QuestionAnsweringOutputElement, # noqa: F401 + QuestionAnsweringParameters, # noqa: F401 + SentenceSimilarityInput, # noqa: F401 + SentenceSimilarityInputData, # noqa: F401 + SummarizationInput, # noqa: F401 + SummarizationOutput, # noqa: F401 + SummarizationParameters, # noqa: F401 + SummarizationTruncationStrategy, # noqa: F401 + TableQuestionAnsweringInput, # noqa: F401 + TableQuestionAnsweringInputData, # noqa: F401 + TableQuestionAnsweringOutputElement, # noqa: F401 + TableQuestionAnsweringParameters, # noqa: F401 + Text2TextGenerationInput, # noqa: F401 + Text2TextGenerationOutput, # noqa: F401 + Text2TextGenerationParameters, # noqa: F401 + Text2TextGenerationTruncationStrategy, # noqa: F401 + TextClassificationInput, # noqa: F401 + TextClassificationOutputElement, # noqa: F401 + TextClassificationOutputTransform, # noqa: F401 + TextClassificationParameters, # noqa: F401 + TextGenerationInput, # noqa: F401 + TextGenerationInputGenerateParameters, # noqa: F401 + TextGenerationInputGrammarType, # noqa: F401 + TextGenerationOutput, # noqa: F401 + TextGenerationOutputBestOfSequence, # noqa: F401 + TextGenerationOutputDetails, # noqa: F401 + TextGenerationOutputFinishReason, # noqa: F401 + TextGenerationOutputPrefillToken, # noqa: F401 + TextGenerationOutputToken, # noqa: F401 + TextGenerationStreamOutput, # noqa: F401 + TextGenerationStreamOutputStreamDetails, # noqa: F401 + TextGenerationStreamOutputToken, # noqa: F401 + TextToAudioEarlyStoppingEnum, # noqa: F401 + TextToAudioGenerationParameters, # noqa: F401 + TextToAudioInput, # noqa: F401 + TextToAudioOutput, # noqa: F401 + TextToAudioParameters, # noqa: F401 + TextToImageInput, # noqa: F401 + TextToImageOutput, # noqa: F401 + TextToImageParameters, # noqa: F401 + TextToSpeechEarlyStoppingEnum, # noqa: F401 + TextToSpeechGenerationParameters, # noqa: F401 + TextToSpeechInput, # noqa: F401 + TextToSpeechOutput, # noqa: F401 + TextToSpeechParameters, # noqa: F401 + TextToVideoInput, # noqa: F401 + TextToVideoOutput, # noqa: F401 + TextToVideoParameters, # noqa: F401 + TokenClassificationAggregationStrategy, # noqa: F401 + TokenClassificationInput, # noqa: F401 + TokenClassificationOutputElement, # noqa: F401 + TokenClassificationParameters, # noqa: F401 + TranslationInput, # noqa: F401 + TranslationOutput, # noqa: F401 + TranslationParameters, # noqa: F401 + TranslationTruncationStrategy, # noqa: F401 + TypeEnum, # noqa: F401 + VideoClassificationInput, # noqa: F401 + VideoClassificationOutputElement, # noqa: F401 + VideoClassificationOutputTransform, # noqa: F401 + VideoClassificationParameters, # noqa: F401 + VisualQuestionAnsweringInput, # noqa: F401 + VisualQuestionAnsweringInputData, # noqa: F401 + VisualQuestionAnsweringOutputElement, # noqa: F401 + VisualQuestionAnsweringParameters, # noqa: F401 + ZeroShotClassificationInput, # noqa: F401 + ZeroShotClassificationOutputElement, # noqa: F401 + ZeroShotClassificationParameters, # noqa: F401 + ZeroShotImageClassificationInput, # noqa: F401 + ZeroShotImageClassificationOutputElement, # noqa: F401 + ZeroShotImageClassificationParameters, # noqa: F401 + ZeroShotObjectDetectionBoundingBox, # noqa: F401 + ZeroShotObjectDetectionInput, # noqa: F401 + ZeroShotObjectDetectionOutputElement, # noqa: F401 + ZeroShotObjectDetectionParameters, # noqa: F401 + ) + from .inference._mcp.agent import Agent # noqa: F401 + from .inference._mcp.mcp_client import MCPClient # noqa: F401 + from .repocard import ( + DatasetCard, # noqa: F401 + ModelCard, # noqa: F401 + RepoCard, # noqa: F401 + SpaceCard, # noqa: F401 + metadata_eval_result, # noqa: F401 + metadata_load, # noqa: F401 + metadata_save, # noqa: F401 + metadata_update, # noqa: F401 + ) + from .repocard_data import ( + CardData, # noqa: F401 + DatasetCardData, # noqa: F401 + EvalResult, # noqa: F401 + ModelCardData, # noqa: F401 + SpaceCardData, # noqa: F401 + ) + from .serialization import ( + StateDictSplit, # noqa: F401 + get_torch_storage_id, # noqa: F401 + get_torch_storage_size, # noqa: F401 + load_state_dict_from_file, # noqa: F401 + load_torch_model, # noqa: F401 + save_torch_model, # noqa: F401 + save_torch_state_dict, # noqa: F401 + split_state_dict_into_shards_factory, # noqa: F401 + split_torch_state_dict_into_shards, # noqa: F401 + ) + from .serialization._dduf import ( + DDUFEntry, # noqa: F401 + export_entries_as_dduf, # noqa: F401 + export_folder_as_dduf, # noqa: F401 + read_dduf_file, # noqa: F401 + ) + from .utils import ( + ASYNC_CLIENT_FACTORY_T, # noqa: F401 + CLIENT_FACTORY_T, # noqa: F401 + CachedFileInfo, # noqa: F401 + CachedIncompleteFileInfo, # noqa: F401 + CachedRepoInfo, # noqa: F401 + CachedRevisionInfo, # noqa: F401 + CacheNotFound, # noqa: F401 + CorruptedCacheException, # noqa: F401 + DeleteCacheStrategy, # noqa: F401 + HFCacheInfo, # noqa: F401 + HfUri, # noqa: F401 + cached_assets_path, # noqa: F401 + close_session, # noqa: F401 + dump_environment_info, # noqa: F401 + get_async_session, # noqa: F401 + get_session, # noqa: F401 + get_token, # noqa: F401 + hf_raise_for_status, # noqa: F401 + logging, # noqa: F401 + parse_hf_uri, # noqa: F401 + scan_cache_dir, # noqa: F401 + set_async_client_factory, # noqa: F401 + set_client_factory, # noqa: F401 + ) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c35a6e552767637a84954f54a9548adb4b07a5fa Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_buckets.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_buckets.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec93bb2fcb858c257cffe4e263afd07f521e22c0 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_buckets.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_api.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..27e3cb594c5a428b13244ee09a1d6c2e0d39839d Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_api.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_scheduler.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_scheduler.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7e6b387abf9c5c5112b097967c82d9980b61fcbe Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_scheduler.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_dataset_viewer.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_dataset_viewer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0dcaf9fbd20a0d77655dd1f3959a47be8d210c7 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_dataset_viewer.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_eval_results.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_eval_results.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c88acb06a47af4c94559fbda4586af65ff16e17b Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_eval_results.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_inference_endpoints.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_inference_endpoints.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..edb281b7c38904663d647831d1ceba60ba8d3769 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_inference_endpoints.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_jobs_api.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_jobs_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..683b11e87b1ba20d3a25fb5f7b85165e25ec276c Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_jobs_api.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_local_folder.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_local_folder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0ff4b80be5302647c1a70c5f580d82b0406e0b8f Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_local_folder.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_login.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_login.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7f5801517b679a877d0562d71b8fc1e77ddaafd Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_login.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_oauth.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_oauth.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ca11cbfd4945efefbaa629ae65a1fbb70acce32 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_oauth.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_oidc.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_oidc.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c65780e8b4c17f1daa1e27b1d4e6da903cd54df2 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_oidc.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_sandbox.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_sandbox.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7b3b45b8948492783946a5bcda23fbac8a2aded3 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_sandbox.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_sandbox_cache.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_sandbox_cache.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3ceea97378a0ee6377ea51363875c8d780491dc8 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_sandbox_cache.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_snapshot_download.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_snapshot_download.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..07b13bcfa5ad42b6c8c9be01128a8c95f205fe3a Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_snapshot_download.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_space_api.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_space_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5da041f81de30a657ec9a21f68444c504c91dd42 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_space_api.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_tensorboard_logger.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_tensorboard_logger.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4baceedfaf697585e9d18cd1a9f3a7dd311f1d6b Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_tensorboard_logger.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_tree_cache.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_tree_cache.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..60783171adf59cdb7fe9cf8af5cc3c2c9c9767ec Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_tree_cache.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_upload_large_folder.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_upload_large_folder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a48ebd5cb7d1a97d9283116c6418d813f1c4d367 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_upload_large_folder.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_upload_pipeline.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_upload_pipeline.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6878c8849c980c4b33818f9862027b1b4003fa4d Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_upload_pipeline.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_payload.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_payload.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea65e12a985fed4afb3a4950c6405f9f66c99177 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_payload.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_server.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_server.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61f7d9dbe340797fecc242b04608f84529963b7b Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_server.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/community.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/community.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..21a52864f0823662807a8d4ca0c040d11cb09f67 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/community.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/constants.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/constants.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f1129fafba7ad9fcab484d6bb71bb5c1f767c36c Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/constants.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/dataclasses.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/dataclasses.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7b62b14a6efd83d5d63ab1299e879fcb494cffbf Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/dataclasses.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/errors.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/errors.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2d55ad5fca471acd9d63b93d71566fa5dfa654b4 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/errors.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/fastai_utils.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/fastai_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1c3fbd07d9bdb7c91ac0bc66005b805e30fa400e Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/fastai_utils.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/file_download.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/file_download.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50aa3119b2ef6770e576dfd40ddd5eb4dfddc1c4 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/file_download.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hf_api.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hf_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..543265d0d0a958a86ea79f5eadb4e7e8a510ec1a --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hf_api.cpython-312.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6208e94e6f1a1d433ee348bdc67cbb776f59da176e6d996bcc8e45fd8e2e24a +size 666991 diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hf_file_system.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hf_file_system.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f893ec8785c5de38f4c251e1a0b2f10e09b20cd4 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hf_file_system.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hub_mixin.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hub_mixin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d5dadea1c0f635119094e8a521eb9236386bfdf5 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hub_mixin.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/lfs.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/lfs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6307cdf31154e98e70fa4c234b4fd907a4a69120 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/lfs.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d87131e408ec96a5433be587c4105abed05ecdd Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard_data.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard_data.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66aa903e60545b136deaf9b5c06c361ca956f786 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard_data.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_buckets.py b/venv/lib/python3.12/site-packages/huggingface_hub/_buckets.py new file mode 100644 index 0000000000000000000000000000000000000000..6a1ac867af35eae74eb56489ab90c9bcbf746b46 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_buckets.py @@ -0,0 +1,1244 @@ +# Copyright 2026-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shared logic for bucket operations. + +This module contains the core buckets logic used by both the CLI and the Python API. +""" + +import fnmatch +import json +import mimetypes +import os +import stat +import sys +import time +from collections.abc import Iterator +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal + +from . import constants, logging +from .errors import BucketNotFoundError +from .utils import ( + HfUri, + StatusLine, + XetFileData, + disable_progress_bars, + enable_progress_bars, + parse_datetime, + parse_hf_uri, +) +from .utils._hf_uris import _looks_like_hf_url + + +if TYPE_CHECKING: + from .hf_api import HfApi + + +logger = logging.get_logger(__name__) + + +BUCKET_PREFIX = "hf://buckets/" +_SYNC_TIME_WINDOW_MS = 1000 # 1s safety-window for file modification time comparisons + + +# ============================================================================= +# Bucket data structures +# ============================================================================= + + +@dataclass +class BucketInfo: + """ + Contains information about a bucket on the Hub. This object is returned by [`bucket_info`] and [`list_buckets`]. + + Attributes: + id (`str`): + ID of the bucket. + private (`bool`): + Is the bucket private. + created_at (`datetime`): + Date of creation of the bucket on the Hub. + size (`int`): + Size of the bucket in bytes. + total_files (`int`): + Total number of files in the bucket. + """ + + id: str + private: bool + created_at: datetime + size: int + total_files: int + + def __init__(self, **kwargs): + self.id = kwargs.pop("id") + self.private = kwargs.pop("private") + self.created_at = parse_datetime(kwargs.pop("createdAt")) + self.size = kwargs.pop("size") + self.total_files = kwargs.pop("totalFiles") + self.__dict__.update(**kwargs) + + +@dataclass +class _BucketAddFile: + source: str | Path | bytes + destination: str + + xet_hash: str | None = field(default=None) + size: int | None = field(default=None) + mtime: int = field(init=False) + content_type: str | None = field(init=False) + + def __post_init__(self) -> None: + self.content_type = None + if isinstance(self.source, (str, Path)): # guess content type from source path + self.content_type = mimetypes.guess_type(self.source)[0] + if self.content_type is None: # or default to destination path content type + self.content_type = mimetypes.guess_type(self.destination)[0] + + self.mtime = int( + os.path.getmtime(self.source) * 1000 if not isinstance(self.source, bytes) else time.time() * 1000 + ) + + +@dataclass +class _BucketCopyFile: + destination: str + xet_hash: str + source_repo_type: str # "model", "dataset", "space", "bucket" + source_repo_id: str + size: int | None = field(default=None) + mtime: int = field(init=False) + content_type: str | None = field(init=False) + + def __post_init__(self) -> None: + self.content_type = mimetypes.guess_type(self.destination)[0] + self.mtime = int(time.time() * 1000) + + +@dataclass +class _BucketDeleteFile: + path: str + + +@dataclass(frozen=True) +class BucketFileMetadata: + """Data structure containing information about a file in a bucket. + + Returned by [`get_bucket_file_metadata`]. + + Args: + size (`int`): + Size of the file in bytes. + xet_file_data (`XetFileData`): + Xet information for the file (hash and refresh route). + """ + + size: int + xet_file_data: XetFileData + + +@dataclass +class BucketUrl: + """Describes a bucket URL on the Hub. + + `BucketUrl` is returned by [`create_bucket`]. At initialization, the URL is parsed to populate properties: + - endpoint (`str`) + - namespace (`str`) + - bucket_id (`str`) + - url (`str`) + - uri (`HfUri`) + + Args: + url (`str`): + String value of the bucket url. + endpoint (`str`, *optional*): + Endpoint of the Hub. Defaults to . + """ + + url: str + endpoint: str = "" + namespace: str = field(init=False) + bucket_id: str = field(init=False) + uri: HfUri = field(init=False) + + def __post_init__(self) -> None: + self.endpoint = self.endpoint or constants.ENDPOINT + + # Parse URL: expected format is `{endpoint}/buckets/{namespace}/{bucket_name}` + url_path = self.url.replace(self.endpoint, "").strip("/") + # Remove leading "buckets/" prefix + if url_path.startswith("buckets/"): + url_path = url_path[len("buckets/") :] + parsed = _parse_bucket_uri(url_path) + if parsed.path_in_repo: + raise ValueError(f"Unable to parse bucket URL: {self.url}") + self.namespace = parsed.id.split("/")[0] + self.bucket_id = parsed.id + self.uri = parsed + + +@dataclass +class BucketFile: + """ + Contains information about a file in a bucket on the Hub. This object is returned by [`list_bucket_tree`]. + + Similar to [`RepoFile`] but for files in buckets. + """ + + type: Literal["file"] + path: str + size: int + xet_hash: str + mtime: datetime | None + uploaded_at: datetime | None + + def __init__(self, **kwargs): + self.type = kwargs.pop("type") + self.path = kwargs.pop("path") + self.size = kwargs.pop("size") + self.xet_hash = kwargs.pop("xetHash") + mtime = kwargs.pop("mtime", None) + self.mtime = parse_datetime(mtime) if mtime else None + uploaded_at = kwargs.pop("uploadedAt", None) + self.uploaded_at = parse_datetime(uploaded_at) if uploaded_at else None + + +@dataclass +class BucketFolder: + """ + Contains information about a directory in a bucket on the Hub. This object is returned by [`list_bucket_tree`]. + + Similar to [`RepoFolder`] but for directories in buckets. + """ + + type: Literal["directory"] + path: str + uploaded_at: datetime | None + + def __init__(self, **kwargs): + self.type = kwargs.pop("type") + self.path = kwargs.pop("path") + uploaded_at = kwargs.pop("uploadedAt", None) or kwargs.pop("uploaded_at", None) + self.uploaded_at = ( + (uploaded_at if isinstance(uploaded_at, datetime) else parse_datetime(uploaded_at)) + if uploaded_at + else None + ) + + +# ============================================================================= +# Bucket path parsing +# ============================================================================= + + +def _parse_bucket_uri(path: str) -> HfUri: + """Parse a bucket path into a HfUri. + + Accepts: + - `hf://buckets/namespace/name(/path/in/repo)` URIs, + - Hugging Face web URLs such as `https://huggingface.co/buckets/namespace/name(/tree/path)`, + - plain `namespace/name(/path/in/repo)` paths. + """ + if path.startswith(constants.HF_PROTOCOL) or _looks_like_hf_url(path): + # Don't use 'if is_hf_uri(...)' here as we prefer 'parse_hf_uri(...)' to raise the exact error message. + parsed = parse_hf_uri(path) + if not parsed.is_bucket: + raise ValueError(f"Invalid bucket path: {path}. Must be a bucket URI (hf://buckets/...).") + return parsed + parts = path.split("/", 2) + if len(parts) < 2 or not parts[0] or not parts[1]: + raise ValueError(f"Invalid bucket path: '{path}'. Expected format: namespace/bucket_name") + bucket_id = f"{parts[0]}/{parts[1]}" + prefix = "/".join(parts[2:]) + return HfUri(type="bucket", id=bucket_id, path_in_repo=prefix) + + +def _is_bucket_path(path: str) -> bool: + """Check if a path is a bucket path. + + Do not raise if the path is not a hf:// URI. + Raise if the path is a hf:// URI but with an incorrect format. + """ + if not path.startswith(constants.HF_PROTOCOL): + return False + return parse_hf_uri(path).is_bucket + + +# ============================================================================= +# Sync data structures +# ============================================================================= + + +@dataclass +class SyncOperation: + """Represents a sync operation to be performed.""" + + action: Literal["upload", "download", "delete", "skip"] + path: str + size: int | None = None + reason: str = "" + local_mtime: str | None = None + remote_mtime: str | None = None + bucket_file: BucketFile | None = None # BucketFile when available (not serialized to plan file) + + +@dataclass +class SyncPlan: + """Represents a complete sync plan.""" + + source: str + dest: str + timestamp: str + operations: list[SyncOperation] = field(default_factory=list) + + def summary(self) -> dict[str, int | str]: + uploads = sum(1 for op in self.operations if op.action == "upload") + downloads = sum(1 for op in self.operations if op.action == "download") + deletes = sum(1 for op in self.operations if op.action == "delete") + skips = sum(1 for op in self.operations if op.action == "skip") + total_size = sum(op.size or 0 for op in self.operations if op.action in ("upload", "download")) + return { + "uploads": uploads, + "downloads": downloads, + "deletes": deletes, + "skips": skips, + "total_size": total_size, + } + + +# ============================================================================= +# Filter matching +# ============================================================================= + + +class FilterMatcher: + """Matches file paths against include/exclude patterns.""" + + def __init__( + self, + include_patterns: list[str] | None = None, + exclude_patterns: list[str] | None = None, + filter_rules: list[tuple[str, str]] | None = None, + ): + """Initialize the filter matcher. + + Args: + include_patterns: Patterns to include (from --include) + exclude_patterns: Patterns to exclude (from --exclude) + filter_rules: Rules from filter file as list of ("+"/"-", pattern) tuples + """ + self.include_patterns = include_patterns or [] + self.exclude_patterns = exclude_patterns or [] + self.filter_rules = filter_rules or [] + + def matches(self, path: str) -> bool: + """Check if a path should be included based on the filter rules. + + Filtering rules: + - Filters are evaluated in order, first matching rule decides + - If no rules match, include by default (unless include patterns are specified) + """ + # First check filter rules from file (in order) + for sign, pattern in self.filter_rules: + if fnmatch.fnmatch(path, pattern): + return sign == "+" + + # Then check CLI patterns + for pattern in self.exclude_patterns: + if fnmatch.fnmatch(path, pattern): + return False + + for pattern in self.include_patterns: + if fnmatch.fnmatch(path, pattern): + return True + + # If include patterns were specified but none matched, exclude + if self.include_patterns: + return False + + # Default: include + return True + + +def _parse_filter_file(filter_file: str) -> list[tuple[str, str]]: + """Parse a filter file and return a list of (sign, pattern) tuples. + + Filter file format: + - Lines starting with "+" are include patterns + - Lines starting with "-" are exclude patterns + - Empty lines and lines starting with "#" are ignored + """ + rules = [] + with open(filter_file) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("+"): + rules.append(("+", line[1:].strip())) + elif line.startswith("-"): + rules.append(("-", line[1:].strip())) + else: + # Default to include if no prefix + rules.append(("+", line)) + return rules + + +# ============================================================================= +# File listing +# ============================================================================= + + +def _stat_local(path: str) -> tuple[int, float] | None: + """Stat a local file and return (size, mtime_ms). + + Returns None if the path is missing or is a directory. Uses a single + ``os.stat`` call so callers don't pay for multiple syscalls per file. + """ + try: + st = os.stat(path) + except OSError: + return None + if stat.S_ISDIR(st.st_mode): + return None + return st.st_size, st.st_mtime * 1000 + + +def _list_local_files(local_path: str) -> Iterator[tuple[str, int, float]]: + """List all files in a local directory. + + Yields: + tuple: (relative_path, size, mtime_ms) for each file + """ + local_path = os.path.abspath(local_path) + if not os.path.isdir(local_path): + raise ValueError(f"Local path must be a directory: {local_path}") + + for root, _, files in os.walk(local_path): + for filename in files: + full_path = os.path.join(root, filename) + stat_info = _stat_local(full_path) + if stat_info is None: + continue + rel_path = os.path.relpath(full_path, local_path) + # Normalize to forward slashes for consistency + rel_path = rel_path.replace(os.sep, "/") + yield rel_path, stat_info[0], stat_info[1] + + +def _list_remote_files(api: "HfApi", bucket_id: str, prefix: str) -> Iterator[tuple[str, int, float, Any]]: + """List all files in a bucket with a given prefix. + + Yields: + tuple: (relative_path, size, mtime_ms, bucket_file) for each file. + bucket_file is the BucketFile object from list_bucket_tree. + """ + for item in api.list_bucket_tree(bucket_id, prefix=prefix or None, recursive=True): + if isinstance(item, BucketFolder): + continue + path = item.path + # Remove prefix from path to get relative path + # Only strip prefix if it's followed by "/" (directory boundary) or is exact match + if prefix: + if path.startswith(prefix + "/"): + rel_path = path[len(prefix) + 1 :] + elif path == prefix: + # Exact match: the file IS the prefix (e.g., single file download) + rel_path = path.rsplit("/", 1)[-1] if "/" in path else path + else: + # Path doesn't match prefix pattern (e.g., "submarine.txt" for prefix "sub") + # Skip this file - it was returned by the API but doesn't belong to this prefix + continue + else: + rel_path = path + mtime_ms = item.mtime.timestamp() * 1000 if item.mtime else 0 + yield rel_path, item.size, mtime_ms, item + + +# ============================================================================= +# Sync plan computation +# ============================================================================= + + +def _mtime_to_iso(mtime_ms: float) -> str: + """Convert mtime in milliseconds to ISO format string.""" + return datetime.fromtimestamp(mtime_ms / 1000, tz=timezone.utc).isoformat() + + +def _compare_files_for_sync( + *, + path: str, + action: Literal["upload", "download"], + source_size: int, + source_mtime: float, + dest_size: int, + dest_mtime: float, + source_newer_label: str, + dest_newer_label: str, + ignore_sizes: bool, + ignore_times: bool, + ignore_existing: bool, + bucket_file: Any | None = None, +) -> SyncOperation: + """Compare source and dest files and return the appropriate sync operation. + + This is a unified helper for both upload and download directions. + + Args: + path: Relative file path + action: "upload" or "download" + source_size: Size of the source file (bytes) + source_mtime: Mtime of the source file (milliseconds) + dest_size: Size of the destination file (bytes) + dest_mtime: Mtime of the destination file (milliseconds) + source_newer_label: Label when source is newer (e.g., "local newer" or "remote newer") + dest_newer_label: Label when dest is newer (e.g., "remote newer" or "local newer") + ignore_sizes: Only compare mtime + ignore_times: Only compare size + ignore_existing: Skip files that exist on receiver + bucket_file: BucketFile object (for downloads only) + + Returns: + SyncOperation describing the action to take + """ + local_mtime_iso = _mtime_to_iso(source_mtime if action == "upload" else dest_mtime) + remote_mtime_iso = _mtime_to_iso(dest_mtime if action == "upload" else source_mtime) + + base_kwargs: dict[str, Any] = { + "path": path, + "size": source_size, + "local_mtime": local_mtime_iso, + "remote_mtime": remote_mtime_iso, + } + + if ignore_existing: + return SyncOperation(action="skip", reason="exists on receiver (--ignore-existing)", **base_kwargs) + + size_differs = source_size != dest_size + source_newer = (source_mtime - dest_mtime) > _SYNC_TIME_WINDOW_MS + + if ignore_sizes: + if source_newer: + return SyncOperation(action=action, reason=source_newer_label, bucket_file=bucket_file, **base_kwargs) + else: + dest_newer = (dest_mtime - source_mtime) > _SYNC_TIME_WINDOW_MS + skip_reason = dest_newer_label if dest_newer else "same mtime" + return SyncOperation(action="skip", reason=skip_reason, **base_kwargs) + elif ignore_times: + if size_differs: + return SyncOperation(action=action, reason="size differs", bucket_file=bucket_file, **base_kwargs) + else: + return SyncOperation(action="skip", reason="same size", **base_kwargs) + else: + if size_differs or source_newer: + reason = "size differs" if size_differs else source_newer_label + return SyncOperation(action=action, reason=reason, bucket_file=bucket_file, **base_kwargs) + else: + return SyncOperation(action="skip", reason="identical", **base_kwargs) + + +def _compute_sync_plan( + source: str, + dest: str, + api: "HfApi", + delete: bool = False, + ignore_times: bool = False, + ignore_sizes: bool = False, + existing: bool = False, + ignore_existing: bool = False, + filter_matcher: FilterMatcher | None = None, + status: Any | None = None, +) -> SyncPlan: + """Compute the sync plan by comparing source and destination. + + Returns: + SyncPlan with all operations to be performed + """ + filter_matcher = filter_matcher or FilterMatcher() + is_upload = not _is_bucket_path(source) and _is_bucket_path(dest) + is_download = _is_bucket_path(source) and not _is_bucket_path(dest) + + if not is_upload and not is_download: + raise ValueError("One of source or dest must be a bucket path (hf://buckets/...) and the other must be local.") + + plan = SyncPlan( + source=source, + dest=dest, + timestamp=datetime.now(timezone.utc).isoformat(), + ) + + remote_total: int | None = None + if is_upload: + # Local -> Remote + local_path = os.path.abspath(source) + parsed = _parse_bucket_uri(dest) + bucket_id, prefix = parsed.id, parsed.path_in_repo + + if not os.path.isdir(local_path): + raise ValueError(f"Source must be a directory: {local_path}") + + # Get local and remote file lists + local_files = {} + for rel_path, size, mtime_ms in _list_local_files(local_path): + if filter_matcher.matches(rel_path): + local_files[rel_path] = (size, mtime_ms) + if status: + status.update(f"Scanning local directory ({len(local_files)} files)") + if status: + status.done(f"Scanning local directory ({len(local_files)} files)") + + remote_files = {} + if status: + try: + remote_total = api.bucket_info(bucket_id).total_files + except Exception: + pass + try: + for rel_path, size, mtime_ms, _ in _list_remote_files(api, bucket_id, prefix): + if filter_matcher.matches(rel_path): + remote_files[rel_path] = (size, mtime_ms) + if status: + total_str = f"/{remote_total}" if remote_total is not None else "" + status.update(f"Scanning remote bucket ({len(remote_files)}{total_str} files)") + except BucketNotFoundError: + # Bucket doesn't exist yet - this is expected for new uploads + logger.debug(f"Bucket '{bucket_id}' not found, treating as empty.") + if status: + status.done(f"Scanning remote bucket ({len(remote_files)} files)") + + # Compare files + all_paths = set(local_files.keys()) | set(remote_files.keys()) + if status: + status.done(f"Comparing files ({len(all_paths)} paths)") + for path in sorted(all_paths): + local_info = local_files.get(path) + remote_info = remote_files.get(path) + + if local_info and not remote_info: + # New file + if existing: + # --existing: skip new files + plan.operations.append( + SyncOperation( + action="skip", + path=path, + size=local_info[0], + reason="new file (--existing)", + local_mtime=_mtime_to_iso(local_info[1]), + ) + ) + else: + plan.operations.append( + SyncOperation( + action="upload", + path=path, + size=local_info[0], + reason="new file", + local_mtime=_mtime_to_iso(local_info[1]), + ) + ) + elif local_info and remote_info: + # File exists in both - use helper to determine action + local_size, local_mtime = local_info + remote_size, remote_mtime = remote_info + plan.operations.append( + _compare_files_for_sync( + path=path, + action="upload", + source_size=local_size, + source_mtime=local_mtime, + dest_size=remote_size, + dest_mtime=remote_mtime, + source_newer_label="local newer", + dest_newer_label="remote newer", + ignore_sizes=ignore_sizes, + ignore_times=ignore_times, + ignore_existing=ignore_existing, + ) + ) + elif not local_info and remote_info and delete: + # File only in remote and --delete mode + plan.operations.append( + SyncOperation( + action="delete", + path=path, + size=remote_info[0], + reason="not in source (--delete)", + remote_mtime=_mtime_to_iso(remote_info[1]), + ) + ) + + else: + # Remote -> Local (download) + parsed = _parse_bucket_uri(source) + bucket_id, prefix = parsed.id, parsed.path_in_repo + local_path = os.path.abspath(dest) + + # Get remote and local file lists + remote_files = {} + bucket_file_map: dict[str, Any] = {} + if status: + try: + remote_total = api.bucket_info(bucket_id).total_files + except Exception: + pass + for rel_path, size, mtime_ms, bucket_file in _list_remote_files(api, bucket_id, prefix): + if filter_matcher.matches(rel_path): + remote_files[rel_path] = (size, mtime_ms) + bucket_file_map[rel_path] = bucket_file + if status: + total_str = f"/{remote_total}" if remote_total is not None else "" + status.update(f"Scanning remote bucket ({len(remote_files)}{total_str} files)") + if status: + status.done(f"Scanning remote bucket ({len(remote_files)} files)") + + local_files = {} + if os.path.isdir(local_path): + if delete: + # Full walk needed to discover local-only files for deletion. + for rel_path, size, mtime_ms in _list_local_files(local_path): + if filter_matcher.matches(rel_path): + local_files[rel_path] = (size, mtime_ms) + if status: + status.update(f"Scanning local directory ({len(local_files)} files)") + else: + # Without --delete, the plan only depends on paths that exist + # remotely. Stat just those instead of walking the whole tree, + # which can take minutes when dest sits in a large directory + # like ~/.cache/huggingface/. + for rel_path in remote_files: + local_file = os.path.join(local_path, rel_path) + stat_info = _stat_local(local_file) + if stat_info is None: + continue + local_files[rel_path] = stat_info + if status: + status.update(f"Scanning local directory ({len(local_files)} files)") + if status: + status.done(f"Scanning local directory ({len(local_files)} files)") + + # Compare files + all_paths = set(remote_files.keys()) | set(local_files.keys()) + if status: + status.done(f"Comparing files ({len(all_paths)} paths)") + for path in sorted(all_paths): + remote_info = remote_files.get(path) + local_info = local_files.get(path) + + if remote_info and not local_info: + # New file + if existing: + # --existing: skip new files + plan.operations.append( + SyncOperation( + action="skip", + path=path, + size=remote_info[0], + reason="new file (--existing)", + remote_mtime=_mtime_to_iso(remote_info[1]), + ) + ) + else: + plan.operations.append( + SyncOperation( + action="download", + path=path, + size=remote_info[0], + reason="new file", + remote_mtime=_mtime_to_iso(remote_info[1]), + bucket_file=bucket_file_map.get(path), + ) + ) + elif remote_info and local_info: + # File exists in both - use helper to determine action + remote_size, remote_mtime = remote_info + local_size, local_mtime = local_info + plan.operations.append( + _compare_files_for_sync( + path=path, + action="download", + source_size=remote_size, + source_mtime=remote_mtime, + dest_size=local_size, + dest_mtime=local_mtime, + source_newer_label="remote newer", + dest_newer_label="local newer", + ignore_sizes=ignore_sizes, + ignore_times=ignore_times, + ignore_existing=ignore_existing, + bucket_file=bucket_file_map.get(path), + ) + ) + elif not remote_info and local_info and delete: + # File only in local and --delete mode + plan.operations.append( + SyncOperation( + action="delete", + path=path, + size=local_info[0], + reason="not in source (--delete)", + local_mtime=_mtime_to_iso(local_info[1]), + ) + ) + + return plan + + +# ============================================================================= +# Plan serialization +# ============================================================================= + + +def _write_plan(plan: SyncPlan, f) -> None: + """Write a sync plan as JSONL to a file-like object.""" + # Write header + header = { + "type": "header", + "source": plan.source, + "dest": plan.dest, + "timestamp": plan.timestamp, + "summary": plan.summary(), + } + f.write(json.dumps(header) + "\n") + + # Write operations + for op in plan.operations: + op_dict: dict[str, Any] = { + "type": "operation", + "action": op.action, + "path": op.path, + "reason": op.reason, + } + if op.size is not None: + op_dict["size"] = op.size + if op.local_mtime is not None: + op_dict["local_mtime"] = op.local_mtime + if op.remote_mtime is not None: + op_dict["remote_mtime"] = op.remote_mtime + f.write(json.dumps(op_dict) + "\n") + + +def _save_plan(plan: SyncPlan, plan_file: str) -> None: + """Save a sync plan to a JSONL file.""" + with open(plan_file, "w") as f: + _write_plan(plan, f) + + +def _load_plan(plan_file: str) -> SyncPlan: + """Load a sync plan from a JSONL file.""" + with open(plan_file) as f: + lines = f.readlines() + + if not lines: + raise ValueError(f"Empty plan file: {plan_file}") + + # Parse header + header = json.loads(lines[0]) + if header.get("type") != "header": + raise ValueError("Invalid plan file: expected header as first line") + + plan = SyncPlan( + source=header["source"], + dest=header["dest"], + timestamp=header["timestamp"], + ) + + # Parse operations + for line in lines[1:]: + op_dict = json.loads(line) + if op_dict.get("type") != "operation": + continue + plan.operations.append( + SyncOperation( + action=op_dict["action"], + path=op_dict["path"], + size=op_dict.get("size"), + reason=op_dict.get("reason", ""), + local_mtime=op_dict.get("local_mtime"), + remote_mtime=op_dict.get("remote_mtime"), + ) + ) + + return plan + + +# ============================================================================= +# Plan execution +# ============================================================================= + + +def _execute_plan(plan: SyncPlan, api: "HfApi", verbose: bool = False, status: Any | None = None) -> None: + """Execute a sync plan.""" + is_upload = not _is_bucket_path(plan.source) and _is_bucket_path(plan.dest) + is_download = _is_bucket_path(plan.source) and not _is_bucket_path(plan.dest) + + if is_upload: + local_path = os.path.abspath(plan.source) + parsed = _parse_bucket_uri(plan.dest) + bucket_id, prefix = parsed.id, parsed.path_in_repo + + # Collect operations + add_files: list[tuple[str | Path | bytes, str]] = [] + delete_paths: list[str] = [] + + for op in plan.operations: + match op.action: + case "upload": + local_file = os.path.join(local_path, op.path) + remote_path = f"{prefix}/{op.path}" if prefix else op.path + if verbose: + print(f" Uploading: {op.path} ({op.reason})") + add_files.append((local_file, remote_path)) + case "delete": + remote_path = f"{prefix}/{op.path}" if prefix else op.path + if verbose: + print(f" Deleting: {op.path} ({op.reason})") + delete_paths.append(remote_path) + case "skip" if verbose: + print(f" Skipping: {op.path} ({op.reason})") + + # Execute batch operations + if add_files or delete_paths: + if status: + parts = [] + if add_files: + parts.append(f"uploading {len(add_files)} files") + if delete_paths: + parts.append(f"deleting {len(delete_paths)} files") + status.done(", ".join(parts).capitalize()) + api.batch_bucket_files( + bucket_id, + add=add_files or None, + delete=delete_paths or None, + ) + + elif is_download: + parsed = _parse_bucket_uri(plan.source) + bucket_id, prefix = parsed.id, parsed.path_in_repo + local_path = os.path.abspath(plan.dest) + + # Ensure local directory exists + os.makedirs(local_path, exist_ok=True) + + # Collect download operations + download_files: list[tuple[str | BucketFile, str | Path]] = [] + delete_files: list[str] = [] + + for op in plan.operations: + if op.action == "download": + local_file = os.path.join(local_path, op.path) + # Ensure parent directory exists + os.makedirs(os.path.dirname(local_file), exist_ok=True) + if verbose: + print(f" Downloading: {op.path} ({op.reason})") + # Use BucketFile when available (avoids extra metadata fetch per file) + if op.bucket_file is not None: + download_files.append((op.bucket_file, local_file)) + else: + remote_path = f"{prefix}/{op.path}" if prefix else op.path + download_files.append((remote_path, local_file)) + elif op.action == "delete": + local_file = os.path.join(local_path, op.path) + if verbose: + print(f" Deleting: {op.path} ({op.reason})") + delete_files.append(local_file) + elif op.action == "skip" and verbose: + print(f" Skipping: {op.path} ({op.reason})") + + # Execute downloads + if len(download_files) > 0: + if status: + status.done(f"Downloading {len(download_files)} files") + api.download_bucket_files(bucket_id, download_files) + + # Execute deletes + if status and delete_files: + status.done(f"Deleting {len(delete_files)} local files") + for file_path in delete_files: + if os.path.exists(file_path): + os.remove(file_path) + # Remove empty parent directories + parent = os.path.dirname(file_path) + while parent != local_path: + try: + os.rmdir(parent) + parent = os.path.dirname(parent) + except OSError: + break + + +def _print_plan_summary(plan: SyncPlan) -> None: + """Print a summary of the sync plan.""" + summary = plan.summary() + print(f"Sync plan: {plan.source} -> {plan.dest}") + print(f" Uploads: {summary['uploads']}") + print(f" Downloads: {summary['downloads']}") + print(f" Deletes: {summary['deletes']}") + print(f" Skips: {summary['skips']}") + + +# ============================================================================= +# Public sync function (Python API) +# ============================================================================= + + +def sync_bucket_internal( + source: str | None = None, + dest: str | None = None, + *, + api: "HfApi", + delete: bool = False, + ignore_times: bool = False, + ignore_sizes: bool = False, + existing: bool = False, + ignore_existing: bool = False, + include: list[str] | None = None, + exclude: list[str] | None = None, + filter_from: str | None = None, + plan: str | None = None, + apply: str | None = None, + dry_run: bool = False, + verbose: bool = False, + quiet: bool = False, + token: bool | str | None = None, +) -> SyncPlan: + """Sync files between a local directory and a bucket. + + This is equivalent to the ``hf buckets sync`` CLI command. One of ``source`` or ``dest`` must be a bucket path + (``hf://buckets/...``) and the other must be a local directory path. + + Args: + source (`str`, *optional*): + Source path: local directory or ``hf://buckets/namespace/bucket_name(/prefix)``. + Required unless using ``apply``. + dest (`str`, *optional*): + Destination path: local directory or ``hf://buckets/namespace/bucket_name(/prefix)``. + Required unless using ``apply``. + api ([`HfApi`]): + The HfApi instance to use for API calls. + delete (`bool`, *optional*, defaults to `False`): + Delete destination files not present in source. + ignore_times (`bool`, *optional*, defaults to `False`): + Skip files only based on size, ignoring modification times. + ignore_sizes (`bool`, *optional*, defaults to `False`): + Skip files only based on modification times, ignoring sizes. + existing (`bool`, *optional*, defaults to `False`): + Skip creating new files on receiver (only update existing files). + ignore_existing (`bool`, *optional*, defaults to `False`): + Skip updating files that exist on receiver (only create new files). + include (`list[str]`, *optional*): + Include files matching patterns (fnmatch-style). + exclude (`list[str]`, *optional*): + Exclude files matching patterns (fnmatch-style). + filter_from (`str`, *optional*): + Path to a filter file with include/exclude rules. + plan (`str`, *optional*): + Save sync plan to this JSONL file instead of executing. + apply (`str`, *optional*): + Apply a previously saved plan file. When set, ``source`` and ``dest`` are not needed. + dry_run (`bool`, *optional*, defaults to `False`): + Print sync plan to stdout as JSONL without executing. + verbose (`bool`, *optional*, defaults to `False`): + Show detailed per-file operations. + quiet (`bool`, *optional*, defaults to `False`): + Suppress all output and progress bars. + token (Union[bool, str, None], optional): + A valid user access token. If not provided, the locally saved token will be used. + + Returns: + [`SyncPlan`]: The computed (or loaded) sync plan. + + Raises: + `ValueError`: If arguments are invalid (e.g., both paths are remote, conflicting options). + + Example: + ```python + >>> from huggingface_hub import HfApi + >>> api = HfApi() + + # Upload local directory to bucket + >>> api.sync_bucket("./data", "hf://buckets/username/my-bucket") + + # Download bucket to local directory + >>> api.sync_bucket("hf://buckets/username/my-bucket", "./data") + + # Sync with delete and filtering + >>> api.sync_bucket( + ... "./data", + ... "hf://buckets/username/my-bucket", + ... delete=True, + ... include=["*.safetensors"], + ... ) + + # Dry run: preview what would be synced + >>> plan = api.sync_bucket("./data", "hf://buckets/username/my-bucket", dry_run=True) + >>> plan.summary() + {'uploads': 3, 'downloads': 0, 'deletes': 0, 'skips': 1, 'total_size': 4096} + + # Save plan for review, then apply + >>> api.sync_bucket("./data", "hf://buckets/username/my-bucket", plan="sync-plan.jsonl") + >>> api.sync_bucket(apply="sync-plan.jsonl") + ``` + """ + # Build API with token if needed + if token is not None: + from .hf_api import HfApi + + api = HfApi( + endpoint=api.endpoint, + token=token, + library_name=api.library_name, + library_version=api.library_version, + user_agent=api.user_agent, + headers=api.headers, + ) + # --- Apply mode --- + if apply: + if source or dest: + raise ValueError("Cannot specify source/dest when using apply.") + if plan is not None: + raise ValueError("Cannot specify both plan and apply.") + if delete: + raise ValueError("Cannot specify delete when using apply.") + if ignore_times: + raise ValueError("Cannot specify ignore_times when using apply.") + if ignore_sizes: + raise ValueError("Cannot specify ignore_sizes when using apply.") + if include: + raise ValueError("Cannot specify include when using apply.") + if exclude: + raise ValueError("Cannot specify exclude when using apply.") + if filter_from: + raise ValueError("Cannot specify filter_from when using apply.") + if existing: + raise ValueError("Cannot specify existing when using apply.") + if ignore_existing: + raise ValueError("Cannot specify ignore_existing when using apply.") + if dry_run: + raise ValueError("Cannot specify dry_run when using apply.") + + sync_plan = _load_plan(apply) + status = StatusLine(enabled=not quiet) + if not quiet: + _print_plan_summary(sync_plan) + print("Executing plan...") + + if quiet: + disable_progress_bars() + try: + _execute_plan(sync_plan, api, verbose=verbose, status=status) + finally: + if quiet: + enable_progress_bars() + + if not quiet: + print("Sync completed.") + + return sync_plan + + # --- Normal mode --- + if not source or not dest: + raise ValueError("Both source and dest are required (unless using apply).") + + source_is_bucket = _is_bucket_path(source) + dest_is_bucket = _is_bucket_path(dest) + + if source_is_bucket and dest_is_bucket: + raise ValueError("Remote to remote sync is not supported. One path must be local.") + + if not source_is_bucket and not dest_is_bucket: + raise ValueError("One of source or dest must be a bucket path (hf://buckets/...).") + + if ignore_times and ignore_sizes: + raise ValueError("Cannot specify both ignore_times and ignore_sizes.") + + if existing and ignore_existing: + raise ValueError("Cannot specify both existing and ignore_existing.") + + if dry_run and plan: + raise ValueError("Cannot specify both dry_run and plan.") + + # Validate local path + if source_is_bucket: + if os.path.exists(dest) and not os.path.isdir(dest): + raise ValueError(f"Destination must be a directory: {dest}") + else: + if not os.path.isdir(source): + raise ValueError(f"Source must be an existing directory: {source}") + + # Build filter matcher + filter_rules = None + if filter_from: + filter_rules = _parse_filter_file(filter_from) + + filter_matcher = FilterMatcher( + include_patterns=include, + exclude_patterns=exclude, + filter_rules=filter_rules, + ) + + # Compute sync plan + status = StatusLine(enabled=not quiet and not dry_run) + sync_plan = _compute_sync_plan( + source=source, + dest=dest, + api=api, + delete=delete, + ignore_times=ignore_times, + ignore_sizes=ignore_sizes, + existing=existing, + ignore_existing=ignore_existing, + filter_matcher=filter_matcher, + status=status, + ) + + if dry_run: + _write_plan(sync_plan, sys.stdout) + return sync_plan + + if plan: + _save_plan(sync_plan, plan) + if not quiet: + _print_plan_summary(sync_plan) + print(f"Plan saved to: {plan}") + return sync_plan + + # Execute plan + if not quiet: + _print_plan_summary(sync_plan) + + summary = sync_plan.summary() + if summary["uploads"] == 0 and summary["downloads"] == 0 and summary["deletes"] == 0: + if not quiet: + print("Nothing to sync.") + return sync_plan + + if not quiet: + print("Syncing...") + + if quiet: + disable_progress_bars() + try: + _execute_plan(sync_plan, api, verbose=verbose, status=status) + finally: + if quiet: + enable_progress_bars() + + if not quiet: + print("Sync completed.") + + return sync_plan diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_commit_api.py b/venv/lib/python3.12/site-packages/huggingface_hub/_commit_api.py new file mode 100644 index 0000000000000000000000000000000000000000..112445e500535f4de92148b17f25cf0cd5f37749 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_commit_api.py @@ -0,0 +1,1074 @@ +""" +Type definitions and utilities for the `create_commit` API +""" + +import base64 +import io +import json +import os +import warnings +from collections import defaultdict +from collections.abc import Iterable, Iterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass, field +from itertools import groupby +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Any, BinaryIO, Literal, NamedTuple, Union + +from . import constants +from .errors import EntryNotFoundError +from .file_download import hf_hub_url +from .lfs import UploadInfo, lfs_upload, post_lfs_batch_info +from .utils import ( + FORBIDDEN_FOLDERS, + are_progress_bars_disabled, + chunk_iterable, + get_session, + hf_raise_for_status, + hf_thread_map, + http_backoff, + logging, + sha, + tqdm_stream_file, + validate_hf_hub_args, +) +from .utils import tqdm as hf_tqdm +from .utils._runtime import is_xet_available + + +if TYPE_CHECKING: + from .hf_api import CommitInfo, RepoFile + + +logger = logging.get_logger(__name__) + + +UploadMode = Literal["lfs", "regular"] + +# Max is 1,000 per request on the Hub for HfApi.get_paths_info +# Otherwise we get: +# HfHubHTTPError: 413 Client Error: Payload Too Large for url: https://huggingface.co/api/datasets/xxx (Request ID: xxx)\n\ntoo many parameters +# See https://github.com/huggingface/huggingface_hub/issues/1503 +FETCH_LFS_BATCH_SIZE = 500 +DUPLICATE_LFS_BATCH_SIZE = 500 + +UPLOAD_BATCH_MAX_NUM_FILES = 256 + + +@dataclass +class CommitOperationDelete: + """ + Data structure holding necessary info to delete a file or a folder from a repository + on the Hub. + + Args: + path_in_repo (`str`): + Relative filepath in the repo, for example: `"checkpoints/1fec34a/weights.bin"` + for a file or `"checkpoints/1fec34a/"` for a folder. + is_folder (`bool` or `Literal["auto"]`, *optional*) + Whether the Delete Operation applies to a folder or not. If "auto", the path + type (file or folder) is guessed automatically by looking if path ends with + a "/" (folder) or not (file). To explicitly set the path type, you can set + `is_folder=True` or `is_folder=False`. + """ + + path_in_repo: str + is_folder: bool | Literal["auto"] = "auto" + + def __post_init__(self): + self.path_in_repo = _validate_path_in_repo(self.path_in_repo) + + if self.is_folder == "auto": + self.is_folder = self.path_in_repo.endswith("/") + if not isinstance(self.is_folder, bool): + raise ValueError( + f"Wrong value for `is_folder`. Must be one of [`True`, `False`, `'auto'`]. Got '{self.is_folder}'." + ) + + +@dataclass +class CommitOperationCopy: + """ + Data structure holding necessary info to copy a file in a repository on the Hub. + + Both LFS files and regular files are supported. LFS files are copied server-side while regular files are + downloaded and re-uploaded as part of the commit. + + Cross-repository copies are supported by setting `src_repo_id` and `src_repo_type`. For cross-repo LFS copies, + the LFS objects are duplicated to the destination repository before the commit is created. This is handled + automatically by [`create_commit`]. Note that cross-repository copies only work within the same + [storage region](https://huggingface.co/docs/hub/storage-regions); copying across regions is not supported. + + Note: you can combine a [`CommitOperationCopy`] and a [`CommitOperationDelete`] to rename an LFS file on the Hub. + + Args: + src_path_in_repo (`str`): + Relative filepath in the repo of the file to be copied, e.g. `"checkpoints/1fec34a/weights.bin"`. + path_in_repo (`str`): + Relative filepath in the repo where to copy the file, e.g. `"checkpoints/1fec34a/weights_copy.bin"`. + src_revision (`str`, *optional*): + The git revision of the file to be copied. Can be any valid git revision. + Default to the target commit revision. + src_repo_id (`str`, *optional*): + The source repository to copy from (e.g. `"username/source-model"`). + Default to the destination repository (intra-repo copy). + src_repo_type (`str`, *optional*): + The type of the source repository (`"model"`, `"dataset"` or `"space"`). + Required when `src_repo_id` is set. + """ + + src_path_in_repo: str + path_in_repo: str + src_revision: str | None = None + src_repo_id: str | None = None + src_repo_type: str | None = None + # set to the OID of the file to be copied if it has already been uploaded + # useful to determine if a commit will be empty or not. + _src_oid: str | None = None + # set to the OID of the file to copy to if it has already been uploaded + # useful to determine if a commit will be empty or not. + _dest_oid: str | None = None + # set to True once cross-repo LFS files have been duplicated to the destination repo + _is_duplicated: bool = False + + def __post_init__(self): + self.src_path_in_repo = _validate_path_in_repo(self.src_path_in_repo) + self.path_in_repo = _validate_path_in_repo(self.path_in_repo) + if self.src_repo_id is not None and self.src_repo_type is None: + raise ValueError("`src_repo_type` is required when `src_repo_id` is set.") + if self.src_repo_type is not None and self.src_repo_id is None: + raise ValueError("`src_repo_id` is required when `src_repo_type` is set.") + + +@dataclass +class CommitOperationAdd: + """ + Data structure holding necessary info to upload a file to a repository on the Hub. + + Args: + path_in_repo (`str`): + Relative filepath in the repo, for example: `"checkpoints/1fec34a/weights.bin"` + path_or_fileobj (`str`, `Path`, `bytes`, or `BinaryIO`): + Either: + - a path to a local file (as `str` or `pathlib.Path`) to upload + - a buffer of bytes (`bytes`) holding the content of the file to upload + - a "file object" (subclass of `io.BufferedIOBase`), typically obtained + with `open(path, "rb")`. It must support `seek()` and `tell()` methods. + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If `path_or_fileobj` is not one of `str`, `Path`, `bytes` or `io.BufferedIOBase`. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If `path_or_fileobj` is a `str` or `Path` but not a path to an existing file. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If `path_or_fileobj` is a `io.BufferedIOBase` but it doesn't support both + `seek()` and `tell()`. + """ + + path_in_repo: str + path_or_fileobj: str | Path | bytes | BinaryIO + upload_info: UploadInfo = field(init=False, repr=False) + + # Internal attributes + + # set to "lfs" or "regular" once known + _upload_mode: UploadMode | None = field(init=False, repr=False, default=None) + + # set to True if .gitignore rules prevent the file from being uploaded as LFS + # (server-side check) + _should_ignore: bool | None = field(init=False, repr=False, default=None) + + # set to the remote OID of the file if it has already been uploaded + # useful to determine if a commit will be empty or not + _remote_oid: str | None = field(init=False, repr=False, default=None) + + # set to True once the file has been uploaded as LFS + _is_uploaded: bool = field(init=False, repr=False, default=False) + + # set to True once the file has been committed + _is_committed: bool = field(init=False, repr=False, default=False) + + def __post_init__(self) -> None: + """Validates `path_or_fileobj` and compute `upload_info`.""" + self.path_in_repo = _validate_path_in_repo(self.path_in_repo) + + # Validate `path_or_fileobj` value + if isinstance(self.path_or_fileobj, Path): + self.path_or_fileobj = str(self.path_or_fileobj) + if isinstance(self.path_or_fileobj, str): + path_or_fileobj = os.path.normpath(os.path.expanduser(self.path_or_fileobj)) + if not os.path.isfile(path_or_fileobj): + raise ValueError(f"Provided path: '{path_or_fileobj}' is not a file on the local file system") + elif not isinstance(self.path_or_fileobj, (io.BufferedIOBase, bytes)): + # ^^ Inspired from: https://stackoverflow.com/questions/44584829/how-to-determine-if-file-is-opened-in-binary-or-text-mode + raise ValueError( + "path_or_fileobj must be either an instance of str, bytes or" + " io.BufferedIOBase. If you passed a file-like object, make sure it is" + " in binary mode." + ) + if isinstance(self.path_or_fileobj, io.BufferedIOBase): + try: + self.path_or_fileobj.tell() + self.path_or_fileobj.seek(0, os.SEEK_CUR) + except (OSError, AttributeError) as exc: + raise ValueError( + "path_or_fileobj is a file-like object but does not implement seek() and tell()" + ) from exc + + # Compute "upload_info" attribute + if isinstance(self.path_or_fileobj, str): + self.upload_info = UploadInfo.from_path(self.path_or_fileobj) + elif isinstance(self.path_or_fileobj, bytes): + self.upload_info = UploadInfo.from_bytes(self.path_or_fileobj) + else: + self.upload_info = UploadInfo.from_fileobj(self.path_or_fileobj) + + @contextmanager + def as_file(self, with_tqdm: bool = False) -> Iterator[BinaryIO]: + """ + A context manager that yields a file-like object allowing to read the underlying + data behind `path_or_fileobj`. + + Args: + with_tqdm (`bool`, *optional*, defaults to `False`): + If True, iterating over the file object will display a progress bar. Only + works if the file-like object is a path to a file. Pure bytes and buffers + are not supported. + + Example: + + ```python + >>> operation = CommitOperationAdd( + ... path_in_repo="remote/dir/weights.h5", + ... path_or_fileobj="./local/weights.h5", + ... ) + CommitOperationAdd(path_in_repo='remote/dir/weights.h5', path_or_fileobj='./local/weights.h5') + + >>> with operation.as_file() as file: + ... content = file.read() + + >>> with operation.as_file(with_tqdm=True) as file: + ... while True: + ... data = file.read(1024) + ... if not data: + ... break + config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s] + + >>> with operation.as_file(with_tqdm=True) as file: + ... httpx.put(..., data=file) + config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s] + ``` + """ + if isinstance(self.path_or_fileobj, str) or isinstance(self.path_or_fileobj, Path): + if with_tqdm: + with tqdm_stream_file(self.path_or_fileobj) as file: + yield file + else: + with open(self.path_or_fileobj, "rb") as file: + yield file + elif isinstance(self.path_or_fileobj, bytes): + yield io.BytesIO(self.path_or_fileobj) + elif isinstance(self.path_or_fileobj, io.BufferedIOBase): + prev_pos = self.path_or_fileobj.tell() + yield self.path_or_fileobj + self.path_or_fileobj.seek(prev_pos, io.SEEK_SET) + + def b64content(self) -> bytes: + """ + The base64-encoded content of `path_or_fileobj` + + Returns: `bytes` + """ + with self.as_file() as file: + return base64.b64encode(file.read()) + + @property + def _local_oid(self) -> str | None: + """Return the OID of the local file. + + This OID is then compared to `self._remote_oid` to check if the file has changed compared to the remote one. + If the file did not change, we won't upload it again to prevent empty commits. + + For LFS files, the OID corresponds to the SHA256 of the file content (used a LFS ref). + For regular files, the OID corresponds to the SHA1 of the file content. + Note: this is slightly different to git OID computation since the oid of an LFS file is usually the git-SHA1 of the + pointer file content (not the actual file content). However, using the SHA256 is enough to detect changes + and more convenient client-side. + """ + if self._upload_mode is None: + return None + elif self._upload_mode == "lfs": + return self.upload_info.sha256.hex() + else: + # Regular file => compute sha1 + # => no need to read by chunk since the file is guaranteed to be <=5MB. + with self.as_file() as file: + return sha.git_hash(file.read()) + + +def _validate_path_in_repo(path_in_repo: str) -> str: + # Validate `path_in_repo` value to prevent a server-side issue + if path_in_repo.startswith("/"): + path_in_repo = path_in_repo[1:] + if path_in_repo == "." or path_in_repo == ".." or path_in_repo.startswith("../"): + raise ValueError(f"Invalid `path_in_repo` in CommitOperation: '{path_in_repo}'") + if path_in_repo.startswith("./"): + path_in_repo = path_in_repo[2:] + for forbidden in FORBIDDEN_FOLDERS: + if any(part == forbidden for part in path_in_repo.split("/")): + raise ValueError( + f"Invalid `path_in_repo` in CommitOperation: cannot update files under a '{forbidden}/' folder (path:" + f" '{path_in_repo}')." + ) + return path_in_repo + + +CommitOperation = Union[CommitOperationAdd, CommitOperationCopy, CommitOperationDelete] + + +def _warn_on_overwriting_operations(operations: list[CommitOperation]) -> None: + """ + Warn user when a list of operations is expected to overwrite itself in a single + commit. + + Rules: + - If a filepath is updated by multiple `CommitOperationAdd` operations, a warning + message is triggered. + - If a filepath is updated at least once by a `CommitOperationAdd` and then deleted + by a `CommitOperationDelete`, a warning is triggered. + - If a `CommitOperationDelete` deletes a filepath that is then updated by a + `CommitOperationAdd`, no warning is triggered. This is usually useless (no need to + delete before upload) but can happen if a user deletes an entire folder and then + add new files to it. + """ + nb_additions_per_path: dict[str, int] = defaultdict(int) + for operation in operations: + path_in_repo = operation.path_in_repo + if isinstance(operation, CommitOperationAdd): + if nb_additions_per_path[path_in_repo] > 0: + warnings.warn( + "About to update multiple times the same file in the same commit:" + f" '{path_in_repo}'. This can cause undesired inconsistencies in" + " your repo." + ) + nb_additions_per_path[path_in_repo] += 1 + for parent in PurePosixPath(path_in_repo).parents: + # Also keep track of number of updated files per folder + # => warns if deleting a folder overwrite some contained files + nb_additions_per_path[str(parent)] += 1 + if isinstance(operation, CommitOperationDelete): + if nb_additions_per_path[str(PurePosixPath(path_in_repo))] > 0: + if operation.is_folder: + warnings.warn( + "About to delete a folder containing files that have just been" + f" updated within the same commit: '{path_in_repo}'. This can" + " cause undesired inconsistencies in your repo." + ) + else: + warnings.warn( + "About to delete a file that have just been updated within the" + f" same commit: '{path_in_repo}'. This can cause undesired" + " inconsistencies in your repo." + ) + + +@validate_hf_hub_args +def _upload_files( + *, + additions: list[CommitOperationAdd], + repo_type: str, + repo_id: str, + headers: dict[str, str], + endpoint: str | None = None, + num_threads: int = 5, + revision: str | None = None, + create_pr: bool | None = None, +): + """ + Uploads the files through the Xet protocol if possible, otherwise through the legacy LFS protocol. + + The Xet path does not require any Python-side sha256 computation: hashing happens inside `hf_xet` + while chunking the files (single read pass) and is backfilled on the operations afterwards. + """ + has_buffered_io_data = any(isinstance(op.path_or_fileobj, io.BufferedIOBase) for op in additions) + if is_xet_available(): + if not has_buffered_io_data: + _upload_xet_files( + additions=additions, + repo_type=repo_type, + repo_id=repo_id, + headers=headers, + endpoint=endpoint, + revision=revision, + create_pr=create_pr, + ) + return + logger.warning( + "Uploading files as a binary IO buffer is not supported by Xet Storage. Falling back to HTTP upload." + ) + + # Legacy LFS path: sha256 is required by the LFS batch endpoint => compute missing ones (in parallel). + _compute_missing_sha256s(additions, num_threads=num_threads) + + lfs_actions: list[dict[str, Any]] = [] + lfs_oid2addop: dict[str, CommitOperationAdd] = {} + for chunk in chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES): + chunk_list = [op for op in chunk] + actions_chunk, errors_chunk, _ = post_lfs_batch_info( + upload_infos=[op.upload_info for op in chunk_list], + repo_id=repo_id, + repo_type=repo_type, + revision=revision, + endpoint=endpoint, + headers=headers, + token=None, # already passed in 'headers' + transfers=["basic", "multipart"], + ) + if errors_chunk: + message = "\n".join( + [ + f"Encountered error for file with OID {err.get('oid')}: `{err.get('error', {}).get('message')}" + for err in errors_chunk + ] + ) + raise ValueError(f"LFS batch API returned errors:\n{message}") + lfs_actions.extend(actions_chunk) + for op in chunk_list: + lfs_oid2addop[op.upload_info.sha256.hex()] = op + + if len(lfs_actions) > 0: + _upload_lfs_files( + actions=lfs_actions, + oid2addop=lfs_oid2addop, + headers=headers, + endpoint=endpoint, + num_threads=num_threads, + ) + + +def _compute_missing_sha256s(additions: list[CommitOperationAdd], num_threads: int) -> None: + """Compute the sha256 of the operations that don't have one yet, in parallel.""" + not_hashed = [op for op in additions if not op.upload_info.is_hashed] + if len(not_hashed) == 0: + return + logger.info(f"Computing sha256 for {len(not_hashed)} files.") + if len(not_hashed) == 1: + _ = not_hashed[0].upload_info.sha256 + return + with ThreadPoolExecutor(max_workers=num_threads) as executor: + list(executor.map(lambda op: op.upload_info.sha256, not_hashed)) + + +@validate_hf_hub_args +def _upload_lfs_files( + *, + actions: list[dict[str, Any]], + oid2addop: dict[str, CommitOperationAdd], + headers: dict[str, str], + endpoint: str | None = None, + num_threads: int = 5, +): + """ + Uploads the content of `additions` to the Hub using the large file storage protocol. + + Relevant external documentation: + - LFS Batch API: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md + + Args: + actions (`list[dict[str, Any]]`): + LFS batch actions returned by the server. + oid2addop (`dict[str, CommitOperationAdd]`): + A dictionary mapping the OID of the file to the corresponding `CommitOperationAdd` object. + headers (`dict[str, str]`): + Headers to use for the request, including authorization headers and user agent. + endpoint (`str`, *optional*): + The endpoint to use for the request. Defaults to `constants.ENDPOINT`. + num_threads (`int`, *optional*): + The number of concurrent threads to use when uploading. Defaults to 5. + + Raises: + [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) + If an upload failed for any reason + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + headers (`dict[str, str]`): + Headers to use for the request, including authorization headers and user agent. + num_threads (`int`, *optional*): + The number of concurrent threads to use when uploading. Defaults to 5. + revision (`str`, *optional*): + The git revision to upload to. + + Raises: + [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) + If an upload failed for any reason + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the server returns malformed responses + [`HfHubHTTPError`] + If the LFS batch endpoint returned an HTTP error. + """ + # Filter out files already present upstream + filtered_actions = [] + for action in actions: + if action.get("actions") is None: + logger.debug( + f"Content of file {oid2addop[action['oid']].path_in_repo} is already present upstream - skipping upload." + ) + else: + filtered_actions.append(action) + + # Upload according to server-provided actions + def _wrapped_lfs_upload(batch_action) -> None: + try: + operation = oid2addop[batch_action["oid"]] + lfs_upload(operation=operation, lfs_batch_action=batch_action, headers=headers, endpoint=endpoint) + except Exception as exc: + raise RuntimeError(f"Error while uploading '{operation.path_in_repo}' to the Hub.") from exc + + if len(filtered_actions) == 1: + logger.debug("Uploading 1 LFS file to the Hub") + _wrapped_lfs_upload(filtered_actions[0]) + else: + logger.debug( + f"Uploading {len(filtered_actions)} LFS files to the Hub using up to {num_threads} threads concurrently" + ) + hf_thread_map( + _wrapped_lfs_upload, + filtered_actions, + desc=f"Upload {len(filtered_actions)} LFS files", + max_workers=num_threads, + tqdm_class=hf_tqdm, + ) + + +@validate_hf_hub_args +def _upload_xet_files( + *, + additions: list[CommitOperationAdd], + repo_type: str, + repo_id: str, + headers: dict[str, str], + endpoint: str | None = None, + revision: str | None = None, + create_pr: bool | None = None, +): + """ + Uploads the content of `additions` to the Hub using the xet storage protocol. + This chunks the files and deduplicates the chunks before uploading them to xetcas storage. + + Args: + additions (`` of `CommitOperationAdd`): + The files to be uploaded. + repo_type (`str`): + Type of the repo (e.g. `"model"`, `"dataset"`, `"space"`). + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + headers (`dict[str, str]`): + Headers to use for the request, including authorization headers and user agent. + endpoint: (`str`, *optional*): + The endpoint to use for the xetcas service. Defaults to `constants.ENDPOINT`. + revision (`str`, *optional*): + The git revision to upload to. + create_pr (`bool`, *optional*): + Whether or not to create a Pull Request with that commit. + + Raises: + [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) + If an upload failed for any reason. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the server returns malformed responses or if the user is unauthorized to upload to xet storage. + [`HfHubHTTPError`] + If the LFS batch endpoint returned an HTTP error. + + **How it works:** + The file upload system uses Xet storage, which is a content-addressable storage system that breaks files into chunks + for efficient storage and transfer. + + ``session.new_upload_commit()`` manages uploading files by: + - Registering upload tasks and starting upload immediately in the background + - Breaking files into smaller chunks for efficient storage + - Avoiding duplicate storage by recognizing identical chunks across files + - Connecting to a storage server (CAS server) that manages these chunks + + Authentication works transparently: the upload commit accepts a ``token_refresh_url`` + that is used to refresh the short-lived xet write token as needed. + + The upload process works like this: + 1. Upload tasks run in parallel: + 1.1. Read the file content from a path, bytes array, or a stream. + 1.2. Split the file content into smaller chunks based on content patterns: each chunk gets a unique ID based on what's in it. + 1.3. For each chunk: + - Check if it already exists in storage. + - Skip uploading chunks that already exist. + 1.4. Group chunks into larger blocks for efficient transfer. + 1.5. Upload these blocks to the storage server. + 1.6. Assemble the file manifest locally and send it to the server for validation. + 2. Return reference files that contain information about the uploaded files, which can be used later to download them. + """ + if len(additions) == 0: + return + + # at this point, we know that hf_xet is installed + from .utils._xet import ( + XetTokenType, + abort_xet_session, + get_xet_session, + xet_connection_info_refresh_url, + xet_headers_without_auth, + ) + from .utils._xet_progress_reporting import XetUploadProgressReporter + + refresh_url = xet_connection_info_refresh_url( + token_type=XetTokenType.WRITE, + repo_id=repo_id, + repo_type=repo_type, + revision=revision, + endpoint=endpoint, + ) + if create_pr: + refresh_url += "?create_pr=1" + + xet_headers = xet_headers_without_auth(headers) + + import hf_xet + + session = get_xet_session() + progress = None + + def _sha256_arg(op: CommitOperationAdd): + # If the sha256 is already known, pass it to avoid recomputation. Otherwise let hf_xet + # compute it while chunking the file (single read pass) and backfill it afterwards. + return op.upload_info.sha256.hex() if op.upload_info.is_hashed else hf_xet.COMPUTE_SHA256 + + try: + if not are_progress_bars_disabled(): + progress = XetUploadProgressReporter() + progress_callback = progress.update_progress + else: + progress_callback = None + + all_bytes_ops = [op for op in additions if isinstance(op.path_or_fileobj, bytes)] + all_paths_ops = [op for op in additions if isinstance(op.path_or_fileobj, (str, Path))] + + handles: list[tuple[CommitOperationAdd, Any]] = [] + with session.new_upload_commit( + token_refresh_url=refresh_url, + token_refresh_headers=headers, + custom_headers=xet_headers, + progress_callback=progress_callback, + ) as commit: + for op in all_paths_ops: + handles.append((op, commit.start_upload_file(str(op.path_or_fileobj), sha256=_sha256_arg(op)))) + for op in all_bytes_ops: + handles.append((op, commit.start_upload_bytes(op.path_or_fileobj, sha256=_sha256_arg(op)))) + + # Backfill sha256 computed by hf_xet (needed later for the commit payload). + for op, handle in handles: + if not op.upload_info.is_hashed: + op.upload_info.sha256 = bytes.fromhex(handle.result().xet_info.sha256) + except KeyboardInterrupt: + abort_xet_session() + raise + finally: + if progress is not None: + progress.close() + + +def _validate_preupload_info(preupload_info: dict): + files = preupload_info.get("files") + if not isinstance(files, list): + raise ValueError("preupload_info is improperly formatted") + for file_info in files: + if not ( + isinstance(file_info, dict) + and isinstance(file_info.get("path"), str) + and isinstance(file_info.get("uploadMode"), str) + and (file_info["uploadMode"] in ("lfs", "regular")) + ): + raise ValueError("preupload_info is improperly formatted:") + return preupload_info + + +@validate_hf_hub_args +def _fetch_upload_modes( + additions: Iterable[CommitOperationAdd], + repo_type: str, + repo_id: str, + headers: dict[str, str], + revision: str, + endpoint: str | None = None, + create_pr: bool = False, + gitignore_content: str | None = None, +) -> None: + """ + Requests the Hub "preupload" endpoint to determine whether each input file should be uploaded as a regular git blob, + as a git LFS blob, or as a XET file. Input `additions` are mutated in-place with the upload mode. + + Args: + additions (`Iterable` of :class:`CommitOperationAdd`): + Iterable of :class:`CommitOperationAdd` describing the files to + upload to the Hub. + repo_type (`str`): + Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + headers (`dict[str, str]`): + Headers to use for the request, including authorization headers and user agent. + revision (`str`): + The git revision to upload the files to. Can be any valid git revision. + gitignore_content (`str`, *optional*): + The content of the `.gitignore` file to know which files should be ignored. The order of priority + is to first check if `gitignore_content` is passed, then check if the `.gitignore` file is present + in the list of files to commit and finally default to the `.gitignore` file already hosted on the Hub + (if any). + Raises: + [`~utils.HfHubHTTPError`] + If the Hub API returned an error. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the Hub API response is improperly formatted. + """ + endpoint = endpoint if endpoint is not None else constants.ENDPOINT + + # Fetch upload mode (LFS or regular) chunk by chunk. + upload_modes: dict[str, UploadMode] = {} + should_ignore_info: dict[str, bool] = {} + oid_info: dict[str, str | None] = {} + + for chunk in chunk_iterable(additions, 256): + payload: dict = { + "files": [ + { + "path": op.path_in_repo, + "sample": base64.b64encode(op.upload_info.sample).decode("ascii"), + "size": op.upload_info.size, + } + for op in chunk + ] + } + if gitignore_content is not None: + payload["gitIgnore"] = gitignore_content + + resp = http_backoff( + "POST", + f"{endpoint}/api/{repo_type}s/{repo_id}/preupload/{revision}", + json=payload, + headers=headers, + params={"create_pr": "1"} if create_pr else None, + ) + hf_raise_for_status(resp) + preupload_info = _validate_preupload_info(resp.json()) + upload_modes.update(**{file["path"]: file["uploadMode"] for file in preupload_info["files"]}) + should_ignore_info.update(**{file["path"]: file["shouldIgnore"] for file in preupload_info["files"]}) + oid_info.update(**{file["path"]: file.get("oid") for file in preupload_info["files"]}) + + # Set upload mode for each addition operation + for addition in additions: + addition._upload_mode = upload_modes[addition.path_in_repo] + addition._should_ignore = should_ignore_info[addition.path_in_repo] + addition._remote_oid = oid_info[addition.path_in_repo] + + # Empty files cannot be uploaded as LFS (S3 would fail with a 501 Not Implemented) + # => empty files are uploaded as "regular" to still allow users to commit them. + for addition in additions: + if addition.upload_info.size == 0: + addition._upload_mode = "regular" + + +class _CopySource(NamedTuple): + src_repo_id: str | None + src_repo_type: str | None + path: str + revision: str | None + + +@validate_hf_hub_args +def _fetch_files_to_copy( + copies: Iterable[CommitOperationCopy], + repo_type: str, + repo_id: str, + headers: dict[str, str], + revision: str, + endpoint: str | None = None, +) -> dict[_CopySource, Union["RepoFile", bytes]]: + """ + Fetch information about the files to copy. + + For LFS files, we only need their metadata (file size and sha256) while for regular files + we need to download the raw content from the Hub. + + Args: + copies (`Iterable` of :class:`CommitOperationCopy`): + Iterable of :class:`CommitOperationCopy` describing the files to + copy on the Hub. + repo_type (`str`): + Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + headers (`dict[str, str]`): + Headers to use for the request, including authorization headers and user agent. + revision (`str`): + The git revision to upload the files to. Can be any valid git revision. + + Returns: `dict[_CopySource, Union[RepoFile, bytes]]]` + Key is `(src_repo_id, src_repo_type, path, revision)`. For intra-repo copies, + `src_repo_id` and `src_repo_type` are `None`. + Value is the raw content as bytes (for regular files) or the file information as a RepoFile (for LFS files). + + Raises: + [`~utils.HfHubHTTPError`] + If the Hub API returned an error. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the Hub API response is improperly formatted. + """ + from .hf_api import HfApi, RepoFolder + + hf_api = HfApi(endpoint=endpoint, headers=headers) + copies = list(copies) + files_to_copy: dict[_CopySource, Union["RepoFile", bytes]] = {} + oid_info: dict[tuple[str, str | None], str | None] = {} + + # 1. Fetch OIDs for destination paths in batches. + dest_paths = [op.path_in_repo for op in copies] + for batch in chunk_iterable(dest_paths, FETCH_LFS_BATCH_SIZE): + dest_repo_files = hf_api.get_paths_info( + repo_id=repo_id, + paths=list(batch), + revision=revision, + repo_type=repo_type, + ) + for file in dest_repo_files: + if not isinstance(file, RepoFolder): + oid_info[(file.path, revision)] = file.blob_id + + # 2. Fetch source file info, grouped by (src_repo_id, src_repo_type, src_revision). + copies.sort(key=lambda op: (op.src_repo_id or "", op.src_repo_type or "", op.src_revision or "")) + for (src_repo_id_key, src_repo_type_key, src_revision_key), group in groupby( + copies, key=lambda op: (op.src_repo_id, op.src_repo_type, op.src_revision) + ): + operations = list(group) + is_cross_repo = src_repo_id_key is not None + eff_repo_id = src_repo_id_key or repo_id + eff_repo_type = src_repo_type_key or repo_type + eff_revision = src_revision_key or ("main" if is_cross_repo else revision) + + src_paths = [op.src_path_in_repo for op in operations] + for paths_batch in chunk_iterable(src_paths, FETCH_LFS_BATCH_SIZE): + src_repo_files = hf_api.get_paths_info( + repo_id=eff_repo_id, + paths=list(paths_batch), + revision=eff_revision, + repo_type=eff_repo_type, + ) + for src_repo_file in src_repo_files: + if isinstance(src_repo_file, RepoFolder): + raise NotImplementedError("Copying a folder is not implemented.") + source = _CopySource(src_repo_id_key, src_repo_type_key, src_repo_file.path, src_revision_key) + if src_repo_file.lfs: + files_to_copy[source] = src_repo_file + else: + url = hf_hub_url( + endpoint=endpoint, + repo_type=eff_repo_type, + repo_id=eff_repo_id, + revision=eff_revision, + filename=src_repo_file.path, + ) + response = get_session().get(url, headers=headers) + hf_raise_for_status(response) + files_to_copy[source] = response.content + if not is_cross_repo: + oid_info[(src_repo_file.path, src_revision_key)] = src_repo_file.blob_id + + for operation in operations: + key = _CopySource(src_repo_id_key, src_repo_type_key, operation.src_path_in_repo, src_revision_key) + if key not in files_to_copy: + source_desc = f" from {eff_repo_type}s/{eff_repo_id}" if is_cross_repo else "" + raise EntryNotFoundError( + f"Cannot copy {operation.src_path_in_repo} at revision " + f"{eff_revision}{source_desc}: file is missing on repo." + ) + operation._dest_oid = oid_info.get((operation.path_in_repo, revision)) + if not is_cross_repo: + operation._src_oid = oid_info.get((operation.src_path_in_repo, operation.src_revision)) + + return files_to_copy + + +def _prepare_commit_payload( + operations: Iterable[CommitOperation], + files_to_copy: dict[_CopySource, Union["RepoFile", bytes]], + commit_message: str, + commit_description: str | None = None, + parent_commit: str | None = None, +) -> Iterable[dict[str, Any]]: + """ + Builds the payload to POST to the `/commit` API of the Hub. + + Payload is returned as an iterator so that it can be streamed as a ndjson in the + POST request. + + For more information, see: + - https://github.com/huggingface/huggingface_hub/issues/1085#issuecomment-1265208073 + - http://ndjson.org/ + """ + commit_description = commit_description if commit_description is not None else "" + + # 1. Send a header item with the commit metadata + header_value = {"summary": commit_message, "description": commit_description} + if parent_commit is not None: + header_value["parentCommit"] = parent_commit + yield {"key": "header", "value": header_value} + + nb_ignored_files = 0 + + # 2. Send operations, one per line + for operation in operations: + # Skip ignored files + if isinstance(operation, CommitOperationAdd) and operation._should_ignore: + logger.debug(f"Skipping file '{operation.path_in_repo}' in commit (ignored by gitignore file).") + nb_ignored_files += 1 + continue + + # 2.a. Case adding a regular file + if isinstance(operation, CommitOperationAdd) and operation._upload_mode == "regular": + yield { + "key": "file", + "value": { + "content": operation.b64content().decode(), + "path": operation.path_in_repo, + "encoding": "base64", + }, + } + # 2.b. Case adding an LFS file + elif isinstance(operation, CommitOperationAdd) and operation._upload_mode == "lfs": + yield { + "key": "lfsFile", + "value": { + "path": operation.path_in_repo, + "algo": "sha256", + "oid": operation.upload_info.sha256.hex(), + "size": operation.upload_info.size, + }, + } + # 2.c. Case deleting a file or folder + elif isinstance(operation, CommitOperationDelete): + yield { + "key": "deletedFolder" if operation.is_folder else "deletedFile", + "value": {"path": operation.path_in_repo}, + } + # 2.d. Case copying a file or folder + elif isinstance(operation, CommitOperationCopy): + source = _CopySource( + operation.src_repo_id, + operation.src_repo_type, + operation.src_path_in_repo, + operation.src_revision, + ) + file_to_copy = files_to_copy[source] + if isinstance(file_to_copy, bytes): + yield { + "key": "file", + "value": { + "content": base64.b64encode(file_to_copy).decode(), + "path": operation.path_in_repo, + "encoding": "base64", + }, + } + elif file_to_copy.lfs: + yield { + "key": "lfsFile", + "value": { + "path": operation.path_in_repo, + "algo": "sha256", + "oid": file_to_copy.lfs.sha256, + }, + } + else: + raise ValueError( + "Malformed files_to_copy (should be raw file content as bytes or RepoFile objects with LFS info." + ) + # 2.e. Never expected to happen + else: + raise ValueError( + f"Unknown operation to commit. Operation: {operation}. Upload mode:" + f" {getattr(operation, '_upload_mode', None)}" + ) + + if nb_ignored_files > 0: + logger.info(f"Skipped {nb_ignored_files} file(s) in commit (ignored by gitignore file).") + + +def _send_commit( + *, + operations: Iterable[CommitOperation], + files_to_copy: dict[_CopySource, Union["RepoFile", bytes]], + commit_message: str, + commit_description: str, + repo_type: str, + repo_id: str, + headers: dict[str, str], + revision: str, + endpoint: str | None = None, + parent_commit: str | None = None, + create_pr: bool = False, + hot_reload: bool = False, + retry_on_error: bool = False, +) -> "CommitInfo": + """ + Build the ndjson payload for `operations` and POST it to the `/commit` endpoint of the Hub. + + `revision` must already be URL-quoted. Operations must be ready to commit: upload modes fetched + and LFS/Xet content uploaded. + + If `retry_on_error` is `True`, the POST is retried with exponential backoff on transient errors. + Caution: if a commit succeeds but its response is lost, the retry will create a duplicate commit + (and a duplicate pull request if `create_pr` is set) — only enable it when this is acceptable. + + Returns a [`CommitInfo`] built from the server response. + + Raises: + [`~utils.HfHubHTTPError`] + If the Hub API returned an error. + """ + endpoint = endpoint if endpoint is not None else constants.ENDPOINT + payload = _prepare_commit_payload( + operations=operations, + files_to_copy=files_to_copy, + commit_message=commit_message, + commit_description=commit_description, + parent_commit=parent_commit, + ) + data = b"".join(json.dumps(item).encode() + b"\n" for item in payload) + # See https://github.com/huggingface/huggingface_hub/issues/1085#issuecomment-1265208073 + headers = {"Content-Type": "application/x-ndjson", **headers} + url = f"{endpoint}/api/{repo_type}s/{repo_id}/commit/{revision}" + + params: dict[str, str] = {} + if create_pr: + params["create_pr"] = "1" + if hot_reload: + params["hot_reload"] = "1" + + if retry_on_error: + response = http_backoff("POST", url, headers=headers, content=data, params=params) + else: + response = get_session().post(url, headers=headers, content=data, params=params) + hf_raise_for_status(response, endpoint_name="commit") + + commit_data = response.json() + from .hf_api import CommitInfo + + return CommitInfo( + commit_url=commit_data["commitUrl"], + commit_message=commit_message, + commit_description=commit_description, + oid=commit_data["commitOid"], + pr_url=commit_data["pullRequestUrl"] if create_pr else None, + _endpoint=endpoint, + ) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_commit_scheduler.py b/venv/lib/python3.12/site-packages/huggingface_hub/_commit_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..4aa4b8bf19400f318251838da0245f1aad2f1dbc --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_commit_scheduler.py @@ -0,0 +1,353 @@ +import atexit +import logging +import os +import time +from concurrent.futures import Future +from dataclasses import dataclass +from io import SEEK_END, SEEK_SET, BytesIO +from pathlib import Path +from threading import Lock, Thread +from typing import Optional + +from .hf_api import DEFAULT_IGNORE_PATTERNS, CommitInfo, CommitOperationAdd, HfApi +from .utils import filter_repo_objects + + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _FileToUpload: + """Temporary dataclass to store info about files to upload. Not meant to be used directly.""" + + local_path: Path + path_in_repo: str + size_limit: int + last_modified: float + + +class CommitScheduler: + """ + Scheduler to upload a local folder to the Hub at regular intervals (e.g. push to hub every 5 minutes). + + The recommended way to use the scheduler is to use it as a context manager. This ensures that the scheduler is + properly stopped and the last commit is triggered when the script ends. The scheduler can also be stopped manually + with the `stop` method. Checkout the [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#scheduled-uploads) + to learn more about how to use it. + + Args: + repo_id (`str`): + The id of the repo to commit to. + folder_path (`str` or `Path`): + Path to the local folder to upload regularly. + every (`int` or `float`, *optional*): + The number of minutes between each commit. Defaults to 5 minutes. + path_in_repo (`str`, *optional*): + Relative path of the directory in the repo, for example: `"checkpoints/"`. Defaults to the root folder + of the repository. + repo_type (`str`, *optional*): + The type of the repo to commit to. Defaults to `model`. + revision (`str`, *optional*): + The revision of the repo to commit to. Defaults to `main`. + private (`bool`, *optional*): + Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists. + token (`str`, *optional*): + The token to use to commit to the repo. Defaults to the token saved on the machine. + allow_patterns (`list[str]` or `str`, *optional*): + If provided, only files matching at least one pattern are uploaded. + ignore_patterns (`list[str]` or `str`, *optional*): + If provided, files matching any of the patterns are not uploaded. + squash_history (`bool`, *optional*): + Whether to squash the history of the repo after each commit. Defaults to `False`. Squashing commits is + useful to avoid degraded performances on the repo when it grows too large. + hf_api (`HfApi`, *optional*): + The [`HfApi`] client to use to commit to the Hub. Can be set with custom settings (user agent, token,...). + + Example: + ```py + >>> from pathlib import Path + >>> from huggingface_hub import CommitScheduler + + # Scheduler uploads every 10 minutes + >>> csv_path = Path("watched_folder/data.csv") + >>> CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path=csv_path.parent, every=10) + + >>> with csv_path.open("a") as f: + ... f.write("first line") + + # Some time later (...) + >>> with csv_path.open("a") as f: + ... f.write("second line") + ``` + + Example using a context manager: + ```py + >>> from pathlib import Path + >>> from huggingface_hub import CommitScheduler + + >>> with CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path="watched_folder", every=10) as scheduler: + ... csv_path = Path("watched_folder/data.csv") + ... with csv_path.open("a") as f: + ... f.write("first line") + ... (...) + ... with csv_path.open("a") as f: + ... f.write("second line") + + # Scheduler is now stopped and last commit have been triggered + ``` + """ + + def __init__( + self, + *, + repo_id: str, + folder_path: str | Path, + every: int | float = 5, + path_in_repo: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + private: bool | None = None, + token: str | None = None, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + squash_history: bool = False, + hf_api: Optional["HfApi"] = None, + ) -> None: + self.api = hf_api or HfApi(token=token) + + # Folder + self.folder_path = Path(folder_path).expanduser().resolve() + self.path_in_repo = path_in_repo or "" + self.allow_patterns = allow_patterns + + if ignore_patterns is None: + ignore_patterns = [] + elif isinstance(ignore_patterns, str): + ignore_patterns = [ignore_patterns] + self.ignore_patterns = ignore_patterns + DEFAULT_IGNORE_PATTERNS + + if self.folder_path.is_file(): + raise ValueError(f"'folder_path' must be a directory, not a file: '{self.folder_path}'.") + self.folder_path.mkdir(parents=True, exist_ok=True) + + # Repository + repo_url = self.api.create_repo(repo_id=repo_id, private=private, repo_type=repo_type, exist_ok=True) + self.repo_id = repo_url.repo_id + self.repo_type = repo_type + self.revision = revision + self.token = token + + # Keep track of already uploaded files + self.last_uploaded: dict[Path, float] = {} # key is local path, value is timestamp + + # Scheduler + if not every > 0: + raise ValueError(f"'every' must be a positive integer, not '{every}'.") + self.lock = Lock() + self.every = every + self.squash_history = squash_history + + logger.info(f"Scheduled job to push '{self.folder_path}' to '{self.repo_id}' every {self.every} minutes.") + self._scheduler_thread = Thread(target=self._run_scheduler, daemon=True) + self._scheduler_thread.start() + atexit.register(self._push_to_hub) + + self.__stopped = False + + def stop(self) -> None: + """Stop the scheduler. + + A stopped scheduler cannot be restarted. Mostly for tests purposes. + """ + self.__stopped = True + + def __enter__(self) -> "CommitScheduler": + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + # Upload last changes before exiting + self.trigger().result() + self.stop() + return + + def _run_scheduler(self) -> None: + """Dumb thread waiting between each scheduled push to Hub.""" + while True: + self.last_future = self.trigger() + time.sleep(self.every * 60) + if self.__stopped: + break + + def trigger(self) -> Future: + """Trigger a `push_to_hub` and return a future. + + This method is automatically called every `every` minutes. You can also call it manually to trigger a commit + immediately, without waiting for the next scheduled commit. + """ + return self.api.run_as_future(self._push_to_hub) + + def _push_to_hub(self) -> CommitInfo | None: + if self.__stopped: # If stopped, already scheduled commits are ignored + return None + + logger.info("(Background) scheduled commit triggered.") + try: + value = self.push_to_hub() + if self.squash_history: + logger.info("(Background) squashing repo history.") + self.api.super_squash_history(repo_id=self.repo_id, repo_type=self.repo_type, branch=self.revision) + return value + except Exception as e: + logger.error(f"Error while pushing to Hub: {e}") # Depending on the setup, error might be silenced + raise + + def push_to_hub(self) -> CommitInfo | None: + """ + Push folder to the Hub and return the commit info. + + > [!WARNING] + > This method is not meant to be called directly. It is run in the background by the scheduler, respecting a + > queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency + > issues. + + The default behavior of `push_to_hub` is to assume an append-only folder. It lists all files in the folder and + uploads only changed files. If no changes are found, the method returns without committing anything. If you want + to change this behavior, you can inherit from [`CommitScheduler`] and override this method. This can be useful + for example to compress data together in a single file before committing. For more details and examples, check + out our [integration guide](https://huggingface.co/docs/huggingface_hub/main/en/guides/upload#scheduled-uploads). + """ + # Check files to upload (with lock) + with self.lock: + logger.debug("Listing files to upload for scheduled commit.") + + # List files from folder (taken from `_prepare_upload_folder_additions`) + relpath_to_abspath = { + path.relative_to(self.folder_path).as_posix(): path + for path in sorted(self.folder_path.glob("**/*")) # sorted to be deterministic + if path.is_file() + } + prefix = f"{self.path_in_repo.strip('/')}/" if self.path_in_repo else "" + + # Filter with pattern + filter out unchanged files + retrieve current file size + files_to_upload: list[_FileToUpload] = [] + for relpath in filter_repo_objects( + relpath_to_abspath.keys(), allow_patterns=self.allow_patterns, ignore_patterns=self.ignore_patterns + ): + local_path = relpath_to_abspath[relpath] + stat = local_path.stat() + if self.last_uploaded.get(local_path) is None or self.last_uploaded[local_path] != stat.st_mtime: + files_to_upload.append( + _FileToUpload( + local_path=local_path, + path_in_repo=prefix + relpath, + size_limit=stat.st_size, + last_modified=stat.st_mtime, + ) + ) + + # Return if nothing to upload + if len(files_to_upload) == 0: + logger.debug("Dropping schedule commit: no changed file to upload.") + return None + + # Convert `_FileToUpload` as `CommitOperationAdd` (=> compute file shas + limit to file size) + logger.debug("Removing unchanged files since previous scheduled commit.") + add_operations = [ + CommitOperationAdd( + # Cap the file to its current size, even if the user append data to it while a scheduled commit is happening + path_or_fileobj=PartialFileIO(file_to_upload.local_path, size_limit=file_to_upload.size_limit), + path_in_repo=file_to_upload.path_in_repo, + ) + for file_to_upload in files_to_upload + ] + + # Upload files (append mode expected - no need for lock) + logger.debug("Uploading files for scheduled commit.") + commit_info = self.api.create_commit( + repo_id=self.repo_id, + repo_type=self.repo_type, + operations=add_operations, + commit_message="Scheduled Commit", + revision=self.revision, + ) + + # Successful commit: keep track of the latest "last_modified" for each file + for file in files_to_upload: + self.last_uploaded[file.local_path] = file.last_modified + return commit_info + + +class PartialFileIO(BytesIO): + """A file-like object that reads only the first part of a file. + + Useful to upload a file to the Hub when the user might still be appending data to it. Only the first part of the + file is uploaded (i.e. the part that was available when the filesystem was first scanned). + + In practice, only used internally by the CommitScheduler to regularly push a folder to the Hub with minimal + disturbance for the user. The object is passed to `CommitOperationAdd`. + + Only supports `read`, `tell` and `seek` methods. + + Args: + file_path (`str` or `Path`): + Path to the file to read. + size_limit (`int`): + The maximum number of bytes to read from the file. If the file is larger than this, only the first part + will be read (and uploaded). + """ + + def __init__(self, file_path: str | Path, size_limit: int) -> None: + self._file_path = Path(file_path) + self._file = self._file_path.open("rb") + self._size_limit = min(size_limit, os.fstat(self._file.fileno()).st_size) + + def __del__(self) -> None: + self._file.close() + return super().__del__() + + def __repr__(self) -> str: + return f"" + + def __len__(self) -> int: + return self._size_limit + + def __getattribute__(self, name: str): + if name.startswith("_") or name in ("read", "tell", "seek", "fileno"): # only 4 public methods supported + return super().__getattribute__(name) + raise NotImplementedError(f"PartialFileIO does not support '{name}'.") + + def fileno(self): + raise AttributeError("PartialFileIO does not have a fileno.") + + def tell(self) -> int: + """Return the current file position.""" + return self._file.tell() + + def seek(self, __offset: int, __whence: int = SEEK_SET) -> int: + """Change the stream position to the given offset. + + Behavior is the same as a regular file, except that the position is capped to the size limit. + """ + if __whence == SEEK_END: + # SEEK_END => set from the truncated end + __offset = len(self) + __offset + __whence = SEEK_SET + + pos = self._file.seek(__offset, __whence) + if pos > self._size_limit: + return self._file.seek(self._size_limit) + return pos + + def read(self, __size: int | None = -1) -> bytes: + """Read at most `__size` bytes from the file. + + Behavior is the same as a regular file, except that it is capped to the size limit. + """ + current = self._file.tell() + if __size is None or __size < 0: + # Read until file limit + truncated_size = self._size_limit - current + else: + # Read until file limit or __size + truncated_size = min(__size, self._size_limit - current) + return self._file.read(truncated_size) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_dataset_viewer.py b/venv/lib/python3.12/site-packages/huggingface_hub/_dataset_viewer.py new file mode 100644 index 0000000000000000000000000000000000000000..ada15a7f0c3fd2d8d67d4fe9d803598b32896bb2 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_dataset_viewer.py @@ -0,0 +1,172 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os +import shutil +import subprocess +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Union + +from . import constants +from .utils import get_token + + +if TYPE_CHECKING: + import duckdb + + +@dataclass(frozen=True) +class DatasetParquetEntry: + """Represents a single parquet file available for a dataset on the Hub.""" + + config: str + split: str + url: str + size: int + + +def execute_raw_sql_query(sql_query: str, *, token: str | bool | None = None) -> list[dict[str, Any]]: + normalized_query = sql_query.strip().rstrip(";").strip() + _raise_on_forbidden_query(normalized_query) + + connection = None + try: + connection = _get_duckdb_connection(token=token) + relation = connection.sql(normalized_query) + if relation is None: + raise ValueError("SQL query must return rows.") + + if isinstance(relation, _DuckDBCliRelation): + # DuckDB binary => run CLI => parse JSON + return relation.execute() + else: + # DuckDB Python API => fetch columns + rows => convert to dicts + columns = tuple(column[0] for column in relation.description) + rows = tuple(tuple(row) for row in relation.fetchall()) + return [dict(zip(columns, row)) for row in rows] + finally: + if connection is not None: + connection.close() + + +def _raise_on_forbidden_query(query: str) -> None: + if len(query) == 0: + raise ValueError("SQL query cannot be empty.") + + # DuckDB CLI meta-commands are dot-prefixed words (e.g. `.shell`, `.output`). + # Let's forbid them for now but allow SQL expressions like `.5` that can legitimately start a line. + for line in query.splitlines(): + stripped = line.lstrip() + if stripped.startswith(".") and stripped[1:2].isalpha(): + raise ValueError("DuckDB CLI meta-commands are not allowed in SQL queries.") + + +def _get_duckdb_connection( + token: str | bool | None, +) -> Union["duckdb.DuckDBPyConnection", "_DuckDBCliConnection"]: + try: + # If DuckDB is installed as a Python package, use it! + import duckdb + except ImportError as error: + # Otherwise, use the DuckDB CLI binary. + duckdb_binary = shutil.which("duckdb") + if duckdb_binary is None: + raise ImportError( + "DuckDB is required for `hf datasets sql`. Install the Python package with `pip install duckdb` or " + "install the DuckDB CLI binary (for example `brew install duckdb`)." + ) from error + return _DuckDBCliConnection(binary_path=duckdb_binary, token=token) + + # Create a new connection (Python API). + connection = duckdb.connect() + try: + for statement in _build_duckdb_secret_statements(token): + connection.execute(statement) + return connection + except Exception: + connection.close() + raise + + +@dataclass +class _DuckDBCliConnection: + """DuckDB connection. + + Mimics the DuckDB Python API, but runs the queries via the DuckDB CLI binary. + """ + + binary_path: str + token: str | bool | None + + def __post_init__(self) -> None: + self._setup_statements = _build_duckdb_secret_statements(self.token) + + def sql(self, query: str) -> "_DuckDBCliRelation": + return _DuckDBCliRelation(binary_path=self.binary_path, setup_statements=self._setup_statements, query=query) + + def close(self) -> None: + pass + + +@dataclass +class _DuckDBCliRelation: + """DuckDB relation. + + Mimics the DuckDB Python API, but runs the queries via the DuckDB CLI binary. + """ + + binary_path: str + setup_statements: list[str] + query: str + + def execute(self) -> list[dict[str, Any]]: + # Build the DuckDB CLI input. + setup = [] + if self.setup_statements: + setup = [ + f".output {os.devnull}", + *(f"{stmt};" for stmt in self.setup_statements), + ".output", + ] + full_query = "\n".join(setup + [self.query + ";"]) + + # Run DuckDB binary + result = subprocess.run( + [self.binary_path, "-json"], + input=full_query, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + error_message = result.stderr.strip() or result.stdout.strip() or "DuckDB CLI command failed." + raise RuntimeError(error_message) + + # Parse JSON output and return + return json.loads(result.stdout.strip()) + + +def _build_duckdb_secret_statements(token: str | bool | None) -> list[str]: + if token is None or token is True: + token = get_token() + + if not token: + return [] + + escaped_token = token.replace("'", "''") + escaped_endpoint = constants.ENDPOINT.replace("'", "''") + return [ + f"CREATE OR REPLACE SECRET hf_hub_token (TYPE HTTP, BEARER_TOKEN '{escaped_token}', SCOPE '{escaped_endpoint}')", + f"CREATE OR REPLACE SECRET hf_token (TYPE HUGGINGFACE, TOKEN '{escaped_token}')", + ] diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_eval_results.py b/venv/lib/python3.12/site-packages/huggingface_hub/_eval_results.py new file mode 100644 index 0000000000000000000000000000000000000000..77aee2b595e979a0d4e83a71f7bea81a6f2e4d33 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_eval_results.py @@ -0,0 +1,211 @@ +"""Evaluation results utilities for the `.eval_results/*.yaml` format. + +See https://huggingface.co/docs/hub/eval-results for more details. +Specifications are available at https://github.com/huggingface/hub-docs/blob/main/eval_results.yaml. +""" + +from dataclasses import dataclass +from typing import Any + + +@dataclass +class EvalResultEntry: + """ + Evaluation result entry for the `.eval_results/*.yaml` format. + + Represents evaluation scores stored in model repos that automatically appear on + the model page and the benchmark dataset's leaderboard. + + For the legacy `model-index` format in `README.md`, use [`EvalResult`] instead. + + See https://huggingface.co/docs/hub/eval-results for more details. + + Args: + dataset_id (`str`): + Benchmark dataset ID from the Hub. Example: "cais/hle", "Idavidrein/gpqa". + task_id (`str`): + Task identifier within the benchmark. Example: "gpqa_diamond". + value (`Any`): + The metric value. Example: 20.90. + dataset_revision (`str`, *optional*): + Git SHA of the benchmark dataset. + verify_token (`str`, *optional*): + A signature that can be used to prove that evaluation is provably auditable and reproducible. + date (`str`, *optional*): + When the evaluation was run (ISO-8601 datetime). Defaults to git commit time. + source_url (`str`, *optional*): + Link to the evaluation source (e.g., https://huggingface.co/spaces/SaylorTwift/smollm3-mmlu-pro). Required if `source_name`, `source_user`, or `source_org` is provided. + source_name (`str`, *optional*): + Display name for the source. Example: "Eval Logs". + source_user (`str`, *optional*): + HF user name for attribution. Example: "celinah". + source_org (`str`, *optional*): + HF org name for attribution. Example: "cais". + notes (`str`, *optional*): + Details about the evaluation setup. Example: "tools", "no-tools", "chain-of-thought". + + Example: + ```python + >>> from huggingface_hub import EvalResultEntry + >>> # Minimal example with required fields only + >>> result = EvalResultEntry( + ... dataset_id="Idavidrein/gpqa", + ... task_id="gpqa_diamond", + ... value=0.412, + ... ) + >>> # Full example with all fields + >>> result = EvalResultEntry( + ... dataset_id="cais/hle", + ... task_id="default", + ... value=20.90, + ... dataset_revision="5503434ddd753f426f4b38109466949a1217c2bb", + ... verify_token="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...", + ... date="2025-01-15T10:30:00Z", + ... source_url="https://huggingface.co/datasets/cais/hle", + ... source_name="CAIS HLE", + ... source_org="cais", + ... notes="no-tools", + ... ) + + ``` + """ + + dataset_id: str + task_id: str + value: Any + dataset_revision: str | None = None + verify_token: str | None = None + date: str | None = None + source_url: str | None = None + source_name: str | None = None + source_user: str | None = None + source_org: str | None = None + notes: str | None = None + + def __post_init__(self) -> None: + if ( + self.source_name is not None or self.source_user is not None or self.source_org is not None + ) and self.source_url is None: + raise ValueError( + "If `source_name`, `source_user`, or `source_org` is provided, `source_url` must also be provided." + ) + + +def eval_result_entries_to_yaml(entries: list[EvalResultEntry]) -> list[dict[str, Any]]: + """Convert a list of [`EvalResultEntry`] objects to a YAML-serializable list of dicts. + + This produces the format expected in `.eval_results/*.yaml` files. + + Args: + entries (`list[EvalResultEntry]`): + List of evaluation result entries to serialize. + + Returns: + `list[dict[str, Any]]`: A list of dictionaries ready to be dumped to YAML. + + Example: + ```python + >>> from huggingface_hub import EvalResultEntry, eval_result_entries_to_yaml + >>> entries = [ + ... EvalResultEntry(dataset_id="cais/hle", task_id="default", value=20.90), + ... EvalResultEntry(dataset_id="Idavidrein/gpqa", task_id="gpqa_diamond", value=0.412), + ... ] + >>> yaml_data = eval_result_entries_to_yaml(entries) + >>> yaml_data[0] + {'dataset': {'id': 'cais/hle', 'task_id': 'default'}, 'value': 20.9} + + ``` + + To upload eval results to the Hub: + ```python + >>> import yaml + >>> from huggingface_hub import upload_file, EvalResultEntry, eval_result_entries_to_yaml + >>> entries = [ + ... EvalResultEntry(dataset_id="cais/hle", task_id="default", value=20.90), + ... ] + >>> yaml_content = yaml.dump(eval_result_entries_to_yaml(entries)) + >>> upload_file( + ... path_or_fileobj=yaml_content.encode(), + ... path_in_repo=".eval_results/hle.yaml", + ... repo_id="your-username/your-model", + ... ) + + ``` + """ + result = [] + for entry in entries: + # build the dataset object + dataset: dict[str, Any] = {"id": entry.dataset_id, "task_id": entry.task_id} + if entry.dataset_revision is not None: + dataset["revision"] = entry.dataset_revision + + data: dict[str, Any] = {"dataset": dataset, "value": entry.value} + if entry.verify_token is not None: + data["verifyToken"] = entry.verify_token + if entry.date is not None: + data["date"] = entry.date + # build the source object + if entry.source_url is not None: + source: dict[str, Any] = {"url": entry.source_url} + if entry.source_name is not None: + source["name"] = entry.source_name + if entry.source_user is not None: + source["user"] = entry.source_user + if entry.source_org is not None: + source["org"] = entry.source_org + data["source"] = source + if entry.notes is not None: + data["notes"] = entry.notes + + result.append(data) + return result + + +def parse_eval_result_entries(data: list[dict[str, Any]]) -> list[EvalResultEntry]: + """Parse a list of dicts into [`EvalResultEntry`] objects. + + This parses the `.eval_results/*.yaml` format. For the legacy `model-index` format, + use [`model_index_to_eval_results`] instead. + + Args: + data (`list[dict[str, Any]]`): + A list of dictionaries (e.g., parsed from YAML or API response). + + Returns: + `list[EvalResultEntry]`: A list of evaluation result entry objects. + + Example: + ```python + >>> from huggingface_hub import parse_eval_result_entries + >>> data = [ + ... {"dataset": {"id": "cais/hle", "task_id": "default"}, "value": 20.90}, + ... {"dataset": {"id": "Idavidrein/gpqa", "task_id": "gpqa_diamond"}, "value": 0.412}, + ... ] + >>> entries = parse_eval_result_entries(data) + >>> entries[0].dataset_id + 'cais/hle' + >>> entries[0].value + 20.9 + + ``` + """ + entries = [] + for item in data: + entry_data = item.get("data", item) + dataset = entry_data.get("dataset", {}) + source = entry_data.get("source", {}) + entry = EvalResultEntry( + dataset_id=dataset["id"], + value=entry_data["value"], + task_id=dataset["task_id"], + dataset_revision=dataset.get("revision"), + verify_token=entry_data.get("verifyToken"), + date=entry_data.get("date"), + source_url=source.get("url") if source else None, + source_name=source.get("name") if source else None, + source_user=source.get("user") if source else None, + source_org=source.get("org") if source else None, + notes=entry_data.get("notes"), + ) + entries.append(entry) + return entries diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__init__.py b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ef08599ad40fcb6d55fac1c397597f68b8b52084 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f82c2b1ef65674a72ab13d49821bec4d198e043d Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/client.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef9f3455af93e0a757853e567160832fec619490 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/client.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/sse_client.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/sse_client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a6d94abe91377ed5424b5ab05e72d5e02ccec898 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/sse_client.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/types.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd701ce82d1fa3f68eaa58c52c2f6eccd47ff112 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/__pycache__/types.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/client.py b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/client.py new file mode 100644 index 0000000000000000000000000000000000000000..1c3dc30938ce81e32886d5e585bdab8a33fe5218 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/client.py @@ -0,0 +1,130 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +import time +from collections import deque +from collections.abc import Iterator +from typing import Literal, TypedDict + +import httpx + +from ..utils._headers import build_hf_headers +from ..utils._http import hf_raise_for_status +from .sse_client import SSEClient +from .types import ApiGetReloadEventSourceData, ApiGetReloadRequest + + +HOT_RELOADING_PORT = 7887 +CLIENT_TIMEOUT = 20 + + +class MultiReplicaStreamWarning(TypedDict): + kind: Literal["warning"] + message: str + + +class MultiReplicaStreamEvent(TypedDict): + kind: Literal["event"] + event: ApiGetReloadEventSourceData + + +class MultiReplicaStreamReplicaHash(TypedDict): + kind: Literal["replicaHash"] + hash: str + + +class MultiReplicaStreamFullMatch(TypedDict): + kind: Literal["fullMatch"] + + +class ReloadClient: + def __init__( + self, + *, + host: str, + subdomain: str, + replica_hash: str, + token: str | None, + ): + base_host = host.replace(subdomain, f"{subdomain}--{HOT_RELOADING_PORT}") + self.replica_hash = replica_hash + self.client = httpx.Client( + base_url=f"{base_host}/--replicas/+{replica_hash}", + headers=build_hf_headers(token=token), + timeout=CLIENT_TIMEOUT, + ) + + def get_reload(self, reload_id: str) -> Iterator[ApiGetReloadEventSourceData] | int: + req = ApiGetReloadRequest(reloadId=reload_id) + with self.client.stream("POST", "/get-reload", json=req) as res: + if res.status_code != 200: + return res.status_code + hf_raise_for_status(res) + for event in SSEClient(res.iter_bytes()).events(): + if event.event == "message": + yield json.loads(event.data) + return None + + +def multi_replica_reload_events( + commit_sha: str, + host: str, + subdomain: str, + replica_hashes: list[str], + token: str | None, + max_retries: int = 10, +) -> Iterator[ + MultiReplicaStreamWarning | MultiReplicaStreamEvent | MultiReplicaStreamReplicaHash | MultiReplicaStreamFullMatch +]: + clients = [ + ReloadClient( + host=host, + subdomain=subdomain, + replica_hash=hash, + token=token, + ) + for hash in replica_hashes + ] + + first_client_events: dict[int, ApiGetReloadEventSourceData] = {} + for client_index, client in enumerate(clients): + if len(clients) > 1: + yield {"kind": "replicaHash", "hash": client.replica_hash} + + retries = 0 + while isinstance((events := client.get_reload(commit_sha)), int): + if (retries := retries + 1) > max_retries: + raise Exception("Too many retries reached") + if (status_code := events) not in (200, 204): + raise Exception(f"Unexpected {status_code=} on `ReloadClient.get_reload`") + subject = "reloadId" if status_code == 204 else "replica" + yield {"kind": "warning", "message": f"Retrying on unexpected {subject} not found"} + time.sleep(2) + + full_match = True + replay: deque[ApiGetReloadEventSourceData] = deque() + for event_index, event in enumerate(events): + if client_index == 0: + first_client_events[event_index] = event + elif full_match := full_match and first_client_events.get(event_index) == event: + replay.append(event) + continue + while replay: + yield {"kind": "event", "event": replay.popleft()} + yield {"kind": "event", "event": event} + + if client_index > 0 and full_match: + yield {"kind": "fullMatch"} diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/sse_client.py b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/sse_client.py new file mode 100644 index 0000000000000000000000000000000000000000..2dab959bbd62045adff1d8d51438acd5bd062265 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/sse_client.py @@ -0,0 +1,144 @@ +""" +Vendored file: Server Side Events (SSE) client for Python. + +Source: +- Author: Maxime Petazzoni +- Repository: https://github.com/mpetazzoni/sseclient +- File: https://github.com/mpetazzoni/sseclient/blob/main/sseclient/__init__.py + +License: +- Apache-2.0 (from upstream project) + +Provides a generator of SSE received through an existing HTTP response. +""" + +import logging + +__author__ = 'Maxime Petazzoni ' +__email__ = 'maxime.petazzoni@bulix.org' +__all__ = ['SSEClient'] + +_FIELD_SEPARATOR = ':' + + +class SSEClient: + """Implementation of a SSE client. + + See http://www.w3.org/TR/2009/WD-eventsource-20091029/ for the + specification. + """ + + def __init__(self, event_source, char_enc='utf-8'): + """Initialize the SSE client over an existing, ready to consume + event source. + + The event source is expected to be a binary stream and have a close() + method. That would usually be something that implements + io.BinaryIOBase, like an httplib or urllib3 HTTPResponse object. + """ + self._logger = logging.getLogger(self.__class__.__module__) + self._logger.debug('Initialized SSE client from event source %s', + event_source) + self._event_source = event_source + self._char_enc = char_enc + + def _read(self): + """Read the incoming event source stream and yield event chunks. + + Unfortunately it is possible for some servers to decide to break an + event into multiple HTTP chunks in the response. It is thus necessary + to correctly stitch together consecutive response chunks and find the + SSE delimiter (empty new line) to yield full, correct event chunks.""" + data = b'' + for chunk in self._event_source: + for line in chunk.splitlines(True): + data += line + if data.endswith((b'\r\r', b'\n\n', b'\r\n\r\n')): + yield data + data = b'' + if data: + yield data + + def events(self): + for chunk in self._read(): + event = Event() + # Split before decoding so splitlines() only uses \r and \n + for line in chunk.splitlines(): + # Decode the line. + line = line.decode(self._char_enc) + + # Lines starting with a separator are comments and are to be + # ignored. + if not line.strip() or line.startswith(_FIELD_SEPARATOR): + continue + + data = line.split(_FIELD_SEPARATOR, 1) + field = data[0] + + # Ignore unknown fields. + if field not in event.__dict__: + self._logger.debug('Saw invalid field %s while parsing ' + 'Server Side Event', field) + continue + + if len(data) > 1: + # From the spec: + # "If value starts with a single U+0020 SPACE character, + # remove it from value." + if data[1].startswith(' '): + value = data[1][1:] + else: + value = data[1] + else: + # If no value is present after the separator, + # assume an empty value. + value = '' + + # The data field may come over multiple lines and their values + # are concatenated with each other. + if field == 'data': + event.__dict__[field] += value + '\n' + else: + event.__dict__[field] = value + + # Events with no data are not dispatched. + if not event.data: + continue + + # If the data field ends with a newline, remove it. + if event.data.endswith('\n'): + event.data = event.data[0:-1] + + # Empty event names default to 'message' + event.event = event.event or 'message' + + # Dispatch the event + self._logger.debug('Dispatching %s...', event) + yield event + + def close(self): + """Manually close the event source stream.""" + self._event_source.close() + + +class Event: + """Representation of an event from the event stream.""" + + def __init__(self, id=None, event='message', data='', retry=None): + self.id = id + self.event = event + self.data = data + self.retry = retry + + def __str__(self): + s = f'{self.event} event' + if self.id: + s += f' #{self.id}' + if self.data: + s += ', {} byte{}'.format(len(self.data), + 's' if len(self.data) else '') + else: + s += ', no data' + if self.retry: + s += f', retry in {self.retry}ms' + return s diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/types.py b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/types.py new file mode 100644 index 0000000000000000000000000000000000000000..c5f892d6287f5b581edca8fdf64c1827e114621b --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_hot_reload/types.py @@ -0,0 +1,121 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Literal, TypedDict + +from typing_extensions import NotRequired + + +class ReloadRegion(TypedDict): + startLine: int + startCol: int + endLine: int + endCol: int + + +class ReloadOperationObject(TypedDict): + kind: Literal["add", "update", "delete"] + region: ReloadRegion + objectType: str + objectName: str + + +class ReloadOperationRun(TypedDict): + kind: Literal["run"] + region: ReloadRegion + codeLines: str + stdout: NotRequired[str] + stderr: NotRequired[str] + + +class ReloadOperationException(TypedDict): + kind: Literal["exception"] + region: ReloadRegion + traceback: str + + +class ReloadOperationError(TypedDict): + kind: Literal["error"] + traceback: str + + +class ReloadOperationUI(TypedDict): + kind: Literal["ui"] + updated: bool + + +class ReloadOperationFile(TypedDict): + kind: Literal["file"] + created: bool + + +class ApiCreateReloadRequest(TypedDict): + filepath: str + contents: str + reloadId: NotRequired[str] + + +class ApiCreateReloadResponseSuccess(TypedDict): + status: Literal["created"] + reloadId: str + + +class ApiCreateReloadResponseError(TypedDict): + status: Literal["alreadyReloading", "fileNotFound"] + + +class ApiCreateReloadResponse(TypedDict): + res: ApiCreateReloadResponseError | ApiCreateReloadResponseSuccess + + +class ApiGetReloadRequest(TypedDict): + reloadId: str + + +class ApiGetReloadEventSourceData(TypedDict): + data: ( + ReloadOperationError + | ReloadOperationException + | ReloadOperationObject + | ReloadOperationRun + | ReloadOperationUI + | ReloadOperationFile + ) + + +class ApiGetStatusRequest(TypedDict): + revision: str + + +class ApiGetStatusResponse(TypedDict): + reloading: bool + uncommited: list[str] + + +class ApiFetchContentsRequest(TypedDict): + filepath: str + + +class ApiFetchContentsResponseError(TypedDict): + status: Literal["fileNotFound"] + + +class ApiFetchContentsResponseSuccess(TypedDict): + status: Literal["ok"] + contents: str + + +class ApiFetchContentsResponse(TypedDict): + res: ApiFetchContentsResponseError | ApiFetchContentsResponseSuccess diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_inference_endpoints.py b/venv/lib/python3.12/site-packages/huggingface_hub/_inference_endpoints.py new file mode 100644 index 0000000000000000000000000000000000000000..3a48f47784551d91f36b0b73b84c1ad76283c500 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_inference_endpoints.py @@ -0,0 +1,419 @@ +import time +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from typing import TYPE_CHECKING, Optional + +from huggingface_hub.errors import InferenceEndpointError, InferenceEndpointTimeoutError + +from .utils import get_session, logging, parse_datetime + + +if TYPE_CHECKING: + from .hf_api import HfApi + from .inference._client import InferenceClient + from .inference._generated._async_client import AsyncInferenceClient + +logger = logging.get_logger(__name__) + + +class InferenceEndpointStatus(str, Enum): + PENDING = "pending" + INITIALIZING = "initializing" + UPDATING = "updating" + UPDATE_FAILED = "updateFailed" + RUNNING = "running" + PAUSED = "paused" + FAILED = "failed" + SCALED_TO_ZERO = "scaledToZero" + + +class InferenceEndpointType(str, Enum): + PUBlIC = "public" + PROTECTED = "protected" # deprecated, use AUTHENTICATED instead + AUTHENTICATED = "authenticated" + PRIVATE = "private" + + +class InferenceEndpointScalingMetric(str, Enum): + PENDING_REQUESTS = "pendingRequests" + HARDWARE_USAGE = "hardwareUsage" + + +@dataclass +class InferenceEndpoint: + """ + Contains information about a deployed Inference Endpoint. + + Args: + name (`str`): + The unique name of the Inference Endpoint. + namespace (`str`): + The namespace where the Inference Endpoint is located. + repository (`str`): + The name of the model repository deployed on this Inference Endpoint. + status ([`InferenceEndpointStatus`]): + The current status of the Inference Endpoint. + url (`str`, *optional*): + The URL of the Inference Endpoint, if available. Only a deployed Inference Endpoint will have a URL. + framework (`str`): + The machine learning framework used for the model. + revision (`str`): + The specific model revision deployed on the Inference Endpoint. + task (`str`): + The task associated with the deployed model. + created_at (`datetime.datetime`): + The timestamp when the Inference Endpoint was created. + updated_at (`datetime.datetime`): + The timestamp of the last update of the Inference Endpoint. + type ([`InferenceEndpointType`]): + The type of the Inference Endpoint (public, authenticated, private). + raw (`dict`): + The raw dictionary data returned from the API. + token (`str` or `bool`, *optional*): + Authentication token for the Inference Endpoint, if set when requesting the API. Will default to the + locally saved token if not provided. Pass `token=False` if you don't want to send your token to the server. + + Example: + ```python + >>> from huggingface_hub import get_inference_endpoint + >>> endpoint = get_inference_endpoint("my-text-to-image") + >>> endpoint + InferenceEndpoint(name='my-text-to-image', ...) + + # Get status + >>> endpoint.status + 'running' + >>> endpoint.url + 'https://my-text-to-image.region.vendor.endpoints.huggingface.cloud' + + # Run inference + >>> endpoint.client.text_to_image(...) + + # Pause endpoint to save $$$ + >>> endpoint.pause() + + # ... + # Resume and wait for deployment + >>> endpoint.resume() + >>> endpoint.wait() + >>> endpoint.client.text_to_image(...) + ``` + """ + + # Field in __repr__ + name: str = field(init=False) + namespace: str + repository: str = field(init=False) + status: InferenceEndpointStatus = field(init=False) + health_route: str = field(init=False) + url: str | None = field(init=False) + + # Other fields + framework: str = field(repr=False, init=False) + revision: str = field(repr=False, init=False) + task: str = field(repr=False, init=False) + created_at: datetime = field(repr=False, init=False) + updated_at: datetime = field(repr=False, init=False) + type: InferenceEndpointType = field(repr=False, init=False) + + # Raw dict from the API + raw: dict = field(repr=False) + + # Internal fields + _token: str | bool | None = field(repr=False, compare=False) + _api: "HfApi" = field(repr=False, compare=False) + + @classmethod + def from_raw( + cls, raw: dict, namespace: str, token: str | bool | None = None, api: Optional["HfApi"] = None + ) -> "InferenceEndpoint": + """Initialize object from raw dictionary.""" + if api is None: + from .hf_api import HfApi + + api = HfApi() + if token is None: + token = api.token + + # All other fields are populated in __post_init__ + return cls(raw=raw, namespace=namespace, _token=token, _api=api) + + def __post_init__(self) -> None: + """Populate fields from raw dictionary.""" + self._populate_from_raw() + + @property + def client(self) -> "InferenceClient": + """Returns a client to make predictions on this Inference Endpoint. + + Returns: + [`InferenceClient`]: an inference client pointing to the deployed endpoint. + + Raises: + [`InferenceEndpointError`]: If the Inference Endpoint is not yet deployed. + """ + if self.url is None: + raise InferenceEndpointError( + "Cannot create a client for this Inference Endpoint as it is not yet deployed. " + "Please wait for the Inference Endpoint to be deployed using `endpoint.wait()` and try again." + ) + from .inference._client import InferenceClient + + return InferenceClient( + model=self.url, + token=self._token, # type: ignore # boolean token shouldn't be possible. In practice it's ok. + ) + + @property + def async_client(self) -> "AsyncInferenceClient": + """Returns a client to make predictions on this Inference Endpoint. + + Returns: + [`AsyncInferenceClient`]: an asyncio-compatible inference client pointing to the deployed endpoint. + + Raises: + [`InferenceEndpointError`]: If the Inference Endpoint is not yet deployed. + """ + if self.url is None: + raise InferenceEndpointError( + "Cannot create a client for this Inference Endpoint as it is not yet deployed. " + "Please wait for the Inference Endpoint to be deployed using `endpoint.wait()` and try again." + ) + from .inference._generated._async_client import AsyncInferenceClient + + return AsyncInferenceClient( + model=self.url, + token=self._token, # type: ignore # boolean token shouldn't be possible. In practice it's ok. + ) + + def wait(self, timeout: int | None = None, refresh_every: int = 5) -> "InferenceEndpoint": + """Wait for the Inference Endpoint to be deployed. + + Information from the server will be fetched every 1s. If the Inference Endpoint is not deployed after `timeout` + seconds, a [`InferenceEndpointTimeoutError`] will be raised. The [`InferenceEndpoint`] will be mutated in place with the latest + data. + + Args: + timeout (`int`, *optional*): + The maximum time to wait for the Inference Endpoint to be deployed, in seconds. If `None`, will wait + indefinitely. + refresh_every (`int`, *optional*): + The time to wait between each fetch of the Inference Endpoint status, in seconds. Defaults to 5s. + + Returns: + [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. + + Raises: + [`InferenceEndpointError`] + If the Inference Endpoint ended up in a failed state. + [`InferenceEndpointTimeoutError`] + If the Inference Endpoint is not deployed after `timeout` seconds. + """ + if timeout is not None and timeout < 0: + raise ValueError("`timeout` cannot be negative.") + if refresh_every <= 0: + raise ValueError("`refresh_every` must be positive.") + + start = time.time() + while True: + if self.status == InferenceEndpointStatus.FAILED: + raise InferenceEndpointError( + f"Inference Endpoint {self.name} failed to deploy. Please check the logs for more information." + ) + if self.status == InferenceEndpointStatus.UPDATE_FAILED: + raise InferenceEndpointError( + f"Inference Endpoint {self.name} failed to update. Please check the logs for more information." + ) + if self.status == InferenceEndpointStatus.RUNNING and self.url is not None: + # Verify the endpoint is actually reachable + _health_url = f"{self.url.rstrip('/')}/{self.health_route.lstrip('/')}" + response = get_session().get(_health_url, headers=self._api._build_hf_headers(token=self._token)) + if response.status_code == 200: + logger.info("Inference Endpoint is ready to be used.") + return self + + if timeout is not None: + if time.time() - start > timeout: + raise InferenceEndpointTimeoutError("Timeout while waiting for Inference Endpoint to be deployed.") + logger.info(f"Inference Endpoint is not deployed yet ({self.status}). Waiting {refresh_every}s...") + time.sleep(refresh_every) + self.fetch() + + def fetch(self) -> "InferenceEndpoint": + """Fetch latest information about the Inference Endpoint. + + Returns: + [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. + """ + obj = self._api.get_inference_endpoint(name=self.name, namespace=self.namespace, token=self._token) # type: ignore [arg-type] + self.raw = obj.raw + self._populate_from_raw() + return self + + def update( + self, + *, + # Compute update + accelerator: str | None = None, + instance_size: str | None = None, + instance_type: str | None = None, + min_replica: int | None = None, + max_replica: int | None = None, + scale_to_zero_timeout: int | None = None, + # Model update + repository: str | None = None, + framework: str | None = None, + revision: str | None = None, + task: str | None = None, + custom_image: dict | None = None, + secrets: dict[str, str] | None = None, + ) -> "InferenceEndpoint": + """Update the Inference Endpoint. + + This method allows the update of either the compute configuration, the deployed model, or both. All arguments are + optional but at least one must be provided. + + This is an alias for [`HfApi.update_inference_endpoint`]. The current object is mutated in place with the + latest data from the server. + + Args: + accelerator (`str`, *optional*): + The hardware accelerator to be used for inference (e.g. `"cpu"`). + instance_size (`str`, *optional*): + The size or type of the instance to be used for hosting the model (e.g. `"x4"`). + instance_type (`str`, *optional*): + The cloud instance type where the Inference Endpoint will be deployed (e.g. `"intel-icl"`). + min_replica (`int`, *optional*): + The minimum number of replicas (instances) to keep running for the Inference Endpoint. + max_replica (`int`, *optional*): + The maximum number of replicas (instances) to scale to for the Inference Endpoint. + scale_to_zero_timeout (`int`, *optional*): + The duration in minutes before an inactive endpoint is scaled to zero. + + repository (`str`, *optional*): + The name of the model repository associated with the Inference Endpoint (e.g. `"gpt2"`). + framework (`str`, *optional*): + The machine learning framework used for the model (e.g. `"custom"`). + revision (`str`, *optional*): + The specific model revision to deploy on the Inference Endpoint (e.g. `"6c0e6080953db56375760c0471a8c5f2929baf11"`). + task (`str`, *optional*): + The task on which to deploy the model (e.g. `"text-classification"`). + custom_image (`dict`, *optional*): + A custom Docker image to use for the Inference Endpoint. This is useful if you want to deploy an + Inference Endpoint running on the `text-generation-inference` (TGI) framework (see examples). + secrets (`dict[str, str]`, *optional*): + Secret values to inject in the container environment. + Returns: + [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. + """ + # Make API call + obj = self._api.update_inference_endpoint( + name=self.name, + namespace=self.namespace, + accelerator=accelerator, + instance_size=instance_size, + instance_type=instance_type, + min_replica=min_replica, + max_replica=max_replica, + scale_to_zero_timeout=scale_to_zero_timeout, + repository=repository, + framework=framework, + revision=revision, + task=task, + custom_image=custom_image, + secrets=secrets, + token=self._token, # type: ignore [arg-type] + ) + + # Mutate current object + self.raw = obj.raw + self._populate_from_raw() + return self + + def pause(self) -> "InferenceEndpoint": + """Pause the Inference Endpoint. + + A paused Inference Endpoint will not be charged. It can be resumed at any time using [`InferenceEndpoint.resume`]. + This is different from scaling the Inference Endpoint to zero with [`InferenceEndpoint.scale_to_zero`], which + would be automatically restarted when a request is made to it. + + This is an alias for [`HfApi.pause_inference_endpoint`]. The current object is mutated in place with the + latest data from the server. + + Returns: + [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. + """ + obj = self._api.pause_inference_endpoint(name=self.name, namespace=self.namespace, token=self._token) # type: ignore [arg-type] + self.raw = obj.raw + self._populate_from_raw() + return self + + def resume(self, running_ok: bool = True) -> "InferenceEndpoint": + """Resume the Inference Endpoint. + + This is an alias for [`HfApi.resume_inference_endpoint`]. The current object is mutated in place with the + latest data from the server. + + Args: + running_ok (`bool`, *optional*): + If `True`, the method will not raise an error if the Inference Endpoint is already running. Defaults to + `True`. + + Returns: + [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. + """ + obj = self._api.resume_inference_endpoint( + name=self.name, namespace=self.namespace, running_ok=running_ok, token=self._token + ) # type: ignore [arg-type] + self.raw = obj.raw + self._populate_from_raw() + return self + + def scale_to_zero(self) -> "InferenceEndpoint": + """Scale Inference Endpoint to zero. + + An Inference Endpoint scaled to zero will not be charged. It will be resumed on the next request to it, with a + cold start delay. This is different from pausing the Inference Endpoint with [`InferenceEndpoint.pause`], which + would require a manual resume with [`InferenceEndpoint.resume`]. + + This is an alias for [`HfApi.scale_to_zero_inference_endpoint`]. The current object is mutated in place with the + latest data from the server. + + Returns: + [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data. + """ + obj = self._api.scale_to_zero_inference_endpoint(name=self.name, namespace=self.namespace, token=self._token) # type: ignore [arg-type] + self.raw = obj.raw + self._populate_from_raw() + return self + + def delete(self) -> None: + """Delete the Inference Endpoint. + + This operation is not reversible. If you don't want to be charged for an Inference Endpoint, it is preferable + to pause it with [`InferenceEndpoint.pause`] or scale it to zero with [`InferenceEndpoint.scale_to_zero`]. + + This is an alias for [`HfApi.delete_inference_endpoint`]. + """ + self._api.delete_inference_endpoint(name=self.name, namespace=self.namespace, token=self._token) # type: ignore [arg-type] + + def _populate_from_raw(self) -> None: + """Populate fields from raw dictionary. + + Called in __post_init__ + each time the Inference Endpoint is updated. + """ + # Repr fields + self.name = self.raw["name"] + self.repository = self.raw["model"]["repository"] + self.status = self.raw["status"]["state"] + self.url = self.raw["status"].get("url") + self.health_route = self.raw["healthRoute"] + + # Other fields + self.framework = self.raw["model"]["framework"] + self.revision = self.raw["model"]["revision"] + self.task = self.raw["model"]["task"] + self.created_at = parse_datetime(self.raw["status"]["createdAt"]) + self.updated_at = parse_datetime(self.raw["status"]["updatedAt"]) + self.type = self.raw["type"] diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_jobs_api.py b/venv/lib/python3.12/site-packages/huggingface_hub/_jobs_api.py new file mode 100644 index 0000000000000000000000000000000000000000..d19b501e4f489213da40527715e9c2777c959da4 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_jobs_api.py @@ -0,0 +1,626 @@ +# Copyright 2025-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib +import platform +import re +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from pathlib import Path +from typing import Any + +from huggingface_hub import constants +from huggingface_hub._space_api import Volume +from huggingface_hub.utils._datetime import parse_datetime + + +class JobHardware(str, Enum): + """ + Enumeration of hardware flavors available to run Jobs on the Hub. + + Value can be compared to a string: + ```py + assert JobHardware.CPU_BASIC == "cpu-basic" + ``` + + Both enums are kept in sync with the Hub API by `utils/check_hardware_flavors.py`. + """ + + # CPU + CPU_BASIC = "cpu-basic" + CPU_UPGRADE = "cpu-upgrade" + CPU_PERFORMANCE = "cpu-performance" + CPU_XL = "cpu-xl" + + # GPU + T4_SMALL = "t4-small" + T4_MEDIUM = "t4-medium" + L4X1 = "l4x1" + L4X4 = "l4x4" + L40SX1 = "l40sx1" + L40SX4 = "l40sx4" + L40SX8 = "l40sx8" + A10G_SMALL = "a10g-small" + A10G_LARGE = "a10g-large" + A10G_LARGEX2 = "a10g-largex2" + A10G_LARGEX4 = "a10g-largex4" + A100_LARGE = "a100-large" + A100X4 = "a100x4" + A100X8 = "a100x8" + H200 = "h200" + H200X2 = "h200x2" + H200X4 = "h200x4" + H200X8 = "h200x8" + RTX_PRO_6000 = "rtx-pro-6000" + RTX_PRO_6000X2 = "rtx-pro-6000x2" + RTX_PRO_6000X4 = "rtx-pro-6000x4" + RTX_PRO_6000X8 = "rtx-pro-6000x8" + + +class JobStage(str, Enum): + """ + Enumeration of possible stage of a Job on the Hub. + + Value can be compared to a string: + ```py + assert JobStage.COMPLETED == "COMPLETED" + ``` + Possible values are: `COMPLETED`, `CANCELED`, `ERROR`, `DELETED`, `SCHEDULING`, `RUNNING`. + Taken from https://github.com/huggingface/moon-landing/blob/main/server/job_types/JobInfo.ts#L61 (private url). + """ + + # Copied from moon-landing > server > lib > Job.ts + COMPLETED = "COMPLETED" + CANCELED = "CANCELED" + ERROR = "ERROR" + DELETED = "DELETED" + SCHEDULING = "SCHEDULING" + RUNNING = "RUNNING" + + +# Stages indicating the Job has reached a terminal state and will not run further. +TERMINAL_JOB_STAGES = (JobStage.COMPLETED, JobStage.CANCELED, JobStage.ERROR, JobStage.DELETED) + +# URL prefixes identifying an image that points to a HF Space rather than a Docker image. +_SPACE_IMAGE_PREFIXES = ( + "https://huggingface.co/spaces/", + "https://hf.co/spaces/", + "huggingface.co/spaces/", + "hf.co/spaces/", +) + + +@dataclass +class JobStatus: + stage: JobStage + message: str | None + expose_urls: list[str] | None + ssh_url: str | None + + +@dataclass +class JobOwner: + id: str + name: str + type: str + + +@dataclass +class JobDurations: + """ + Timing breakdown for a Job, computed server-side. + + Args: + scheduling_secs (`int` or `None`): + Seconds the job spent in the scheduling stage before starting to run. + `None` if the job never reached the running stage. + running_secs (`int` or `None`): + Seconds the job has been or was running. Recomputed on each request + while the job is in progress. `None` if the job never started running. + total_secs (`int` or `None`): + Total seconds elapsed since the job was created. Recomputed on each + request while the job is in progress. + """ + + scheduling_secs: int | None + running_secs: int | None + total_secs: int | None + + def __init__(self, **kwargs) -> None: + self.scheduling_secs = kwargs.get("schedulingSecs", kwargs.get("scheduling_secs")) + self.running_secs = kwargs.get("runningSecs", kwargs.get("running_secs")) + self.total_secs = kwargs.get("totalSecs", kwargs.get("total_secs")) + + +@dataclass +class JobInitiator: + """ + Contains information about what triggered a Job. + + Args: + type (`str`): Initiator kind, for example `"user"`, `"org"`, `"scheduled-job"`, or `"duplicated-job"`. + id (`str`): Identifier of the initiator. + name (`str` or `None`): Human-readable name when available, usually for user/org initiators. + """ + + type: str + id: str + name: str | None = None + + +@dataclass +class JobInfo: + """ + Contains information about a Job. + + Args: + id (`str`): + Job ID. + created_at (`datetime` or `None`): + When the Job was created. + started_at (`datetime` or `None`): + When the Job started running. None while the Job is still scheduling. + finished_at (`datetime` or `None`): + When the Job finished. None while the Job is still scheduling or running. + docker_image (`str` or `None`): + The Docker image from Docker Hub used for the Job. + Can be None if space_id is present instead. + space_id (`str` or `None`): + The Docker image from Hugging Face Spaces used for the Job. + Can be None if docker_image is present instead. + command (`list[str]` or `None`): + Command of the Job, e.g. `["python", "-c", "print('hello world')"]` + arguments (`list[str]` or `None`): + Arguments passed to the command + environment (`dict[str]` or `None`): + Environment variables of the Job as a dictionary. + secrets (`dict[str]` or `None`): + Secret environment variables of the Job (encrypted). + flavor (`str` or `None`): + Flavor for the hardware. See [`JobHardware`] for possible values. + E.g. `"cpu-basic"`. + labels (`dict[str, str]` or `None`): + Labels to attach to the job (key-value pairs). + volumes (`list[Volume]` or `None`): + Volumes mounted in the job container (buckets, models, datasets, spaces). + status: (`JobStatus` or `None`): + Status of the Job, e.g. `JobStatus(stage="RUNNING", message=None)` + See [`JobStage`] for possible stage values. + durations (`JobDurations` or `None`): + Timing breakdown of the Job. Present for all job states including SCHEDULING. + owner: (`JobOwner` or `None`): + Owner of the Job, e.g. `JobOwner(id="5e9ecfc04957053f60648a3e", name="lhoestq", type="user")` + initiator (`JobInitiator` or `None`): + What triggered the Job, e.g. `JobInitiator(type="scheduled-job", id="...")` for a cron-triggered run. + expose_urls (`list[str]` or `None`): + Public URLs through which the Job's exposed ports are reachable (one per port exposed via `expose=`), + e.g. `["https://687fb701029421ae5549d998--8000.hf.jobs"]`. `None` when no port is exposed. + Accessing a URL requires an HF token with read access to the Job's namespace. + ssh_url (`str` or `None`): + SSH endpoint of the Job, e.g. `"ssh://687fb701029421ae5549d998@ssh.hf.jobs"`. Only present when the Job + was started with `ssh=True`. Connecting requires write access to the Job's namespace and an SSH public + key registered on the Hub (https://huggingface.co/settings/keys). + + Example: + + ```python + >>> from huggingface_hub import run_job + >>> job = run_job( + ... image="python:3.12", + ... command=["python", "-c", "print('Hello from the cloud!')"] + ... ) + >>> job + JobInfo(id='687fb701029421ae5549d998', created_at=datetime.datetime(2025, 7, 22, 16, 6, 25, 79000, tzinfo=datetime.timezone.utc), started_at=datetime.datetime(2025, 7, 22, 16, 6, 31, 79000, tzinfo=datetime.timezone.utc), finished_at=None, docker_image='python:3.12', space_id=None, command=['python', '-c', "print('Hello from the cloud!')"], arguments=[], environment={}, secrets={}, flavor='cpu-basic', labels=None, status=JobStatus(stage='RUNNING', message=None), durations=JobDurations(scheduling_secs=6, running_secs=2, total_secs=8), owner=JobOwner(id='5e9ecfc04957053f60648a3e', name='lhoestq', type='user'), initiator=JobInitiator(type='user', id='5e9ecfc04957053f60648a3e', name='lhoestq'), endpoint='https://huggingface.co', url='https://huggingface.co/jobs/lhoestq/687fb701029421ae5549d998') + >>> job.id + '687fb701029421ae5549d998' + >>> job.url + 'https://huggingface.co/jobs/lhoestq/687fb701029421ae5549d998' + >>> job.status.stage + 'RUNNING' + ``` + """ + + id: str + created_at: datetime | None + started_at: datetime | None + finished_at: datetime | None + docker_image: str | None + space_id: str | None + command: list[str] | None + arguments: list[str] | None + environment: dict[str, Any] | None + secrets: dict[str, Any] | None + flavor: JobHardware | None + labels: dict[str, str] | None + volumes: list[Volume] | None + status: JobStatus + durations: JobDurations | None + owner: JobOwner + initiator: JobInitiator | None + + # Inferred fields + endpoint: str + url: str + + def __init__(self, **kwargs) -> None: + self.id = kwargs["id"] + created_at = kwargs.get("createdAt") or kwargs.get("created_at") + self.created_at = parse_datetime(created_at) if created_at else None + started_at = kwargs.get("startedAt") or kwargs.get("started_at") + self.started_at = parse_datetime(started_at) if started_at else None + finished_at = kwargs.get("finishedAt") or kwargs.get("finished_at") + self.finished_at = parse_datetime(finished_at) if finished_at else None + self.docker_image = kwargs.get("dockerImage") or kwargs.get("docker_image") + self.space_id = kwargs.get("spaceId") or kwargs.get("space_id") + owner = kwargs.get("owner", {}) + self.owner = JobOwner(id=owner["id"], name=owner["name"], type=owner["type"]) + self.command = kwargs.get("command") + self.arguments = kwargs.get("arguments") + self.environment = kwargs.get("environment") + self.secrets = kwargs.get("secrets") + self.flavor = kwargs.get("flavor") + self.labels = kwargs.get("labels") + volumes = kwargs.get("volumes") + self.volumes = [Volume(**v) for v in volumes] if volumes else None + status = kwargs.get("status", {}) + self.status = JobStatus( + stage=status["stage"], + message=status.get("message"), + expose_urls=status.get("exposeUrls"), + ssh_url=status.get("sshUrl"), + ) + durations = kwargs.get("durations") + self.durations = JobDurations(**durations) if durations else None + initiator = kwargs.get("initiator") + self.initiator = ( + JobInitiator(type=initiator["type"], id=initiator["id"], name=initiator.get("name")) if initiator else None + ) + + # Inferred fields + self.endpoint = kwargs.get("endpoint", constants.ENDPOINT) + self.url = f"{self.endpoint}/jobs/{self.owner.name}/{self.id}" + + +@dataclass +class JobSpec: + docker_image: str | None + space_id: str | None + command: list[str] | None + arguments: list[str] | None + environment: dict[str, Any] | None + secrets: dict[str, Any] | None + flavor: JobHardware | None + timeout: int | None + tags: list[str] | None + arch: str | None + labels: dict[str, str] | None + volumes: list[Volume] | None + + def __init__(self, **kwargs) -> None: + self.docker_image = kwargs.get("dockerImage") or kwargs.get("docker_image") + self.space_id = kwargs.get("spaceId") or kwargs.get("space_id") + self.command = kwargs.get("command") + self.arguments = kwargs.get("arguments") + self.environment = kwargs.get("environment") + self.secrets = kwargs.get("secrets") + self.flavor = kwargs.get("flavor") + self.timeout = kwargs.get("timeout") + self.tags = kwargs.get("tags") + self.arch = kwargs.get("arch") + self.labels = kwargs.get("labels") + volumes = kwargs.get("volumes") + self.volumes = [Volume(**v) for v in volumes] if volumes else None + + +@dataclass +class LastJobInfo: + id: str + at: datetime + + def __init__(self, **kwargs) -> None: + self.id = kwargs["id"] + self.at = parse_datetime(kwargs["at"]) + + +@dataclass +class ScheduledJobStatus: + last_job: LastJobInfo | None + next_job_run_at: datetime | None + + def __init__(self, **kwargs) -> None: + last_job = kwargs.get("lastJob") or kwargs.get("last_job") + self.last_job = LastJobInfo(**last_job) if last_job else None + next_job_run_at = kwargs.get("nextJobRunAt") or kwargs.get("next_job_run_at") + self.next_job_run_at = parse_datetime(str(next_job_run_at)) if next_job_run_at else None + + +@dataclass +class ScheduledJobInfo: + """ + Contains information about a Job. + + Args: + id (`str`): + Scheduled Job ID. + created_at (`datetime` or `None`): + When the scheduled Job was created. + tags (`list[str]` or `None`): + The tags of the scheduled Job. + schedule (`str` or `None`): + One of "@annually", "@yearly", "@monthly", "@weekly", "@daily", "@hourly", or a + CRON schedule expression (e.g., '0 9 * * 1' for 9 AM every Monday). + suspend (`bool` or `None`): + Whether the scheduled job is suspended (paused). + concurrency (`bool` or `None`): + Whether multiple instances of this Job can run concurrently. + status (`ScheduledJobStatus` or `None`): + Status of the scheduled Job. + owner: (`JobOwner` or `None`): + Owner of the scheduled Job, e.g. `JobOwner(id="5e9ecfc04957053f60648a3e", name="lhoestq", type="user")` + job_spec: (`JobSpec` or `None`): + Specifications of the Job. + + Example: + + ```python + >>> from huggingface_hub import run_job + >>> scheduled_job = create_scheduled_job( + ... image="python:3.12", + ... command=["python", "-c", "print('Hello from the cloud!')"], + ... schedule="@hourly", + ... ) + >>> scheduled_job.id + '687fb701029421ae5549d999' + >>> scheduled_job.status.next_job_run_at + datetime.datetime(2025, 7, 22, 17, 6, 25, 79000, tzinfo=datetime.timezone.utc) + ``` + """ + + id: str + created_at: datetime | None + job_spec: JobSpec + schedule: str | None + suspend: bool | None + concurrency: bool | None + status: ScheduledJobStatus + owner: JobOwner + + def __init__(self, **kwargs) -> None: + self.id = kwargs["id"] + created_at = kwargs.get("createdAt") or kwargs.get("created_at") + self.created_at = parse_datetime(created_at) if created_at else None + self.job_spec = JobSpec(**(kwargs.get("job_spec") or kwargs.get("jobSpec", {}))) + self.schedule = kwargs.get("schedule") + self.suspend = kwargs.get("suspend") + self.concurrency = kwargs.get("concurrency") + status = kwargs.get("status", {}) + self.status = ScheduledJobStatus( + last_job=status.get("last_job") or status.get("lastJob"), + next_job_run_at=status.get("next_job_run_at") or status.get("nextJobRunAt"), + ) + owner = kwargs.get("owner", {}) + self.owner = JobOwner(id=owner["id"], name=owner["name"], type=owner["type"]) + + +@dataclass +class JobAccelerator: + """ + Contains information about a Job accelerator (GPU). + + Args: + type (`str`): + Type of accelerator, e.g. `"gpu"`. + model (`str`): + Model of accelerator, e.g. `"T4"`, `"A10G"`, `"A100"`, `"L4"`, `"L40S"`. + quantity (`str`): + Number of accelerators, e.g. `"1"`, `"2"`, `"4"`, `"8"`. + vram (`str`): + Total VRAM, e.g. `"16 GB"`, `"24 GB"`. + manufacturer (`str`): + Manufacturer of the accelerator, e.g. `"Nvidia"`. + """ + + type: str + model: str + quantity: str + vram: str + manufacturer: str + + def __init__(self, **kwargs) -> None: + self.type = kwargs["type"] + self.model = kwargs["model"] + self.quantity = kwargs["quantity"] + self.vram = kwargs["vram"] + self.manufacturer = kwargs["manufacturer"] + + +@dataclass +class JobHardwareInfo: + """ + Contains information about available Job hardware. + + Args: + name (`str`): + Machine identifier, e.g. `"cpu-basic"`, `"a10g-large"`. + pretty_name (`str`): + Human-readable name, e.g. `"CPU Basic"`, `"Nvidia A10G - large"`. + cpu (`str`): + CPU specification, e.g. `"2 vCPU"`, `"12 vCPU"`. + ram (`str`): + RAM specification, e.g. `"16 GB"`, `"46 GB"`. + ephemeral_storage (`str`): + Ephemeral storage specification, e.g. `"20 GB"`, `"100 GB"`. + accelerator (`JobAccelerator` or `None`): + GPU/accelerator details if available. + unit_cost_micro_usd (`int`): + Cost in micro-dollars per unit, e.g. `167` (= $0.000167). + unit_cost_usd (`float`): + Cost in USD per unit, e.g. `0.000167`. + unit_label (`str`): + Cost unit period, e.g. `"minute"`. + + Example: + + ```python + >>> from huggingface_hub import list_jobs_hardware + >>> hardware_list = list_jobs_hardware() + >>> hardware_list[0] + JobHardwareInfo(name='cpu-basic', pretty_name='CPU Basic', cpu='2 vCPU', ram='16 GB', ephemeral_storage='20 GB', accelerator=None, unit_cost_micro_usd=167, unit_cost_usd=0.000167, unit_label='minute') + >>> hardware_list[0].name + 'cpu-basic' + ``` + """ + + name: str + pretty_name: str + cpu: str + ram: str + ephemeral_storage: str + accelerator: JobAccelerator | None + unit_cost_micro_usd: int + unit_cost_usd: float + unit_label: str + + def __init__(self, **kwargs) -> None: + self.name = kwargs["name"] + self.pretty_name = kwargs["prettyName"] + self.cpu = kwargs["cpu"] + self.ram = kwargs["ram"] + self.ephemeral_storage = kwargs.get("ephemeralStorage", "N/A") + accelerator = kwargs.get("accelerator") + self.accelerator = JobAccelerator(**accelerator) if accelerator else None + self.unit_cost_micro_usd = kwargs["unitCostMicroUSD"] + self.unit_cost_usd = kwargs["unitCostUSD"] + self.unit_label = kwargs["unitLabel"] + + +def _derive_job_volume_name(source: str | Path) -> str: + """Derive a stable remote folder name for a local directory synced as a Job volume. + + The name is `{dirname}-{hash}` where the hash fingerprints the absolute path and the machine's + hostname: re-syncing the same directory from the same machine reuses the same remote folder + (so only changed files are re-uploaded), while same-named directories from different paths or + machines don't collide. + """ + resolved = Path(source).expanduser().resolve() + digest = hashlib.sha256(f"{platform.node()}:{resolved}".encode()).hexdigest()[:8] + dirname = resolved.name.replace(" ", "_") or "root" + return f"{dirname}-{digest}" + + +def _sanitize_job_name(name: str) -> str: + """Sanitize a string so it is a valid Job `name` label. + + Job names must match `^[a-zA-Z0-9._-]*$` and are also stored as tags, which only allow + alphanumerics, `-` and `_`. Every other character (including `/`, `:` and `.`) is replaced with `-`. + """ + return re.sub(r"[^a-zA-Z0-9_-]", "-", name) + + +def _short_invocation_hash(parts: list[str]) -> str: + """Short, stable hash of a Job invocation (image/script + command line). + + Appended to auto-generated names so reruns of the same command share a name while different ones don't. + """ + return hashlib.sha256("\x00".join(parts).encode()).hexdigest()[:8] + + +def _default_job_name_from_image(image: str, command: list[str]) -> str: + """Derive a default Job name from a Docker image or Space reference and its command. + + A short hash of the full command line is appended to group reruns of the same command. + e.g. python:3.12 + [foo, --truc] -> python-3-12-1a2b3c4d + hf.co/spaces/lhoestq/duckdb -> lhoestq-duckdb- + pytorch/pytorch:2.6.0-cuda12.4-... -> pytorch-2-6-0-cuda12-4-...- + """ + for prefix in _SPACE_IMAGE_PREFIXES: + if image.startswith(prefix): + base = _sanitize_job_name(image[len(prefix) :] or image) + break + else: + base = _sanitize_job_name(image.rstrip("/").split("/")[-1] or image) # drop registry host and namespace + return f"{base}-{_short_invocation_hash([image, *command])}" + + +def _default_job_name_from_script(script: str, script_args: list[str]) -> str: + """Derive a default Job name from a UV script path, URL, or command and its arguments. + + A short hash of the full command line is appended to group reruns of the same command. + e.g. my_script.py + [--epochs, 3] -> my_script-1a2b3c4d + https://.../sft.py?raw=1 -> sft- + lighteval -> lighteval- + """ + name = script.split("?", 1)[0].split("#", 1)[0].rstrip("/").split("/")[-1] + if name.endswith(".py"): + name = name[: -len(".py")] + base = _sanitize_job_name(name or script) + return f"{base}-{_short_invocation_hash([script, *script_args])}" + + +def _create_job_spec( + *, + image: str, + command: list[str], + env: dict[str, Any] | None, + secrets: dict[str, Any] | None, + flavor: JobHardware | str | None, + timeout: int | float | str | None, + name: str | None = None, + labels: dict[str, str] | None = None, + volumes: list[Volume] | None = None, + expose: list[int] | None = None, + ssh: bool = False, +) -> dict[str, Any]: + if name is not None: + if labels is not None and "name" in labels: + raise ValueError("`name` and the `name` key in `labels` cannot both be provided.") + labels = {**(labels or {}), "name": name} + + # prepare job spec to send to HF Jobs API + job_spec: dict[str, Any] = { + "command": command, + "arguments": [], + "environment": env or {}, + "flavor": flavor or JobHardware.CPU_BASIC, + } + # secrets are optional + if secrets: + job_spec["secrets"] = secrets + # timeout is optional + if timeout: + time_units_factors = {"s": 1, "m": 60, "h": 3600, "d": 3600 * 24} + if isinstance(timeout, str) and timeout[-1] in time_units_factors: + job_spec["timeoutSeconds"] = int(float(timeout[:-1]) * time_units_factors[timeout[-1]]) + else: + job_spec["timeoutSeconds"] = int(timeout) + # labels are optional + if labels: + job_spec["labels"] = labels + # volumes are optional + if volumes: + job_spec["volumes"] = [vol.to_dict() for vol in volumes] + # expose ports through the jobs proxy + if expose: + job_spec["expose"] = {"ports": expose} + # make the job container reachable over SSH + if ssh: + job_spec["ssh"] = {"enabled": True} + # input is either from docker hub or from HF spaces + for prefix in _SPACE_IMAGE_PREFIXES: + if image.startswith(prefix): + job_spec["spaceId"] = image[len(prefix) :] + break + else: + job_spec["dockerImage"] = image + return job_spec diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_local_folder.py b/venv/lib/python3.12/site-packages/huggingface_hub/_local_folder.py new file mode 100644 index 0000000000000000000000000000000000000000..eb9de7b741eeab39a2f84401546a57000293791c --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_local_folder.py @@ -0,0 +1,484 @@ +# Copyright 2024-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to handle the `../.cache/huggingface` folder in local directories. + +First discussed in https://github.com/huggingface/huggingface_hub/issues/1738 to store +download metadata when downloading files from the hub to a local directory (without +using the cache). + +./.cache/huggingface folder structure: +[4.0K] data +├── [4.0K] .cache +│ └── [4.0K] huggingface +│ └── [4.0K] download +│ ├── [ 16] file.parquet.metadata +│ ├── [ 16] file.txt.metadata +│ └── [4.0K] folder +│ └── [ 16] file.parquet.metadata +│ +├── [6.5G] file.parquet +├── [1.5K] file.txt +└── [4.0K] folder + └── [ 16] file.parquet + + +Download metadata file structure: +``` +# file.txt.metadata +11c5a3d5811f50298f278a704980280950aedb10 +a16a55fda99d2f2e7b69cce5cf93ff4ad3049930 +1712656091.123 + +# file.parquet.metadata +11c5a3d5811f50298f278a704980280950aedb10 +7c5d3f4b8b76583b422fcb9189ad6c89d5d97a094541ce8932dce3ecabde1421 +1712656091.123 +} +``` +""" + +import base64 +import hashlib +import logging +import os +import time +from dataclasses import dataclass +from pathlib import Path + +from .utils import WeakFileLock + + +logger = logging.getLogger(__name__) + +CACHEDIR_TAG_CONTENT = ( + "Signature: 8a477f597d28d172789f06886806bc55\n" + "# This file is a cache directory tag created by huggingface_hub.\n" + "# For information about cache directory tags, see:\n" + "#\thttps://bford.info/cachedir/\n" +) + + +@dataclass +class LocalDownloadFilePaths: + """ + Paths to the files related to a download process in a local dir. + + Returned by [`get_local_download_paths`]. + + Attributes: + file_path (`Path`): + Path where the file will be saved. + lock_path (`Path`): + Path to the lock file used to ensure atomicity when reading/writing metadata. + metadata_path (`Path`): + Path to the metadata file. + """ + + file_path: Path + lock_path: Path + metadata_path: Path + + def incomplete_path(self, etag: str) -> Path: + """Return the path where a file will be temporarily downloaded before being moved to `file_path`.""" + path = self.metadata_path.parent / f"{_short_hash(self.metadata_path.name)}.{etag}.incomplete" + resolved_path = str(path.resolve()) + # Some Windows versions do not allow for paths longer than 255 characters. + # In this case, we must specify it as an extended path by using the "\\?\" prefix. + if os.name == "nt" and len(resolved_path) > 255 and not resolved_path.startswith("\\\\?\\"): + path = Path("\\\\?\\" + resolved_path) + return path + + +@dataclass(frozen=True) +class LocalUploadFilePaths: + """ + Paths to the files related to an upload process in a local dir. + + Returned by [`get_local_upload_paths`]. + + Attributes: + path_in_repo (`str`): + Path of the file in the repo. + file_path (`Path`): + Path where the file will be saved. + lock_path (`Path`): + Path to the lock file used to ensure atomicity when reading/writing metadata. + metadata_path (`Path`): + Path to the metadata file. + """ + + path_in_repo: str + file_path: Path + lock_path: Path + metadata_path: Path + + +@dataclass +class LocalDownloadFileMetadata: + """ + Metadata about a file in the local directory related to a download process. + + Attributes: + filename (`str`): + Path of the file in the repo. + commit_hash (`str`): + Commit hash of the file in the repo. + etag (`str`): + ETag of the file in the repo. Used to check if the file has changed. + For LFS files, this is the sha256 of the file. For regular files, it corresponds to the git hash. + timestamp (`int`): + Unix timestamp of when the metadata was saved i.e. when the metadata was accurate. + """ + + filename: str + commit_hash: str + etag: str + timestamp: float + + +@dataclass +class LocalUploadFileMetadata: + """ + Metadata about a file in the local directory related to an upload process. + """ + + size: int + + # Default values correspond to "we don't know yet" + timestamp: float | None = None + should_ignore: bool | None = None + sha256: str | None = None + upload_mode: str | None = None + remote_oid: str | None = None + is_uploaded: bool = False + is_committed: bool = False + + def save(self, paths: LocalUploadFilePaths) -> None: + """Save the metadata to disk.""" + with WeakFileLock(paths.lock_path): + with paths.metadata_path.open("w") as f: + new_timestamp = time.time() + f.write(str(new_timestamp) + "\n") + + f.write(str(self.size)) # never None + f.write("\n") + + if self.should_ignore is not None: + f.write(str(int(self.should_ignore))) + f.write("\n") + + if self.sha256 is not None: + f.write(self.sha256) + f.write("\n") + + if self.upload_mode is not None: + f.write(self.upload_mode) + f.write("\n") + + if self.remote_oid is not None: + f.write(self.remote_oid) + f.write("\n") + + f.write(str(int(self.is_uploaded)) + "\n") + f.write(str(int(self.is_committed)) + "\n") + + self.timestamp = new_timestamp + + +def get_local_download_paths(local_dir: Path, filename: str) -> LocalDownloadFilePaths: + """Compute paths to the files related to a download process. + + Folders containing the paths are all guaranteed to exist. + + Args: + local_dir (`Path`): + Path to the local directory in which files are downloaded. + filename (`str`): + Path of the file in the repo. + + Return: + [`LocalDownloadFilePaths`]: the paths to the files (file_path, lock_path, metadata_path, incomplete_path). + """ + # filename is the path in the Hub repository (separated by '/') + # make sure to have a cross-platform transcription + sanitized_filename = os.path.join(*filename.split("/")) + if os.name == "nt": + if sanitized_filename.startswith("..\\") or "\\..\\" in sanitized_filename: + raise ValueError( + f"Invalid filename: cannot handle filename '{sanitized_filename}' on Windows. Please ask the repository" + " owner to rename this file." + ) + file_path = local_dir / sanitized_filename + metadata_path = _huggingface_dir(local_dir) / "download" / f"{sanitized_filename}.metadata" + lock_path = metadata_path.with_suffix(".lock") + + # Some Windows versions do not allow for paths longer than 255 characters. + # In this case, we must specify it as an extended path by using the "\\?\" prefix + if os.name == "nt": + if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255: + file_path = Path("\\\\?\\" + os.path.abspath(file_path)) + lock_path = Path("\\\\?\\" + os.path.abspath(lock_path)) + metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path)) + + file_path.parent.mkdir(parents=True, exist_ok=True) + metadata_path.parent.mkdir(parents=True, exist_ok=True) + return LocalDownloadFilePaths(file_path=file_path, lock_path=lock_path, metadata_path=metadata_path) + + +def get_local_upload_paths(local_dir: Path, filename: str) -> LocalUploadFilePaths: + """Compute paths to the files related to an upload process. + + Folders containing the paths are all guaranteed to exist. + + Args: + local_dir (`Path`): + Path to the local directory that is uploaded. + filename (`str`): + Path of the file in the repo. + + Return: + [`LocalUploadFilePaths`]: the paths to the files (file_path, lock_path, metadata_path). + """ + # filename is the path in the Hub repository (separated by '/') + # make sure to have a cross-platform transcription + sanitized_filename = os.path.join(*filename.split("/")) + if os.name == "nt": + if sanitized_filename.startswith("..\\") or "\\..\\" in sanitized_filename: + raise ValueError( + f"Invalid filename: cannot handle filename '{sanitized_filename}' on Windows. Please ask the repository" + " owner to rename this file." + ) + file_path = local_dir / sanitized_filename + metadata_path = _huggingface_dir(local_dir) / "upload" / f"{sanitized_filename}.metadata" + lock_path = metadata_path.with_suffix(".lock") + + # Some Windows versions do not allow for paths longer than 255 characters. + # In this case, we must specify it as an extended path by using the "\\?\" prefix + if os.name == "nt": + if not str(local_dir).startswith("\\\\?\\") and len(os.path.abspath(lock_path)) > 255: + file_path = Path("\\\\?\\" + os.path.abspath(file_path)) + lock_path = Path("\\\\?\\" + os.path.abspath(lock_path)) + metadata_path = Path("\\\\?\\" + os.path.abspath(metadata_path)) + + file_path.parent.mkdir(parents=True, exist_ok=True) + metadata_path.parent.mkdir(parents=True, exist_ok=True) + return LocalUploadFilePaths( + path_in_repo=filename, file_path=file_path, lock_path=lock_path, metadata_path=metadata_path + ) + + +def read_download_metadata(local_dir: Path, filename: str) -> LocalDownloadFileMetadata | None: + """Read metadata about a file in the local directory related to a download process. + + Args: + local_dir (`Path`): + Path to the local directory in which files are downloaded. + filename (`str`): + Path of the file in the repo. + + Return: + `[LocalDownloadFileMetadata]` or `None`: the metadata if it exists, `None` otherwise. + """ + paths = get_local_download_paths(local_dir, filename) + with WeakFileLock(paths.lock_path): + if paths.metadata_path.exists(): + try: + with paths.metadata_path.open() as f: + commit_hash = f.readline().strip() + etag = f.readline().strip() + timestamp = float(f.readline().strip()) + metadata = LocalDownloadFileMetadata( + filename=filename, + commit_hash=commit_hash, + etag=etag, + timestamp=timestamp, + ) + except Exception as e: + # remove the metadata file if it is corrupted / not the right format + logger.warning( + f"Invalid metadata file {paths.metadata_path}: {e}. Removing it from disk and continue." + ) + try: + paths.metadata_path.unlink() + except Exception as e: + logger.warning(f"Could not remove corrupted metadata file {paths.metadata_path}: {e}") + return None + + try: + # check if the file exists and hasn't been modified since the metadata was saved + stat = paths.file_path.stat() + if ( + stat.st_mtime - 1 <= metadata.timestamp + ): # allow 1s difference as stat.st_mtime might not be precise + return metadata + logger.info(f"Ignored metadata for '{filename}' (outdated). Will re-compute hash.") + except FileNotFoundError: + # file does not exist => metadata is outdated + return None + return None + + +def read_upload_metadata(local_dir: Path, filename: str) -> LocalUploadFileMetadata: + """Read metadata about a file in the local directory related to an upload process. + + TODO: factorize logic with `read_download_metadata`. + + Args: + local_dir (`Path`): + Path to the local directory in which files are downloaded. + filename (`str`): + Path of the file in the repo. + + Return: + `[LocalUploadFileMetadata]` or `None`: the metadata if it exists, `None` otherwise. + """ + paths = get_local_upload_paths(local_dir, filename) + with WeakFileLock(paths.lock_path): + if paths.metadata_path.exists(): + try: + with paths.metadata_path.open() as f: + timestamp = float(f.readline().strip()) + + size = int(f.readline().strip()) # never None + + _should_ignore = f.readline().strip() + should_ignore = None if _should_ignore == "" else bool(int(_should_ignore)) + + _sha256 = f.readline().strip() + sha256 = None if _sha256 == "" else _sha256 + + _upload_mode = f.readline().strip() + upload_mode = None if _upload_mode == "" else _upload_mode + if upload_mode not in (None, "regular", "lfs"): + raise ValueError(f"Invalid upload mode in metadata {paths.path_in_repo}: {upload_mode}") + + _remote_oid = f.readline().strip() + remote_oid = None if _remote_oid == "" else _remote_oid + + is_uploaded = bool(int(f.readline().strip())) + is_committed = bool(int(f.readline().strip())) + + metadata = LocalUploadFileMetadata( + timestamp=timestamp, + size=size, + should_ignore=should_ignore, + sha256=sha256, + upload_mode=upload_mode, + remote_oid=remote_oid, + is_uploaded=is_uploaded, + is_committed=is_committed, + ) + except Exception as e: + # remove the metadata file if it is corrupted / not the right format + logger.warning( + f"Invalid metadata file {paths.metadata_path}: {e}. Removing it from disk and continue." + ) + try: + paths.metadata_path.unlink() + except Exception as e: + logger.warning(f"Could not remove corrupted metadata file {paths.metadata_path}: {e}") + + # corrupted metadata => we don't know anything expect its size + return LocalUploadFileMetadata(size=paths.file_path.stat().st_size) + + # TODO: can we do better? + if ( + metadata.timestamp is not None + and metadata.is_uploaded # file was uploaded + and not metadata.is_committed # but not committed + and time.time() - metadata.timestamp > 20 * 3600 # and it's been more than 20 hours + ): # => we consider it as garbage-collected by S3 + metadata.is_uploaded = False + + # check if the file exists and hasn't been modified since the metadata was saved + try: + if metadata.timestamp is not None and paths.file_path.stat().st_mtime <= metadata.timestamp: + return metadata + logger.info(f"Ignored metadata for '{filename}' (outdated). Will re-compute hash.") + except FileNotFoundError: + # file does not exist => metadata is outdated + pass + + # empty metadata => we don't know anything expect its size + return LocalUploadFileMetadata(size=paths.file_path.stat().st_size) + + +def write_download_metadata(local_dir: Path, filename: str, commit_hash: str, etag: str) -> None: + """Write metadata about a file in the local directory related to a download process. + + Args: + local_dir (`Path`): + Path to the local directory in which files are downloaded. + """ + paths = get_local_download_paths(local_dir, filename) + with WeakFileLock(paths.lock_path): + with paths.metadata_path.open("w") as f: + f.write(f"{commit_hash}\n{etag}\n{time.time()}\n") + + +def _huggingface_dir(local_dir: Path) -> Path: + """Return the path to the `.cache/huggingface` directory in a local directory.""" + # Wrap in lru_cache to avoid overwriting the .gitignore file if called multiple times + path = local_dir / ".cache" / "huggingface" + # Without long path support enabled, Windows caps directory paths at 247 characters + # (MAX_PATH minus room for an 8.3 file name), so creating the `.cache/huggingface` directory + # (and the bookkeeping files below) fails for a deep `local_dir`. + # Use the extended-length "\\?\" prefix for the filesystem operations, matching what + # `get_local_download_paths`/`get_local_upload_paths` already do for the download/upload paths. + # The un-prefixed `path` is still returned so callers keep re-deriving/prefixing as before. + target = path + if os.name == "nt": + abs_path = os.path.abspath(path) + if len(abs_path) > 247 and not abs_path.startswith("\\\\?\\"): + target = Path("\\\\?\\" + abs_path) + target.mkdir(exist_ok=True, parents=True) + + # Create a CACHEDIR.TAG so backup tools can skip this directory. + _create_cachedir_tag(target) + + # Create a .gitignore file in the .cache/huggingface directory if it doesn't exist + # Should be thread-safe enough like this. + gitignore = target / ".gitignore" + gitignore_lock = target / ".gitignore.lock" + if not gitignore.exists(): + try: + with WeakFileLock(gitignore_lock, timeout=0.1): + gitignore.write_text("*") + except IndexError: + pass + except OSError: # TimeoutError, FileNotFoundError, PermissionError, etc. + pass + try: + gitignore_lock.unlink() + except OSError: + pass + return path + + +def _create_cachedir_tag(cache_dir: Path) -> None: + """Create a CACHEDIR.TAG file in ``cache_dir`` if one does not already exist. + + The tag follows the `Cache Directory Tagging Standard `_ + so that backup tools can recognize and skip cache directories. + """ + tag_path = cache_dir / "CACHEDIR.TAG" + if not tag_path.exists(): + try: + tag_path.write_text(CACHEDIR_TAG_CONTENT) + except OSError: + pass + + +def _short_hash(filename: str) -> str: + return base64.urlsafe_b64encode(hashlib.sha1(filename.encode()).digest()).decode() diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_login.py b/venv/lib/python3.12/site-packages/huggingface_hub/_login.py new file mode 100644 index 0000000000000000000000000000000000000000..f8fcdc55b778f9423701664aa29ca65f08c4a229 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_login.py @@ -0,0 +1,559 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains methods to log in to the Hub.""" + +import html +import os +import subprocess +import sys +import time +from datetime import datetime +from getpass import getpass +from pathlib import Path + +from . import constants +from .errors import DeviceCodeError +from .utils import ( + ANSI, + get_token, + is_google_colab, + is_notebook, + list_credential_helpers, + logging, + run_subprocess, + select_choice, + set_git_credential, + tabulate, + unset_git_credential, +) +from .utils._auth import ( + _get_token_by_name, + _get_token_from_environment, + _get_token_from_file, + _get_token_from_google_colab, + _read_stored_tokens_full, + _save_stored_tokens_full, + _save_token, + _write_secret, + get_stored_tokens, +) +from .utils._oauth_device import OAuthTokenResponse, poll_device_token, request_device_code + + +logger = logging.get_logger(__name__) + + +def login( + token: str | None = None, + *, + add_to_git_credential: bool = False, + skip_if_logged_in: bool = True, +) -> None: + """Login the machine to access the Hub. + + The `token` is persisted in cache and set as a git credential. Once done, the machine + is logged in and the access token will be available across all `huggingface_hub` + components. If `token` is not provided, a browser-based OAuth flow is used to + authenticate: open a URL, enter a short code, and the token is retrieved and saved. + In a terminal, you can also choose to paste an existing access token instead. + + To log in from outside of a script, one can also use `hf auth login` which is + a cli command that wraps [`login`]. + + > [!TIP] + > When the token is not passed, [`login`] will automatically detect if the script runs + > in a notebook or not. However, this detection might not be accurate due to the + > variety of notebooks that exists nowadays. If that is the case, you can always force + > the UI by using [`notebook_login`] or [`interpreter_login`]. + + Args: + token (`str`, *optional*): + User access token to generate from https://huggingface.co/settings/token. + add_to_git_credential (`bool`, defaults to `False`): + If `True`, token will be set as git credential. If no git credential helper + is configured, a warning will be displayed to the user. Only used when `token` + is provided; ignored by the browser-based flow. + skip_if_logged_in (`bool`, defaults to `True`): + If `True`, do not prompt for token if user is already logged in. + Set to `False` to force re-login. In CLI, use `--force` instead. + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If an organization token is passed. Only personal account tokens are valid + to log in. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If token is invalid. + [`DeviceCodeError`] + If the browser-based login fails (authorization denied, code expired, ...). + """ + if token is not None: + if not add_to_git_credential: + logger.info( + "The token has not been saved to the git credentials helper. Pass " + "`add_to_git_credential=True` in this function directly or " + "`--add-to-git-credential` if using via `hf`CLI if " + "you want to set the git credential as well." + ) + _validate_and_save_token(token, add_to_git_credential=add_to_git_credential) + return + if add_to_git_credential: + logger.warning( + "`add_to_git_credential=True` is only supported when a token is passed directly. " + "It is ignored by the browser-based login." + ) + if is_notebook(): + notebook_login(skip_if_logged_in=skip_if_logged_in) + else: + interpreter_login(skip_if_logged_in=skip_if_logged_in) + + +def logout(token_name: str | None = None) -> None: + """Logout the machine from the Hub. + + Token is deleted from the machine and removed from git credential. + + Args: + token_name (`str`, *optional*): + Name of the access token to logout from. If `None`, will log out from all saved access tokens. + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError): + If the access token name is not found. + """ + if get_token() is None and not get_stored_tokens(): # No active token and no saved access tokens + logger.warning("Not logged in!") + return + if not token_name: + # Delete all saved access tokens and token + for file_path in (constants.HF_TOKEN_PATH, constants.HF_STORED_TOKENS_PATH): + try: + Path(file_path).unlink() + except FileNotFoundError: + pass + logger.info("Successfully logged out from all access tokens.") + else: + _logout_from_token(token_name) + logger.info(f"Successfully logged out from access token: {token_name}.") + + unset_git_credential() + + # Check if still logged in + if _get_token_from_google_colab() is not None: + raise OSError( + "You are automatically logged in using a Google Colab secret.\n" + "To log out, you must unset the `HF_TOKEN` secret in your Colab settings." + ) + if _get_token_from_environment() is not None: + raise OSError( + "Token has been deleted from your machine but you are still logged in.\n" + "To log out, you must clear out both `HF_TOKEN` and `HUGGING_FACE_HUB_TOKEN` environment variables." + ) + + +def auth_switch(token_name: str, add_to_git_credential: bool = False) -> None: + """Switch to a different access token. + + Args: + token_name (`str`): + Name of the access token to switch to. + add_to_git_credential (`bool`, defaults to `False`): + If `True`, token will be set as git credential. If no git credential helper + is configured, a warning will be displayed to the user. If `token` is `None`, + the value of `add_to_git_credential` is ignored and will be prompted again + to the end user. + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError): + If the access token name is not found. + """ + token = _get_token_by_name(token_name) + if not token: + raise ValueError(f"Access token {token_name} not found in {constants.HF_STORED_TOKENS_PATH}") + # Write token to HF_TOKEN_PATH + _set_active_token(token_name, add_to_git_credential) + logger.info(f"The current active token is: {token_name}") + token_from_environment = _get_token_from_environment() + if token_from_environment is not None and token_from_environment != token: + logger.warning( + "The environment variable `HF_TOKEN` is set and will override the access token you've just switched to." + ) + + +def auth_list() -> None: + """List all stored access tokens.""" + # Resolve the current token before reading the file: `get_token()` may refresh an OAuth + # token and rewrite the stored tokens on the way. + current_token = get_token() + stored_tokens = _read_stored_tokens_full() + + if not stored_tokens: + if _get_token_from_environment(): + logger.info("No stored access tokens found.") + logger.warning("Note: Environment variable `HF_TOKEN` is set and is the current active token.") + else: + logger.info("No access tokens found.") + return + show_expires = any("expires_at" in fields for fields in stored_tokens.values()) + headers = [" ", "name", "token"] + (["expires"] if show_expires else []) + + current_token_name = None + rows: list[list[str | int]] = [] + for token_name, fields in stored_tokens.items(): + token = fields.get("hf_token", "") + if token == current_token: + current_token_name = token_name + masked_token = f"{token[:3]}****{token[-4:]}" if token != "" else token + row: list[str | int] = ["*" if token == current_token else "", token_name, masked_token] + if show_expires: + row.append(_format_expiration(fields.get("expires_at"))) + rows.append(row) + print(tabulate(rows, headers=headers)) + + if _get_token_from_environment(): + logger.warning( + "\nNote: Environment variable `HF_TOKEN` is set and is the current active token independently from the stored tokens listed above." + ) + elif current_token_name is None: + logger.warning( + "\nNote: No active token is set and no environment variable `HF_TOKEN` is found. Use `hf auth login` to log in." + ) + + +### +# Device Code OAuth login (RFC 8628) +### + + +def _device_code_login() -> None: + """Run the Device Code OAuth flow: request a code, prompt the user to authorize it in a browser, + poll for the token and save it.""" + device_info = request_device_code() + + # The complete URI has the code pre-filled when the server supports it. + print(f"\n Open this URL in your browser:\n {device_info['verification_uri_complete']}\n") + print(f" And enter the code: {device_info['user_code']}\n") + + print(" Waiting for authorization", end="", flush=True) + try: + response = poll_device_token(device_info, on_pending=lambda: print(".", end="", flush=True)) + finally: + print() # newline after the progress dots, also on failure + + _save_oauth_token(response) + + +def _save_oauth_token(response: OAuthTokenResponse) -> tuple[str, str]: + """Validate and persist a token response from the device code flow, including refresh metadata.""" + expires_in = response.get("expires_in") + token_name, username = _validate_and_save_token( + response["access_token"], + add_to_git_credential=False, + refresh_token=response.get("refresh_token"), + expires_at=int(time.time()) + int(expires_in) if expires_in else None, + ) + if note := _expiration_note(response): + logger.info(f"Note: {note}") + return token_name, username + + +def _expiration_note(response: OAuthTokenResponse) -> str | None: + """Human-readable note about the lifetime of a freshly obtained OAuth token, if known.""" + expires_in = response.get("expires_in") + if not expires_in: + return None + if response.get("refresh_token"): + return "This token will be refreshed automatically when it expires." + days = max(1, int(expires_in) // 86400) + return f"This token expires in {days} days. Log in again to renew it." + + +### +# Interpreter-based login (text) +### + + +def interpreter_login(*, skip_if_logged_in: bool = True) -> None: + """ + Displays a prompt to log in to the HF website and store the token. + + This is equivalent to [`login`] without passing a token when not run in a notebook. + [`interpreter_login`] is useful if you want to force the use of the terminal prompt + instead of a notebook flow. + + For more details, see [`login`]. + + Args: + skip_if_logged_in (`bool`, defaults to `True`): + If `True`, do not prompt for token if user is already logged in. + Set to `False` to force re-login. In CLI, use `--force` instead. + """ + if skip_if_logged_in and get_token() is not None: + logger.info("User is already logged in. Use `hf auth login --force` to force re-login.") + return + + if get_token() is not None: + logger.info("Note: a token is already saved on this machine. Logging in again will replace the active token.") + + if _prompt_login_method() == "token": + _paste_token_login() + else: + _device_code_login() + + +def _prompt_login_method() -> str: + """Ask the user how to log in: "browser" (default) or "token". Never prompts without a TTY.""" + if sys.stdin is None or not sys.stdin.isatty(): + return "browser" + choice = select_choice("How would you like to log in?", ["Log in with your browser", "Paste an access token"]) + return "browser" if choice == 0 else "token" + + +def _paste_token_login() -> None: + logger.info( + " To log in, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens ." + ) + if os.name == "nt": + logger.info("Token can be pasted using 'Right-Click'.") + token = getpass("Enter your token (input will not be visible): ") + _validate_and_save_token(token=token, add_to_git_credential=False) + + +### +# Notebook-based login +### + + +def notebook_login(*, skip_if_logged_in: bool = True) -> None: + """ + Displays a prompt to log in to the HF website and store the token. + + This is equivalent to [`login`] without passing a token when run in a notebook. + [`notebook_login`] is useful if you want to force the use of the notebook flow + instead of a prompt in the terminal. + + For more details, see [`login`]. + + Args: + skip_if_logged_in (`bool`, defaults to `True`): + If `True`, do not prompt for token if user is already logged in. + Set to `False` to force re-login. In CLI, use `--force` instead. + """ + if skip_if_logged_in and get_token() is not None: + logger.info("User is already logged in. Use `hf auth login --force` to force re-login.") + return + + try: + from IPython.display import HTML, display # type: ignore + except ImportError: + # Not in a notebook environment: fall back to the terminal flow + interpreter_login(skip_if_logged_in=False) + return + + device_info = request_device_code() + # Escape server-provided values: they end up in raw notebook HTML. + verification_uri = html.escape(device_info["verification_uri"]) + verification_uri_complete = html.escape(device_info["verification_uri_complete"]) + + display( + HTML( + '
Hugging Face

' + "

To log in, open this URL and enter the code:

" + f'

{verification_uri}

' + '

' + f"{html.escape(device_info['user_code'])}

" + ) + ) + display(HTML("
Waiting for authorization...
")) + try: + response = poll_device_token(device_info) + except DeviceCodeError as e: + display(HTML(f"
Login failed: {html.escape(str(e))}
")) + return + + try: + token_name, username = _save_oauth_token(response) + except Exception as error: + display(HTML(f"
{html.escape(str(error))}
")) + return + + message = f"Login successful. Logged in as {html.escape(username)} (token: {html.escape(token_name)})." + if note := _expiration_note(response): + message += f"
{html.escape(note)}" + display(HTML(f"
{message}
")) + + +### +# Login private helpers +### + + +def _validate_and_save_token( + token: str, + add_to_git_credential: bool, + refresh_token: str | None = None, + expires_at: int | None = None, +) -> tuple[str, str]: + """Validate a token against the Hub, save it to the stored tokens file and set it as active. + + The token is stored under its `displayName` from the whoami response, or `oauth-{username}` + for OAuth tokens (which have no display name). + + Args: + token (`str`): + The access token. + add_to_git_credential (`bool`): + Whether to save the token to the git credential helpers. + refresh_token (`str`, *optional*): + OAuth refresh token to persist alongside the access token. + expires_at (`int`, *optional*): + Unix timestamp at which the access token expires. + + Returns: + `tuple[str, str]`: The token name and the username. + """ + from .hf_api import whoami # avoid circular import + + if token.startswith("api_org"): + raise ValueError("You must use your personal account token, not an organization token.") + + token_info = whoami(token) + username = token_info["name"] + + access_token_info = (token_info.get("auth") or {}).get("accessToken") or {} + if role := access_token_info.get("role"): + logger.info(f"Token is valid (permission: {role}).") + else: + logger.info("Token is valid.") + + token_name = access_token_info.get("displayName") or f"oauth-{username}" + + # Store token locally + _save_token(token=token, token_name=token_name, refresh_token=refresh_token, expires_at=expires_at) + # Set active token + _set_active_token(token_name=token_name, add_to_git_credential=add_to_git_credential) + logger.info("Login successful.") + if _get_token_from_environment(): + logger.warning( + "Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured." + ) + else: + logger.info(f"The current active token is: `{token_name}`") + return token_name, username + + +def _logout_from_token(token_name: str) -> None: + """Logout from a specific access token. + + Args: + token_name (`str`): + The name of the access token to logout from. + """ + stored_tokens = _read_stored_tokens_full() + # If there is no access tokens saved or the access token name is not found, do nothing + if token_name not in stored_tokens: + return + + fields = stored_tokens.pop(token_name) + _save_stored_tokens_full(stored_tokens) + + if fields.get("hf_token") == _get_token_from_file(): + logger.warning(f"Active token '{token_name}' has been deleted.") + Path(constants.HF_TOKEN_PATH).unlink(missing_ok=True) + + +def _format_expiration(expires_at: str | None) -> str: + """Format an `expires_at` unix timestamp for display in `auth list`.""" + if not expires_at: + return "" + try: + timestamp = int(expires_at) + except ValueError: + return "" + date_str = datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d") + return f"{date_str} (expired)" if timestamp < time.time() else date_str + + +def _set_active_token( + token_name: str, + add_to_git_credential: bool, +) -> None: + """Set the active access token. + + Args: + token_name (`str`): + The name of the token to set as active. + """ + token = _get_token_by_name(token_name) + if not token: + raise ValueError(f"Token {token_name} not found in {constants.HF_STORED_TOKENS_PATH}") + if add_to_git_credential: + if _is_git_credential_helper_configured(): + set_git_credential(token) + logger.info( + "Your token has been saved in your configured git credential helpers" + + f" ({','.join(list_credential_helpers())})." + ) + else: + logger.warning("Token has not been saved to git credential helper.") + # Write token to HF_TOKEN_PATH + _write_secret(Path(constants.HF_TOKEN_PATH), token) + logger.info(f"Your token has been saved to {constants.HF_TOKEN_PATH}") + + +def _is_git_credential_helper_configured() -> bool: + """Check if a git credential helper is configured. + + Warns user if not the case (except for Google Colab where "store" is set by default + by `huggingface_hub`). + """ + helpers = list_credential_helpers() + if len(helpers) > 0: + return True # Do not warn: at least 1 helper is set + + # Only in Google Colab to avoid the warning message + # See https://github.com/huggingface/huggingface_hub/issues/1043#issuecomment-1247010710 + if is_google_colab(): + _set_store_as_git_credential_helper_globally() + return True # Do not warn: "store" is used by default in Google Colab + + # Otherwise, warn user + print( + ANSI.red( + "Cannot authenticate through git-credential as no helper is defined on your" + " machine.\nYou might have to re-authenticate when pushing to the Hugging" + " Face Hub.\nRun the following command in your terminal in case you want to" + " set the 'store' credential helper as default.\n\ngit config --global" + " credential.helper store\n\nRead" + " https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more" + " details." + ) + ) + return False + + +def _set_store_as_git_credential_helper_globally() -> None: + """Set globally the credential.helper to `store`. + + To be used only in Google Colab as we assume the user doesn't care about the git + credential config. It is the only particular case where we don't want to display the + warning message in [`notebook_login()`]. + + Related: + - https://github.com/huggingface/huggingface_hub/issues/1043 + - https://github.com/huggingface/huggingface_hub/issues/1051 + - https://git-scm.com/docs/git-credential-store + """ + try: + run_subprocess("git config --global credential.helper store") + except subprocess.CalledProcessError as exc: + raise OSError(exc.stderr) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_oauth.py b/venv/lib/python3.12/site-packages/huggingface_hub/_oauth.py new file mode 100644 index 0000000000000000000000000000000000000000..cac82ada87262370031740ea6d7c9234aec44de0 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_oauth.py @@ -0,0 +1,460 @@ +import datetime +import hashlib +import logging +import os +import time +import urllib.parse +import warnings +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal + +from . import constants +from .hf_api import whoami +from .utils import experimental, get_token + + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + import fastapi + + +@dataclass +class OAuthOrgInfo: + """ + Information about an organization linked to a user logged in with OAuth. + + Attributes: + sub (`str`): + Unique identifier for the org. OpenID Connect field. + name (`str`): + The org's full name. OpenID Connect field. + preferred_username (`str`): + The org's username. OpenID Connect field. + picture (`str`): + The org's profile picture URL. OpenID Connect field. + plan (`str`, *optional*): + The org's plan (e.g., "enterprise", "team"). Hugging Face field. + can_pay (`Optional[bool]`, *optional*): + Whether the org has a payment method set up. Hugging Face field. + role_in_org (`Optional[str]`, *optional*): + The user's role in the org. Hugging Face field. + security_restrictions (`Optional[list[Literal["ip", "token-policy", "mfa", "sso"]]]`, *optional*): + Array of security restrictions that the user hasn't completed for this org. Possible values: "ip", "token-policy", "mfa", "sso". Hugging Face field. + """ + + sub: str + name: str + preferred_username: str + picture: str + plan: str | None = None + can_pay: bool | None = None + role_in_org: str | None = None + security_restrictions: list[Literal["ip", "token-policy", "mfa", "sso"]] | None = None + + +@dataclass +class OAuthUserInfo: + """ + Information about a user logged in with OAuth. + + Attributes: + sub (`str`): + Unique identifier for the user, even in case of rename. OpenID Connect field. + name (`str`): + The user's full name. OpenID Connect field. + preferred_username (`str`): + The user's username. OpenID Connect field. + email_verified (`Optional[bool]`, *optional*): + Indicates if the user's email is verified. OpenID Connect field. + email (`Optional[str]`, *optional*): + The user's email address. OpenID Connect field. + picture (`str`): + The user's profile picture URL. OpenID Connect field. + profile (`str`): + The user's profile URL. OpenID Connect field. + website (`Optional[str]`, *optional*): + The user's website URL. OpenID Connect field. + is_pro (`bool`): + Whether the user is a pro user. Hugging Face field. + can_pay (`Optional[bool]`, *optional*): + Whether the user has a payment method set up. Hugging Face field. + orgs (`Optional[list[OrgInfo]]`, *optional*): + List of organizations the user is part of. Hugging Face field. + """ + + sub: str + name: str + preferred_username: str + email_verified: bool | None + email: str | None + picture: str + profile: str + website: str | None + is_pro: bool + can_pay: bool | None + orgs: list[OAuthOrgInfo] | None + + +@dataclass +class OAuthInfo: + """ + Information about the OAuth login. + + Attributes: + access_token (`str`): + The access token. + access_token_expires_at (`datetime.datetime`): + The expiration date of the access token. + user_info ([`OAuthUserInfo`]): + The user information. + state (`str`, *optional*): + State passed to the OAuth provider in the original request to the OAuth provider. + scope (`str`): + Granted scope. + """ + + access_token: str + access_token_expires_at: datetime.datetime + user_info: OAuthUserInfo + state: str | None + scope: str + + +@experimental +def attach_huggingface_oauth(app: "fastapi.FastAPI", route_prefix: str = "/"): + """ + Add OAuth endpoints to a FastAPI app to enable OAuth login with Hugging Face. + + How to use: + - Call this method on your FastAPI app to add the OAuth endpoints. + - Inside your route handlers, call `parse_huggingface_oauth(request)` to retrieve the OAuth info. + - If user is logged in, an [`OAuthInfo`] object is returned with the user's info. If not, `None` is returned. + - In your app, make sure to add links to `/oauth/huggingface/login` and `/oauth/huggingface/logout` for the user to log in and out. + + Example: + ```py + from huggingface_hub import attach_huggingface_oauth, parse_huggingface_oauth + + # Create a FastAPI app + app = FastAPI() + + # Add OAuth endpoints to the FastAPI app + attach_huggingface_oauth(app) + + # Add a route that greets the user if they are logged in + @app.get("/") + def greet_json(request: Request): + # Retrieve the OAuth info from the request + oauth_info = parse_huggingface_oauth(request) # e.g. OAuthInfo dataclass + if oauth_info is None: + return {"msg": "Not logged in!"} + return {"msg": f"Hello, {oauth_info.user_info.preferred_username}!"} + ``` + """ + # TODO: handle generic case (handling OAuth in a non-Space environment with custom dev values) (low priority) + + # Add SessionMiddleware to the FastAPI app to store the OAuth info in the session. + # Session Middleware requires a secret key to sign the cookies. Let's use a hash + # of the OAuth secret key to make it unique to the Space + updated in case OAuth + # config gets updated. When ran locally, we use an empty string as a secret key. + try: + from starlette.middleware.sessions import SessionMiddleware + except ImportError as e: + raise ImportError( + "Cannot initialize OAuth to due a missing library. Please run `pip install huggingface_hub[oauth]` or add " + "`huggingface_hub[oauth]` to your requirements.txt file in order to install the required dependencies." + ) from e + session_secret = (constants.OAUTH_CLIENT_SECRET or "") + "-v1" + app.add_middleware( + SessionMiddleware, # type: ignore + secret_key=hashlib.sha256(session_secret.encode()).hexdigest(), + same_site="none", + https_only=True, + ) # type: ignore + + # Add OAuth endpoints to the FastAPI app: + # - {route_prefix}/oauth/huggingface/login + # - {route_prefix}/oauth/huggingface/callback + # - {route_prefix}/oauth/huggingface/logout + # If the app is running in a Space, OAuth is enabled normally. + # Otherwise, we mock the endpoints to make the user log in with a fake user profile - without any calls to hf.co. + route_prefix = route_prefix.strip("/") + if os.getenv("SPACE_ID") is not None: + logger.info("OAuth is enabled in the Space. Adding OAuth routes.") + _add_oauth_routes(app, route_prefix=route_prefix) + else: + logger.info("App is not running in a Space. Adding mocked OAuth routes.") + _add_mocked_oauth_routes(app, route_prefix=route_prefix) + + +def parse_huggingface_oauth(request: "fastapi.Request") -> OAuthInfo | None: + """ + Returns the information from a logged-in user as a [`OAuthInfo`] object. + + For flexibility and future-proofing, this method is very lax in its parsing and does not raise errors. + Missing fields are set to `None` without a warning. + + Return `None`, if the user is not logged in (no info in session cookie). + + See [`attach_huggingface_oauth`] for an example on how to use this method. + """ + if "oauth_info" not in request.session: + logger.debug("No OAuth info in session.") + return None + + logger.debug("Parsing OAuth info from session.") + oauth_data = request.session["oauth_info"] + user_data = oauth_data.get("userinfo", {}) + orgs_data = user_data.get("orgs", []) + + orgs = ( + [ + OAuthOrgInfo( + sub=org.get("sub"), + name=org.get("name"), + preferred_username=org.get("preferred_username"), + picture=org.get("picture"), + plan=org.get("plan"), + can_pay=org.get("canPay"), + role_in_org=org.get("roleInOrg"), + security_restrictions=org.get("securityRestrictions"), + ) + for org in orgs_data + ] + if orgs_data + else None + ) + + user_info = OAuthUserInfo( + sub=user_data.get("sub"), + name=user_data.get("name"), + preferred_username=user_data.get("preferred_username"), + email_verified=user_data.get("email_verified"), + email=user_data.get("email"), + picture=user_data.get("picture"), + profile=user_data.get("profile"), + website=user_data.get("website"), + is_pro=user_data.get("isPro"), + can_pay=user_data.get("canPay"), + orgs=orgs, + ) + + return OAuthInfo( + access_token=oauth_data.get("access_token"), + access_token_expires_at=datetime.datetime.fromtimestamp(oauth_data.get("expires_at")), + user_info=user_info, + state=oauth_data.get("state"), + scope=oauth_data.get("scope"), + ) + + +def _add_oauth_routes(app: "fastapi.FastAPI", route_prefix: str) -> None: + """Add OAuth routes to the FastAPI app (login, callback handler and logout).""" + try: + import fastapi + from authlib.integrations.base_client.errors import MismatchingStateError + from authlib.integrations.starlette_client import OAuth + from fastapi.responses import RedirectResponse + except ImportError as e: + raise ImportError( + "Cannot initialize OAuth to due a missing library. Please run `pip install huggingface_hub[oauth]` or add " + "`huggingface_hub[oauth]` to your requirements.txt file." + ) from e + + # Check environment variables + msg = ( + "OAuth is required but '{}' environment variable is not set. Make sure you've enabled OAuth in your Space by" + " setting `hf_oauth: true` in the Space metadata." + ) + if constants.OAUTH_CLIENT_ID is None: + raise ValueError(msg.format("OAUTH_CLIENT_ID")) + if constants.OAUTH_CLIENT_SECRET is None: + raise ValueError(msg.format("OAUTH_CLIENT_SECRET")) + if constants.OAUTH_SCOPES is None: + raise ValueError(msg.format("OAUTH_SCOPES")) + if constants.OPENID_PROVIDER_URL is None: + raise ValueError(msg.format("OPENID_PROVIDER_URL")) + + # Register OAuth server + oauth = OAuth() + oauth.register( + name="huggingface", + client_id=constants.OAUTH_CLIENT_ID, + client_secret=constants.OAUTH_CLIENT_SECRET, + client_kwargs={"scope": constants.OAUTH_SCOPES}, + server_metadata_url=constants.OPENID_PROVIDER_URL + "/.well-known/openid-configuration", + ) + + login_uri, callback_uri, logout_uri = _get_oauth_uris(route_prefix) + + # Register OAuth endpoints + @app.get(login_uri) + async def oauth_login(request: fastapi.Request) -> RedirectResponse: + """Endpoint that redirects to HF OAuth page.""" + redirect_uri = _generate_redirect_uri(request) + return await oauth.huggingface.authorize_redirect(request, redirect_uri) # type: ignore + + @app.get(callback_uri) + async def oauth_redirect_callback(request: fastapi.Request) -> RedirectResponse: + """Endpoint that handles the OAuth callback.""" + try: + oauth_info = await oauth.huggingface.authorize_access_token(request) # type: ignore + except MismatchingStateError: + # Parse query params + nb_redirects = int(request.query_params.get("_nb_redirects", 0)) + target_url = request.query_params.get("_target_url") + + # Build redirect URI with the same query params as before and bump nb_redirects count + query_params: dict[str, int | str] = {"_nb_redirects": nb_redirects + 1} + if target_url: + query_params["_target_url"] = target_url + + redirect_uri = f"{login_uri}?{urllib.parse.urlencode(query_params)}" + + # If the user is redirected more than 3 times, it is very likely that the cookie is not working properly. + # (e.g. browser is blocking third-party cookies in iframe). In this case, redirect the user in the + # non-iframe view. + if nb_redirects > constants.OAUTH_MAX_REDIRECTS: + host = os.environ.get("SPACE_HOST") + if host is None: # cannot happen in a Space + raise RuntimeError( + "App is not running in a Space (SPACE_HOST environment variable is not set). Cannot redirect to non-iframe view." + ) from None + host_url = "https://" + host.rstrip("/") + return RedirectResponse(host_url + redirect_uri) + + # Redirect the user to the login page again + return RedirectResponse(redirect_uri) + + # OAuth login worked => store the user info in the session and redirect + logger.debug("Successfully logged in with OAuth. Storing user info in session.") + request.session["oauth_info"] = oauth_info + return RedirectResponse(_get_redirect_target(request)) + + @app.get(logout_uri) + async def oauth_logout(request: fastapi.Request) -> RedirectResponse: + """Endpoint that logs out the user (e.g. delete info from cookie session).""" + logger.debug("Logged out with OAuth. Removing user info from session.") + request.session.pop("oauth_info", None) + return RedirectResponse(_get_redirect_target(request)) + + +def _add_mocked_oauth_routes(app: "fastapi.FastAPI", route_prefix: str = "/") -> None: + """Add fake oauth routes if app is run locally and OAuth is enabled. + + Using OAuth will have the same behavior as in a Space but instead of authenticating with HF, a mocked user profile + is added to the session. + """ + try: + import fastapi + from fastapi.responses import RedirectResponse + from starlette.datastructures import URL + except ImportError as e: + raise ImportError( + "Cannot initialize OAuth to due a missing library. Please run `pip install huggingface_hub[oauth]` or add " + "`huggingface_hub[oauth]` to your requirements.txt file." + ) from e + + warnings.warn( + "OAuth is not supported outside of a Space environment. To help you debug your app locally, the oauth endpoints" + " are mocked to return your profile and token. To make it work, your machine must be logged in to Huggingface." + ) + mocked_oauth_info = _get_mocked_oauth_info() + + login_uri, callback_uri, logout_uri = _get_oauth_uris(route_prefix) + + # Define OAuth routes + @app.get(login_uri) + async def oauth_login(request: fastapi.Request) -> RedirectResponse: + """Fake endpoint that redirects to HF OAuth page.""" + # Define target (where to redirect after login) + redirect_uri = _generate_redirect_uri(request) + return RedirectResponse(callback_uri + "?" + urllib.parse.urlencode({"_target_url": redirect_uri})) + + @app.get(callback_uri) + async def oauth_redirect_callback(request: fastapi.Request) -> RedirectResponse: + """Endpoint that handles the OAuth callback.""" + request.session["oauth_info"] = mocked_oauth_info + return RedirectResponse(_get_redirect_target(request)) + + @app.get(logout_uri) + async def oauth_logout(request: fastapi.Request) -> RedirectResponse: + """Endpoint that logs out the user (e.g. delete cookie session).""" + request.session.pop("oauth_info", None) + logout_url = URL("/").include_query_params(**request.query_params) + return RedirectResponse(url=logout_url, status_code=302) # see https://github.com/gradio-app/gradio/pull/9659 + + +def _generate_redirect_uri(request: "fastapi.Request") -> str: + if "_target_url" in request.query_params: + # if `_target_url` already in query params => respect it + target = request.query_params["_target_url"] + else: + # otherwise => keep query params + target = "/?" + urllib.parse.urlencode(request.query_params) + + redirect_uri = request.url_for("oauth_redirect_callback").include_query_params(_target_url=target) + redirect_uri_as_str = str(redirect_uri) + if redirect_uri.netloc.endswith(".hf.space"): + # In Space, FastAPI redirect as http but we want https + redirect_uri_as_str = redirect_uri_as_str.replace("http://", "https://") + return redirect_uri_as_str + + +def _get_redirect_target(request: "fastapi.Request", default_target: str = "/") -> str: + return request.query_params.get("_target_url", default_target) + + +def _get_mocked_oauth_info() -> dict: + token = get_token() + if token is None: + raise ValueError( + "Your machine must be logged in to HF to debug an OAuth app locally. Please" + " run `hf auth login` or set `HF_TOKEN` as environment variable " + "with one of your access token. You can generate a new token in your " + "settings page (https://huggingface.co/settings/tokens)." + ) + + user = whoami() + if user["type"] != "user": + raise ValueError( + "Your machine is not logged in with a personal account. Please use a " + "personal access token. You can generate a new token in your settings page" + " (https://huggingface.co/settings/tokens)." + ) + + return { + "access_token": token, + "token_type": "bearer", + "expires_in": 8 * 60 * 60, # 8 hours + "id_token": "FOOBAR", + "scope": "openid profile", + "refresh_token": "hf_oauth__refresh_token", + "expires_at": int(time.time()) + 8 * 60 * 60, # 8 hours + "userinfo": { + "sub": "0123456789", + "name": user["fullname"], + "preferred_username": user["name"], + "profile": f"https://huggingface.co/{user['name']}", + "picture": user["avatarUrl"], + "website": "", + "aud": "00000000-0000-0000-0000-000000000000", + "auth_time": 1691672844, + "nonce": "aaaaaaaaaaaaaaaaaaa", + "iat": 1691672844, + "exp": 1691676444, + "iss": "https://huggingface.co", + }, + } + + +def _get_oauth_uris(route_prefix: str = "/") -> tuple[str, str, str]: + route_prefix = route_prefix.strip("/") + if route_prefix: + route_prefix = f"/{route_prefix}" + return ( + f"{route_prefix}/oauth/huggingface/login", + f"{route_prefix}/oauth/huggingface/callback", + f"{route_prefix}/oauth/huggingface/logout", + ) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_oidc.py b/venv/lib/python3.12/site-packages/huggingface_hub/_oidc.py new file mode 100644 index 0000000000000000000000000000000000000000..3a7dba20418a2ece15afa5b5c1cbc127eb8f32cd --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_oidc.py @@ -0,0 +1,163 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Keyless CI/CD authentication via OIDC token exchange ("Trusted Publishers"). + +A CI job proves its identity to the Hub with a short-lived OIDC id token minted by its CI +provider (e.g. GitHub Actions), then exchanges it at ``POST {ENDPOINT}/oauth/token`` (RFC 8693) +for a short-lived Hugging Face token — no long-lived ``HF_TOKEN`` secret to store. + +This module is self-contained: it only handles minting the provider id token and the exchange. +It deliberately does not register a public API or a CLI verb; the integration point is the token +resolution in ``utils/_auth.py`` (see ``_get_token_from_oidc``). + +Docs: https://huggingface.co/docs/hub/trusted-publishers +""" + +import os +from enum import Enum + +from . import constants +from .errors import OIDCError +from .utils import get_session, hf_raise_for_status + + +# RFC 8693 token-exchange grant + id-token subject type (see trusted-publishers docs). +_TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange" +_ID_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:id_token" + + +class Provider(str, Enum): + """CI providers that can mint an OIDC id token natively. GitHub Actions only for now.""" + + GITHUB = "github" + + +def detect_provider() -> Provider | None: + """Detect the CI provider able to mint an OIDC id token, or `None` if not in a supported CI.""" + if os.environ.get("GITHUB_ACTIONS") == "true": + return Provider.GITHUB + return None + + +def _get_github_oidc_token(audience: str) -> str: + """Mint an OIDC id token from the GitHub Actions runtime. + + Relies on the `ACTIONS_ID_TOKEN_REQUEST_URL` / `ACTIONS_ID_TOKEN_REQUEST_TOKEN` env vars, + which GitHub only injects when the job declares `permissions: id-token: write`. + """ + request_url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL") + request_token = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN") + if not request_url or not request_token: + raise OIDCError( + "Cannot request an OIDC id token from GitHub Actions. Make sure the workflow job sets " + "`permissions: id-token: write`. See " + "https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect" + ) + response = get_session().get( + request_url, + params={"audience": audience}, + headers={"Authorization": f"Bearer {request_token}"}, + ) + hf_raise_for_status(response) + return response.json()["value"] + + +def get_oidc_token(*, provider: Provider | str | None = None, audience: str | None = None) -> str: + """Mint a raw OIDC id token (JWT) from the current CI provider. + + Args: + provider (`str`, *optional*): + CI provider to use. Auto-detected from the environment when omitted. + audience (`str`, *optional*): + The `aud` claim to request. Defaults to `constants.ENDPOINT` so it matches the endpoint + that validates it (respects `HF_ENDPOINT`/staging). + + Returns: + `str`: The raw id token (JWT) to pass to [`exchange_oidc_token`]. + """ + audience = audience or constants.ENDPOINT + provider = provider or detect_provider() + supported = ", ".join(p.value for p in Provider) + if provider is None: + raise OIDCError(f"No supported CI OIDC provider detected. Trusted Publishers currently supports: {supported}.") + if provider == Provider.GITHUB: + return _get_github_oidc_token(audience) + raise NotImplementedError(f"OIDC provider '{provider}' is not supported yet. Supported: {supported}.") + + +def exchange_oidc_token(*, subject_token: str, resource: str, endpoint: str | None = None) -> dict: + """Exchange a CI OIDC id token for a short-lived Hugging Face token (RFC 8693). + + Args: + subject_token (`str`): + The raw OIDC id token (JWT) from the CI provider. Its `aud` claim must be the Hub URL. + resource (`str`): + What to scope the token to: a Hub repo (`namespace/name`, `datasets/namespace/name`, + `spaces/namespace/name`, `kernels/namespace/name`) for a write token, or a bare Hub + username for a read-only `gated-repos` token. + endpoint (`str`, *optional*): + Hub endpoint. Defaults to `constants.ENDPOINT` (respects `HF_ENDPOINT`/staging). + + Returns: + `dict`: The token-exchange response, e.g. + `{"access_token": "hf_jwt_…", "token_type": "bearer", "expires_in": 3600, ...}`. + """ + response = get_session().post( + f"{endpoint or constants.ENDPOINT}/oauth/token", + json={ + "grant_type": _TOKEN_EXCHANGE_GRANT_TYPE, + "subject_token_type": _ID_TOKEN_TYPE, + "subject_token": subject_token, + "resource": resource, + }, + ) + hf_raise_for_status(response) + return response.json() + + +def oidc_login( + *, + resource: str, + subject_token: str | None = None, + provider: Provider | str | None = None, + audience: str | None = None, + endpoint: str | None = None, +) -> dict: + """Mint a CI OIDC id token and exchange it for a Hugging Face token. + + Convenience wrapper around [`get_oidc_token`] + [`exchange_oidc_token`]. Returns the raw + exchange response (it does not persist anything — the caller decides what to do with the token). + + Args: + resource (`str`): + Repo or username to scope the token to. See [`exchange_oidc_token`]. + subject_token (`str`, *optional*): + A pre-minted OIDC id token to exchange directly. Use this for CI providers not yet + supported natively (e.g. GitLab): mint the id token in your job and pass it here. When + omitted, the token is minted from the detected `provider`. + provider (`str`, *optional*): + CI provider. Auto-detected when omitted. Ignored when `subject_token` is provided. + audience (`str`, *optional*): + The `aud` claim to request. Defaults to the resolved `endpoint`, so it matches the + endpoint that validates it. + endpoint (`str`, *optional*): + Hub endpoint. Defaults to `constants.ENDPOINT`. + + Returns: + `dict`: The token-exchange response (`access_token`, `token_type`, `expires_in`, ...). + """ + endpoint = endpoint or constants.ENDPOINT + if subject_token is None: + subject_token = get_oidc_token(provider=provider, audience=audience or endpoint) + return exchange_oidc_token(subject_token=subject_token, resource=resource, endpoint=endpoint) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_sandbox.py b/venv/lib/python3.12/site-packages/huggingface_hub/_sandbox.py new file mode 100644 index 0000000000000000000000000000000000000000..c3c7b93a1828a7089ee8c1897f19753065318cf5 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_sandbox.py @@ -0,0 +1,1764 @@ +# coding=utf-8 +# Copyright 2026-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib +import hmac +import json +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from secrets import token_hex +from typing import Any, BinaryIO, Callable, Iterator, List, Literal, overload + +import httpx + +from . import constants +from ._sandbox_cache import ( + CachedHost, + delete_pool_cache, + read_pool_cache, + save_pool_cache, +) +from ._space_api import Volume +from .errors import HfHubHTTPError, SandboxCommandError, SandboxError +from .hf_api import HfApi, JobInfo +from .utils import get_token, logging +from .utils._parsing import parse_duration + + +logger = logging.get_logger(__name__) + +# Port the sandbox server listens on inside the job. Deliberately uncommon so that typical user ports stay free. +SANDBOX_SERVER_PORT = 49983 + +# Stable marker present on every sandbox job to ease filtering +SANDBOX_LABEL = "hf-sandbox" +# Sandbox mode: "dedicated" or "pool" to ease filtering +MODE_LABEL = "hf-sandbox-mode" +MODE_DEDICATED = "dedicated" +MODE_POOL = "pool" +# Pool name, to scope host reuse to a named group (see SandboxPool(name=...)). Present on pool +# host jobs only. Pool config (capacity, idle timeout) lives in the host's env vars, read via +# inspect_job — labels are kept for filtering/grouping only. +POOL_LABEL = "hf-sandbox-pool" +# Per-job public nonce the sandbox token is derived from (see _derive_sandbox_token), so +# `Sandbox.connect(id)` can recompute the token from any machine with no local state. +NONCE_LABEL = "hf-sandbox-nonce" + +DEFAULT_IMAGE = "python:3.12" + +DEFAULT_IDLE_TIMEOUT = 10 * 60 # 10 minutes +SANDBOX_MAX_LIFETIME = "24h" + +DEFAULT_SANDBOXES_PER_HOST = 50 + +SHARED_ID_SEP = "." + +# Job stages in which a sandbox/host is finished and needs no teardown. +_TERMINAL_STAGES = ("COMPLETED", "ERROR", "DELETED", "CANCELED") + +# Safety bound on create()'s pack-retry loop +_MAX_PACK_ROUNDS = 8 + +# hf-mount path where the server bucket is mounted on every sandbox job +_SERVER_MOUNT_PATH = "/.hf-sbx-server" + +# Job startup script (needs only /bin/sh). The server bucket is public, so the download is +# unauthenticated: no HF credential is ever placed in the job environment (see `_derive_sandbox_token`). +_BOOTSTRAP_DOWNLOAD = """\ +set -e +d=/tmp/.sbx-server +if command -v wget >/dev/null 2>&1; then wget -q -O "$d" "$SBX_SERVER_URL" +elif command -v curl >/dev/null 2>&1; then curl -fsSL -o "$d" "$SBX_SERVER_URL" +else cp "$SBX_SERVER_MOUNT/sbx-server" "$d"; fi +chmod +x "$d" +unset SBX_SERVER_URL SBX_SERVER_MOUNT +exec "$d" +""" + + +def _derive_sandbox_token(hf_token: str, nonce: str) -> str: + """Derive the per-sandbox auth token from the user's HF token and the sandbox nonce. + + Stateless: any machine holding the same HF token can recompute it from the + nonce stored in the job's labels, so `Sandbox.connect(job_id)` needs no local state. + The HF token itself is never sent to the sandbox. + """ + return hmac.new(hf_token.encode(), f"hf-sandbox:{nonce}".encode(), hashlib.sha256).hexdigest() + + +def _duration_to_secs(duration: int | float | str) -> int: + """Parse a duration like 300, "300s", "10m", "2h", "1d" into seconds.""" + if isinstance(duration, (int, float)): + return int(duration) + return parse_duration(duration) + + +@dataclass +class SandboxCommandResult: + """Result of a command executed in a sandbox with [`Sandbox.run`].""" + + exit_code: int | None + stdout: str + stderr: str + signal: int | None = None + timed_out: bool = False + duration_ms: int = 0 + + @property + def ok(self) -> bool: + return self.exit_code == 0 + + def __repr__(self) -> str: + out = self.stdout if len(self.stdout) <= 80 else self.stdout[:77] + "..." + return f"SandboxCommandResult(exit_code={self.exit_code}, stdout={out!r}, duration_ms={self.duration_ms})" + + +@dataclass +class SandboxProcess: + """A background process started in a sandbox with [`Sandbox.run`]`(..., background=True)`. + + List a sandbox's processes with [`Sandbox.processes`] and stop one with [`SandboxProcess.kill`]. + Completed processes stay in the listing until the sandbox is deleted, so `running` and + `exit_code` tell whether a process is still alive or already exited (as of when it was listed). + """ + + pid: int + cmd: str | List[str] + # Back-reference to the sandbox, used by `kill()`. Excluded from repr/eq so a process + # stays a plain data object (and two with the same pid compare equal). + _sandbox: "Sandbox" = field(repr=False, compare=False) + tag: str | None = None + started_at_ms: int | None = None + running: bool = True + exit_code: int | None = None + + def kill(self) -> None: + """Terminate the background process (idempotent server-side).""" + self._sandbox._request("DELETE", f"/processes/{self.pid}") + + +@dataclass +class FileEntry: + """A file or directory inside a sandbox.""" + + name: str + path: str + type: Literal["file", "dir", "symlink"] + size: int + mtime_ms: int | None = None + mode: str = "" + + +class SandboxFiles: + """Filesystem operations inside a sandbox, available as [`Sandbox.files`]. + + In shared (pool) mode, paths are rooted at the sandbox's private home — the + only place its code can write — so a leading `/` is taken relative to that + home. In dedicated mode, paths are absolute on the container filesystem. + """ + + # Above this size, transfers are split into ranged requests over parallel + # connections: a single TCP stream through the jobs proxy is limited by the + # bandwidth-delay product (~2 MiB/s at ~100ms RTT); parallel streams scale it. + PARALLEL_THRESHOLD = 2 * 1024 * 1024 + PARALLEL_CHUNK_SIZE = 1 * 1024 * 1024 + PARALLEL_MAX_WORKERS = 16 + + def __init__(self, sandbox: "Sandbox") -> None: + self._sandbox = sandbox + + def read(self, path: str) -> bytes: + """Read a file from the sandbox and return its content as bytes.""" + size = self.stat(path).size + if size > self.PARALLEL_THRESHOLD: + return b"".join(self._read_ranges(path, size)) + response = self._sandbox._request("GET", "/files/read", params={"path": path}) + return response.content + + def read_text(self, path: str, encoding: str = "utf-8") -> str: + """Read a file from the sandbox and return its content as a string.""" + return self.read(path).decode(encoding) + + def write(self, path: str, data: str | bytes | BinaryIO, mode: str | None = None) -> None: + """Write content to a file in the sandbox (parent directories are created).""" + if isinstance(data, str): + data = data.encode() + elif not isinstance(data, bytes): + data = data.read() # binary file object + if len(data) > self.PARALLEL_THRESHOLD: + self._write_ranges(path, data, mode) + return + params = {"path": path} + if mode is not None: + params["mode"] = mode + self._sandbox._request("PUT", "/files/write", params=params, content=data) + + def upload(self, local_path: str | Path, path: str, mode: str | None = None) -> None: + """Upload a local file to the sandbox.""" + size = Path(local_path).stat().st_size + if size > self.PARALLEL_THRESHOLD: + self._write_ranges(path, Path(local_path).read_bytes(), mode) + return + with open(local_path, "rb") as f: + self.write(path, f, mode=mode) + + def download(self, path: str, local_path: str | Path) -> None: + """Download a file from the sandbox.""" + size = self.stat(path).size + if size > self.PARALLEL_THRESHOLD: + with open(local_path, "wb") as f: + for part in self._read_ranges(path, size): + f.write(part) + return + with self._sandbox._stream("GET", "/files/read", params={"path": path}) as response: + with open(local_path, "wb") as f: + for chunk in response.iter_bytes(chunk_size=1024 * 1024): + f.write(chunk) + + def _ranges(self, size: int) -> List[tuple[int, int]]: + chunk = self.PARALLEL_CHUNK_SIZE + return [(offset, min(chunk, size - offset)) for offset in range(0, size, chunk)] + + def _parallel(self, items: List[Any], fn: Callable[[Any], Any]) -> List[Any]: + """Run `fn(item)` over items concurrently. + + All workers share the sandbox's `httpx.Client`, which is thread-safe and pools + connections, so parallel transfers fan out over several streams at once. + """ + workers = min(self.PARALLEL_MAX_WORKERS, len(items)) + with ThreadPoolExecutor(workers) as executor: + return list(executor.map(fn, items)) + + def _read_ranges(self, path: str, size: int) -> List[bytes]: + def fetch(rng: tuple[int, int]) -> bytes: + offset, length = rng + response = self._sandbox._request( + "GET", "/files/read", params={"path": path, "offset": offset, "length": length} + ) + return response.content + + return self._parallel(self._ranges(size), fetch) + + def _write_ranges(self, path: str, data: bytes, mode: str | None) -> None: + def push(rng: tuple[int, int]) -> None: + offset, length = rng + params: dict[str, Any] = {"path": path, "offset": offset} + if mode is not None: + params["mode"] = mode + self._sandbox._request("PUT", "/files/write", params=params, content=data[offset : offset + length]) + + self._parallel(self._ranges(len(data)), push) + + def list(self, path: str) -> List[FileEntry]: + """List a directory in the sandbox.""" + response = self._sandbox._request("GET", "/files/list", params={"path": path}) + return [FileEntry(**entry) for entry in response.json()["entries"]] + + def stat(self, path: str) -> FileEntry: + """Get metadata of a file or directory in the sandbox.""" + response = self._sandbox._request("GET", "/files/stat", params={"path": path}) + return FileEntry(**response.json()) + + def exists(self, path: str) -> bool: + """Check whether a path exists in the sandbox.""" + try: + self.stat(path) + return True + except SandboxError as e: + if e.status_code == 404: + return False + raise # network/auth/server errors must not be silently reported as "missing" + + def delete(self, path: str, recursive: bool = False) -> None: + """Delete a file or directory in the sandbox.""" + params = {"path": path} + if recursive: + params["recursive"] = "1" + self._sandbox._request("DELETE", "/files/delete", params=params) + + def mkdir(self, path: str) -> None: + """Create a directory (and parents) in the sandbox.""" + self._sandbox._request("POST", "/files/mkdir", params={"path": path}) + + +def _exec_payload(cmd: str | List[str], shell: bool | None) -> dict[str, Any]: + """Build the `cmd`/`shell` part of an `/exec` payload, validating their consistency.""" + if shell is True and not isinstance(cmd, str): + raise ValueError("shell=True requires `cmd` to be a shell command string, not a list.") + if shell is False and isinstance(cmd, str): + raise ValueError("shell=False requires `cmd` to be an argv list (e.g. ['echo', 'hi']), not a string.") + payload: dict[str, Any] = {"cmd": cmd} + if shell is not None: + payload["shell"] = shell + return payload + + +def _iter_events(response: httpx.Response) -> Iterator[dict]: + """Iterate NDJSON events from a streaming response, skipping keepalive pings.""" + for line in response.iter_lines(): + if not line: + continue + event = json.loads(line) + if event.get("event") != "ping": + yield event + + +def _raise_for_status(response: httpx.Response) -> None: + """Read the error body and raise a SandboxError (works for streaming responses too).""" + response.read() # no-op for buffered responses, reads the body for streaming ones + try: + message = response.json()["error"] + except Exception: + message = response.text[:500] + raise SandboxError(f"Sandbox API error ({response.status_code}): {message}", status_code=response.status_code) + + +class _SandboxServer: + """HTTP transport to one `sbx-server` instance — a dedicated job or a shared host. + + Owns the `httpx.Client`, the base URL and the auth headers. + In dedicated mode a server is paired 1:1 with its [`Sandbox` + In pool mode one server (one host job) is shared by many sandboxes, and `live`/`capacity` track packing. + """ + + def __init__( + self, + *, + job_id: str, + owner: str, + image: str | None, + base_url: str, + nonce: str, + sandbox_token: str, + api: HfApi, + max_connections: int = 10, + capacity: int = 0, + ) -> None: + self.job_id = job_id + self.owner = owner + self._image = image + self.base_url = base_url + # Public nonce the sandbox token is derived from; kept so a host can be persisted + # to (and rebuilt from) the pool cache without re-reading the job labels. + self.nonce = nonce + self._api = api + self._auth_token = _effective_token(api) + self._sandbox_token = sandbox_token + # Packing bookkeeping (shared mode only). + self.capacity = capacity + self.live = 0 + # False only for hosts rebuilt from the (best-effort) pool cache: their job may + # be gone, so the first failed request drops them instead of failing the create. + self.verified = True + # httpx.Client is thread-safe, so a single client serves both sequential requests + # and the concurrent workers used for parallel file transfers / many sandboxes. + self._client = httpx.Client( + headers={ + "Authorization": f"Bearer {self._auth_token}", + "X-Sandbox-Token": sandbox_token, + }, + limits=httpx.Limits(max_connections=max_connections, max_keepalive_connections=max_connections), + follow_redirects=True, + ) + + @classmethod + def from_job( + cls, + *, + job: JobInfo, + nonce: str, + sandbox_token: str, + api: HfApi, + max_connections: int = 10, + capacity: int = 0, + ) -> "_SandboxServer": + """Build a server from a freshly fetched job (reads its exposed server URL).""" + return cls( + job_id=job.id, + owner=job.owner.name, + image=job.docker_image or job.space_id, + base_url=_find_server_url(job), + nonce=nonce, + sandbox_token=sandbox_token, + api=api, + max_connections=max_connections, + capacity=capacity, + ) + + @property + def image(self) -> str | None: + return self._image + + def request(self, method: str, path: str, **kwargs) -> httpx.Response: + """Request to the in-job server. Raises SandboxError on API errors.""" + timeout = kwargs.pop("timeout", httpx.Timeout(60.0, connect=10.0)) + response = self._client.request(method, self.base_url + path, timeout=timeout, **kwargs) + if response.status_code >= 400: + _raise_for_status(response) + return response + + @contextmanager + def stream(self, method: str, path: str, **kwargs) -> Iterator[httpx.Response]: + """Streaming request to the in-job server. Raises SandboxError on API errors.""" + timeout = kwargs.pop("timeout", httpx.Timeout(70.0, connect=10.0)) # server pings every 15s + with self._client.stream(method, self.base_url + path, timeout=timeout, **kwargs) as response: + if response.status_code >= 400: + _raise_for_status(response) + yield response + + def close(self) -> None: + self._client.close() + + def cancel_job(self) -> None: + self._api.cancel_job(job_id=self.job_id, namespace=self.owner) + + def wait_ready(self, start_timeout: float) -> None: + """Poll /health until the server answers; fail fast (with logs) if the job dies.""" + deadline = time.time() + start_timeout + last_job_check = 0.0 + while time.time() < deadline: + try: + response = self._client.get(self.base_url + "/health", timeout=httpx.Timeout(5.0)) + if response.status_code == 200: + return + except httpx.RequestError: + pass + if time.time() - last_job_check > 2.0: + last_job_check = time.time() + job = self._api.inspect_job(job_id=self.job_id, namespace=self.owner) + if job.status.stage in _TERMINAL_STAGES: + logs = _tail_job_logs(self._api, self.job_id, namespace=self.owner) + raise SandboxError( + f"Sandbox job {self.job_id} terminated during startup " + f"(status: {job.status.stage}, message: {job.status.message}).{logs}" + ) + time.sleep(0.15) + raise SandboxError(f"Sandbox job {self.job_id} did not become ready within {start_timeout:.0f}s.") + + +class _KillMethod: + """Lets `kill` work both as a classmethod and an instance method.""" + + @overload + def __get__(self, instance: None, owner: type) -> Callable[..., None]: ... + @overload + def __get__(self, instance: "Sandbox", owner: type) -> Callable[[], None]: ... + def __get__(self, instance: "Sandbox | None", owner: type) -> Callable[..., None]: + if instance is not None: + return instance._kill + + def kill(sandbox_id: str, *, namespace: str | None = None, token: str | None = None) -> None: + owner.connect(sandbox_id, namespace=namespace, token=token).kill() # type: ignore[attr-defined] + + return kill + + +class Sandbox: + """An isolated cloud machine running on Hugging Face Jobs. + + Create a dedicated one with [`Sandbox.create`] (one job per sandbox), or get many cheap shared ones from a [`SandboxPool`]. + Reattach to a running sandbox from anywhere with [`Sandbox.connect`]. Use as a context manager to terminate it on exit: + + ```python + >>> from huggingface_hub import Sandbox + >>> with Sandbox.create(image="python:3.12") as sbx: + ... print(sbx.run("python --version").stdout) + ``` + """ + + def __init__( + self, + *, + id: str, + server: _SandboxServer, + local_id: str | None, + owns_sandbox: bool, + owns_server: bool, + ) -> None: + self.id = id + self._server = server + # None in dedicated mode; the host-local sandbox id in shared mode. + self._local_id = local_id + # Path prefix for all in-server operations: dedicated routes live under + # /v1/*, shared ones under /v1/sandboxes//*. + self._base_path = "/v1" if local_id is None else f"/v1/sandboxes/{local_id}" + # Whether exiting a `with` block terminates the sandbox (True for sandboxes + # we created, False for ones reattached via connect()). + self._owns_sandbox = owns_sandbox + # Whether closing/killing this sandbox also closes the HTTP client. False + # for pool sandboxes, whose client (the host) is owned by the pool. + self._owns_server = owns_server + # Set by SandboxPool to free a packing slot when a shared sandbox is killed. + self._on_kill: Callable[["Sandbox"], None] | None = None + self._killed = False + self.files = SandboxFiles(self) + + # ------------------------------------------------------------------ lifecycle + + @classmethod + def create( + cls, + image: str = DEFAULT_IMAGE, + *, + flavor: str = "cpu-basic", + idle_timeout: int | float | str | None = DEFAULT_IDLE_TIMEOUT, + env: dict[str, Any] | None = None, + secrets: dict[str, Any] | None = None, + volumes: List[Volume] | None = None, + namespace: str | None = None, + forward_hf_token: bool = False, + start_timeout: float = 120.0, + token: str | None = None, + ) -> "Sandbox": + """Create a dedicated sandbox (one HF Job) and block until it is ready (~7s on cpu-basic). + + Each sandbox is a full isolated VM, so this is the right choice for GPU + workloads or untrusted code. To fan out many cheap CPU sandboxes instead, use + [`SandboxPool`]. + + The job runs with a fixed 24h maximum lifetime; `idle_timeout` is the real + keeper — an idle sandbox shuts itself down well before that. + + Args: + image (`str`, *optional*, defaults to `"python:3.12"`): + Any Docker image with `/bin/sh` (Docker Hub or `hf.co/spaces/...`). + flavor (`str`, *optional*, defaults to `"cpu-basic"`): + Hardware flavor, e.g. `"cpu-basic"`, `"a10g-small"`. See `hf jobs hardware`. + idle_timeout (`int` or `float` or `str`, *optional*, defaults to `600`): + Auto-shutdown after this much inactivity (no API calls, no running + processes). Defaults to 10 minutes; pass `None` to disable. + env (`dict[str, Any]`, *optional*): + Environment variables available in the sandbox. + secrets (`dict[str, Any]`, *optional*): + Secret environment variables (encrypted server-side). + volumes (`List[Volume]`, *optional*): + HF repos/buckets to mount, see [`Volume`]. + namespace (`str`, *optional*): + User or org namespace to run under (defaults to current user). + forward_hf_token (`bool`, *optional*, defaults to `False`): + If True, your HF token is injected as `HF_TOKEN` (opt-in). + start_timeout (`float`, *optional*, defaults to `120.0`): + Max seconds to wait for the sandbox to become ready. + token (`str`, *optional*): + HF token override. + + The image only needs `/bin/sh`. The sandbox server is downloaded at startup with + `wget`/`curl` if available, otherwise read off an always-mounted server bucket (which + adds ~2-3s to cold start, so shipping `wget`/`curl` keeps it fast). + """ + api = HfApi(token=token) + hf_token = _effective_token(api) + nonce = token_hex(16) + sandbox_token = _derive_sandbox_token(hf_token, nonce) + + command, job_env, job_secrets, job_volumes = _bootstrap_job_spec( + api, + hf_token, + env=env, + secrets=secrets, + volumes=volumes, + idle_timeout=idle_timeout, + forward_hf_token=forward_hf_token, + sandbox_token=sandbox_token, + ) + + job = api.run_job( + image=image, + command=command, + env=job_env, + secrets=job_secrets, + flavor=flavor, + timeout=SANDBOX_MAX_LIFETIME, + labels={SANDBOX_LABEL: "1", MODE_LABEL: MODE_DEDICATED, NONCE_LABEL: nonce}, + volumes=job_volumes or None, + expose=[SANDBOX_SERVER_PORT], + namespace=namespace, + ) + server: "_SandboxServer | None" = None + try: + server = _SandboxServer.from_job( + job=job, + nonce=nonce, + sandbox_token=sandbox_token, + api=api, + max_connections=SandboxFiles.PARALLEL_MAX_WORKERS + 2, + ) + server.wait_ready(start_timeout) + except Exception: + # run_job already started a billable job; cancel it before re-raising so it + # doesn't linger as an orphan (e.g. if the server port isn't exposed or startup fails). + try: + api.cancel_job(job_id=job.id, namespace=job.owner.name) + except Exception as e: + logger.warning(f"Failed to cancel sandbox job {job.id} after startup failure: {e}") + if server is not None: + server.close() + raise + return cls(id=job.id, server=server, local_id=None, owns_sandbox=True, owns_server=True) + + @classmethod + def connect(cls, sandbox_id: str, *, namespace: str | None = None, token: str | None = None) -> "Sandbox": + """Reattach to a running sandbox from anywhere, using only its id.""" + api = HfApi(token=token) + sandbox_id, namespace = _split_sandbox_id(sandbox_id, namespace) + if SHARED_ID_SEP in sandbox_id: + host_job_id, local_id = sandbox_id.split(SHARED_ID_SEP, 1) + server = _connect_host(api, host_job_id, namespace=namespace) + try: + existing = {item["id"] for item in server.request("GET", "/v1/sandboxes").json()} + if local_id not in existing: + raise SandboxError(f"Sandbox {sandbox_id} no longer exists on host {host_job_id}.") + except Exception: + server.close() # don't leak the HTTP client when the host is gone/unreachable + raise + return cls(id=sandbox_id, server=server, local_id=local_id, owns_sandbox=False, owns_server=True) + + job = api.inspect_job(job_id=sandbox_id, namespace=namespace) + labels = job.labels or {} + nonce = labels.get(NONCE_LABEL) + if labels.get(SANDBOX_LABEL) != "1" or nonce is None: + raise SandboxError(f"Job {sandbox_id} is not a sandbox (missing '{SANDBOX_LABEL}' label).") + if labels.get(MODE_LABEL) == MODE_POOL: + raise SandboxError( + f"Job {sandbox_id} is a sandbox host, not a single sandbox. Connect to one of its " + f"sandboxes with id '{SHARED_ID_SEP}'." + ) + if job.status.stage != "RUNNING": + raise SandboxError(f"Sandbox {sandbox_id} is not running (status: {job.status.stage}).") + sandbox_token = _derive_sandbox_token(_effective_token(api), nonce) + server = _SandboxServer.from_job( + job=job, + nonce=nonce, + sandbox_token=sandbox_token, + api=api, + max_connections=SandboxFiles.PARALLEL_MAX_WORKERS + 2, + ) + return cls(id=job.id, server=server, local_id=None, owns_sandbox=False, owns_server=True) + + # `kill` is both a classmethod (`Sandbox.kill(id)`) and an instance method (`sbx.kill()`), + # dispatched by the `_KillMethod` descriptor; `_kill` holds the instance behaviour. + kill = _KillMethod() + + def _kill(self) -> None: + """Terminate the sandbox. Idempotent. + + Dedicated sandboxes cancel their underlying job; shared sandboxes are + removed from their host (freeing a slot) while the host keeps running. + """ + if self._killed: + return + try: + if self._local_id is None: + self._server.cancel_job() + else: + self._server.request("DELETE", f"/v1/sandboxes/{self._local_id}") + except Exception as e: + # Don't mark as killed: a later kill() call should retry so nothing leaks. + logger.warning(f"Failed to kill sandbox {self.id}: {e}") + return + self._killed = True + if self._on_kill is not None: + self._on_kill(self) + if self._owns_server: + self._server.close() + + def close(self) -> None: + """Release the local HTTP client without terminating the sandbox. Idempotent. + + No-op for pool sandboxes (the client belongs to the pool's host). + """ + if self._owns_server: + self._server.close() + + def __enter__(self) -> "Sandbox": + return self + + def __exit__(self, *exc_info) -> None: + if self._owns_sandbox: + self.kill() + else: + self.close() + + # ------------------------------------------------------------------ exec + + @overload + def run( + self, + cmd: str | List[str], + *, + shell: bool | None = ..., + env: dict[str, Any] | None = ..., + cwd: str | None = ..., + timeout: float | None = ..., + stdin: str | None = ..., + on_stdout: Callable[[str], None] | None = ..., + on_stderr: Callable[[str], None] | None = ..., + check: bool = ..., + background: Literal[False] = ..., + ) -> SandboxCommandResult: ... + + @overload + def run( + self, + cmd: str | List[str], + *, + shell: bool | None = ..., + env: dict[str, Any] | None = ..., + cwd: str | None = ..., + background: Literal[True], + ) -> SandboxProcess: ... + + def run( + self, + cmd: str | List[str], + *, + shell: bool | None = None, + env: dict[str, Any] | None = None, + cwd: str | None = None, + timeout: float | None = None, + stdin: str | None = None, + on_stdout: Callable[[str], None] | None = None, + on_stderr: Callable[[str], None] | None = None, + check: bool = True, + background: bool = False, + ) -> SandboxCommandResult | SandboxProcess: + """Run a command in the sandbox and wait for it, streaming output live. + + With `background=True` the command is started detached and `run` returns a + [`SandboxProcess`] immediately, without waiting for it to finish — handy for + servers and other long-running processes. List them later with [`Sandbox.processes`] + and stop one with [`SandboxProcess.kill`]. The streaming/wait-only options + (`timeout`, `stdin`, `on_stdout`, `on_stderr`, `check`) don't apply in that mode. + + Args: + cmd (`str` or `List[str]`): + A shell command string (run with `/bin/sh -c`) or an argv list (exec'd directly). + shell (`bool`, *optional*): + Force the execution mode instead of inferring it from the type of `cmd`. + `True` runs through `/bin/sh -c` and requires `cmd` to be a string; `False` + exec's `cmd` directly and requires it to be an argv list. `None` (default) + infers from the type. Set it explicitly to avoid the type-driven footgun (e.g. + `["echo hi"]` being exec'd as a single program named `"echo hi"`). + env (`dict[str, Any]`, *optional*): + Extra environment variables for this command. + cwd (`str`, *optional*): + Working directory. + timeout (`float`, *optional*): + Kill the command (whole process group) after this many seconds. + stdin (`str`, *optional*): + Data to write to the command's stdin. + on_stdout (`Callable[[str], None]`, *optional*): + Callback invoked with stdout chunks as they arrive. + on_stderr (`Callable[[str], None]`, *optional*): + Callback invoked with stderr chunks as they arrive. + check (`bool`, *optional*, defaults to `True`): + If True, raise [`SandboxCommandError`] on non-zero exit. + background (`bool`, *optional*, defaults to `False`): + If True, start the command detached and return a [`SandboxProcess`] right + away instead of waiting for it and returning a [`SandboxCommandResult`]. + + Returns: a [`SandboxCommandResult`] (with `exit_code`, `stdout`, `stderr`, + `duration_ms`), or a [`SandboxProcess`] when `background=True`. + """ + payload = _exec_payload(cmd, shell) + if env: + payload["env"] = env + if cwd: + payload["cwd"] = cwd + if background: + data = self._request("POST", "/processes", json=payload).json() + return SandboxProcess(pid=data["pid"], cmd=cmd, tag=data.get("tag"), _sandbox=self) + if timeout is not None: + payload["timeout"] = timeout + if stdin is not None: + payload["stdin"] = stdin + + stdout_parts: list[str] = [] + stderr_parts: list[str] = [] + result: SandboxCommandResult | None = None + with self._stream("POST", "/exec", json=payload) as response: + for event in _iter_events(response): + if event["event"] == "stdout": + stdout_parts.append(event["data"]) + if on_stdout is not None: + on_stdout(event["data"]) + elif event["event"] == "stderr": + stderr_parts.append(event["data"]) + if on_stderr is not None: + on_stderr(event["data"]) + elif event["event"] == "exit": + result = SandboxCommandResult( + exit_code=event["exit_code"], + stdout="".join(stdout_parts), + stderr="".join(stderr_parts), + signal=event.get("signal"), + timed_out=event.get("timed_out", False), + duration_ms=event.get("duration_ms", 0), + ) + if result is None: + raise SandboxError("connection lost while running command") + if check and (result.exit_code != 0 or result.timed_out): + raise SandboxCommandError(cmd=cmd, result=result) + return result + + def processes(self) -> List[SandboxProcess]: + """List the background processes of this sandbox. + + Returns the processes started with [`Sandbox.run`]`(..., background=True)`; stop one + with [`SandboxProcess.kill`]. Completed processes stay listed (with `running=False` and + their `exit_code`) until the sandbox is deleted. + """ + data = self._request("GET", "/processes").json() + return [ + SandboxProcess( + pid=p["pid"], + cmd=p["cmd"], + tag=p.get("tag"), + started_at_ms=p.get("started_at_ms"), + running=p["running"], + exit_code=p.get("exit_code"), + _sandbox=self, + ) + for p in data + ] + + # ------------------------------------------------------------------ misc + + @property + def image(self) -> str | None: + return self._server.image + + @property + def host_id(self) -> str | None: + """For a shared/pool sandbox, the job id of the host running it (else None).""" + return self._server.job_id if self._local_id is not None else None + + # ------------------------------------------------------------------ port proxy + + def proxy_url_for(self, port: int | str, path: str = "/", *, scheme: str = "https://") -> str: + """Public URL that proxies through to a server running *inside* this sandbox. + + Requests to the returned URL are forwarded by the in-job sandbox server to a + server you started in the sandbox on `port`, including WebSocket (`ws(s)://`) + upgrades and streamed responses. Pair it with [`proxy_headers`] for auth. + + How the sandbox must listen on `port`: + + - **Pool / shared sandbox**: it cannot bind a TCP port (Landlock), so bind a + **unix socket** at `$SBX_PROXY_DIR/.sock` (the `SBX_PROXY_DIR` env var + is set in every sandbox). E.g. `uvicorn app:app --uds $SBX_PROXY_DIR/8000.sock`. + - **Dedicated sandbox**: bind a normal TCP port on `127.0.0.1:`. (You can + also expose the port directly via the job proxy without going through here.) + + Args: + port (`int` or `str`): + The port (pool: the `` of the unix socket) the inner server listens on. + path (`str`, *optional*, defaults to `"/"`): + Path on the inner server to point at, e.g. `"/ws"`. + scheme (`str`, *optional*, defaults to `"https://"`): + URL scheme to build the link with. Defaults to `"https://"`; pass + `"wss://"` for a WebSocket client (the proxy is protocol-agnostic, so only + the client-side scheme changes). + + Returns: + `str`: a URL like `https://--49983.hf.jobs/v1/.../proxy/8000/ws` (or + `wss://...` with `scheme="wss://"`). + + Example: + ```python + >>> url = sandbox.proxy_url_for(8000, "/ws", scheme="wss://") + >>> import websockets + >>> async with websockets.connect(url, additional_headers=sandbox.proxy_headers) as ws: + ... await ws.send("hello") + ``` + """ + # base_url is always https://...; swap in the requested scheme (e.g. wss://) for the client. + host_and_rest = self._server.base_url.split("://", 1)[-1] + path = path if path.startswith("/") else "/" + path + return f"{scheme}{host_and_rest}{self._base_path}/proxy/{port}{path}" + + @property + def proxy_headers(self) -> dict[str, str]: + """Auth headers to send with [`proxy_url_for`] requests (HF token + sandbox token).""" + return { + "Authorization": f"Bearer {self._server._auth_token}", + "X-Sandbox-Token": self._server._sandbox_token, + } + + def __repr__(self) -> str: + return f"Sandbox(id={self.id!r}, image={self.image!r})" + + # ------------------------------------------------------------------ internals + + def _request(self, method: str, resource: str, **kwargs) -> httpx.Response: + return self._server.request(method, self._base_path + resource, **kwargs) + + @contextmanager + def _stream(self, method: str, resource: str, **kwargs) -> Iterator[httpx.Response]: + with self._server.stream(method, self._base_path + resource, **kwargs) as response: + yield response + + +class SandboxPool: + """A fleet of shared "host" jobs, each packing many landlock-isolated sandboxes. + + One host is one billed HF Job (a VM); it runs the sandbox server and multiplexes + up to `sandboxes_per_host` lightweight sandboxes, isolated from each other by + uid + the Landlock LSM. This makes large fan-outs cheap (the VM cost is shared + across all its sandboxes) and fast (creating a sandbox is ~one proxy round-trip + once a host is warm). Best for many parallel CPU sandboxes such as RL rollouts; + for GPU or strong VM-level isolation between mutually-distrusting workloads, use + [`Sandbox.create`] instead. + + The constructor pre-provisions `warm_up` hosts (default 1) and blocks until they are + ready; further hosts are then provisioned on demand as sandboxes are requested, and all + are torn down on `close()` (or when idle, via `idle_timeout`). The user never manages jobs: + + ```python + >>> from huggingface_hub import SandboxPool + >>> with SandboxPool(image="python:3.12", flavor="cpu-basic", warm_up=2) as pool: + ... boxes = [pool.create() for _ in range(100)] # packed across the warm hosts + ... print(boxes[0].run("echo hi").stdout) + hi + ``` + + `create()` makes **one** sandbox at a time: it reuses a host that still has free + capacity before booting a new one, so you grow on demand as work arrives. To avoid + a cold start on the first few calls, pre-provision hosts with `warm_up` (or + [`warm`]). Warm hosts are discovered via job labels, so reuse works **across + processes** too (a fresh pool with the same `image`/`flavor`/`name` attaches to + hosts an earlier run left behind): + + ```python + >>> pool = SandboxPool(image="python:3.12") + >>> sbx = pool.create() # finds a warm host (here or in another process), else boots one + ``` + """ + + def __init__( + self, + image: str = DEFAULT_IMAGE, + *, + flavor: str = "cpu-basic", + sandboxes_per_host: int = DEFAULT_SANDBOXES_PER_HOST, + warm_up: int = 1, + max_hosts: int | None = None, + name: str | None = None, + idle_timeout: int | float | str | None = DEFAULT_IDLE_TIMEOUT, + namespace: str | None = None, + start_timeout: float = 120.0, + token: str | None = None, + _connect_mode: bool = False, + ) -> None: + """Configure a pool and pre-provision `warm_up` hosts (blocks until they are ready). + + Env/secrets are *not* set here: they belong to each sandbox and are passed to + `create(env=...)`, so sandboxes in the same pool can have different environments. + + Args: + image (`str`, *optional*, defaults to `"python:3.12"`): + Docker image for the hosts (needs `/bin/sh`). All sandboxes in the + pool share this image. + flavor (`str`, *optional*, defaults to `"cpu-basic"`): + Hardware flavor for the host jobs (e.g. `"cpu-basic"`). + sandboxes_per_host (`int`, *optional*, defaults to `50`): + How many sandboxes to pack per host (per VM density). + warm_up (`int`, *optional*, defaults to `1`): + How many hosts to pre-provision in the constructor (which blocks + until they are ready), so an initial burst of `create()` calls doesn't pay + a host cold start each. Existing warm hosts (from the cache / other processes) + count towards it, so only the shortfall is booted; capped by `max_hosts`. + Defaults to 1 (a single host). + max_hosts (`int`, *optional*): + Optional cap on the number of host jobs (a cost ceiling). When + reached and all hosts are full, `create()` raises. + name (`str`, *optional*): + Pool name, used as the `hf-sandbox-pool` job label so the pool is + discoverable (e.g. `hf sandbox pool ls`, `connect()`). `create()` reuses + running hosts carrying this label (including from other processes) before + booting new ones, so distinct names keep separate pools from sharing hosts. + A random name is generated when omitted. + idle_timeout (`int` or `float` or `str`, *optional*, defaults to `600`): + Host idle timeout — a host shuts down once it has had no + sandboxes for this long (a billing backstop). Each sandbox also has its + own idle timeout, set at `create()`. Pass `None` to disable. + namespace (`str`, *optional*): + User or org namespace to run hosts under. + start_timeout (`float`, *optional*, defaults to `120.0`): + Max seconds to wait for a host to become ready. + token (`str`, *optional*): + HF token override. + """ + if sandboxes_per_host < 1: + raise ValueError("sandboxes_per_host must be >= 1.") + if warm_up < 1: + raise ValueError("warm_up must be >= 1.") + self._api = HfApi(token=token) + self.image = image + self.flavor = flavor + self.sandboxes_per_host = sandboxes_per_host + self._warm_up = warm_up + self.max_hosts = max_hosts + self.name = name if name is not None else f"pool-{token_hex(6)}" + self._idle_timeout = idle_timeout + self._namespace = namespace + self._start_timeout = start_timeout + self._hosts: List[_SandboxServer] = [] + self._lock = threading.Lock() + # Held across the whole one-time warm-up so concurrent first create() calls block until + # it finishes (and see the warm hosts) instead of racing past a half-set flag. + self._warmup_lock = threading.Lock() + # Serializes on-demand host creation within the process: a burst of create() calls that + # all find every host full boots one host at a time (each new host frees + # `sandboxes_per_host` slots for the threads still queued behind the lock) instead of + # one host per thread. The CLI is a single process per command, so this is a no-op there. + self._boot_lock = threading.Lock() + self._closed = False + # Whether the one-time warm-up (seed cache + pre-provision `warm_up` hosts) ran. + self._warmed_up = False + # Set for connect()'d handles: the pool must already exist, so create() refuses to + # silently resurrect it by booting a fresh host if discovery confirms every host is gone. + # Such a handle also never boots during construction — it attaches lazily on create(). + self._require_live_host = _connect_mode + # Whether close()/`__exit__` cancels the host jobs. True for a pool we created; False + # for a connect()'d handle, which only releases its HTTP clients on exit — the shared + # hosts (possibly serving other clients) are left running, like Sandbox.connect(). + self._owns_hosts = not _connect_mode + # Job ids of cached hosts we found dead this session; pruned from the cache on save. + self._dead_host_ids: set[str] = set() + + # Pre-provision `warm_up` hosts now so the constructor returns only once the pool is + # warm. A connect()'d handle skips this: it must never boot during construction (it can + # only attach to existing hosts, lazily on the first create()). + if not _connect_mode: + self._ensure_warmed_up() + + # ------------------------------------------------------------------ public API + + @classmethod + def connect(cls, pool_id: str, *, namespace: str | None = None, token: str | None = None) -> "SandboxPool": + """Reattach to a running pool by id, from any machine — no local state needed. + + Finds a running host labelled with `pool_id` and rebuilds the pool's config + (image/flavor/density/host-idle) from that host job's spec and env vars, returning + a [`SandboxPool`] ready to `create()` more sandboxes — packing onto the running + hosts, or booting a duplicate (same config) when they are full. + + Raises [`SandboxError`] if no running host is found (a pool stops existing once + all of its hosts are gone — idle-timed-out or killed). + + Args: + pool_id (`str`): + The id returned when the pool was first created. + namespace (`str`, *optional*): + Namespace to search for the pool's hosts (defaults to yours). + token (`str`, *optional*): + HF token override. + """ + # Fast path: rebuild the pool from the local best-effort cache, with no HTTP at all. + # The cached hosts are seeded (and verified) lazily on the next create(); if they are + # all stale, create() falls back to label discovery exactly like a cold connect would. + cache = read_pool_cache(pool_id) + if cache is not None: + return cls( + image=cache.image, + flavor=cache.flavor, + sandboxes_per_host=cache.sandboxes_per_host, + max_hosts=cache.max_hosts, + name=pool_id, + idle_timeout=cache.idle_timeout, + namespace=cache.namespace if namespace is None else namespace, + token=token, + _connect_mode=True, # attach to existing hosts; never boot during construction + ) + + # Cold path: find a running host via labels and rebuild the config from its job spec. + api = HfApi(token=token) + job = _find_pool_host_job(api, pool_id, namespace=namespace) + env = _host_env(api, job, namespace=namespace) + idle_raw = env.get("SBX_IDLE_TIMEOUT") + max_hosts_raw = env.get("SBX_MAX_HOSTS") + return cls( + image=job.docker_image or job.space_id or DEFAULT_IMAGE, + flavor=str(job.flavor) if job.flavor is not None else "cpu-basic", + sandboxes_per_host=int(env.get("SBX_CAPACITY", DEFAULT_SANDBOXES_PER_HOST)), + max_hosts=int(max_hosts_raw) if max_hosts_raw is not None else None, + name=pool_id, + idle_timeout=int(idle_raw) if idle_raw is not None else None, + namespace=namespace, + token=token, + _connect_mode=True, # attach to existing hosts; never boot during construction + ) + + def warm(self, num_hosts: int = 1) -> List[str]: + """Ensure `num_hosts` empty host(s) are running and leave them running. Returns the + pool's host job ids. + + Used to "create" a pool up front: the hosts carry the pool label and config (in + their env vars), so a later `SandboxPool.connect(pool_id)` (even from another + machine) finds them and spawns sandboxes without a cold start. The hosts keep + billing until killed or idle. + + Adopts hosts already running for this pool (found via job labels) before booting, + so a `warm()` after `connect()` — or a repeated `warm()` — tops up to `num_hosts` + instead of duplicating live hosts and blowing past `max_hosts`. + """ + if self._closed: + raise SandboxError("This SandboxPool is closed.") + self._discover_hosts() + with self._lock: + shortfall = num_hosts - len(self._hosts) + if shortfall > 0: + booted = self._provision_hosts(shortfall) + with self._lock: + self._hosts.extend(booted) + with self._warmup_lock: + self._warmed_up = True # explicit warm-up satisfies create()'s one-time warm-up + self._save_cache() + return self.host_ids + + def create( + self, + *, + env: dict[str, Any] | None = None, + idle_timeout: int | float | str | None = DEFAULT_IDLE_TIMEOUT, + forward_hf_token: bool = False, + ) -> "Sandbox": + """Create one sandbox, provisioning a host if needed. + + Reuses a host with free capacity (this pool's, or a warm host found via job labels + / the local cache) before booting a new one, so a `create()` against a warm host + costs ~one round-trip. Call it repeatedly to fan out; use `warm_up` (or [`warm`]) + to pre-provision hosts and avoid a cold start on the first calls. If a host fills + up under us (another process packed it) or a cached host is gone, the sandbox is + re-placed on another host (or a fresh one). + + Args: + env (`dict[str, Any]`, *optional*): + Environment variables for this sandbox (each sandbox gets its own). + idle_timeout (`int` or `float` or `str`, *optional*, defaults to `600`): + Per-sandbox idle timeout — a sandbox is evicted from its host + after this much inactivity (no API calls, no running process). Distinct + from the host idle timeout. Pass `None` to disable. + forward_hf_token (`bool`, *optional*, defaults to `False`): + If True, inject your HF token as `HF_TOKEN` in the sandbox + (opt-in). Unlike a dedicated sandbox's `secrets`, a pooled sandbox's env is + delivered to the host server at creation (never stored in the host job), so + it doesn't appear in any job's metadata. + """ + if self._closed: + raise SandboxError("This SandboxPool is closed.") + sandbox_env = dict(env or {}) + if forward_hf_token: + sandbox_env["HF_TOKEN"] = _effective_token(self._api) + # None disables per-sandbox idle eviction (mirrors Sandbox.create): leave it out of + # the create body entirely rather than sending 0, which the server reads as "evict now". + idle_secs = _duration_to_secs(idle_timeout) if idle_timeout is not None else None + + new_hosts: List[_SandboxServer] = [] + discovered = False + rounds = 0 + try: + # One-time warm-up: seed from cache (no HTTP) and pre-provision `warm_up` hosts. + # Warm-up hosts are pool-level (like `warm()`), so they survive a failed create(). + self._ensure_warmed_up() + while True: + # 1. Reserve a slot on a host we already track. + host = self._reserve_one() + # 2. If none free, attach (once) to warm hosts discovered via labels. + if host is None and not discovered: + discovered = True + self._discover_hosts() + host = self._reserve_one() + # 3. Still none? Boot one host (unless we must not resurrect a dead pool). + if host is None: + if self._require_live_host and discovered and not self._hosts: + # connect()'d to a pool whose hosts are all gone (stale cache + nothing + # found via labels): don't resurrect it under the same id, just report it. + delete_pool_cache(self.name) + raise SandboxError( + f"No running host found for pool '{self.name}'. The pool has stopped " + "(all its hosts were killed or idle-timed-out); create a new one." + ) + host = self._boot_one_host(new_hosts) + # 4. Create the sandbox on the reserved host. `None` means the host filled up + # (another process packed it) or a stale cached host is gone — retry. + if host is not None: + try: + sandbox = self._create_one(host, sandbox_env, idle_secs) + except Exception: + # The create POST failed on a live host: release the slot we optimistically + # reserved so the host isn't left permanently one sandbox short. + with self._lock: + host.live = max(0, host.live - 1) + raise + if sandbox is not None: + self._save_cache() + return sandbox + # sandbox is None: the host is full (packed by another process) or gone. + # Either way it's no longer a host we exclusively own, so stop tracking it + # for teardown — cancelling it on a later round's failure would kill another + # tenant's sandboxes (full host) or hit an already-dead job (gone host). + if host in new_hosts: + new_hosts.remove(host) + discovered = False # re-scan for room / boot a host next round + rounds += 1 + if rounds > _MAX_PACK_ROUNDS: + raise SandboxError( + "Could not place sandbox: hosts kept reporting full. Raise max_hosts or sandboxes_per_host." + ) + except Exception: + # All-or-nothing: tear down hosts we booted in this call. + with self._lock: + self._hosts = [h for h in self._hosts if h not in new_hosts] + for host in new_hosts: + try: + host.cancel_job() + except Exception: + pass + finally: + host.close() + raise + + @property + def num_hosts(self) -> int: + """Number of host jobs currently provisioned.""" + with self._lock: + return len(self._hosts) + + @property + def num_sandboxes(self) -> int: + """Number of sandboxes currently handed out (across all hosts).""" + with self._lock: + return sum(host.live for host in self._hosts) + + @property + def host_ids(self) -> List[str]: + """Job ids of the provisioned hosts.""" + with self._lock: + return [host.job_id for host in self._hosts] + + def close(self) -> None: + """Release the pool. Idempotent. + + For a pool we created, this terminates all host jobs (and therefore all their + sandboxes). For a `connect()`'d handle it only releases the local HTTP clients: the + shared hosts may be serving other clients, so — like [`Sandbox.connect`] — leaving a + `with` block must not tear them down. Terminate a connected pool's hosts explicitly + with `hf sandbox pool delete `. + """ + with self._lock: + hosts = self._hosts + self._hosts = [] + self._closed = True + for host in hosts: + try: + if self._owns_hosts: + host.cancel_job() + except Exception as e: + logger.warning(f"Failed to cancel sandbox host {host.job_id}: {e}") + finally: + host.close() + if self._owns_hosts: + delete_pool_cache(self.name) # the pool's hosts are gone; don't leave a stale entry + + def __enter__(self) -> "SandboxPool": + return self + + def __exit__(self, *exc_info) -> None: + self.close() + + # ------------------------------------------------------------------ internals + + def _ensure_warmed_up(self) -> None: + """One-time: seed from the cache and pre-provision up to `warm_up` hosts. + + Cheap by default: seeding is local (no HTTP), and if the cache already gives at + least `warm_up` hosts we trust them and skip discovery/booting (dead ones are + pruned lazily by `create()`). Only when short do we list_jobs and boot the + shortfall in parallel, capped by `max_hosts`. + + Runs at most once per pool, under `_warmup_lock`: concurrent first `create()` calls + block here until the warm-up completes — and see its hosts — rather than racing past + a half-set flag and each booting their own. `_warmed_up` is only set once the work + succeeds, so a failed warm-up is retried (seeding/discovery dedupe by job id). + + Warm-up hosts are pool-level (like [`warm`]): they are not torn down if the + triggering `create()` later fails — `close()` (or the `with` block) reclaims them. + """ + with self._warmup_lock: + if self._warmed_up: + return + self._seed_hosts_from_cache() + target = self._warm_up if self.max_hosts is None else min(self._warm_up, self.max_hosts) + with self._lock: + shortfall = target - len(self._hosts) + if shortfall > 0: + self._discover_hosts() + # A connect()'d pool must never boot during warm-up: it may only attach to + # existing hosts. Booting is left to create()'s loop, guarded by + # `_require_live_host`, so a stopped pool is reported instead of resurrected. + if not self._require_live_host: + with self._lock: + shortfall = target - len(self._hosts) + if shortfall > 0: + booted = self._provision_hosts(shortfall) + with self._lock: + self._hosts.extend(booted) + self._warmed_up = True + # Persist the warmed hosts so the next process (e.g. `hf sandbox create --pool` after a + # bare `hf sandbox pool create`) hits the cache fast path instead of re-discovering. + if self._hosts: + self._save_cache() + + def _reserve_one(self) -> "_SandboxServer | None": + """Reserve one slot on the first host with free capacity (under lock), else None.""" + with self._lock: + for host in self._hosts: + if host.capacity - host.live > 0: + host.live += 1 + return host + return None + + def _boot_one_host(self, new_hosts: List["_SandboxServer"]) -> "_SandboxServer | None": + """Provision (or adopt) one host and reserve a slot on it. Returns None to retry. + + Held under `_boot_lock` so that, within a process, only one host is booted at a time: + a burst of create() calls that all found every host full queue here, and each new host + frees `sandboxes_per_host` slots for the threads still waiting — so they reuse it instead + of each booting their own. Before booting, we reuse a slot freed by a concurrent boot and, + failing that, adopt a host already SCHEDULING for this pool (here or in another process) + rather than piling on a duplicate. + """ + with self._boot_lock: + # A concurrent boot (this process) may have added a host with free slots while we + # waited for the lock — reuse it instead of booting another. + host = self._reserve_one() + if host is not None: + return host + # A host already coming up for this pool (ours, or another process'): wait for it + # to start and adopt it instead of booting a duplicate. + if self._adopt_pending_host(): + host = self._reserve_one() + if host is not None: + return host + booted = self._provision_hosts(1) + new_hosts.extend(booted) + with self._lock: + self._hosts.extend(booted) + return self._reserve_one() + + def _adopt_pending_host(self) -> bool: + """Wait for and adopt a host already SCHEDULING for this pool, if any. + + Avoids over-provisioning when a host is already on its way up for this pool (started by + another process, or an earlier create() in this one): rather than booting a duplicate, + wait for it to reach RUNNING and adopt it via discovery. Returns True if a pending host + was found (whether or not it eventually came up), False if none is scheduling. + """ + known = {host.job_id for host in self._hosts} + pending = next( + ( + job + for job in self._api.list_jobs( + status="SCHEDULING", + labels={MODE_LABEL: MODE_POOL, POOL_LABEL: self.name}, + namespace=self._namespace, + ) + if job.id not in known + ), + None, + ) + if pending is None: + return False + logger.debug(f"Pool '{self.name}' host {pending.id} is already SCHEDULING; waiting for it instead of booting.") + deadline = time.time() + self._start_timeout + while time.time() < deadline: + stage = self._api.inspect_job(job_id=pending.id, namespace=self._namespace).status.stage + if stage not in ("SCHEDULING", "RUNNING"): + return False # it died before starting up; let the caller boot a fresh one + if stage == "RUNNING": + # RUNNING precedes "server ready" (the host still has to fetch + exec sbx-server), + # so discovery only adopts it once its server answers — keep polling until it does. + self._discover_hosts() + if any(host.job_id == pending.id for host in self._hosts): + return True + time.sleep(1.0) + return False + + def _discover_hosts(self) -> None: + """Attach to running host jobs that match this pool (image/flavor/name). + + Lets `create()` reuse a host warmed by an earlier call or another process + instead of booting a new one. Hosts are found via job labels; each adopted + host's free capacity is read from the server, so packing stays accurate. + """ + known = {host.job_id for host in self._hosts} + matches = [ + job + for job in self._api.list_jobs( + status="RUNNING", + labels={MODE_LABEL: MODE_POOL, POOL_LABEL: self.name}, + namespace=self._namespace, + ) + if job.id not in known + ] + + for job in matches: + server = None + try: + server = _connect_host(self._api, job.id, namespace=self._namespace) + # Capacity comes from the host's SBX_CAPACITY env; otherwise an adopted host + # would silently pack at the pool default and mis-pack. + env = _host_env(self._api, job, namespace=self._namespace) + server.capacity = int(env.get("SBX_CAPACITY", self.sandboxes_per_host)) + server.live = len(server.request("GET", "/v1/sandboxes").json()) + except (SandboxError, httpx.HTTPError, HfHubHTTPError) as e: + # Host died (e.g. deleted between list_jobs and inspect_job), is still starting + # up, or is unreachable: skip it, closing the client if one was opened. + logger.debug(f"Skipping host {job.id} during discovery: {e}") + if server is not None: + server.close() + continue + with self._lock: + if any(host.job_id == job.id for host in self._hosts): + server.close() # adopted concurrently by another thread + else: + self._hosts.append(server) + + def _provision_hosts(self, num_new: int) -> List[_SandboxServer]: + """Boot `num_new` host jobs in parallel, respecting `max_hosts`.""" + with self._lock: + current = len(self._hosts) + if self.max_hosts is not None and current + num_new > self.max_hosts: + allowed = self.max_hosts - current + raise SandboxError( + f"Pool needs {num_new} more host(s) but max_hosts={self.max_hosts} " + f"allows only {max(0, allowed)} more. Raise max_hosts or kill some sandboxes." + ) + # Boot in parallel, but collect every result before re-raising: if one boot fails, + # the others may have already started billable host jobs, so cancel those instead of + # leaking them (executor.map would surface the first error and drop the rest). + with ThreadPoolExecutor(max_workers=min(num_new, 32)) as executor: + futures = [executor.submit(self._boot_host) for _ in range(num_new)] + booted: List[_SandboxServer] = [] + error: Exception | None = None + for future in futures: + try: + booted.append(future.result()) + except Exception as e: + error = e + if error is not None: + for server in booted: + try: + server.cancel_job() + except Exception: + pass + finally: + server.close() + raise error + return booted + + def _boot_host(self) -> _SandboxServer: + """Start one host job and wait until its server is ready.""" + hf_token = _effective_token(self._api) + nonce = token_hex(16) + sandbox_token = _derive_sandbox_token(hf_token, nonce) + command, job_env, job_secrets, job_volumes = _bootstrap_job_spec( + self._api, + hf_token, + env=None, # host job carries no user env; sandboxes get env per create request + secrets=None, + volumes=None, + idle_timeout=self._idle_timeout, + forward_hf_token=False, + sandbox_token=sandbox_token, + ) + # Host mode: per-sandbox idle eviction + empty-host shutdown (vs the dedicated + # whole-job watchdog). + job_env["SBX_HOST_MODE"] = "1" + # Server-authoritative packing cap: the host refuses creates past this (the client + # then packs onto / boots another host). Avoids cross-process over-commit. Read + # back (with SBX_IDLE_TIMEOUT) by `connect()` to rebuild the pool — labels stay for + # filtering only. + job_env["SBX_CAPACITY"] = str(self.sandboxes_per_host) + # Persist the optional cost ceiling on the host too, so a later connect() (e.g. the CLI + # `pool create --max-hosts N` then `create --pool ` flow) rebuilds the cap instead of + # defaulting to unlimited and provisioning past it. + if self.max_hosts is not None: + job_env["SBX_MAX_HOSTS"] = str(self.max_hosts) + labels = {SANDBOX_LABEL: "1", MODE_LABEL: MODE_POOL, POOL_LABEL: self.name, NONCE_LABEL: nonce} + job = self._api.run_job( + image=self.image, + command=command, + env=job_env, + secrets=job_secrets, + flavor=self.flavor, + timeout=SANDBOX_MAX_LIFETIME, + labels=labels, + volumes=job_volumes or None, + expose=[SANDBOX_SERVER_PORT], + namespace=self._namespace, + ) + server: "_SandboxServer | None" = None + try: + # from_job is inside the try: if it raises (e.g. the server port isn't exposed), + # the already-started host job must still be cancelled so it doesn't linger billing. + server = _SandboxServer.from_job( + job=job, + nonce=nonce, + sandbox_token=sandbox_token, + api=self._api, + max_connections=min(self.sandboxes_per_host + 8, 256), + capacity=self.sandboxes_per_host, + ) + server.wait_ready(self._start_timeout) + except Exception: + try: + self._api.cancel_job(job_id=job.id, namespace=job.owner.name) + except Exception as e: + logger.warning(f"Failed to cancel sandbox host {job.id} after startup failure: {e}") + if server is not None: + server.close() + raise + return server + + def _create_one(self, host: "_SandboxServer", env: dict[str, Any], idle_secs: int | None) -> "Sandbox | None": + """Create one sandbox on a reserved `host`. Returns the sandbox, or None to retry. + + None means either the host filled up between our reservation and the create + (server-authoritative capacity — another client packed it, so we mark it full and + place the sandbox elsewhere) or a host rebuilt from the cache is gone/unreachable + (dropped and re-placed via discovery / a fresh boot). + """ + body: dict[str, Any] = {"count": 1} + if idle_secs is not None: + body["idle_timeout_secs"] = idle_secs + if env: + body["env"] = env + try: + data = host.request("POST", "/v1/sandboxes", json=body).json() + except (SandboxError, httpx.HTTPError) as e: + if host.verified: + raise # a host we booted/discovered this session failing is a real error + logger.debug(f"Dropping unreachable cached host {host.job_id}: {e}") + self._drop_host(host) + return None + host.verified = True # it answered, so it's a real live host now + sandboxes = data.get("sandboxes") or [] + if int(data.get("rejected", 0)) or not sandboxes: + with self._lock: + host.live = host.capacity # the host is full; stop reserving it + return None + item = sandboxes[0] + sandbox = Sandbox( + id=f"{host.job_id}{SHARED_ID_SEP}{item['id']}", + server=host, + local_id=item["id"], + owns_sandbox=True, + owns_server=False, + ) + sandbox._on_kill = self._on_sandbox_killed + return sandbox + + def _on_sandbox_killed(self, sandbox: Sandbox) -> None: + """Free the packing slot of a shared sandbox that was killed.""" + with self._lock: + sandbox._server.live = max(0, sandbox._server.live - 1) + + def _drop_host(self, host: _SandboxServer) -> None: + """Forget a host found dead this session (and mark it for cache pruning).""" + with self._lock: + self._hosts = [h for h in self._hosts if h is not host] + self._dead_host_ids.add(host.job_id) + host.close() + + # ------------------------------------------------------------------ cache (best effort) + + def _seed_hosts_from_cache(self) -> None: + """Adopt the pool's cached hosts without any HTTP (rebuilt from cached URL + nonce). + + Best-effort and unverified: each adopted host is confirmed on the first successful + request (and dropped on the first failure, see `_create_one`). Hosts the cache + believes are full are skipped — label discovery re-checks them with fresh counts if + the seeded ones don't satisfy the request, so a stale-full entry never blocks a create. + """ + cache = read_pool_cache(self.name) + if cache is None: + return + hf_token = _effective_token(self._api) + with self._lock: + known = {host.job_id for host in self._hosts} + for ch in cache.hosts: + if ch.job_id in known or ch.live >= ch.capacity: + continue + server = _SandboxServer( + job_id=ch.job_id, + owner=ch.owner, + image=self.image, + base_url=ch.base_url, + nonce=ch.nonce, + sandbox_token=_derive_sandbox_token(hf_token, ch.nonce), + api=self._api, + max_connections=min(self.sandboxes_per_host + 8, 256), + capacity=ch.capacity, + ) + server.live = ch.live + server.verified = False + self._hosts.append(server) + + def _save_cache(self) -> None: + """Persist the pool config + current hosts (with their live counts) for next time.""" + with self._lock: + hosts = [ + CachedHost( + job_id=host.job_id, + owner=host.owner, + base_url=host.base_url, + nonce=host.nonce, + capacity=host.capacity, + live=host.live, + updated_at=time.time(), + ) + for host in self._hosts + ] + dead = set(self._dead_host_ids) + save_pool_cache( + self.name, + image=self.image, + flavor=self.flavor, + sandboxes_per_host=self.sandboxes_per_host, + max_hosts=self.max_hosts, + idle_timeout=_duration_to_secs(self._idle_timeout) if self._idle_timeout is not None else None, + namespace=self._namespace, + hosts=hosts, + dead_host_ids=dead, + ) + + +def _effective_token(api: HfApi) -> str: + token = api.token if isinstance(api.token, str) else get_token() + if not token: + raise SandboxError("A Hugging Face token is required to use sandboxes. Run `hf auth login` first.") + return token + + +def _bootstrap_job_spec( + api: HfApi, + hf_token: str, + *, + env: dict[str, Any] | None, + secrets: dict[str, Any] | None, + volumes: List[Volume] | None, + idle_timeout: int | float | str | None, + forward_hf_token: bool, + sandbox_token: str, +) -> tuple[list[str], dict[str, Any], dict[str, Any], List[Volume]]: + """Build the (command, env, secrets, volumes) to launch a job running sbx-server. + + Shared by dedicated sandboxes and shared hosts: both fetch and exec the same unified + `sbx-server` binary at startup (via `/bin/sh`), downloading it with wget/curl, or + reading it off the always-mounted server bucket when the image ships neither. + """ + # Reserved SBX_* keys go last so user-provided env/secrets can't override them + # (e.g. clobbering SBX_PORT would break the proxy, SBX_TOKEN would break auth). + job_env: dict[str, Any] = {**(env or {}), "SBX_PORT": str(SANDBOX_SERVER_PORT)} + job_secrets: dict[str, Any] = {**(secrets or {}), "SBX_TOKEN": sandbox_token} + job_volumes = list(volumes or []) + if idle_timeout is not None: + job_env["SBX_IDLE_TIMEOUT"] = str(_duration_to_secs(idle_timeout)) + if forward_hf_token: + job_secrets["HF_TOKEN"] = hf_token + + job_env["SBX_SERVER_URL"] = f"{api.endpoint}/buckets/{constants.SANDBOX_SERVER_BUCKET}/resolve/sbx-server" + # Always mount the server bucket as a transparent fallback for images without wget/curl. + # It's only read (paying the ~2-3s FUSE cost) when the bootstrap script can't download. + job_env["SBX_SERVER_MOUNT"] = _SERVER_MOUNT_PATH + job_volumes.append( + Volume(type="bucket", source=constants.SANDBOX_SERVER_BUCKET, mount_path=_SERVER_MOUNT_PATH, read_only=True) + ) + command = ["/bin/sh", "-c", _BOOTSTRAP_DOWNLOAD] + return command, job_env, job_secrets, job_volumes + + +def _host_env(api: HfApi, job: JobInfo, *, namespace: str | None) -> dict[str, Any]: + """Return a host job's env vars (where pool config lives)""" + env = job.environment if isinstance(job.environment, dict) else {} + if "SBX_CAPACITY" not in env: + env = api.inspect_job(job_id=job.id, namespace=namespace).environment or {} + env = env if isinstance(env, dict) else {} + return env + + +def _find_pool_host_job(api: HfApi, pool_id: str, *, namespace: str | None = None) -> JobInfo: + """Return any running host job belonging to `pool_id` (found via the pool label).""" + for job in api.list_jobs( + status="RUNNING", labels={MODE_LABEL: MODE_POOL, POOL_LABEL: pool_id}, namespace=namespace + ): + return job + raise SandboxError( + f"No running host found for pool '{pool_id}'. The pool has stopped " + "(all its hosts were killed or idle-timed-out); create a new one." + ) + + +def _connect_host(api: HfApi, host_job_id: str, *, namespace: str | None = None) -> _SandboxServer: + """Reattach to a running host job and return its server transport.""" + job = api.inspect_job(job_id=host_job_id, namespace=namespace) + labels = job.labels or {} + nonce = labels.get(NONCE_LABEL) + if nonce is None or labels.get(MODE_LABEL) != MODE_POOL: + raise SandboxError(f"Job {host_job_id} is not a sandbox host.") + if job.status.stage != "RUNNING": + raise SandboxError(f"Sandbox host {host_job_id} is not running (status: {job.status.stage}).") + return _SandboxServer.from_job( + job=job, + nonce=nonce, + sandbox_token=_derive_sandbox_token(_effective_token(api), nonce), + api=api, + max_connections=SandboxFiles.PARALLEL_MAX_WORKERS + 2, + ) + + +def _split_sandbox_id(sandbox_id: str, namespace: str | None) -> tuple[str, str | None]: + """Accept `namespace/sandbox_id` ids (as shown in the Hub UI), like `hf jobs` does.""" + if "/" not in sandbox_id: + return sandbox_id, namespace + extracted_namespace, parsed_id = sandbox_id.split("/", 1) + if not extracted_namespace or not parsed_id or "/" in parsed_id: + raise SandboxError(f"Sandbox id must be 'sandbox_id' or 'namespace/sandbox_id', got {sandbox_id!r}.") + if namespace is not None and namespace != extracted_namespace: + raise SandboxError( + f"Conflicting namespace: got namespace={namespace!r} but sandbox id implies namespace={extracted_namespace!r}." + ) + return parsed_id, extracted_namespace + + +def _find_server_url(job: JobInfo) -> str: + for url in job.status.expose_urls or []: + if f"--{SANDBOX_SERVER_PORT}." in url: + return url + raise SandboxError(f"Job {job.id} does not expose the sandbox server port {SANDBOX_SERVER_PORT}.") + + +def _tail_job_logs(api: HfApi, job_id: str, *, namespace: str | None = None, limit: int = 20) -> str: + try: + lines = list(api.fetch_job_logs(job_id=job_id, namespace=namespace))[-limit:] + except Exception: + return "" + return " Last logs:\n" + "\n".join(f" {line}" for line in lines) if lines else "" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_sandbox_cache.py b/venv/lib/python3.12/site-packages/huggingface_hub/_sandbox_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..25ddca75be16347006d4946f80cbda12311820f6 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_sandbox_cache.py @@ -0,0 +1,159 @@ +# coding=utf-8 +# Copyright 2026-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Best-effort local cache for SandboxPool hosts (host/pool mode).""" + +import json +import os +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import List + +from . import constants +from .utils import WeakFileLock, logging + + +logger = logging.get_logger(__name__) + +# Bump if the on-disk layout changes incompatibly; older/newer files are ignored on read. +_CACHE_VERSION = 1 + +# A write should never block a sandbox creation for long: the cache is best-effort, so we +# rather skip persisting than wait on a stuck lock. +_LOCK_TIMEOUT = 5.0 + + +@dataclass +class CachedHost: + """A single host Job of a pool, as last seen by some process. + + `base_url` + `nonce` are everything needed to rebuild the in-job server transport + (`_SandboxServer`) without an `inspect_job` round-trip: the per-sandbox auth token is + re-derived from the user's HF token and `nonce` (see `_derive_sandbox_token`). + """ + + job_id: str + owner: str # namespace the host job runs under (for cancel/inspect) + base_url: str # exposed sbx-server URL (does not change while the job lives) + nonce: str # public nonce from the job label; derives the sandbox auth token + capacity: int # SBX_CAPACITY: max sandboxes the host packs + live: int # sandboxes last observed on the host (best-effort, may be stale) + updated_at: float = 0.0 + + +@dataclass +class PoolCache: + """Cached view of one pool: its config (to boot new hosts) + its known hosts.""" + + pool_id: str + image: str + flavor: str + sandboxes_per_host: int + max_hosts: int | None + idle_timeout: int | None + namespace: str | None + hosts: List[CachedHost] = field(default_factory=list) + version: int = _CACHE_VERSION + updated_at: float = 0.0 + + +def _pools_dir() -> Path: + return Path(constants.HF_HOME) / "sandbox" / "pools" + + +def pool_cache_path(pool_id: str) -> Path: + """Path of the cache file for `pool_id` (no I/O).""" + if any(c in pool_id for c in ("/", "\\", "\x00")) or pool_id in (".", ".."): + raise ValueError(f"Invalid pool id: {pool_id!r}") + return _pools_dir() / f"{pool_id}.json" + + +def read_pool_cache(pool_id: str) -> PoolCache | None: + """Return the cached view of `pool_id`, or `None` if missing/corrupt/incompatible.""" + try: + path = pool_cache_path(pool_id) + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + if data.get("version") != _CACHE_VERSION: + return None + hosts = [CachedHost(**h) for h in data.pop("hosts", [])] + return PoolCache(**data, hosts=hosts) + except FileNotFoundError: + return None + except Exception as e: + logger.debug(f"Ignoring unreadable sandbox pool cache for {pool_id!r}: {e}") + return None + + +def save_pool_cache( + pool_id: str, + *, + image: str, + flavor: str, + sandboxes_per_host: int, + max_hosts: int | None, + idle_timeout: int | None, + namespace: str | None, + hosts: List[CachedHost], + dead_host_ids: set[str] | None = None, +) -> None: + """Merge `hosts` into the cache for `pool_id` (best-effort, never raises). + + Concurrency-safe: under a file lock, the on-disk hosts are read, then `hosts` are + upserted by `job_id` and `dead_host_ids` removed, so a process only adds/updates what + it learned and never drops hosts another process discovered. The result is written + atomically. The pool config is refreshed from the arguments. + """ + dead = dead_host_ids or set() + try: + path = pool_cache_path(pool_id) + path.parent.mkdir(parents=True, exist_ok=True) + with WeakFileLock(str(path) + ".lock", timeout=_LOCK_TIMEOUT): + existing = read_pool_cache(pool_id) + merged = {h.job_id: h for h in (existing.hosts if existing else [])} + for host in hosts: + merged[host.job_id] = host + for job_id in dead: + merged.pop(job_id, None) + cache = PoolCache( + pool_id=pool_id, + image=image, + flavor=flavor, + sandboxes_per_host=sandboxes_per_host, + max_hosts=max_hosts, + idle_timeout=idle_timeout, + namespace=namespace, + hosts=list(merged.values()), + updated_at=time.time(), + ) + _atomic_write(path, cache) + except Exception as e: + logger.debug(f"Could not write sandbox pool cache for {pool_id!r}: {e}") + + +def delete_pool_cache(pool_id: str) -> None: + """Remove the cache file for `pool_id` (best-effort, never raises).""" + try: + pool_cache_path(pool_id).unlink(missing_ok=True) + except Exception as e: + logger.debug(f"Could not delete sandbox pool cache for {pool_id}: {e}") + + +def _atomic_write(path: Path, cache: PoolCache) -> None: + """Write the cache via a temp file + `os.replace` so readers never see a partial file.""" + tmp = path.parent / f"{path.name}.{os.getpid()}.tmp" + with tmp.open("w", encoding="utf-8") as f: + json.dump(asdict(cache), f, indent=2) + os.replace(tmp, path) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_snapshot_download.py b/venv/lib/python3.12/site-packages/huggingface_hub/_snapshot_download.py new file mode 100644 index 0000000000000000000000000000000000000000..9a6fdbfe9e2aa0308d1f2823b4b00f0c46a5eb21 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_snapshot_download.py @@ -0,0 +1,682 @@ +import os +from pathlib import Path +from typing import Literal, overload + +import httpx +from tqdm.auto import tqdm as base_tqdm + +from . import constants +from ._tree_cache import TreeCacheEntry, read_tree_cache, tree_cache_folder_for_local_dir, write_tree_cache +from .errors import ( + CachedRepoTreeNotFoundError, + DryRunError, + GatedRepoError, + HfHubHTTPError, + IncompleteSnapshotError, + LocalEntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, +) +from .file_download import REGEX_COMMIT_HASH, DryRunFileInfo, hf_hub_download, repo_folder_name +from .hf_api import DatasetInfo, HfApi, KernelInfo, ModelInfo, RepoFile, SpaceInfo +from .utils import OfflineModeIsEnabled, filter_repo_objects, logging, validate_hf_hub_args +from .utils._xet_progress_reporting import ( + XET_BYTES_BAR_FORMAT, + XET_TRANSFER_BAR_FORMAT, + _finish_transfer_bar, + _set_aggregate_rate_postfix, + _update_transfer_bar, +) +from .utils.tqdm import _create_progress_bar, hf_thread_map +from .utils.tqdm import tqdm as hf_tqdm + + +logger = logging.get_logger(__name__) + + +@overload +def snapshot_download( + repo_id: str, + *, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + library_name: str | None = None, + library_version: str | None = None, + user_agent: dict | str | None = None, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + force_download: bool = False, + token: bool | str | None = None, + local_files_only: bool = False, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + max_workers: int = 8, + tqdm_class: type[base_tqdm] | None = None, + headers: dict[str, str] | None = None, + endpoint: str | None = None, + dry_run: Literal[False] = False, +) -> str: ... + + +@overload +def snapshot_download( + repo_id: str, + *, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + library_name: str | None = None, + library_version: str | None = None, + user_agent: dict | str | None = None, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + force_download: bool = False, + token: bool | str | None = None, + local_files_only: bool = False, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + max_workers: int = 8, + tqdm_class: type[base_tqdm] | None = None, + headers: dict[str, str] | None = None, + endpoint: str | None = None, + dry_run: Literal[True] = True, +) -> list[DryRunFileInfo]: ... + + +@overload +def snapshot_download( + repo_id: str, + *, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + library_name: str | None = None, + library_version: str | None = None, + user_agent: dict | str | None = None, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + force_download: bool = False, + token: bool | str | None = None, + local_files_only: bool = False, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + max_workers: int = 8, + tqdm_class: type[base_tqdm] | None = None, + headers: dict[str, str] | None = None, + endpoint: str | None = None, + dry_run: bool = False, +) -> str | list[DryRunFileInfo]: ... + + +@validate_hf_hub_args +def snapshot_download( + repo_id: str, + *, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + library_name: str | None = None, + library_version: str | None = None, + user_agent: dict | str | None = None, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + force_download: bool = False, + token: bool | str | None = None, + local_files_only: bool = False, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + max_workers: int = 8, + tqdm_class: type[base_tqdm] | None = None, + headers: dict[str, str] | None = None, + endpoint: str | None = None, + dry_run: bool = False, +) -> str | list[DryRunFileInfo]: + """Download repo files. + + Download a whole snapshot of a repo's files at the specified revision. This is useful when you want all files from + a repo because you don't know which ones you will need _a priori_. All files are nested in a folder to keep their + path and filename relative to that folder. You can also filter which files to download by using `allow_patterns` + and `ignore_patterns`. + + If `local_dir` is provided, the file structure from the repo will be replicated in this location. When using this + option, the `cache_dir` will not be used, and a `.cache/huggingface/` folder will be created at the root of `local_dir` + to store some metadata related to the downloaded files. While this mechanism is not as robust as the main + cache system, it's optimized for regularly pulling the latest version of a repository. + + An alternative would be to clone the repo, but this requires git and git-lfs to be installed and properly + configured. It is also not possible to filter which files to download when cloning a repository using git. + + Args: + repo_id (`str`): + A user or an organization name and a repo name separated by a `/`. + repo_type (`str`, *optional*): + Set to `"dataset"`, `"space"` or `"kernel"` if downloading from a dataset, space or kernel repo, + `None` or `"model"` if downloading from a model. Default is `None`. + revision (`str`, *optional*): + An optional Git revision id, which can be a branch name, a tag, or a + commit hash. + cache_dir (`str`, `Path`, *optional*): + Path to the folder where cached files are stored. + local_dir (`str` or `Path`, *optional*): + If provided, the downloaded files will be placed under this directory. + library_name (`str`, *optional*): + The name of the library to which the object corresponds. + library_version (`str`, *optional*): + The version of the library. + user_agent (`str`, `dict`, *optional*): + The user-agent info in the form of a dictionary or a string. + etag_timeout (`float`, *optional*, defaults to `10`): + When fetching ETag, how many seconds to wait for the server to send + data before giving up, which is passed to `httpx.request`. + force_download (`bool`, *optional*, defaults to `False`): + Whether the file should be downloaded even if it already exists in the local cache. + token (`str`, `bool`, *optional*): + A token to be used for the download. + - If `True`, the token is read from the HuggingFace config + folder. + - If a string, it's used as the authentication token. + headers (`dict`, *optional*): + Additional headers to include in the request. Those headers take precedence over the others. + endpoint (`str`, *optional*): + The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, do not download any files even if they are not in `cache_dir` or `local_dir`. + allow_patterns (`list[str]` or `str`, *optional*): + If provided, only files matching at least one pattern are downloaded. + ignore_patterns (`list[str]` or `str`, *optional*): + If provided, files matching any of the patterns are not downloaded. + max_workers (`int`, *optional*): + Number of concurrent threads to download files (1 thread = 1 file download). + Defaults to 8. + tqdm_class (`tqdm`, *optional*): + If provided, overwrites the default behavior for the progress bar. Passed + argument must inherit from `tqdm.auto.tqdm` or at least mimic its behavior. + Note that the `tqdm_class` is not passed to each individual download. + Defaults to the custom HF progress bar that can be disabled by setting + `HF_HUB_DISABLE_PROGRESS_BARS` environment variable. + dry_run (`bool`, *optional*, defaults to `False`): + If `True`, perform a dry run without actually downloading the files. Returns a list of + [`DryRunFileInfo`] objects containing information about what would be downloaded. + + Returns: + `str` or list of [`DryRunFileInfo`]: + - If `dry_run=False`: Local snapshot path. + - If `dry_run=True`: A list of [`DryRunFileInfo`] objects containing download information. + + Raises: + [`~utils.RepositoryNotFoundError`] + If the repository to download from cannot be found. This may be because it doesn't exist + or because it is set to `private` and you do not have access. + [`~utils.RevisionNotFoundError`] + If the revision to download from cannot be found. + [`~errors.IncompleteSnapshotError`] + If the Hub cannot be reached (offline, connection issue, or `local_files_only=True`) and the + cached snapshot is missing some of the requested files. + [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) + If `token=True` and the token cannot be found. + [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if + ETag cannot be determined. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If some parameter value is invalid. + """ + if cache_dir is None: + cache_dir = constants.HF_HUB_CACHE + cache_dir = str(Path(cache_dir).expanduser().resolve()) + if local_dir is not None: + local_dir = str(Path(local_dir).expanduser().resolve()) + if revision is None: + revision = constants.DEFAULT_REVISION + + if repo_type is None: + repo_type = "model" + if repo_type not in constants.REPO_TYPES_WITH_KERNEL: + raise ValueError( + f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES_WITH_KERNEL)}" + ) + + storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type)) + + # Folder under which the per-commit tree listing (`trees/.json`) is cached on disk. + tree_cache_folder = tree_cache_folder_for_local_dir(local_dir) if local_dir is not None else storage_folder + + api = HfApi( + library_name=library_name, + library_version=library_version, + user_agent=user_agent, + endpoint=endpoint, + headers=headers, + token=token, + ) + + repo_info: ModelInfo | DatasetInfo | SpaceInfo | KernelInfo | None = None + api_call_error: Exception | None = None + if not local_files_only: + # try/except logic to handle different errors => taken from `hf_hub_download` + try: + # if we have internet connection we want to list files to download + repo_info = api.repo_info(repo_id=repo_id, repo_type=repo_type, revision=revision) + except httpx.ProxyError: + # Actually raise on proxy error + raise + except (httpx.ConnectError, httpx.TimeoutException, OfflineModeIsEnabled) as error: + # Internet connection is down + # => will try to use local files only + api_call_error = error + pass + except RevisionNotFoundError: + # The repo was found but the revision doesn't exist on the Hub (never existed or got deleted) + raise + except HfHubHTTPError as error: + # Multiple reasons for an http error: + # - Repository is private and invalid/missing token sent + # - Repository is gated and invalid/missing token sent + # - Hub is down (error 500 or 504) + # => let's switch to 'local_files_only=True' to check if the files are already cached. + # (if it's not the case, the error will be re-raised) + api_call_error = error + pass + + # At this stage, if `repo_info` is None it means either: + # - internet connection is down + # - internet connection is deactivated (local_files_only=True or HF_HUB_OFFLINE=True) + # - repo is private/gated and invalid/missing token sent + # - Hub is down + # => let's look if we can find the appropriate folder in the cache: + # - if the specified revision is a commit hash, look inside "snapshots". + # - f the specified revision is a branch or tag, look inside "refs". + # => if local_dir is not None, we will return the path to the local folder if it exists. + if repo_info is None: + if dry_run: + raise DryRunError( + "Dry run cannot be performed as the repository cannot be accessed. Please check your internet connection or authentication token." + ) from api_call_error + + # Try to get which commit hash corresponds to the specified revision + commit_hash = None + if REGEX_COMMIT_HASH.match(revision): + commit_hash = revision + else: + ref_path = os.path.join(storage_folder, "refs", revision) + if os.path.exists(ref_path): + # retrieve commit_hash from refs file + with open(ref_path) as f: + commit_hash = f.read() + + # Try to locate snapshot folder for this commit hash + if commit_hash is not None and local_dir is None: + snapshot_folder = os.path.join(storage_folder, "snapshots", commit_hash) + if os.path.exists(snapshot_folder): + # The folder exists, but may be partial (e.g. after an interrupted download): only return it + # if the cached tree listing confirms it is complete. + _raise_if_incomplete_snapshot( + tree_cache_folder=tree_cache_folder, + commit_hash=commit_hash, + base_dir=snapshot_folder, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + repo_id=repo_id, + revision=revision, + api_call_error=api_call_error, + ) + return snapshot_folder + + # If local_dir is not None, return it if it exists and is complete + if local_dir is not None: + local_dir = Path(local_dir) + if local_dir.is_dir() and any(local_dir.iterdir()): + if commit_hash is not None: + _raise_if_incomplete_snapshot( + tree_cache_folder=tree_cache_folder, + commit_hash=commit_hash, + base_dir=str(local_dir), + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + repo_id=repo_id, + revision=revision, + api_call_error=api_call_error, + ) + logger.warning( + f"Returning existing local_dir `{local_dir}` as remote repo cannot be accessed in `snapshot_download` ({api_call_error})." + ) + return str(local_dir.resolve()) + # If we couldn't find the appropriate folder on disk, raise an error. + if local_files_only: + raise LocalEntryNotFoundError( + "Cannot find an appropriate cached snapshot folder for the specified revision on the local disk and " + "outgoing traffic has been disabled. To enable repo look-ups and downloads online, pass " + "'local_files_only=False' as input." + ) + elif isinstance(api_call_error, OfflineModeIsEnabled): + raise LocalEntryNotFoundError( + "Cannot find an appropriate cached snapshot folder for the specified revision on the local disk and " + "outgoing traffic has been disabled. To enable repo look-ups and downloads online, set " + "'HF_HUB_OFFLINE=0' as environment variable." + ) from api_call_error + elif isinstance(api_call_error, (RepositoryNotFoundError, GatedRepoError)) or ( + isinstance(api_call_error, HfHubHTTPError) and api_call_error.response.status_code == 401 + ): + # Repo not found, gated, or specific authentication error => let's raise the actual error + raise api_call_error + else: + # Otherwise: most likely a connection issue or Hub downtime => let's warn the user + raise LocalEntryNotFoundError( + f"Got: {api_call_error.__class__.__name__}: {api_call_error}" + "\nAn error happened while trying to locate the files on the Hub, and we cannot find the appropriate" + " snapshot folder for the specified revision on the local disk. Please check your internet connection" + " and try again." + ) from api_call_error + + # At this stage, internet connection is up and running + # => let's download the files! + assert repo_info.sha is not None, "Repo info returned from server must have a revision sha." + commit_hash = repo_info.sha + + # Retrieve /tree listing from cache or fetch it + tree_entries = read_tree_cache(tree_cache_folder, commit_hash) + if tree_entries is None: + tree_entries = { + f.path: TreeCacheEntry( + size=f.size, + blob_id=f.blob_id, + lfs_sha256=f.lfs.sha256 if f.lfs is not None else None, + lfs_size=f.lfs.size if f.lfs is not None else None, + xet_hash=f.xet_hash, + ) + for f in api.list_repo_tree(repo_id=repo_id, recursive=True, revision=commit_hash, repo_type=repo_type) + if isinstance(f, RepoFile) + } + if not dry_run: + write_tree_cache(tree_cache_folder, commit_hash, tree_entries) + + filtered_repo_files = list( + filter_repo_objects( + items=tree_entries.keys(), + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + ) + ) + tqdm_desc = f"Fetching {len(filtered_repo_files)} files" + if dry_run: + tqdm_desc = "[dry-run] " + tqdm_desc + + snapshot_folder = os.path.join(storage_folder, "snapshots", commit_hash) + # if passed revision is not identical to commit_hash + # then revision has to be a branch name or tag name. + # In that case store a ref. + if revision != commit_hash: + ref_path = os.path.join(storage_folder, "refs", revision) + try: + os.makedirs(os.path.dirname(ref_path), exist_ok=True) + with open(ref_path, "w") as f: + f.write(commit_hash) + except OSError as e: + logger.warning(f"Ignored error while writing commit hash to {ref_path}: {e}.") + + results: list[str | DryRunFileInfo] = [] + + # User can use its own tqdm class or the default one from `huggingface_hub.utils` + tqdm_class = tqdm_class or hf_tqdm + + # Create progress bars for the bytes downloaded. + # Transfer bytes are received from the network; reconstruction bytes are written to disk. + transfer_progress = _create_progress_bar( + cls=tqdm_class, + log_level=logger.getEffectiveLevel(), + name="huggingface_hub.snapshot_download.transfer", + desc="Downloading bytes", + total=0, + initial=0, + unit="B", + unit_scale=True, + bar_format=XET_TRANSFER_BAR_FORMAT, + ) + + reconstruct_progress = _create_progress_bar( + cls=tqdm_class, + log_level=logger.getEffectiveLevel(), + name="huggingface_hub.snapshot_download", + desc="Reconstructing (incomplete total...)", + total=0, + initial=0, + unit="B", + unit_scale=True, + bar_format=XET_BYTES_BAR_FORMAT, + ) + + class _AggregatedTqdm: + """Fake tqdm object to aggregate progress into the parent snapshot progress bars. + + In practice, the `_AggregatedTqdm` object won't be displayed; it's just used to update + the `reconstruct_progress` and `transfer_progress` bars from each thread/file download. + """ + + def __init__(self, *args, **kwargs): + # Adjust the total of the parent progress bar + total = kwargs.pop("total", None) + if total is not None: + reconstruct_progress.total = (reconstruct_progress.total or 0) + total + transfer_progress.total = (transfer_progress.total or 0) + total + reconstruct_progress.refresh() + + # Adjust initial of the parent progress bar + initial = kwargs.pop("initial", 0) + if initial: + reconstruct_progress.update(initial) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + pass + + def close(self) -> None: + pass + + def update(self, n: int | float | None = 1) -> None: + reconstruct_progress.update(n) + + def update_transfer(self, n: int | float | None = 1) -> None: + _update_transfer_bar(transfer_progress, int(n or 0)) + + def set_postfix_str(self, postfix: str, refresh: bool = False) -> None: + # Discard the caller's per-file rate; report the summed rate across all files instead. + _set_aggregate_rate_postfix(reconstruct_progress) + + def set_transfer_postfix_str(self, postfix: str, refresh: bool = False) -> None: + _set_aggregate_rate_postfix(transfer_progress) + + # Pass the commit_hash as revision to hf_hub_download to skip network call if: + # - file is cached + # - or xet file with metadata cached in /tree cache + def _inner_hf_hub_download(repo_file: str) -> None: + results.append( + hf_hub_download( # type: ignore + repo_id, + filename=repo_file, + repo_type=repo_type, + revision=commit_hash, + endpoint=endpoint, + cache_dir=cache_dir, + local_dir=local_dir, + library_name=library_name, + library_version=library_version, + user_agent=user_agent, + etag_timeout=etag_timeout, + force_download=force_download, + token=token, + headers=headers, + tqdm_class=_AggregatedTqdm, # type: ignore + dry_run=dry_run, + ) + ) + + hf_thread_map( + _inner_hf_hub_download, + filtered_repo_files, + desc=tqdm_desc, + max_workers=max_workers, + tqdm_class=tqdm_class, + ) + + _finish_transfer_bar(transfer_progress) + transfer_progress.set_description("Download complete") + reconstruct_progress.set_description("Reconstruction complete") + + if dry_run: + assert all(isinstance(r, DryRunFileInfo) for r in results) + return results # type: ignore + + if local_dir is not None: + return str(os.path.realpath(local_dir)) + return snapshot_folder + + +def _raise_if_incomplete_snapshot( + *, + tree_cache_folder: str, + commit_hash: str, + base_dir: str, + allow_patterns: list[str] | str | None, + ignore_patterns: list[str] | str | None, + repo_id: str, + revision: str, + api_call_error: Exception | None, +) -> None: + """Raise [`IncompleteSnapshotError`] if the cached tree listing shows `base_dir` misses requested files. + + If the tree listing is not cached we cannot tell, so we do nothing and the caller keeps returning the + folder as-is. Otherwise every expected file (after pattern filtering) must exist under `base_dir`. + """ + tree_entries = read_tree_cache(tree_cache_folder, commit_hash) + if tree_entries is None: + return + expected = filter_repo_objects( + items=tree_entries.keys(), allow_patterns=allow_patterns, ignore_patterns=ignore_patterns + ) + missing = [path for path in expected if not _local_file_exists(base_dir, path)] + if not missing: + return + + sample = ", ".join(missing[:3]) + if len(missing) > 3: + sample += f", ... ({len(missing) - 3} more)" + if api_call_error is not None: + reason = f"The Hub could not be reached ({api_call_error.__class__.__name__}: {api_call_error})." + else: + reason = "Outgoing traffic is disabled ('local_files_only=True')." + raise IncompleteSnapshotError( + f"The cached snapshot for '{repo_id}' (revision '{revision}', commit {commit_hash}) is incomplete: " + f"{len(missing)} file(s) are missing ({sample}). {reason} Re-run the download with network access " + "to complete the snapshot.", + snapshot_path=base_dir, + ) from api_call_error + + +@validate_hf_hub_args +def get_cached_repo_tree( + repo_id: str, + *, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, +) -> list[RepoFile]: + """Return the cached tree listing of a repo at a given revision, without any network call. + + The tree listing is the set of files (with their download metadata) of a repo at a commit. It is populated + on disk as a side effect of [`snapshot_download`] (see the `trees/.json` cache files) and is + used to skip network calls on subsequent downloads. This function exposes that cache directly. + + If you need the current tree listing of a repo on the Hub, use [`list_repo_tree`] instead. + + Args: + repo_id (`str`): + A user or an organization name and a repo name separated by a `/`. + repo_type (`str`, *optional*): + Set to `"dataset"`, `"space"` or `"kernel"` if listing from a dataset, space or kernel repo, + `None` or `"model"` if listing from a model. Default is `None`. + revision (`str`, *optional*): + An optional Git revision id, which can be a branch name, a tag, or a commit hash. Defaults to the + default branch. Branch/tag names are resolved to a commit hash using the local cache (`refs/`). + cache_dir (`str`, `Path`, *optional*): + Path to the folder where cached files are stored. Defaults to the value of `HF_HUB_CACHE`. + local_dir (`str` or `Path`, *optional*): + If provided, read the tree listing cached by a `local_dir` download (from + `local_dir/.cache/huggingface/`) instead of the main cache. Branch/tag revisions are still resolved + to a commit hash using the main cache (`cache_dir`). + + Returns: + `list[RepoFile]`: The list of [`RepoFile`] objects cached for this revision. + + Raises: + [`~errors.CachedRepoTreeNotFoundError`] + If no tree listing is cached for the requested revision (e.g. the repo was never downloaded at this revision). + + Example: + ```py + >>> from huggingface_hub import get_cached_repo_tree + >>> files = get_cached_repo_tree("openai-community/gpt2") + >>> [f.path for f in files] + ['.gitattributes', 'config.json', 'model.safetensors', ...] + ``` + """ + if cache_dir is None: + cache_dir = constants.HF_HUB_CACHE + cache_dir = str(Path(cache_dir).expanduser().resolve()) + if local_dir is not None: + local_dir = str(Path(local_dir).expanduser().resolve()) + if revision is None: + revision = constants.DEFAULT_REVISION + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + if repo_type not in constants.REPO_TYPES_WITH_KERNEL: + raise ValueError( + f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES_WITH_KERNEL)}" + ) + + storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type)) + + # For `local_dir` downloads the tree listing lives under `local_dir/.cache/huggingface/`; otherwise it lives + # in the per-repo `storage_folder`. Refs are always recorded in the main cache, so we resolve them there. + tree_cache_folder = tree_cache_folder_for_local_dir(local_dir) if local_dir is not None else storage_folder + + # The tree cache is keyed by commit hash. Resolve the revision to a commit hash: either it already is one, + # or it's a branch/tag name recorded in `refs/` by a previous download. + if REGEX_COMMIT_HASH.match(revision): + commit_hash = revision + else: + ref_path = os.path.join(storage_folder, "refs", revision) + if not os.path.isfile(ref_path): + raise CachedRepoTreeNotFoundError( + f"No cached tree listing found for '{repo_id}' (revision '{revision}', repo_type '{repo_type}'): " + f"the revision is not a commit hash and no matching ref is cached in '{storage_folder}'. " + "Download the repo (e.g. with `snapshot_download`) to populate the cache first." + ) + with open(ref_path) as f: + commit_hash = f.read() + + tree_entries = read_tree_cache(tree_cache_folder, commit_hash) + if tree_entries is None: + raise CachedRepoTreeNotFoundError( + f"No cached tree listing found for '{repo_id}' (revision '{revision}', commit '{commit_hash}', " + f"repo_type '{repo_type}') in '{tree_cache_folder}'. Download the repo (e.g. with `snapshot_download`) " + "to populate the cache first." + ) + + return [ + RepoFile(path=path, size=entry.size, oid=entry.blob_id, xetHash=entry.xet_hash) + for path, entry in tree_entries.items() + ] + + +def _local_file_exists(base_dir: str, path: str) -> bool: + """Check whether a repo file (path relative to `base_dir`, '/'-separated) exists on disk. + + On Windows, paths longer than 255 characters must be prefixed with `\\\\?\\`, otherwise `os.path.isfile` reports an + existing file as missing. + """ + full_path = os.path.join(base_dir, *path.split("/")) + if os.name == "nt" and len(os.path.abspath(full_path)) > 255 and not full_path.startswith("\\\\?\\"): + full_path = "\\\\?\\" + os.path.abspath(full_path) + return os.path.isfile(full_path) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_space_api.py b/venv/lib/python3.12/site-packages/huggingface_hub/_space_api.py new file mode 100644 index 0000000000000000000000000000000000000000..7d00fd56a78a6d17b82d81dc4e8816da9a45518c --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_space_api.py @@ -0,0 +1,395 @@ +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import Literal + +from huggingface_hub.utils import HfMount, HfUri, parse_datetime + + +class SpaceStage(str, Enum): + """ + Enumeration of possible stage of a Space on the Hub. + + Value can be compared to a string: + ```py + assert SpaceStage.BUILDING == "BUILDING" + ``` + + Taken from https://github.com/huggingface/moon-landing/blob/main/server/repo_types/SpaceInfo.ts#L61 (private url). + """ + + # Copied from moon-landing > server > repo_types > SpaceInfo.ts (private repo) + NO_APP_FILE = "NO_APP_FILE" + CONFIG_ERROR = "CONFIG_ERROR" + BUILDING = "BUILDING" + BUILD_ERROR = "BUILD_ERROR" + RUNNING = "RUNNING" + RUNNING_BUILDING = "RUNNING_BUILDING" + RUNTIME_ERROR = "RUNTIME_ERROR" + DELETING = "DELETING" + STOPPED = "STOPPED" + PAUSED = "PAUSED" + APP_STARTING = "APP_STARTING" + RUNNING_APP_STARTING = "RUNNING_APP_STARTING" + + +INTERMEDIATE_SPACE_STAGES = ( + SpaceStage.BUILDING, + SpaceStage.RUNNING_BUILDING, + SpaceStage.APP_STARTING, + SpaceStage.RUNNING_APP_STARTING, +) + +TERMINAL_SPACE_STAGES = ( + SpaceStage.RUNNING, + SpaceStage.BUILD_ERROR, + SpaceStage.RUNTIME_ERROR, + SpaceStage.CONFIG_ERROR, + SpaceStage.NO_APP_FILE, + SpaceStage.STOPPED, + SpaceStage.PAUSED, + SpaceStage.DELETING, +) + + +class SpaceHardware(str, Enum): + """ + Enumeration of hardwares available to run your Space on the Hub. + + Value can be compared to a string: + ```py + assert SpaceHardware.CPU_BASIC == "cpu-basic" + ``` + + Taken from https://github.com/huggingface-internal/moon-landing/blob/main/server/repo_types/SpaceHardwareFlavor.ts (private url). + """ + + # CPU + CPU_BASIC = "cpu-basic" + CPU_UPGRADE = "cpu-upgrade" + + # ZeroGPU + ZERO_A10G = "zero-a10g" + + # GPU + T4_SMALL = "t4-small" + T4_MEDIUM = "t4-medium" + L4X1 = "l4x1" + L4X4 = "l4x4" + L40SX1 = "l40sx1" + L40SX4 = "l40sx4" + L40SX8 = "l40sx8" + A10G_SMALL = "a10g-small" + A10G_LARGE = "a10g-large" + A10G_LARGEX2 = "a10g-largex2" + A10G_LARGEX4 = "a10g-largex4" + A100_LARGE = "a100-large" + A100X4 = "a100x4" + A100X8 = "a100x8" + + +class SpaceStorage(str, Enum): + """ + Enumeration of persistent storage available for your Space on the Hub. + + Value can be compared to a string: + ```py + assert SpaceStorage.SMALL == "small" + ``` + + Taken from https://github.com/huggingface/moon-landing/blob/main/server/repo_types/SpaceHardwareFlavor.ts#L24 (private url). + """ + + SMALL = "small" + MEDIUM = "medium" + LARGE = "large" + + +@dataclass +class Volume: + """ + Describes a volume to mount in a Space or Job container. + + Args: + type (`str`): + Type of volume: `"bucket"`, `"model"`, `"dataset"`, or `"space"`. + source (`str`): + Source identifier, e.g. `"username/my-bucket"` or `"username/my-model"`. + mount_path (`str`): + Mount path inside the container, e.g. `"/data"`. Must start with `/`. + revision (`str` or `None`): + Git revision (only for repos, defaults to `"main"`). + read_only (`bool` or `None`): + Read-only mount. Forced `True` for repos, defaults to `False` for buckets. + path (`str` or `None`): + Subfolder prefix inside the bucket/repo to mount, e.g. `"path/to/dir"`. + """ + + type: Literal["bucket", "model", "dataset", "space"] + source: str + mount_path: str + revision: str | None = None + read_only: bool | None = None + path: str | None = None + + def __init__(self, **kwargs) -> None: + self.type = kwargs.get("type", "model") + self.source = kwargs["source"] + mount_path = kwargs.get("mountPath") + self.mount_path = mount_path if mount_path is not None else kwargs["mount_path"] + self.revision = kwargs.get("revision") + read_only = kwargs.get("readOnly") + self.read_only = read_only if read_only is not None else kwargs.get("read_only") + self.path = kwargs.get("path") + + def to_dict(self) -> dict: + """Serialize to the JSON payload expected by the Hub API.""" + data: dict = { + "type": self.type, + "source": self.source, + "mountPath": self.mount_path, + } + if self.revision is not None: + data["revision"] = self.revision + if self.read_only is not None: + data["readOnly"] = self.read_only + if self.path is not None: + data["path"] = self.path + return data + + def to_uri(self) -> str: + """Return the volume as an HF mount URI in the format expected by the CLI.""" + return HfMount( + source=HfUri(type=self.type, id=self.source, revision=self.revision, path_in_repo=self.path or ""), + mount_path=self.mount_path, + read_only=self.read_only, + ).to_uri() + + +@dataclass +class SpaceHotReloading: + status: Literal["created", "canceled"] + replica_statuses: list[tuple[str, str | None]] # See _hot_reloading_types.ApiCreateReloadResponse.res.status + raw: dict + + def __init__(self, data: dict) -> None: + self.status = data["status"] + self.replica_statuses = data["replicaStatuses"] + self.raw = data + + +@dataclass +class SpaceRuntime: + """ + Contains information about the current runtime of a Space. + + Args: + stage (`str`): + Current stage of the space. Example: RUNNING. + hardware (`str` or `None`): + Current hardware of the space. Example: "cpu-basic". Can be `None` if Space + is `BUILDING` for the first time. + requested_hardware (`str` or `None`): + Requested hardware. Can be different from `hardware` especially if the request + has just been made. Example: "t4-medium". Can be `None` if no hardware has + been requested yet. + sleep_time (`int` or `None`): + Number of seconds the Space will be kept alive after the last request. By default (if value is `None`), the + Space will never go to sleep if it's running on an upgraded hardware, while it will go to sleep after 48 + hours on a free 'cpu-basic' hardware. For more details, see https://huggingface.co/docs/hub/spaces-gpus#sleep-time. + volumes (`list[Volume]` or `None`): + List of volumes mounted in the Space. Each volume is a [`Volume`] object describing its type, source, + mount path, and optional settings. `None` if no volumes are attached. + raw (`dict`): + Raw response from the server. Contains more information about the Space + runtime like number of replicas, number of cpu, memory size,... + """ + + stage: SpaceStage + hardware: SpaceHardware | None + requested_hardware: SpaceHardware | None + sleep_time: int | None + storage: SpaceStorage | None + dev_mode: bool + hot_reloading: SpaceHotReloading | None + volumes: list[Volume] | None + raw: dict + + def __init__(self, data: dict) -> None: + self.stage = data["stage"] + self.hardware = data.get("hardware", {}).get("current") + self.requested_hardware = data.get("hardware", {}).get("requested") + self.sleep_time = data.get("gcTimeout") + self.storage = data.get("storage") + self.dev_mode = data.get("devMode", False) + self.hot_reloading = SpaceHotReloading(raw_hr) if (raw_hr := data.get("hotReloading")) is not None else None + raw_volumes = data.get("volumes") + self.volumes = [Volume(**v) for v in raw_volumes] if raw_volumes is not None else None + self.raw = data + + +@dataclass +class SpaceSecret: + """ + Contains information about a secret of a Space. + + Secret values are write-only and cannot be read back. Only the key, description, + and last update time are returned by the API. + + Args: + key (`str`): + Secret key. Example: `"GITHUB_API_KEY"` + description (`str` or None): + Description of the secret. Example: `"Github API key to access the Github API"`. + updated_at (`datetime` or None): + datetime of the last update of the secret (if the secret has been updated at least once). + """ + + key: str + description: str | None + updated_at: datetime | None + + def __init__(self, key: str, values: dict) -> None: + self.key = key + self.description = values.get("description") + updated_at = values.get("updatedAt") + self.updated_at = parse_datetime(updated_at) if updated_at is not None else None + + +@dataclass +class SpaceVariable: + """ + Contains information about the current variables of a Space. + + Args: + key (`str`): + Variable key. Example: `"MODEL_REPO_ID"` + value (`str`): + Variable value. Example: `"the_model_repo_id"`. + description (`str` or None): + Description of the variable. Example: `"Model Repo ID of the implemented model"`. + updatedAt (`datetime` or None): + datetime of the last update of the variable (if the variable has been updated at least once). + """ + + key: str + value: str + description: str | None + updated_at: datetime | None + + def __init__(self, key: str, values: dict) -> None: + self.key = key + self.value = values["value"] + self.description = values.get("description") + updated_at = values.get("updatedAt") + self.updated_at = parse_datetime(updated_at) if updated_at is not None else None + + +@dataclass +class SpaceSearchResult: + """A single result from the Spaces semantic search API. + + Returned by [`HfApi.search_spaces`]. + + Attributes: + id (`str`): + ID of the Space (e.g. `"username/repo-name"`). + author (`str`): + Author of the Space. + title (`str`): + Display title of the Space. + emoji (`str` or `None`): + Emoji icon of the Space. + sdk (`str` or `None`): + SDK used by the Space (e.g. `"gradio"`, `"docker"`, `"static"`). + likes (`int`): + Number of likes. + private (`bool`): + Whether the Space is private. + tags (`list[str]` or `None`): + List of tags. + runtime ([`SpaceRuntime`] or `None`): + Runtime information (stage, hardware, etc.). + ai_short_description (`str` or `None`): + AI-generated short description. + ai_category (`str` or `None`): + AI-generated category (e.g. `"Image Generation"`). + semantic_relevancy_score (`float` or `None`): + Semantic relevancy score (0-1) relative to the search query. + trending_score (`int` or `None`): + Trending score. + """ + + id: str + author: str + title: str + emoji: str | None + sdk: str | None + likes: int + private: bool + tags: list[str] | None + runtime: SpaceRuntime | None + ai_short_description: str | None + ai_category: str | None + semantic_relevancy_score: float | None + trending_score: int | None + + def __init__(self, data: dict) -> None: + runtime = data.get("runtime") + self.id = data["id"] + self.author = data.get("author", "") + self.title = data.get("title", "") + self.emoji = data.get("emoji") + self.sdk = data.get("sdk") + self.likes = data.get("likes", 0) + self.private = data.get("private", False) + self.tags = data.get("tags") + self.runtime = SpaceRuntime(runtime) if runtime else None + self.ai_short_description = data.get("ai_short_description") + self.ai_category = data.get("ai_category") + self.semantic_relevancy_score = data.get("semanticRelevancyScore") + self.trending_score = data.get("trendingScore") + + +@dataclass +class SpaceTemplate: + """ + Contains information about a Space template available on the Hub. + + Returned by [`HfApi.list_space_templates`]. The `repo_id` can be passed as `space_template` + to [`HfApi.create_repo`] to seed a new Space from that template. + + Args: + name (`str`): + Human-friendly name of the template (e.g. `"JupyterLab"`, `"chatbot"`). + repo_id (`str`): + Repo id of the template Space (e.g. `"SpacesExamples/jupyterlab"`). + sdk (`str`): + SDK the template is built with (e.g. `"gradio"`, `"docker"`, `"static"`). + preferred_private (`bool`): + Whether Spaces created from this template are recommended to be private. + """ + + name: str + repo_id: str + sdk: str + preferred_private: bool + + def __init__(self, data: dict) -> None: + self.name = data["name"] + self.repo_id = data["repoId"] + self.sdk = data["sdk"] + self.preferred_private = data["preferredPrivate"] diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_tensorboard_logger.py b/venv/lib/python3.12/site-packages/huggingface_hub/_tensorboard_logger.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a0d222990e25297feaa72c846b94212e0ebcfc --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_tensorboard_logger.py @@ -0,0 +1,189 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains a logger to push training logs to the Hub, using Tensorboard.""" + +from pathlib import Path + +from ._commit_scheduler import CommitScheduler +from .errors import EntryNotFoundError +from .repocard import ModelCard +from .utils import experimental + + +# Depending on user's setup, SummaryWriter can come either from 'tensorboardX' +# or from 'torch.utils.tensorboard'. Both are compatible so let's try to load +# from either of them. +try: + from tensorboardX import SummaryWriter as _RuntimeSummaryWriter + + is_summary_writer_available = True +except ImportError: + try: + from torch.utils.tensorboard import SummaryWriter as _RuntimeSummaryWriter + + is_summary_writer_available = True + except ImportError: + # Dummy class to avoid failing at import. Will raise on instance creation. + class _DummySummaryWriter: + pass + + _RuntimeSummaryWriter = _DummySummaryWriter # type: ignore[assignment] # ty: ignore[conflicting-declarations] + is_summary_writer_available = False + + +class HFSummaryWriter(_RuntimeSummaryWriter): + """ + Wrapper around the tensorboard's `SummaryWriter` to push training logs to the Hub. + + Data is logged locally and then pushed to the Hub asynchronously. Pushing data to the Hub is done in a separate + thread to avoid blocking the training script. In particular, if the upload fails for any reason (e.g. a connection + issue), the main script will not be interrupted. Data is automatically pushed to the Hub every `commit_every` + minutes (default to every 5 minutes). + + > [!WARNING] + > `HFSummaryWriter` is experimental. Its API is subject to change in the future without prior notice. + + Args: + repo_id (`str`): + The id of the repo to which the logs will be pushed. + logdir (`str`, *optional*): + The directory where the logs will be written. If not specified, a local directory will be created by the + underlying `SummaryWriter` object. + commit_every (`int` or `float`, *optional*): + The frequency (in minutes) at which the logs will be pushed to the Hub. Defaults to 5 minutes. + squash_history (`bool`, *optional*): + Whether to squash the history of the repo after each commit. Defaults to `False`. Squashing commits is + useful to avoid degraded performances on the repo when it grows too large. + repo_type (`str`, *optional*): + The type of the repo to which the logs will be pushed. Defaults to "model". + repo_revision (`str`, *optional*): + The revision of the repo to which the logs will be pushed. Defaults to "main". + repo_private (`bool`, *optional*): + Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists. + path_in_repo (`str`, *optional*): + The path to the folder in the repo where the logs will be pushed. Defaults to "tensorboard/". + repo_allow_patterns (`list[str]` or `str`, *optional*): + A list of patterns to include in the upload. Defaults to `"*.tfevents.*"`. Check out the + [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-folder) for more details. + repo_ignore_patterns (`list[str]` or `str`, *optional*): + A list of patterns to exclude in the upload. Check out the + [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-folder) for more details. + token (`str`, *optional*): + Authentication token. Will default to the stored token. See https://huggingface.co/settings/token for more + details + kwargs: + Additional keyword arguments passed to `SummaryWriter`. + + Examples: + ```diff + # Taken from https://pytorch.org/docs/stable/tensorboard.html + - from torch.utils.tensorboard import SummaryWriter + + from huggingface_hub import HFSummaryWriter + + import numpy as np + + - writer = SummaryWriter() + + writer = HFSummaryWriter(repo_id="username/my-trained-model") + + for n_iter in range(100): + writer.add_scalar('Loss/train', np.random.random(), n_iter) + writer.add_scalar('Loss/test', np.random.random(), n_iter) + writer.add_scalar('Accuracy/train', np.random.random(), n_iter) + writer.add_scalar('Accuracy/test', np.random.random(), n_iter) + ``` + + ```py + >>> from huggingface_hub import HFSummaryWriter + + # Logs are automatically pushed every 15 minutes (5 by default) + when exiting the context manager + >>> with HFSummaryWriter(repo_id="test_hf_logger", commit_every=15) as logger: + ... logger.add_scalar("a", 1) + ... logger.add_scalar("b", 2) + ``` + """ + + @experimental + def __new__(cls, *args, **kwargs) -> "HFSummaryWriter": + if not is_summary_writer_available: + raise ImportError( + "You must have `tensorboard` installed to use `HFSummaryWriter`. Please run `pip install --upgrade" + " tensorboardX` first." + ) + return super().__new__(cls) + + def __init__( + self, + repo_id: str, + *, + logdir: str | None = None, + commit_every: int | float = 5, + squash_history: bool = False, + repo_type: str | None = None, + repo_revision: str | None = None, + repo_private: bool | None = None, + path_in_repo: str | None = "tensorboard", + repo_allow_patterns: list[str] | str | None = "*.tfevents.*", + repo_ignore_patterns: list[str] | str | None = None, + token: str | None = None, + **kwargs, + ): + # Initialize SummaryWriter + super().__init__(logdir=logdir, **kwargs) + + # Check logdir has been correctly initialized and fail early otherwise. In practice, SummaryWriter takes care of it. + if not isinstance(self.logdir, str): + raise ValueError(f"`self.logdir` must be a string. Got '{self.logdir}' of type {type(self.logdir)}.") + + # Append logdir name to `path_in_repo` + if path_in_repo is None or path_in_repo == "": + path_in_repo = Path(self.logdir).name + else: + path_in_repo = path_in_repo.strip("/") + "/" + Path(self.logdir).name + + # Initialize scheduler + self.scheduler = CommitScheduler( + folder_path=self.logdir, + path_in_repo=path_in_repo, + repo_id=repo_id, + repo_type=repo_type, + revision=repo_revision, + private=repo_private, + token=token, + allow_patterns=repo_allow_patterns, + ignore_patterns=repo_ignore_patterns, + every=commit_every, + squash_history=squash_history, + ) + + # Exposing some high-level info at root level + self.repo_id = self.scheduler.repo_id + self.repo_type = self.scheduler.repo_type + self.repo_revision = self.scheduler.revision + + # Add `hf-summary-writer` tag to the model card metadata + try: + card = ModelCard.load(repo_id_or_path=self.repo_id, repo_type=self.repo_type) + except EntryNotFoundError: + card = ModelCard("") + tags = card.data.get("tags", []) + if "hf-summary-writer" not in tags: + tags.append("hf-summary-writer") + card.data["tags"] = tags + card.push_to_hub(repo_id=self.repo_id, repo_type=self.repo_type) + + def __exit__(self, exc_type, exc_val, exc_tb): + """Push to hub in a non-blocking way when exiting the logger's context manager.""" + super().__exit__(exc_type, exc_val, exc_tb) + future = self.scheduler.trigger() + future.result() diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_tree_cache.py b/venv/lib/python3.12/site-packages/huggingface_hub/_tree_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..09fe01f8e9d2fb1d7264647e3456e261bed95f50 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_tree_cache.py @@ -0,0 +1,138 @@ +# Copyright 2026-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""On-disk cache for repository tree listings. + +A tree listing is the set of files (with their download metadata) contained in a repo at a given commit. Because a +commit hash is immutable, its tree listing never changes and can be cached forever without any invalidation logic. + +The listing is stored as a human-readable JSON file under `/trees/.json`. The +folder depends on the download target: the per-repo `storage_folder` for `cache_dir` downloads, or +`local_dir/.cache/huggingface/` for `local_dir` downloads (see `tree_cache_folder_for_local_dir`). + +```json +{ + "format_version": 1, + "files": { + "config.json": {"size": 519, "blob_id": ""}, + "model.safetensors": {"size": 1234, "blob_id": "...", "lfs_sha256": "", "lfs_size": 1234, "xet_hash": "..."} + } +} +``` +""" + +import json +import os +import tempfile +import threading +from dataclasses import dataclass + +from .utils import logging + + +logger = logging.get_logger(__name__) + +TREE_CACHE_FORMAT_VERSION = 1 + +# In-memory cache of parsed tree listings, keyed by absolute file path. +_IN_MEMORY_TREE_CACHE: dict[str, "dict[str, TreeCacheEntry]"] = {} +_IN_MEMORY_TREE_CACHE_LOCK = threading.Lock() + + +@dataclass(frozen=True) +class TreeCacheEntry: + """Raw metadata of a single file in a cached tree listing, mirroring the `/tree` endpoint fields.""" + + size: int + blob_id: str + lfs_sha256: str | None = None + lfs_size: int | None = None + xet_hash: str | None = None + + def to_json(self) -> dict: + info: dict = {"size": self.size, "blob_id": self.blob_id} + if self.lfs_sha256 is not None: + info["lfs_sha256"] = self.lfs_sha256 + info["lfs_size"] = self.lfs_size + if self.xet_hash is not None: + info["xet_hash"] = self.xet_hash + return info + + @classmethod + def from_json(cls, info: dict) -> "TreeCacheEntry": + return cls( + size=info["size"], + blob_id=info["blob_id"], + lfs_sha256=info.get("lfs_sha256"), + lfs_size=info.get("lfs_size"), + xet_hash=info.get("xet_hash"), + ) + + +def _tree_cache_path(tree_cache_folder: str, commit_hash: str) -> str: + return os.path.join(tree_cache_folder, "trees", f"{commit_hash}.json") + + +def tree_cache_folder_for_local_dir(local_dir: str) -> str: + """Folder under which the `trees/` cache lives for a `local_dir` download.""" + return os.path.join(local_dir, ".cache", "huggingface") + + +def read_tree_cache(tree_cache_folder: str, commit_hash: str) -> dict[str, TreeCacheEntry] | None: + """Return the cached tree listing for a commit hash, or `None` if not cached (or unreadable).""" + path = _tree_cache_path(tree_cache_folder, commit_hash) + with _IN_MEMORY_TREE_CACHE_LOCK: + if path in _IN_MEMORY_TREE_CACHE: + return _IN_MEMORY_TREE_CACHE[path] + entries = _read_tree_cache_from_disk(path) + if entries is not None: + with _IN_MEMORY_TREE_CACHE_LOCK: + _IN_MEMORY_TREE_CACHE[path] = entries + return entries + + +def _read_tree_cache_from_disk(path: str) -> dict[str, TreeCacheEntry] | None: + try: + with open(path, encoding="utf-8") as f: + data = json.load(f) + if data.get("format_version") != TREE_CACHE_FORMAT_VERSION: + # Unknown format (e.g. written by a newer version) => ignore and re-fetch. + return None + return {file_path: TreeCacheEntry.from_json(info) for file_path, info in data["files"].items()} + except FileNotFoundError: + return None + except (OSError, ValueError, KeyError, TypeError) as e: + logger.warning(f"Ignoring corrupted tree cache file {path}: {e}") + return None + + +def write_tree_cache(tree_cache_folder: str, commit_hash: str, entries: dict[str, TreeCacheEntry]) -> None: + """Write the tree listing of a commit hash to the cache (ignoring any failures).""" + path = _tree_cache_path(tree_cache_folder, commit_hash) + data = { + "format_version": TREE_CACHE_FORMAT_VERSION, + "files": {file_path: entries[file_path].to_json() for file_path in sorted(entries)}, + } + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(dir=os.path.dirname(path), suffix=".tmp") + with os.fdopen(tmp_fd, "w", encoding="utf-8") as f: + json.dump(data, f, indent=1) + os.replace(tmp_path, path) + except OSError as e: + logger.warning(f"Ignored error while writing tree cache file {path}: {e}") + return + + # Seed the in-memory cache so later readers of this commit skip re-reading and re-parsing the file. + with _IN_MEMORY_TREE_CACHE_LOCK: + _IN_MEMORY_TREE_CACHE[path] = dict(entries) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_upload_large_folder.py b/venv/lib/python3.12/site-packages/huggingface_hub/_upload_large_folder.py new file mode 100644 index 0000000000000000000000000000000000000000..2fd0045e6776d2170c5ef31548916ba0fd0470a1 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_upload_large_folder.py @@ -0,0 +1,769 @@ +# Copyright 2024-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import enum +import logging +import os +import queue +import shutil +import sys +import threading +import time +import traceback +from datetime import datetime +from pathlib import Path +from threading import Lock +from typing import TYPE_CHECKING, Any +from urllib.parse import quote + +from ._commit_api import CommitOperationAdd, UploadInfo, _fetch_upload_modes +from ._local_folder import LocalUploadFileMetadata, LocalUploadFilePaths, get_local_upload_paths, read_upload_metadata +from .constants import DEFAULT_REVISION, REPO_TYPES +from .utils import DEFAULT_IGNORE_PATTERNS, _format_size, filter_repo_objects, tqdm +from .utils._runtime import is_xet_available +from .utils.sha import sha_fileobj + + +if TYPE_CHECKING: + from .hf_api import HfApi + +logger = logging.getLogger(__name__) + +WAITING_TIME_IF_NO_TASKS = 10 # seconds +MAX_NB_FILES_FETCH_UPLOAD_MODE = 100 +COMMIT_SIZE_SCALE: list[int] = [20, 50, 75, 100, 125, 200, 250, 400, 600, 1000] + +UPLOAD_BATCH_SIZE_XET = 256 # Max 256 files per upload batch for XET-enabled repos +UPLOAD_BATCH_SIZE_LFS = 1 # Otherwise, batches of 1 for regular LFS upload + +# Repository limits (from https://huggingface.co/docs/hub/repositories-recommendations) +MAX_FILES_PER_REPO = 100_000 # Recommended maximum number of files per repository +MAX_FILES_PER_FOLDER = 10_000 # Recommended maximum number of files per folder +MAX_FILE_SIZE_GB = 200 # Recommended maximum for individual file size (split larger files) +RECOMMENDED_FILE_SIZE_GB = 20 # Recommended maximum for individual file size + + +def _validate_upload_limits(paths_list: list[LocalUploadFilePaths]) -> None: + """ + Validate upload against repository limits and warn about potential issues. + + Args: + paths_list: List of file paths to be uploaded + + Warns about: + - Too many files in the repository (>100k) + - Too many entries (files or subdirectories) in a single folder (>10k) + - Files exceeding size limits (>20GB recommended, >200GB maximum) + """ + logger.info("Running validation checks on files to upload...") + + # Check 1: Total file count + if len(paths_list) > MAX_FILES_PER_REPO: + logger.warning( + f"You are about to upload {len(paths_list):,} files. " + f"This exceeds the recommended limit of {MAX_FILES_PER_REPO:,} files per repository.\n" + f"Consider:\n" + f" - Splitting your data into multiple repositories\n" + f" - Using fewer, larger files (e.g., parquet files)\n" + f" - See: https://huggingface.co/docs/hub/repositories-recommendations" + ) + + # Check 2: Files and subdirectories per folder + # Track immediate children (files and subdirs) for each folder + from collections import defaultdict + + entries_per_folder: dict[str, Any] = defaultdict(lambda: {"files": 0, "subdirs": set()}) + + for paths in paths_list: + path = Path(paths.path_in_repo) + parts = path.parts + + # Count this file in its immediate parent directory + parent = str(path.parent) if str(path.parent) != "." else "." + entries_per_folder[parent]["files"] += 1 + + # Track immediate subdirectories for each parent folder + # Walk through the path components to track parent-child relationships + for i, child in enumerate(parts[:-1]): + parent = "." if i == 0 else "/".join(parts[:i]) + entries_per_folder[parent]["subdirs"].add(child) + + # Check limits for each folder + for folder, data in entries_per_folder.items(): + file_count = data["files"] + subdir_count = len(data["subdirs"]) + total_entries = file_count + subdir_count + + if total_entries > MAX_FILES_PER_FOLDER: + folder_display = "root" if folder == "." else folder + logger.warning( + f"Folder '{folder_display}' contains {total_entries:,} entries " + f"({file_count:,} files and {subdir_count:,} subdirectories). " + f"This exceeds the recommended {MAX_FILES_PER_FOLDER:,} entries per folder.\n" + "Consider reorganising into sub-folders." + ) + + # Check 3: File sizes + large_files = [] + very_large_files = [] + + for paths in paths_list: + size = paths.file_path.stat().st_size + size_gb = size / 1_000_000_000 # Use decimal GB as per Hub limits + + if size_gb > MAX_FILE_SIZE_GB: + very_large_files.append((paths.path_in_repo, size_gb)) + elif size_gb > RECOMMENDED_FILE_SIZE_GB: + large_files.append((paths.path_in_repo, size_gb)) + + # Warn about very large files (>200GB) + if very_large_files: + files_str = "\n - ".join(f"{path}: {size:.1f}GB" for path, size in very_large_files[:5]) + more_str = f"\n ... and {len(very_large_files) - 5} more files" if len(very_large_files) > 5 else "" + logger.warning( + f"Found {len(very_large_files)} files exceeding the {MAX_FILE_SIZE_GB}GB recommended maximum:\n" + f" - {files_str}{more_str}\n" + f"Consider splitting these files into smaller chunks." + ) + + # Warn about large files (>20GB) + if large_files: + files_str = "\n - ".join(f"{path}: {size:.1f}GB" for path, size in large_files[:5]) + more_str = f"\n ... and {len(large_files) - 5} more files" if len(large_files) > 5 else "" + logger.warning( + f"Found {len(large_files)} files larger than {RECOMMENDED_FILE_SIZE_GB}GB (recommended limit):\n" + f" - {files_str}{more_str}\n" + f"Large files may slow down loading and processing." + ) + + logger.info("Validation checks complete.") + + +def upload_large_folder_internal( + api: "HfApi", + repo_id: str, + folder_path: str | Path, + *, + repo_type: str, # Repo type is required! + revision: str | None = None, + private: bool | None = None, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + num_workers: int | None = None, + print_report: bool = True, + print_report_every: int = 60, +): + """Upload a large folder to the Hub in the most resilient way possible. + + See [`HfApi.upload_large_folder`] for the full documentation. + """ + # 1. Check args and setup + if repo_type is None: + raise ValueError( + "For large uploads, `repo_type` is explicitly required. Please set it to `model`, `dataset` or `space`." + " If you are using the CLI, pass it as `--repo-type=model`." + ) + if repo_type not in REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}") + if revision is None: + revision = DEFAULT_REVISION + + folder_path = Path(folder_path).expanduser().resolve() + if not folder_path.is_dir(): + raise ValueError(f"Provided path: '{folder_path}' is not a directory") + + if ignore_patterns is None: + ignore_patterns = [] + elif isinstance(ignore_patterns, str): + ignore_patterns = [ignore_patterns] + ignore_patterns += DEFAULT_IGNORE_PATTERNS + + if num_workers is None: + nb_cores = os.cpu_count() or 1 + num_workers = max(nb_cores // 2, 1) # Use at most half of cpu cores + + # 2. Create repo if missing + repo_url = api.create_repo(repo_id=repo_id, repo_type=repo_type, private=private, exist_ok=True) + logger.info(f"Repo created: {repo_url}") + repo_id = repo_url.repo_id + + # Warn on too many commits + try: + commits = api.list_repo_commits(repo_id=repo_id, repo_type=repo_type, revision=revision) + commit_count = len(commits) + if commit_count > 500: + logger.warning( + f"\n{'=' * 80}\n" + f"WARNING: This repository has {commit_count} commits.\n" + f"Repositories with a large number of commits can experience performance issues.\n" + f"\n" + f"Consider squashing your commit history using `super_squash_history()`.\n" + "To do so, you need to stop this process, run the snippet below and restart the upload command." + f" from huggingface_hub import super_squash_history\n" + f" super_squash_history(repo_id='{repo_id}', repo_type='{repo_type}')\n" + f"\n" + f"Note: This is a non-revertible operation. See the documentation for more details:\n" + f"https://huggingface.co/docs/huggingface_hub/main/en/package_reference/hf_api#huggingface_hub.HfApi.super_squash_history\n" + f"{'=' * 80}\n" + ) + except Exception as e: + # Don't fail the upload if we can't check commit count + logger.debug(f"Could not check commit count: {e}") + + # 2.1 Check if xet is enabled to set batch file upload size + upload_batch_size = UPLOAD_BATCH_SIZE_XET if is_xet_available() else UPLOAD_BATCH_SIZE_LFS + + # 3. List files to upload + filtered_paths_list = filter_repo_objects( + (path.relative_to(folder_path).as_posix() for path in folder_path.glob("**/*") if path.is_file()), + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + ) + paths_list = [get_local_upload_paths(folder_path, relpath) for relpath in filtered_paths_list] + logger.info(f"Found {len(paths_list)} candidate files to upload") + + # Validate upload against repository limits + _validate_upload_limits(paths_list) + + logger.info("Starting upload...") + + # Read metadata for each file + items = [ + (paths, read_upload_metadata(folder_path, paths.path_in_repo)) + for paths in tqdm(paths_list, desc="Recovering from metadata files") + ] + + # 4. Start workers + status = LargeUploadStatus(items, upload_batch_size) + threads = [ + threading.Thread( + target=_worker_job, + kwargs={ + "status": status, + "api": api, + "repo_id": repo_id, + "repo_type": repo_type, + "revision": revision, + }, + ) + for _ in range(num_workers) + ] + + for thread in threads: + thread.start() + + # 5. Print regular reports + if print_report: + print("\n\n" + status.current_report()) + last_report_ts = time.time() + while True: + time.sleep(1) + if time.time() - last_report_ts >= print_report_every: + if print_report: + _print_overwrite(status.current_report()) + last_report_ts = time.time() + if status.is_done(): + logger.info("Is done: exiting main loop") + break + + for thread in threads: + thread.join() + + logger.info(status.current_report()) + logger.info("Upload is complete!") + + +#################### +# Logic to manage workers and synchronize tasks +#################### + + +class WorkerJob(enum.Enum): + SHA256 = enum.auto() + GET_UPLOAD_MODE = enum.auto() + PREUPLOAD_LFS = enum.auto() + COMMIT = enum.auto() + WAIT = enum.auto() # if no tasks are available but we don't want to exit + + +JOB_ITEM_T = tuple[LocalUploadFilePaths, LocalUploadFileMetadata] + + +class LargeUploadStatus: + """Contains information, queues and tasks for a large upload process.""" + + def __init__(self, items: list[JOB_ITEM_T], upload_batch_size: int = 1): + self.items = items + self.queue_sha256: "queue.Queue[JOB_ITEM_T]" = queue.Queue() + self.queue_get_upload_mode: "queue.Queue[JOB_ITEM_T]" = queue.Queue() + self.queue_preupload_lfs: "queue.Queue[JOB_ITEM_T]" = queue.Queue() + self.queue_commit: "queue.Queue[JOB_ITEM_T]" = queue.Queue() + self.lock = Lock() + + self.nb_workers_sha256: int = 0 + self.nb_workers_get_upload_mode: int = 0 + self.nb_workers_preupload_lfs: int = 0 + self.upload_batch_size: int = upload_batch_size + self.nb_workers_commit: int = 0 + self.nb_workers_waiting: int = 0 + self.last_commit_attempt: float | None = None + + self._started_at = datetime.now() + self._chunk_idx: int = 1 + self._chunk_lock: Lock = Lock() + + # Setup queues + for item in self.items: + paths, metadata = item + if metadata.sha256 is None: + self.queue_sha256.put(item) + elif metadata.upload_mode is None: + self.queue_get_upload_mode.put(item) + elif metadata.upload_mode == "lfs" and not metadata.is_uploaded: + self.queue_preupload_lfs.put(item) + elif not metadata.is_committed: + self.queue_commit.put(item) + else: + logger.debug(f"Skipping file {paths.path_in_repo} (already uploaded and committed)") + + def target_chunk(self) -> int: + with self._chunk_lock: + return COMMIT_SIZE_SCALE[self._chunk_idx] + + def update_chunk(self, success: bool, nb_items: int, duration: float) -> None: + with self._chunk_lock: + if not success: + logger.warning(f"Failed to commit {nb_items} files at once. Will retry with less files in next batch.") + self._chunk_idx -= 1 + elif nb_items >= COMMIT_SIZE_SCALE[self._chunk_idx] and duration < 40: + logger.info(f"Successfully committed {nb_items} at once. Increasing the limit for next batch.") + self._chunk_idx += 1 + + self._chunk_idx = max(0, min(self._chunk_idx, len(COMMIT_SIZE_SCALE) - 1)) + + def current_report(self) -> str: + """Generate a report of the current status of the large upload.""" + nb_hashed = 0 + size_hashed = 0 + nb_preuploaded = 0 + nb_lfs = 0 + nb_lfs_unsure = 0 + size_preuploaded = 0 + nb_committed = 0 + size_committed = 0 + total_size = 0 + ignored_files = 0 + total_files = 0 + + with self.lock: + for _, metadata in self.items: + if metadata.should_ignore: + ignored_files += 1 + continue + total_size += metadata.size + total_files += 1 + if metadata.sha256 is not None: + nb_hashed += 1 + size_hashed += metadata.size + if metadata.upload_mode == "lfs": + nb_lfs += 1 + if metadata.upload_mode is None: + nb_lfs_unsure += 1 + if metadata.is_uploaded: + nb_preuploaded += 1 + size_preuploaded += metadata.size + if metadata.is_committed: + nb_committed += 1 + size_committed += metadata.size + total_size_str = _format_size(total_size) + + now = datetime.now() + now_str = now.strftime("%Y-%m-%d %H:%M:%S") + elapsed = now - self._started_at + elapsed_str = str(elapsed).split(".")[0] # remove milliseconds + + message = "\n" + "-" * 10 + message += f" {now_str} ({elapsed_str}) " + message += "-" * 10 + "\n" + + message += "Files: " + message += f"hashed {nb_hashed}/{total_files} ({_format_size(size_hashed)}/{total_size_str}) | " + message += f"pre-uploaded: {nb_preuploaded}/{nb_lfs} ({_format_size(size_preuploaded)}/{total_size_str})" + if nb_lfs_unsure > 0: + message += f" (+{nb_lfs_unsure} unsure)" + message += f" | committed: {nb_committed}/{total_files} ({_format_size(size_committed)}/{total_size_str})" + message += f" | ignored: {ignored_files}\n" + + message += "Workers: " + message += f"hashing: {self.nb_workers_sha256} | " + message += f"get upload mode: {self.nb_workers_get_upload_mode} | " + message += f"pre-uploading: {self.nb_workers_preupload_lfs} | " + message += f"committing: {self.nb_workers_commit} | " + message += f"waiting: {self.nb_workers_waiting}\n" + message += "-" * 51 + + return message + + def is_done(self) -> bool: + with self.lock: + return all(metadata.is_committed or metadata.should_ignore for _, metadata in self.items) + + +def _worker_job( + status: LargeUploadStatus, + api: "HfApi", + repo_id: str, + repo_type: str, + revision: str, +): + """ + Main process for a worker. The worker will perform tasks based on the priority list until all files are uploaded + and committed. If no tasks are available, the worker will wait for 10 seconds before checking again. + + If a task fails for any reason, the item(s) are put back in the queue for another worker to pick up. + + Read `upload_large_folder` docstring for more information on how tasks are prioritized. + """ + while True: + next_job: tuple[WorkerJob, list[JOB_ITEM_T]] | None = None + + # Determine next task + next_job = _determine_next_job(status) + if next_job is None: + return + job, items = next_job + + # Perform task + match job: + case WorkerJob.SHA256: + item = items[0] # single item + try: + _compute_sha256(item) + status.queue_get_upload_mode.put(item) + except KeyboardInterrupt: + raise + except Exception as e: + logger.error(f"Failed to compute sha256: {e}") + traceback.format_exc() + status.queue_sha256.put(item) + + with status.lock: + status.nb_workers_sha256 -= 1 + + case WorkerJob.GET_UPLOAD_MODE: + try: + _get_upload_mode(items, api=api, repo_id=repo_id, repo_type=repo_type, revision=revision) + except KeyboardInterrupt: + raise + except Exception as e: + logger.error(f"Failed to get upload mode: {e}") + traceback.format_exc() + + # Items are either: + # - dropped (if should_ignore) + # - put in LFS queue (if LFS) + # - put in commit queue (if regular) + # - or put back (if error occurred). + for item in items: + _, metadata = item + if metadata.should_ignore: + continue + match metadata.upload_mode: + case "lfs": + status.queue_preupload_lfs.put(item) + case "regular": + status.queue_commit.put(item) + case _: + status.queue_get_upload_mode.put(item) + + with status.lock: + status.nb_workers_get_upload_mode -= 1 + + case WorkerJob.PREUPLOAD_LFS: + try: + _preupload_lfs(items, api=api, repo_id=repo_id, repo_type=repo_type, revision=revision) + for item in items: + status.queue_commit.put(item) + except KeyboardInterrupt: + raise + except Exception as e: + logger.error(f"Failed to preupload LFS: {e}") + traceback.format_exc() + for item in items: + status.queue_preupload_lfs.put(item) + + with status.lock: + status.nb_workers_preupload_lfs -= 1 + + case WorkerJob.COMMIT: + start_ts = time.time() + success = True + try: + _commit(items, api=api, repo_id=repo_id, repo_type=repo_type, revision=revision) + except KeyboardInterrupt: + raise + except Exception as e: + logger.error(f"Failed to commit: {e}") + traceback.format_exc() + for item in items: + status.queue_commit.put(item) + success = False + duration = time.time() - start_ts + status.update_chunk(success, len(items), duration) + with status.lock: + status.last_commit_attempt = time.time() + status.nb_workers_commit -= 1 + + case WorkerJob.WAIT: + time.sleep(WAITING_TIME_IF_NO_TASKS) + with status.lock: + status.nb_workers_waiting -= 1 + + +def _determine_next_job(status: LargeUploadStatus) -> tuple[WorkerJob, list[JOB_ITEM_T]] | None: + with status.lock: + # 1. Commit if more than 5 minutes since last commit attempt (and at least 1 file) + if ( + status.nb_workers_commit == 0 + and status.queue_commit.qsize() > 0 + and status.last_commit_attempt is not None + and time.time() - status.last_commit_attempt > 5 * 60 + ): + status.nb_workers_commit += 1 + logger.debug("Job: commit (more than 5 minutes since last commit attempt)") + return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk())) + + # 2. Commit if at least 100 files are ready to commit + elif status.nb_workers_commit == 0 and status.queue_commit.qsize() >= 150: + status.nb_workers_commit += 1 + logger.debug("Job: commit (>100 files ready)") + return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk())) + + # 3. Get upload mode if at least 100 files + elif status.queue_get_upload_mode.qsize() >= MAX_NB_FILES_FETCH_UPLOAD_MODE: + status.nb_workers_get_upload_mode += 1 + logger.debug(f"Job: get upload mode (>{MAX_NB_FILES_FETCH_UPLOAD_MODE} files ready)") + return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode, MAX_NB_FILES_FETCH_UPLOAD_MODE)) + + # 4. Preupload LFS file if at least `status.upload_batch_size` files and no worker is preuploading LFS + elif status.queue_preupload_lfs.qsize() >= status.upload_batch_size and status.nb_workers_preupload_lfs == 0: + status.nb_workers_preupload_lfs += 1 + logger.debug("Job: preupload LFS (no other worker preuploading LFS)") + return (WorkerJob.PREUPLOAD_LFS, _get_n(status.queue_preupload_lfs, status.upload_batch_size)) + + # 5. Compute sha256 if at least 1 file and no worker is computing sha256 + elif status.queue_sha256.qsize() > 0 and status.nb_workers_sha256 == 0: + status.nb_workers_sha256 += 1 + logger.debug("Job: sha256 (no other worker computing sha256)") + return (WorkerJob.SHA256, _get_one(status.queue_sha256)) + + # 6. Get upload mode if at least 1 file and no worker is getting upload mode + elif status.queue_get_upload_mode.qsize() > 0 and status.nb_workers_get_upload_mode == 0: + status.nb_workers_get_upload_mode += 1 + logger.debug("Job: get upload mode (no other worker getting upload mode)") + return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode, MAX_NB_FILES_FETCH_UPLOAD_MODE)) + + # 7. Preupload LFS file if at least `status.upload_batch_size` files + elif status.queue_preupload_lfs.qsize() >= status.upload_batch_size: + status.nb_workers_preupload_lfs += 1 + logger.debug("Job: preupload LFS") + return (WorkerJob.PREUPLOAD_LFS, _get_n(status.queue_preupload_lfs, status.upload_batch_size)) + + # 8. Compute sha256 if at least 1 file + elif status.queue_sha256.qsize() > 0: + status.nb_workers_sha256 += 1 + logger.debug("Job: sha256") + return (WorkerJob.SHA256, _get_one(status.queue_sha256)) + + # 9. Get upload mode if at least 1 file + elif status.queue_get_upload_mode.qsize() > 0: + status.nb_workers_get_upload_mode += 1 + logger.debug("Job: get upload mode") + return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode, MAX_NB_FILES_FETCH_UPLOAD_MODE)) + + # 10. Preupload LFS file if at least 1 file + elif status.queue_preupload_lfs.qsize() > 0: + status.nb_workers_preupload_lfs += 1 + logger.debug("Job: preupload LFS") + return (WorkerJob.PREUPLOAD_LFS, _get_n(status.queue_preupload_lfs, status.upload_batch_size)) + + # 11. Commit if at least 1 file and 1 min since last commit attempt + elif ( + status.nb_workers_commit == 0 + and status.queue_commit.qsize() > 0 + and status.last_commit_attempt is not None + and time.time() - status.last_commit_attempt > 1 * 60 + ): + status.nb_workers_commit += 1 + logger.debug("Job: commit (1 min since last commit attempt)") + return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk())) + + # 12. Commit if at least 1 file all other queues are empty and all workers are waiting + # e.g. when it's the last commit + elif ( + status.nb_workers_commit == 0 + and status.queue_commit.qsize() > 0 + and status.queue_sha256.qsize() == 0 + and status.queue_get_upload_mode.qsize() == 0 + and status.queue_preupload_lfs.qsize() == 0 + and status.nb_workers_sha256 == 0 + and status.nb_workers_get_upload_mode == 0 + and status.nb_workers_preupload_lfs == 0 + ): + status.nb_workers_commit += 1 + logger.debug("Job: commit") + return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk())) + + # 13. If all queues are empty, exit + elif all(metadata.is_committed or metadata.should_ignore for _, metadata in status.items): + logger.info("All files have been processed! Exiting worker.") + return None + + # 14. If no task is available, wait + else: + status.nb_workers_waiting += 1 + logger.debug(f"No task available, waiting... ({WAITING_TIME_IF_NO_TASKS}s)") + return (WorkerJob.WAIT, []) + + +#################### +# Atomic jobs (sha256, get_upload_mode, preupload_lfs, commit) +#################### + + +def _compute_sha256(item: JOB_ITEM_T) -> None: + """Compute sha256 of a file and save it in metadata.""" + paths, metadata = item + if metadata.sha256 is None: + with paths.file_path.open("rb") as f: + metadata.sha256 = sha_fileobj(f).hex() + metadata.save(paths) + + +def _get_upload_mode(items: list[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None: + """Get upload mode for each file and update metadata. + + Also receive info if the file should be ignored. + """ + additions = [_build_hacky_operation(item) for item in items] + _fetch_upload_modes( + additions=additions, + repo_type=repo_type, + repo_id=repo_id, + headers=api._build_hf_headers(), + revision=quote(revision, safe=""), + endpoint=api.endpoint, + ) + for item, addition in zip(items, additions): + paths, metadata = item + metadata.upload_mode = addition._upload_mode + metadata.should_ignore = addition._should_ignore + metadata.remote_oid = addition._remote_oid + metadata.save(paths) + + +def _preupload_lfs(items: list[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None: + """Preupload LFS files and update metadata.""" + additions = [_build_hacky_operation(item) for item in items] + api.preupload_lfs_files( + repo_id=repo_id, + repo_type=repo_type, + revision=revision, + additions=additions, + ) + + for paths, metadata in items: + metadata.is_uploaded = True + metadata.save(paths) + + +def _commit(items: list[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None: + """Commit files to the repo.""" + additions = [_build_hacky_operation(item) for item in items] + api.create_commit( + repo_id=repo_id, + repo_type=repo_type, + revision=revision, + operations=additions, + commit_message="Add files using upload-large-folder tool", + ) + for paths, metadata in items: + metadata.is_committed = True + metadata.save(paths) + + +#################### +# Hacks with CommitOperationAdd to bypass checks/sha256 calculation +#################### + + +class HackyCommitOperationAdd(CommitOperationAdd): + def __post_init__(self) -> None: + if isinstance(self.path_or_fileobj, Path): + self.path_or_fileobj = str(self.path_or_fileobj) + + +def _build_hacky_operation(item: JOB_ITEM_T) -> HackyCommitOperationAdd: + paths, metadata = item + operation = HackyCommitOperationAdd(path_in_repo=paths.path_in_repo, path_or_fileobj=paths.file_path) + with paths.file_path.open("rb") as file: + sample = file.peek(512)[:512] + if metadata.sha256 is None: + raise ValueError("sha256 must have been computed by now!") + operation.upload_info = UploadInfo(sha256=bytes.fromhex(metadata.sha256), size=metadata.size, sample=sample) + operation._upload_mode = metadata.upload_mode # type: ignore + operation._should_ignore = metadata.should_ignore + operation._remote_oid = metadata.remote_oid + operation._is_uploaded = metadata.is_uploaded + if metadata.is_uploaded and metadata.upload_mode == "lfs": + operation.path_or_fileobj = b"" + return operation + + +#################### +# Misc helpers +#################### + + +def _get_one(queue: "queue.Queue[JOB_ITEM_T]") -> list[JOB_ITEM_T]: + return [queue.get()] + + +def _get_n(queue: "queue.Queue[JOB_ITEM_T]", n: int) -> list[JOB_ITEM_T]: + return [queue.get() for _ in range(min(queue.qsize(), n))] + + +def _print_overwrite(report: str) -> None: + """Print a report, overwriting the previous lines. + + Since tqdm in using `sys.stderr` to (re-)write progress bars, we need to use `sys.stdout` + to print the report. + + Note: works well only if no other process is writing to `sys.stdout`! + """ + report += "\n" + # Get terminal width + terminal_width = shutil.get_terminal_size().columns + + # Count number of lines that should be cleared + nb_lines = sum(len(line) // terminal_width + 1 for line in report.splitlines()) + + # Clear previous lines based on the number of lines in the report + for _ in range(nb_lines): + sys.stdout.write("\r\033[K") # Clear line + sys.stdout.write("\033[F") # Move cursor up one line + + # Print the new report, filling remaining space with whitespace + sys.stdout.write(report) + sys.stdout.write(" " * (terminal_width - len(report.splitlines()[-1]))) + sys.stdout.flush() diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_upload_pipeline.py b/venv/lib/python3.12/site-packages/huggingface_hub/_upload_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..d575b039d1e0ce19eeb792033772979b6c0c0ca0 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_upload_pipeline.py @@ -0,0 +1,684 @@ +# coding=utf-8 +# Copyright 2026-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Streamed, multi-commit upload of a folder on top of the Xet upload protocol. + +How it works: + +- The **coordinator** (caller's thread) walks the list of files and asks the Hub, 256 files at a + time, what each file is (regular git blob, xet file, ignored). Regular files are accumulated + directly; xet files are registered into a `XetSession` upload-commit, which chunks, deduplicates, + retries and uploads them in the background while the coordinator keeps going. No Python-side + sha256 computation: `hf_xet` computes it during chunking (single read pass over each file). +- Whenever enough files have accumulated (adaptive batch size), the batch is handed over to the + **committer** thread which joins the xet uploads, drops unchanged files, and creates a git + commit for the batch. While a batch is being committed, the coordinator is already uploading + the next one. +- Interrupted uploads are resumable by simply re-running the same call: already-committed files + are dropped (no-op detection against the remote oid) and already-uploaded chunks are + deduplicated by the xet storage backend, transferring ~0 bytes. +""" + +import queue +import shutil +import sys +import threading +import time +from typing import TYPE_CHECKING, Any, Callable +from urllib.parse import quote + +from . import constants +from ._commit_api import ( + CommitOperationAdd, + CommitOperationDelete, + _fetch_upload_modes, + _send_commit, + _warn_on_overwriting_operations, +) +from .errors import RepositoryNotFoundError +from .utils import are_progress_bars_disabled, logging +from .utils._xet import ( + XetTokenType, + abort_xet_session, + get_xet_session, + xet_connection_info_refresh_url, + xet_headers_without_auth, +) + + +if TYPE_CHECKING: + from .hf_api import CommitInfo, HfApi + +logger = logging.get_logger(__name__) + +# Number of files sent to the "preupload" endpoint per call (server-side limit). +PREUPLOAD_BATCH_SIZE = 256 + +# Files per git commit: adaptive, scaled up after fast commits and down after failures. +COMMIT_SIZE_SCALE = [20, 50, 75, 100, 125, 200, 250, 400, 600, 1000] +INITIAL_COMMIT_SIZE_INDEX = 6 # start at 256 files per commit +TARGET_COMMIT_DURATION = 40.0 # seconds; scale up batch size if commits are faster than this +MAX_COMMIT_INTERVAL = 5 * 60.0 # seconds; force a commit if the current batch is older than this + +# Budget of regular-file content per commit (regular files are base64-encoded in the payload). +REGULAR_CONTENT_BYTES_BUDGET = 100 * 1024 * 1024 + +_SENTINEL = object() # Sentinel value for the batch queue to indicate the end of the upload + +# Live display tuning +_BAR_WIDTH = 20 +_REFRESH_INTERVAL = 0.5 # seconds between redraws on a TTY +_NON_TTY_LOG_INTERVAL = 30.0 # seconds between summary logs when stderr is not a TTY + + +def _bar(current: float, total: float, width: int = _BAR_WIDTH) -> str: + if total <= 0: + return "░" * width + filled = int(min(current / total, 1.0) * width) + return "█" * filled + "░" * (width - filled) + + +def _format_bytes(n: float) -> str: + for unit in ("B", "kB", "MB", "GB", "TB"): + if abs(n) < 1000: + if n < 10: + return f"{n:.2f}{unit}" + elif n < 100: + return f"{n:.1f}{unit}" + return f"{n:.0f}{unit}" + n /= 1000 + return f"{n:.1f}PB" + + +class _LiveDisplay: + """Three-line live progress display on stderr:: + + Preparing ████████████████████ 11,100 / 11,100 ✓ + Uploading ██████████████░░░░░░ 580 / 603 files 3.8GB · 19.7MB/s + Committing ██████████████████░░ 10,800 / 11,100 14 commits + + A small renderer thread redraws the three lines in-place every ~0.5 s on a TTY + (worker threads only update counters under a lock). When stderr is not a TTY, + it falls back to a periodic ``logger.info`` summary instead. + + Disabling progress bars (e.g. agent output mode) only turns off the TTY renderer: + the non-TTY log summaries are gated by the logger verbosity alone, so consumers + tailing stderr during a long upload still see periodic progress. + """ + + _N_LINES = 3 + + def __init__(self, total_files: int, enabled: bool = True) -> None: + self._total = total_files + self._tty = enabled and sys.stderr.isatty() + self._active = self._tty or logger.isEnabledFor(logging.INFO) + self._lock = threading.Lock() + self._drawn = False + self._stop_event = threading.Event() + self._thread: threading.Thread | None = None + + # Counters (written by coordinator/committer threads, read by the renderer) + self._prepared = 0 + self._ignored = 0 + self._xet_total = 0 + self._xet_done: set[str] = set() # item names; unique across batches + self._committed = 0 # committed or skipped-as-unchanged + self._nb_commits = 0 + + # Xet transfer bytes, summed across (possibly concurrent) upload-commits + self._xet_bytes = 0 + self._speed_ema = 0.0 + self._prev_bytes = 0 + self._prev_time: float | None = None + + # -- lifecycle (main thread) ------------------------------------------------ + + def start(self) -> None: + if not self._active: + return + if self._tty: + sys.stderr.write(f"Found {self._total:,} files to upload\n") + sys.stderr.flush() + else: + logger.info(f"Found {self._total:,} files to upload") + self._thread = threading.Thread(target=self._render_loop, name="hf-upload-display", daemon=True) + self._thread.start() + + def close(self) -> None: + if self._thread is not None: + self._stop_event.set() + self._thread.join() + if self._tty: + with self._lock: + self._redraw() # final state + + # -- counter updates (coordinator / committer / xet callback threads) -------- + + def notify_prepared(self, n: int) -> None: + with self._lock: + self._prepared += n + + def notify_ignored(self, n: int) -> None: + with self._lock: + self._ignored += n + + def notify_xet_registered(self, n: int) -> None: + with self._lock: + self._xet_total += n + + def notify_xet_uploaded(self, names: list[str]) -> None: + with self._lock: + self._xet_done.update(names) + + def notify_skipped(self, n: int) -> None: + with self._lock: + self._committed += n + + def notify_commit(self, n_files: int) -> None: + with self._lock: + self._committed += n_files + self._nb_commits += 1 + + def new_xet_callback(self) -> "Callable | None": + """Progress callback for one ``new_upload_commit``. + + The byte counters in ``group_report`` are cumulative *per upload-commit* and several + upload-commits can be in flight at once (one finalizing, one filling), so each commit + gets its own closure tracking its own previous value; increments are summed globally. + """ + if not self._active: + return None + prev = 0 + + def callback(group_report: Any, item_reports: Any) -> None: + nonlocal prev + with self._lock: + completed = group_report.total_transfer_bytes_completed + self._xet_bytes += max(0, completed - prev) + prev = completed + for item in item_reports.values(): + if item.total_bytes > 0 and item.bytes_completed == item.total_bytes: + self._xet_done.add(item.item_name) + + return callback + + # -- rendering (display thread) ---------------------------------------------- + + def _render_loop(self) -> None: + last_log = 0.0 + while not self._stop_event.wait(_REFRESH_INTERVAL): + with self._lock: + self._update_speed() + if self._tty: + self._redraw() + elif time.monotonic() - last_log >= _NON_TTY_LOG_INTERVAL: + logger.info(self._summary()) + last_log = time.monotonic() + + def _update_speed(self) -> None: + now = time.monotonic() + if self._prev_time is not None and now > self._prev_time: + rate = (self._xet_bytes - self._prev_bytes) / (now - self._prev_time) + self._speed_ema = rate if self._speed_ema == 0 else 0.3 * rate + 0.7 * self._speed_ema + self._prev_time = now + self._prev_bytes = self._xet_bytes + + def _redraw(self) -> None: + if self._drawn: + sys.stderr.write(f"\033[{self._N_LINES}A") + width = shutil.get_terminal_size().columns + for line in (self._line_preparing(), self._line_uploading(), self._line_committing()): + truncated = line[: width - 4] + "..." if len(line) > width - 1 else line + sys.stderr.write(f"\r\033[K{truncated}\n") + sys.stderr.flush() + self._drawn = True + + def _line_preparing(self) -> str: + done = " ✓" if self._prepared >= self._total else "" + return f" Preparing {_bar(self._prepared, self._total)} {self._prepared:,} / {self._total:,}{done}" + + def _line_uploading(self) -> str: + if self._xet_total == 0: + bar = _bar(1, 1) if self._prepared >= self._total else _bar(0, 1) + return f" Uploading {bar} -" + n_done = len(self._xet_done) + parts = [] + if self._xet_bytes > 0: + parts.append(_format_bytes(self._xet_bytes)) + if self._speed_ema > 0: + parts.append(f"{_format_bytes(self._speed_ema)}/s") + extra = f" {' · '.join(parts)}" if parts else "" + done = " ✓" if self._prepared >= self._total and n_done >= self._xet_total else "" + return f" Uploading {_bar(n_done, self._xet_total)} {n_done:,} / {self._xet_total:,} files{extra}{done}" + + def _line_committing(self) -> str: + effective = self._total - self._ignored + commits_str = f" {self._nb_commits} commits" if self._nb_commits > 1 else "" + done = " ✓" if self._committed >= effective > 0 else "" + return ( + f" Committing {_bar(self._committed, effective)} {self._committed:,} / {effective:,}{commits_str}{done}" + ) + + def _summary(self) -> str: + return ( + f"Uploading... {self._prepared:,}/{self._total:,} files checked, " + f"{len(self._xet_done):,}/{self._xet_total:,} uploaded ({_format_bytes(self._xet_bytes)} transferred), " + f"{self._committed:,} committed in {self._nb_commits} commit(s)" + ) + + +class _CommitPacer: + """Adaptive number of files per commit, to stay below server-side commit timeouts.""" + + def __init__(self) -> None: + self._index = INITIAL_COMMIT_SIZE_INDEX + + @property + def target(self) -> int: + return COMMIT_SIZE_SCALE[self._index] + + def record_success(self, duration: float, nb_files: int) -> None: + if duration < TARGET_COMMIT_DURATION and nb_files >= self.target: + self._index = min(self._index + 1, len(COMMIT_SIZE_SCALE) - 1) + elif duration > TARGET_COMMIT_DURATION: + self._index = max(self._index - 1, 0) + + def record_failure(self) -> None: + self._index = max(self._index - 1, 0) + + +class _Batch: + """A group of files destined to a single git commit, with their in-flight xet uploads.""" + + def __init__(self) -> None: + self.ops: list[CommitOperationAdd] = [] + self.regular_bytes: int = 0 + self.xet_commit: Any = None # XetUploadCommit, opened lazily + self.handles: list[tuple[CommitOperationAdd, Any]] = [] # (op, XetFileUpload) + self.created_at: float = time.monotonic() + + +class _UploadPipeline: + def __init__( + self, + api: "HfApi", + *, + repo_id: str, + repo_type: str, + add_operations: list[CommitOperationAdd], + delete_operations: list[CommitOperationDelete], + commit_message: str, + commit_description: str | None, + token: str | bool | None, + revision: str | None, + create_pr: bool, + parent_commit: str | None, + ) -> None: + self.api = api + self.repo_id = repo_id + self.repo_type = repo_type + self.add_operations = add_operations + self.delete_operations = delete_operations + self.commit_message = commit_message + self.commit_description = commit_description + self.token = token + self.headers = api._build_hf_headers(token=token) + self.revision = revision or constants.DEFAULT_REVISION + self.create_pr = create_pr + self.parent_commit = parent_commit + + # The base revision is used by the coordinator for ALL preupload calls and the xet token + # refresh URL, with the `create_pr` flag — exactly like `create_commit` does. It never + # changes during the run, even after a PR has been created. + self.base_revision_quoted = quote(self.revision, safe="") + + # Committer state (mutated by the committer thread only) + self.commit_revision_quoted = self.base_revision_quoted # switched to the PR ref once created + self.pr_url: str | None = None + self.pr_revision: str | None = None + self.nb_commits = 0 + self.last_commit_info: "CommitInfo | None" = None + self.pacer = _CommitPacer() + + # Pipeline plumbing + self.batch_queue: queue.Queue = queue.Queue(maxsize=1) + self.errors: list[BaseException] = [] + self.abort_event = threading.Event() + self.display = _LiveDisplay(total_files=len(add_operations), enabled=not are_progress_bars_disabled()) + + # All xet uploads share the same token refresh URL. With `create_pr`, the final ref is not + # known in advance: `?create_pr=1` makes the server grant a token valid for PR refs. + refresh_url = xet_connection_info_refresh_url( + token_type=XetTokenType.WRITE, + repo_id=repo_id, + repo_type=repo_type, + revision=self.base_revision_quoted, + endpoint=api.endpoint, + ) + if create_pr: + refresh_url += "?create_pr=1" + self.xet_session = get_xet_session() + self.xet_commit_kwargs = { + "token_refresh_url": refresh_url, + "token_refresh_headers": self.headers, + "custom_headers": xet_headers_without_auth(self.headers), + } + + # `.gitignore` rules are enforced server-side: forward the local one if it's being uploaded. + self.gitignore_content: str | None = None + for op in add_operations: + if op.path_in_repo == ".gitignore": + with op.as_file() as f: + self.gitignore_content = f.read().decode() + break + + def run(self) -> "CommitInfo": + _warn_on_overwriting_operations([*self.delete_operations, *self.add_operations]) + committer = threading.Thread(target=self._committer_loop, name="hf-upload-committer", daemon=True) + committer.start() + self.display.start() + try: + self._coordinator_loop() + except BaseException: + self.abort_event.set() + abort_xet_session() + raise + finally: + if self.abort_event.is_set(): + # The committer exits on its own once the queue is drained (see `_committer_loop`). + # Bound the wait so a xet call blocked on the (aborted) session can never hang the + # shutdown — the committer is a daemon thread. + committer.join(timeout=10) + else: + self.batch_queue.put(_SENTINEL) + committer.join() + self.display.close() + if self.abort_event.is_set() and self.pr_revision is not None: + logger.warning( + f"Upload to pull request {self.pr_url} did not complete. To resume into the" + f' same PR, re-run with `revision="{self.pr_revision}"` (without `create_pr=True`). Re-running' + " with `create_pr=True` would open a new pull request." + ) + if self.errors: + raise self.errors[0] + return self._final_commit_info() + + # ---------------------------------------------------------------- coordinator + + def _coordinator_loop(self) -> None: + import hf_xet + + batch = _Batch() + for start in range(0, len(self.add_operations), PREUPLOAD_BATCH_SIZE): + if self.abort_event.is_set(): + self._abort_batch(batch) + return + chunk = self.add_operations[start : start + PREUPLOAD_BATCH_SIZE] + try: + _fetch_upload_modes( + additions=chunk, + repo_type=self.repo_type, + repo_id=self.repo_id, + headers=self.headers, + revision=self.base_revision_quoted, + endpoint=self.api.endpoint, + create_pr=self.create_pr, + gitignore_content=self.gitignore_content, + ) + except RepositoryNotFoundError as e: + from .hf_api import _CREATE_COMMIT_NO_REPO_ERROR_MESSAGE + + e.append_to_message(_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE) + raise + self.display.notify_prepared(len(chunk)) + for op in chunk: + if op._should_ignore: + logger.debug(f"Skipping upload for '{op.path_in_repo}' (ignored by gitignore rules).") + self.display.notify_ignored(1) + continue + if op._upload_mode == "regular": + batch.regular_bytes += op.upload_info.size + else: + if batch.xet_commit is None: + batch.xet_commit = self.xet_session.new_upload_commit( + progress_callback=self.display.new_xet_callback(), **self.xet_commit_kwargs + ) + # Upload starts immediately in the background. sha256 is computed by hf_xet + # while chunking, unless already known (e.g. resumed operations). + sha_arg = op.upload_info.sha256.hex() if op.upload_info.is_hashed else hf_xet.COMPUTE_SHA256 + if isinstance(op.path_or_fileobj, bytes): + handle = batch.xet_commit.start_upload_bytes( + op.path_or_fileobj, sha256=sha_arg, name=op.path_in_repo + ) + else: + handle = batch.xet_commit.start_upload_file(str(op.path_or_fileobj), sha256=sha_arg) + batch.handles.append((op, handle)) + self.display.notify_xet_registered(1) + batch.ops.append(op) + + if ( + len(batch.ops) >= self.pacer.target + or batch.regular_bytes >= REGULAR_CONTENT_BYTES_BUDGET + or (time.monotonic() - batch.created_at > MAX_COMMIT_INTERVAL and len(batch.ops) > 0) + ): + self._enqueue(batch) + batch = _Batch() + self._enqueue(batch) + + def _enqueue(self, batch: _Batch) -> None: + if len(batch.ops) == 0 and not (self.nb_commits == 0 and len(self.delete_operations) > 0): + return + # Blocks if a batch is already waiting: natural backpressure on scanning/uploading. + while not self.abort_event.is_set(): + try: + self.batch_queue.put(batch, timeout=1.0) + return + except queue.Full: + continue + self._abort_batch(batch) + + def _abort_batch(self, batch: _Batch) -> None: + if batch.xet_commit is not None: + try: + batch.xet_commit.abort() + except Exception: + pass + + # ---------------------------------------------------------------- committer + + def _committer_loop(self) -> None: + while True: + try: + batch = self.batch_queue.get(timeout=0.5) + except queue.Empty: + if self.abort_event.is_set(): + return # aborted: exit once the queue is drained, no sentinel needed + continue + if batch is _SENTINEL: + return + try: + if not self.abort_event.is_set(): + self._process_batch(batch) + else: + self._abort_batch(batch) + except BaseException as e: + self._abort_batch(batch) + self.errors.append(e) + self.abort_event.set() + + def _process_batch(self, batch: _Batch) -> None: + # 1. Wait for all xet uploads of this batch and finalize them (atomic xet commit). Files + # can only be referenced by a git commit once their xet upload-commit is finalized. + if batch.xet_commit is not None: + batch.xet_commit.wait_to_finish() + for op, handle in batch.handles: + if not op.upload_info.is_hashed: + op.upload_info.sha256 = bytes.fromhex(handle.result().xet_info.sha256) + op._is_uploaded = True + # Files whose last progress tick was missed are still done at this point. + self.display.notify_xet_uploaded( + [ + str(op.path_or_fileobj) if not isinstance(op.path_or_fileobj, bytes) else op.path_in_repo + for op, _ in batch.handles + ] + ) + + # 2. Drop files that have not changed compared to the remote (prevents empty commits). + # Their chunks were deduplicated anyway (~0 bytes transferred). + ops_to_commit = [] + for op in batch.ops: + if op._remote_oid is not None and op._remote_oid == op._local_oid: + logger.debug(f"Skipping commit for '{op.path_in_repo}' (file unchanged).") + self.display.notify_skipped(1) + continue + ops_to_commit.append(op) + + # 3. Create the git commit(s). On failure, scale down and split the batch. + if len(ops_to_commit) > 0 or (self.nb_commits == 0 and len(self.delete_operations) > 0): + self._commit_with_split(ops_to_commit) + + def _commit_with_split(self, ops: list[CommitOperationAdd]) -> None: + try: + self._do_commit(ops) + except Exception as e: + self.pacer.record_failure() + if len(ops) <= COMMIT_SIZE_SCALE[0]: + raise + logger.warning(f"Commit of {len(ops)} files failed ({e!r}). Retrying in smaller chunks.") + target = self.pacer.target + for start in range(0, len(ops), target): + self._commit_with_split(ops[start : start + target]) + + def _do_commit(self, ops: list[CommitOperationAdd]) -> None: + if self.create_pr and self.pr_revision is None: + # Create the (draft) pull request explicitly and push every commit to its ref. Committing + # with `?create_pr=1` instead would risk opening a second PR if the commit POST is retried + # after a lost response. Created lazily so that a fully-unchanged upload opens no PR. + # Note: PRs created this way are always opened against the default branch, hence the + # `create_pr` + `revision` combination being rejected in `upload_folder`. + pr = self.api.create_pull_request( + repo_id=self.repo_id, + title=self.commit_message, + token=self.token, + description=self.commit_description, + repo_type=self.repo_type, + ) + if pr.git_reference is None: + raise ValueError("Server did not return a git reference for the created pull request.") + self.pr_url = pr.url + self.pr_revision = pr.git_reference + self.commit_revision_quoted = quote(pr.git_reference, safe="") + + operations: list[Any] = list(ops) + if self.nb_commits == 0: + # Deletions and `parent_commit` ride the first commit. + operations = list(self.delete_operations) + operations + + commit_message = ( + self.commit_message if self.nb_commits == 0 else f"{self.commit_message} (part {self.nb_commits + 1})" + ) + t0 = time.monotonic() + # Retried with backoff on transient errors: safe because the commit targets an explicit + # ref (`?create_pr=1` is never used, see above). + self.last_commit_info = _send_commit( + operations=operations, + files_to_copy={}, + commit_message=commit_message, + commit_description=self.commit_description or "", + repo_type=self.repo_type, + repo_id=self.repo_id, + headers=self.headers, + revision=self.commit_revision_quoted, + endpoint=self.api.endpoint, + parent_commit=self.parent_commit if self.nb_commits == 0 else None, + retry_on_error=True, + ) + duration = time.monotonic() - t0 + self.pacer.record_success(duration, len(ops)) + self.nb_commits += 1 + + for op in ops: + op._is_committed = True + self.display.notify_commit(len(ops)) + logger.debug(f"Committed {len(ops)} file(s) in {duration:.1f}s: {self.last_commit_info.commit_url}") + + # ---------------------------------------------------------------- result + + def _final_commit_info(self) -> "CommitInfo": + from .hf_api import CommitInfo + + if self.last_commit_info is None: + # Nothing was committed (everything unchanged/ignored): mimic `create_commit` and + # return info about the latest commit on the target revision. + logger.warning("No files have been modified since last commit. Skipping to prevent empty commit.") + info = self.api.repo_info( + repo_id=self.repo_id, repo_type=self.repo_type, revision=self.revision, token=self.token + ) + url_prefix = self.api.endpoint + if self.repo_type != constants.REPO_TYPE_MODEL: + url_prefix = f"{url_prefix}/{self.repo_type}s" + return CommitInfo( + commit_url=f"{url_prefix}/{self.repo_id}/commit/{info.sha}", + commit_message=self.commit_message, + commit_description=self.commit_description or "", + oid=info.sha, # type: ignore + _endpoint=self.api.endpoint, + ) + if self.nb_commits > 1: + logger.info(f"Upload completed in {self.nb_commits} commits.") + if self.pr_url is not None: + # PR upload: attach the PR info (commit responses don't carry it; the PR is created separately). + return CommitInfo( + commit_url=self.last_commit_info.commit_url, + commit_message=self.last_commit_info.commit_message, + commit_description=self.last_commit_info.commit_description, + oid=self.last_commit_info.oid, + pr_url=self.pr_url, + _endpoint=self.api.endpoint, + ) + return self.last_commit_info + + +def pipelined_upload( + api: "HfApi", + *, + repo_id: str, + repo_type: str, + add_operations: list[CommitOperationAdd], + delete_operations: list[CommitOperationDelete], + commit_message: str, + commit_description: str | None = None, + token: str | bool | None = None, + revision: str | None = None, + create_pr: bool = False, + parent_commit: str | None = None, +) -> "CommitInfo": + """Upload a prepared list of operations through the streamed multi-commit pipeline. + + Requires `hf_xet` to be installed. See module docstring for the architecture. + """ + + return _UploadPipeline( + api, + repo_id=repo_id, + repo_type=repo_type, + add_operations=add_operations, + delete_operations=delete_operations, + commit_message=commit_message, + commit_description=commit_description, + token=token, + revision=revision, + create_pr=create_pr, + parent_commit=parent_commit, + ).run() diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_webhooks_payload.py b/venv/lib/python3.12/site-packages/huggingface_hub/_webhooks_payload.py new file mode 100644 index 0000000000000000000000000000000000000000..976a97a1d30a5105d26f4cb3746be2c9d10d1e4b --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_webhooks_payload.py @@ -0,0 +1,136 @@ +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains data structures to parse the webhooks payload.""" + +from typing import Literal + +from .utils import is_pydantic_available + + +if is_pydantic_available(): + from pydantic import BaseModel +else: + # Define a dummy BaseModel to avoid import errors when pydantic is not installed + # Import error will be raised when trying to use the class + + class BaseModel: # type: ignore [no-redef] + def __init__(self, *args, **kwargs) -> None: + raise ImportError( + "You must have `pydantic` installed to use `WebhookPayload`. This is an optional dependency that" + " should be installed separately. Please run `pip install --upgrade pydantic` and retry." + ) + + +# This is an adaptation of the ReportV3 interface implemented in moon-landing. V0, V1 and V2 have been ignored as they +# are not in used anymore. To keep in sync when format is updated in +# https://github.com/huggingface/moon-landing/blob/main/server/lib/HFWebhooks.ts (internal link). + + +WebhookEvent_T = Literal[ + "create", + "delete", + "move", + "update", +] +RepoChangeEvent_T = Literal[ + "add", + "move", + "remove", + "update", +] +RepoType_T = Literal[ + "dataset", + "model", + "space", +] +DiscussionStatus_T = Literal[ + "closed", + "draft", + "open", + "merged", +] +SupportedWebhookVersion = Literal[3] + + +class ObjectId(BaseModel): + id: str + + +class WebhookPayloadUrl(BaseModel): + web: str + api: str | None = None + + +class WebhookPayloadMovedTo(BaseModel): + name: str + owner: ObjectId + + +class WebhookPayloadWebhook(ObjectId): + version: SupportedWebhookVersion + + +class WebhookPayloadEvent(BaseModel): + action: WebhookEvent_T + scope: str + + +class WebhookPayloadDiscussionChanges(BaseModel): + base: str + mergeCommitId: str | None = None + + +class WebhookPayloadComment(ObjectId): + author: ObjectId + hidden: bool + content: str | None = None + url: WebhookPayloadUrl + + +class WebhookPayloadDiscussion(ObjectId): + num: int + author: ObjectId + url: WebhookPayloadUrl + title: str + isPullRequest: bool + status: DiscussionStatus_T + changes: WebhookPayloadDiscussionChanges | None = None + pinned: bool | None = None + + +class WebhookPayloadRepo(ObjectId): + owner: ObjectId + head_sha: str | None = None + name: str + private: bool + subdomain: str | None = None + tags: list[str] | None = None + type: Literal["dataset", "model", "space"] + url: WebhookPayloadUrl + + +class WebhookPayloadUpdatedRef(BaseModel): + ref: str + oldSha: str | None = None + newSha: str | None = None + + +class WebhookPayload(BaseModel): + event: WebhookPayloadEvent + repo: WebhookPayloadRepo + discussion: WebhookPayloadDiscussion | None = None + comment: WebhookPayloadComment | None = None + webhook: WebhookPayloadWebhook + movedTo: WebhookPayloadMovedTo | None = None + updatedRefs: list[WebhookPayloadUpdatedRef] | None = None diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/_webhooks_server.py b/venv/lib/python3.12/site-packages/huggingface_hub/_webhooks_server.py new file mode 100644 index 0000000000000000000000000000000000000000..8e327d5ee882c503baf3e3a34a8dad4ec3bf6ef2 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/_webhooks_server.py @@ -0,0 +1,376 @@ +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains `WebhooksServer` and `webhook_endpoint` to create a webhook server easily.""" + +import atexit +import inspect +import os +from collections.abc import Callable +from functools import wraps +from typing import TYPE_CHECKING, Any, Optional + +from .utils import experimental, is_fastapi_available, is_gradio_available + + +if TYPE_CHECKING: + import gradio as gr + from fastapi import Request + +if is_fastapi_available(): + from fastapi import FastAPI, Request + from fastapi.responses import JSONResponse +else: + # Will fail at runtime if FastAPI is not available + FastAPI = Request = JSONResponse = None # type: ignore + + +_global_app: Optional["WebhooksServer"] = None # ty: ignore[invalid-type-form] +_is_local = os.environ.get("SPACE_ID") is None + + +@experimental +class WebhooksServer: + """ + The [`WebhooksServer`] class lets you create an instance of a Gradio app that can receive Huggingface webhooks. + These webhooks can be registered using the [`~WebhooksServer.add_webhook`] decorator. Webhook endpoints are added to + the app as a POST endpoint to the FastAPI router. Once all the webhooks are registered, the `launch` method has to be + called to start the app. + + It is recommended to accept [`WebhookPayload`] as the first argument of the webhook function. It is a Pydantic + model that contains all the information about the webhook event. The data will be parsed automatically for you. + + Check out the [webhooks guide](../guides/webhooks_server) for a step-by-step tutorial on how to set up your + WebhooksServer and deploy it on a Space. + + > [!WARNING] + > `WebhooksServer` is experimental. Its API is subject to change in the future. + + > [!WARNING] + > You must have `gradio` installed to use `WebhooksServer` (`pip install --upgrade gradio`). + + Args: + ui (`gradio.Blocks`, optional): + A Gradio UI instance to be used as the Space landing page. If `None`, a UI displaying instructions + about the configured webhooks is created. + webhook_secret (`str`, optional): + A secret key to verify incoming webhook requests. You can set this value to any secret you want as long as + you also configure it in your [webhooks settings panel](https://huggingface.co/settings/webhooks). You + can also set this value as the `WEBHOOK_SECRET` environment variable. If no secret is provided, the + webhook endpoints are opened without any security. + + Example: + + ```python + import gradio as gr + from huggingface_hub import WebhooksServer, WebhookPayload + + with gr.Blocks() as ui: + ... + + app = WebhooksServer(ui=ui, webhook_secret="my_secret_key") + + @app.add_webhook("/say_hello") + async def hello(payload: WebhookPayload): + return {"message": "hello"} + + app.launch() + ``` + """ + + def __new__(cls, *args, **kwargs) -> "WebhooksServer": # ty: ignore[invalid-type-form] + if not is_gradio_available(): + raise ImportError( + "You must have `gradio` installed to use `WebhooksServer`. Please run `pip install --upgrade gradio`" + " first." + ) + if not is_fastapi_available(): + raise ImportError( + "You must have `fastapi` installed to use `WebhooksServer`. Please run `pip install --upgrade fastapi`" + " first." + ) + return super().__new__(cls) + + def __init__( + self, + ui: Optional["gr.Blocks"] = None, + webhook_secret: str | None = None, + ) -> None: + self._ui = ui + + self.webhook_secret = webhook_secret or os.getenv("WEBHOOK_SECRET") + self.registered_webhooks: dict[str, Callable] = {} + _warn_on_empty_secret(self.webhook_secret) + + def add_webhook(self, path: str | None = None) -> Callable: + """ + Decorator to add a webhook to the [`WebhooksServer`] server. + + Args: + path (`str`, optional): + The URL path to register the webhook function. If not provided, the function name will be used as the + path. In any case, all webhooks are registered under `/webhooks`. + + Raises: + ValueError: If the provided path is already registered as a webhook. + + Example: + ```python + from huggingface_hub import WebhooksServer, WebhookPayload + + app = WebhooksServer() + + @app.add_webhook + async def trigger_training(payload: WebhookPayload): + if payload.repo.type == "dataset" and payload.event.action == "update": + # Trigger a training job if a dataset is updated + ... + + app.launch() + ``` + """ + # Usage: directly as decorator. Example: `@app.add_webhook` + if callable(path): + # If path is a function, it means it was used as a decorator without arguments + return self.add_webhook()(path) + + # Usage: provide a path. Example: `@app.add_webhook(...)` + @wraps(FastAPI.post) + def _inner_post(*args, **kwargs): + func = args[0] + abs_path = f"/webhooks/{(path or func.__name__).strip('/')}" + if abs_path in self.registered_webhooks: + raise ValueError(f"Webhook {abs_path} already exists.") + self.registered_webhooks[abs_path] = func + + return _inner_post + + def launch(self, prevent_thread_lock: bool = False, **launch_kwargs: Any) -> None: + """Launch the Gradio app and register webhooks to the underlying FastAPI server. + + Input parameters are forwarded to Gradio when launching the app. + """ + ui = self._ui or self._get_default_ui() + + # Start Gradio App + # - as non-blocking so that webhooks can be added afterwards + # - as shared if launch locally (to debug webhooks) + launch_kwargs.setdefault("share", _is_local) + self.fastapi_app, _, _ = ui.launch(prevent_thread_lock=True, **launch_kwargs) + + # Register webhooks to FastAPI app + for path, func in self.registered_webhooks.items(): + # Add secret check if required + if self.webhook_secret is not None: + func = _wrap_webhook_to_check_secret(func, webhook_secret=self.webhook_secret) + + # Add route to FastAPI app + self.fastapi_app.post(path)(func) + + # Print instructions and block main thread + space_host = os.environ.get("SPACE_HOST") + url = "https://" + space_host if space_host is not None else (ui.share_url or ui.local_url) + if url is None: + raise ValueError("Cannot find the URL of the app. Please provide a valid `ui` or update `gradio` version.") + url = url.strip("/") + message = "\nWebhooks are correctly setup and ready to use:" + message += "\n" + "\n".join(f" - POST {url}{webhook}" for webhook in self.registered_webhooks) + message += "\nGo to https://huggingface.co/settings/webhooks to setup your webhooks." + print(message) + + if not prevent_thread_lock: + ui.block_thread() + + def _get_default_ui(self) -> "gr.Blocks": + """Default UI if not provided (lists webhooks and provides basic instructions).""" + import gradio as gr + + with gr.Blocks() as ui: + gr.Markdown("# This is an app to process 🤗 Webhooks") + gr.Markdown( + "Webhooks are a foundation for MLOps-related features. They allow you to listen for new changes on" + " specific repos or to all repos belonging to particular set of users/organizations (not just your" + " repos, but any repo). Check out this [guide](https://huggingface.co/docs/hub/webhooks) to get to" + " know more about webhooks on the Huggingface Hub." + ) + gr.Markdown( + f"{len(self.registered_webhooks)} webhook(s) are registered:" + + "\n\n" + + "\n ".join( + f"- [{webhook_path}]({_get_webhook_doc_url(webhook.__name__, webhook_path)})" + for webhook_path, webhook in self.registered_webhooks.items() + ) + ) + gr.Markdown( + "Go to https://huggingface.co/settings/webhooks to setup your webhooks." + + "\nYou app is running locally. Please look at the logs to check the full URL you need to set." + if _is_local + else ( + "\nThis app is running on a Space. You can find the corresponding URL in the options menu" + " (top-right) > 'Embed the Space'. The URL looks like 'https://{username}-{repo_name}.hf.space'." + ) + ) + return ui + + +@experimental +def webhook_endpoint(path: str | None = None) -> Callable: + """Decorator to start a [`WebhooksServer`] and register the decorated function as a webhook endpoint. + + This is a helper to get started quickly. If you need more flexibility (custom landing page or webhook secret), + you can use [`WebhooksServer`] directly. You can register multiple webhook endpoints (to the same server) by using + this decorator multiple times. + + Check out the [webhooks guide](../guides/webhooks_server) for a step-by-step tutorial on how to set up your + server and deploy it on a Space. + + > [!WARNING] + > `webhook_endpoint` is experimental. Its API is subject to change in the future. + + > [!WARNING] + > You must have `gradio` installed to use `webhook_endpoint` (`pip install --upgrade gradio`). + + Args: + path (`str`, optional): + The URL path to register the webhook function. If not provided, the function name will be used as the path. + In any case, all webhooks are registered under `/webhooks`. + + Examples: + The default usage is to register a function as a webhook endpoint. The function name will be used as the path. + The server will be started automatically at exit (i.e. at the end of the script). + + ```python + from huggingface_hub import webhook_endpoint, WebhookPayload + + @webhook_endpoint + async def trigger_training(payload: WebhookPayload): + if payload.repo.type == "dataset" and payload.event.action == "update": + # Trigger a training job if a dataset is updated + ... + + # Server is automatically started at the end of the script. + ``` + + Advanced usage: register a function as a webhook endpoint and start the server manually. This is useful if you + are running it in a notebook. + + ```python + from huggingface_hub import webhook_endpoint, WebhookPayload + + @webhook_endpoint + async def trigger_training(payload: WebhookPayload): + if payload.repo.type == "dataset" and payload.event.action == "update": + # Trigger a training job if a dataset is updated + ... + + # Start the server manually + trigger_training.launch() + ``` + """ + if callable(path): + # If path is a function, it means it was used as a decorator without arguments + return webhook_endpoint()(path) + + @wraps(WebhooksServer.add_webhook) + def _inner(func: Callable) -> Callable: + app = _get_global_app() + app.add_webhook(path)(func) + if len(app.registered_webhooks) == 1: + # Register `app.launch` to run at exit (only once) + atexit.register(app.launch) + + @wraps(app.launch) + def _launch_now(): + # Run the app directly (without waiting atexit) + atexit.unregister(app.launch) + app.launch() + + func.launch = _launch_now # type: ignore + return func + + return _inner + + +def _get_global_app() -> WebhooksServer: # ty: ignore[invalid-type-form] + global _global_app + if _global_app is None: + _global_app = WebhooksServer() + return _global_app + + +def _warn_on_empty_secret(webhook_secret: str | None) -> None: + if webhook_secret is None: + print("Webhook secret is not defined. This means your webhook endpoints will be open to everyone.") + print( + "To add a secret, set `WEBHOOK_SECRET` as environment variable or pass it at initialization: " + "\n\t`app = WebhooksServer(webhook_secret='my_secret', ...)`" + ) + print( + "For more details about webhook secrets, please refer to" + " https://huggingface.co/docs/hub/webhooks#webhook-secret." + ) + else: + print("Webhook secret is correctly defined.") + + +def _get_webhook_doc_url(webhook_name: str, webhook_path: str) -> str: + """Returns the anchor to a given webhook in the docs (experimental)""" + return "/docs#/default/" + webhook_name + webhook_path.replace("/", "_") + "_post" + + +def _wrap_webhook_to_check_secret(func: Callable, webhook_secret: str) -> Callable: + """Wraps a webhook function to check the webhook secret before calling the function. + + This is a hacky way to add the `request` parameter to the function signature. Since FastAPI based itself on route + parameters to inject the values to the function, we need to hack the function signature to retrieve the `Request` + object (and hence the headers). A far cleaner solution would be to use a middleware. However, since + `fastapi==0.90.1`, a middleware cannot be added once the app has started. And since the FastAPI app is started by + Gradio internals (and not by us), we cannot add a middleware. + + This method is called only when a secret has been defined by the user. If a request is sent without the + "x-webhook-secret", the function will return a 401 error (unauthorized). If the header is sent but is incorrect, + the function will return a 403 error (forbidden). + + Inspired by https://stackoverflow.com/a/33112180. + """ + initial_sig = inspect.signature(func) + + @wraps(func) + async def _protected_func(request: Request, **kwargs): + request_secret = request.headers.get("x-webhook-secret") + if request_secret is None: + return JSONResponse({"error": "x-webhook-secret header not set."}, status_code=401) + if request_secret != webhook_secret: + return JSONResponse({"error": "Invalid webhook secret."}, status_code=403) + + # Inject `request` in kwargs if required + if "request" in initial_sig.parameters: + kwargs["request"] = request + + # Handle both sync and async routes + if inspect.iscoroutinefunction(func): + return await func(**kwargs) + else: + return func(**kwargs) + + # Update signature to include request + if "request" not in initial_sig.parameters: + _protected_func.__signature__ = initial_sig.replace( # type: ignore + parameters=( + inspect.Parameter(name="request", kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Request), + ) + + tuple(initial_sig.parameters.values()) + ) + + # Return protected route + return _protected_func diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__init__.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..15c9257674e1f57ed2437e5725dc11e082e13e7f --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from huggingface_hub.cli._output import out # noqa: F401 diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca430abb12473979f4903ead8bdc8fcebd9d92b2 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_city_game.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_city_game.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5433de00645df1955e1eca5d44173eed262371b9 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_city_game.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_cli_utils.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_cli_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d431e685fea3395bdd0c8462fa12b6e735bf1cee Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_cli_utils.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_completion.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_completion.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de650fead26540cf7959485f4e9868a6d03a33d3 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_completion.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_cp.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_cp.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5dfae72720cd1de6ec2cf546894c1c6520512259 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_cp.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_errors.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_errors.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e7643298b07b81adeef2200137ad4b175f700580 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_errors.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_file_listing.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_file_listing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d2851bf0a9cadc1414b42857d96749c6764b856c Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_file_listing.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_framework.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_framework.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..97dc983463d120f16ddcc60e6c7d82068d9d636c Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_framework.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_help_formatter.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_help_formatter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7de423e3f51925d73e18d8729c54d4825d0335c7 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_help_formatter.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_output.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_output.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..407010d92ba5fdbd71ce5ab2dc2e252b44c05b77 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_output.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_skills.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_skills.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..78d0f5bf84632b92cfa3ffdaa18237f2006daf1f Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/_skills.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/auth.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/auth.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..11eae772a65ea5eedde2694765d9314cc4a93e4b Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/auth.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/buckets.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/buckets.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f8e80930ac9a6f8083026bd06744a162b337154 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/buckets.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/cache.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/cache.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..512568594c790551e1465deb837405d5aab0599c Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/cache.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/collections.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/collections.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be3f8d6340b3945a97eedd44836e10a456bfe1c5 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/collections.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/datasets.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/datasets.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e64eb239699663d17608ac2abb1b1bb22c9714bf Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/datasets.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/deprecated_cli.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/deprecated_cli.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f1e8ea8800a7a7ea4f49d1b55e1ae14b8baac56 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/deprecated_cli.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/discussions.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/discussions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f58ea57922cc872df92359b8c8c863a8966c608f Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/discussions.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/download.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/download.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c6314b843262c4cad890711ea5618c3be8170ac Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/download.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/extensions.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/extensions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..91f525f367e835c9e7cc63c51d675586ae7d8900 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/extensions.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/hf.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/hf.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a57905e1b6fa8ab6b47e7f9e2863565662c8a8b5 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/hf.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/inference_endpoints.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/inference_endpoints.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8a31768bf057b49dfc70dd665b6c1dc020945e0 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/inference_endpoints.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/jobs.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/jobs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9fa4bbadec9aa5cf34a8f1bc350d8ed0f8052419 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/jobs.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/lfs.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/lfs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d418ed835e721d4b5d5bc5f5950a40ddcaef1cda Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/lfs.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/models.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..03a978b393a02efbc9161f4561eb58d616fee1b0 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/models.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/papers.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/papers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab4437f33d95a9654208cda53584befe36976b8e Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/papers.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/repo_files.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/repo_files.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..20c5a1daccb14300f7fa3c56c7d70288bd09a3ed Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/repo_files.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/repos.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/repos.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..474f79eba4f5cbd3fb6d4f8e8adeb3ed993a8950 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/repos.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/sandbox.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/sandbox.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d5ae8782ac777d7452da4f6da5e60905344af3a4 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/sandbox.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/skills.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/skills.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ffc166802e834579ccf55ed3dca7d81866e8e955 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/skills.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/spaces.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/spaces.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb27faae9aa835ebaac4200bff5951184ba46998 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/spaces.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/system.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/system.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..37523e0adb29e66be94f7436bbdb4c1ba23059fb Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/system.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/upload.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/upload.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4814735d23bddcb50c4704919194d10554a093a Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/upload.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/upload_large_folder.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/upload_large_folder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..35847a0326fe485cf508d79ad8f1ece0addb8b95 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/upload_large_folder.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/webhooks.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/webhooks.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..58e65d469673f8f05eeb82305b53cef4110c0ad5 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/cli/__pycache__/webhooks.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/_city_game.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_city_game.py new file mode 100644 index 0000000000000000000000000000000000000000..85c84ddb7c01efe3e11f08c1c499104a72ab262f --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_city_game.py @@ -0,0 +1,661 @@ +"""Interactive isometric city explorer — easter egg for `hf repos ls --explore`.""" + +import dataclasses +import math +import os +import random +import re +import select +import shutil +import sys +import time + +from huggingface_hub.hf_api import RepoStorageInfo + +from ._file_listing import format_size + + +Color = tuple[int, int, int] + +# (top_face, left_face, right_face) — lighter to darker for 3D effect +_TYPE_COLORS: dict[str, tuple[Color, Color, Color]] = { + "model": ((175, 148, 240), (138, 112, 208), (105, 80, 180)), + "dataset": ((245, 128, 128), (222, 92, 92), (190, 60, 60)), + "space": ((245, 175, 85), (218, 140, 55), (185, 110, 30)), + "bucket": ((112, 185, 242), (70, 150, 220), (40, 118, 192)), +} +_EXTRA_COLORS: tuple[Color, Color, Color] = ((168, 176, 188), (128, 136, 148), (90, 98, 110)) +_GRID_COLOR: Color = (178, 182, 190) + +_DX = 4 # isometric half-width (pixels) +_DY = 2 # isometric half-height (pixels) +_MAX_H = 16 # tallest tile (pixels) +_MIN_H = 1 +_COLS = 6 +_EXT = 1 # grid extension beyond tiles +_MAX_TILES = 30 + +# Cursor sprite — pixel art arrow pointer +_OUTLINE: Color = (30, 30, 30) +_FILL: Color = (255, 255, 255) + +_CURSOR_GRID = [ + " X ", + " XWX ", + "XWWWX", + " XWX ", + " X ", +] +_CURSOR_PALETTE: dict[str, Color] = { + "X": _OUTLINE, + "W": _FILL, +} +_CURSOR_H = len(_CURSOR_GRID) + +_MOVE_FRAMES = 8 +_MOVE_DELAY = 0.03 +_CURSOR_PAD = _CURSOR_H + 16 +_GAP = 3 +_MIN_TERM_W = 100 +_MIN_TERM_H = 24 +_SUMMARY_W = 24 + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class TileInfo: + grid_row: int + grid_col: int + height: int + top: Color + left: Color + right: Color + repo: RepoStorageInfo | None + + +@dataclasses.dataclass +class CityData: + tiles: list[TileInfo] + rows: int + cols: int + x_off: int + y_off: int + buf_w: int + buf_h: int + total_storage: int + extra_count: int + extra_storage: int + all_repos: list[RepoStorageInfo] + + +# --------------------------------------------------------------------------- +# City layout +# --------------------------------------------------------------------------- + + +def _prepare_city_data(repos: list[RepoStorageInfo]) -> CityData: + sorted_repos = sorted(repos, key=lambda r: r.storage, reverse=True) + display = sorted_repos[:_MAX_TILES] + extra_count = max(0, len(sorted_repos) - _MAX_TILES) + extra_storage = sum(r.storage for r in sorted_repos[_MAX_TILES:]) + total_storage = sum(r.storage for r in repos) + max_storage = max(1, display[0].storage) + + n = len(display) + (1 if extra_count > 0 else 0) + cols = min(n, _COLS) + rows = math.ceil(n / cols) if cols > 0 else 1 + + tiles: list[TileInfo] = [] + for i, repo in enumerate(display): + r, c = divmod(i, cols) + h = max(_MIN_H, round(math.sqrt(repo.storage / max_storage) * _MAX_H)) + top, left, right = _TYPE_COLORS.get(repo.type, _EXTRA_COLORS) + tiles.append(TileInfo(r, c, h, top, left, right, repo)) + if extra_count > 0: + r, c = divmod(len(display), cols) + h = max(_MIN_H, round(math.sqrt(extra_storage / max_storage) * _MAX_H)) + tiles.append(TileInfo(r, c, h, *_EXTRA_COLORS, None)) + + r_lo, r_hi = -_EXT, rows - 1 + _EXT + c_lo, c_hi = -_EXT, cols - 1 + _EXT + + xs: list[int] = [] + ys: list[int] = [] + for rr in range(r_lo, r_hi + 1): + for cc in range(c_lo, c_hi + 1): + cx, cy = (cc - rr) * _DX, (cc + rr) * _DY + xs.extend([cx - _DX, cx + _DX]) + ys.extend([cy, cy + 2 * _DY]) + for tile in tiles: + ys.append((tile.grid_col + tile.grid_row) * _DY - tile.height) + + x_off = -min(xs) + y_off = -min(ys) + buf_w = max(xs) - min(xs) + 1 + buf_h = max(ys) - min(ys) + 1 + if buf_h % 2: + buf_h += 1 + + return CityData( + tiles=tiles, + rows=rows, + cols=cols, + x_off=x_off, + y_off=y_off, + buf_w=buf_w, + buf_h=buf_h, + total_storage=total_storage, + extra_count=extra_count, + extra_storage=extra_storage, + all_repos=repos, + ) + + +# --------------------------------------------------------------------------- +# Drawing primitives +# --------------------------------------------------------------------------- + + +def _draw_diamond_outline(buf: list[list[Color | None]], cx: int, cy: int) -> None: + t = (cx, cy) + r = (cx + _DX, cy + _DY) + b = (cx, cy + 2 * _DY) + ll = (cx - _DX, cy + _DY) + _draw_line(buf, *t, *r, _GRID_COLOR) + _draw_line(buf, *r, *b, _GRID_COLOR) + _draw_line(buf, *b, *ll, _GRID_COLOR) + _draw_line(buf, *ll, *t, _GRID_COLOR) + + +def _draw_block( + buf: list[list[Color | None]], + cx: int, + cy: int, + h: int, + top: Color, + left: Color, + right: Color, +) -> None: + _fill_poly( + buf, + [(cx - _DX, cy + _DY - h), (cx, cy + 2 * _DY - h), (cx, cy + 2 * _DY), (cx - _DX, cy + _DY)], + left, + ) + _fill_poly( + buf, + [(cx, cy + 2 * _DY - h), (cx + _DX, cy + _DY - h), (cx + _DX, cy + _DY), (cx, cy + 2 * _DY)], + right, + ) + _fill_poly( + buf, + [(cx, cy - h), (cx + _DX, cy + _DY - h), (cx, cy + 2 * _DY - h), (cx - _DX, cy + _DY - h)], + top, + ) + + +def _fill_poly(buf: list[list[Color | None]], verts: list[tuple[int, int]], color: Color) -> None: + bh = len(buf) + bw = len(buf[0]) if buf else 0 + all_y = [v[1] for v in verts] + y0 = max(0, min(all_y)) + y1 = min(bh - 1, max(all_y)) + n = len(verts) + for y in range(y0, y1 + 1): + xl: float = float("inf") + xr: float = float("-inf") + for i in range(n): + ax, ay = verts[i] + bx, by = verts[(i + 1) % n] + if ay == by: + if y == ay: + xl = min(xl, float(min(ax, bx))) + xr = max(xr, float(max(ax, bx))) + continue + if not (min(ay, by) <= y <= max(ay, by)): + continue + t = (y - ay) / (by - ay) + ix = ax + t * (bx - ax) + xl = min(xl, ix) + xr = max(xr, ix) + if xl <= xr: + for x in range(max(0, round(xl)), min(bw, round(xr) + 1)): + buf[y][x] = color + + +def _draw_line(buf: list[list[Color | None]], x0: int, y0: int, x1: int, y1: int, color: Color) -> None: + bh = len(buf) + bw = len(buf[0]) if buf else 0 + dx = abs(x1 - x0) + dy = abs(y1 - y0) + steps = max(dx, dy) + if steps == 0: + if 0 <= y0 < bh and 0 <= x0 < bw: + buf[y0][x0] = color + return + xi = (x1 - x0) / steps + yi = (y1 - y0) / steps + fx, fy = float(x0), float(y0) + for _ in range(steps + 1): + px, py = round(fx), round(fy) + if 0 <= py < bh and 0 <= px < bw: + buf[py][px] = color + fx += xi + fy += yi + + +# --------------------------------------------------------------------------- +# Pixel buffer → terminal +# --------------------------------------------------------------------------- + +_ANSI_RE = re.compile(r"\033\[[0-9;]*m") + + +def _strip_ansi(s: str) -> str: + return _ANSI_RE.sub("", s) + + +def _visible_len(s: str) -> int: + return len(_strip_ansi(s)) + + +def _pixels_to_lines(buf: list[list[Color | None]]) -> list[str]: + height = len(buf) + width = len(buf[0]) if buf else 0 + lines: list[str] = [] + for row in range(0, height, 2): + last = -1 + for col in range(width - 1, -1, -1): + top = buf[row][col] + bot = buf[row + 1][col] if row + 1 < height else None + if top or bot: + last = col + break + if last < 0: + lines.append("") + continue + + parts: list[str] = [] + cfg: Color | None = None + cbg: Color | None = None + + for col in range(last + 1): + top = buf[row][col] + bot = buf[row + 1][col] if row + 1 < height else None + + if not top and not bot: + if cfg is not None or cbg is not None: + parts.append("\033[0m") + cfg = cbg = None + parts.append(" ") + continue + + if top and bot and top == bot: + nfg, nbg, ch = top, None, "█" + elif top and bot: + nfg, nbg, ch = bot, top, "▄" + elif top: + nfg, nbg, ch = top, None, "▀" + else: + nfg, nbg, ch = bot, None, "▄" # type: ignore[assignment] + + esc = "" + if nfg != cfg: + esc += f"\033[38;2;{nfg[0]};{nfg[1]};{nfg[2]}m" + cfg = nfg + if nbg != cbg: + esc += "\033[49m" if nbg is None else f"\033[48;2;{nbg[0]};{nbg[1]};{nbg[2]}m" + cbg = nbg + parts.append(esc + ch) + + if cfg is not None or cbg is not None: + parts.append("\033[0m") + lines.append("".join(parts)) + return lines + + +# --------------------------------------------------------------------------- +# Rendering +# --------------------------------------------------------------------------- + + +def _render_base_buffer(city: CityData) -> list[list[Color | None]]: + buf: list[list[Color | None]] = [[None] * city.buf_w for _ in range(city.buf_h)] + + for tile in city.tiles: + cx = city.x_off + (tile.grid_col - tile.grid_row) * _DX + cy = city.y_off + (tile.grid_col + tile.grid_row) * _DY + _draw_diamond_outline(buf, cx, cy) + + sorted_tiles = sorted(city.tiles, key=lambda t: (t.grid_row + t.grid_col, t.grid_col)) + for tile in sorted_tiles: + cx = city.x_off + (tile.grid_col - tile.grid_row) * _DX + cy = city.y_off + (tile.grid_col + tile.grid_row) * _DY + _draw_block(buf, cx, cy, tile.height, tile.top, tile.left, tile.right) + + return buf + + +# --------------------------------------------------------------------------- +# Summary panel +# --------------------------------------------------------------------------- + + +def _colored_square(color: Color) -> str: + return f"\033[38;2;{color[0]};{color[1]};{color[2]}m■\033[0m" + + +def _build_summary( + repos: list[RepoStorageInfo], + total_storage: int, + extra_count: int, +) -> list[str]: + lines: list[str] = [""] + lines.append(" Storage Overview") + lines.append(" " + "─" * 16) + lines.append(f" {format_size(total_storage, human_readable=True)} total") + lines.append("") + + order = ["model", "dataset", "space", "bucket"] + labels = {"model": "Models", "dataset": "Datasets", "space": "Spaces", "bucket": "Buckets"} + for rtype in order: + group = [r for r in repos if r.type == rtype] + if not group: + continue + storage = sum(r.storage for r in group) + sq = _colored_square(_TYPE_COLORS[rtype][0]) + lines.append(f" {sq} {labels[rtype]}") + lines.append(f" {len(group)} repos · {format_size(storage, human_readable=True)}") + lines.append("") + + if extra_count > 0: + sq = _colored_square(_EXTRA_COLORS[0]) + lines.append(f" {sq} +{extra_count} more repos") + + return lines + + +# --------------------------------------------------------------------------- +# Cursor +# --------------------------------------------------------------------------- + + +def _build_cursor() -> list[tuple[int, int, Color]]: + pixels: list[tuple[int, int, Color]] = [] + for ri, row in enumerate(_CURSOR_GRID): + for ci, ch in enumerate(row): + if ch in _CURSOR_PALETTE: + pixels.append((ci - len(row) // 2, ri - _CURSOR_H + 1, _CURSOR_PALETTE[ch])) + return pixels + + +_CURSOR_PIXELS = _build_cursor() + + +# --------------------------------------------------------------------------- +# Interactive game +# --------------------------------------------------------------------------- + + +def run_city_game(repos: list[RepoStorageInfo]) -> None: + """Launch the interactive city explorer.""" + if not repos: + print("No repositories found.") + return + + try: + import termios + import tty + except ImportError: + print("Interactive mode requires a Unix-like terminal (Linux/macOS).") + return + + if not sys.stdin.isatty() or not sys.stdout.isatty(): + print("Interactive mode requires a terminal.") + return + + term = shutil.get_terminal_size() + if term.columns < _MIN_TERM_W or term.lines < _MIN_TERM_H: + print(f"Your terminal is {term.columns}×{term.lines} characters.") + print(f"Please resize to at least {_MIN_TERM_W}×{_MIN_TERM_H} to explore the city!") + return + + city = _prepare_city_data(repos) + + tiles_with_repos = [t for t in city.tiles if t.repo is not None] + start_tile = random.choice(tiles_with_repos) if tiles_with_repos else city.tiles[0] + + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + try: + tty.setraw(fd) + sys.stdout.write("\033[?1049h\033[?25l\033[2J") + sys.stdout.flush() + _game_loop(city, start_tile.grid_row, start_tile.grid_col) + finally: + sys.stdout.write("\033[?25h\033[?1049l") + sys.stdout.flush() + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + + +def _game_loop(city: CityData, cur_row: int, cur_col: int) -> None: + tile_map: dict[tuple[int, int], TileInfo] = {(t.grid_row, t.grid_col): t for t in city.tiles} + city = dataclasses.replace(city, buf_h=city.buf_h + _CURSOR_PAD, y_off=city.y_off + _CURSOR_PAD) + base_buf = _render_base_buffer(city) + + summary = _build_summary(city.all_repos, city.total_storage, city.extra_count) + + # Intro: cursor drops onto starting tile + tx, ty = _tile_top_center(city, cur_row, cur_col, tile_map) + for i in range(1, _MOVE_FRAMES + 1): + t = i / _MOVE_FRAMES + t = t * t * (3 - 2 * t) + drop_y = ty - 16 * (1 - t) + frame = _copy_buf(base_buf) + _highlight_tile(frame, city, tile_map[(cur_row, cur_col)]) + _draw_cursor(frame, tx, round(drop_y)) + _present(city, frame, tile_map.get((cur_row, cur_col)), summary) + time.sleep(_MOVE_DELAY) + + while True: + cx, cy = _tile_top_center(city, cur_row, cur_col, tile_map) + frame = _copy_buf(base_buf) + _highlight_tile(frame, city, tile_map[(cur_row, cur_col)]) + _draw_cursor(frame, cx, cy) + _present(city, frame, tile_map.get((cur_row, cur_col)), summary) + + key = _read_key() + if key in ("q", "Q", "esc", "\x03"): + return + + dr, dc = _key_to_direction(key) + if dr == 0 and dc == 0: + continue + + nr, nc = cur_row + dr, cur_col + dc + if (nr, nc) not in tile_map: + continue + + ex, ey = _tile_top_center(city, nr, nc, tile_map) + for i in range(1, _MOVE_FRAMES + 1): + t = i / _MOVE_FRAMES + t = t * t * (3 - 2 * t) + bx = cx + (ex - cx) * t + by = cy + (ey - cy) * t + frame = _copy_buf(base_buf) + _highlight_tile(frame, city, tile_map[(nr, nc)]) + _draw_cursor(frame, round(bx), round(by)) + _present(city, frame, tile_map.get((nr, nc)), summary) + time.sleep(_MOVE_DELAY) + + cur_row, cur_col = nr, nc + + +def _tile_top_center(city: CityData, row: int, col: int, tile_map: dict[tuple[int, int], TileInfo]) -> tuple[int, int]: + tile = tile_map.get((row, col)) + h = tile.height if tile else 1 + cx = city.x_off + (col - row) * _DX + cy = city.y_off + (col + row) * _DY + return cx, cy + _DY - h + + +def _key_to_direction(key: str) -> tuple[int, int]: + match key: + case "w" | "W" | "\x1b[A": + return -1, 0 + case "s" | "S" | "\x1b[B": + return 1, 0 + case "a" | "A" | "\x1b[D": + return 0, -1 + case "d" | "D" | "\x1b[C": + return 0, 1 + case _: + return 0, 0 + + +def _draw_cursor(buf: list[list[Color | None]], cx: int, cy: int) -> None: + bh = len(buf) + bw = len(buf[0]) if buf else 0 + for dx, dy, color in _CURSOR_PIXELS: + px, py = cx + dx, cy + dy + if 0 <= py < bh and 0 <= px < bw: + buf[py][px] = color + + +def _highlight_tile(buf: list[list[Color | None]], city: CityData, tile: TileInfo) -> None: + cx = city.x_off + (tile.grid_col - tile.grid_row) * _DX + cy = city.y_off + (tile.grid_col + tile.grid_row) * _DY + h = tile.height + _fill_poly( + buf, + [(cx, cy - h), (cx + _DX, cy + _DY - h), (cx, cy + 2 * _DY - h), (cx - _DX, cy + _DY - h)], + _brighten(tile.top, 35), + ) + + +def _brighten(color: Color, amount: int) -> Color: + return (min(255, color[0] + amount), min(255, color[1] + amount), min(255, color[2] + amount)) + + +def _present( + city: CityData, + buf: list[list[Color | None]], + tile: TileInfo | None, + summary: list[str], +) -> None: + city_lines = _pixels_to_lines(buf) + while city_lines and not _strip_ansi(city_lines[0]).strip(): + city_lines.pop(0) + while city_lines and not _strip_ansi(city_lines[-1]).strip(): + city_lines.pop() + + city_w = max((_visible_len(line) for line in city_lines), default=0) + term = shutil.get_terminal_size() + panel_max_w = max(20, term.columns - city_w - _SUMMARY_W - 2 * _GAP) + + info = _build_info_panel(tile, city, panel_max_w) + + n = max(len(summary), len(city_lines), len(info)) + summary_lo = max(0, (n - len(summary)) // 2) + info_lo = max(0, (n - len(info)) // 2) + + lines: list[str] = [] + for i in range(n): + si = i - summary_lo + lt = summary[si] if 0 <= si < len(summary) else "" + lpad = max(0, _SUMMARY_W - _visible_len(lt)) + + ct = city_lines[i] if i < len(city_lines) else "" + cpad = max(0, city_w - _visible_len(ct)) + + ri = i - info_lo + rt = info[ri] if 0 <= ri < len(info) else "" + + lines.append(lt + " " * lpad + " " * _GAP + ct + " " * cpad + " " * _GAP + rt) + + lines.append("") + lines.append(" \033[90mWASD/Arrows: move · Q/ESC: quit\033[0m") + + while len(lines) < term.lines - 1: + lines.append("") + + output = "\033[H" + for line in lines[: term.lines - 1]: + output += line + "\033[K\r\n" + sys.stdout.write(output) + sys.stdout.flush() + + +def _build_info_panel(tile: TileInfo | None, city: CityData, max_w: int) -> list[str]: + reset = "\033[0m" + gray = "\033[90m" + bold = "\033[1m" + indent = " " + content_w = max_w - len(indent) + + lines: list[str] = [""] + lines.append(f"{indent}{bold}City Explorer{reset}") + lines.append(indent + "─" * min(22, content_w)) + lines.append("") + + if tile is None: + lines.append(f"{indent}{gray}Move to a tile") + lines.append(f"{indent}to see details.{reset}") + return lines + + if tile.repo is None: + lines.append(f"{indent}{gray}+{city.extra_count} more repos{reset}") + lines.append(f"{indent}{gray}{format_size(city.extra_storage, human_readable=True)} combined{reset}") + return lines + + repo = tile.repo + name = repo.id + if len(name) > content_w: + name = name[: content_w - 3] + "..." + lines.append(f"{indent}{bold}{name}{reset}") + lines.append("") + + type_ansi = { + "model": "\033[38;2;175;148;240m", + "dataset": "\033[38;2;245;128;128m", + "space": "\033[38;2;245;175;85m", + "bucket": "\033[38;2;112;185;242m", + } + tc = type_ansi.get(repo.type, "") + + lines.append(f"{indent}Type {tc}{repo.type}{reset}") + lines.append(f"{indent}Visibility {repo.visibility}") + lines.append(f"{indent}Storage {format_size(repo.storage, human_readable=True)}") + lines.append(f"{indent}Usage {repo.storage_percent:.1f}%") + lines.append("") + + bar_w = min(18, content_w) + filled = max(0, min(bar_w, round(repo.storage_percent / 100 * bar_w))) + lines.append(f"{indent}{tc}{'█' * filled}{gray}{'░' * (bar_w - filled)}{reset}") + + return lines + + +def _copy_buf(buf: list[list[Color | None]]) -> list[list[Color | None]]: + return [row[:] for row in buf] + + +def _read_key() -> str: + fd = sys.stdin.fileno() + ch = os.read(fd, 1) + if ch == b"\x1b": + if _has_input(fd, 0.05): + ch2 = os.read(fd, 1) + if ch2 == b"[" and _has_input(fd, 0.05): + ch3 = os.read(fd, 1) + return f"\x1b[{ch3.decode()}" + return "esc" + return ch.decode("utf-8", errors="replace") + + +def _has_input(fd: int, timeout: float) -> bool: + r, _, _ = select.select([fd], [], [], timeout) + return bool(r) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/_cli_utils.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_cli_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0bb0e25aabb394328d930a9458d126f110d19950 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_cli_utils.py @@ -0,0 +1,1069 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains CLI utilities (styling, helpers).""" + +import difflib +import importlib.metadata +import os +import re +import shlex +import subprocess +import sys +import time +from collections.abc import Callable, Sequence +from enum import Enum +from pathlib import Path +from typing import TYPE_CHECKING, Annotated, Any, Literal, TypeVar, cast + +import click + +from huggingface_hub import Volume, __version__, constants +from huggingface_hub.errors import CLIError +from huggingface_hub.utils import ( + get_session, + hf_raise_for_status, + installation_method, + logging, + parse_hf_mount, +) +from huggingface_hub.utils._dotenv import load_dotenv + +from ._framework import Argument, HfCommand, HfGroup, Option +from ._help_formatter import StyledContext +from ._output import OutputFormat, out + + +logger = logging.get_logger() + +# Arbitrary default limit for models/datasets/spaces list commands. +REPO_LIST_DEFAULT_LIMIT = 30 + +if TYPE_CHECKING: + from huggingface_hub.hf_api import HfApi + + +def get_hf_api(token: str | None = None) -> "HfApi": + # Import here to avoid circular import + from huggingface_hub.hf_api import HfApi + + return HfApi(token=token, library_name="huggingface-cli", library_version=__version__) + + +#### TYPER UTILS + +CLI_REFERENCE_URL = "https://huggingface.co/docs/huggingface_hub/en/guides/cli" + + +def generate_epilog(examples: list[str], docs_anchor: str | None = None) -> str: + """Generate an epilog with examples and a Learn More section. + + Args: + examples: List of example commands (without the `$ ` prefix). + docs_anchor: Optional anchor for the docs URL (e.g., "#hf-download"). + + Returns: + Formatted epilog string. + """ + docs_url = f"{CLI_REFERENCE_URL}{docs_anchor}" if docs_anchor else CLI_REFERENCE_URL + examples_str = "\n".join(f" $ {ex}" for ex in examples) + return f"""\ +Examples +{examples_str} + +Learn more + Use `hf --help` for more information about a command. + Read the documentation at {docs_url} +""" + + +TOPIC_T = Literal["main", "help"] | str +FallbackHandlerT = Callable[[list[str], set[str]], int | None] +ExpandPropertyT = TypeVar("ExpandPropertyT", bound=str) + + +def _format_epilog_no_indent(epilog: str | None, ctx: click.Context, formatter: click.HelpFormatter) -> None: + """Write the epilog without indentation.""" + if epilog: + formatter.write_paragraph() + for line in epilog.split("\n"): + formatter.write_text(line) + + +_ALIAS_SPLIT = re.compile(r"\s*\|\s*") + + +class HFCliTyperGroup(HfGroup): + """ + CLI Group that: + - lists commands alphabetically within sections. + - separates commands by topic (main, help, etc.). + - formats epilog without extra indentation. + - supports aliases via pipe-separated names (e.g. ``name="list | ls"``). + - consumes the global formatting flags (``--format``, ``--json``, ``-q`` / ``--quiet``, ``--no-truncate``) + anywhere in the args of a leaf command and applies them to ``out``, so leaf + commands don't need to declare these options themselves. + - rewrites ``spaces/user/repo`` to ``user/repo --type space`` for commands that accept ``--type``. + - enriches "No such option" / "No such command" errors with available options or commands. + """ + + context_class = StyledContext + + def invoke(self, ctx: click.Context) -> None: + """Enrich unknown-option errors with available options or subcommands. + + Catches `NoSuchOption` raised during subcommand `make_context()` + (option parsing). For leaf commands (e.g. `hf repos create --test`) + we list the command's options; for groups (e.g. `hf cache --test`) + we list subcommands since groups have no user-facing options. + """ + try: + return super().invoke(ctx) + except click.NoSuchOption as e: + if e.ctx is not None and e.ctx.command is not None: + cmd = e.ctx.command + if isinstance(cmd, click.Group): + # Group has no user-facing options -> show subcommands instead + items = [ + (name, sub.get_short_help_str(limit=80)) + for name in cmd.list_commands(e.ctx) + if (sub := cmd.get_command(e.ctx, name)) is not None and not sub.hidden + ] + _enrich_usage_error(e, "commands", items) + else: + # Leaf command -> show its options using Click's rich formatting + items = [ + record + for p in cmd.get_params(e.ctx) + if isinstance(p, click.Option) and not p.hidden and (record := p.get_help_record(e.ctx)) + ] + _enrich_usage_error(e, "options", items) + raise + + def resolve_command(self, ctx: click.Context, args: list[str]) -> tuple: + cmd_name = args[0] if args and not args[0].startswith("-") else None + cmd = self.get_command(ctx, cmd_name) if cmd_name else None + + if cmd is not None: + self._rewrite_repo_type_prefix(cmd, args) + + try: + name, resolved_cmd, sub_args = super().resolve_command(ctx, args) + except click.UsageError as e: + # Unknown subcommand -> add fuzzy suggestions and list available commands. + if cmd is None and cmd_name is not None: + # Expand aliases ("list | ls" → ["list", "ls"]) for accurate fuzzy matching. + visible_names = [ + alias + for key, registered in self.commands.items() + if not registered.hidden + for alias in _ALIAS_SPLIT.split(key) + ] + matches = difflib.get_close_matches(cmd_name, visible_names) + if matches: + suggestions = ", ".join(f"'{m}'" for m in matches) + setattr(e, "message", f"{e.message.rstrip('.')}. Did you mean {suggestions}?") + items = [ + (name, sub.get_short_help_str(limit=80)) + for name in self.list_commands(ctx) + if (sub := self.get_command(ctx, name)) is not None and not sub.hidden + ] + _enrich_usage_error(e, "commands", items) + raise + + # If we just resolved a leaf command, eagerly consume any global formatting + # flags (--format / --json / -q / --quiet / --no-truncate) from its args before click parses + # them. Group resolution is recursive — leaves (and only leaves) need this. + if resolved_cmd is not None and not isinstance(resolved_cmd, click.Group): + _consume_format_flags_for_leaf(resolved_cmd, sub_args) + + return name, resolved_cmd, sub_args + + @staticmethod + def _rewrite_repo_type_prefix(cmd: click.Command, args: list[str]) -> None: + """Rewrite prefixed repo IDs (e.g. ``spaces/user/repo``) to ``user/repo --type space``. + + Only applies to commands that have a ``--type`` / ``--repo-type`` option and + at least one repo-ID positional argument (any ``click.Argument`` whose name + ends with ``_id``, e.g. ``repo_id``, ``from_id``, ``to_id``). When the + token that maps to such an argument matches ``{prefix}/org/repo`` (where + *prefix* is one of ``spaces``, ``datasets``, or ``models``), the prefix is + stripped and an implicit ``--type {type}`` is appended. An error is raised + if ``--type`` is also provided explicitly or if multiple prefixed arguments + disagree on the repo type. + + Only repo-ID positional slots are inspected so that other positional + arguments (filenames, local paths, patterns …) are never misinterpreted as + prefixed repo IDs. + """ + has_type_option = any(isinstance(param, click.Option) and "--type" in param.opts for param in cmd.params) + if not has_type_option: + return + + # Locate all repo-ID positional arguments and their indices among Arguments. + repo_id_positions: set[int] = set() + arg_idx = 0 + for param in cmd.params: + if isinstance(param, click.Argument): + if param.name in ("repo_id", "from_id", "to_id"): + repo_id_positions.add(arg_idx) + arg_idx += 1 + + if not repo_id_positions: + return + + # Build a set of option names that consume a following value token. + value_options: set[str] = set() + for param in cmd.params: + if isinstance(param, click.Option) and not param.is_flag: + for opt in (*param.opts, *param.secondary_opts): + value_options.add(opt) + + # Walk through args (skipping args[0] = command name) to map positional + # slots to their indices in `args`. + positional_count = 0 + repo_id_arg_indices: list[int] = [] + i = 1 + while i < len(args): + arg = args[i] + if arg == "--": + break # everything after -- is positional literal; stop rewriting + if arg.startswith("-"): + if "=" in arg or arg not in value_options: + i += 1 # flag or --opt=val — single token + else: + i += 2 # value-taking option — skip the value too + else: + if positional_count in repo_id_positions: + repo_id_arg_indices.append(i) + positional_count += 1 + i += 1 + + if not repo_id_arg_indices: + return + + # Check each repo-ID arg for a type prefix and collect rewrites. + inferred_type: str | None = None + first_prefix: str | None = None + rewrites: list[tuple[int, str]] = [] # (args index, new value without prefix) + + for arg_index in repo_id_arg_indices: + parts = args[arg_index].split("/", 2) + if len(parts) != 3 or parts[0] not in constants.REPO_TYPES_MAPPING: + continue + prefix = parts[0] + mapped_type = constants.REPO_TYPES_MAPPING[prefix] + if inferred_type is not None and mapped_type != inferred_type: + raise click.UsageError(f"Conflicting repo type prefixes: '{first_prefix}/' and '{prefix}/'.") + inferred_type = mapped_type + first_prefix = prefix + rewrites.append((arg_index, f"{parts[1]}/{parts[2]}")) + + if not rewrites: + return + + # Error if --type / --repo-type was also provided explicitly. + if any( + arg == "--type" or arg.startswith("--type=") or arg == "--repo-type" or arg.startswith("--repo-type=") + for arg in args + ): + raise click.UsageError( + f"Ambiguous repo type: got prefix '{first_prefix}/' in repo ID and explicit --type. Use one or the other." + ) + + # Apply all rewrites and append --type once. + for arg_index, new_value in rewrites: + args[arg_index] = new_value + args.extend(["--type", inferred_type]) # type: ignore + + def get_command(self, ctx: click.Context, cmd_name: str) -> click.Command | None: + # Try exact match first + cmd = super().get_command(ctx, cmd_name) + if cmd is not None: + return cmd + # Fall back to alias lookup: check if cmd_name matches any alias + # taken from https://github.com/fastapi/typer/issues/132#issuecomment-2417492805 + for registered_name, registered_cmd in self.commands.items(): + aliases = _ALIAS_SPLIT.split(registered_name) + if cmd_name in aliases: + return registered_cmd + return None + + def _alias_map(self) -> dict[str, list[str]]: + """Build a mapping from primary command name to its aliases (if any).""" + result: dict[str, list[str]] = {} + for registered_name in self.commands: + parts = _ALIAS_SPLIT.split(registered_name) + primary = parts[0] + result[primary] = parts[1:] + return result + + def format_commands(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: + topics: dict[str, list] = {} + alias_map = self._alias_map() + + for name in self.list_commands(ctx): + cmd = self.get_command(ctx, name) + if cmd is None or cmd.hidden: + continue + help_text = cmd.get_short_help_str(limit=formatter.width) + aliases = alias_map.get(name, []) + if aliases: + help_text = f"{help_text} [alias: {', '.join(aliases)}]" + topic = getattr(cmd, "topic", "main") + topics.setdefault(topic, []).append((name, help_text)) + + with formatter.section("Main commands"): + formatter.write_dl(topics["main"]) + for topic in sorted(topics.keys()): + if topic == "main": + continue + with formatter.section(f"{topic.capitalize()} commands"): + formatter.write_dl(topics[topic]) + + def format_epilog(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: + # Collect only the first example from each command (to keep group help concise) + # Full examples are shown in individual subcommand help (e.g. `hf buckets sync --help`) + all_examples: list[str] = [] + for name in self.list_commands(ctx): + cmd = self.get_command(ctx, name) + if cmd is None or cmd.hidden: + continue + cmd_examples = getattr(cmd, "examples", []) + if cmd_examples: + all_examples.append(cmd_examples[0]) + + if all_examples: + epilog = generate_epilog(all_examples) + _format_epilog_no_indent(epilog, ctx, formatter) + elif self.epilog: + _format_epilog_no_indent(self.epilog, ctx, formatter) + + def list_commands(self, ctx: click.Context) -> list[str]: # type: ignore[name-defined] + # For aliased commands ("list | ls"), use the primary name (first entry). + primary_names: list[str] = [] + for name in self.commands: + primary = _ALIAS_SPLIT.split(name)[0] + primary_names.append(primary) + return sorted(primary_names) + + def command( # type: ignore # adds topic/examples on top of HfGroup.command + self, + name: str | None = None, + *, + topic: TOPIC_T = "main", + examples: list[str] | None = None, + epilog: str | None = None, + **kwargs: Any, + ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + # Generate the epilog from examples when not provided explicitly, then build the + # command with a topic/examples-aware command class. + if epilog is None and examples: + epilog = generate_epilog(examples) + return super().command(name, cls=HFCliCommand(topic, examples), epilog=epilog, **kwargs) + + +_FORMATTING_OPTIONS_HELP_RECORDS: list[tuple[str, str]] = [ + ( + "--format [auto|human|agent|json|quiet]", + "Output format. Defaults to 'auto' which picks 'agent' or 'human' based on the terminal.", + ), + ("--json", "JSON output. Equivalent to '--format json'."), + ("-q, --quiet", "Quiet output (one ID per line). Equivalent to '--format quiet'."), + ("--no-truncate", "Do not truncate scalar values in human tables (list/dict columns stay shortened)."), +] + + +def _format_formatting_options_section(formatter: click.HelpFormatter) -> None: + with formatter.section("Formatting options"): + formatter.write_dl(_FORMATTING_OPTIONS_HELP_RECORDS) + + +def _has_local_formatting_option(cmd: click.Command) -> bool: + """Return True if the command defines its own --format, --json or --quiet / -q. + + Used to skip the global formatting flag pre-processor and the duplicated "Formatting options" help section for + legacy commands like 'hf jobs ls' that have their own format/quiet options. + """ + for param in cmd.params: + if not isinstance(param, click.Option): + continue + opts = (*param.opts, *param.secondary_opts) + if "--format" in opts or "--json" in opts or "--quiet" in opts or "-q" in opts: + return True + return False + + +def _consume_format_flags_for_leaf(cmd: click.Command, args: list[str]) -> None: + """Apply global formatting flags from 'args' to a leaf command. + + Two modes, depending on the command: + + * **Pass-through commands** (ignore_unknown_options=True, e.g. 'hf extensions exec'): + args are forwarded verbatim to an external binary; we don't touch them. + + * **Legacy commands with a local --format option** (e.g. 'hf jobs ls' whose '--format' accepts Go templates): + the global flags are rewritten in-place to the legacy form ('--json' → '--format json', '--quiet'/'-q' → '--format quiet' + when the cmd has no own '--quiet') so click can parse them locally. This preserves backwards compatibility with the previous shorthand behavior. + + * **Modern commands** (no local format/quiet/json options): the flags '--format ' / '--json' / '--quiet' / '-q' are stripped from 'args' and applied to the singleton 'out'. + + '--no-truncate' is stripped for all non-pass-through commands; when present, human table cells are not truncated. + + Raises click.UsageError if multiple conflicting flags are supplied (e.g. '--json' together with '--format table'). + """ + if cmd.context_settings.get("ignore_unknown_options"): + return + + no_truncate = _consume_no_truncate_flags(args) + out.set_no_truncate(no_truncate) + + has_local_format = False + has_local_quiet = False + has_local_json = False + for param in cmd.params: + if not isinstance(param, click.Option): + continue + opts = (*param.opts, *param.secondary_opts) + if "--format" in opts: + has_local_format = True + if "--quiet" in opts or "-q" in opts: + has_local_quiet = True + if "--json" in opts: + has_local_json = True + + if has_local_format: + _rewrite_legacy_shorthands(args, rewrite_json=not has_local_json, rewrite_quiet=not has_local_quiet) + return + + # Strip --format/--json/-q/--quiet from 'args' and apply to 'out' + chosen_mode: OutputFormat = OutputFormat.auto + chosen_flag: str | None = None + + def _check_conflict(new_flag: str) -> None: + # Reject any second formatting flag before parsing values, so the user gets + # a "mutually exclusive" error rather than e.g. an "invalid value" error + # from the second flag's argument. + if chosen_flag is not None: + raise click.UsageError(f"'{chosen_flag}' and '{new_flag}' are mutually exclusive.") + + i = 0 + while i < len(args): + arg = args[i] + if arg == "--": + break # everything after '--' is a positional literal + if arg == "--format": + _check_conflict("--format") + if i + 1 >= len(args): + raise click.UsageError("Option '--format' requires a value.") + chosen_mode = _parse_format_value(args[i + 1]) + chosen_flag = "--format" + del args[i : i + 2] # --format value => 2 args removed + continue + if arg.startswith("--format="): + _check_conflict("--format") + chosen_mode = _parse_format_value(arg[len("--format=") :]) + chosen_flag = "--format" + del args[i : i + 1] + continue + if arg == "--json": + _check_conflict("--json") + chosen_mode = OutputFormat.json + chosen_flag = "--json" + del args[i : i + 1] + continue + if arg in ("-q", "--quiet"): + _check_conflict(arg) + chosen_mode = OutputFormat.quiet + chosen_flag = arg + del args[i : i + 1] + continue + i += 1 + + out.set_mode(chosen_mode) + + +def _consume_no_truncate_flags(args: list[str]) -> bool: + """Strip all global --no-truncate flags from args and return whether any was provided.""" + no_truncate = False + i = 0 + while i < len(args): + arg = args[i] + if arg == "--": + break # everything after '--' is a positional literal + if arg == "--no-truncate": + no_truncate = True + del args[i : i + 1] + continue + if arg.startswith("--no-truncate="): + raise click.UsageError("Option '--no-truncate' does not take a value.") + i += 1 + return no_truncate + + +def _rewrite_legacy_shorthands(args: list[str], *, rewrite_json: bool, rewrite_quiet: bool) -> None: + """Rewrite --json / -q / --quiet to --format ... for legacy commands. + + Used for commands like 'hf jobs ls' that still own their '--format' option. + The rewrite lets users keep using the global shorthand while click parses + '--format ' locally. + """ + has_format_in_args = any(arg == "--format" or arg.startswith("--format=") for arg in args) + + if rewrite_json and "--json" in args: + if has_format_in_args: + raise click.UsageError("'--json' and '--format' are mutually exclusive.") + idx = args.index("--json") + args[idx : idx + 1] = ["--format", "json"] + has_format_in_args = True + + if rewrite_quiet: + flag = "-q" if "-q" in args else ("--quiet" if "--quiet" in args else None) + if flag is not None: + if has_format_in_args: + raise click.UsageError(f"'{flag}' and '--format' are mutually exclusive.") + idx = args.index(flag) + args[idx : idx + 1] = ["--format", "quiet"] + + +def _parse_format_value(value: str) -> "OutputFormat": + try: + return OutputFormat(value) + except ValueError: + valid = ", ".join(m.value for m in OutputFormat) + raise click.UsageError(f"Invalid value for '--format': '{value}'. Valid values: {valid}.") from None + + +def _enrich_usage_error(error: click.UsageError, label: str, items: list[tuple[str, str]]) -> None: + """Append a list of available options or commands to a usage error message.""" + if not items or error.ctx is None or f"Available {label} for" in error.message: + return + cmd_path = error.ctx.command_path + lines = [f"\n\nAvailable {label} for '{cmd_path}':"] + for name, help_text in items: + lines.append(f" {name:30s} {help_text}") + lines.append(f"\nRun '{cmd_path} --help' for full details.") + if isinstance(error, click.NoSuchOption) and error.possibilities: + lines.append(f"\nDid you mean: {', '.join(sorted(error.possibilities))}?") + setattr(error, "possibilities", []) + setattr(error, "message", error.message + "\n".join(lines)) + + +def fallback_typer_group_factory( + fallback_handler: FallbackHandlerT, + extra_commands_provider: Callable[[], list[tuple[str, str]]] | None = None, +) -> type[HFCliTyperGroup]: + """Return a Typer group class that runs a fallback handler before command resolution.""" + + class FallbackTyperGroup(HFCliTyperGroup): + def resolve_command(self, ctx: click.Context, args: list[str]) -> tuple: + fallback_exit_code = fallback_handler(args, set(self.commands.keys())) + if fallback_exit_code is not None: + raise SystemExit(fallback_exit_code) + return super().resolve_command(ctx, args) + + def format_commands(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: + super().format_commands(ctx, formatter) + if extra_commands_provider is not None: + entries = extra_commands_provider() + if entries: + with formatter.section("Extension commands"): + formatter.write_dl(entries) + + return FallbackTyperGroup + + +def HFCliCommand(topic: TOPIC_T, examples: list[str] | None = None) -> type[HfCommand]: + def format_epilog(self: click.Command, ctx: click.Context, formatter: click.HelpFormatter) -> None: + _format_epilog_no_indent(self.epilog, ctx, formatter) + + def format_options(self: HfCommand, ctx: click.Context, formatter: click.HelpFormatter) -> None: + HfCommand.format_options(self, ctx, formatter) + # Skip the section for commands that define their own --format / --quiet / --json, + # or for pass-through commands that forward args to an external binary. + if _has_local_formatting_option(self): + return + if self.context_settings.get("ignore_unknown_options"): + return + _format_formatting_options_section(formatter) + + def parse_args(self: click.Command, ctx: click.Context, args: list[str]) -> list[str]: + # Show help when a command with required arguments is invoked without any args + # (mirrors group behavior: `hf jobs` prints help, so `hf download` should too). + if not args and not ctx.resilient_parsing: + if any(isinstance(p, click.Argument) and p.required for p in self.params): + click.echo(ctx.get_help(), color=ctx.color) + ctx.exit() + return HfCommand.parse_args(self, ctx, args) + + return type( + f"HfCommand{topic.capitalize()}", + (HfCommand,), + { + "context_class": StyledContext, + "topic": topic, + "examples": examples or [], + "format_epilog": format_epilog, + "format_options": format_options, + "parse_args": parse_args, + }, + ) + + +def typer_factory(help: str, epilog: str | None = None, cls: type[HFCliTyperGroup] | None = None) -> "HFCliTyperGroup": + """Create a CLI command group with consistent settings. + + The returned group is the app: register commands with ``@group.command(...)``, + subgroups with ``group.add_group(sub, name=...)``, and a group-level callback + with ``@group.callback(...)``. + + Args: + help: Help text for the group. + epilog: Optional epilog text (use `generate_epilog` to create one). + cls: Optional group class to use (defaults to `HFCliTyperGroup`). + + Returns: + A configured `HFCliTyperGroup` instance. + """ + if cls is None: + cls = HFCliTyperGroup + return cls( + help=help, + epilog=epilog, + no_args_is_help=True, + # Increase max content width for better readability + context_settings={ + "max_content_width": 120, + "help_option_names": ["-h", "--help"], + }, + ) + + +class SoftChoice(click.Choice): + """A click Choice that suggests choices for autocompletion/docs but accepts any string. + + Unlike `click.Choice`, unknown values are passed through as-is instead of raising an error. + This makes CLI options future-compatible when new server-side values are added. + + Accepts either a sequence of strings or an Enum class: + ```python + SoftChoice(SpaceHardware) # from an enum + SoftChoice(["a", "b", "c"]) # from a list + ``` + """ + + def __init__(self, choices: Sequence[str] | type[Enum]) -> None: + values = ( + [m.value for m in choices] if isinstance(choices, type) and issubclass(choices, Enum) else list(choices) + ) + super().__init__(values, case_sensitive=True) + + def convert(self, value: Any, param: click.Parameter | None, ctx: click.Context | None) -> str: + try: + return super().convert(value, param, ctx) + except click.exceptions.BadParameter: + return str(value) + + +class RepoType(str, Enum): + model = "model" + dataset = "dataset" + space = "space" + + +RepoIdArg = Annotated[ + str, + Argument( + help="The ID of the repo (e.g. `username/repo-name` or `spaces/username/repo-name`).", + ), +] + + +RepoTypeOpt = Annotated[ + RepoType, + Option( + "--type", + "--repo-type", + help="The type of repository (model, dataset, or space).", + ), +] + +# Same as `RepoTypeOpt` but optional (defaults to `None` rather than `model`). Used by commands that +# accept an `hf://` URI as repo id: a `None` default lets us tell apart "user did not pass --repo-type" +# from "user explicitly passed --repo-type model", which is required to detect conflicts with the URI. +RepoTypeOptionalOpt = Annotated[ + RepoType | None, + Option( + "--type", + "--repo-type", + help="The type of repository (model, dataset, or space).", + show_default="model", + ), +] + +TokenOpt = Annotated[ + str | None, + Option( + help="A User Access Token generated from https://huggingface.co/settings/tokens.", + ), +] + +PrivateOpt = Annotated[ + bool | None, + Option( + help="Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists.", + ), +] + +RevisionOpt = Annotated[ + str | None, + Option( + help="Git revision id which can be a branch name, a tag, or a commit hash.", + ), +] + + +LimitOpt = Annotated[ + int, + Option(help="Limit the number of results."), +] + +AuthorOpt = Annotated[ + str | None, + Option(help="Filter by author or organization."), +] + +FilterOpt = Annotated[ + list[str] | None, + Option(help="Filter by tags (e.g. 'text-classification'). Can be used multiple times."), +] + +SearchOpt = Annotated[ + str | None, + Option(help="Search query."), +] + + +# --- Env / Secrets shared options and parsing helpers (used by jobs, repos, etc.) --- + +EnvOpt = Annotated[ + list[str] | None, + Option( + "-e", + "--env", + help="Set environment variables. E.g. --env ENV=value", + ), +] + +SecretsOpt = Annotated[ + list[str] | None, + Option( + "-s", + "--secrets", + help=( + "Set secret environment variables. E.g. --secrets SECRET=value" + " or `--secrets HF_TOKEN` to pass your Hugging Face token." + ), + ), +] + +EnvFileOpt = Annotated[ + str | None, + Option( + "--env-file", + help="Read in a file of environment variables.", + ), +] + +SecretsFileOpt = Annotated[ + str | None, + Option( + help="Read in a file of secret environment variables.", + ), +] + + +def _get_extended_environ() -> dict[str, str]: + """Return a copy of ``os.environ`` with the user's HF token injected (if available).""" + from huggingface_hub import get_token + + extended_environ = os.environ.copy() + if (token := get_token()) is not None: + extended_environ["HF_TOKEN"] = token + return extended_environ + + +def parse_env_map( + env: list[str] | None = None, + env_file: str | None = None, +) -> dict[str, str | None]: + """Parse ``-e``/``--env``/``-s``/``--secrets`` and ``--env-file``/``--secrets-file`` CLI args into a dict. + + Uses an extended environment that includes the user's HF token so that + bare ``--secrets HF_TOKEN`` resolves correctly. + """ + extended_environ = _get_extended_environ() + env_map: dict[str, str | None] = {} + if env_file: + env_map.update(load_dotenv(Path(env_file).read_text(), environ=extended_environ)) + for env_value in env or []: + env_map.update(load_dotenv(env_value, environ=extended_environ)) + return env_map + + +def env_map_to_key_value_list(env_map: dict[str, str | None]) -> list[dict[str, str]] | None: + """Convert an env/secrets dict to the ``[{"key": ..., "value": ...}]`` format used by the Hub API.""" + if not env_map: + return None + return [{"key": k, "value": v or ""} for k, v in env_map.items()] + + +VolumesOpt = Annotated[ + list[str] | None, + Option( + "-v", + "--volume", + help="Mount one or more volumes. Format: hf://[TYPE/]SOURCE:/MOUNT_PATH[:ro]. " + "TYPE is one of: models, datasets, spaces, buckets. " + "TYPE defaults to models if omitted. " + "models, datasets and spaces are always mounted read-only. buckets are read+write by default. " + "E.g. -v hf://org/m:/data or -v hf://datasets/org/ds:/data or -v hf://buckets/org/b:/mnt:ro", + ), +] + + +def parse_volumes(volumes: list[str] | None) -> "list[Volume] | None": + """Parse volume specs from CLI arguments. + + Format: hf://[TYPE/]SOURCE[/PATH]:/MOUNT_PATH[:ro|:rw] + Where TYPE is one of: models, datasets, spaces, buckets (defaults to models if omitted). + SOURCE is the repo/bucket identifier (e.g. 'username/my-model'). + PATH is an optional subfolder inside the repo/bucket. + MOUNT_PATH starts with '/'. + Optional ':ro' or ':rw' suffix for read-only or read-write. + + Examples: + hf://my-org/my-model:/data (model, implicit type) + hf://models/my-org/my-model:/data (model, explicit type) + hf://datasets/my-org/my-dataset:/data:ro + hf://buckets/my-org/my-bucket:/mnt + hf://spaces/my-org/my-space:/app + hf://datasets/org/ds/train:/data (with path inside repo) + hf://buckets/org/b/sub/dir:/mnt (with path inside bucket) + """ + if not volumes: + return None + + result: list[Volume] = [] + for raw_spec in volumes: + mount = parse_hf_mount(raw_spec) + result.append( + Volume( + type=mount.source.type, + source=mount.source.id, + mount_path=mount.mount_path, + read_only=mount.read_only, + path=mount.source.path_in_repo or None, + revision=mount.source.revision or None, + ) + ) + return result + + +def make_expand_properties_parser(valid_properties: Sequence[ExpandPropertyT]): + """Create a callback to parse and validate comma-separated expand properties.""" + + def _parse_expand_properties(value: str | None) -> list[ExpandPropertyT] | None: + if value is None: + return None + properties = [p.strip() for p in value.split(",")] + for prop in properties: + if prop not in valid_properties: + raise click.BadParameter( + f"Invalid expand property: '{prop}'. Valid values are: {', '.join(valid_properties)}" + ) + return [cast(ExpandPropertyT, prop) for prop in properties] + + return _parse_expand_properties + + +### SSH + + +SshIdentityFileOpt = Annotated[ + Path | None, + Option("-i", "--identity-file", help="Path to the SSH identity file (forwarded to `ssh -i`)."), +] + +SshDryRunOpt = Annotated[ + bool, + Option("--dry-run", help="Print the SSH command instead of running it."), +] + + +def exec_ssh( + destination: str, *, port: int | None = None, identity_file: Path | None = None, dry_run: bool = False +) -> None: + """Run an interactive `ssh` command to `destination` (`user@host`) and exit with its return code. + + With `dry_run`, print the command instead of running it. + """ + cmd = ["ssh"] + if identity_file is not None: + cmd += ["-i", str(identity_file)] + if port is not None: + cmd += ["-p", str(port)] + cmd.append(destination) + if dry_run: + out.text(shlex.join(cmd)) + return + out.text(f"Running `{shlex.join(cmd)}`") + result = subprocess.run(cmd) + raise click.exceptions.Exit(code=result.returncode) + + +### PyPI VERSION CHECKER + + +def check_cli_update(library: Literal["huggingface_hub", "transformers"]) -> None: + """ + Check whether a newer version of a library is available on PyPI. + + If a newer version is found, print a hint pointing at `hf update`. + + If current version is a pre-release (e.g. `1.0.0.rc1`), or a dev version (e.g. `1.0.0.dev1`), no check is performed. + If `HF_HUB_DISABLE_UPDATE_CHECK` is set, the check is skipped entirely. + + This function is called at the entry point of the CLI. It only performs the check once every 24 hours, and any error + during the check is caught and logged, to avoid breaking the CLI. + + Args: + library: The library to check for updates. Currently supports "huggingface_hub" and "transformers". + """ + try: + _check_cli_update(library) + except Exception: + # We don't want the CLI to fail on version checks, no matter the reason. + logger.debug("Error while checking for CLI update.", exc_info=True) + + +def _check_cli_update(library: Literal["huggingface_hub", "transformers"]) -> None: + if constants.HF_HUB_DISABLE_UPDATE_CHECK: + return + + current_version = importlib.metadata.version(library) + + # Skip if current version is a pre-release or dev version + if any(tag in current_version for tag in ["rc", "dev"]): + return + + # Skip if already checked in the last 24 hours + if os.path.exists(constants.CHECK_FOR_UPDATE_DONE_PATH): + mtime = os.path.getmtime(constants.CHECK_FOR_UPDATE_DONE_PATH) + if (time.time() - mtime) < 24 * 3600: + return + + # Touch the file to mark that we did the check now + Path(constants.CHECK_FOR_UPDATE_DONE_PATH).parent.mkdir(parents=True, exist_ok=True) + Path(constants.CHECK_FOR_UPDATE_DONE_PATH).touch() + + # Check latest version from the appropriate registry + if library == "huggingface_hub" and installation_method() == "brew": + latest_version = _fetch_latest_brew_version() + else: + latest_version = _fetch_latest_pypi_version(library) + if latest_version is None or current_version == latest_version: + return + + if library == "huggingface_hub": + update_command = _get_huggingface_hub_update_command() + else: + update_command = _get_transformers_update_command() + + message = f"A new version of {library} ({latest_version}) is available! You are using version {current_version}." + if update_command is not None: + match library: + case "huggingface_hub": + message += "\nTo update, run: hf update" + case _: + message += f"\nTo update, run: {' '.join(update_command)}" + out.hint(message) + + +def _fetch_latest_pypi_version(library: str) -> str | None: + """Fetch the latest version of a library from PyPI. Returns None if the request fails.""" + try: + response = get_session().get(f"https://pypi.org/pypi/{library}/json", timeout=2) + hf_raise_for_status(response) + return response.json()["info"]["version"] + except Exception: + logger.debug("Error while fetching latest version from PyPI.", exc_info=True) + return None + + +def _fetch_latest_brew_version() -> str | None: + """Fetch the latest version of the `hf` formula from the Homebrew registry. Returns None if the request fails.""" + try: + response = get_session().get("https://formulae.brew.sh/api/formula/hf.json", timeout=2) + hf_raise_for_status(response) + return response.json()["versions"]["stable"] + except Exception: + logger.debug("Error while fetching latest version from Homebrew.", exc_info=True) + return None + + +def run_update() -> int: + """Run the install-method-appropriate update command for the `hf` CLI. + + Raises CLIError if the installation method can't be determined. + Returns the subprocess exit code on success/failure of the update itself. + """ + cmd = _get_huggingface_hub_update_command() + if cmd is None: + raise CLIError( + "Cannot determine how to update huggingface_hub (unknown installation method). Please update manually." + ) + return subprocess.call(cmd) + + +def _get_huggingface_hub_update_command() -> list[str] | None: + """Return the command to update huggingface_hub as an argv list, or None if the installation method is unknown.""" + match installation_method(): + case "brew": + return ["brew", "upgrade", "hf"] + case "hf_installer" if os.name == "nt": + return ["powershell", "-NoProfile", "-Command", "iwr -useb https://hf.co/cli/install.ps1 | iex"] + case "hf_installer": + return ["bash", "-c", "curl -LsSf https://hf.co/cli/install.sh | bash -"] + case "pip": + return [sys.executable, "-m", "pip", "install", "-U", "huggingface_hub"] + case _: + return None + + +def _get_transformers_update_command() -> list[str] | None: + """Return the command to update transformers as an argv list, or None if the installation method is unknown.""" + match installation_method(): + case "hf_installer" if os.name == "nt": + return [ + "powershell", + "-NoProfile", + "-Command", + "iwr -useb https://hf.co/cli/install.ps1 | iex -WithTransformers", + ] + case "hf_installer": + return ["bash", "-c", "curl -LsSf https://hf.co/cli/install.sh | bash -s -- --with-transformers"] + case "pip": + return [sys.executable, "-m", "pip", "install", "-U", "transformers"] + case _: + return None diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/_completion.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_completion.py new file mode 100644 index 0000000000000000000000000000000000000000..cbbb3d1c0ab2e0835eca7692f31644731094fe15 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_completion.py @@ -0,0 +1,105 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shell completion for the ``hf`` CLI, built on Click's native completion. + +Click generates completion scripts activated by the ``_HF_COMPLETE`` env var (this +works out of the box because ``hf`` is a Click command). This module exposes the two +conveniences Typer used to provide — ``--install-completion`` / ``--show-completion`` +— as thin eager options over that machinery. +""" + +import os +from pathlib import Path +from typing import Annotated + +import click +from click.shell_completion import get_completion_class + +from ._framework import Option + + +_COMPLETE_VAR = "_HF_COMPLETE" + +# Shells with a shared rc file: the line that activates completion (appended once). +_RC_ACTIVATION: dict[str, tuple[Path, str]] = { + "bash": (Path.home() / ".bashrc", f'eval "$({_COMPLETE_VAR}=bash_source hf)"'), + "zsh": (Path.home() / ".zshrc", f'eval "$({_COMPLETE_VAR}=zsh_source hf)"'), +} +# Fish auto-loads per-command files from its completions directory. The file is dedicated +# to `hf`, so the full script is written there directly (overwriting any stale version) +# instead of re-generating it via `hf` on every shell startup. +_FISH_COMPLETION_PATH = Path.home() / ".config" / "fish" / "completions" / "hf.fish" + + +def _detect_shell() -> str: + return Path(os.environ.get("SHELL", "")).name or "bash" + + +def _completion_script(shell: str) -> str: + # Imported lazily to avoid a circular import (hf.py imports the options below). + from .hf import app + + completion_cls = get_completion_class(shell) + if completion_cls is None: + raise click.ClickException(f"Shell '{shell}' is not supported for completion.") + return completion_cls(app, {}, "hf", _COMPLETE_VAR).source() + + +def _show_completion(value: bool) -> None: + if not value: + return + click.echo(_completion_script(_detect_shell())) + raise click.exceptions.Exit() + + +def _install_completion(value: bool) -> None: + if not value: + return + shell = _detect_shell() + if shell == "fish": + path = _FISH_COMPLETION_PATH + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(_completion_script("fish")) + elif shell in _RC_ACTIVATION: + path, activation = _RC_ACTIVATION[shell] + path.parent.mkdir(parents=True, exist_ok=True) + existing = path.read_text() if path.exists() else "" + if activation not in existing: + with path.open("a") as file: + file.write(f"\n{activation}\n") + else: + raise click.ClickException(f"Shell '{shell}' is not supported for completion.") + click.echo(f"{shell} completion installed in {path}. Restart your shell for it to take effect.") + raise click.exceptions.Exit() + + +InstallCompletionOpt = Annotated[ + bool, + Option( + "--install-completion", + callback=_install_completion, + is_eager=True, + help="Install completion for the current shell.", + ), +] + +ShowCompletionOpt = Annotated[ + bool, + Option( + "--show-completion", + callback=_show_completion, + is_eager=True, + help="Show completion for the current shell, to copy it or customize the installation.", + ), +] diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/_cp.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_cp.py new file mode 100644 index 0000000000000000000000000000000000000000..e163cc6596188e052ae177bebaac1cf6786669fb --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_cp.py @@ -0,0 +1,253 @@ +# Copyright 2026-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shared ``cp`` command to copy files between local paths, repositories and buckets. + +This single command backs three identical CLI entry points: ``hf cp`` (top-level), +``hf repos cp`` and ``hf buckets cp``. It supports any source/destination combination +of local file, repo/bucket ``hf://`` URI, and ``-`` (stdin/stdout), with two exceptions: +- bucket-to-repo copies are not supported (server limitation), and +- local-to-local copies (use a regular ``cp`` for that). +""" + +import os +import sys +from dataclasses import replace +from typing import Annotated, Literal + +import click + +from huggingface_hub import HfApi +from huggingface_hub.errors import CLIError +from huggingface_hub.utils import HfUri, SoftTemporaryDirectory, disable_progress_bars, is_hf_uri, parse_hf_uri + +from ._cli_utils import TokenOpt, get_hf_api +from ._framework import Argument +from ._output import out + + +CP_EXAMPLES = [ + # Download (repo or bucket -> local / stdout) + "hf cp hf://username/my-model/config.json", + "hf cp hf://username/my-model/config.json ./config.json", + "hf cp hf://datasets/username/my-dataset/data.csv ./data/", + "hf cp hf://buckets/username/my-bucket/config.json -", + # Upload (local / stdin -> repo or bucket) + "hf cp ./model.safetensors hf://username/my-model/model.safetensors", + "hf cp ./config.json hf://buckets/username/my-bucket/logs/", + "hf cp - hf://buckets/username/my-bucket/config.json", + # Remote to remote (repo/bucket -> repo/bucket, server-side when possible) + "hf cp hf://username/source-model/ hf://username/dest-model/", + "hf cp hf://datasets/username/my-dataset/processed/ hf://buckets/username/my-bucket/processed/", + "hf cp hf://buckets/username/my-bucket/logs/ hf://buckets/username/archive-bucket/ # copies contents only", +] + + +# Which alias registered the command, used to restrict the remote endpoint type (see `_enforce_context`). +CpContext = Literal["repos", "buckets"] + + +def make_cp(context: CpContext | None = None): + """Build the ``cp`` command function for a given alias. + + The three entry points (`hf cp`, `hf repos cp`, `hf buckets cp`) share the exact same logic; + 'context' only adds a guardrail on the remote endpoint type (see `_enforce_context`). + """ + + def cp( + src: Annotated[ + str, + Argument(help="Source: local file, hf:// URI (repo or bucket), or - for stdin."), + ], + dst: Annotated[ + str | None, + Argument(help="Destination: local path, hf:// URI (repo or bucket), or - for stdout."), + ] = None, + token: TokenOpt = None, + ) -> None: + """Copy files between local paths, repositories, and buckets. + + Handles uploads (local/stdin -> repo/bucket), downloads (repo/bucket -> local/stdout) and + remote-to-remote copies (repo/bucket -> repo/bucket). Bucket-to-repo and local-to-local + copies are not supported. For directories, use `hf upload`/`hf download` (repos) or + `hf buckets sync` (buckets). Remote-to-remote copies only work within the same storage + region (https://huggingface.co/docs/hub/storage-regions). + """ + _enforce_context(context, src, dst) + _run_cp(src, dst, token) + + return cp + + +def _enforce_context(context: CpContext | None, src: str, dst: str | None) -> None: + """Guardrail for the `hf repos cp` / `hf buckets cp` aliases. + + These aliases are exact duplicates of `hf cp`, so a bare `hf repos cp` could otherwise touch a + bucket (and vice versa). We validate the type of the remote side: the destination for uploads and + remote-to-remote copies, or the source when downloading to a local path / stdout. The top-level + `hf cp` (i.e. 'context' is None) accepts any combination. + """ + if context is None: + return + # The remote endpoint is the destination when it is an hf:// URI, otherwise the source (download). + remote = dst if (dst is not None and is_hf_uri(dst)) else src + if not is_hf_uri(remote): + return + if context == "repos" and parse_hf_uri(remote).is_bucket: + raise CLIError("`hf repos cp` only works with repositories. Use `hf cp` or `hf buckets cp` for buckets.") + if context == "buckets" and not parse_hf_uri(remote).is_bucket: + raise CLIError("`hf buckets cp` only works with buckets. Use `hf cp` or `hf repos cp` for repositories.") + + +def _run_cp(src: str, dst: str | None, token: str | None) -> None: + api = get_hf_api(token=token) + + src_is_stdin = src == "-" + dst_is_stdout = dst == "-" + src_is_hf = is_hf_uri(src) + dst_is_hf = dst is not None and is_hf_uri(dst) + + # --- Remote to remote: delegate to copy_files (repo/bucket -> repo/bucket) --- + if src_is_hf and dst_is_hf: + assert dst is not None # guaranteed by dst_is_hf + api.copy_files(src, dst) + out.result("Copied", src=src, dst=dst) + return + + # --- At least one side must be a remote hf:// URI (rules out local->local, stdin->local, etc.) --- + if not src_is_hf and not dst_is_hf: + if dst is None: + raise click.BadParameter("Missing destination. Provide a repo or bucket hf:// URI as DST.") + raise click.BadParameter( + "One of SRC or DST must be a repo (hf://username/...) or bucket (hf://buckets/...) URI." + ) + + # --- Download: repo/bucket -> local file or stdout --- + if src_is_hf: + if dst_is_stdout: + _download_file_to_stdout(api, src) + return + _download_file_to_local(api, src, dst) + return + + # --- Upload: local file or stdin -> repo/bucket --- + assert dst is not None # guaranteed: reaching here means dst_is_hf is True + _upload_file_to_remote(api, src, dst, src_is_stdin=src_is_stdin) + + +def _download_file_to_stdout(api: HfApi, src: str) -> None: + uri = parse_hf_uri(src) + filename = _source_filename(uri, src) + # Suppress progress bars to avoid polluting the piped output. + with disable_progress_bars(): + with SoftTemporaryDirectory() as tmp_dir: + tmp_path = os.path.join(tmp_dir, filename) + _download_single(api, uri, tmp_path) + with open(tmp_path, "rb") as f: + while chunk := f.read(32_000_000): # 32MB chunks + sys.stdout.buffer.write(chunk) + + +def _download_file_to_local(api: HfApi, src: str, dst: str | None) -> None: + uri = parse_hf_uri(src) + filename = _source_filename(uri, src) + + if dst is None: + local_path = filename + elif os.path.isdir(dst) or dst.endswith(os.sep) or dst.endswith("/"): + local_path = os.path.join(dst, filename) + else: + local_path = dst + + parent_dir = os.path.dirname(local_path) + if parent_dir: + os.makedirs(parent_dir, exist_ok=True) + + _download_single(api, uri, local_path) + out.result("Downloaded", src=src, dst=local_path) + + +def _download_single(api: HfApi, uri: HfUri, local_path: str) -> None: + """Download a single file (repo or bucket) to ``local_path``. + + Used by `_download_file_to_local` and `_download_file_to_stdout`. + """ + if uri.is_bucket: + api.download_bucket_files(uri.id, [(uri.path_in_repo, local_path)], raise_on_missing_files=True) + else: + # Download into a temporary folder next to the destination (rather than the shared cache) + # so the final move stays on the same filesystem and is instant. The temp folder is + # cleaned up automatically once the move is complete. + parent_dir = os.path.dirname(local_path) or "." + with SoftTemporaryDirectory(prefix=".tmp", dir=parent_dir) as tmp_dir: + downloaded_path = api.hf_hub_download( + repo_id=uri.id, + repo_type=uri.type, + filename=uri.path_in_repo, + revision=uri.revision, + local_dir=tmp_dir, + ) + os.replace(downloaded_path, local_path) + + +def _source_filename(uri: HfUri, src: str) -> str: + if uri.path_in_repo == "" or src.endswith("/"): + raise click.BadParameter( + "Source path must include a file name, not just a repo/bucket or directory path." + " Use `hf download` or `hf buckets sync` to copy directories." + ) + return uri.path_in_repo.rsplit("/", 1)[-1] + + +def _upload_file_to_remote(api: HfApi, src: str, dst: str, *, src_is_stdin: bool) -> None: + uri = parse_hf_uri(dst) + + if src_is_stdin: + if uri.path_in_repo == "" or dst.endswith("/"): + raise click.BadParameter("Stdin upload requires a full destination path including filename.") + data = sys.stdin.buffer.read() + _upload_single(api, uri, data, uri.path_in_repo) + out.result("Uploaded", src="stdin", dst=uri.to_uri()) + return + + if os.path.isdir(src): + raise click.BadParameter( + "Source must be a file, not a directory. Use `hf upload` or `hf buckets sync` for directories." + ) + if not os.path.isfile(src): + raise click.BadParameter(f"Source file not found: {src}") + + prefix = uri.path_in_repo + if prefix == "": + remote_path = os.path.basename(src) + elif dst.endswith("/"): + remote_path = prefix + "/" + os.path.basename(src) + else: + remote_path = prefix + + _upload_single(api, uri, src, remote_path) + out.result("Uploaded", src=src, dst=replace(uri, path_in_repo=remote_path).to_uri()) + + +def _upload_single(api: HfApi, uri: HfUri, source: str | bytes, remote_path: str) -> None: + """Upload a single file or bytes (to a repo or bucket).""" + if uri.is_bucket: + api.batch_bucket_files(uri.id, add=[(source, remote_path)]) + else: + api.upload_file( + path_or_fileobj=source, + path_in_repo=remote_path, + repo_id=uri.id, + repo_type=uri.type, + revision=uri.revision, + ) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/_errors.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_errors.py new file mode 100644 index 0000000000000000000000000000000000000000..45e98da0907f2b4b559c48fedc41b652eb438499 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_errors.py @@ -0,0 +1,155 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CLI error handling utilities.""" + +import traceback +from collections.abc import Callable + +from huggingface_hub.errors import ( + BucketNotFoundError, + CLIError, + CLIExtensionInstallError, + DeviceCodeError, + EntryNotFoundError, + GatedRepoError, + HfHubHTTPError, + HfUriError, + IncompleteSnapshotError, + LocalEntryNotFoundError, + LocalTokenNotFoundError, + OfflineModeIsEnabled, + OIDCError, + RemoteEntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, +) + + +def _format_repo_not_found(error: RepositoryNotFoundError) -> str: + label = error.repo_type.capitalize() if error.repo_type else "Repository" + if error.repo_id: + msg = f"{label} '{error.repo_id}' not found." + else: + msg = f"{label} not found." + msg += "\nIf the repo is private, make sure you are authenticated and your token has the required permissions." + + msg += "\nIf the repo does not exist, create it with: " + if error.repo_id is not None: + type_flag = f" --type {error.repo_type}" if error.repo_type and error.repo_type != "model" else "" + msg += f"hf repos create {error.repo_id}{type_flag}" + else: + msg += "hf repos create " + + return msg + + +def _format_gated_repo(error: GatedRepoError) -> str: + label = error.repo_type if error.repo_type else "repository" + if error.repo_id: + return f"Access denied. {label.capitalize()} '{error.repo_id}' requires approval." + return f"Access denied. This {label} requires approval." + + +def _format_bucket_not_found(error: BucketNotFoundError) -> str: + if error.bucket_id: + msg = f"Bucket '{error.bucket_id}' not found." + cmd = f"hf buckets create {error.bucket_id}" + else: + msg = "Bucket not found." + cmd = "hf buckets create " + msg += "\nIf the bucket is private, make sure you are authenticated and your token has the required permissions." + msg += f"\nIf the bucket does not exist, create it with: {cmd}" + return msg + + +def _format_entry_not_found(error: RemoteEntryNotFoundError) -> str: + label = error.repo_type if error.repo_type else "repository" + url = str(error.response.url) if error.response else None + if error.repo_id: + msg = f"File not found in {label} '{error.repo_id}'." + else: + msg = f"File not found in {label}." + if url: + msg += f"\nURL: {url}" + return msg + + +def _format_local_entry_not_found(error: LocalEntryNotFoundError) -> str: + cause = error.__cause__ + if cause is not None: + return f"Local entry not found. {cause}" + return f"Local entry not found. {error}" + + +def _format_incomplete_snapshot(error: IncompleteSnapshotError) -> str: + msg = _format_local_entry_not_found(error) + msg += f"\nIncomplete snapshot available at: {error.snapshot_path}" + return msg + + +def _format_revision_not_found(error: RevisionNotFoundError) -> str: + label = error.repo_type if error.repo_type else "repository" + if error.repo_id: + return f"Revision not found in {label} '{error.repo_id}'." + return f"Revision not found in {label}. Check the revision parameter." + + +def _format_cli_error(error: CLIError) -> str: + """No traceback, just the error message.""" + return str(error) + + +def _format_cli_extension_install_error(error: CLIExtensionInstallError) -> str: + """Format a CLI extension installation error. + + The error is likely to be a tricky subprocess error to investigate. In this specific case we want to format the + traceback of the root cause while keeping the "nicely formatted" error message of the CLIExtensionInstallError + as a 1-line message. + """ + cause_tb = ( + "".join(traceback.format_exception(type(error.__cause__), error.__cause__, error.__cause__.__traceback__)) + if error.__cause__ is not None + else "" + ) + return f"{cause_tb}\n{error}" + + +CLI_ERROR_MAPPINGS: dict[type[Exception], Callable[..., str]] = { + OfflineModeIsEnabled: lambda error: str(error), + # GatedRepoError must come before RepositoryNotFoundError (it's a subclass). + GatedRepoError: _format_gated_repo, + BucketNotFoundError: _format_bucket_not_found, + RepositoryNotFoundError: _format_repo_not_found, + RevisionNotFoundError: _format_revision_not_found, + LocalTokenNotFoundError: lambda _: "Not logged in. Run 'hf auth login' first.", + OIDCError: lambda error: f"OIDC Exchange failed. {error}", + DeviceCodeError: lambda error: f"Login failed: {error}", + RemoteEntryNotFoundError: _format_entry_not_found, + # IncompleteSnapshotError must come before LocalEntryNotFoundError (it's a subclass). + IncompleteSnapshotError: _format_incomplete_snapshot, + LocalEntryNotFoundError: _format_local_entry_not_found, + EntryNotFoundError: lambda error: str(error), + HfHubHTTPError: lambda error: str(error), + HfUriError: lambda error: f"Invalid HF URI: {error.uri}. {error.msg}", + ValueError: lambda error: f"Invalid value. {error}", + CLIExtensionInstallError: _format_cli_extension_install_error, + CLIError: _format_cli_error, +} + + +def format_known_exception(error: Exception) -> str | None: + for exc_type, formatter in CLI_ERROR_MAPPINGS.items(): + if isinstance(error, exc_type): + return formatter(error) + return None diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/_file_listing.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_file_listing.py new file mode 100644 index 0000000000000000000000000000000000000000..bfa0bbef2cdea3d17ced555c0c19ea103bdb5a99 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_file_listing.py @@ -0,0 +1,225 @@ +# Copyright 2026-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shared helpers for listing files in buckets and repos (tree view, flat view, formatting).""" + +import json +from datetime import datetime +from typing import Sequence + +import click + +from huggingface_hub._buckets import BucketFile, BucketFolder +from huggingface_hub.hf_api import RepoFile, RepoFolder + +from ._cli_utils import get_hf_api +from ._output import OutputFormat, _dataclass_to_dict, out + + +BucketItem = BucketFile | BucketFolder +RepoItem = RepoFile | RepoFolder +ListingItem = BucketItem | RepoItem + + +def get_item_date(item: ListingItem) -> datetime | None: + """Extract date from an item, supporting both repo items (last_commit.date) and bucket items (mtime/uploaded_at).""" + match item: + case BucketFile(mtime=mtime) if mtime is not None: + return mtime + case BucketFile(uploaded_at=uploaded_at) | BucketFolder(uploaded_at=uploaded_at) if uploaded_at is not None: + return uploaded_at + case RepoFile(last_commit=last_commit) | RepoFolder(last_commit=last_commit) if last_commit is not None: + return last_commit.date + case _: + return None + + +def format_size(size: int | float, human_readable: bool = False) -> str: + """Format a size in bytes.""" + if not human_readable: + return str(size) + + for unit in ["B", "KB", "MB", "GB", "TB"]: + if size < 1000: + if unit == "B": + return f"{size} {unit}" + return f"{size:.1f} {unit}" + size /= 1000 + return f"{size:.1f} PB" + + +def format_date(dt: datetime | None, human_readable: bool = False) -> str: + """Format a datetime to a readable date string.""" + if dt is None: + return "" + if human_readable: + return dt.strftime("%b %d %H:%M") + return dt.strftime("%Y-%m-%d %H:%M:%S") + + +def build_tree( + items: Sequence[BucketItem] | Sequence[RepoItem], + human_readable: bool = False, + quiet: bool = False, +) -> list[str]: + """Build a tree representation of files and directories. + + Produces ASCII tree with size and date columns before the tree connector. + When quiet=True, only the tree structure is shown (no size/date). + """ + tree: dict = {} + + for item in items: + parts = item.path.split("/") + current = tree + for part in parts[:-1]: + if part not in current: + current[part] = {"__children__": {}} + current = current[part]["__children__"] + + final_part = parts[-1] + if isinstance(item, BucketFolder | RepoFolder): + if final_part not in current: + current[final_part] = {"__children__": {}} + else: + current[final_part] = {"__item__": item} + + prefix_width = 0 + max_size_width = 0 + max_date_width = 0 + if not quiet: + for item in items: + if isinstance(item, BucketFile | RepoFile): + size_str = format_size(item.size, human_readable) + max_size_width = max(max_size_width, len(size_str)) + date_str = format_date(get_item_date(item), human_readable) + max_date_width = max(max_date_width, len(date_str)) + if max_size_width > 0: + prefix_width = max_size_width + 2 + max_date_width + + lines: list[str] = [] + _render_tree( + tree, + lines, + "", + prefix_width=prefix_width, + max_size_width=max_size_width, + human_readable=human_readable, + ) + return lines + + +def _render_tree( + node: dict, + lines: list[str], + indent: str, + prefix_width: int = 0, + max_size_width: int = 0, + human_readable: bool = False, +) -> None: + """Recursively render a tree structure with size+date prefix.""" + sorted_items = sorted(node.items()) + for i, (name, value) in enumerate(sorted_items): + is_last = i == len(sorted_items) - 1 + connector = "└── " if is_last else "├── " + + is_dir = "__children__" in value + children = value.get("__children__", {}) + + if prefix_width > 0: + if is_dir: + prefix = " " * prefix_width + else: + item = value.get("__item__") + if item is not None: + size_str = format_size(item.size, human_readable) + date_str = format_date(get_item_date(item), human_readable) + prefix = f"{size_str:>{max_size_width}} {date_str}" + else: + prefix = " " * prefix_width + lines.append(f"{prefix} {indent}{connector}{name}{'/' if is_dir else ''}") + else: + lines.append(f"{indent}{connector}{name}{'/' if is_dir else ''}") + + if children: + child_indent = indent + (" " if is_last else "│ ") + _render_tree( + children, + lines, + child_indent, + prefix_width=prefix_width, + max_size_width=max_size_width, + human_readable=human_readable, + ) + + +def list_repo_files_cmd( + repo_id: str, + repo_type: str, + human_readable: bool, + as_tree: bool, + recursive: bool, + revision: str | None, + token: str | None, +) -> None: + """List files in a repo on the Hub. Used by models/datasets/spaces ls commands.""" + if as_tree and out.mode == OutputFormat.json: + raise click.BadParameter("Cannot use --tree with --format json.") + + api = get_hf_api(token=token) + items = list(api.list_repo_tree(repo_id, recursive=recursive, revision=revision, repo_type=repo_type, expand=True)) + print_file_listing(items, human_readable=human_readable, as_tree=as_tree, recursive=recursive) + + +def print_file_listing( + items: Sequence[BucketItem] | Sequence[RepoItem], + *, + human_readable: bool = False, + as_tree: bool = False, + recursive: bool = False, +) -> None: + """Print a file listing in the appropriate format based on the current output mode. + + Supports tree, json, quiet, and flat human-readable views. Works with both + BucketFile/BucketFolder and RepoFile/RepoFolder items. + """ + if not items: + out.text("(empty)") + return + + has_directories = any(isinstance(item, BucketFolder | RepoFolder) for item in items) + + if as_tree: + quiet = out.mode == OutputFormat.quiet + for line in build_tree(items, human_readable=human_readable, quiet=quiet): + print(line) + elif out.mode == OutputFormat.json: + print(json.dumps([_dataclass_to_dict(item) for item in items], indent=2)) + elif out.mode == OutputFormat.quiet: + for item in items: + if isinstance(item, BucketFolder | RepoFolder): + print(f"{item.path}/") + else: + print(item.path) + else: + for item in items: + if isinstance(item, BucketFolder | RepoFolder): + date_str = format_date(get_item_date(item), human_readable) + print(f"{'':>12} {date_str:>19} {item.path}/") + else: + size_str = format_size(item.size, human_readable) + date_str = format_date(get_item_date(item), human_readable) + print(f"{size_str:>12} {date_str:>19} {item.path}") + + if not recursive and has_directories: + out.hint("Use -R to list files recursively.") diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/_framework.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_framework.py new file mode 100644 index 0000000000000000000000000000000000000000..26c8382fb94c1ced57f662b68e72c96e27a46cff --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_framework.py @@ -0,0 +1,645 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Minimal declaration layer over Click 8.x for the ``hf`` CLI. + +This module vendors *only* the slice of Typer the CLI actually uses: turning +annotated function signatures (``Annotated[T, Option(...)]`` / ``Argument(...)``) +into Click parameters, and rendering argument/option help the way Typer did. +It exists so the CLI depends on Click's stable public API instead of Typer's +now-internal vendored Click (``typer._click``), which Typer reserves the right +to refactor at any release. + +Deliberately *not* implemented (add a case here only when a command needs it): +rich rendering, shell-completion install commands, prompts/confirmation options, +env-var options, ``File``/``UUID`` params, ``count`` options, ``default_factory``. +Click's own machinery already handles exceptions, aborts, exits, and native +``_HF_COMPLETE`` shell completion, so none of Typer's ``main()`` overrides are +reproduced here. + +The interesting logic mirrors four Typer functions, kept faithful so ``--help`` +output stays byte-identical: +- :func:`_get_click_type` <- ``typer.main.get_click_type`` +- :func:`_build_click_param` <- ``typer.main.get_click_param`` +- :func:`_make_handler` <- ``typer.main.get_callback`` +- :class:`HfArgument` / :class:`HfOption` help records <- ``typer.core.Typer*`` +""" + +import enum +import functools +import inspect +from collections.abc import Callable, Sequence +from datetime import datetime +from pathlib import Path +from types import UnionType +from typing import Annotated, Any, Literal, Union, get_args, get_origin + +import click + + +def get_command_name(name: str) -> str: + """Normalize a Python identifier to a CLI name (``force_download`` -> ``force-download``).""" + return name.lower().replace("_", "-") + + +# --------------------------------------------------------------------------- +# 1. Declaration markers — replace ``typer.Option`` / ``typer.Argument``. +# Attached in ``Annotated[T, Option(...)]``; the ``= value`` in the function +# signature carries the default (never the marker). +# --------------------------------------------------------------------------- + + +class ParameterInfo: + """Base marker holding the (tiny) set of Click knobs the CLI actually sets. + + ``callback`` is a Typer-style value parser ``fn(value) -> parsed`` — it does NOT + receive Click's ``(ctx, param, value)``. It consumes the enum/list convertors, so + ``value`` is already converted when it runs. + """ + + def __init__( + self, + *param_decls: str, + help: str | None = None, + show_default: bool | str = True, + hidden: bool = False, + is_eager: bool = False, + callback: Callable[..., Any] | None = None, + min: int | None = None, + click_type: click.ParamType | None = None, + ) -> None: + self.param_decls = list(param_decls) + self.help = help + self.show_default = show_default + self.hidden = hidden + self.is_eager = is_eager + self.callback = callback + self.min = min + self.click_type = click_type + + +class Option(ParameterInfo): + """Declare a CLI option, e.g. ``Annotated[str | None, Option("--revision")]``.""" + + +class Argument(ParameterInfo): + """Declare a positional CLI argument, e.g. ``Annotated[str, Argument()]``.""" + + +# --------------------------------------------------------------------------- +# 2. Annotation -> Click type + value convertors. +# --------------------------------------------------------------------------- + + +def _get_click_type(annotation: Any, info: ParameterInfo) -> click.ParamType: + """Map a (already unwrapped) annotation to a Click ``ParamType``.""" + if info.click_type is not None: + return info.click_type + if annotation is str: + return click.STRING + if annotation is bool: + return click.BOOL + if annotation is int: + return click.IntRange(min=info.min) if info.min is not None else click.INT + if annotation is float: + return click.FloatRange(min=info.min) if info.min is not None else click.FLOAT + if annotation is Path: + return click.Path(path_type=Path) + if annotation is datetime: + return click.DateTime() + if isinstance(annotation, type) and issubclass(annotation, enum.Enum): + values = [item.value for item in annotation] + if not all(isinstance(value, str) for value in values): + # A non-str member default would fail Choice validation at parse time with a + # cryptic error; fail at build time with a clear one instead. + raise TypeError(f"Unsupported enum {annotation.__name__!r}: only str-valued enums are supported.") + return click.Choice(values) + if get_origin(annotation) is Literal: + return click.Choice([str(value) for value in get_args(annotation)]) + raise TypeError(f"Unsupported CLI annotation {annotation!r}. Add a case in `_framework._get_click_type()`.") + + +def _enum_convertor(enum_type: type[enum.Enum]) -> Callable[[Any], Any]: + """Map a parsed choice string back to its ``Enum`` member (mirrors Typer).""" + value_map = {str(member.value): member for member in enum_type} + + def convertor(value: Any) -> Any: + if value is not None: + return value_map.get(str(value)) + return None + + return convertor + + +def _list_convertor( + inner: Callable[[Any], Any] | None, default_value: Any | None +) -> Callable[[Sequence[Any] | None], list[Any] | None]: + """Turn Click's tuple (from ``multiple=True``) into a list, applying ``inner`` per item.""" + + def convertor(value: Sequence[Any] | None) -> list[Any] | None: + if value is None or (default_value is None and len(value) == 0): + return None + return [inner(item) if inner else item for item in value] + + return convertor + + +def _make_convertor( + annotation: Any, is_list: bool, default_value: Any | None, info: ParameterInfo +) -> Callable[[Any], Any] | None: + """Build the value convertor for a param (``None`` when no coercion is needed). + + An explicit ``click_type`` (e.g. ``SoftChoice``) owns its own conversion, so no + enum convertor is added on top of it. + """ + inner = None + if info.click_type is None and isinstance(annotation, type) and issubclass(annotation, enum.Enum): + inner = _enum_convertor(annotation) + if is_list: + return _list_convertor(inner, default_value) + return inner + + +def _unwrap_optional_and_list(annotation: Any) -> tuple[Any, bool]: + """Strip ``T | None`` / ``Optional[T]`` and ``list[T]``; return ``(base_type, is_list)``.""" + origin = get_origin(annotation) + if origin in (Union, UnionType): + non_none = [arg for arg in get_args(annotation) if arg is not type(None)] + if len(non_none) != 1: + raise TypeError(f"Unsupported union annotation {annotation!r}; only ``T | None`` is supported.") + annotation = non_none[0] + origin = get_origin(annotation) + if origin in (list, Sequence): + return get_args(annotation)[0], True + return annotation, False + + +def _split_annotated(annotation: Any) -> tuple[Any, ParameterInfo | None]: + """Split ``Annotated[T, Option(...)]`` into ``(T, marker)``; ``(annotation, None)`` otherwise.""" + if get_origin(annotation) is Annotated: + args = get_args(annotation) + marker = next((meta for meta in args[1:] if isinstance(meta, ParameterInfo)), None) + if marker is None and (typer_marker := next((meta for meta in args[1:] if _is_typer_marker(meta)), None)): + marker = _from_typer_marker(typer_marker, from_annotated=True) + return args[0], marker + return annotation, None + + +# --------------------------------------------------------------------------- +# Typer-marker compatibility shim (transition helper). +# +# ``typer_factory`` is exposed publicly and downstream CLIs (e.g. `transformers`) +# still register commands whose parameters carry ``typer.Option`` / +# ``typer.Argument`` markers. Recognize those markers structurally (no typer +# import required) and translate the fields this framework supports, so such +# commands keep working while downstream migrates. +# TODO: remove once transformers pins huggingface_hub>=1.22.0. +# --------------------------------------------------------------------------- + + +def _is_typer_marker(obj: Any) -> bool: + return type(obj).__module__.partition(".")[0] == "typer" and type(obj).__name__ in ("OptionInfo", "ArgumentInfo") + + +def _from_typer_marker(meta: Any, *, from_annotated: bool) -> ParameterInfo: + """Translate a ``typer.models.OptionInfo`` / ``ArgumentInfo`` into our marker.""" + param_decls = list(getattr(meta, "param_decls", None) or ()) + if type(meta).__name__ == "OptionInfo": + # In ``Annotated[...]`` usage typer stores the first flag name in ``default`` + # (``Option("--flag", "-f")`` -> ``default="--flag"``, ``param_decls=("-f",)``). + default = getattr(meta, "default", ...) + if from_annotated and isinstance(default, str): + param_decls = [default, *param_decls] + cls: type[ParameterInfo] = Option + else: + cls = Argument + return cls( + *param_decls, + help=getattr(meta, "help", None), + show_default=getattr(meta, "show_default", True), + hidden=getattr(meta, "hidden", False), + is_eager=getattr(meta, "is_eager", False), + callback=getattr(meta, "callback", None), + min=getattr(meta, "min", None), + click_type=getattr(meta, "click_type", None), + ) + + +# --------------------------------------------------------------------------- +# 3. Signature -> Click parameters (mirrors ``typer.main.get_click_param``). +# --------------------------------------------------------------------------- + + +def _wrap_param_callback( + user_callback: Callable[[Any], Any], convertor: Callable[[Any], Any] | None +) -> Callable[[click.Context, click.Parameter, Any], Any]: + """Adapt a Typer-style value parser ``fn(value) -> parsed`` to a Click param callback.""" + + def click_callback(ctx: click.Context, param: click.Parameter, value: Any) -> Any: + return user_callback(convertor(value) if convertor else value) + + return click_callback + + +def _build_click_param( + name: str, annotation: Any, info: ParameterInfo | None, signature_default: Any +) -> tuple[click.Parameter, Callable[[Any], Any] | None]: + """Build a Click ``Parameter`` (and its post-parse value convertor) for one function param.""" + required = signature_default is inspect.Parameter.empty + default_value = None if required else signature_default + if isinstance(default_value, enum.Enum): + default_value = default_value.value + if info is None: + # A bare annotation with no marker: required positional -> Argument, else Option. + info = Argument() if required else Option() + + base_type, is_list = _unwrap_optional_and_list(annotation) + convertor = _make_convertor(base_type, is_list, default_value, info) + + # A user ``callback`` consumes the value itself (and the convertor with it); otherwise the + # convertor runs later in the command handler wrapper. This keeps conversion single-pass. + if info.callback is not None: + param_callback: Callable[..., Any] | None = _wrap_param_callback(info.callback, convertor) + handler_convertor: Callable[[Any], Any] | None = None + else: + param_callback = None + handler_convertor = convertor + + if isinstance(info, Argument): + argument = HfArgument( + [name], + type=_get_click_type(base_type, info), + required=required, + default=default_value, + nargs=-1 if is_list else 1, + is_eager=info.is_eager, + callback=param_callback, + help=info.help, + show_default=info.show_default, + hidden=info.hidden, + ) + return argument, handler_convertor + + # Option: prepend the Python name so ``click.Parameter.name`` matches the function kwarg, + # regardless of the user-facing flag names ("-v"/"--version", "--type"/"--repo-type", ...). + is_flag = base_type is bool + decls = [name] + if info.param_decls: + decls.extend(info.param_decls) + else: + kebab = get_command_name(name) + decls.append(f"--{kebab}/--no-{kebab}" if is_flag else f"--{kebab}") + option = HfOption( + decls, + # Click infers the flag type itself; passing a type alongside ``is_flag`` is rejected. + type=None if is_flag else _get_click_type(base_type, info), + required=required, + default=default_value, + is_flag=is_flag, + multiple=is_list, + is_eager=info.is_eager, + callback=param_callback, + help=info.help, + show_default=info.show_default, + hidden=info.hidden, + ) + return option, handler_convertor + + +def _build_params( + func: Callable[..., Any], +) -> tuple[list[click.Parameter], dict[str, Callable[[Any], Any]], str | None]: + """Introspect ``func`` and return ``(params, convertors, context_param_name)``.""" + params: list[click.Parameter] = [] + convertors: dict[str, Callable[[Any], Any]] = {} + context_param_name: str | None = None + for name, sig_param in inspect.signature(func).parameters.items(): + # Read raw signature annotations rather than ``typing.get_type_hints``: the CLI never uses + # ``from __future__ import annotations``, so annotations are already live objects, and + # get_type_hints mangles ``Annotated[X | None, ...]`` on Python 3.10 (drops the origin). + annotation = sig_param.annotation if sig_param.annotation is not inspect.Parameter.empty else str + base_type, info = _split_annotated(annotation) + if isinstance(base_type, type) and ( + issubclass(base_type, click.Context) + # typer.Context subclasses typer's *vendored* click since typer 0.26, so an + # issubclass check against real click misses it. Match it structurally. + or (base_type.__module__.partition(".")[0] == "typer" and base_type.__name__ == "Context") + ): + context_param_name = name + continue + signature_default = sig_param.default + if _is_typer_marker(signature_default): + # Old typer style: the marker *is* the signature default and carries the value. + if info is None: + info = _from_typer_marker(signature_default, from_annotated=False) + marker_default = getattr(signature_default, "default", ...) + signature_default = inspect.Parameter.empty if marker_default is ... else marker_default + click_param, convertor = _build_click_param(name, base_type, info, signature_default) + if convertor is not None and click_param.name is not None: + convertors[click_param.name] = convertor + params.append(click_param) + return params, convertors, context_param_name + + +def _make_handler( + func: Callable[..., Any], + convertors: dict[str, Callable[[Any], Any]], + context_param_name: str | None, +) -> Callable[..., Any]: + """Wrap ``func`` so Click's kwargs map back to it, applying convertors and injecting the ctx.""" + + def handler(**kwargs: Any) -> Any: + call_kwargs = {key: (convertors[key](value) if key in convertors else value) for key, value in kwargs.items()} + if context_param_name is not None: + call_kwargs[context_param_name] = click.get_current_context() + return func(**call_kwargs) + + functools.update_wrapper(handler, func) + return handler + + +# --------------------------------------------------------------------------- +# 4. Click parameter subclasses — port Typer's help records so arguments show up +# in ``--help`` (plain ``click.Argument`` renders none) and defaults/metavars +# read identically. Rich/env-var branches are dropped (never enabled here). +# --------------------------------------------------------------------------- + + +def _split_opt(opt: str) -> tuple[str, str]: + """Split an option string into ``(prefix, name)`` (Click's ``split_opt``, inlined).""" + first = opt[:1] + if first.isalnum(): + return "", opt + if opt[1:2] == first: + return opt[:2], opt[2:] + return first, opt[1:] + + +def _extract_default_help_str(param: click.Parameter, ctx: click.Context) -> Any: + # Resilient parsing avoids type casting failing while rendering the default. + resilient = ctx.resilient_parsing + ctx.resilient_parsing = True + try: + return param.get_default(ctx, call=False) + finally: + ctx.resilient_parsing = resilient + + +def _default_string( + param: "HfArgument | HfOption", ctx: click.Context, show_default_is_str: bool, default_value: Any +) -> str: + if show_default_is_str: + return f"({param.show_default})" + if isinstance(default_value, (list, tuple)): + return ", ".join(_default_string(param, ctx, show_default_is_str, item) for item in default_value) + if isinstance(default_value, enum.Enum): + return str(default_value.value) + if inspect.isfunction(default_value): + return "(dynamic)" + if isinstance(param, HfOption) and param.is_bool_flag and param.secondary_opts: + # Boolean toggle: show the opt name (without prefix) matching the current default. + if default_value: + return _split_opt(param.opts[0])[1] if param.opts else str(default_value) + return _split_opt(param.secondary_opts[0])[1] + if isinstance(param, HfOption) and param.is_bool_flag and not param.secondary_opts and not default_value: + return "" + return str(default_value) + + +def _describe_number_range(param_type: "click.IntRange | click.FloatRange") -> str: + """Human-readable range hint like ``x>=1`` or ``1<=x<10``. + + Reproduces click's private ``_NumberRangeBase._describe_range()`` using the public + ``IntRange``/``FloatRange`` attributes, so the framework stays on Click's public API. + """ + if param_type.min is None: + return f"x{'<' if param_type.max_open else '<='}{param_type.max}" + if param_type.max is None: + return f"x{'>' if param_type.min_open else '>='}{param_type.min}" + left = "<" if param_type.min_open else "<=" + right = "<" if param_type.max_open else "<=" + return f"{param_type.min}{left}x{right}{param_type.max}" + + +def _build_help_extra(param: "HfArgument | HfOption", ctx: click.Context, base_help: str) -> str: + """Append the ``[default: ...; required]`` suffix to a help string (shared by arg/option).""" + extra: list[str] = [] + default_value = _extract_default_help_str(param, ctx) + show_default_is_str = isinstance(param.show_default, str) + if show_default_is_str or (default_value is not None and (param.show_default or ctx.show_default)): + default_string = _default_string(param, ctx, show_default_is_str, default_value) + if default_string: + extra.append(f"default: {default_string}") + # Numeric range hints are shown for options only (matches Typer; arguments omit them). + if isinstance(param, HfOption) and isinstance(param.type, (click.IntRange, click.FloatRange)): + range_str = _describe_number_range(param.type) + if range_str: + extra.append(range_str) + if param.required: + extra.append("required") + if extra: + suffix = f"[{'; '.join(extra)}]" + return f"{base_help} {suffix}" if base_help else suffix + return base_help + + +class HfArgument(click.Argument): + """Positional argument that renders help text and a metavar (Click's does neither).""" + + def __init__( + self, + param_decls: Sequence[str], + *, + help: str | None = None, + show_default: bool | str = True, + hidden: bool = False, + **attrs: Any, + ) -> None: + self.help = help + self.show_default = show_default + self.hidden = hidden + super().__init__(param_decls, **attrs) + + def make_metavar(self, ctx: click.Context) -> str: + if self.metavar is not None: + var = self.metavar + if not self.required and not var.startswith("["): + var = f"[{var}]" + return var + var = (self.name or "").upper() + if not self.required: + var = f"[{var}]" + type_var = self.type.get_metavar(self, ctx=ctx) + if type_var: + var += f":{type_var}" + if self.nargs != 1: + var += "..." + return var + + def get_help_record(self, ctx: click.Context) -> tuple[str, str] | None: + if self.hidden: + return None + return self.make_metavar(ctx=ctx), _build_help_extra(self, ctx, self.help or "") + + +class HfOption(click.Option): + """Option whose help record ports Typer's default/metavar rendering verbatim.""" + + show_default: bool | str + + def get_help_record(self, ctx: click.Context) -> tuple[str, str] | None: + if self.hidden: + return None + + any_prefix_is_slash = False + + def _write_opts(opts: Sequence[str]) -> str: + nonlocal any_prefix_is_slash + rv, any_slashes = click.formatting.join_options(opts) + if any_slashes: + any_prefix_is_slash = True + if not self.is_flag and not self.count: + rv += f" {self.make_metavar(ctx=ctx)}" + return rv + + rv = [_write_opts(self.opts)] + if self.secondary_opts: + rv.append(_write_opts(self.secondary_opts)) + + help_text = _build_help_extra(self, ctx, self.help or "") + return ("; " if any_prefix_is_slash else " / ").join(rv), help_text + + +# --------------------------------------------------------------------------- +# 5. Command / group base classes and the decorator API (replace ``typer.Typer``). +# --------------------------------------------------------------------------- + + +def _format_params(command: click.Command, ctx: click.Context, formatter: click.HelpFormatter) -> None: + """Render params split into "Arguments" and "Options" sections (mirrors Typer).""" + args: list[tuple[str, str]] = [] + opts: list[tuple[str, str]] = [] + for param in command.get_params(ctx): + record = param.get_help_record(ctx) + if record is None: + continue + if param.param_type_name == "argument": + args.append(record) + elif param.param_type_name == "option": + opts.append(record) + if args: + with formatter.section("Arguments"): + formatter.write_dl(args) + if opts: + with formatter.section("Options"): + formatter.write_dl(opts) + + +class HfCommand(click.Command): + """Leaf command that renders arguments and options in separate help sections.""" + + def format_options(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: + _format_params(self, ctx, formatter) + + +def build_command( + func: Callable[..., Any], + *, + name: str | None = None, + cls: type[click.Command] | None = None, + help: str | None = None, + epilog: str | None = None, + short_help: str | None = None, + options_metavar: str = "[OPTIONS]", + add_help_option: bool = True, + no_args_is_help: bool = False, + hidden: bool = False, + deprecated: bool = False, + context_settings: dict[str, Any] | None = None, +) -> click.Command: + """Build a Click ``Command`` from a function with ``Annotated`` params. + + Replaces ``typer.main.get_command_from_info``. + """ + params, convertors, context_param_name = _build_params(func) + handler = _make_handler(func, convertors, context_param_name) + command_help = inspect.cleandoc(help) if help else inspect.getdoc(func) + return (cls or HfCommand)( + name=name if name is not None else get_command_name(func.__name__), + callback=handler, + params=params, + help=command_help, + epilog=epilog, + short_help=short_help, + options_metavar=options_metavar, + add_help_option=add_help_option, + no_args_is_help=no_args_is_help, + hidden=hidden, + deprecated=deprecated, + context_settings=context_settings or {}, + ) + + +class HfGroup(click.Group): + """Command group with the decorator API the CLI relies on (``command``/``callback``/``add_group``). + + Subclasses (see ``HFCliTyperGroup``) layer on styling, aliases, topics and error + enrichment; this base only wires functions to Click via :func:`build_command`. + """ + + #: Command class used by ``@group.command()`` unless overridden per call. + command_class: type[click.Command] = HfCommand + + def format_options(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: + _format_params(self, ctx, formatter) + self.format_commands(ctx, formatter) + + def list_commands(self, ctx: click.Context) -> list[str]: + # Preserve declaration order rather than Click's alphabetical default. + return list(self.commands) + + def command( # type: ignore # deliberately narrows click.Group.command (builds from annotated signatures) + self, name: str | None = None, *, cls: type[click.Command] | None = None, **kwargs: Any + ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + command = build_command(func, name=name, cls=cls or self.command_class, **kwargs) + self.add_command(command, command.name) + return func + + return decorator + + def add_group(self, group: click.Group, *, name: str, hidden: bool = False) -> None: + """Register a subgroup under ``name`` (which may carry pipe aliases, e.g. ``"repos | repo"``).""" + group.name = name + group.hidden = hidden + self.add_command(group, name) + + def group_callback( + self, *, invoke_without_command: bool = False + ) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Register the function invoked for this group (named ``group_callback`` because Click + already uses the ``callback`` attribute for the group's own handler).""" + + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + params, convertors, context_param_name = _build_params(func) + self.callback = _make_handler(func, convertors, context_param_name) + self.params = [*self.params, *params] + self.invoke_without_command = invoke_without_command + if invoke_without_command: + # Match Typer: the subcommand is optional when the group runs without one. + self.subcommand_metavar = "[COMMAND] [ARGS]..." + if self.help is None and func.__doc__: + self.help = inspect.cleandoc(func.__doc__) + return func + + return decorator diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/_help_formatter.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_help_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..19c2909b7be9a151d912db851e6493f2b6e46bd3 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_help_formatter.py @@ -0,0 +1,34 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Pretty ANSI help formatter for the `hf` CLI.""" + +from collections.abc import Iterable + +import click + +from huggingface_hub.utils import ANSI + + +class StyledHelpFormatter(click.HelpFormatter): + def write_heading(self, heading: str) -> None: + styled = ANSI.underline(heading + ":") + self.write(f"{'':>{self.current_indent}}{styled}\n") + + def write_dl(self, rows: Iterable[tuple[str, str]], col_max: int = 30, col_spacing: int = 2) -> None: + rows = [(ANSI.bold(first), second) for first, second in rows] + super().write_dl(rows, col_max=col_max, col_spacing=col_spacing) + + +class StyledContext(click.Context): + formatter_class = StyledHelpFormatter diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/_output.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_output.py new file mode 100644 index 0000000000000000000000000000000000000000..9806476feee1285b8e91f1e773caea879862a76e --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_output.py @@ -0,0 +1,360 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Output framework for the `hf` CLI.""" + +import dataclasses +import datetime +import json +import re +import shutil +import sys +from collections.abc import Sequence +from enum import Enum +from typing import Any, cast + +import click + +from huggingface_hub.errors import ConfirmationError +from huggingface_hub.utils import ANSI, StatusLine, disable_progress_bars, is_agent, tabulate + + +class OutputFormat(str, Enum): + """Output format for CLI commands with auto detection of agent/human mode.""" + + agent = "agent" + auto = "auto" + human = "human" + json = "json" + quiet = "quiet" + + +def _print_flush(*values: Any, **kwargs: Any) -> None: + """Like `print`, but always flushed: some CLI flows block on user action right after + printing (e.g. the device-code login), so output must not stay buffered.""" + print(*values, **kwargs, flush=True) + + +class Output: + """Output sink for the `hf` CLI. + + Mode is resolved once at init time based on `is_agent()` auto-detection + and can be overridden per-command via `set_mode()`. + """ + + mode: OutputFormat + no_truncate: bool + + def __init__(self) -> None: + self.no_truncate = False + self.set_mode() + + def set_mode(self, mode: OutputFormat = OutputFormat.auto) -> None: + """Override the output mode (called once at startup and again per '--format' flag).""" + if mode == OutputFormat.auto: + mode = OutputFormat.agent if is_agent() else OutputFormat.human + self.mode = mode + if mode != OutputFormat.human: + disable_progress_bars() + + def set_no_truncate(self, no_truncate: bool) -> None: + """Toggle off cell truncation for human table output.""" + self.no_truncate = no_truncate + + def is_quiet(self) -> bool: + return self.mode == OutputFormat.quiet + + def text(self, msg: str | None = None, *, human: str | None = None, agent: str | None = None) -> None: + """Print a free-form text message to stdout.""" + if msg is not None: + if human is not None or agent is not None: + raise ValueError("Cannot mix 'msg' with 'human'/'agent'.") + human = msg + agent = _strip_ansi(msg) + + match self.mode: + case OutputFormat.human: + if human is not None: + _print_flush(human) + case OutputFormat.agent: + if agent is not None: + _print_flush(agent) + # json/quiet: no-op + + def table( + self, + items: Sequence[dict[str, Any]], + *, + headers: list[str] | None = None, + id_key: str | None = None, + alignments: dict[str, str] | None = None, + ) -> None: + """Print tabular data to stdout. + + Args: + items: List of dicts. Headers are auto-detected from keys if not provided. + headers: Explicit column names. If None, derived from dict keys (all-None columns filtered). + id_key: Key to print in quiet mode. If None, uses the first header. + alignments: Optional mapping of header name to "left" or "right". Defaults to "left". + """ + if not items: + match self.mode: + case OutputFormat.agent | OutputFormat.human: + _print_flush("No results found.") + case OutputFormat.json: + _print_flush("[]") + return + + if headers is None: + all_columns = list(items[0].keys()) + headers = [col for col in all_columns if any(item.get(col) is not None for item in items)] + rows = [[item.get(h) for h in headers] for item in items] + + match self.mode: + case OutputFormat.human: # padded table, adaptive truncation, SCREAMING_SNAKE headers + screaming_headers = [_to_header(h) for h in headers] + formatted_rows: list[list[str]] = [[_format_table_value_human(v) for v in row] for row in rows] + + is_truncated = _truncate_columns(screaming_headers, formatted_rows, no_truncate=self.no_truncate) + + inferred = {**_infer_alignments(headers, rows), **(alignments or {})} + screaming_alignments = {_to_header(k): v for k, v in inferred.items()} + _print_flush( + tabulate( + cast("list[list[str | int]]", formatted_rows), + headers=screaming_headers, + alignments=screaming_alignments, + ), + ) + if is_truncated: + self.hint("Use `--no-truncate` or `--format json` to display full values.") + case OutputFormat.agent: # TSV, no truncation, full timestamps + _print_flush("\t".join(headers)) + for row in rows: + _print_flush("\t".join(_format_table_cell_agent(v) for v in row)) + case OutputFormat.json: # compact JSON array + _print_flush(json.dumps(list(items), default=str)) + case OutputFormat.quiet: # id_key column (or first column), one per line + quiet_key = id_key or headers[0] + for item in items: + _print_flush(item.get(quiet_key, "")) + + def dict(self, data: Any, *, id_key: str | None = None) -> None: + """Print structured data as JSON in all modes (indented for human, compact otherwise). + + Accepts a dict or a dataclass. + """ + if dataclasses.is_dataclass(data) and not isinstance(data, type): + data = _dataclass_to_dict(data) + if self.mode == OutputFormat.quiet and id_key is not None: + _print_flush(data.get(id_key, "")) + return + indent = 2 if self.mode == OutputFormat.human else None + _print_flush(json.dumps(data, indent=indent, default=str)) + + def result(self, message: str, **data: Any) -> None: + """Print a success summary to stdout.""" + match self.mode: + case OutputFormat.human: # ✓ message + key: value lines + parts = [ANSI.green(f"✓ {message}")] + for k, v in data.items(): + if v is not None: + parts.append(f" {k}: {v}") + _print_flush("\n".join(parts)) + case OutputFormat.agent: # key=val pairs, space-separated + parts = [f"{k}={v}" for k, v in data.items() if v is not None] + _print_flush(" ".join(parts) if parts else message) + case OutputFormat.json: # json.dumps(data), message ignored + _print_flush(json.dumps(data, default=str) if data else "") + case OutputFormat.quiet: # first value only + values = list(data.values()) + if values: + _print_flush(values[0]) + + def confirm(self, message: str, *, default: bool = False, yes: bool = False, confirm_param: str = "--yes") -> None: + """ + Ask for confirmation. Raises `ConfirmationError` in non-human modes. + """ + if yes: + return + if self.mode != OutputFormat.human: + raise ConfirmationError(f"{message} Use {confirm_param} to skip confirmation.") + click.confirm(message, default=default, abort=True) + + def status(self, message: str | None = None) -> StatusLine: + """Return a status line that emits only in human mode (no-op otherwise).""" + status = StatusLine(enabled=self.mode == OutputFormat.human) + if message is not None: + status.update(message) + return status + + def warning(self, message: str) -> None: + """Print a non-fatal warning to stderr (all modes).""" + if self.mode == OutputFormat.human: + _print_flush(ANSI.yellow(f"Warning: {message}"), file=sys.stderr) + else: + _print_flush(f"Warning: {message}", file=sys.stderr) + + def error(self, message: str) -> None: + """Print an error to stderr (all modes).""" + if self.mode == OutputFormat.human: + _print_flush(ANSI.red(f"Error: {message}"), file=sys.stderr) + else: + _print_flush(f"Error: {message}", file=sys.stderr) + + def log(self, message: str) -> None: + """Print a text message to stderr (human: gray, json/agent: plain text). + + Suppressed in quiet mode. Kept in json mode (like agent) since agents + commonly run with ``--format json`` and the message goes to stderr so + it never pollutes the parsed stdout. + """ + if self.mode == OutputFormat.quiet: + return + if self.mode == OutputFormat.human: + _print_flush(ANSI.gray(message), file=sys.stderr) + else: + _print_flush(message, file=sys.stderr) + + def hint(self, message: str) -> None: + """Print a helpful hint to stderr (human: gray, json/agent: plain text). + + Suppressed in quiet mode. Kept in json mode (like agent) since agents + commonly run with ``--format json`` and the next-command hints are useful + there; hints go to stderr so they never pollute the parsed stdout. + """ + self.log(f"Hint: {message}") + + +# HELPERS + + +def _serialize_value(v: object) -> object: + """Recursively serialize a value to be JSON-compatible.""" + if isinstance(v, datetime.datetime): + return v.isoformat() + elif isinstance(v, dict): + return {key: _serialize_value(val) for key, val in v.items() if val is not None} + elif isinstance(v, list): + return [_serialize_value(item) for item in v] + return v + + +def _dataclass_to_dict(info: Any) -> dict[str, Any]: + """Convert a dataclass to a json-serializable dict.""" + return {k: _serialize_value(v) for k, v in dataclasses.asdict(info).items() if v is not None} + + +_ANSI_RE = re.compile(r"\033\[[0-9;]*m") + + +def _strip_ansi(text: str) -> str: + return _ANSI_RE.sub("", text) + + +def _single_line(text: str) -> str: + return " ".join(text.split()) + + +def _to_header(name: str) -> str: + """Convert a camelCase or PascalCase string to SCREAMING_SNAKE_CASE.""" + s = re.sub(r"([a-z])([A-Z])", r"\1_\2", name) + return s.upper() + + +def _infer_alignments(headers: list[str], rows: list[list[Any]]) -> dict[str, str]: + """Return ``{"col": "right"}`` for columns where every non-None value is numeric.""" + result: dict[str, str] = {} + for c, h in enumerate(headers): + if all(row[c] is None or (isinstance(row[c], (int, float)) and not isinstance(row[c], bool)) for row in rows): + result[h] = "right" + return result + + +def _format_table_value_human(value: Any) -> str: + """Convert a value to string for terminal display.""" + if value is None: + return "" + if isinstance(value, bool): + return "✔" if value else "" + if isinstance(value, datetime.datetime): + return value.strftime("%Y-%m-%d") + if isinstance(value, str) and re.match(r"^\d{4}-\d{2}-\d{2}T", value): + return value[:10] + if isinstance(value, str): + return _single_line(value) + if isinstance(value, list): + return ", ".join(_format_table_value_human(v) for v in value) + elif isinstance(value, dict): + if "name" in value: # Likely to be a user or org => print name + return _single_line(str(value["name"])) + return _single_line(json.dumps(value)) + return _single_line(str(value)) + + +def _truncate_columns( + headers: list[str], + rows: list[list[str]], + *, + no_truncate: bool, +) -> bool: + """Truncate cells in-place to fit the current terminal width. + + Returns `True` if any cell was truncated, so the caller can emit a hint. + `shutil.get_terminal_size` is cross-platform: it honors `$COLUMNS`, then + queries the OS-native API, then falls back to `(80, 24)`. + """ + if no_truncate or not rows: + return False + + n = len(headers) + # Per-column natural width: longest of header label and cell values. + natural = [max(len(headers[c]), *(len(rows[r][c]) for r in range(len(rows)))) for c in range(n)] + + # `max(0, n - 1)` accounts for the single-space separator between columns. + budget = shutil.get_terminal_size().columns - max(0, n - 1) + if sum(natural) <= budget: + return False + + # Shrink the widest column 1 char at a time. Floors keep the header label + # visible; the `4` is the smallest cap that still shows "x..." (one content + # char plus the "..." marker). + caps = natural.copy() + min_widths = [max(len(h), 4) for h in headers] + while sum(caps) > budget: + widest = max( + (i for i, w in enumerate(caps) if w > min_widths[i]), + key=lambda i: caps[i], + default=-1, + ) + if widest < 0: + break # everything at floor — table wraps slightly + caps[widest] -= 1 + + truncated = False + for row in rows: + for c, cell in enumerate(row): + if len(cell) > caps[c]: + truncated = True + row[c] = cell[: caps[c] - 3] + "..." + return truncated + + +def _format_table_cell_agent(value: Any) -> str: + """Format a cell value for agent TSV output (ISO timestamps, tabs escaped).""" + if isinstance(value, datetime.datetime): + return value.isoformat() + return _single_line(str(value)) + + +out = Output() diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/_skills.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_skills.py new file mode 100644 index 0000000000000000000000000000000000000000..664dd4bcdfb06bd62aad6bfe76815b5507f53beb --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/_skills.py @@ -0,0 +1,311 @@ +"""Internal helpers for Hugging Face marketplace skill installation and upgrades.""" + +import json +import re +import shutil +import tempfile +from collections.abc import Callable +from dataclasses import dataclass, replace +from pathlib import Path, PurePosixPath +from typing import Any, Literal + +from huggingface_hub._buckets import BucketFile +from huggingface_hub.errors import CLIError + +from ..utils import disable_progress_bars +from ._cli_utils import get_hf_api + + +DEFAULT_SKILL_ID = "hf-cli" +DEFAULT_SKILLS_BUCKET_ID = "huggingface/skills" +MARKETPLACE_PATH = "marketplace.json" +# Empty marker file dropped into managed skill installs so `hf skills update` knows +# to touch them and leave user-placed skill dirs alone. Filename is historical (used +# to be a JSON manifest with a revision); we keep it for backward compat with installs +# made by previous versions. +MANAGED_MARKER_FILENAME = ".hf-skill-manifest.json" + +SkillUpdateStatus = Literal["up_to_date", "unmanaged", "source_unreachable"] + + +@dataclass(frozen=True) +class MarketplaceSkill: + name: str + repo_path: str + description: str | None = None + + +@dataclass(frozen=True) +class SkillUpdateInfo: + name: str + skill_dir: Path + status: SkillUpdateStatus + detail: str | None = None + + +def add_skill(skill_name: str, destination_root: Path, force: bool = False) -> Path: + """Resolve a marketplace skill by name and install it.""" + api = get_hf_api() + with disable_progress_bars(): + marketplace_skills = _load_marketplace_skills(api) + skill = _select_marketplace_skill(marketplace_skills, skill_name) + if skill is None: + raise CLIError( + f"Skill '{skill_name}' not found in {DEFAULT_SKILLS_BUCKET_ID}. " + "Try `hf skills add` to install `hf-cli` or use a known skill name." + ) + return _install_marketplace_skill(api, skill, destination_root, force=force) + + +def install_generated_skill(content: str, destination_root: Path, force: bool = False) -> Path: + """Install the `hf-cli` skill from locally generated SKILL.md content (no bucket download).""" + + def populate(install_dir: Path) -> None: + install_dir.mkdir(parents=True, exist_ok=True) + (install_dir / "SKILL.md").write_text(content, encoding="utf-8") + (install_dir / MANAGED_MARKER_FILENAME).touch() + + return _install_skill(DEFAULT_SKILL_ID, destination_root, populate=populate, force=force) + + +def update_skills(roots: list[Path], selector: str | None = None, *, hf_cli_content: str) -> list[SkillUpdateInfo]: + """Re-sync managed skill installs (`hf-cli` is rewritten from `hf_cli_content`, the rest from the bucket).""" + skill_dirs = _iter_unique_skill_dirs(roots) + if selector is not None: + selector_lower = selector.strip().lower() + skill_dirs = [d for d in skill_dirs if d.name.lower() == selector_lower] + if not skill_dirs: + raise CLIError(f"No installed skill matches '{selector}'. Install it with `hf skills add {selector}`.") + + # `hf-cli` is regenerated locally, so only hit the marketplace when another managed skill needs it. + needs_marketplace = any(d.name != DEFAULT_SKILL_ID and (d / MANAGED_MARKER_FILENAME).exists() for d in skill_dirs) + api = None + marketplace_skills: dict[str, MarketplaceSkill] = {} + if needs_marketplace: + api = get_hf_api() + with disable_progress_bars(): + marketplace_skills = {skill.name.lower(): skill for skill in _load_marketplace_skills(api)} + + return [_apply_single_update(api, skill_dir, marketplace_skills, hf_cli_content) for skill_dir in skill_dirs] + + +def _load_marketplace_skills(api) -> list[MarketplaceSkill]: + payload = _load_marketplace_payload(api) + plugins = payload.get("plugins") + if not isinstance(plugins, list): + raise CLIError("Invalid marketplace payload: expected a top-level 'plugins' list.") + + skills: list[MarketplaceSkill] = [] + for plugin in plugins: + if not isinstance(plugin, dict): + continue + name = plugin.get("name") + source = plugin.get("source") + if not isinstance(name, str) or not isinstance(source, str): + continue + description = plugin.get("description") + skills.append( + MarketplaceSkill( + name=name, + repo_path=_normalize_repo_path(source), + description=description if isinstance(description, str) else None, + ) + ) + return skills + + +def _install_marketplace_skill(api, skill: MarketplaceSkill, destination_root: Path, force: bool = False) -> Path: + """Install a marketplace skill into a local skills directory.""" + + def populate(install_dir: Path) -> None: + install_dir.mkdir(parents=True, exist_ok=True) + bucket_files = _list_skill_files(api, skill) + _download_skill_files(api, skill, bucket_files, install_dir) + _validate_installed_skill_dir(install_dir) + (install_dir / MANAGED_MARKER_FILENAME).touch() + + return _install_skill(skill.name, destination_root, populate=populate, force=force) + + +_VALID_SKILL_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") + + +def _install_skill( + name: str, + destination_root: Path, + populate: Callable[[Path], None], + force: bool = False, +) -> Path: + """Install a skill into ``destination_root`` by calling ``populate(install_dir)`` to fill it. + + Used by both the marketplace install (populate = download from bucket) and the + locally-generated install (populate = write content). When the install already + exists and ``force`` is set, the new content is staged in a sibling tempdir and + atomically swapped in, so the existing install stays intact if ``populate`` + fails halfway through. + """ + # `name` may come from the remote marketplace payload and the install dir is removed on + # reinstall: validate it as defense-in-depth against path traversal. + if not _VALID_SKILL_NAME.fullmatch(name): + raise CLIError(f"Invalid skill name '{name}'.") + destination_root = destination_root.expanduser().resolve() + destination_root.mkdir(parents=True, exist_ok=True) + install_dir = destination_root / name + already_exists = install_dir.exists() + + if already_exists and not force: + raise FileExistsError(f"Skill already exists: {install_dir}") + + if already_exists: + with tempfile.TemporaryDirectory(dir=destination_root, prefix=f".{install_dir.name}.install-") as tmp_dir_str: + staged_dir = Path(tmp_dir_str) / install_dir.name + populate(staged_dir) + _atomic_replace_directory(existing_dir=install_dir, staged_dir=staged_dir) + return install_dir + + try: + populate(install_dir) + except Exception: + if install_dir.exists(): + shutil.rmtree(install_dir) + raise + return install_dir + + +def _load_marketplace_payload(api) -> dict[str, Any]: + with tempfile.TemporaryDirectory() as tmp_dir: + local_path = Path(tmp_dir) / "marketplace.json" + api.download_bucket_files( + DEFAULT_SKILLS_BUCKET_ID, + [(MARKETPLACE_PATH, local_path)], + raise_on_missing_files=True, + ) + parsed = json.loads(local_path.read_text(encoding="utf-8")) + + if not isinstance(parsed, dict): + raise CLIError("Invalid marketplace payload: expected a JSON object.") + return parsed + + +def _select_marketplace_skill(skills: list[MarketplaceSkill], selector: str) -> MarketplaceSkill | None: + selector_lower = selector.strip().lower() + for skill in skills: + if skill.name.lower() == selector_lower: + return skill + return None + + +def _normalize_repo_path(path: str) -> str: + normalized = path.strip() + while normalized.startswith("./"): + normalized = normalized[2:] + normalized = normalized.strip("/") + if not normalized: + raise CLIError("Invalid marketplace entry: empty source path.") + return normalized + + +def _validate_installed_skill_dir(skill_dir: Path) -> None: + skill_file = skill_dir / "SKILL.md" + if not skill_file.is_file(): + raise RuntimeError(f"Installed skill is missing SKILL.md: {skill_file}") + + +def _list_skill_files(api, skill: MarketplaceSkill) -> list[BucketFile]: + """List all files under `skill.repo_path` in the marketplace bucket.""" + prefix = skill.repo_path.rstrip("/") + files: list[BucketFile] = [ + item + for item in api.list_bucket_tree(DEFAULT_SKILLS_BUCKET_ID, prefix=prefix, recursive=True) + if isinstance(item, BucketFile) + ] + if not files: + raise FileNotFoundError(f"Path '{prefix}' not found in bucket '{DEFAULT_SKILLS_BUCKET_ID}'.") + return files + + +def _download_skill_files(api, skill: MarketplaceSkill, files: list[BucketFile], install_dir: Path) -> None: + """Download bucket files into `install_dir`.""" + prefix = skill.repo_path.rstrip("/") + prefix_with_slash = f"{prefix}/" + + # `list_bucket_tree(prefix=...)` matches as a raw string prefix, so e.g. asking for + # "skills/gradio" can also return "skills/gradio-tools/...". Filter on the trailing + # slash to keep only files actually inside the directory, then strip it so files land + # directly under `install_dir` preserving any nested structure. + download_specs: list[tuple[str | BucketFile, str | Path]] = [] + for bucket_file in files: + if not bucket_file.path.startswith(prefix_with_slash): + continue + relative = bucket_file.path[len(prefix_with_slash) :] + local_file = install_dir.joinpath(*PurePosixPath(relative).parts) + local_file.parent.mkdir(parents=True, exist_ok=True) + download_specs.append((bucket_file, local_file)) + + if not download_specs: + raise FileNotFoundError(f"No files found under '{prefix}' in bucket '{DEFAULT_SKILLS_BUCKET_ID}'.") + + api.download_bucket_files(DEFAULT_SKILLS_BUCKET_ID, download_specs) + + +def _atomic_replace_directory(existing_dir: Path, staged_dir: Path) -> None: + backup_dir = staged_dir.parent / f"{existing_dir.name}.backup" + try: + existing_dir.rename(backup_dir) + staged_dir.rename(existing_dir) + shutil.rmtree(backup_dir) + except Exception: + if backup_dir.exists() and not existing_dir.exists(): + backup_dir.rename(existing_dir) + raise + + +def _iter_unique_skill_dirs(roots: list[Path]) -> list[Path]: + seen: set[Path] = set() + discovered: list[Path] = [] + for root in roots: + root = root.expanduser().resolve() + if not root.is_dir(): + continue + for child in sorted(root.iterdir()): + if child.name.startswith("."): + continue + if not child.is_dir() and not child.is_symlink(): + continue + resolved = child.resolve() + if resolved in seen or not resolved.is_dir(): + continue + seen.add(resolved) + discovered.append(resolved) + return discovered + + +def _apply_single_update( + api, skill_dir: Path, marketplace_skills: dict[str, MarketplaceSkill], hf_cli_content: str +) -> SkillUpdateInfo: + base = SkillUpdateInfo(name=skill_dir.name, skill_dir=skill_dir, status="unmanaged") + + if not (skill_dir / MANAGED_MARKER_FILENAME).exists(): + return base + + if skill_dir.name == DEFAULT_SKILL_ID: + try: + install_generated_skill(hf_cli_content, skill_dir.parent, force=True) + except Exception as exc: + return replace(base, status="source_unreachable", detail=str(exc)) + return replace(base, status="up_to_date") + + skill = marketplace_skills.get(skill_dir.name.lower()) + if skill is None: + return replace( + base, + status="source_unreachable", + detail=f"Skill '{skill_dir.name}' is no longer available in {DEFAULT_SKILLS_BUCKET_ID}.", + ) + + try: + _install_marketplace_skill(api, skill, skill_dir.parent, force=True) + except Exception as exc: + return replace(base, status="source_unreachable", detail=str(exc)) + + return replace(base, status="up_to_date") diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/auth.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/auth.py new file mode 100644 index 0000000000000000000000000000000000000000..e04bfa316eef8899045b3d53b62ad88809bb35ab --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/auth.py @@ -0,0 +1,174 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to authenticate to the Hugging Face Hub and interact with your repositories.""" + +from typing import Annotated + +import click + +from huggingface_hub.constants import ENDPOINT +from huggingface_hub.hf_api import whoami + +from .._login import _save_oauth_token, auth_list, auth_switch, login, logout +from ..errors import CLIError +from ..utils import get_stored_tokens, get_token, logging, select_choice +from ..utils._oauth_device import poll_device_token, request_device_code +from ._cli_utils import TokenOpt, typer_factory +from ._framework import Option +from ._output import OutputFormat, out + + +logger = logging.get_logger(__name__) + + +auth_cli = typer_factory(help="Manage authentication (login, logout, etc.).") + + +@auth_cli.command( + "login", + examples=[ + "hf auth login", + "hf auth login --token $HF_TOKEN", + "hf auth login --token $HF_TOKEN --add-to-git-credential", + "hf auth login --force", + ], +) +def auth_login( + token: TokenOpt = None, + add_to_git_credential: Annotated[ + bool, + Option( + help="Save to git credential helper. Useful only if you plan to run git commands directly.", + ), + ] = False, + force: Annotated[ + bool, + Option( + help="Force re-login even if already logged in.", + ), + ] = False, +) -> None: + """Login from your browser, or using a token from huggingface.co/settings/tokens.""" + if token is not None or out.mode == OutputFormat.human: + # `--token` bypasses any prompt; in human mode the gh-style menu lives in `login()`. + login(token=token, add_to_git_credential=add_to_git_credential, skip_if_logged_in=not force) + return + + # Logging in is an interactive flow: besides human mode, only agent mode is supported. + if out.mode != OutputFormat.agent: + raise CLIError( + "`hf auth login` is interactive and does not support --format json/quiet. " + "Pass --token for a non-interactive login." + ) + + # agent mode: never prompt; print instructions the agent can relay to its user. + if not force and get_token() is not None: + out.text(agent="Already logged in. Use `hf auth login --force` to re-login.") + return + device_info = request_device_code() + out.text( + agent=( + f"Ask the user to open {device_info['verification_uri_complete']} in a browser and enter the code " + f"{device_info['user_code']}. The code expires in {device_info['expires_in']} seconds. " + "Waiting for authorization..." + ) + ) + response = poll_device_token(device_info) + token_name, username = _save_oauth_token(response) + out.text(agent=f"Login successful: logged in as {username} (token saved as '{token_name}').") + + +@auth_cli.command( + "logout", + examples=["hf auth logout", "hf auth logout --token-name my-token"], +) +def auth_logout( + token_name: Annotated[ + str | None, + Option(help="Name of token to logout"), + ] = None, +) -> None: + """Logout from a specific token.""" + logout(token_name=token_name) + + +def _select_token_name() -> str | None: + token_names = list(get_stored_tokens().keys()) + + if not token_names: + logger.error("No stored tokens found. Please login first.") + return None + + if out.mode != OutputFormat.human: + raise CLIError("Use --token-name to select a token in non-interactive mode.") + return token_names[select_choice("Select a token to switch to:", token_names)] + + +@auth_cli.command( + "switch", + examples=["hf auth switch", "hf auth switch --token-name my-token"], +) +def auth_switch_cmd( + token_name: Annotated[ + str | None, + Option( + help="Name of the token to switch to", + ), + ] = None, + add_to_git_credential: Annotated[ + bool, + Option( + help="Save to git credential helper. Useful only if you plan to run git commands directly.", + ), + ] = False, +) -> None: + """Switch between access tokens.""" + if token_name is None: + token_name = _select_token_name() + if token_name is None: + print("No token name provided. Aborting.") + raise click.exceptions.Exit() + auth_switch(token_name, add_to_git_credential=add_to_git_credential) + + +@auth_cli.command("list | ls", examples=["hf auth list"]) +def auth_list_cmd() -> None: + """List all stored access tokens.""" + auth_list() + + +@auth_cli.command("token", examples=["hf auth token", "hf auth token | xargs curl -H 'Authorization: Bearer {}'"]) +def auth_token() -> None: + """Print the current access token to stdout.""" + token = get_token() + if token is None: + out.error("Not logged in. Run `hf auth login` first.") + raise click.exceptions.Exit(code=1) + print(token) + out.hint("Run `hf auth whoami` to see which account this token belongs to.") + + +@auth_cli.command("whoami", examples=["hf auth whoami", "hf auth whoami --format json"]) +def auth_whoami() -> None: + """Find out which huggingface.co account you are logged in as.""" + + token = get_token() + if token is None: + out.error("Not logged in") + raise click.exceptions.Exit(code=1) + + info = whoami(token) + orgs = ",".join(org["name"] for org in info["orgs"]) or None + endpoint = ENDPOINT if ENDPOINT != "https://huggingface.co" else None + out.result("Logged in", user=info["name"], orgs=orgs, endpoint=endpoint) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/buckets.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/buckets.py new file mode 100644 index 0000000000000000000000000000000000000000..9b84e77014902c14da41466e461ea854d75f1595 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/buckets.py @@ -0,0 +1,679 @@ +# Copyright 2025-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with buckets via the CLI.""" + +from typing import Annotated + +import click + +from huggingface_hub import logging +from huggingface_hub._buckets import ( + BUCKET_PREFIX, + BucketFile, + FilterMatcher, + _parse_bucket_uri, +) + +from ..hf_api import REPO_REGIONS +from ._cli_utils import ( + SearchOpt, + TokenOpt, + get_hf_api, + typer_factory, +) +from ._cp import make_cp +from ._file_listing import format_size, print_file_listing +from ._framework import Argument, Option +from ._output import OutputFormat, out + + +logger = logging.get_logger(__name__) + + +buckets_cli = typer_factory(help="Commands to interact with buckets.") + + +@buckets_cli.command( + name="create", + examples=[ + "hf buckets create my-bucket", + "hf buckets create user/my-bucket", + "hf buckets create hf://buckets/user/my-bucket", + "hf buckets create user/my-bucket --private", + "hf buckets create user/my-bucket --exist-ok", + "hf buckets create user/my-bucket --region us", + ], +) +def create( + bucket_id: Annotated[ + str, + Argument( + help="Bucket ID: bucket_name, namespace/bucket_name, or hf://buckets/namespace/bucket_name", + ), + ], + private: Annotated[ + bool, + Option( + "--private", + help="Create a private bucket.", + ), + ] = False, + region: Annotated[ + REPO_REGIONS | None, + Option( + "--region", + help="Cloud region in which to create the bucket. Can be one of 'us' or 'eu'. Requires Team plan or above.", + ), + ] = None, + exist_ok: Annotated[ + bool, + Option( + "--exist-ok", + help="Do not raise an error if the bucket already exists.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + """Create a new bucket.""" + api = get_hf_api(token=token) + + if bucket_id.startswith(BUCKET_PREFIX): + parsed = _parse_bucket_uri(bucket_id) + if parsed.path_in_repo: + raise click.BadParameter( + f"Cannot specify a prefix for bucket creation: {bucket_id}." + f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name." + ) + bucket_id = parsed.id + + bucket_url = api.create_bucket( + bucket_id, + private=private if private else None, + region=region, + exist_ok=exist_ok, + ) + out.result("Bucket created", uri=bucket_url.uri.to_uri(), url=bucket_url.url) + + +def _is_bucket_id(argument: str) -> bool: + """Check if argument is a bucket ID (namespace/name) vs just a namespace.""" + if argument.startswith(BUCKET_PREFIX): + path = argument[len(BUCKET_PREFIX) :] + else: + path = argument + return "/" in path + + +@buckets_cli.command( + name="list | ls", + examples=[ + "hf buckets list", + "hf buckets list huggingface", + 'hf buckets list --search "my-prefix"', + "hf buckets list user/my-bucket", + "hf buckets list user/my-bucket -R", + "hf buckets list user/my-bucket -h", + "hf buckets list user/my-bucket --tree", + "hf buckets list user/my-bucket --tree -h", + "hf buckets list hf://buckets/user/my-bucket", + "hf buckets list user/my-bucket/sub -R", + ], +) +def list_cmd( + argument: Annotated[ + str | None, + Argument( + help=( + "Namespace (user or org) to list buckets, or bucket ID" + " (namespace/bucket_name(/prefix) or hf://buckets/...) to list files." + ), + ), + ] = None, + human_readable: Annotated[ + bool, + Option( + "--human-readable", + "-h", + help="Show sizes in human readable format.", + ), + ] = False, + as_tree: Annotated[ + bool, + Option( + "--tree", + help="List files in tree format (only for listing files).", + ), + ] = False, + recursive: Annotated[ + bool, + Option( + "--recursive", + "-R", + help="List files recursively (only for listing files).", + ), + ] = False, + search: SearchOpt = None, + token: TokenOpt = None, +) -> None: + """List buckets or files in a bucket. + + When called with no argument or a namespace, lists buckets. + When called with a bucket ID (namespace/bucket_name), lists files in the bucket. + """ + # Determine mode: listing buckets or listing files + is_file_mode = argument is not None and _is_bucket_id(argument) + + if is_file_mode: + if search is not None: + raise click.BadParameter("Cannot use --search when listing files.") + _list_files( + argument=argument, # type: ignore + human_readable=human_readable, + as_tree=as_tree, + recursive=recursive, + token=token, + ) + else: + _list_buckets( + namespace=argument, + search=search, + human_readable=human_readable, + as_tree=as_tree, + recursive=recursive, + token=token, + ) + + +def _list_buckets( + namespace: str | None, + search: str | None, + human_readable: bool, + as_tree: bool, + recursive: bool, + token: str | None, +) -> None: + """List buckets in a namespace.""" + # Validate incompatible flags + if as_tree: + raise click.BadParameter("Cannot use --tree when listing buckets.") + if recursive: + raise click.BadParameter("Cannot use --recursive when listing buckets.") + + # Handle hf://buckets/namespace format + if namespace is not None and namespace.startswith(BUCKET_PREFIX): + namespace = namespace[len(BUCKET_PREFIX) :] + # Strip trailing slash if any + namespace = namespace.rstrip("/") + + api = get_hf_api(token=token) + items = [ + { + "id": bucket.id, + "private": bucket.private, + "size": format_size(bucket.size, human_readable) if human_readable else bucket.size, + "total_files": bucket.total_files, + "created_at": bucket.created_at, + } + for bucket in api.list_buckets(namespace=namespace, search=search) + ] + out.table(items, alignments={"size": "right"}) + + +def _list_files( + argument: str, + human_readable: bool, + as_tree: bool, + recursive: bool, + token: str | None, +) -> None: + """List files in a bucket.""" + if as_tree and out.mode == OutputFormat.json: + raise click.BadParameter("Cannot use --tree with --format json.") + + api = get_hf_api(token=token) + parsed = _parse_bucket_uri(argument) + items = list( + api.list_bucket_tree( + parsed.id, + prefix=parsed.path_in_repo or None, + recursive=recursive, + ) + ) + + print_file_listing(items, human_readable=human_readable, as_tree=as_tree, recursive=recursive) + + +@buckets_cli.command( + name="info", + examples=[ + "hf buckets info user/my-bucket", + "hf buckets info hf://buckets/user/my-bucket", + ], +) +def info( + bucket_id: Annotated[ + str, + Argument( + help="Bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name", + ), + ], + token: TokenOpt = None, +) -> None: + """Get info about a bucket.""" + api = get_hf_api(token=token) + parsed = _parse_bucket_uri(bucket_id) + bucket = api.bucket_info(parsed.id) + out.dict(bucket, id_key="id") + + +@buckets_cli.command( + name="delete", + examples=[ + "hf buckets delete user/my-bucket", + "hf buckets delete hf://buckets/user/my-bucket", + "hf buckets delete user/my-bucket --yes", + "hf buckets delete user/my-bucket --missing-ok", + ], +) +def delete( + bucket_id: Annotated[ + str, + Argument( + help="Bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name", + ), + ], + yes: Annotated[ + bool, + Option( + "--yes", + "-y", + help="Skip confirmation prompt.", + ), + ] = False, + missing_ok: Annotated[ + bool, + Option( + "--missing-ok", + help="Do not raise an error if the bucket does not exist.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + """Delete a bucket. + + This deletes the entire bucket and all its contents. Use `hf buckets rm` to remove individual files. + """ + if bucket_id.startswith(BUCKET_PREFIX): + parsed = _parse_bucket_uri(bucket_id) + if parsed.path_in_repo: + raise click.BadParameter( + f"Cannot specify a prefix for bucket deletion: {bucket_id}." + f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name." + ) + bucket_id = parsed.id + elif "/" not in bucket_id: + raise click.BadParameter( + f"Invalid bucket ID: {bucket_id}." + f" Must be in format namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name." + ) + + out.confirm(f"Are you sure you want to delete bucket '{bucket_id}'?", yes=yes) + + api = get_hf_api(token=token) + api.delete_bucket(bucket_id, missing_ok=missing_ok) + out.result("Bucket deleted", bucket_id=bucket_id) + + +@buckets_cli.command( + name="remove | rm", + examples=[ + "hf buckets remove user/my-bucket/file.txt", + "hf buckets rm hf://buckets/user/my-bucket/file.txt", + "hf buckets rm user/my-bucket/logs/ --recursive", + 'hf buckets rm user/my-bucket --recursive --include "*.tmp"', + "hf buckets rm user/my-bucket/data/ --recursive --dry-run", + ], +) +def remove( + argument: Annotated[ + str, + Argument( + help=( + "Bucket path: namespace/bucket_name/path or hf://buckets/namespace/bucket_name/path." + " With --recursive, namespace/bucket_name is also accepted to target all files." + ), + ), + ], + recursive: Annotated[ + bool, + Option( + "--recursive", + "-R", + help="Remove files recursively under the given prefix.", + ), + ] = False, + yes: Annotated[ + bool, + Option( + "--yes", + "-y", + help="Skip confirmation prompt.", + ), + ] = False, + dry_run: Annotated[ + bool, + Option( + "--dry-run", + help="Preview what would be deleted without actually deleting.", + ), + ] = False, + include: Annotated[ + list[str] | None, + Option( + help="Include only files matching pattern (can specify multiple). Requires --recursive.", + ), + ] = None, + exclude: Annotated[ + list[str] | None, + Option( + help="Exclude files matching pattern (can specify multiple). Requires --recursive.", + ), + ] = None, + token: TokenOpt = None, +) -> None: + """Remove files from a bucket. + + To delete an entire bucket, use `hf buckets delete` instead. + """ + parsed = _parse_bucket_uri(argument) + bucket_id = parsed.id + prefix = parsed.path_in_repo + + if prefix == "" and not recursive: + raise click.BadParameter( + f"No file path specified. To remove files, provide a path" + f" (e.g. '{bucket_id}/FILE') or use --recursive to remove all files." + f" To delete the entire bucket, use `hf buckets delete {bucket_id}`." + ) + + if (include or exclude) and not recursive: + raise click.BadParameter("--include and --exclude require --recursive.") + + api = get_hf_api(token=token) + + if recursive: + status = out.status("Listing files from remote") + + all_files: list[BucketFile] = [] + for item in api.list_bucket_tree( + bucket_id, + prefix=prefix or None, + recursive=True, + ): + if isinstance(item, BucketFile): + all_files.append(item) + status.update(f"Listing files from remote ({len(all_files)} files)") + status.done(f"Listing files from remote ({len(all_files)} files)") + + if include or exclude: + matcher = FilterMatcher(include_patterns=include, exclude_patterns=exclude) + matched_files = [f for f in all_files if matcher.matches(f.path)] + else: + matched_files = all_files + + file_paths = [f.path for f in matched_files] + total_size = sum(f.size for f in matched_files) + size_str = format_size(total_size, human_readable=True) + + if not file_paths: + out.text("No files to remove.") + return + + count_label = f"{len(file_paths)} file(s) totaling {size_str}" + + if not yes and not dry_run: + out.text("\n".join(f" {path}" for path in file_paths)) + out.confirm(f"Remove {count_label} from '{bucket_id}'?", yes=False) + + if dry_run: + out.text("\n".join(f"delete: {BUCKET_PREFIX}{bucket_id}/{path}" for path in file_paths)) + out.text(f"(dry run) {count_label} would be removed.") + return + + api.batch_bucket_files(bucket_id, delete=file_paths) + out.result( + f"Removed {count_label} from '{bucket_id}'", + bucket_id=bucket_id, + files_deleted=len(file_paths), + size=size_str, + ) + + else: + file_path = prefix + if not file_path: + raise click.BadParameter("File path cannot be empty.") + + if dry_run: + out.text(f"delete: {BUCKET_PREFIX}{bucket_id}/{file_path}") + out.text("(dry run) 1 file would be removed.") + return + + out.confirm(f"Remove '{file_path}' from '{bucket_id}'?", yes=yes) + + api.batch_bucket_files(bucket_id, delete=[file_path]) + out.result("File removed", path=file_path, bucket_id=bucket_id) + + +@buckets_cli.command( + name="move", + examples=[ + "hf buckets move user/old-bucket user/new-bucket", + "hf buckets move user/my-bucket my-org/my-bucket", + "hf buckets move hf://buckets/user/old-bucket hf://buckets/user/new-bucket", + ], +) +def move( + from_id: Annotated[ + str, + Argument( + help="Source bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name", + ), + ], + to_id: Annotated[ + str, + Argument( + help="Destination bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_name", + ), + ], + token: TokenOpt = None, +) -> None: + """Move (rename) a bucket to a new name or namespace.""" + # Parse from_id + parsed_from = _parse_bucket_uri(from_id) + if parsed_from.path_in_repo: + raise click.BadParameter( + f"Cannot specify a prefix for bucket move: {from_id}." + f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name." + ) + + # Parse to_id + parsed_to = _parse_bucket_uri(to_id) + if parsed_to.path_in_repo: + raise click.BadParameter( + f"Cannot specify a prefix for bucket move: {to_id}." + f" Use namespace/bucket_name or {BUCKET_PREFIX}namespace/bucket_name." + ) + + api = get_hf_api(token=token) + api.move_bucket(from_id=parsed_from.id, to_id=parsed_to.id) + out.result("Bucket moved", from_id=parsed_from.id, to_id=parsed_to.id) + + +# ============================================================================= +# Sync command +# ============================================================================= + + +@buckets_cli.command( + name="sync", + examples=[ + "hf buckets sync ./data hf://buckets/user/my-bucket", + "hf buckets sync hf://buckets/user/my-bucket ./data", + "hf buckets sync ./data hf://buckets/user/my-bucket --delete", + 'hf buckets sync hf://buckets/user/my-bucket ./data --include "*.safetensors" --exclude "*.tmp"', + "hf buckets sync ./data hf://buckets/user/my-bucket --plan sync-plan.jsonl", + "hf buckets sync --apply sync-plan.jsonl", + "hf buckets sync ./data hf://buckets/user/my-bucket --dry-run", + "hf buckets sync ./data hf://buckets/user/my-bucket --dry-run | jq .", + ], +) +def sync( + source: Annotated[ + str | None, + Argument( + help="Source path: local directory or hf://buckets/namespace/bucket_name(/prefix)", + ), + ] = None, + dest: Annotated[ + str | None, + Argument( + help="Destination path: local directory or hf://buckets/namespace/bucket_name(/prefix)", + ), + ] = None, + delete: Annotated[ + bool, + Option( + help="Delete destination files not present in source.", + ), + ] = False, + ignore_times: Annotated[ + bool, + Option( + "--ignore-times", + help="Skip files only based on size, ignoring modification times.", + ), + ] = False, + ignore_sizes: Annotated[ + bool, + Option( + "--ignore-sizes", + help="Skip files only based on modification times, ignoring sizes.", + ), + ] = False, + plan: Annotated[ + str | None, + Option( + help="Save sync plan to JSONL file for review instead of executing.", + ), + ] = None, + apply: Annotated[ + str | None, + Option( + help="Apply a previously saved plan file.", + ), + ] = None, + dry_run: Annotated[ + bool, + Option( + "--dry-run", + help="Print sync plan to stdout as JSONL without executing.", + ), + ] = False, + include: Annotated[ + list[str] | None, + Option( + help="Include files matching pattern (can specify multiple).", + ), + ] = None, + exclude: Annotated[ + list[str] | None, + Option( + help="Exclude files matching pattern (can specify multiple).", + ), + ] = None, + filter_from: Annotated[ + str | None, + Option( + help="Read include/exclude patterns from file.", + ), + ] = None, + existing: Annotated[ + bool, + Option( + "--existing", + help="Skip creating new files on receiver (only update existing files).", + ), + ] = False, + ignore_existing: Annotated[ + bool, + Option( + "--ignore-existing", + help="Skip updating files that exist on receiver (only create new files).", + ), + ] = False, + verbose: Annotated[ + bool, + Option( + "--verbose", + "-v", + help="Show detailed logging with reasoning.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + """Sync files between local directory and a bucket.""" + api = get_hf_api(token=token) + api.sync_bucket( + source=source, + dest=dest, + delete=delete, + ignore_times=ignore_times, + ignore_sizes=ignore_sizes, + existing=existing, + ignore_existing=ignore_existing, + include=include, + exclude=exclude, + filter_from=filter_from, + plan=plan, + apply=apply, + dry_run=dry_run, + verbose=verbose, + quiet=out.is_quiet(), + ) + if plan and not out.is_quiet(): + out.hint(f"Run `hf buckets sync --apply {plan}` to execute this plan.") + + +# ============================================================================= +# Cp command +# ============================================================================= + + +# `hf buckets cp` is an alias for the top-level `hf cp` command (see `cli/_cp.py`). +buckets_cli.command( + name="cp", + examples=[ + # Download (repo or bucket -> local / stdout) + "hf buckets cp hf://buckets/username/my-bucket/config.json config.json", + "hf buckets cp hf://buckets/username/my-bucket/data.csv data/", + "hf buckets cp hf://buckets/username/my-bucket/config.json -", + # Upload (local / stdin -> bucket) + "hf buckets cp model.safetensors hf://buckets/username/my-bucket/model.safetensors", + "hf buckets cp config.json hf://buckets/username/my-bucket/logs/", + "hf buckets cp - hf://buckets/username/my-bucket/config.json", + # Remote to remote (repo or bucket -> bucket) + "hf buckets cp hf://buckets/username/my-bucket/data.csv hf://buckets/username/dest-bucket/", + "hf buckets cp hf://buckets/username/source-bucket/logs/ hf://buckets/username/dest-bucket/logs/", + ], +)(make_cp("buckets")) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/cache.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..df13ee6b0b41aa92b9ee4a825f40d54730a4af61 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/cache.py @@ -0,0 +1,815 @@ +# Copyright 2025-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains the 'hf cache' command group with cache management subcommands.""" + +import re +import time +from collections import defaultdict +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from enum import Enum +from typing import Annotated, Any + +import click + +from huggingface_hub.errors import CLIError + +from ..utils import ( + ANSI, + CachedRepoInfo, + CachedRevisionInfo, + CacheNotFound, + HFCacheInfo, + _format_size, + parse_hf_uri, + scan_cache_dir, +) +from ..utils._parsing import parse_duration, parse_size +from ._cli_utils import RepoIdArg, RepoTypeOpt, RevisionOpt, TokenOpt, get_hf_api, typer_factory +from ._framework import Argument, Option +from ._output import out + + +cache_cli = typer_factory(help="Manage local cache directory.") + + +#### Cache helper utilities + + +@dataclass(frozen=True) +class _DeletionResolution: + revisions: frozenset[str] + selected: dict[CachedRepoInfo, frozenset[CachedRevisionInfo]] + missing: tuple[str, ...] + + +_FILTER_PATTERN = re.compile(r"^(?P[a-zA-Z_]+)\s*(?P==|!=|>=|<=|>|<|=)\s*(?P.+)$") +_ALLOWED_OPERATORS = {"=", "!=", ">", "<", ">=", "<="} +_FILTER_KEYS = {"accessed", "modified", "refs", "size", "type"} +_SORT_KEYS = {"accessed", "modified", "name", "size"} +_SORT_PATTERN = re.compile(r"^(?P[a-zA-Z_]+)(?::(?Pasc|desc))?$") +_SORT_DEFAULT_ORDER = { + # Default ordering: accessed/modified/size are descending (newest/biggest first), name is ascending + "accessed": "desc", + "modified": "desc", + "size": "desc", + "name": "asc", +} + + +# Dynamically generate SortOptions enum from _SORT_KEYS +_sort_options_dict = {} +for key in sorted(_SORT_KEYS): + _sort_options_dict[key] = key + _sort_options_dict[f"{key}_asc"] = f"{key}:asc" + _sort_options_dict[f"{key}_desc"] = f"{key}:desc" + +SortOptions = Enum("SortOptions", _sort_options_dict, type=str, module=__name__) # type: ignore + + +@dataclass(frozen=True) +class CacheDeletionCounts: + """Simple counters summarizing cache deletions for CLI messaging.""" + + repo_count: int + partial_revision_count: int + total_revision_count: int + + +CacheEntry = tuple[CachedRepoInfo, CachedRevisionInfo | None] +RepoRefsMap = dict[CachedRepoInfo, frozenset[str]] + + +def summarize_deletions( + selected_by_repo: Mapping[CachedRepoInfo, frozenset[CachedRevisionInfo]], +) -> CacheDeletionCounts: + """Summarize deletions across repositories.""" + repo_count = 0 + total_revisions = 0 + revisions_in_full_repos = 0 + + for repo, revisions in selected_by_repo.items(): + total_revisions += len(revisions) + if len(revisions) == len(repo.revisions): + repo_count += 1 + revisions_in_full_repos += len(revisions) + + partial_revision_count = total_revisions - revisions_in_full_repos + return CacheDeletionCounts(repo_count, partial_revision_count, total_revisions) + + +def _prune_summary(revision_count: int, incomplete_count: int) -> str: + """Build the human-readable summary of what `hf cache prune` is about to delete.""" + parts: list[str] = [] + if revision_count: + parts.append(f"{revision_count} unreferenced revision(s)") + if incomplete_count: + parts.append(f"{incomplete_count} incomplete download(s)") + return " and ".join(parts) + + +def print_cache_selected_revisions(selected_by_repo: Mapping[CachedRepoInfo, frozenset[CachedRevisionInfo]]) -> None: + """Pretty-print selected cache revisions during confirmation prompts.""" + for repo in sorted(selected_by_repo.keys(), key=lambda repo: (repo.repo_type, repo.repo_id.lower())): + repo_key = f"{repo.repo_type}/{repo.repo_id}" + revisions = sorted(selected_by_repo[repo], key=lambda rev: rev.commit_hash) + if len(revisions) == len(repo.revisions): + out.text(f" - {repo_key} (entire repo)") + continue + + out.text(f" - {repo_key}:") + for revision in revisions: + refs = " ".join(sorted(revision.refs)) or "(detached)" + out.text(f" {revision.commit_hash} [{refs}] {revision.size_on_disk_str}") + + +def build_cache_index( + hf_cache_info: HFCacheInfo, +) -> tuple[ + dict[str, CachedRepoInfo], + dict[str, tuple[CachedRepoInfo, CachedRevisionInfo]], +]: + """Create lookup tables so CLI commands can resolve repo ids and revisions quickly.""" + repo_lookup: dict[str, CachedRepoInfo] = {} + revision_lookup: dict[str, tuple[CachedRepoInfo, CachedRevisionInfo]] = {} + for repo in hf_cache_info.repos: + repo_key = repo.cache_id.lower() + repo_lookup[repo_key] = repo + for revision in repo.revisions: + revision_lookup[revision.commit_hash.lower()] = (repo, revision) + return repo_lookup, revision_lookup + + +def _repo_cache_id_from_target(target: str) -> str: + """Return the cache id matching a repo target passed to `hf cache rm`.""" + if not target.startswith("hf://"): + return target + + uri = parse_hf_uri(target) + if not uri.is_repo: + raise CLIError("Only repository hf:// URIs are supported by `hf cache rm`.") + if uri.revision is not None or uri.path_in_repo: + raise CLIError("Only repo-level hf:// URIs are supported by `hf cache rm` for now.") + return f"{uri.type}/{uri.id}" + + +def collect_cache_entries( + hf_cache_info: HFCacheInfo, *, include_revisions: bool +) -> tuple[list[CacheEntry], RepoRefsMap]: + """Flatten cache metadata into rows consumed by `hf cache ls`.""" + entries: list[CacheEntry] = [] + repo_refs_map: RepoRefsMap = {} + sorted_repos = sorted(hf_cache_info.repos, key=lambda repo: (repo.repo_type, repo.repo_id.lower())) + for repo in sorted_repos: + repo_refs_map[repo] = frozenset({ref for revision in repo.revisions for ref in revision.refs}) + if include_revisions: + for revision in sorted(repo.revisions, key=lambda rev: rev.commit_hash): + entries.append((repo, revision)) + else: + entries.append((repo, None)) + if include_revisions: + entries.sort( + key=lambda entry: ( + entry[0].cache_id, + entry[1].commit_hash if entry[1] is not None else "", + ) + ) + else: + entries.sort(key=lambda entry: entry[0].cache_id) + return entries, repo_refs_map + + +def compile_cache_filter( + expr: str, repo_refs_map: RepoRefsMap +) -> Callable[[CachedRepoInfo, CachedRevisionInfo | None, float], bool]: + """Convert a `hf cache ls` filter expression into the yes/no test we apply to each cache entry before displaying it.""" + match = _FILTER_PATTERN.match(expr.strip()) + if not match: + raise ValueError(f"Invalid filter expression: '{expr}'.") + + key = match.group("key").lower() + op = match.group("op") + value_raw = match.group("value").strip() + + if op not in _ALLOWED_OPERATORS: + raise ValueError(f"Unsupported operator '{op}' in filter '{expr}'. Must be one of {list(_ALLOWED_OPERATORS)}.") + + if key not in _FILTER_KEYS: + raise ValueError(f"Unsupported filter key '{key}' in '{expr}'. Must be one of {list(_FILTER_KEYS)}.") + # at this point we know that key is in `_FILTER_KEYS` + if key == "size": + size_threshold = parse_size(value_raw) + return lambda repo, revision, _: _compare_numeric( + revision.size_on_disk if revision is not None else repo.size_on_disk, + op, + size_threshold, + ) + + if key in {"modified", "accessed"}: + seconds = parse_duration(value_raw.strip()) + + def _time_filter(repo: CachedRepoInfo, revision: CachedRevisionInfo | None, now: float) -> bool: + timestamp = ( + repo.last_accessed + if key == "accessed" + else revision.last_modified + if revision is not None + else repo.last_modified + ) + if timestamp is None: + return False + return _compare_numeric(now - timestamp, op, seconds) + + return _time_filter + + if key == "type": + expected = value_raw.lower() + + if op != "=": + raise ValueError(f"Only '=' is supported for 'type' filters. Got '{op}'.") + + def _type_filter(repo: CachedRepoInfo, revision: CachedRevisionInfo | None, _: float) -> bool: + return repo.repo_type.lower() == expected + + return _type_filter + + else: # key == "refs" + if op != "=": + raise ValueError(f"Only '=' is supported for 'refs' filters. Got {op}.") + + def _refs_filter(repo: CachedRepoInfo, revision: CachedRevisionInfo | None, _: float) -> bool: + refs = revision.refs if revision is not None else repo_refs_map.get(repo, frozenset()) + return value_raw.lower() in [ref.lower() for ref in refs] + + return _refs_filter + + +def _compare_numeric(left: float | None, op: str, right: float) -> bool: + """Evaluate numeric comparisons for filters.""" + if left is None: + return False + + comparisons = { + "=": left == right, + "!=": left != right, + ">": left > right, + "<": left < right, + ">=": left >= right, + "<=": left <= right, + } + + if op not in comparisons: + raise ValueError(f"Unsupported numeric comparison operator: {op}") + + return comparisons[op] + + +def compile_cache_sort(sort_expr: str) -> tuple[Callable[[CacheEntry], tuple[Any, ...]], bool]: + """Convert a `hf cache ls` sort expression into a key function for sorting entries. + + Returns: + A tuple of (key_function, reverse_flag) where reverse_flag indicates whether + to sort in descending order (True) or ascending order (False). + """ + match = _SORT_PATTERN.match(sort_expr.strip().lower()) + if not match: + raise ValueError(f"Invalid sort expression: '{sort_expr}'. Expected format: 'key' or 'key:asc' or 'key:desc'.") + + key = match.group("key").lower() + explicit_order = match.group("order") + + if key not in _SORT_KEYS: + raise ValueError(f"Unsupported sort key '{key}' in '{sort_expr}'. Must be one of {list(_SORT_KEYS)}.") + + # Use explicit order if provided, otherwise use default for the key + order = explicit_order if explicit_order else _SORT_DEFAULT_ORDER[key] + reverse = order == "desc" + + def _sort_key(entry: CacheEntry) -> tuple[Any, ...]: + repo, revision = entry + + if key == "name": + # Sort by cache_id (repo type/id) + value: Any = repo.cache_id.lower() + return (value,) + + if key == "size": + # Use revision size if available, otherwise repo size + value = revision.size_on_disk if revision is not None else repo.size_on_disk + return (value,) + + if key == "accessed": + # For revisions, accessed is not available per-revision, use repo's last_accessed + # For repos, use repo's last_accessed + value = repo.last_accessed if repo.last_accessed is not None else 0.0 + return (value,) + + if key == "modified": + # Use revision's last_modified if available, otherwise repo's last_modified + if revision is not None: + value = revision.last_modified if revision.last_modified is not None else 0.0 + else: + value = repo.last_modified if repo.last_modified is not None else 0.0 + return (value,) + + # Should never reach here due to validation above + raise ValueError(f"Unsupported sort key: {key}") + + return _sort_key, reverse + + +def _resolve_deletion_targets(hf_cache_info: HFCacheInfo, targets: list[str]) -> _DeletionResolution: + """Resolve the deletion targets into a deletion resolution.""" + repo_lookup, revision_lookup = build_cache_index(hf_cache_info) + + selected: dict[CachedRepoInfo, set[CachedRevisionInfo]] = defaultdict(set) + revisions: set[str] = set() + missing: list[str] = [] + + for raw_target in targets: + target = raw_target.strip() + if not target: + continue + lowered = target.lower() + + if re.fullmatch(r"[0-9a-fA-F]{40}", lowered): + match = revision_lookup.get(lowered) + if match is None: + missing.append(raw_target) + continue + repo, revision = match + selected[repo].add(revision) + revisions.add(revision.commit_hash) + continue + + matched_repo = repo_lookup.get(_repo_cache_id_from_target(target).lower()) + if matched_repo is None: + missing.append(raw_target) + continue + + for revision in matched_repo.revisions: + selected[matched_repo].add(revision) + revisions.add(revision.commit_hash) + + frozen_selected = {repo: frozenset(revs) for repo, revs in selected.items()} + return _DeletionResolution( + revisions=frozenset(revisions), + selected=frozen_selected, + missing=tuple(missing), + ) + + +#### Cache CLI commands + + +@cache_cli.command( + "list | ls", + examples=[ + "hf cache ls", + "hf cache ls --revisions", + 'hf cache ls --filter "size>1GB" --limit 20', + "hf cache ls --format json", + ], +) +def ls( + cache_dir: Annotated[ + str | None, + Option( + help="Cache directory to scan (defaults to Hugging Face cache).", + ), + ] = None, + revisions: Annotated[ + bool, + Option( + help="Include revisions in the output instead of aggregated repositories.", + ), + ] = False, + filter: Annotated[ + list[str] | None, + Option( + "-f", + "--filter", + help="Filter entries (e.g. 'size>1GB', 'type=model', 'accessed>7d'). Can be used multiple times.", + ), + ] = None, + sort: Annotated[ + SortOptions | None, + Option( + help="Sort entries by key. Supported keys: 'accessed', 'modified', 'name', 'size'. " + "Append ':asc' or ':desc' to explicitly set the order (e.g., 'modified:asc'). " + "Defaults: 'accessed', 'modified', 'size' default to 'desc' (newest/biggest first); " + "'name' defaults to 'asc' (alphabetical).", + ), + ] = None, + limit: Annotated[ + int | None, + Option( + help="Limit the number of results returned. Returns only the top N entries after sorting.", + ), + ] = None, + show_warnings: Annotated[ + bool, + Option( + help="Show warnings about cache inconsistencies.", + ), + ] = False, +) -> None: + """List cached repositories or revisions.""" + try: + hf_cache_info = scan_cache_dir(cache_dir) + except CacheNotFound as exc: + raise CLIError(f"Cache directory not found: {exc.cache_dir}") from exc + + filters = filter or [] + + entries, repo_refs_map = collect_cache_entries(hf_cache_info, include_revisions=revisions) + try: + filter_fns = [compile_cache_filter(expr, repo_refs_map) for expr in filters] + except ValueError as exc: + raise click.BadParameter(str(exc)) from exc + + now = time.time() + for fn in filter_fns: + entries = [entry for entry in entries if fn(entry[0], entry[1], now)] + + # Apply sorting if requested + if sort: + try: + sort_key_fn, reverse = compile_cache_sort(sort.value) + entries.sort(key=sort_key_fn, reverse=reverse) + except ValueError as exc: + raise click.BadParameter(str(exc)) from exc + + # Apply limit if requested + if limit is not None: + if limit < 0: + raise click.BadParameter(f"Limit must be a positive integer, got {limit}.") + entries = entries[:limit] + + if revisions: + items = [ + { + "id": repo.cache_id, + "repo_id": repo.repo_id, + "repo_type": repo.repo_type, + "revision": revision.commit_hash, + "snapshot_path": str(revision.snapshot_path), + "size": revision.size_on_disk_str, + "last_modified": revision.last_modified_str, + "refs": sorted(revision.refs), + } + for repo, revision in entries + if revision is not None + ] + out.table( + items, + headers=["id", "revision", "size", "last_modified", "refs"], + id_key="revision", + alignments={"size": "right"}, + ) + else: + items = [ + { + "id": repo.cache_id, + "repo_id": repo.repo_id, + "repo_type": repo.repo_type, + "size": repo.size_on_disk_str, + "last_accessed": repo.last_accessed_str or "", + "last_modified": repo.last_modified_str, + "refs": sorted(repo_refs_map.get(repo, frozenset())), + } + for repo, _ in entries + ] + out.table( + items, + headers=["id", "size", "last_accessed", "last_modified", "refs"], + id_key="id", + alignments={"size": "right"}, + ) + + if entries: + unique_repos = {repo for repo, _ in entries} + repo_count = len(unique_repos) + if revisions: + revision_count = sum(1 for _, rev in entries if rev is not None) + total_size = sum(rev.size_on_disk for _, rev in entries if rev is not None) + else: + revision_count = sum(len(repo.revisions) for repo in unique_repos) + total_size = sum(repo.size_on_disk for repo in unique_repos) + out.text( + ANSI.bold( + f"\nFound {repo_count} repo(s) for a total of {revision_count} revision(s)" + f" and {_format_size(total_size)} on disk." + ) + ) + + if len(hf_cache_info.warnings): + if show_warnings: + for warning in hf_cache_info.warnings: + out.warning(str(warning).rstrip(".") + ". Repo ignored.") + else: + out.warning( + f"Found {len(hf_cache_info.warnings)} cache inconsistencies. Re-run with `--show-warnings` to display them." + ) + + incomplete_files = hf_cache_info.incomplete_files + if incomplete_files: + out.hint( + f"Found {len(incomplete_files)} incomplete download(s) totalling " + f"{_format_size(hf_cache_info.incomplete_size_on_disk)}. " + "Remove them with 'hf cache prune'." + ) + + +@cache_cli.command( + examples=[ + "hf cache rm model/gpt2", + "hf cache rm hf://models/openai-community/gpt2", + "hf cache rm ", + "hf cache rm model/gpt2 --dry-run", + "hf cache rm model/gpt2 --yes", + ], +) +def rm( + targets: Annotated[ + list[str], + Argument( + help="One or more repo IDs (e.g. model/bert-base-uncased), repo-level hf:// URIs, or revision hashes to delete.", + ), + ], + cache_dir: Annotated[ + str | None, + Option( + help="Cache directory to scan (defaults to Hugging Face cache).", + ), + ] = None, + yes: Annotated[ + bool, + Option( + "-y", + "--yes", + help="Skip confirmation prompt.", + ), + ] = False, + dry_run: Annotated[ + bool, + Option( + help="Preview deletions without removing anything.", + ), + ] = False, +) -> None: + """Remove cached repositories or revisions.""" + try: + hf_cache_info = scan_cache_dir(cache_dir) + except CacheNotFound as exc: + raise CLIError(f"Cache directory not found: {exc.cache_dir}") from exc + + resolution = _resolve_deletion_targets(hf_cache_info, targets) + + if resolution.missing: + details = "\n".join(f" - {entry}" for entry in resolution.missing) + out.warning(f"Could not find in cache:\n{details}") + + if len(resolution.revisions) == 0: + out.text("Nothing to delete.") + raise click.exceptions.Exit(code=0) + + strategy = hf_cache_info.delete_revisions(*sorted(resolution.revisions)) + counts = summarize_deletions(resolution.selected) + + summary_parts: list[str] = [] + if counts.repo_count: + summary_parts.append(f"{counts.repo_count} repo(s)") + if counts.partial_revision_count: + summary_parts.append(f"{counts.partial_revision_count} revision(s)") + if not summary_parts: + summary_parts.append(f"{counts.total_revision_count} revision(s)") + + summary_text = " and ".join(summary_parts) + out.text(f"About to delete {summary_text} totalling {strategy.expected_freed_size_str}.") + print_cache_selected_revisions(resolution.selected) + + if dry_run: + out.result( + "Dry run: no files were deleted.", + dry_run=True, + repos=counts.repo_count, + revisions=counts.total_revision_count, + size=strategy.expected_freed_size_str, + ) + return + + out.confirm("Proceed with deletion?", yes=yes) + + strategy.execute() + counts = summarize_deletions(resolution.selected) + out.result( + f"Deleted {counts.repo_count} repo(s) and {counts.total_revision_count} revision(s);" + f" freed {strategy.expected_freed_size_str}.", + repos_deleted=counts.repo_count, + revisions_deleted=counts.total_revision_count, + freed=strategy.expected_freed_size_str, + ) + + +@cache_cli.command(examples=["hf cache prune", "hf cache prune --dry-run"]) +def prune( + cache_dir: Annotated[ + str | None, + Option( + help="Cache directory to scan (defaults to Hugging Face cache).", + ), + ] = None, + yes: Annotated[ + bool, + Option( + "-y", + "--yes", + help="Skip confirmation prompt.", + ), + ] = False, + dry_run: Annotated[ + bool, + Option( + help="Preview deletions without removing anything.", + ), + ] = False, +) -> None: + """Remove detached revisions and incomplete downloads from the cache.""" + try: + hf_cache_info = scan_cache_dir(cache_dir) + except CacheNotFound as exc: + raise CLIError(f"Cache directory not found: {exc.cache_dir}") from exc + + selected: dict[CachedRepoInfo, frozenset[CachedRevisionInfo]] = {} + revisions: set[str] = set() + for repo in hf_cache_info.repos: + detached = frozenset(revision for revision in repo.revisions if len(revision.refs) == 0) + if not detached: + continue + selected[repo] = detached + revisions.update(revision.commit_hash for revision in detached) + + incomplete_files = hf_cache_info.incomplete_files + + if len(revisions) == 0 and not incomplete_files: + out.text("No unreferenced revisions or incomplete downloads found. Nothing to prune.") + return + + strategy = hf_cache_info.delete_revisions(*sorted(revisions)) + counts = summarize_deletions(selected) + total_freed = strategy.expected_freed_size + hf_cache_info.incomplete_size_on_disk + + summary = _prune_summary(counts.total_revision_count, len(incomplete_files)) + out.text(f"About to delete {summary} ({_format_size(total_freed)} total).") + print_cache_selected_revisions(selected) + + if dry_run: + out.result( + "Dry run: no files were deleted.", + dry_run=True, + revisions=counts.total_revision_count, + incomplete=len(incomplete_files), # might be overstated but it's fine + size=_format_size(total_freed), + ) + return + + out.confirm("Proceed?", yes=yes) + + strategy.execute() + for incomplete_file in incomplete_files: + try: + incomplete_file.file_path.unlink() + except FileNotFoundError: + pass # already removed (e.g. by a full-repo deletion above) + except OSError as exc: + out.warning(f"Could not delete incomplete file {incomplete_file.file_path}: {exc}") + out.result( + f"Deleted {summary}; freed {_format_size(total_freed)}.", + revisions_deleted=counts.total_revision_count, + incomplete_deleted=len(incomplete_files), + freed=_format_size(total_freed), + ) + + +@cache_cli.command( + examples=[ + "hf cache verify gpt2", + "hf cache verify gpt2 --revision refs/pr/1", + "hf cache verify my-dataset --repo-type dataset", + ], +) +def verify( + repo_id: RepoIdArg, + repo_type: RepoTypeOpt = RepoTypeOpt.model, + revision: RevisionOpt = None, + cache_dir: Annotated[ + str | None, + Option( + help="Cache directory to use when verifying files from cache (defaults to Hugging Face cache).", + ), + ] = None, + local_dir: Annotated[ + str | None, + Option( + help="If set, verify files under this directory instead of the cache.", + ), + ] = None, + fail_on_missing_files: Annotated[ + bool, + Option( + "--fail-on-missing-files", + help="Fail if some files exist on the remote but are missing locally.", + ), + ] = False, + fail_on_extra_files: Annotated[ + bool, + Option( + "--fail-on-extra-files", + help="Fail if some files exist locally but are not present on the remote revision.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + """Verify checksums for a single repo revision from cache or a local directory. + + Examples: + - Verify main revision in cache: `hf cache verify gpt2` + - Verify specific revision: `hf cache verify gpt2 --revision refs/pr/1` + - Verify dataset: `hf cache verify karpathy/fineweb-edu-100b-shuffle --repo-type dataset` + - Verify local dir: `hf cache verify deepseek-ai/DeepSeek-OCR --local-dir /path/to/repo` + """ + + if local_dir is not None and cache_dir is not None: + out.error("Cannot pass both --local-dir and --cache-dir. Use one or the other.") + raise click.exceptions.Exit(code=2) + + api = get_hf_api(token=token) + + result = api.verify_repo_checksums( + repo_id=repo_id, + repo_type=repo_type.value if hasattr(repo_type, "value") else str(repo_type), + revision=revision, + local_dir=local_dir, + cache_dir=cache_dir, + token=token, + ) + + exit_code = 0 + + if result.mismatches: + details = "\n".join( + f" - {m['path']}: expected {m['expected']} ({m['algorithm']}), got {m['actual']}" + for m in result.mismatches + ) + out.text(f"❌ Checksum verification failed for the following file(s):\n{details}") + exit_code = 1 + + if result.missing_paths: + if fail_on_missing_files: + details = "\n".join(f" - {p}" for p in result.missing_paths) + out.text(f"❌ Missing files (present remotely, absent locally):\n{details}") + exit_code = 1 + else: + out.warning( + f"{len(result.missing_paths)} remote file(s) are missing locally. " + "Use --fail-on-missing-files for details." + ) + + if result.extra_paths: + if fail_on_extra_files: + details = "\n".join(f" - {p}" for p in result.extra_paths) + out.text(f"❌ Extra files (present locally, absent remotely):\n{details}") + exit_code = 1 + else: + out.warning( + f"{len(result.extra_paths)} local file(s) do not exist on the remote repo. " + "Use --fail-on-extra-files for details." + ) + + verified_location = result.verified_path + + if exit_code != 0: + out.error( + f"Verification failed for '{repo_id}' ({repo_type.value}) in {verified_location}.\n Revision: {result.revision}" + ) + raise click.exceptions.Exit(code=exit_code) + + out.result( + f"Verified {result.checked_count} file(s) for {repo_type.value} '{repo_id}'. All checksums match.", + repo_id=repo_id, + repo_type=repo_type.value, + checked=result.checked_count, + path=str(verified_location), + ) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/collections.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/collections.py new file mode 100644 index 0000000000000000000000000000000000000000..984b11cdc081a3e8e37cc74e0d21fe6320322437 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/collections.py @@ -0,0 +1,293 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with collections on the Hugging Face Hub.""" + +import enum +from typing import Annotated, get_args + +from huggingface_hub.hf_api import CollectionItemType_T, CollectionSort_T + +from ._cli_utils import LimitOpt, TokenOpt, get_hf_api, typer_factory +from ._framework import Argument, Option +from ._output import _dataclass_to_dict, out + + +# Build enums dynamically from Literal types to avoid duplication +_COLLECTION_ITEM_TYPES = get_args(CollectionItemType_T) +CollectionItemType = enum.Enum("CollectionItemType", {t: t for t in _COLLECTION_ITEM_TYPES}, type=str) # type: ignore[misc] + +_COLLECTION_SORT_OPTIONS = get_args(CollectionSort_T) +CollectionSort = enum.Enum("CollectionSort", {s: s for s in _COLLECTION_SORT_OPTIONS}, type=str) # type: ignore[misc] + + +collections_cli = typer_factory(help="Interact with collections on the Hub.") + + +@collections_cli.command( + "list | ls", + examples=[ + "hf collections ls", + "hf collections ls --owner nvidia", + "hf collections ls --item models/teknium/OpenHermes-2.5-Mistral-7B --limit 10", + ], +) +def collections_ls( + owner: Annotated[ + str | None, + Option(help="Filter by owner username or organization."), + ] = None, + item: Annotated[ + str | None, + Option( + help='Filter collections containing a specific item (e.g., "models/gpt2", "datasets/squad", "papers/2311.12983").' + ), + ] = None, + sort: Annotated[ + CollectionSort | None, + Option(help="Sort results by last modified, trending, or upvotes."), + ] = None, + limit: LimitOpt = 10, + token: TokenOpt = None, +) -> None: + """List collections on the Hub.""" + api = get_hf_api(token=token) + sort_key = sort.value if sort else None + results = [ + _dataclass_to_dict(collection) + for collection in api.list_collections( + owner=owner, + item=item, + sort=sort_key, # type: ignore[arg-type] + limit=limit, + ) + ] + out.table(results) + + +@collections_cli.command( + "info", + examples=[ + "hf collections info username/my-collection-slug", + ], +) +def collections_info( + collection_slug: Annotated[str, Argument(help="The collection slug (e.g., 'username/collection-slug').")], + token: TokenOpt = None, +) -> None: + """Get info about a collection on the Hub.""" + api = get_hf_api(token=token) + collection = api.get_collection(collection_slug) + out.dict(collection) + + +@collections_cli.command( + "create", + examples=[ + 'hf collections create "My Models"', + 'hf collections create "My Models" --description "A collection of my favorite models" --private', + 'hf collections create "Org Collection" --namespace my-org', + ], +) +def collections_create( + title: Annotated[str, Argument(help="The title of the collection.")], + namespace: Annotated[ + str | None, + Option(help="The namespace (username or organization). Defaults to the authenticated user."), + ] = None, + description: Annotated[ + str | None, + Option(help="A description for the collection (max 150 characters)."), + ] = None, + private: Annotated[ + bool, + Option(help="Create a private collection."), + ] = False, + exists_ok: Annotated[ + bool, + Option(help="Do not raise an error if the collection already exists."), + ] = False, + token: TokenOpt = None, +) -> None: + """Create a new collection on the Hub.""" + api = get_hf_api(token=token) + collection = api.create_collection( + title=title, + namespace=namespace, + description=description, + private=private, + exists_ok=exists_ok, + ) + out.result("Collection created", slug=collection.slug, url=collection.url) + + +@collections_cli.command( + "update", + examples=[ + 'hf collections update username/my-collection --title "New Title"', + 'hf collections update username/my-collection --description "Updated description"', + "hf collections update username/my-collection --private --theme green", + ], +) +def collections_update( + collection_slug: Annotated[str, Argument(help="The collection slug (e.g., 'username/collection-slug').")], + title: Annotated[ + str | None, + Option(help="The new title for the collection."), + ] = None, + description: Annotated[ + str | None, + Option(help="The new description for the collection (max 150 characters)."), + ] = None, + position: Annotated[ + int | None, + Option(help="The new position of the collection in the owner's list."), + ] = None, + private: Annotated[ + bool | None, + Option(help="Whether the collection should be private."), + ] = None, + theme: Annotated[ + str | None, + Option(help="The theme color for the collection (e.g., 'green', 'blue')."), + ] = None, + token: TokenOpt = None, +) -> None: + """Update a collection's metadata on the Hub.""" + api = get_hf_api(token=token) + collection = api.update_collection_metadata( + collection_slug=collection_slug, + title=title, + description=description, + position=position, + private=private, + theme=theme, + ) + out.result("Collection updated", slug=collection.slug, url=collection.url) + + +@collections_cli.command( + "delete", + examples=[ + "hf collections delete username/my-collection", + "hf collections delete username/my-collection --missing-ok", + ], +) +def collections_delete( + collection_slug: Annotated[str, Argument(help="The collection slug (e.g., 'username/collection-slug').")], + missing_ok: Annotated[ + bool, + Option(help="Do not raise an error if the collection doesn't exist."), + ] = False, + token: TokenOpt = None, +) -> None: + """Delete a collection from the Hub.""" + api = get_hf_api(token=token) + api.delete_collection(collection_slug, missing_ok=missing_ok) + out.result("Collection deleted", slug=collection_slug) + + +@collections_cli.command( + "add-item", + examples=[ + "hf collections add-item username/my-collection moonshotai/kimi-k2 model", + 'hf collections add-item username/my-collection Qwen/DeepPlanning dataset --note "Useful dataset"', + "hf collections add-item username/my-collection Tongyi-MAI/Z-Image space", + ], +) +def collections_add_item( + collection_slug: Annotated[str, Argument(help="The collection slug (e.g., 'username/collection-slug').")], + item_id: Annotated[str, Argument(help="The ID of the item to add (repo_id for repos, paper ID for papers).")], + item_type: Annotated[ + CollectionItemType, + Argument(help="The type of item (model, dataset, space, paper, collection, or bucket)."), + ], + note: Annotated[ + str | None, + Option(help="A note to attach to the item (max 500 characters)."), + ] = None, + exists_ok: Annotated[ + bool, + Option(help="Do not raise an error if the item is already in the collection."), + ] = False, + token: TokenOpt = None, +) -> None: + """Add an item to a collection.""" + api = get_hf_api(token=token) + collection = api.add_collection_item( + collection_slug=collection_slug, + item_id=item_id, + item_type=item_type.value, # type: ignore[arg-type] + note=note, + exists_ok=exists_ok, + ) + out.result("Item added to collection", slug=collection_slug, url=collection.url) + + +@collections_cli.command( + "update-item", + examples=[ + 'hf collections update-item username/my-collection ITEM_OBJECT_ID --note "Updated note"', + "hf collections update-item username/my-collection ITEM_OBJECT_ID --position 0", + ], +) +def collections_update_item( + collection_slug: Annotated[str, Argument(help="The collection slug (e.g., 'username/collection-slug').")], + item_object_id: Annotated[ + str, + Argument(help="The ID of the item in the collection (from 'item_object_id' field, not the repo_id)."), + ], + note: Annotated[ + str | None, + Option(help="A new note for the item (max 500 characters)."), + ] = None, + position: Annotated[ + int | None, + Option(help="The new position of the item in the collection."), + ] = None, + token: TokenOpt = None, +) -> None: + """Update an item in a collection.""" + api = get_hf_api(token=token) + api.update_collection_item( + collection_slug=collection_slug, + item_object_id=item_object_id, + note=note, + position=position, + ) + out.result("Item updated in collection", slug=collection_slug) + + +@collections_cli.command("delete-item") +def collections_delete_item( + collection_slug: Annotated[str, Argument(help="The collection slug (e.g., 'username/collection-slug').")], + item_object_id: Annotated[ + str, + Argument( + help="The ID of the item in the collection (retrieved from `item_object_id` field returned by 'hf collections info'." + ), + ], + missing_ok: Annotated[ + bool, + Option(help="Do not raise an error if the item doesn't exist."), + ] = False, + token: TokenOpt = None, +) -> None: + """Delete an item from a collection.""" + api = get_hf_api(token=token) + api.delete_collection_item( + collection_slug=collection_slug, + item_object_id=item_object_id, + missing_ok=missing_ok, + ) + out.result("Item deleted from collection", slug=collection_slug) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/datasets.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..15625504da845c384f7ab77b2b553e69be76d30f --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/datasets.py @@ -0,0 +1,274 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with datasets on the Hugging Face Hub.""" + +import enum +from typing import Annotated, get_args + +import click + +from huggingface_hub._dataset_viewer import execute_raw_sql_query +from huggingface_hub.errors import CLIError, RepositoryNotFoundError, RevisionNotFoundError +from huggingface_hub.hf_api import DatasetSort_T, ExpandDatasetProperty_T +from huggingface_hub.repocard import DatasetCard + +from ._cli_utils import ( + REPO_LIST_DEFAULT_LIMIT, + AuthorOpt, + FilterOpt, + LimitOpt, + RevisionOpt, + SearchOpt, + TokenOpt, + get_hf_api, + make_expand_properties_parser, + typer_factory, +) +from ._file_listing import list_repo_files_cmd +from ._framework import Argument, Option +from ._output import _dataclass_to_dict, out + + +_EXPAND_PROPERTIES = sorted(get_args(ExpandDatasetProperty_T)) +_SORT_OPTIONS = get_args(DatasetSort_T) +DatasetSortEnum = enum.Enum("DatasetSortEnum", {s: s for s in _SORT_OPTIONS}, type=str) # type: ignore[misc] + + +ExpandOpt = Annotated[ + str | None, + Option( + help=f"Comma-separated properties to return. When used, only the listed properties (and id) are returned. Example: '--expand=downloads,likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.", + callback=make_expand_properties_parser(_EXPAND_PROPERTIES), + ), +] + + +datasets_cli = typer_factory(help="Interact with datasets on the Hub.") + + +@datasets_cli.command( + "list | ls", + examples=[ + "hf datasets ls", + "hf datasets ls --sort downloads --limit 10", + 'hf datasets ls --search "code"', + "hf datasets ls --filter benchmark:official", + "hf datasets ls HuggingFaceFW/fineweb", + "hf datasets ls HuggingFaceFW/fineweb -R", + "hf datasets ls HuggingFaceFW/fineweb --tree -h", + ], +) +def datasets_ls( + repo_id: Annotated[ + str | None, + Argument(help="Dataset ID (e.g. `username/repo-name`) to list files from. If omitted, lists datasets."), + ] = None, + search: SearchOpt = None, + author: AuthorOpt = None, + filter: FilterOpt = None, + sort: Annotated[ + DatasetSortEnum | None, + Option(help="Sort results."), + ] = None, + limit: LimitOpt = REPO_LIST_DEFAULT_LIMIT, + expand: ExpandOpt = None, + human_readable: Annotated[ + bool, + Option("--human-readable", "-h", help="Show sizes in human readable format (only for listing files)."), + ] = False, + as_tree: Annotated[ + bool, + Option("--tree", help="List files in tree format (only for listing files)."), + ] = False, + recursive: Annotated[ + bool, + Option("--recursive", "-R", help="List files recursively (only for listing files)."), + ] = False, + revision: RevisionOpt = None, + token: TokenOpt = None, +) -> None: + """List datasets on the Hub, or files in a dataset repo. + + When called with no argument, lists datasets on the Hub. + When called with a dataset ID, lists files in that dataset repo. + """ + if repo_id is not None: + if search is not None: + raise click.BadParameter("Cannot use --search when listing files.") + if author is not None: + raise click.BadParameter("Cannot use --author when listing files.") + if filter is not None: + raise click.BadParameter("Cannot use --filter when listing files.") + if sort is not None: + raise click.BadParameter("Cannot use --sort when listing files.") + if limit != REPO_LIST_DEFAULT_LIMIT: + raise click.BadParameter("Cannot use --limit when listing files.") + if expand is not None: + raise click.BadParameter("Cannot use --expand when listing files.") + return list_repo_files_cmd( + repo_id=repo_id, + repo_type="dataset", + human_readable=human_readable, + as_tree=as_tree, + recursive=recursive, + revision=revision, + token=token, + ) + + if as_tree: + raise click.BadParameter("Cannot use --tree when listing datasets.") + if recursive: + raise click.BadParameter("Cannot use --recursive when listing datasets.") + if human_readable: + raise click.BadParameter("Cannot use --human-readable when listing datasets.") + if revision is not None: + raise click.BadParameter("Cannot use --revision when listing datasets.") + + api = get_hf_api(token=token) + sort_key = sort.value if sort else None + results = [ + _dataclass_to_dict(dataset_info) + for dataset_info in api.list_datasets( + filter=filter, + author=author, + search=search, + sort=sort_key, + limit=limit, + expand=expand, # type: ignore + ) + ] + out.table(results) + + +@datasets_cli.command( + "leaderboard", + examples=[ + "hf datasets leaderboard SWE-bench/SWE-bench_Verified", + "hf datasets leaderboard SWE-bench/SWE-bench_Verified --limit 5 --format json", + "hf datasets ls --filter benchmark:official # list available leaderboards", + ], +) +def datasets_leaderboard( + dataset_id: Annotated[str, Argument(help="The benchmark dataset ID (e.g. `SWE-bench/SWE-bench_Verified`).")], + limit: LimitOpt = 20, + token: TokenOpt = None, +) -> None: + """List model scores from a dataset leaderboard. This command helps find the best models for a task or compare models by benchmark scores. Use 'hf datasets ls --filter benchmark:official' to list available leaderboards.""" + api = get_hf_api(token=token) + leaderboard = api.get_dataset_leaderboard(repo_id=dataset_id) + results = [_dataclass_to_dict(entry) for entry in leaderboard[:limit]] + out.table( + results, + headers=["rank", "model_id", "value", "source"], + id_key="model_id", + ) + out.hint("Use 'hf datasets ls --filter benchmark:official' to list available leaderboards.") + if leaderboard: + out.hint(f"Use 'hf models info {leaderboard[0].model_id}' to get details about a model.") + + +@datasets_cli.command( + "info", + examples=[ + "hf datasets info HuggingFaceFW/fineweb", + "hf datasets info my-dataset --expand downloads,likes,tags", + ], +) +def datasets_info( + dataset_id: Annotated[str, Argument(help="The dataset ID (e.g. `username/repo-name`).")], + revision: RevisionOpt = None, + expand: ExpandOpt = None, + token: TokenOpt = None, +) -> None: + """Get info about a dataset on the Hub.""" + api = get_hf_api(token=token) + try: + info = api.dataset_info(repo_id=dataset_id, revision=revision, expand=expand) # type: ignore + except RepositoryNotFoundError as e: + raise CLIError(f"Dataset '{dataset_id}' not found.") from e + except RevisionNotFoundError as e: + raise CLIError(f"Revision '{revision}' not found on '{dataset_id}'.") from e + out.dict(info) + + +@datasets_cli.command( + "parquet", + examples=[ + "hf datasets parquet cfahlgren1/hub-stats", + "hf datasets parquet cfahlgren1/hub-stats --subset models", + "hf datasets parquet cfahlgren1/hub-stats --split train", + "hf datasets parquet cfahlgren1/hub-stats --format json", + ], +) +def datasets_parquet( + dataset_id: Annotated[str, Argument(help="The dataset ID (e.g. `username/repo-name`).")], + subset: Annotated[str | None, Option("--subset", help="Filter parquet entries by subset/config.")] = None, + split: Annotated[str | None, Option(help="Filter parquet entries by split.")] = None, + token: TokenOpt = None, +) -> None: + """List parquet file URLs available for a dataset.""" + api = get_hf_api(token=token) + entries = api.list_dataset_parquet_files(repo_id=dataset_id, config=subset) + filtered = [entry for entry in entries if split is None or entry.split == split] + results = [ + {"subset": entry.config, "split": entry.split, "url": entry.url, "size": entry.size} for entry in filtered + ] + out.table(results, headers=["subset", "split", "url", "size"], id_key="url") + + +@datasets_cli.command( + "sql", + examples=[ + "hf datasets sql \"SELECT COUNT(*) AS rows FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet')\"", + "hf datasets sql \"SELECT * FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet') LIMIT 5\" --format json", + ], +) +def datasets_sql( + sql: Annotated[str, Argument(help="Raw SQL query to execute.")], + token: TokenOpt = None, +) -> None: + """Execute a raw SQL query with DuckDB against dataset parquet URLs.""" + try: + result = execute_raw_sql_query(sql_query=sql, token=token) + except ImportError as e: + raise CLIError(str(e)) from e + out.table(result) + + +@datasets_cli.command( + "card", + examples=[ + "hf datasets card HuggingFaceFW/fineweb", + "hf datasets card HuggingFaceFW/fineweb --metadata", + "hf datasets card HuggingFaceFW/fineweb --metadata --format json", + "hf datasets card HuggingFaceFW/fineweb --text", + ], +) +def datasets_card( + dataset_id: Annotated[str, Argument(help="The dataset ID (e.g. `username/repo-name`).")], + metadata: Annotated[bool, Option("--metadata", help="Output only the metadata from the card.")] = False, + text: Annotated[bool, Option("--text", help="Output only the text body (no metadata).")] = False, + token: TokenOpt = None, +) -> None: + """Get the dataset card (README) for a dataset on the Hub.""" + if metadata and text: + raise CLIError("--metadata and --text are mutually exclusive.") + card = DatasetCard.load(dataset_id, token=token) + if metadata: + out.dict(card.data.to_dict()) + elif text: + out.text(card.text) + else: + out.text(card.content) + out.hint(f"Use `hf datasets card {dataset_id} --metadata` to extract only the card metadata.") diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/deprecated_cli.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/deprecated_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..4fbdd8adaba77accf24fbe0d459b4714d503b781 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/deprecated_cli.py @@ -0,0 +1,35 @@ +"""Deprecated `huggingface-cli` entry point. Warns and exits.""" + +import shutil +import sys + +from ._output import out + + +def main() -> None: + out.warning("`huggingface-cli` is deprecated and no longer works. Use `hf` instead.\n") + + if shutil.which("hf"): + from huggingface_hub.cli._cli_utils import check_cli_update + + check_cli_update("huggingface_hub") + out.hint("`hf` is already installed! Use it directly.\n") + else: + out.hint( + "Install `hf`:\n" + " Standalone (recommended): curl -LsSf https://hf.co/cli/install.sh | bash\n" + " Using Homebrew: brew install hf\n" + " Using pip: pip install huggingface_hub\n", + ) + + out.hint( + "Examples:\n" + " hf auth login\n" + " hf download unsloth/gemma-4-31B-it-GGUF\n" + " hf upload my-cool-model . .\n" + ' hf models ls --search "gemma"\n' + " hf repos ls --format json\n" + " hf jobs run python:3.12 python -c 'print(\"Hello!\")'\n" + " hf --help\n", + ) + sys.exit(1) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/discussions.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/discussions.py new file mode 100644 index 0000000000000000000000000000000000000000..3fd3b4217bfa0d143ba7906d12791b013369c912 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/discussions.py @@ -0,0 +1,497 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with discussions and pull requests on the Hugging Face Hub.""" + +import enum +import sys +from pathlib import Path +from typing import Annotated + +import click + +from huggingface_hub import constants + +from ._cli_utils import ( + AuthorOpt, + LimitOpt, + RepoIdArg, + RepoType, + RepoTypeOpt, + TokenOpt, + get_hf_api, + typer_factory, +) +from ._framework import Argument, Option +from ._output import _dataclass_to_dict, out + + +class DiscussionStatus(str, enum.Enum): + open = "open" + closed = "closed" + merged = "merged" + draft = "draft" + all = "all" + + +class DiscussionKind(str, enum.Enum): + all = "all" + discussion = "discussion" + pull_request = "pull_request" + + +# "merged" and "draft" are valid Discussion statuses but the Hub API filter +# (DiscussionStatusFilter) only accepts "all", "open", "closed". When the user +# asks for merged/draft we fetch with api_status=None (i.e. all) and filter +# client-side. +_CLIENT_SIDE_STATUSES = {"merged", "draft"} + + +DiscussionNumArg = Annotated[ + int, + Argument( + help="The discussion or pull request number.", + min=1, + ), +] + + +def _read_body(body: str | None, body_file: Path | None) -> str | None: + """Resolve body text from --body or --body-file (supports '-' for stdin).""" + if body is not None and body_file is not None: + raise click.BadParameter("Cannot use both --body and --body-file.") + if body_file is not None: + if str(body_file) == "-": + return sys.stdin.read() + return body_file.read_text(encoding="utf-8") + return body + + +discussions_cli = typer_factory(help="Manage discussions and pull requests on the Hub.") + + +@discussions_cli.command( + "list | ls", + examples=[ + "hf discussions list username/my-model", + "hf discussions list username/my-model --kind pull_request --status merged", + "hf discussions list username/my-dataset --type dataset --status closed", + "hf discussions list username/my-model --author alice --format json", + ], +) +def discussion_list( + repo_id: RepoIdArg, + status: Annotated[ + DiscussionStatus, + Option( + "-s", + "--status", + help="Filter by status (open, closed, merged, draft, all).", + ), + ] = DiscussionStatus.open, + kind: Annotated[ + DiscussionKind, + Option( + "-k", + "--kind", + help="Filter by kind (discussion, pull_request, all).", + ), + ] = DiscussionKind.all, + author: AuthorOpt = None, + limit: LimitOpt = 30, + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, +) -> None: + """List discussions and pull requests on a repo.""" + api = get_hf_api(token=token) + + api_status: constants.DiscussionStatusFilter | None + if status == DiscussionStatus.open: + api_status = "open" + elif status == DiscussionStatus.closed: + api_status = "closed" + else: + api_status = None + + api_discussion_type: constants.DiscussionTypeFilter | None + if kind == DiscussionKind.all: + api_discussion_type = None + else: + api_discussion_type = kind.value # type: ignore[assignment] + + discussions = [] + for d in api.get_repo_discussions( + repo_id=repo_id, + author=author, + discussion_type=api_discussion_type, + discussion_status=api_status, + repo_type=repo_type.value, + ): + if status.value in _CLIENT_SIDE_STATUSES and d.status != status.value: + continue + discussions.append(d) + if len(discussions) >= limit: + break + + items = [_dataclass_to_dict(d) for d in discussions] + out.table( + items, + headers=["num", "title", "is_pull_request", "status", "author", "created_at"], + id_key="num", + ) + + +@discussions_cli.command( + "info", + examples=[ + "hf discussions info username/my-model 5", + "hf discussions info username/my-model 5 --format json", + ], +) +def discussion_info( + repo_id: RepoIdArg, + num: DiscussionNumArg, + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, +) -> None: + """Get info about a discussion or pull request.""" + api = get_hf_api(token=token) + details = api.get_discussion_details( + repo_id=repo_id, + discussion_num=num, + repo_type=repo_type.value, + ) + out.dict(details) + + +@discussions_cli.command( + "create", + examples=[ + 'hf discussions create username/my-model --title "Bug report"', + 'hf discussions create username/my-model --title "Feature request" --body "Please add X"', + 'hf discussions create username/my-model --title "Fix typo" --pull-request', + 'hf discussions create username/my-dataset --type dataset --title "Data quality issue"', + ], +) +def discussion_create( + repo_id: RepoIdArg, + title: Annotated[ + str, + Option( + "--title", + help="The title of the discussion or pull request.", + ), + ], + body: Annotated[ + str | None, + Option( + "--body", + help="The description (supports Markdown).", + ), + ] = None, + body_file: Annotated[ + Path | None, + Option( + "--body-file", + help="Read the description from a file. Use '-' for stdin.", + ), + ] = None, + pull_request: Annotated[ + bool, + Option( + "--pull-request", + "--pr", + help="Create a pull request instead of a discussion.", + ), + ] = False, + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, +) -> None: + """Create a new discussion or pull request on a repo.""" + description = _read_body(body, body_file) + api = get_hf_api(token=token) + discussion = api.create_discussion( + repo_id=repo_id, + title=title, + description=description, + repo_type=repo_type.value, + pull_request=pull_request, + ) + kind = "pull request" if pull_request else "discussion" + ref = f"refs/pr/{discussion.num}" if pull_request else None + out.result(f"Created {kind} #{discussion.num} on {repo_id}", num=discussion.num, url=discussion.url, ref=ref) + + +@discussions_cli.command( + "comment", + examples=[ + 'hf discussions comment username/my-model 5 --body "Thanks for reporting!"', + 'hf discussions comment username/my-model 5 --body "LGTM!"', + ], +) +def discussion_comment( + repo_id: RepoIdArg, + num: DiscussionNumArg, + body: Annotated[ + str | None, + Option( + "--body", + help="The comment text (supports Markdown).", + ), + ] = None, + body_file: Annotated[ + Path | None, + Option( + "--body-file", + help="Read the comment from a file. Use '-' for stdin.", + ), + ] = None, + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, +) -> None: + """Comment on a discussion or pull request.""" + comment = _read_body(body, body_file) + if comment is None: + raise click.BadParameter("Either --body or --body-file is required.") + api = get_hf_api(token=token) + api.comment_discussion( + repo_id=repo_id, + discussion_num=num, + comment=comment, + repo_type=repo_type.value, + ) + out.result(f"Commented on #{num} in {repo_id}", num=num, repo=repo_id) + + +@discussions_cli.command( + "edit", + examples=[ + 'hf discussions edit username/my-model 5 abc123 --body "Updated comment."', + "hf discussions edit username/my-model 5 abc123 --body-file fixed.md", + ], +) +def discussion_edit( + repo_id: RepoIdArg, + num: DiscussionNumArg, + comment_id: Annotated[ + str, + Argument( + help="The ID of the comment to edit (see 'hf discussions info ... --format json').", + ), + ], + body: Annotated[ + str | None, + Option( + "--body", + help="The new comment text (supports Markdown).", + ), + ] = None, + body_file: Annotated[ + Path | None, + Option( + "--body-file", + help="Read the new comment from a file. Use '-' for stdin.", + ), + ] = None, + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, +) -> None: + """Edit an existing comment on a discussion or pull request.""" + new_content = _read_body(body, body_file) + if new_content is None: + raise click.BadParameter("Either --body or --body-file is required.") + api = get_hf_api(token=token) + api.edit_discussion_comment( + repo_id=repo_id, + discussion_num=num, + comment_id=comment_id, + new_content=new_content, + repo_type=repo_type.value, + ) + out.result(f"Edited comment {comment_id} on #{num} in {repo_id}", num=num, repo=repo_id, comment_id=comment_id) + + +@discussions_cli.command( + "close", + examples=[ + "hf discussions close username/my-model 5", + 'hf discussions close username/my-model 5 --comment "Closing as resolved."', + ], +) +def discussion_close( + repo_id: RepoIdArg, + num: DiscussionNumArg, + comment: Annotated[ + str | None, + Option( + "--comment", + help="An optional comment to post when closing.", + ), + ] = None, + yes: Annotated[ + bool, + Option( + "--yes", + "-y", + help="Skip confirmation prompt.", + ), + ] = False, + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, +) -> None: + """Close a discussion or pull request.""" + out.confirm(f"Close #{num} on '{repo_id}'?", yes=yes) + api = get_hf_api(token=token) + api.change_discussion_status( + repo_id=repo_id, + discussion_num=num, + new_status="closed", + comment=comment, + repo_type=repo_type.value, + ) + out.result(f"Closed #{num} in {repo_id}", num=num, repo=repo_id) + + +@discussions_cli.command( + "reopen", + examples=[ + "hf discussions reopen username/my-model 5", + 'hf discussions reopen username/my-model 5 --comment "Reopening for further investigation."', + ], +) +def discussion_reopen( + repo_id: RepoIdArg, + num: DiscussionNumArg, + comment: Annotated[ + str | None, + Option( + "--comment", + help="An optional comment to post when reopening.", + ), + ] = None, + yes: Annotated[ + bool, + Option( + "--yes", + "-y", + help="Skip confirmation prompt.", + ), + ] = False, + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, +) -> None: + """Reopen a closed discussion or pull request.""" + out.confirm(f"Reopen #{num} on '{repo_id}'?", yes=yes) + api = get_hf_api(token=token) + api.change_discussion_status( + repo_id=repo_id, + discussion_num=num, + new_status="open", + comment=comment, + repo_type=repo_type.value, + ) + out.result(f"Reopened #{num} in {repo_id}", num=num, repo=repo_id) + + +@discussions_cli.command( + "rename", + examples=[ + 'hf discussions rename username/my-model 5 "Updated title"', + ], +) +def discussion_rename( + repo_id: RepoIdArg, + num: DiscussionNumArg, + new_title: Annotated[ + str, + Argument( + help="The new title.", + ), + ], + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, +) -> None: + """Rename a discussion or pull request.""" + api = get_hf_api(token=token) + api.rename_discussion( + repo_id=repo_id, + discussion_num=num, + new_title=new_title, + repo_type=repo_type.value, + ) + out.result(f"Renamed #{num} in {repo_id}", num=num, repo=repo_id, title=new_title) + + +@discussions_cli.command( + "merge", + examples=[ + "hf discussions merge username/my-model 5", + 'hf discussions merge username/my-model 5 --comment "Merging, thanks!"', + ], +) +def discussion_merge( + repo_id: RepoIdArg, + num: DiscussionNumArg, + comment: Annotated[ + str | None, + Option( + "--comment", + help="An optional comment to post when merging.", + ), + ] = None, + yes: Annotated[ + bool, + Option( + "--yes", + "-y", + help="Skip confirmation prompt.", + ), + ] = False, + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, +) -> None: + """Merge a pull request.""" + out.confirm(f"Merge #{num} on '{repo_id}'?", yes=yes) + api = get_hf_api(token=token) + api.merge_pull_request( + repo_id=repo_id, + discussion_num=num, + comment=comment, + repo_type=repo_type.value, + ) + out.result(f"Merged #{num} in {repo_id}", num=num, repo=repo_id) + + +@discussions_cli.command( + "diff", + examples=[ + "hf discussions diff username/my-model 5", + ], +) +def discussion_diff( + repo_id: RepoIdArg, + num: DiscussionNumArg, + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, +) -> None: + """Show the diff of a pull request.""" + api = get_hf_api(token=token) + details = api.get_discussion_details( + repo_id=repo_id, + discussion_num=num, + repo_type=repo_type.value, + ) + if details.diff: + out.text(details.diff) + else: + out.text("No diff available.") diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/download.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/download.py new file mode 100644 index 0000000000000000000000000000000000000000..3be639228e564697deb45bf6882daab3998e4899 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/download.py @@ -0,0 +1,225 @@ +# Copyright 202-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to download files from the Hub with the CLI.""" + +import warnings +from typing import Annotated + +from huggingface_hub import constants +from huggingface_hub._snapshot_download import snapshot_download +from huggingface_hub.errors import CLIError +from huggingface_hub.file_download import DryRunFileInfo, hf_hub_download +from huggingface_hub.utils import _format_size, parse_hf_uri + +from ._cli_utils import RepoIdArg, RepoType, RepoTypeOptionalOpt, RevisionOpt, TokenOpt +from ._framework import Argument, Option +from ._output import out + + +DOWNLOAD_EXAMPLES = [ + "hf download meta-llama/Llama-3.2-1B-Instruct", + "hf download meta-llama/Llama-3.2-1B-Instruct config.json tokenizer.json", + 'hf download meta-llama/Llama-3.2-1B-Instruct --include "*.safetensors" --exclude "*.bin"', + "hf download meta-llama/Llama-3.2-1B-Instruct --local-dir ./models/llama", + "hf download HuggingFaceM4/FineVision art/ --repo-type dataset", + "hf download hf://datasets/HuggingFaceH4/ultrachat_200k", +] + + +def download( + repo_id: RepoIdArg, + filenames: Annotated[ + list[str] | None, + Argument( + help="Files to download (e.g. `config.json`, `data/metadata.jsonl`).", + ), + ] = None, + repo_type: RepoTypeOptionalOpt = None, + revision: RevisionOpt = None, + include: Annotated[ + list[str] | None, + Option( + help="Glob patterns to include from files to download. eg: *.json", + ), + ] = None, + exclude: Annotated[ + list[str] | None, + Option( + help="Glob patterns to exclude from files to download.", + ), + ] = None, + cache_dir: Annotated[ + str | None, + Option( + help="Directory where to save files.", + ), + ] = None, + local_dir: Annotated[ + str | None, + Option( + help="If set, the downloaded file will be placed under this directory. Check out https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-a-local-folder for more details.", + ), + ] = None, + force_download: Annotated[ + bool, + Option( + help="If True, the files will be downloaded even if they are already cached.", + ), + ] = False, + dry_run: Annotated[ + bool, + Option( + help="If True, perform a dry run without actually downloading the file.", + ), + ] = False, + token: TokenOpt = None, + max_workers: Annotated[ + int, + Option( + help="Maximum number of workers to use for downloading files. Default is 8.", + ), + ] = 8, +) -> None: + """Download files from the Hub.""" + if local_dir is not None and cache_dir is not None: + raise CLIError( + "Cannot use both `--local-dir` and `--cache-dir` at the same time. " + "Use `--cache-dir` (or set the HF_HOME environment variable) for shared caching, " + "or `--local-dir` for a one-off download to a specific directory." + ) + + # `repo_id` may be a plain repo id or an `hf://` URI (e.g. `hf://datasets/my-org/my-dataset@v1.0/data/`). + # When a URI is provided, it is authoritative for the repo type, revision and (optionally) file path, + # so explicit `--repo-type` / `--revision` options are forbidden alongside it. + # We branch on the `hf://` prefix (the user's *intent*) rather than on whether the string parses as a + # valid URI: a malformed URI then surfaces a precise `HfUriError` (formatted globally in `cli/_errors.py`) + # instead of silently falling through to the plain-repo-id path and failing later with an opaque error. + if repo_id.startswith(constants.HF_PROTOCOL): + if repo_type is not None: + raise CLIError(f"'--repo-type' cannot be used with an 'hf://' URI ('{repo_id}').") + if revision is not None: + raise CLIError(f"'--revision' cannot be used with an 'hf://' URI ('{repo_id}').") + uri = parse_hf_uri(repo_id) + if uri.is_bucket: + raise CLIError("Buckets are not supported by `hf download`. Use `hf sync` instead.") + # The URI parser strips trailing slashes, but `hf download` uses a trailing '/' to denote a subfolder + # download (e.g. `data/` -> `data/**`). Re-append it when the URI explicitly ended with '/' so a folder + # URI keeps routing through the subfolder code path below. + path_in_repo = uri.path_in_repo + if path_in_repo and repo_id.endswith("/"): + path_in_repo += "/" + repo_id, repo_type_str, revision = uri.id, uri.type, uri.revision + if path_in_repo: + if filenames: + raise CLIError( + f"Cannot combine a file path in the hf:// URI ('{path_in_repo}') with positional filenames {filenames}." + ) + filenames = [path_in_repo] + else: + repo_type_str = (repo_type or RepoType.model).value + + def run_download() -> str | DryRunFileInfo | list[DryRunFileInfo]: + filenames_list = filenames if filenames is not None else [] + + # Separate subfolder patterns (ending with '/') from regular filenames + # Subfolders like "art/" are converted to include patterns like "art/**" + subfolders = [f for f in filenames_list if f.endswith("/")] + subfolder_patterns = [f"{f.rstrip('/')}/**" for f in subfolders] + regular_filenames = [f for f in filenames_list if not f.endswith("/")] + + # Error if subfolder patterns are combined with --include/--exclude + # Guide user to use --include instead of subfolder argument + if len(subfolder_patterns) > 0: + if include is not None and len(include) > 0: + raise CLIError( + f"Cannot combine subfolder argument ('{subfolders[0]}') with `--include`. " + f'Please use `--include "{subfolders[0]}*"` instead.' + ) + if exclude is not None and len(exclude) > 0: + raise CLIError( + f"Cannot combine subfolder argument ('{subfolders[0]}') with `--exclude`. " + f'Please use `--include "{subfolders[0]}*"` with `--exclude` instead.' + ) + + # Warn user if patterns are ignored (only if regular filenames are provided) + if len(regular_filenames) > 0: + if include is not None and len(include) > 0: + warnings.warn("Ignoring `--include` since filenames have been explicitly set.") + if exclude is not None and len(exclude) > 0: + warnings.warn("Ignoring `--exclude` since filenames have been explicitly set.") + + # Single file to download (not a subfolder): use `hf_hub_download` + if len(regular_filenames) == 1 and len(subfolder_patterns) == 0: + return hf_hub_download( + repo_id=repo_id, + repo_type=repo_type_str, + revision=revision, + filename=regular_filenames[0], + cache_dir=cache_dir, + force_download=force_download, + token=token, + local_dir=local_dir, + library_name="huggingface-cli", + dry_run=dry_run, + ) + + # Otherwise: use `snapshot_download` to ensure all files comes from same revision + if len(regular_filenames) == 0 and len(subfolder_patterns) == 0: + # No filenames provided: use include/exclude patterns + allow_patterns = include + ignore_patterns = exclude + else: + # Combine regular filenames and subfolder patterns as allow_patterns + allow_patterns = regular_filenames + subfolder_patterns + ignore_patterns = None + + return snapshot_download( + repo_id=repo_id, + repo_type=repo_type_str, + revision=revision, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + force_download=force_download, + cache_dir=cache_dir, + token=token, + local_dir=local_dir, + library_name="huggingface-cli", + max_workers=max_workers, + dry_run=dry_run, + ) + + def _print_result(result: str | DryRunFileInfo | list[DryRunFileInfo]) -> None: + if isinstance(result, str): + out.result("Downloaded", path=result) + return + + # Print dry run info + if isinstance(result, DryRunFileInfo): + result = [result] + will_download = [r for r in result if r.will_download] + out.text( + f"[dry-run] Will download {len(will_download)} files" + f" (out of {len(result)})" + f" totalling {_format_size(sum(r.file_size for r in will_download))}." + ) + items = [ + { + "file": info.filename, + "size": _format_size(info.file_size) if info.will_download else "-", + } + for info in sorted(result, key=lambda x: x.filename) + ] + out.table(items) + + _print_result(run_download()) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/extensions.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/extensions.py new file mode 100644 index 0000000000000000000000000000000000000000..c8b8f51936194c6bbd3e1fbef8be4b31400061ab --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/extensions.py @@ -0,0 +1,697 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains helper utilities for hf CLI extensions.""" + +import errno +import json +import os +import re +import shutil +import subprocess +import venv +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path +from typing import Annotated, Literal + +import click + +from huggingface_hub.errors import CLIError, CLIExtensionInstallError, ConfirmationError +from huggingface_hub.utils import get_session, logging + +from ._cli_utils import typer_factory +from ._framework import Argument, Option +from ._output import out + + +DEFAULT_EXTENSION_OWNER = "huggingface" +EXTENSIONS_ROOT = Path("~/.local/share/hf/extensions") +MANIFEST_FILENAME = "manifest.json" +EXTENSIONS_HELP = ( + "Manage hf CLI extensions.\n\n" + "Security Warning: extensions are third-party executables or Python packages. " + "Install only from sources you trust." +) +extensions_cli = typer_factory(help=EXTENSIONS_HELP) +_EXTENSIONS_GITHUB_TOPIC = "hf-extension" +_EXTENSIONS_DOWNLOAD_TIMEOUT = 10 +_EXTENSIONS_PIP_INSTALL_TIMEOUT = 300 + +logger = logging.get_logger(__name__) + + +class _ExtensionUpdateStatus(str, Enum): + UPDATED = "updated" + UP_TO_DATE = "up_to_date" + SKIPPED = "skipped" + + +@dataclass +class ExtensionManifest: + owner: str + repo: str + repo_id: str + short_name: str + executable_path: str + type: Literal["binary", "python"] + installed_at: datetime + description: str | None = None + commit_sha: str | None = None + + @classmethod + def load(cls, path: Path) -> "ExtensionManifest": + manifest_path = path / MANIFEST_FILENAME + if not manifest_path.is_file(): + raise CLIError(f"Manifest file not found at {manifest_path}. Your extension may be corrupted.") + data = json.loads(manifest_path.read_text()) + # Ignore keys not in the dataclass (e.g. fields dropped since the manifest was written). + data = {key: value for key, value in data.items() if key in cls.__dataclass_fields__} + data["installed_at"] = datetime.fromisoformat(data["installed_at"]) + return ExtensionManifest(**data) + + def save(self, path: Path) -> None: + manifest_path = path / MANIFEST_FILENAME + manifest_path.parent.mkdir(parents=True, exist_ok=True) + data = asdict(self) + data["installed_at"] = self.installed_at.isoformat() + manifest_path.write_text(json.dumps(data, indent=2, sort_keys=True)) + + +@extensions_cli.command( + "install", + examples=[ + "hf extensions install hf-claude", + "hf extensions install hanouticelina/hf-claude", + "hf extensions install alvarobartt/hf-mem", + ], +) +def extension_install( + ctx: click.Context, + repo_id: Annotated[ + str, + Argument(help="GitHub extension repository in `[OWNER/]hf-` format."), + ], + force: Annotated[bool, Option("--force", help="Overwrite if already installed.")] = False, +) -> None: + """Install an extension from a public GitHub repository. + + Security warning: this installs a third-party executable or Python package. + Install only from sources you trust. + """ + owner, repo_name, short_name = _normalize_repo_id(repo_id) + root_ctx = ctx.find_root() + reserved_commands = set(getattr(root_ctx.command, "commands", {}).keys()) + if short_name in reserved_commands: + raise CLIError( + f"Cannot install extension '{short_name}' because it conflicts with an existing `hf {short_name}` command." + ) + + extension_dir = _get_extension_dir(short_name) + if extension_dir.exists() and not force: + raise CLIError(f"Extension '{short_name}' is already installed. Use --force to overwrite.") + + branch, description = _fetch_github_repo_info(owner=owner, repo_name=repo_name) + if extension_dir.exists(): + # --force reinstall: only remove the previous install once the repo metadata is resolved. + shutil.rmtree(extension_dir) + manifest = _install_extension( + owner=owner, repo_name=repo_name, short_name=short_name, branch=branch, description=description + ) + ext_type = manifest.type.capitalize() + out.result( + f"{ext_type} extension installed", + source=f"{owner}/{repo_name}", + command=f"hf {short_name}", + ) + out.hint(f"Run it with: hf {short_name}") + + +@extensions_cli.command( + "update", + examples=[ + "hf extensions update", + "hf extensions update hf-claude", + "hf extensions update alvarobartt/hf-mem", + ], +) +def extension_update( + name: Annotated[ + str | None, + Argument( + help=( + "Extension to update (with or without `hf-` prefix, optionally as `OWNER/hf-`). " + "If omitted, all installed extensions are checked and the outdated ones are updated." + ), + ), + ] = None, +) -> None: + """Update installed extension(s) to their latest version.""" + if name is not None: + manifest = _load_installed_extension_for_update(name) + update_status = _update_installed_extension(manifest) + match update_status: + case _ExtensionUpdateStatus.UPDATED: + out.result("Extension updated", name=manifest.short_name, source=manifest.repo_id) + case _ExtensionUpdateStatus.UP_TO_DATE: + out.result(f"Extension '{manifest.short_name}' is already up to date", name=manifest.short_name) + case _ExtensionUpdateStatus.SKIPPED: + pass # warning already emitted by _update_installed_extension + return + + manifests = _list_installed_extensions() + if not manifests: + out.warning("No extensions installed.") + out.hint("Install one with: hf extensions install ") + return + + updated = [] + up_to_date = [] + for manifest in manifests: + out.log(f"Checking '{manifest.short_name}' ({manifest.repo_id})...") + try: + update_status = _update_installed_extension(manifest) + except Exception as error: + # Keep updating the other extensions even if one fails. + out.warning(f"Could not update '{manifest.short_name}' ({manifest.repo_id}): {error}. Skipping.") + continue + match update_status: + case _ExtensionUpdateStatus.UPDATED: + updated.append(manifest.short_name) + case _ExtensionUpdateStatus.UP_TO_DATE: + up_to_date.append(manifest.short_name) + case _ExtensionUpdateStatus.SKIPPED: + pass # warning already emitted by _update_installed_extension + out.result( + "Extensions update complete", + updated=", ".join(updated) if updated else None, + up_to_date=", ".join(up_to_date) if up_to_date else None, + ) + + +@extensions_cli.command( + "exec", + context_settings={"allow_extra_args": True, "allow_interspersed_args": False, "ignore_unknown_options": True}, + examples=[ + "hf extensions exec claude -- --help", + "hf extensions exec claude --model zai-org/GLM-5", + ], +) +def extension_exec( + ctx: click.Context, + name: Annotated[ + str, + Argument(help="Extension name (with or without `hf-` prefix)."), + ], +) -> None: + """Execute an installed extension.""" + short_name = _normalize_extension_name(name) + executable_path = _resolve_installed_executable_path(short_name) + + if not executable_path.is_file(): + raise CLIError(f"Extension '{short_name}' is not installed.") + + exit_code = _execute_extension_binary(executable_path=executable_path, args=list(ctx.args)) + raise click.exceptions.Exit(code=exit_code) + + +@extensions_cli.command("list | ls", examples=["hf extensions list"]) +def extension_list() -> None: + """List installed extension commands.""" + rows = [ + { + "command": f"hf {manifest.short_name}", + "source": str(manifest.repo_id), + "type": str(manifest.type), + "installed": manifest.installed_at.strftime("%Y-%m-%d"), + "description": manifest.description, + } + for manifest in _list_installed_extensions() + ] + out.table(rows, id_key="command") + + +@extensions_cli.command("search", examples=["hf extensions search"]) +def extension_search() -> None: + """Search extensions available on GitHub (tagged with 'hf-extension' topic).""" + response = _github_get( + "https://api.github.com/search/repositories", + params={"q": f"topic:{_EXTENSIONS_GITHUB_TOPIC}", "sort": "stars", "order": "desc", "per_page": 100}, + ) + data = response.json() + + installed = {m.short_name for m in _list_installed_extensions()} + + rows = [] + for repo in data.get("items", []): + short_name = repo["name"].removeprefix("hf-") + rows.append( + { + "name": short_name, + "repo": repo["full_name"], + "stars": repo.get("stargazers_count", 0), + "description": repo.get("description") or "", + "installed": "yes" if short_name in installed else "", + } + ) + + out.table(rows, id_key="repo") + + +@extensions_cli.command("remove | rm", examples=["hf extensions remove claude"]) +def extension_remove( + name: Annotated[ + str, + Argument(help="Extension name to remove (with or without `hf-` prefix)."), + ], +) -> None: + """Remove an installed extension.""" + short_name = _normalize_extension_name(name) + extension_dir = _get_extension_dir(short_name) + + if not extension_dir.is_dir(): + raise CLIError(f"Extension '{short_name}' is not installed.") + + shutil.rmtree(extension_dir) + out.result("Extension removed", name=short_name) + + +### HELPER FUNCTIONS + + +def _list_installed_extensions() -> list[ExtensionManifest]: + """Return manifests for all validly-installed extensions, sorted by directory name.""" + root_dir = EXTENSIONS_ROOT.expanduser() + if not root_dir.is_dir(): + return [] + manifests = [] + for extension_dir in sorted(root_dir.iterdir()): + if not extension_dir.is_dir() or not extension_dir.name.startswith("hf-"): + continue + try: + manifests.append(ExtensionManifest.load(extension_dir)) + except Exception as e: + logger.debug(f"Failed to load manifest for extension '{extension_dir.name}': {e}") + continue + return manifests + + +def list_installed_extensions_for_help() -> list[tuple[str, str]]: + entries = [] + for manifest in _list_installed_extensions(): + tag = f"[extension {manifest.repo_id}]" + help_text = f"{manifest.description} {tag}" if manifest.description is not None else tag + entries.append((manifest.short_name, help_text)) + return entries + + +def dispatch_unknown_top_level_extension(args: list[str], known_commands: set[str]) -> int | None: + if not args: + return None + + command_name = args[0] + if command_name.startswith("-"): + return None + all_known = {a.strip() for cmd in known_commands for a in cmd.split("|")} + if command_name in all_known: + return None + + try: + short_name = _validate_extension_short_name(command_name.removeprefix("hf-"), original_input=command_name) + except CLIError: + return None + + executable_path: Path | None + try: + executable_path = _resolve_installed_executable_path(short_name) + except Exception: + executable_path = _auto_install_official_extension(short_name) + + if executable_path is None or not executable_path.is_file(): + return None + + return _execute_extension_binary(executable_path=executable_path, args=list(args[1:])) + + +def _auto_install_official_extension(short_name: str) -> Path | None: + """Try to auto-install huggingface/hf-. Returns executable path or None.""" + owner, repo_name = DEFAULT_EXTENSION_OWNER, f"hf-{short_name}" + if _get_extension_dir(short_name).exists(): + return None + + try: + branch, description = _fetch_github_repo_info(owner=owner, repo_name=repo_name) + except Exception: + return None + + try: + out.confirm(f"'{short_name}' is an official Hugging Face extension ({owner}/{repo_name}). Install it?") + except ConfirmationError: + return None + try: + manifest = _install_extension( + owner=owner, repo_name=repo_name, short_name=short_name, branch=branch, description=description + ) + return Path(manifest.executable_path).expanduser() + except Exception: + return None + + +def _load_installed_extension_for_update(name: str) -> ExtensionManifest: + short_name = _normalize_extension_name(name) + extension_dir = _get_extension_dir(short_name) + if not extension_dir.is_dir(): + owner, _, _ = name.strip().rpartition("/") + install_target = f"{owner}/hf-{short_name}" if owner else f"hf-{short_name}" + raise CLIError( + f"Extension '{short_name}' is not installed. Install it first with: hf extensions install {install_target}" + ) + return ExtensionManifest.load(extension_dir) + + +def _update_installed_extension(manifest: ExtensionManifest) -> _ExtensionUpdateStatus: + owner, repo_name, short_name = manifest.owner, manifest.repo, manifest.short_name + + try: + branch, description = _fetch_github_repo_info(owner=owner, repo_name=repo_name) + except Exception as error: + out.warning(f"Could not check updates for '{short_name}' ({owner}/{repo_name}): {error}. Skipping.") + return _ExtensionUpdateStatus.SKIPPED + + latest_sha = _fetch_latest_commit_sha(owner=owner, repo_name=repo_name, branch=branch, warn=False) + if latest_sha is None: + out.warning( + f"Could not check updates for '{short_name}' ({owner}/{repo_name}): GitHub is unreachable. Skipping." + ) + return _ExtensionUpdateStatus.SKIPPED + + if latest_sha == manifest.commit_sha: + return _ExtensionUpdateStatus.UP_TO_DATE + + _install_extension( + owner=owner, + repo_name=repo_name, + short_name=short_name, + branch=branch, + description=description, + commit_sha=latest_sha, + ) + return _ExtensionUpdateStatus.UPDATED + + +def _install_extension( + *, + owner: str, + repo_name: str, + short_name: str, + branch: str, + description: str | None = None, + commit_sha: str | None = None, +) -> ExtensionManifest: + """Fetch and install an extension (binary or Python package), then persist its manifest. + + Installs in place: an existing install is overwritten without being removed first, so a failed + update keeps the previous version working. A failed fresh install is cleaned up entirely. + """ + extension_dir = _get_extension_dir(short_name) + fresh_install = not extension_dir.exists() + installed = False + try: + try: + binary = _fetch_remote_binary(owner=owner, repo_name=repo_name, branch=branch, short_name=short_name) + except Exception: + binary = None + + if binary is not None: + executable_path = _install_binary_extension( + extension_dir=extension_dir, short_name=short_name, binary=binary + ) + else: + executable_path = _install_python_extension( + extension_dir=extension_dir, owner=owner, repo_name=repo_name, short_name=short_name, branch=branch + ) + + manifest = ExtensionManifest( + owner=owner, + repo=repo_name, + repo_id=f"{owner}/{repo_name}", + short_name=short_name, + executable_path=str(executable_path), + type="binary" if binary is not None else "python", + installed_at=datetime.now(timezone.utc), + description=_try_fetch_remote_description( + owner=owner, repo_name=repo_name, branch=branch, candidate_description=description + ), + commit_sha=commit_sha or _fetch_latest_commit_sha(owner=owner, repo_name=repo_name, branch=branch), + ) + manifest.save(extension_dir) + installed = True + return manifest + except CLIError: + raise + except subprocess.TimeoutExpired as e: + raise CLIExtensionInstallError( + f"Pip install timed out after {_EXTENSIONS_PIP_INSTALL_TIMEOUT}s for '{owner}/{repo_name}'. " + "See pip output above for details." + ) from e + except subprocess.CalledProcessError as e: + raise CLIExtensionInstallError( + f"Failed to install pip package from '{owner}/{repo_name}' (exit code {e.returncode}). " + "See pip output above for details." + ) from e + except Exception as e: + raise CLIExtensionInstallError(f"Failed to install extension from '{owner}/{repo_name}': {e}") from e + finally: + if not installed and fresh_install: + shutil.rmtree(extension_dir, ignore_errors=True) + + +def _fetch_latest_commit_sha(*, owner: str, repo_name: str, branch: str, warn: bool = True) -> str | None: + """Best-effort fetch of the latest commit SHA for a branch, used to detect available updates.""" + try: + response = _github_get( + f"https://api.github.com/repos/{owner}/{repo_name}/commits/{branch}", + headers={"Accept": "application/vnd.github.sha"}, + ) + return response.text.strip() or None + except Exception as error: + if warn: + out.warning(f"Could not fetch latest commit SHA for '{repo_name}' ({owner}/{repo_name}): {error}") + return None + + +def _fetch_remote_binary(*, owner: str, repo_name: str, branch: str, short_name: str) -> bytes: + executable_name = _get_executable_name(short_name) + raw_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/refs/heads/{branch}/{executable_name}" + response = _github_get(raw_url) + return response.content + + +def _install_binary_extension(*, extension_dir: Path, short_name: str, binary: bytes) -> Path: + extension_dir.mkdir(parents=True, exist_ok=True) + executable_path = extension_dir / _get_executable_name(short_name) + executable_path.write_bytes(binary) + + if os.name != "nt": + os.chmod(executable_path, 0o755) + + return executable_path + + +def _install_python_extension( + *, extension_dir: Path, owner: str, repo_name: str, short_name: str, branch: str +) -> Path: + source_url = f"https://github.com/{owner}/{repo_name}/archive/refs/heads/{branch}.zip" + venv_dir = extension_dir / "venv" + venv_python = _get_venv_bin_path(venv_dir, "python.exe" if os.name == "nt" else "python") + uv_path = shutil.which("uv") + + status = out.status() + if not venv_python.is_file(): + status.update(f"Creating virtual environment in {venv_dir}") + extension_dir.mkdir(parents=True, exist_ok=True) + if uv_path: + subprocess.run([uv_path, "venv", str(venv_dir)], check=True) + else: + venv.EnvBuilder(with_pip=True).create(str(venv_dir)) + status.done(f"Virtual environment created in {venv_dir}") + + status.update(f"Installing package from {source_url}") + if uv_path: + # --reinstall: the source URL is the same for every commit, so cached data must be refreshed. + install_cmd = [uv_path, "pip", "install", "--reinstall", "--python", str(venv_python), source_url] + else: + install_cmd = [ + str(venv_python), + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--no-input", + "--force-reinstall", + source_url, + ] + subprocess.run(install_cmd, check=True, timeout=_EXTENSIONS_PIP_INSTALL_TIMEOUT) + status.done(f"Package installed from {source_url}") + + executable_name = _get_executable_name(short_name) + venv_executable = _get_venv_bin_path(venv_dir, executable_name) + if not venv_executable.is_file(): + raise CLIError( + f"Installed package from '{owner}/{repo_name}' does not expose the required console script " + f"'{executable_name}'." + ) + return venv_executable.resolve() + + +def _try_fetch_remote_description( + owner: str, repo_name: str, branch: str, candidate_description: str | None +) -> str | None: + """Try to fetch project description either from: + - manifest.json + - pyproject.toml + + Only best effort, no error handling. + """ + base = f"https://raw.githubusercontent.com/{owner}/{repo_name}/refs/heads/{branch}" + + # from manifest.json + try: + response = _github_get(f"{base}/{MANIFEST_FILENAME}") + description = response.json().get("description") + if isinstance(description, str): + return description + except Exception: + pass + + # from pyproject.toml + try: + response = _github_get(f"{base}/pyproject.toml") + + # Weak parser but ok for "best effort" + for line in response.text.splitlines(): + line = line.strip() + if line.startswith("description"): + _, _, value = line.partition("=") + return value.strip().strip("\"'") + except Exception: + pass + + # fallback to value fetched from GH API directly + return candidate_description + + +def _get_extension_dir(short_name: str) -> Path: + # Callers validate at the parse boundary already; re-validate here as defense-in-depth since + # this path is rmtree'd on removal. + _validate_extension_short_name(short_name, original_input=short_name) + return EXTENSIONS_ROOT.expanduser() / f"hf-{short_name}" + + +def _github_get(url: str, *, params: dict | None = None, headers: dict | None = None): + """Perform a GitHub GET request. + + Shared by every GitHub/Raw fetch in this module so the timeout and redirect policy are shared. + """ + response = get_session().get( + url, + params=params, + headers=headers, + follow_redirects=True, + timeout=_EXTENSIONS_DOWNLOAD_TIMEOUT, + ) + response.raise_for_status() + return response + + +def _fetch_github_repo_info(*, owner: str, repo_name: str) -> tuple[str, str | None]: + """Fetch `default_branch` + `description` for a GitHub repo from `GET /repos/{owner}/{repo}`.""" + response = _github_get(f"https://api.github.com/repos/{owner}/{repo_name}") + data = response.json() + return data["default_branch"], data.get("description") + + +def _get_executable_name(short_name: str) -> str: + name = f"hf-{short_name}" + if os.name == "nt": + name += ".exe" + return name + + +def _resolve_installed_executable_path(short_name: str) -> Path: + extension_dir = _get_extension_dir(short_name) + manifest = ExtensionManifest.load(extension_dir) + return Path(manifest.executable_path).expanduser() + + +def _get_venv_bin_path(venv_dir: Path, executable_name: str) -> Path: + if os.name == "nt": + return venv_dir / "Scripts" / executable_name + return venv_dir / "bin" / executable_name + + +_ALLOWED_EXTENSION_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") + + +def _validate_extension_short_name(short_name: str, *, original_input: str) -> str: + name = short_name.strip() + if not name: + raise CLIError("Extension name cannot be empty.") + if any(sep in name for sep in ("/", "\\")): + raise CLIError(f"Invalid extension name '{original_input}'.") + if ".." in name or ":" in name: + raise CLIError(f"Invalid extension name '{original_input}'.") + if not _ALLOWED_EXTENSION_NAME.fullmatch(name): + raise CLIError( + f"Invalid extension name '{original_input}'. Allowed characters: letters, digits, '.', '_' and '-'." + ) + return name + + +def _normalize_repo_id(repo_id: str) -> tuple[str, str, str]: + if "://" in repo_id: + raise CLIError("Only GitHub repositories in `[OWNER/]hf-` format are supported.") + + parts = repo_id.split("/") + if len(parts) == 1: + owner = DEFAULT_EXTENSION_OWNER + repo_name = parts[0] + elif len(parts) == 2 and all(parts): + owner, repo_name = parts + else: + raise CLIError(f"Expected `[OWNER/]REPO` format, got '{repo_id}'.") + + if not repo_name.startswith("hf-"): + raise CLIError(f"Extension repository name must start with 'hf-', got '{repo_name}'.") + + short_name = repo_name.removeprefix("hf-") + if not short_name: + raise CLIError("Invalid extension repository name 'hf-'.") + _validate_extension_short_name(short_name, original_input=repo_id) + + return owner, repo_name, short_name + + +def _normalize_extension_name(name: str) -> str: + repo_name = name.strip().rsplit("/", 1)[-1] + return _validate_extension_short_name(repo_name.removeprefix("hf-"), original_input=name) + + +def _execute_extension_binary(executable_path: Path, args: list[str]) -> int: + try: + return subprocess.call([str(executable_path)] + args) + except OSError as e: + if os.name == "nt" or e.errno != errno.ENOEXEC: + raise + return subprocess.call(["sh", str(executable_path)] + args) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/hf.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/hf.py new file mode 100644 index 0000000000000000000000000000000000000000..cefe5e2e050ec10afb4813fcaa6d54b6a0974abc --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/hf.py @@ -0,0 +1,142 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import traceback +from typing import Annotated + +import click + +from huggingface_hub import __version__, constants +from huggingface_hub.cli._cli_utils import check_cli_update, fallback_typer_group_factory, typer_factory +from huggingface_hub.cli._cp import CP_EXAMPLES, make_cp +from huggingface_hub.cli._errors import format_known_exception +from huggingface_hub.cli._output import out +from huggingface_hub.cli.auth import auth_cli +from huggingface_hub.cli.buckets import buckets_cli, sync +from huggingface_hub.cli.cache import cache_cli +from huggingface_hub.cli.collections import collections_cli +from huggingface_hub.cli.datasets import datasets_cli +from huggingface_hub.cli.discussions import discussions_cli +from huggingface_hub.cli.download import DOWNLOAD_EXAMPLES, download +from huggingface_hub.cli.extensions import ( + dispatch_unknown_top_level_extension, + extensions_cli, + list_installed_extensions_for_help, +) +from huggingface_hub.cli.inference_endpoints import ie_cli +from huggingface_hub.cli.jobs import jobs_cli +from huggingface_hub.cli.lfs import lfs_enable_largefiles, lfs_multipart_upload +from huggingface_hub.cli.models import models_cli +from huggingface_hub.cli.papers import papers_cli +from huggingface_hub.cli.repo_files import repo_files_cli +from huggingface_hub.cli.repos import repos_cli +from huggingface_hub.cli.sandbox import sandbox_cli +from huggingface_hub.cli.skills import skills_cli +from huggingface_hub.cli.spaces import spaces_cli +from huggingface_hub.cli.system import env, update, version +from huggingface_hub.cli.upload import UPLOAD_EXAMPLES, upload +from huggingface_hub.cli.upload_large_folder import UPLOAD_LARGE_FOLDER_EXAMPLES, upload_large_folder +from huggingface_hub.cli.webhooks import webhooks_cli +from huggingface_hub.utils import logging + +from ._completion import _COMPLETE_VAR, InstallCompletionOpt, ShowCompletionOpt +from ._framework import Option + + +app = typer_factory( + help="Hugging Face Hub CLI", + cls=fallback_typer_group_factory( + dispatch_unknown_top_level_extension, + extra_commands_provider=list_installed_extensions_for_help, + ), +) + + +def _version_callback(value: bool) -> None: + if value: + print(__version__) + raise click.exceptions.Exit() + + +@app.group_callback(invoke_without_command=True) +def app_callback( + version: Annotated[ + bool | None, Option("-v", "--version", callback=_version_callback, is_eager=True, hidden=True) + ] = None, + install_completion: InstallCompletionOpt = False, + show_completion: ShowCompletionOpt = False, +) -> None: + pass + + +# top level single commands (defined in their respective files) +app.command(examples=CP_EXAMPLES)(make_cp()) +app.command()(sync) +app.command(examples=DOWNLOAD_EXAMPLES)(download) +app.command(examples=UPLOAD_EXAMPLES)(upload) +app.command(examples=UPLOAD_LARGE_FOLDER_EXAMPLES)(upload_large_folder) + +app.command(topic="help")(env) +app.command(topic="help")(update) +app.command(topic="help")(version) + +app.command(hidden=True)(lfs_enable_largefiles) +app.command(hidden=True)(lfs_multipart_upload) + +# command groups +app.add_group(auth_cli, name="auth") +app.add_group(buckets_cli, name="buckets") +app.add_group(cache_cli, name="cache") +app.add_group(collections_cli, name="collections") +app.add_group(datasets_cli, name="datasets") +app.add_group(discussions_cli, name="discussions") +app.add_group(jobs_cli, name="jobs") +app.add_group(models_cli, name="models") +app.add_group(papers_cli, name="papers") +app.add_group(repos_cli, name="repos | repo") +app.add_group(repo_files_cli, name="repo-files", hidden=True) +app.add_group(sandbox_cli, name="sandbox") +app.add_group(skills_cli, name="skills") +app.add_group(spaces_cli, name="spaces") +app.add_group(webhooks_cli, name="webhooks") +app.add_group(ie_cli, name="endpoints") +app.add_group(extensions_cli, name="extensions | ext") + + +def main(): + # Shell-completion requests must stay fast and emit nothing but candidates: + # skip the startup work and let click handle the env var inside `app()`. + if _COMPLETE_VAR not in os.environ: + if not constants.HF_DEBUG: + logging.set_verbosity_info() + check_cli_update("huggingface_hub") + + try: + app() + except Exception as e: + message = format_known_exception(e) + if message: + out.error(message) + if constants.HF_DEBUG: + traceback.print_exc() + else: + out.hint("set HF_DEBUG=1 as environment variable for full traceback.") + sys.exit(1) + raise + + +if __name__ == "__main__": + main() diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/inference_endpoints.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/inference_endpoints.py new file mode 100644 index 0000000000000000000000000000000000000000..a95d5968090def26bce4fc7662b02137dc0f670d --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/inference_endpoints.py @@ -0,0 +1,535 @@ +"""CLI commands for Hugging Face Inference Endpoints.""" + +import shlex +from typing import Annotated + +import click + +from huggingface_hub._inference_endpoints import InferenceEndpointScalingMetric, InferenceEndpointType +from huggingface_hub.errors import CLIError, HfHubHTTPError + +from ._cli_utils import ( + EnvFileOpt, + EnvOpt, + RevisionOpt, + SecretsFileOpt, + SecretsOpt, + SoftChoice, + TokenOpt, + get_hf_api, + parse_env_map, + typer_factory, +) +from ._framework import Argument, Option +from ._output import out + + +ie_cli = typer_factory(help="Manage Hugging Face Inference Endpoints.") + +catalog_app = typer_factory(help="Interact with the Inference Endpoints catalog.") + + +NameArg = Annotated[ + str, + Argument(help="Endpoint name."), +] +NameOpt = Annotated[ + str | None, + Option(help="Endpoint name."), +] + +NamespaceOpt = Annotated[ + str | None, + Option( + help="The namespace associated with the Inference Endpoint. Defaults to the current user's namespace.", + ), +] + + +@ie_cli.command("list | ls", examples=["hf endpoints ls", "hf endpoints ls --namespace my-org"]) +def ls( + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Lists all Inference Endpoints for the given namespace.""" + api = get_hf_api(token=token) + try: + endpoints = api.list_inference_endpoints(namespace=namespace, token=token) + except HfHubHTTPError as error: + out.error(f"Listing failed: {error}") + raise click.exceptions.Exit(code=error.response.status_code) from error + + results = [] + for endpoint in endpoints: + raw = endpoint.raw + status = raw.get("status", {}) + model = raw.get("model", {}) + compute = raw.get("compute", {}) + provider = raw.get("provider", {}) + results.append( + { + "name": raw.get("name", ""), + "model": model.get("repository", "") if isinstance(model, dict) else "", + "status": status.get("state", "") if isinstance(status, dict) else "", + "task": model.get("task", "") if isinstance(model, dict) else "", + "framework": model.get("framework", "") if isinstance(model, dict) else "", + "instance": compute.get("instanceType", "") if isinstance(compute, dict) else "", + "vendor": provider.get("vendor", "") if isinstance(provider, dict) else "", + "region": provider.get("region", "") if isinstance(provider, dict) else "", + } + ) + out.table(results, id_key="name") + + +@ie_cli.command(name="deploy", examples=["hf endpoints deploy my-endpoint --repo gpt2 --framework pytorch ..."]) +def deploy( + name: NameArg, + repo: Annotated[ + str, + Option( + help="The name of the model repository associated with the Inference Endpoint (e.g. 'openai/gpt-oss-120b').", + ), + ], + framework: Annotated[ + str, + Option( + help="The machine learning framework used for the model (e.g. 'vllm').", + ), + ], + accelerator: Annotated[ + str, + Option( + help="The hardware accelerator to be used for inference (e.g. 'cpu').", + ), + ], + instance_size: Annotated[ + str, + Option( + help="The size or type of the instance to be used for hosting the model (e.g. 'x4').", + ), + ], + instance_type: Annotated[ + str, + Option( + help="The cloud instance type where the Inference Endpoint will be deployed (e.g. 'intel-icl').", + ), + ], + region: Annotated[ + str, + Option( + help="The cloud region in which the Inference Endpoint will be created (e.g. 'us-east-1').", + ), + ], + vendor: Annotated[ + str, + Option( + help="The cloud provider or vendor where the Inference Endpoint will be hosted (e.g. 'aws').", + ), + ], + *, + namespace: NamespaceOpt = None, + task: Annotated[ + str | None, + Option( + help="The task on which to deploy the model (e.g. 'text-classification').", + ), + ] = None, + token: TokenOpt = None, + min_replica: Annotated[ + int, + Option( + help="The minimum number of replicas (instances) to keep running for the Inference Endpoint.", + ), + ] = 1, + max_replica: Annotated[ + int, + Option( + help="The maximum number of replicas (instances) to scale to for the Inference Endpoint.", + ), + ] = 1, + scale_to_zero_timeout: Annotated[ + int | None, + Option( + help="The duration in minutes before an inactive endpoint is scaled to zero.", + ), + ] = None, + scaling_metric: Annotated[ + InferenceEndpointScalingMetric | None, + Option( + help="The metric reference for scaling.", + ), + ] = None, + scaling_threshold: Annotated[ + float | None, + Option( + help="The scaling metric threshold used to trigger a scale up. Ignored when scaling metric is not provided.", + ), + ] = None, + revision: RevisionOpt = None, + custom_image: Annotated[ + str | None, + Option( + "--custom-image", + help="Docker image URL for a custom container (e.g. 'nexagi/sglang:v0.5.12'). Requires '--framework custom'.", + ), + ] = None, + health_route: Annotated[ + str | None, + Option( + help="Health check route exposed by the custom container (e.g. '/health'). Requires --custom-image.", + ), + ] = None, + port: Annotated[ + int | None, + Option( + help="Port the custom container listens on (e.g. 30000). Requires --custom-image.", + ), + ] = None, + container_command: Annotated[ + str | None, + Option( + "--container-command", + help=( + "Override the container entrypoint, as a quoted string split into tokens " + '(e.g. "python -m sglang.launch_server"). Requires --custom-image.' + ), + ), + ] = None, + container_args: Annotated[ + str | None, + Option( + "--container-args", + help=( + "Arguments appended to the container entrypoint, as a quoted string split into tokens " + '(e.g. "--tp 8 --reasoning-parser qwen3"). Requires --custom-image.' + ), + ), + ] = None, + env: EnvOpt = None, + env_file: EnvFileOpt = None, + secrets: SecretsOpt = None, + secrets_file: SecretsFileOpt = None, + endpoint_type: Annotated[ + str | None, + Option( + "--type", + click_type=SoftChoice(InferenceEndpointType), + help="Endpoint access type. Defaults to 'authenticated' (token-gated, publicly reachable).", + ), + ] = None, +) -> None: + """Deploy an Inference Endpoint from a Hub repository.""" + # Custom-container knobs only make sense alongside a custom image. + if custom_image is None and (health_route is not None or port is not None or container_command or container_args): + raise CLIError("--health-route, --port, --container-command and --container-args require --custom-image.") + custom_image_dict: dict | None = None + if custom_image is not None: + custom_image_dict = {"url": custom_image} + if health_route is not None: + custom_image_dict["healthRoute"] = health_route + if port is not None: + custom_image_dict["port"] = port + + env_map = {key: value or "" for key, value in parse_env_map(env, env_file).items()} + secrets_map = {key: value or "" for key, value in parse_env_map(secrets, secrets_file).items()} + + # Only forward the values the user actually set and let `create_inference_endpoint` own the defaults. + params: dict = {} + if endpoint_type is not None: + params["type"] = endpoint_type + if custom_image_dict is not None: + params["custom_image"] = custom_image_dict + if container_command: + params["container_command"] = shlex.split(container_command) + if container_args: + params["container_args"] = shlex.split(container_args) + if env_map: + params["env"] = env_map + if secrets_map: + params["secrets"] = secrets_map + + api = get_hf_api(token=token) + endpoint = api.create_inference_endpoint( + name=name, + repository=repo, + framework=framework, + accelerator=accelerator, + instance_size=instance_size, + instance_type=instance_type, + region=region, + vendor=vendor, + namespace=namespace, + task=task, + token=token, + min_replica=min_replica, + max_replica=max_replica, + scaling_metric=scaling_metric, + scaling_threshold=scaling_threshold, + scale_to_zero_timeout=scale_to_zero_timeout, + revision=revision, + **params, + ) + out.dict(endpoint.raw) + out.hint(f"Use 'hf endpoints describe {name}' to check the deployment status.") + + +@catalog_app.command(name="deploy", examples=["hf endpoints catalog deploy --repo meta-llama/Llama-3.2-1B-Instruct"]) +def deploy_from_catalog( + repo: Annotated[ + str, + Option( + help="The name of the model repository associated with the Inference Endpoint (e.g. 'openai/gpt-oss-120b').", + ), + ], + name: NameOpt = None, + accelerator: Annotated[ + str | None, + Option( + help="The hardware accelerator to be used for inference (e.g. 'cpu', 'gpu', 'neuron').", + ), + ] = None, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Deploy an Inference Endpoint from the Model Catalog.""" + api = get_hf_api(token=token) + try: + endpoint = api.create_inference_endpoint_from_catalog( + repo_id=repo, + name=name, + accelerator=accelerator, + namespace=namespace, + token=token, + ) + except HfHubHTTPError as error: + out.error(f"Deployment failed: {error}") + raise click.exceptions.Exit(code=error.response.status_code) from error + + out.dict(endpoint.raw) + + +def list_catalog( + token: TokenOpt = None, +) -> None: + """List available Catalog models.""" + api = get_hf_api(token=token) + try: + models = api.list_inference_catalog(token=token) + except HfHubHTTPError as error: + out.error(f"Catalog fetch failed: {error}") + raise click.exceptions.Exit(code=error.response.status_code) from error + + out.dict({"models": models}) + + +catalog_app.command(name="list | ls", examples=["hf endpoints catalog ls"])(list_catalog) +ie_cli.command(name="list-catalog", hidden=True)(list_catalog) + + +ie_cli.add_group(catalog_app, name="catalog") + + +@ie_cli.command(examples=["hf endpoints describe my-endpoint"]) +def describe( + name: NameArg, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Get information about an existing endpoint.""" + api = get_hf_api(token=token) + try: + endpoint = api.get_inference_endpoint(name=name, namespace=namespace, token=token) + except HfHubHTTPError as error: + out.error(f"Fetch failed: {error}") + raise click.exceptions.Exit(code=error.response.status_code) from error + + out.dict(endpoint.raw) + + +@ie_cli.command(examples=["hf endpoints update my-endpoint --min-replica 2"]) +def update( + name: NameArg, + namespace: NamespaceOpt = None, + repo: Annotated[ + str | None, + Option( + help="The name of the model repository associated with the Inference Endpoint (e.g. 'openai/gpt-oss-120b').", + ), + ] = None, + accelerator: Annotated[ + str | None, + Option( + help="The hardware accelerator to be used for inference (e.g. 'cpu').", + ), + ] = None, + instance_size: Annotated[ + str | None, + Option( + help="The size or type of the instance to be used for hosting the model (e.g. 'x4').", + ), + ] = None, + instance_type: Annotated[ + str | None, + Option( + help="The cloud instance type where the Inference Endpoint will be deployed (e.g. 'intel-icl').", + ), + ] = None, + framework: Annotated[ + str | None, + Option( + help="The machine learning framework used for the model (e.g. 'custom').", + ), + ] = None, + revision: Annotated[ + str | None, + Option( + help="The specific model revision to deploy on the Inference Endpoint (e.g. '6c0e6080953db56375760c0471a8c5f2929baf11').", + ), + ] = None, + task: Annotated[ + str | None, + Option( + help="The task on which to deploy the model (e.g. 'text-classification').", + ), + ] = None, + min_replica: Annotated[ + int | None, + Option( + help="The minimum number of replicas (instances) to keep running for the Inference Endpoint.", + ), + ] = None, + max_replica: Annotated[ + int | None, + Option( + help="The maximum number of replicas (instances) to scale to for the Inference Endpoint.", + ), + ] = None, + scale_to_zero_timeout: Annotated[ + int | None, + Option( + help="The duration in minutes before an inactive endpoint is scaled to zero.", + ), + ] = None, + scaling_metric: Annotated[ + InferenceEndpointScalingMetric | None, + Option( + help="The metric reference for scaling.", + ), + ] = None, + scaling_threshold: Annotated[ + float | None, + Option( + help="The scaling metric threshold used to trigger a scale up. Ignored when scaling metric is not provided.", + ), + ] = None, + token: TokenOpt = None, +) -> None: + """Update an existing endpoint.""" + api = get_hf_api(token=token) + try: + endpoint = api.update_inference_endpoint( + name=name, + namespace=namespace, + repository=repo, + framework=framework, + revision=revision, + task=task, + accelerator=accelerator, + instance_size=instance_size, + instance_type=instance_type, + min_replica=min_replica, + max_replica=max_replica, + scale_to_zero_timeout=scale_to_zero_timeout, + scaling_metric=scaling_metric, + scaling_threshold=scaling_threshold, + token=token, + ) + except HfHubHTTPError as error: + out.error(f"Update failed: {error}") + raise click.exceptions.Exit(code=error.response.status_code) from error + out.dict(endpoint.raw) + + +@ie_cli.command(examples=["hf endpoints delete my-endpoint"]) +def delete( + name: NameArg, + namespace: NamespaceOpt = None, + yes: Annotated[ + bool, + Option("--yes", help="Skip confirmation prompts."), + ] = False, + token: TokenOpt = None, +) -> None: + """Delete an Inference Endpoint permanently.""" + out.confirm(f"Delete endpoint '{name}'?", yes=yes) + + api = get_hf_api(token=token) + try: + api.delete_inference_endpoint(name=name, namespace=namespace, token=token) + except HfHubHTTPError as error: + out.error(f"Delete failed: {error}") + raise click.exceptions.Exit(code=error.response.status_code) from error + + out.result(f"Deleted '{name}'.", name=name) + + +@ie_cli.command(examples=["hf endpoints pause my-endpoint"]) +def pause( + name: NameArg, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Pause an Inference Endpoint.""" + api = get_hf_api(token=token) + try: + endpoint = api.pause_inference_endpoint(name=name, namespace=namespace, token=token) + except HfHubHTTPError as error: + out.error(f"Pause failed: {error}") + raise click.exceptions.Exit(code=error.response.status_code) from error + + out.dict(endpoint.raw) + + +@ie_cli.command(examples=["hf endpoints resume my-endpoint"]) +def resume( + name: NameArg, + namespace: NamespaceOpt = None, + fail_if_already_running: Annotated[ + bool, + Option( + "--fail-if-already-running", + help="If `True`, the method will raise an error if the Inference Endpoint is already running.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + """Resume an Inference Endpoint.""" + api = get_hf_api(token=token) + try: + endpoint = api.resume_inference_endpoint( + name=name, + namespace=namespace, + token=token, + running_ok=not fail_if_already_running, + ) + except HfHubHTTPError as error: + out.error(f"Resume failed: {error}") + raise click.exceptions.Exit(code=error.response.status_code) from error + out.dict(endpoint.raw) + + +@ie_cli.command(examples=["hf endpoints scale-to-zero my-endpoint"]) +def scale_to_zero( + name: NameArg, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Scale an Inference Endpoint to zero.""" + api = get_hf_api(token=token) + try: + endpoint = api.scale_to_zero_inference_endpoint(name=name, namespace=namespace, token=token) + except HfHubHTTPError as error: + out.error(f"Scale To Zero failed: {error}") + raise click.exceptions.Exit(code=error.response.status_code) from error + + out.dict(endpoint.raw) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/jobs.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/jobs.py new file mode 100644 index 0000000000000000000000000000000000000000..18a56e75238a5a639b294a9edf36a1499166a624 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/jobs.py @@ -0,0 +1,1400 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with jobs on the Hugging Face Hub.""" + +import itertools +import multiprocessing +import multiprocessing.pool +import shutil +import time +from collections.abc import Callable, Iterable +from fnmatch import fnmatch +from pathlib import Path +from queue import Empty, Queue +from typing import Annotated, Any, TypeVar +from urllib.parse import urlsplit + +from huggingface_hub import HfApi, JobHardware, JobInfo, JobStage, Volume, constants +from huggingface_hub._jobs_api import TERMINAL_JOB_STAGES +from huggingface_hub.errors import CLIError +from huggingface_hub.utils import logging +from huggingface_hub.utils._cache_manager import _format_size +from huggingface_hub.utils._hf_uris import _split_mount +from huggingface_hub.utils._parsing import format_duration, parse_duration + +from ._cli_utils import ( + EnvFileOpt, + EnvOpt, + SecretsFileOpt, + SecretsOpt, + SoftChoice, + SshDryRunOpt, + SshIdentityFileOpt, + TokenOpt, + exec_ssh, + get_hf_api, + parse_env_map, + parse_volumes, + typer_factory, +) +from ._framework import Argument, Option +from ._output import _dataclass_to_dict, out + + +logger = logging.get_logger(__name__) + + +def _parse_namespace_from_job_id(job_id: str, namespace: str | None) -> tuple[str, str | None]: + """Extract namespace from job_id if provided in 'namespace/job_id' format. + + Allows users to pass job IDs copied from the Hub UI (e.g. 'username/job_id') + instead of only bare job IDs. If the namespace is also provided explicitly via + --namespace and conflicts, a CLIError is raised. + """ + if not job_id: + raise CLIError("Job ID cannot be empty.") + + if job_id.count("/") > 1: + raise CLIError(f"Job ID must be in the form 'job_id' or 'namespace/job_id': '{job_id}'.") + + if "/" not in job_id: + return job_id, namespace + + extracted_namespace, parsed_job_id = job_id.split("/", 1) + if not extracted_namespace or not parsed_job_id: + raise CLIError(f"Job ID must be in the form 'job_id' or 'namespace/job_id': '{job_id}'.") + + if namespace is not None and namespace != extracted_namespace: + raise CLIError( + f"Conflicting namespace: got --namespace='{namespace}' but job ID implies namespace='{extracted_namespace}'" + ) + + return parsed_job_id, extracted_namespace + + +def _parse_and_sync_job_volumes( + volumes: list[str] | None, *, api: HfApi, namespace: str | None +) -> list[Volume] | None: + """Parse `-v` specs for Jobs commands. + + Same as [`parse_volumes`] but the source side can also be a local directory: it is synced to a + bucket via [`HfApi.sync_job_volume`] and the resulting bucket subfolder is mounted (read-only + unless ':rw' is specified). + """ + if not volumes: + return None + + result: list[Volume] = [] + for raw_spec in volumes: + if raw_spec.startswith(constants.HF_PROTOCOL): + result.extend(parse_volumes([raw_spec]) or []) + continue + + # Not a 'hf://' URI: treat the source as a local directory. + source, mount_path, read_only = _split_mount(raw_spec, raw=raw_spec) + if mount_path is None: + raise CLIError( + f"Missing mount path in volume spec '{raw_spec}'. Expected 'LOCAL_DIR:/MOUNT_PATH[:ro|:rw]' (e.g. './data:/data')." + ) + if not Path(source).expanduser().is_dir(): + raise CLIError( + f"Volume source '{source}' is not an existing local directory. " + "To mount a repo or bucket instead, use the 'hf://' syntax (e.g. 'hf://buckets/my-org/my-bucket:/data')." + ) + volume = api.sync_job_volume( + source, + mount_path, + read_only=read_only if read_only is not None else True, + namespace=namespace, + ) + if volume.read_only is False: + out.hint( + f"Volume '{mount_path}' is mounted read-write. Once the job is over, pull back its data with:\n" + f" hf buckets sync hf://buckets/{volume.source}/{volume.path} {source}" + ) + result.append(volume) + return result + + +STATS_UPDATE_MIN_INTERVAL = 0.1 # we set a limit here since there is one update per second per job + +# Common job-related options +ImageArg = Annotated[ + str, + Argument( + help="The Docker image to use.", + ), +] + +ImageOpt = Annotated[ + str | None, + Option( + help="Use a custom Docker image with `uv` installed.", + ), +] + +FlavorOpt = Annotated[ + str | None, + Option( + help="Flavor for the hardware. Run 'hf jobs hardware' to list available flavors. Defaults to `cpu-basic`.", + click_type=SoftChoice(JobHardware), + ), +] + +LabelsOpt = Annotated[ + list[str] | None, + Option( + "-l", + "--label", + help="Set labels. E.g. --label KEY=VALUE or --label LABEL", + ), +] + +NameOpt = Annotated[ + str | None, + Option( + "--name", + help="Name the Job. Stored as the `name` label. Names do not have to be unique. Defaults to the image or script name plus a short hash of the command.", + ), +] + +TimeoutOpt = Annotated[ + str | None, + Option( + help="Max duration: int with s (seconds, default), m (minutes), h (hours) or d (days).", + ), +] + +DetachOpt = Annotated[ + bool, + Option( + "-d", + "--detach", + help="Run the Job in the background and print the Job ID.", + ), +] + +NamespaceOpt = Annotated[ + str | None, + Option( + help="The namespace where the job will be running. Defaults to the current user's namespace.", + ), +] + +ExposeOpt = Annotated[ + list[int] | None, + Option( + "--expose", + help="Expose a container port through the jobs proxy. Repeat the flag for multiple ports (e.g. `--expose 8000 --expose 8001`). Each exposed port is reachable on the public jobs domain; access requires an HF token with read access to the job's namespace.", + ), +] + +SshEnabledOpt = Annotated[ + bool, + Option( + "--ssh", + help="Make the job's container reachable over SSH. Connect with `hf jobs ssh `. Requires an SSH public key registered on https://huggingface.co/settings/keys.", + ), +] + +WithOpt = Annotated[ + list[str] | None, + Option( + "--with", + help="Run with the given packages installed", + ), +] + +PythonOpt = Annotated[ + str | None, + Option( + "-p", + "--python", + help="The Python interpreter to use for the run environment", + ), +] + +SuspendOpt = Annotated[ + bool | None, + Option( + help="Suspend (pause) the scheduled Job", + ), +] + +ConcurrencyOpt = Annotated[ + bool | None, + Option( + help="Allow multiple instances of this Job to run concurrently", + ), +] + +ScheduleArg = Annotated[ + str, + Argument( + help="One of annually, yearly, monthly, weekly, daily, hourly, or a CRON schedule expression.", + ), +] + +ScriptArg = Annotated[ + str, + Argument( + help="UV script to run (local file or URL)", + ), +] + +ScriptArgsArg = Annotated[ + list[str] | None, + Argument( + help="Arguments for the script", + ), +] + + +CommandArg = Annotated[ + list[str], + Argument( + help="The command to run.", + ), +] + +JobIdArg = Annotated[ + str, + Argument( + help="Job ID (or 'namespace/job_id')", + ), +] + +JobIdsArg = Annotated[ + list[str] | None, + Argument( + help="Job IDs (or 'namespace/job_id')", + ), +] + +ScheduledJobIdArg = Annotated[ + str, + Argument( + help="Scheduled Job ID (or 'namespace/scheduled_job_id')", + ), +] + +JobVolumesOpt = Annotated[ + list[str] | None, + Option( + "-v", + "--volume", + help="Mount one or more volumes. Format: hf://[TYPE/]SOURCE:/MOUNT_PATH[:ro|:rw] or LOCAL_DIR:/MOUNT_PATH[:ro|:rw]. " + "TYPE is one of: models, datasets, spaces, buckets. " + "TYPE defaults to models if omitted. " + "models, datasets and spaces are always mounted read-only. buckets are read+write by default. " + "A local directory source is first synced to a bucket and mounted read-only by default. " + "E.g. -v hf://datasets/org/ds:/data or -v hf://buckets/org/b:/mnt:ro or -v ./inputs:/inputs", + ), +] + + +jobs_cli = typer_factory(help="Run and manage Jobs on the Hub.") + + +def _stream_logs_and_check_status(api: HfApi, job: JobInfo) -> None: + """Stream Job logs until the Job ends, then fail the command if the Job did not complete successfully.""" + for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True): + out.text(log) + # The log stream can end while the Job is still scheduling or shutting down: settle the final state. + final = api.wait_for_job(job_id=job.id, namespace=job.owner.name) + if final.status.stage != JobStage.COMPLETED: + message = f": {final.status.message}" if final.status.message else "" + raise CLIError(f"Job {final.id} finished with stage '{final.status.stage}'{message}") + out.text(f"Job {final.id} completed") + + +@jobs_cli.command( + "run", + context_settings={"ignore_unknown_options": True}, + examples=[ + "hf jobs run --name hello-world python:3.12 python -c 'print(\"Hello!\")'", + "hf jobs run --detach python:3.12 python script.py", + "hf jobs run -e FOO=foo python:3.12 python script.py", + "hf jobs run --secrets HF_TOKEN python:3.12 python script.py", + "hf jobs run -v hf://org/my-model:/data -v hf://buckets/org/b:/mnt python:3.12 python script.py", + ], +) +def jobs_run( + image: ImageArg, + command: CommandArg, + env: EnvOpt = None, + secrets: SecretsOpt = None, + name: NameOpt = None, + label: LabelsOpt = None, + volume: JobVolumesOpt = None, + env_file: EnvFileOpt = None, + secrets_file: SecretsFileOpt = None, + flavor: FlavorOpt = None, + timeout: TimeoutOpt = None, + detach: DetachOpt = False, + expose: ExposeOpt = None, + ssh: SshEnabledOpt = False, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Run a Job.""" + env_map = parse_env_map(env, env_file) + secrets_map = parse_env_map(secrets, secrets_file) + + api = get_hf_api(token=token) + job = api.run_job( + image=image, + command=command, + env=env_map, + secrets=secrets_map, + labels=_parse_labels_map(label, name=name), + volumes=_parse_and_sync_job_volumes(volume, api=api, namespace=namespace), + flavor=flavor, + timeout=timeout, + expose=expose, + ssh=ssh, + namespace=namespace, + ) + out.result("Job started", id=job.id, url=job.url) + if not _has_explicit_name(name, label): + auto_name = (job.labels or {}).get("name") + out.hint( + f"Job auto-named '{auto_name}'. Pass `--name` or run " + f"`hf jobs labels {job.owner.name}/{job.id} --name NAME` to rename it." + ) + if isinstance(job.status.expose_urls, list): + urls = "\n".join(f" {url}" for url in job.status.expose_urls) + out.hint(f"Exposed ports are reachable at (requires an HF token with read access to the job):\n{urls}") + if isinstance(job.status.ssh_url, str): + out.hint(f"Use `hf jobs ssh {job.owner.name}/{job.id}` to open an SSH session into the job.") + if detach: + job_ref = f"{job.owner.name}/{job.id}" + out.hint(f"Use `hf jobs logs -f {job_ref}` to stream logs, or `hf jobs inspect {job_ref}` to check status.") + out.hint(f"Use `hf jobs wait {job_ref}` to block until it finishes.") + return + _stream_logs_and_check_status(api, job) + + +@jobs_cli.command( + "logs", + examples=[ + "hf jobs logs ", + "hf jobs logs -f ", + "hf jobs logs --tail 20 ", + "hf jobs logs -f --tail 100 ", + ], +) +def jobs_logs( + job_id: JobIdArg, + follow: Annotated[ + bool, + Option( + "-f", + "--follow", + help="Follow log output (stream until the job completes). Without this flag, only currently available logs are printed.", + ), + ] = False, + tail: Annotated[ + int | None, + Option( + "-n", + "--tail", + help="Number of lines to show from the end of the logs. When combined with --follow, starts streaming from the last N lines.", + ), + ] = None, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Fetch the logs of a Job. + + By default, prints currently available logs and exits (non-blocking). + Use --follow/-f to stream logs in real-time until the job completes. + Use --tail/-n to limit the number of lines returned (server-side when supported). + + Note: following exits when the log stream ends, regardless of whether the Job + succeeded or failed. Run `hf jobs inspect ` to check the final status. + """ + job_id, namespace = _parse_namespace_from_job_id(job_id, namespace) + + api = get_hf_api(token=token) + logs = api.fetch_job_logs(job_id=job_id, namespace=namespace, follow=follow, tail=tail) + for log in logs: + out.text(log) + if follow: + job_ref = f"{namespace}/{job_id}" if namespace else job_id + out.hint(f"Stream ended. Run `hf jobs inspect {job_ref}` to check the final status (e.g. COMPLETED or ERROR).") + + +def _matches_filters(job_properties: dict[str, str], filters: list[tuple[str, str, str]]) -> bool: + """Check if scheduled job matches all specified filters.""" + for key, op_str, pattern in filters: + value = job_properties.get(key) + if value is None: + if op_str == "!=": + continue + return False + match = fnmatch(value.lower(), pattern.lower()) + if (op_str == "=" and not match) or (op_str == "!=" and match): + return False + return True + + +def _clear_line(n: int) -> None: + LINE_UP = "\033[1A" + LINE_CLEAR = "\x1b[2K" + for i in range(n): + print(LINE_UP, end=LINE_CLEAR) + + +def _get_jobs_stats_rows( + job_id: str, metrics_stream: Iterable[dict[str, Any]], table_headers: list[str] +) -> Iterable[tuple[bool, str, list[list[str | int]]]]: + for metrics in metrics_stream: + row = [ + job_id, + f"{metrics['cpu_usage_pct']}%", + round(metrics["cpu_millicores"] / 1000.0, 1), + f"{round(100 * metrics['memory_used_bytes'] / metrics['memory_total_bytes'], 2)}%", + f"{_format_size(metrics['memory_used_bytes'])}B / {_format_size(metrics['memory_total_bytes'])}B", + f"{_format_size(metrics['rx_bps'])}bps / {_format_size(metrics['tx_bps'])}bps", + ] + if metrics["gpus"] and isinstance(metrics["gpus"], dict): + rows = [row] + [[""] * len(row)] * (len(metrics["gpus"]) - 1) + for row, gpu_id in zip(rows, sorted(metrics["gpus"])): + gpu = metrics["gpus"][gpu_id] + row += [ + f"{gpu['utilization']}%", + f"{round(100 * gpu['memory_used_bytes'] / gpu['memory_total_bytes'], 2)}%", + f"{_format_size(gpu['memory_used_bytes'])}B / {_format_size(gpu['memory_total_bytes'])}B", + ] + else: + row += ["N/A"] * (len(table_headers) - len(row)) + rows = [row] + yield False, job_id, rows + yield True, job_id, [] + + +@jobs_cli.command("stats", examples=["hf jobs stats "]) +def jobs_stats( + job_ids: JobIdsArg = None, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Fetch the resource usage statistics and metrics of Jobs""" + if job_ids is not None: + parsed_ids = [] + for job_id in job_ids: + job_id, namespace = _parse_namespace_from_job_id(job_id, namespace) + parsed_ids.append(job_id) + job_ids = parsed_ids + api = get_hf_api(token=token) + if namespace is None: + namespace = api.whoami()["name"] + if job_ids is None: + job_ids = [ + job.id + for job in api.list_jobs(namespace=namespace) + if (job.status.stage if job.status else "UNKNOWN") in ("RUNNING", "UPDATING") + ] + if len(job_ids) == 0: + out.text("No running jobs found") + return + table_headers = [ + "JOB ID", + "CPU %", + "NUM CPU", + "MEM %", + "MEM USAGE", + "NET I/O", + "GPU UTIL %", + "GPU MEM %", + "GPU MEM USAGE", + ] + with multiprocessing.pool.ThreadPool(len(job_ids)) as pool: + rows_per_job_id: dict[str, list[list[str | int]]] = {} + for job_id in job_ids: + row: list[str | int] = [job_id] + row += ["-- / --" if ("/" in header or "USAGE" in header) else "--" for header in table_headers[1:]] + rows_per_job_id[job_id] = [row] + last_update_time = time.time() + total_rows = [row for job_id in rows_per_job_id for row in rows_per_job_id[job_id]] + # In-place refresh (cursor-up + clear) requires a fixed line count and layout — + # `out.table`'s mode-dependent formatting would break it. + print(_tabulate(total_rows, headers=table_headers)) + + kwargs_list = [ + { + "job_id": job_id, + "metrics_stream": api.fetch_job_metrics(job_id=job_id, namespace=namespace), + "table_headers": table_headers, + } + for job_id in job_ids + ] + for done, job_id, rows in iflatmap_unordered(pool, _get_jobs_stats_rows, kwargs_list=kwargs_list): + if done: + rows_per_job_id.pop(job_id, None) + else: + rows_per_job_id[job_id] = rows + now = time.time() + if now - last_update_time >= STATS_UPDATE_MIN_INTERVAL: + _clear_line(2 + len(total_rows)) + total_rows = [row for job_id in rows_per_job_id for row in rows_per_job_id[job_id]] + print(_tabulate(total_rows, headers=table_headers)) + last_update_time = now + + +@jobs_cli.command( + "list | ls | ps", + examples=[ + "hf jobs ls", + "hf jobs ls -a", + "hf jobs ls --status running,scheduling", + "hf jobs ls --label env=prod --label team=ml", + "hf jobs ls --all --label hf-sandbox=1", + ], +) +def jobs_ps( + all: Annotated[ + bool, + Option( + "-a", + "--all", + help="Show all Jobs (default shows running and scheduling). Cannot be combined with --status.", + ), + ] = False, + status: Annotated[ + list[str] | None, + Option( + "--status", + click_type=SoftChoice(JobStage), + help="Only show Jobs with the given status. Comma-separated or repeated, e.g. `--status running,scheduling`.", + ), + ] = None, + label: Annotated[ + list[str] | None, + Option( + "-l", + "--label", + help="Only show Jobs with the given `key=value` label. Repeat to require several labels, e.g. `--label env=prod --label team=ml`.", + ), + ] = None, + limit: Annotated[ + int, + Option( + "--limit", + help="Maximum number of Jobs to display. Set to 0 to show all (no limit).", + ), + ] = 100, + namespace: NamespaceOpt = None, + token: TokenOpt = None, + filter: Annotated[ + list[str] | None, + Option( + "-f", + "--filter", + help="(Deprecated) Use `--status` and `--label` instead.", + ), + ] = None, +) -> None: + """List Jobs. + + Use `--status` to filter by status (see [`JobStage`] for possible values) and `--label` to filter by `key=value` + labels. A Job must match every filter to be listed. + """ + api = get_hf_api(token=token) + + if filter: + out.warning( + f"Ignoring filter '{filter}'." + " `-f`/`--filter` is deprecated and will be removed in a future release. Use `--status`/`--label`." + ) + + if all and status: + raise CLIError("`-a`/`--all` cannot be combined with `--status`.") + + # Status filtering (default to active Jobs, unless `--all` or `--status` is provided). + raw_statuses: list[str] = [] + for value in status or []: + raw_statuses.extend(part.strip() for part in value.split(",") if part.strip()) + + server_statuses: list[str] | None + if raw_statuses: + server_statuses = raw_statuses + elif all: + server_statuses = None + else: + server_statuses = [JobStage.RUNNING.value, JobStage.SCHEDULING.value] + + # Labels filtering + labels: dict[str, str] = {} + for item in label or []: + if "=" not in item: + raise CLIError(f"Invalid label filter '{item}': must be in the form 'key=value'") + key, value = item.split("=") + labels[key] = value + + jobs_iter = api.list_jobs(namespace=namespace, status=server_statuses, labels=labels or None) + + # Apply the display limit. Fetch one extra Job to detect (and warn about) truncation. + truncated = False + if limit > 0: + jobs = list(itertools.islice(jobs_iter, limit + 1)) + if len(jobs) > limit: + truncated = True + jobs = jobs[:limit] + else: + jobs = list(jobs_iter) + + # Build display items. Augment the raw api dict with curated, table-friendly columns. + job_items: list[dict[str, Any]] = [] + for job in jobs: + job_item = _dataclass_to_dict(job) + durations = job_item.get("durations") or {} + cmd = job_item.get("command") or [] + job_item["job_id"] = job_item.get("id", "") + job_item["image/space"] = job_item.get("docker_image") or "N/A" + job_item["command"] = " ".join(cmd) if cmd else "N/A" + job_item["created"] = job_item["created_at"][:19].replace("T", " ") if job_item.get("created_at") else "N/A" + job_item["status"] = (job_item.get("status") or {}).get("stage", "UNKNOWN") + job_item["runtime"] = format_duration(durations.get("running_secs")) + job_items.append(job_item) + + out.table( + job_items, + headers=["job_id", "image/space", "command", "created", "status", "runtime"], + id_key="job_id", + ) + if truncated: + out.hint(f"Output truncated to {limit} Jobs. Use `--limit 0` to show all (or `--limit N`).") + if not job_items: + if raw_statuses or labels: + filters_msg = ", ".join( + [*(f"status={s}" for s in raw_statuses), *(f"label={k}={v}" for k, v in labels.items())] + ) + out.text(f"No jobs matched filters: {filters_msg}") + elif not all: + out.hint("No running jobs. Use `-a`/`--all` to include finished (and failed) jobs.") + + +@jobs_cli.command("hardware", examples=["hf jobs hardware"]) +def jobs_hardware() -> None: + """List available hardware options for Jobs""" + api = get_hf_api() + hardware_list = api.list_jobs_hardware() + items = [] + for hw in hardware_list: + accelerator_info = "" + if hw.accelerator: + accelerator_info = f"{hw.accelerator.quantity}x {hw.accelerator.model} ({hw.accelerator.vram})" + cost_min = f"${hw.unit_cost_usd:.4f}" if hw.unit_cost_usd else "free" + cost_hour = f"${hw.unit_cost_usd * 60:.2f}" if hw.unit_cost_usd else "free" + items.append( + { + "name": hw.name, + "pretty name": hw.pretty_name, + "cpu": hw.cpu, + "ram": hw.ram, + "storage": hw.ephemeral_storage, + "accelerator": accelerator_info, + "cost/min": cost_min, + "cost/hour": cost_hour, + } + ) + out.table(items) + out.hint("Use `hf jobs run --flavor ...` to request a specific hardware flavor.") + + +@jobs_cli.command("inspect", examples=["hf jobs inspect "]) +def jobs_inspect( + job_ids: Annotated[ + list[str], + Argument( + help="Job IDs to inspect (or 'namespace/job_id')", + ), + ], + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Display detailed information on one or more Jobs""" + parsed_ids = [] + for job_id in job_ids: + job_id, namespace = _parse_namespace_from_job_id(job_id, namespace) + parsed_ids.append(job_id) + job_ids = parsed_ids + api = get_hf_api(token=token) + jobs = [api.inspect_job(job_id=job_id, namespace=namespace) for job_id in job_ids] + out.table([_dataclass_to_dict(job) for job in jobs]) + + +@jobs_cli.command("cancel", examples=["hf jobs cancel "]) +def jobs_cancel( + job_id: JobIdArg, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Cancel a Job""" + job_id, namespace = _parse_namespace_from_job_id(job_id, namespace) + api = get_hf_api(token=token) + api.cancel_job(job_id=job_id, namespace=namespace) + out.result("Job cancelled", id=job_id) + + +@jobs_cli.command( + "wait", + examples=[ + "hf jobs wait ", + "hf jobs wait ", + "hf jobs ls -q | xargs hf jobs wait", + ], +) +def jobs_wait( + job_ids: Annotated[ + list[str], + Argument( + help="Job IDs to wait for (or 'namespace/job_id').", + ), + ], + timeout: Annotated[ + str | None, + Option( + help="Max time to wait: int with s (seconds, default), m (minutes), h (hours) or d (days).", + ), + ] = None, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Wait for one or more Jobs to reach a terminal state. + + Blocks until every Job has finished, then exits with code 0 if all Jobs completed + successfully, or a non-zero exit code if any Job was canceled, errored or deleted. + + All Jobs must belong to the same namespace. + """ + parsed_ids = [] + namespaces = set() + for job_id in job_ids: + parsed_id, parsed_namespace = _parse_namespace_from_job_id(job_id, namespace) + parsed_ids.append(parsed_id) + namespaces.add(parsed_namespace) + if len(namespaces) > 1: + raise CLIError( + "All Job IDs must be in the same namespace, got: " + + ", ".join(str(ns) for ns in sorted(namespaces, key=str)) + ) + namespace = namespaces.pop() + timeout_secs = parse_duration(timeout) if timeout is not None else None + + api = get_hf_api(token=token) + status = out.status(f"Waiting for {len(parsed_ids)} Job(s) to finish...") + try: + jobs = api.wait_for_job(parsed_ids, timeout=timeout_secs, namespace=namespace) + except TimeoutError: + status.done("Timed out.") + raise CLIError(f"Timed out after {timeout} waiting for Job(s) to finish.") from None + status.done(f"{len(jobs)} Job(s) finished.") + + out.table([{"id": job.id, "stage": str(job.status.stage), "message": job.status.message} for job in jobs]) + failed = [job for job in jobs if job.status.stage != JobStage.COMPLETED] + if failed: + raise CLIError( + f"{len(failed)} of {len(jobs)} Job(s) did not complete successfully: " + + ", ".join(f"{job.id} ({job.status.stage})" for job in failed) + ) + + +@jobs_cli.command( + "labels", + examples=[ + "hf jobs labels --name training-v2", + "hf jobs labels --label env=prod --label team=ml", + "hf jobs labels --clear", + ], +) +def jobs_labels( + job_id: JobIdArg, + name: NameOpt = None, + label: LabelsOpt = None, + clear: Annotated[bool, Option("--clear", help="Remove all labels from the job.")] = False, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Update labels on a Job. Passing --label replaces all existing labels; passing --name alone keeps them.""" + if not label and name is None and not clear: + raise CLIError( + "Please set a name with --name or at least one label with --label. To remove all labels, pass --clear." + ) + if (label or name is not None) and clear: + raise CLIError( + "Cannot set a name or labels and clear them at the same time. Please use --name/--label or --clear, not both." + ) + job_id, namespace = _parse_namespace_from_job_id(job_id, namespace) + api = get_hf_api(token=token) + if name is not None and not label: + # Naming a Job should not wipe its existing labels: fetch them and merge the name in. + current_labels = api.inspect_job(job_id=job_id, namespace=namespace).labels or {} + labels = {**current_labels, "name": name} + else: + labels = _parse_labels_map(label, name=name) or {} + job = api.update_job_labels(job_id=job_id, labels=labels, namespace=namespace) + out.result("Labels updated", id=job.id) + + +@jobs_cli.command( + "ssh", + examples=[ + "hf jobs ssh ", + "hf jobs ssh --dry-run", + "hf jobs ssh -i ~/.ssh/id_ed25519", + ], +) +def jobs_ssh( + job_id: JobIdArg, + identity_file: SshIdentityFileOpt = None, + dry_run: SshDryRunOpt = False, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """SSH into a running Job. + + If the Job is not yet running, waits until it reaches the RUNNING state before + connecting. Requires the Job to be started with SSH enabled (`hf jobs run --ssh ...`) + and your SSH public key to be registered at https://huggingface.co/settings/keys. + """ + job_id, namespace = _parse_namespace_from_job_id(job_id, namespace) + api = get_hf_api(token=token) + job = api.inspect_job(job_id=job_id, namespace=namespace) + if job.status.ssh_url is None: + raise CLIError("SSH is not enabled on this job. Start a job with SSH support using `hf jobs run --ssh ...`.") + if job.status.stage in TERMINAL_JOB_STAGES: + raise CLIError(f"Cannot SSH into job '{job.id}': job has already finished (stage: '{job.status.stage}').") + if job.status.stage != JobStage.RUNNING: + status = out.status(f"Waiting for job '{job.id}' to be running (stage: '{job.status.stage}')...") + job = api.wait_for_job(job_id=job.id, namespace=namespace, stages=[JobStage.RUNNING]) + if job.status.stage != JobStage.RUNNING: + status.done("Job finished.") + raise CLIError( + f"Cannot SSH into job '{job.id}': job finished before reaching RUNNING (stage: '{job.status.stage}')." + ) + status.done("Job is running.") + ssh_url = urlsplit(job.status.ssh_url) + exec_ssh( + f"{ssh_url.username}@{ssh_url.hostname}", # type: ignore + port=ssh_url.port, + identity_file=identity_file, + dry_run=dry_run, + ) + + +uv_app = typer_factory(help="Run UV scripts (Python with inline dependencies) on HF infrastructure.") +jobs_cli.add_group(uv_app, name="uv") + + +@uv_app.command( + "run", + context_settings={"ignore_unknown_options": True}, + examples=[ + "hf jobs uv run --name my-script my_script.py", + "hf jobs uv run --detach my_script.py", + "hf jobs uv run ml_training.py --flavor a10g-small", + "hf jobs uv run --with transformers train.py", + "hf jobs uv run -v hf://org/my-model:/data -v hf://buckets/org/b:/mnt script.py", + ], +) +def jobs_uv_run( + script: ScriptArg, + script_args: ScriptArgsArg = None, + image: ImageOpt = None, + flavor: FlavorOpt = None, + env: EnvOpt = None, + secrets: SecretsOpt = None, + name: NameOpt = None, + label: LabelsOpt = None, + volume: JobVolumesOpt = None, + env_file: EnvFileOpt = None, + secrets_file: SecretsFileOpt = None, + timeout: TimeoutOpt = None, + detach: DetachOpt = False, + expose: ExposeOpt = None, + ssh: SshEnabledOpt = False, + namespace: NamespaceOpt = None, + token: TokenOpt = None, + with_: WithOpt = None, + python: PythonOpt = None, +) -> None: + """Run a UV script (local file or URL) on HF infrastructure""" + env_map = parse_env_map(env, env_file) + secrets_map = parse_env_map(secrets, secrets_file) + + api = get_hf_api(token=token) + job = api.run_uv_job( + script=script, + script_args=script_args or [], + dependencies=with_, + python=python, + image=image, + env=env_map, + secrets=secrets_map, + labels=_parse_labels_map(label, name=name), + volumes=_parse_and_sync_job_volumes(volume, api=api, namespace=namespace), + flavor=flavor, + timeout=timeout, + expose=expose, + ssh=ssh, + namespace=namespace, + ) + out.result("Job started", id=job.id, url=job.url) + if not _has_explicit_name(name, label): + auto_name = (job.labels or {}).get("name") + out.hint( + f"Job auto-named '{auto_name}'. Pass `--name` or run " + f"`hf jobs labels {job.owner.name}/{job.id} --name NAME` to rename it." + ) + if isinstance(job.status.expose_urls, list): + urls = "\n".join(f" {url}" for url in job.status.expose_urls) + out.hint(f"Exposed ports are reachable at (requires an HF token with read access to the job):\n{urls}") + if isinstance(job.status.ssh_url, str): + out.hint(f"Use `hf jobs ssh {job.owner.name}/{job.id}` to open an SSH session into the job.") + if detach: + job_ref = f"{job.owner.name}/{job.id}" + out.hint(f"Use `hf jobs logs -f {job_ref}` to stream logs, or `hf jobs inspect {job_ref}` to check status.") + out.hint(f"Use `hf jobs wait {job_ref}` to block until it finishes.") + return + _stream_logs_and_check_status(api, job) + + +scheduled_app = typer_factory(help="Create and manage scheduled Jobs on the Hub.") +jobs_cli.add_group(scheduled_app, name="scheduled") + + +@scheduled_app.command( + "run", + context_settings={"ignore_unknown_options": True}, + examples=['hf jobs scheduled run "0 0 * * *" --name daily-script python:3.12 python script.py'], +) +def scheduled_run( + schedule: ScheduleArg, + image: ImageArg, + command: CommandArg, + suspend: SuspendOpt = None, + concurrency: ConcurrencyOpt = None, + env: EnvOpt = None, + secrets: SecretsOpt = None, + name: NameOpt = None, + label: LabelsOpt = None, + volume: JobVolumesOpt = None, + env_file: EnvFileOpt = None, + secrets_file: SecretsFileOpt = None, + flavor: FlavorOpt = None, + timeout: TimeoutOpt = None, + expose: ExposeOpt = None, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Schedule a Job.""" + env_map = parse_env_map(env, env_file) + secrets_map = parse_env_map(secrets, secrets_file) + + api = get_hf_api(token=token) + scheduled_job = api.create_scheduled_job( + image=image, + command=command, + schedule=schedule, + suspend=suspend, + concurrency=concurrency, + env=env_map, + secrets=secrets_map, + labels=_parse_labels_map(label, name=name), + volumes=_parse_and_sync_job_volumes(volume, api=api, namespace=namespace), + flavor=flavor, + timeout=timeout, + expose=expose, + namespace=namespace, + ) + out.result("Scheduled Job created", id=scheduled_job.id) + if not _has_explicit_name(name, label): + auto_name = (scheduled_job.job_spec.labels or {}).get("name") + out.hint( + f"Scheduled Job auto-named '{auto_name}'. Pass `--name` or run " + f"`hf jobs scheduled labels {scheduled_job.owner.name}/{scheduled_job.id} --name NAME` to rename it." + ) + out.hint(f"Use `hf jobs scheduled inspect {scheduled_job.owner.name}/{scheduled_job.id}` to view its details.") + + +@scheduled_app.command("list | ls | ps", examples=["hf jobs scheduled ls"]) +def scheduled_ps( + all: Annotated[ + bool, + Option( + "-a", + "--all", + help="Show all scheduled Jobs (default hides suspended)", + ), + ] = False, + namespace: NamespaceOpt = None, + token: TokenOpt = None, + filter: Annotated[ + list[str] | None, + Option( + "-f", + "--filter", + help="Filter output based on conditions provided (format: key=value)", + ), + ] = None, +) -> None: + """List scheduled Jobs""" + api = get_hf_api(token=token) + scheduled_jobs = api.list_scheduled_jobs(namespace=namespace) + filters: list[tuple[str, str, str]] = [] + for f in filter or []: + if "=" in f: + key, value = f.split("=", 1) + # Negate predicate in case of key!=value + if key.endswith("!"): + op = "!=" + key = key[:-1] + else: + op = "=" + filters.append((key.lower(), op, value.lower())) + else: + out.warning(f"Ignoring invalid filter format '{f}'. Use key=value format.") + + # Filter scheduled jobs (operating on ScheduledJobInfo objects to preserve existing filter behavior) + filtered_jobs = [] + for scheduled_job in scheduled_jobs: + suspend = scheduled_job.suspend or False + if not all and suspend: + continue + image_or_space = scheduled_job.job_spec.docker_image or "N/A" + cmd = scheduled_job.job_spec.command or [] + command_str = " ".join(cmd) if cmd else "N/A" + props = {"id": scheduled_job.id, "image": image_or_space, "suspend": str(suspend), "command": command_str} + if not _matches_filters(props, filters): + continue + filtered_jobs.append(scheduled_job) + + # Build display items. Augment with curated columns. + items: list[dict[str, Any]] = [] + for sj in filtered_jobs: + item = _dataclass_to_dict(sj) + job_spec = item.get("job_spec") or {} + status_dict = item.get("status") or {} + last_job = status_dict.get("last_job") + cmd = job_spec.get("command") or [] + item["image/space"] = job_spec.get("docker_image") or "N/A" + item["command"] = " ".join(cmd) if cmd else "N/A" + item["last_run"] = last_job["at"][:19].replace("T", " ") if last_job and last_job.get("at") else "N/A" + item["next_run"] = ( + status_dict["next_job_run_at"][:19].replace("T", " ") if status_dict.get("next_job_run_at") else "N/A" + ) + item["suspend"] = item.get("suspend") or False + items.append(item) + + out.table( + items, + headers=["id", "schedule", "image/space", "command", "last_run", "next_run", "suspend"], + id_key="id", + ) + if not items and filters: + filters_msg = ", ".join(f"{k}{o}{v}" for k, o, v in filters) + out.text(f"No scheduled jobs matched filters: {filters_msg}") + if items: + first_item_id = items[0]["id"] + out.hint(f"Use `hf jobs scheduled inspect {first_item_id}` to view details about a scheduled job.") + out.hint(f"Use `hf jobs scheduled trigger {first_item_id}` to trigger a scheduled job immediately.") + + +@scheduled_app.command("inspect", examples=["hf jobs scheduled inspect "]) +def scheduled_inspect( + scheduled_job_ids: Annotated[ + list[str], + Argument( + help="Scheduled Job IDs to inspect (or 'namespace/scheduled_job_id')", + ), + ], + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Display detailed information on one or more scheduled Jobs""" + parsed_ids = [] + for job_id in scheduled_job_ids: + job_id, namespace = _parse_namespace_from_job_id(job_id, namespace) + parsed_ids.append(job_id) + scheduled_job_ids = parsed_ids + api = get_hf_api(token=token) + scheduled_jobs = [ + api.inspect_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=namespace) + for scheduled_job_id in scheduled_job_ids + ] + out.table([_dataclass_to_dict(scheduled_job) for scheduled_job in scheduled_jobs]) + + +@scheduled_app.command("delete", examples=["hf jobs scheduled delete "]) +def scheduled_delete( + scheduled_job_id: ScheduledJobIdArg, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Delete a scheduled Job.""" + scheduled_job_id, namespace = _parse_namespace_from_job_id(scheduled_job_id, namespace) + api = get_hf_api(token=token) + api.delete_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=namespace) + out.result("Scheduled Job deleted", id=scheduled_job_id) + + +@scheduled_app.command("suspend", examples=["hf jobs scheduled suspend "]) +def scheduled_suspend( + scheduled_job_id: ScheduledJobIdArg, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Suspend (pause) a scheduled Job.""" + scheduled_job_id, namespace = _parse_namespace_from_job_id(scheduled_job_id, namespace) + api = get_hf_api(token=token) + api.suspend_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=namespace) + out.result("Scheduled Job suspended", id=scheduled_job_id) + out.hint(f"Use `hf jobs scheduled resume {scheduled_job_id}` to resume it.") + + +@scheduled_app.command("resume", examples=["hf jobs scheduled resume "]) +def scheduled_resume( + scheduled_job_id: ScheduledJobIdArg, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Resume (unpause) a scheduled Job.""" + scheduled_job_id, namespace = _parse_namespace_from_job_id(scheduled_job_id, namespace) + api = get_hf_api(token=token) + api.resume_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=namespace) + out.result("Scheduled Job resumed", id=scheduled_job_id) + + +@scheduled_app.command("trigger", examples=["hf jobs scheduled trigger "]) +def scheduled_trigger( + scheduled_job_id: ScheduledJobIdArg, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Trigger a scheduled Job to run immediately (does not change the schedule).""" + scheduled_job_id, namespace = _parse_namespace_from_job_id(scheduled_job_id, namespace) + api = get_hf_api(token=token) + job = api.trigger_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=namespace) + out.result("Scheduled Job triggered", id=job.id, url=job.url) + out.hint(f"Use `hf jobs logs -f {job.owner.name}/{job.id}` to stream logs.") + + +@scheduled_app.command( + "labels", + examples=[ + "hf jobs scheduled labels --name daily-script", + "hf jobs scheduled labels --label env=prod --label team=ml", + "hf jobs scheduled labels --clear", + ], +) +def scheduled_labels( + scheduled_job_id: ScheduledJobIdArg, + name: NameOpt = None, + label: LabelsOpt = None, + clear: Annotated[bool, Option("--clear", help="Remove all labels from the scheduled job.")] = False, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Update labels on a scheduled Job. Passing --label replaces all existing labels; passing --name alone keeps them.""" + if not label and name is None and not clear: + raise CLIError( + "Please set a name with --name or at least one label with --label. To remove all labels, pass --clear." + ) + if (label or name is not None) and clear: + raise CLIError( + "Cannot set a name or labels and clear them at the same time. Please use --name/--label or --clear, not both." + ) + scheduled_job_id, namespace = _parse_namespace_from_job_id(scheduled_job_id, namespace) + api = get_hf_api(token=token) + if name is not None and not label: + # Naming a scheduled Job should not wipe its existing labels: fetch them and merge the name in. + current_labels = ( + api.inspect_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=namespace).job_spec.labels or {} + ) + labels = {**current_labels, "name": name} + else: + labels = _parse_labels_map(label, name=name) or {} + scheduled_job = api.update_scheduled_job_labels( + scheduled_job_id=scheduled_job_id, labels=labels, namespace=namespace + ) + out.result("Labels updated", id=scheduled_job.id) + + +scheduled_uv_app = typer_factory(help="Schedule UV scripts on HF infrastructure.") +scheduled_app.add_group(scheduled_uv_app, name="uv") + + +@scheduled_uv_app.command( + "run", + context_settings={"ignore_unknown_options": True}, + examples=[ + 'hf jobs scheduled uv run "0 0 * * *" --name daily-script script.py', + 'hf jobs scheduled uv run "0 0 * * *" script.py --with pandas', + ], +) +def scheduled_uv_run( + schedule: ScheduleArg, + script: ScriptArg, + script_args: ScriptArgsArg = None, + suspend: SuspendOpt = None, + concurrency: ConcurrencyOpt = None, + image: ImageOpt = None, + flavor: FlavorOpt = None, + env: EnvOpt = None, + secrets: SecretsOpt = None, + name: NameOpt = None, + label: LabelsOpt = None, + volume: JobVolumesOpt = None, + env_file: EnvFileOpt = None, + secrets_file: SecretsFileOpt = None, + timeout: TimeoutOpt = None, + expose: ExposeOpt = None, + namespace: NamespaceOpt = None, + token: TokenOpt = None, + with_: WithOpt = None, + python: PythonOpt = None, +) -> None: + """Run a UV script (local file or URL) on HF infrastructure""" + env_map = parse_env_map(env, env_file) + secrets_map = parse_env_map(secrets, secrets_file) + + api = get_hf_api(token=token) + job = api.create_scheduled_uv_job( + script=script, + script_args=script_args or [], + schedule=schedule, + suspend=suspend, + concurrency=concurrency, + dependencies=with_, + python=python, + image=image, + env=env_map, + secrets=secrets_map, + labels=_parse_labels_map(label, name=name), + volumes=_parse_and_sync_job_volumes(volume, api=api, namespace=namespace), + flavor=flavor, + timeout=timeout, + expose=expose, + namespace=namespace, + ) + out.result("Scheduled Job created", id=job.id) + if not _has_explicit_name(name, label): + auto_name = (job.job_spec.labels or {}).get("name") + out.hint( + f"Scheduled Job auto-named '{auto_name}'. Pass `--name` or run " + f"`hf jobs scheduled labels {job.owner.name}/{job.id} --name NAME` to rename it." + ) + out.hint(f"Use `hf jobs scheduled inspect {job.owner.name}/{job.id}` to view its details.") + + +### UTILS + + +def _has_explicit_name(name: str | None, label: list[str] | None) -> bool: + """Whether the user explicitly named the Job (via `--name` or a `name=` label).""" + return name is not None or any(item.split("=", 1)[0] == "name" for item in label or []) + + +def _parse_labels_map(labels: list[str] | None, *, name: str | None = None) -> dict[str, str] | None: + """Parse label key-value pairs from CLI arguments. + + Args: + labels: List of label strings in KEY=VALUE format. If KEY only, then VALUE is set to empty string. + + Returns: + Dictionary mapping label keys to values, or None if no labels or name provided. + """ + if not labels and name is None: + return None + labels_map: dict[str, str] = {} + for label_var in labels or []: + key, value = label_var.split("=", 1) if "=" in label_var else (label_var, "") + labels_map[key] = value + if name is not None: + if "name" in labels_map: + raise CLIError("--name and --label name=... cannot both be provided.") + labels_map["name"] = name + return labels_map + + +def _tabulate(rows: list[list[str | int]], headers: list[str]) -> str: + """ + Inspired by: + + - stackoverflow.com/a/8356620/593036 + - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data + """ + col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] + terminal_width = max(shutil.get_terminal_size().columns, len(headers) * 12) + while len(headers) + sum(col_widths) > terminal_width: + col_to_minimize = col_widths.index(max(col_widths)) + col_widths[col_to_minimize] //= 2 + if len(headers) + sum(col_widths) <= terminal_width: + col_widths[col_to_minimize] = terminal_width - sum(col_widths) - len(headers) + col_widths[col_to_minimize] + row_format = ("{{:{}}} " * len(headers)).format(*col_widths) + lines = [] + lines.append(row_format.format(*headers)) + lines.append(row_format.format(*["-" * w for w in col_widths])) + for row in rows: + row_format_args = [ + str(x)[: col_width - 3] + "..." if len(str(x)) > col_width else str(x) + for x, col_width in zip(row, col_widths) + ] + lines.append(row_format.format(*row_format_args)) + return "\n".join(lines) + + +T = TypeVar("T") + + +def _write_generator_to_queue(queue: Queue[T], func: Callable[..., Iterable[T]], kwargs: dict) -> None: + for result in func(**kwargs): + queue.put(result) + + +def iflatmap_unordered( + pool: multiprocessing.pool.ThreadPool, + func: Callable[..., Iterable[T]], + *, + kwargs_list: list[dict], +) -> Iterable[T]: + """ + Takes a function that returns an iterable of items, and run it in parallel using threads to return the flattened iterable of items as they arrive. + + This is inspired by those three `map()` variants, and is the mix of all three: + + * `imap()`: like `map()` but returns an iterable instead of a list of results + * `imap_unordered()`: like `imap()` but the output is sorted by time of arrival + * `flatmap()`: like `map()` but given a function which returns a list, `flatmap()` returns the flattened list that is the concatenation of all the output lists + """ + queue: Queue[T] = Queue() + async_results = [pool.apply_async(_write_generator_to_queue, (queue, func, kwargs)) for kwargs in kwargs_list] + try: + while True: + try: + yield queue.get(timeout=0.05) + except Empty: + if all(async_result.ready() for async_result in async_results) and queue.empty(): + break + except KeyboardInterrupt: + pass + finally: + # we get the result in case there's an error to raise + try: + [async_result.get(timeout=0.05) for async_result in async_results] + except multiprocessing.TimeoutError: + pass diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/lfs.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/lfs.py new file mode 100644 index 0000000000000000000000000000000000000000..7b5e8b3ea68e3aabe023e9475e5c97c2fae28209 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/lfs.py @@ -0,0 +1,175 @@ +""" +Implementation of a custom transfer agent for the transfer type "multipart" for +git-lfs. + +Inspired by: +github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py + +Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md + + +To launch debugger while developing: + +``` [lfs "customtransfer.multipart"] +path = /path/to/huggingface_hub/.venv/bin/python args = -m debugpy --listen 5678 +--wait-for-client +/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py +lfs-multipart-upload ```""" + +import json +import os +import subprocess +import sys +from typing import Annotated + +from huggingface_hub.errors import CLIError +from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND + +from ..utils import get_session, hf_raise_for_status, logging +from ..utils._lfs import SliceFileObj +from ._framework import Argument +from ._output import out + + +logger = logging.get_logger(__name__) + + +def lfs_enable_largefiles( + path: Annotated[ + str, + Argument( + help="Local path to repository you want to configure.", + ), + ], +) -> None: + """ + Configure your repository to enable upload of files > 5GB. + + This command sets up git-lfs to use the custom multipart transfer agent + which enables efficient uploading of large files in chunks. + """ + local_path = os.path.abspath(path) + if not os.path.isdir(local_path): + raise CLIError("This does not look like a valid git repo.") + subprocess.run( + "git config lfs.customtransfer.multipart.path hf".split(), + check=True, + cwd=local_path, + ) + subprocess.run( + f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(), + check=True, + cwd=local_path, + ) + out.result("Local repo set up for largefiles", path=local_path) + + +def write_msg(msg: dict): + """Write out the message in Line delimited JSON.""" + msg_str = json.dumps(msg) + "\n" + sys.stdout.write(msg_str) + sys.stdout.flush() + + +def read_msg() -> dict | None: + """Read Line delimited JSON from stdin.""" + msg = json.loads(sys.stdin.readline().strip()) + + if "terminate" in (msg.get("type"), msg.get("event")): + # terminate message received + return None + + if msg.get("event") not in ("download", "upload"): + logger.critical("Received unexpected message") + sys.exit(1) + + return msg + + +def lfs_multipart_upload() -> None: + """Internal git-lfs custom transfer agent for multipart uploads. + + This function implements the custom transfer protocol for git-lfs multipart uploads. + Handles chunked uploads of large files to Hugging Face Hub. + """ + # Immediately after invoking a custom transfer process, git-lfs + # sends initiation data to the process over stdin. + # This tells the process useful information about the configuration. + init_msg = json.loads(sys.stdin.readline().strip()) + if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"): + write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}}) + sys.exit(1) + + # The transfer process should use the information it needs from the + # initiation structure, and also perform any one-off setup tasks it + # needs to do. It should then respond on stdout with a simple empty + # confirmation structure, as follows: + write_msg({}) + + # After the initiation exchange, git-lfs will send any number of + # transfer requests to the stdin of the transfer process, in a serial sequence. + while True: + msg = read_msg() + if msg is None: + # When all transfers have been processed, git-lfs will send + # a terminate event to the stdin of the transfer process. + # On receiving this message the transfer process should + # clean up and terminate. No response is expected. + sys.exit(0) + + oid = msg["oid"] + filepath = msg["path"] + completion_url = msg["action"]["href"] + header = msg["action"]["header"] + chunk_size = int(header.pop("chunk_size")) + presigned_urls: list[str] = list(header.values()) + + # Send a "started" progress event to allow other workers to start. + # Otherwise they're delayed until first "progress" event is reported, + # i.e. after the first 5GB by default (!) + write_msg( + { + "event": "progress", + "oid": oid, + "bytesSoFar": 1, + "bytesSinceLast": 0, + } + ) + + parts = [] + with open(filepath, "rb") as file: + for i, presigned_url in enumerate(presigned_urls): + with SliceFileObj( + file, + seek_from=i * chunk_size, + read_limit=chunk_size, + ) as data: + r = get_session().put(presigned_url, data=data) + hf_raise_for_status(r) + parts.append( + { + "etag": r.headers.get("etag"), + "partNumber": i + 1, + } + ) + # In order to support progress reporting while data is uploading / downloading, + # the transfer process should post messages to stdout + write_msg( + { + "event": "progress", + "oid": oid, + "bytesSoFar": (i + 1) * chunk_size, + "bytesSinceLast": chunk_size, + } + ) + + r = get_session().post( + completion_url, + json={ + "oid": oid, + "parts": parts, + }, + ) + hf_raise_for_status(r) + + write_msg({"event": "complete", "oid": oid}) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/models.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/models.py new file mode 100644 index 0000000000000000000000000000000000000000..7ba7e10fb43ac766469b1c11cf17cd0a735dabbd --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/models.py @@ -0,0 +1,265 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with models on the Hugging Face Hub.""" + +import enum +from typing import Annotated, get_args + +import click + +from huggingface_hub.errors import CLIError, RepositoryNotFoundError, RevisionNotFoundError +from huggingface_hub.hf_api import ExpandModelProperty_T, ModelSort_T +from huggingface_hub.inference._providers import PROVIDER_T +from huggingface_hub.repocard import ModelCard + +from ._cli_utils import ( + REPO_LIST_DEFAULT_LIMIT, + AuthorOpt, + FilterOpt, + LimitOpt, + RevisionOpt, + SearchOpt, + SoftChoice, + TokenOpt, + get_hf_api, + make_expand_properties_parser, + typer_factory, +) +from ._file_listing import list_repo_files_cmd +from ._framework import Argument, Option +from ._output import _dataclass_to_dict, out + + +_EXPAND_PROPERTIES = sorted(get_args(ExpandModelProperty_T)) +_SORT_OPTIONS = get_args(ModelSort_T) +ModelSortEnum = enum.Enum("ModelSortEnum", {s: s for s in _SORT_OPTIONS}, type=str) # type: ignore[misc] +InferenceProviderEnum = enum.Enum( # type: ignore[misc] + "InferenceProviderEnum", {p: p for p in sorted(get_args(PROVIDER_T))}, type=str +) + + +ExpandOpt = Annotated[ + str | None, + Option( + help=f"Comma-separated properties to return. When used, only the listed properties (and id) are returned. Example: '--expand=downloads,likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.", + callback=make_expand_properties_parser(_EXPAND_PROPERTIES), + ), +] + + +models_cli = typer_factory(help="Interact with models on the Hub.") + + +@models_cli.command( + "list | ls", + examples=[ + "hf models ls --sort downloads --limit 10", + 'hf models ls --search "llama" --author meta-llama', + "hf models ls --pipeline-tag text-generation --warm", + "hf models ls --num-parameters min:6B,max:128B --sort likes", + "hf models ls --no-gated --author google", + "hf models ls --apps llama.cpp --apps vllm", + "hf models ls --inference-provider fireworks-ai --sort downloads", + "hf models ls --warm --search llama", + "hf models ls meta-llama/Llama-3.2-1B-Instruct", + "hf models ls meta-llama/Llama-3.2-1B-Instruct -R", + "hf models ls meta-llama/Llama-3.2-1B-Instruct --tree -h", + ], +) +def models_ls( + repo_id: Annotated[ + str | None, + Argument(help="Model ID (e.g. `username/repo-name`) to list files from. If omitted, lists models."), + ] = None, + search: SearchOpt = None, + author: AuthorOpt = None, + filter: FilterOpt = None, + pipeline_tag: Annotated[ + str | None, + Option("--pipeline-tag", help="Filter by pipeline tag (canonical task), e.g. 'summarization'."), + ] = None, + gated: Annotated[ + bool | None, + Option( + "--gated/--no-gated", + help="Filter by gated status. '--gated' for gated only, '--no-gated' for non-gated only.", + ), + ] = None, + apps: Annotated[ + list[str] | None, + Option("--apps", help="Filter by app(s) that can run the model, e.g. 'ollama' or 'vllm'."), + ] = None, + num_parameters: Annotated[ + str | None, + Option(help="Filter by parameter count, e.g. 'min:6B,max:128B'."), + ] = None, + inference_provider: Annotated[ + list[str] | None, + Option( + "--inference-provider", + click_type=SoftChoice(InferenceProviderEnum), + help="Filter by inference provider(s) serving the model, e.g. 'fireworks-ai'.", + ), + ] = None, + warm: Annotated[ + bool, + Option("--warm", help="Only list models currently served by at least one inference provider."), + ] = False, + sort: Annotated[ + ModelSortEnum | None, + Option(help="Sort results."), + ] = None, + limit: LimitOpt = REPO_LIST_DEFAULT_LIMIT, + expand: ExpandOpt = None, + human_readable: Annotated[ + bool, + Option("--human-readable", "-h", help="Show sizes in human readable format (only for listing files)."), + ] = False, + as_tree: Annotated[ + bool, + Option("--tree", help="List files in tree format (only for listing files)."), + ] = False, + recursive: Annotated[ + bool, + Option("--recursive", "-R", help="List files recursively (only for listing files)."), + ] = False, + revision: RevisionOpt = None, + token: TokenOpt = None, +) -> None: + """List models on the Hub, or files in a model repo. + + When called with no argument, lists models on the Hub. + When called with a model ID, lists files in that model repo. + """ + if repo_id is not None: + if search is not None: + raise click.BadParameter("Cannot use --search when listing files.") + if author is not None: + raise click.BadParameter("Cannot use --author when listing files.") + if filter is not None: + raise click.BadParameter("Cannot use --filter when listing files.") + if pipeline_tag is not None: + raise click.BadParameter("Cannot use --pipeline-tag when listing files.") + if gated is not None: + raise click.BadParameter("Cannot use --gated/--no-gated when listing files.") + if apps is not None: + raise click.BadParameter("Cannot use --apps when listing files.") + if num_parameters is not None: + raise click.BadParameter("Cannot use --num-parameters when listing files.") + if inference_provider is not None: + raise click.BadParameter("Cannot use --inference-provider when listing files.") + if warm: + raise click.BadParameter("Cannot use --warm when listing files.") + if sort is not None: + raise click.BadParameter("Cannot use --sort when listing files.") + if limit != REPO_LIST_DEFAULT_LIMIT: + raise click.BadParameter("Cannot use --limit when listing files.") + if expand is not None: + raise click.BadParameter("Cannot use --expand when listing files.") + return list_repo_files_cmd( + repo_id=repo_id, + repo_type="model", + human_readable=human_readable, + as_tree=as_tree, + recursive=recursive, + revision=revision, + token=token, + ) + + if as_tree: + raise click.BadParameter("Cannot use --tree when listing models.") + if recursive: + raise click.BadParameter("Cannot use --recursive when listing models.") + if human_readable: + raise click.BadParameter("Cannot use --human-readable when listing models.") + if revision is not None: + raise click.BadParameter("Cannot use --revision when listing models.") + if warm and inference_provider is not None: + raise click.BadParameter("Cannot use --warm together with --inference-provider.") + api = get_hf_api(token=token) + sort_key = sort.value if sort else None + results = [ + _dataclass_to_dict(model_info) + for model_info in api.list_models( + filter=filter, + author=author, + search=search, + pipeline_tag=pipeline_tag, + gated=gated, + apps=apps, + num_parameters=num_parameters, + inference="warm" if warm else None, + inference_provider=inference_provider, + sort=sort_key, + limit=limit, + expand=expand, # type: ignore + ) + ] + out.table(results) + if (inference_provider is not None or warm) and not expand: + out.hint( + "Use `--expand inferenceProviderMapping` to see which provider serves each model and the provider-specific model id." + ) + + +@models_cli.command( + "info", + examples=[ + "hf models info meta-llama/Llama-3.2-1B-Instruct", + "hf models info Qwen/Qwen3.5-9B --expand downloads,likes,tags", + ], +) +def models_info( + model_id: Annotated[str, Argument(help="The model ID (e.g. `username/repo-name`).")], + revision: RevisionOpt = None, + expand: ExpandOpt = None, + token: TokenOpt = None, +) -> None: + """Get info about a model on the Hub.""" + api = get_hf_api(token=token) + try: + info = api.model_info(repo_id=model_id, revision=revision, expand=expand) # type: ignore + except RepositoryNotFoundError as e: + raise CLIError(f"Model '{model_id}' not found.") from e + except RevisionNotFoundError as e: + raise CLIError(f"Revision '{revision}' not found on '{model_id}'.") from e + out.dict(info) + + +@models_cli.command( + "card", + examples=[ + "hf models card google/gemma-4-31B-it", + "hf models card google/gemma-4-31B-it --metadata", + "hf models card google/gemma-4-31B-it --metadata --format json", + "hf models card google/gemma-4-31B-it --text", + ], +) +def models_card( + model_id: Annotated[str, Argument(help="The model ID (e.g. `username/repo-name`).")], + metadata: Annotated[bool, Option("--metadata", help="Output only the metadata from the card.")] = False, + text: Annotated[bool, Option("--text", help="Output only the text body (no metadata).")] = False, + token: TokenOpt = None, +) -> None: + """Get the model card (README) for a model on the Hub.""" + if metadata and text: + raise CLIError("--metadata and --text are mutually exclusive.") + card = ModelCard.load(model_id, token=token) + if metadata: + out.dict(card.data.to_dict()) + elif text: + out.text(card.text) + else: + out.text(card.content) + out.hint(f"Use `hf models card {model_id} --metadata` to extract only the card metadata.") diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/papers.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/papers.py new file mode 100644 index 0000000000000000000000000000000000000000..5f33db0d90c9febdaea8e26186c13aadc6ea3708 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/papers.py @@ -0,0 +1,167 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with papers on the Hugging Face Hub.""" + +import datetime +import enum +from typing import Annotated, get_args + +from huggingface_hub.errors import CLIError, HfHubHTTPError +from huggingface_hub.hf_api import DailyPapersSort_T + +from ._cli_utils import ( + LimitOpt, + TokenOpt, + get_hf_api, + typer_factory, +) +from ._framework import Argument, Option +from ._output import _dataclass_to_dict, out + + +_SORT_OPTIONS = get_args(DailyPapersSort_T) +PaperSortEnum = enum.Enum("PaperSortEnum", {s: s for s in _SORT_OPTIONS}, type=str) # type: ignore[misc] + + +def _parse_date(value: str | None) -> str | None: + """Parse date option, converting 'today' to current date.""" + if value is None: + return None + if value.lower() == "today": + return datetime.date.today().isoformat() + return value + + +papers_cli = typer_factory(help="Interact with papers on the Hub.") + + +@papers_cli.command( + "list | ls", + examples=[ + "hf papers ls", + "hf papers ls --sort trending", + "hf papers ls --date 2025-01-23", + "hf papers ls --week 2025-W09", + "hf papers ls --submitter akhaliq", + "hf papers ls --format json", + ], +) +def papers_ls( + date: Annotated[ + str | None, + Option( + help="Date in ISO format (YYYY-MM-DD) or 'today'.", + callback=_parse_date, + ), + ] = None, + week: Annotated[ + str | None, + Option(help="ISO week to filter by, e.g. '2025-W09'."), + ] = None, + month: Annotated[ + str | None, + Option(help="Month to filter by in ISO format (YYYY-MM), e.g. '2025-02'."), + ] = None, + submitter: Annotated[ + str | None, + Option(help="Filter by username of the submitter."), + ] = None, + sort: Annotated[ + PaperSortEnum | None, + Option(help="Sort results."), + ] = None, + limit: LimitOpt = 50, + token: TokenOpt = None, +) -> None: + """List daily papers on the Hub.""" + api = get_hf_api(token=token) + sort_key = sort.value if sort else None + results = [] + for paper_info in api.list_daily_papers( + date=date, + week=week, + month=month, + submitter=submitter, + sort=sort_key, + limit=limit, + ): + item = _dataclass_to_dict(paper_info) + submitted_by = item.get("submitted_by") or {} + item["submitted_by_name"] = submitted_by.get("fullname") or submitted_by.get("username") or "" + results.append(item) + out.table( + results, + headers=["id", "title", "upvotes", "comments", "published_at", "submitted_by_name"], + ) + + +@papers_cli.command( + "search", + examples=[ + 'hf papers search "vision language"', + 'hf papers search "attention mechanism" --limit 10', + 'hf papers search "diffusion" --format json', + ], +) +def papers_search( + query: Annotated[str, Argument(help="Search query string.")], + limit: LimitOpt = 20, + token: TokenOpt = None, +) -> None: + """Search papers on the Hub.""" + api = get_hf_api(token=token) + results = [_dataclass_to_dict(paper_info) for paper_info in api.list_papers(query=query, limit=limit)] + out.table(results, headers=["id", "title", "summary", "upvotes", "published_at"]) + + +@papers_cli.command( + "info", + examples=[ + "hf papers info 2601.15621", + ], +) +def papers_info( + paper_id: Annotated[str, Argument(help="The arXiv paper ID (e.g. '2502.08025').")], + token: TokenOpt = None, +) -> None: + """Get info about a paper on the Hub.""" + api = get_hf_api(token=token) + try: + info = api.paper_info(id=paper_id) + except HfHubHTTPError as e: + if e.response.status_code == 404: + raise CLIError(f"Paper '{paper_id}' not found on the Hub.") from e + raise + out.dict(info) + + +@papers_cli.command( + "read", + examples=[ + "hf papers read 2601.15621", + ], +) +def papers_read( + paper_id: Annotated[str, Argument(help="The arXiv paper ID (e.g. '2502.08025').")], + token: TokenOpt = None, +) -> None: + """Read a paper as markdown.""" + api = get_hf_api(token=token) + try: + content = api.read_paper(id=paper_id) + except HfHubHTTPError as e: + if e.response.status_code == 404: + raise CLIError(f"Paper '{paper_id}' not found on the Hub.") from e + raise + out.text(content) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/repo_files.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/repo_files.py new file mode 100644 index 0000000000000000000000000000000000000000..57005845c026a742f67f4985f1a021b02649dcb3 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/repo_files.py @@ -0,0 +1,83 @@ +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Legacy `hf repo-files` command. + +Kept for backward compatibility. Users are nudged to use `hf repos delete-files` instead. +""" + +from typing import Annotated + +from ._cli_utils import ( + RepoIdArg, + RepoType, + RepoTypeOpt, + RevisionOpt, + TokenOpt, + get_hf_api, + typer_factory, +) +from ._framework import Argument, Option +from ._output import out + + +repo_files_cli = typer_factory( + help="(Deprecated) Manage files in a repo on the Hub. Use `hf repos delete-files` instead." +) + + +@repo_files_cli.command( + "delete", +) +def repo_files_delete( + repo_id: RepoIdArg, + patterns: Annotated[ + list[str], + Argument( + help="Glob patterns to match files to delete. Based on fnmatch, '*' matches files recursively.", + ), + ], + repo_type: RepoTypeOpt = RepoType.model, + revision: RevisionOpt = None, + commit_message: Annotated[ + str | None, + Option( + help="The summary / title / first line of the generated commit.", + ), + ] = None, + commit_description: Annotated[ + str | None, + Option( + help="The description of the generated commit.", + ), + ] = None, + create_pr: Annotated[ + bool, + Option( + help="Whether to create a new Pull Request for these changes.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + out.warning("`hf repo-files delete` is deprecated. Use `hf repos delete-files` instead.") + api = get_hf_api(token=token) + url = api.delete_files( + delete_patterns=patterns, + repo_id=repo_id, + repo_type=repo_type.value, + revision=revision, + commit_message=commit_message, + commit_description=commit_description, + create_pr=create_pr, + ) + out.result("Files deleted", repo_id=repo_id, commit_url=url) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/repos.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/repos.py new file mode 100644 index 0000000000000000000000000000000000000000..193e3ada122fa08594e6a821cd5f0c04f44e6c9c --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/repos.py @@ -0,0 +1,633 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with repositories on the Hugging Face Hub.""" + +import enum +from typing import Annotated + +import click + +from huggingface_hub import SpaceHardware, SpaceStorage +from huggingface_hub.cli._cli_utils import SoftChoice +from huggingface_hub.errors import CLIError, HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError +from huggingface_hub.hf_api import REPO_REGIONS + +from ._city_game import run_city_game +from ._cli_utils import ( + REPO_LIST_DEFAULT_LIMIT, + EnvFileOpt, + EnvOpt, + LimitOpt, + PrivateOpt, + RepoIdArg, + RepoType, + RepoTypeOpt, + RevisionOpt, + SearchOpt, + SecretsFileOpt, + SecretsOpt, + TokenOpt, + VolumesOpt, + env_map_to_key_value_list, + get_hf_api, + parse_env_map, + parse_volumes, + typer_factory, +) +from ._cp import make_cp +from ._file_listing import format_size +from ._framework import Argument, Option +from ._output import OutputFormat, out + + +repos_cli = typer_factory(help="Manage repos on the Hub.") + + +@repos_cli.group_callback(invoke_without_command=True) +def _repos_callback(ctx: click.Context) -> None: + if ctx.info_name == "repo": + out.warning("`hf repo` is deprecated in favor of `hf repos`.") + + +class RepoTypeAll(str, enum.Enum): + model = "model" + dataset = "dataset" + space = "space" + bucket = "bucket" + + +class GatedChoices(str, enum.Enum): + auto = "auto" + manual = "manual" + false = "false" + + +PublicOpt = Annotated[ + bool | None, + Option( + "--public", + help="Whether to make the repo public. Ignored if the repo already exists.", + ), +] + +ProtectedOpt = Annotated[ + bool | None, + Option( + "--protected", + help="Whether to make the Space protected (Spaces only). Ignored if the repo already exists.", + ), +] +SpaceHardwareOpt = Annotated[ + str | None, + Option( + "--flavor", + help="Space hardware flavor (e.g. 'cpu-basic', 't4-medium', 'l4x4'). Only for Spaces.", + click_type=SoftChoice(SpaceHardware), + ), +] + +SpaceStorageOpt = Annotated[ + SpaceStorage | None, + Option( + "--storage", + help="(Deprecated, use volumes instead) Space persistent storage tier ('small', 'medium', or 'large'). Only for Spaces.", + ), +] + +SpaceSleepTimeOpt = Annotated[ + int | None, + Option( + "--sleep-time", + help="Seconds of inactivity before the Space is put to sleep. Use -1 to disable. Only for Spaces.", + ), +] + + +tag_cli = typer_factory(help="Manage tags for a repo on the Hub.") +branch_cli = typer_factory(help="Manage branches for a repo on the Hub.") +repos_cli.add_group(tag_cli, name="tag") +repos_cli.add_group(branch_cli, name="branch") + + +@repos_cli.command( + "list | ls", + examples=[ + "hf repos ls", + "hf repos ls --explore", + "hf repos ls --namespace my-org --search bert", + ], +) +def repo_list( + namespace: Annotated[ + str | None, + Option( + help="Organization name. If not provided, lists repos for the authenticated user.", + ), + ] = None, + repo_type: Annotated[ + RepoTypeAll | None, + Option( + "--type", + "--repo-type", + help="Filter by repository type (model, dataset, space, or bucket).", + ), + ] = None, + search: SearchOpt = None, + limit: LimitOpt = REPO_LIST_DEFAULT_LIMIT, + explore: Annotated[ + bool, + Option("--explore", help="Explore your repos as an interactive 3D city."), + ] = False, + token: TokenOpt = None, +) -> None: + """List all repos (models, datasets, spaces, buckets) with storage info.""" + api = get_hf_api(token=token) + repos = list(api.list_user_repos(namespace=namespace)) + if repo_type is not None: + repos = [r for r in repos if r.type == repo_type.value] + if search is not None: + search_lower = search.lower() + repos = [r for r in repos if search_lower in r.id.lower()] + total = len(repos) + + if explore: + if out.mode == OutputFormat.human: + run_city_game(repos) + return + raise CLIError("Repository exploration is only available in terminal.") + + if limit > 0: + repos = repos[:limit] + items = [ + { + "id": r.id, + "type": r.type, + "updated": r.updated_at.strftime("%Y-%m-%d"), + "visibility": r.visibility, + "storage": format_size(r.storage, human_readable=True), + "%_of_total": f"{r.storage_percent:.1f}%", + } + for r in repos + ] + out.table(items, id_key="id", alignments={"storage": "right", "%_of_total": "right"}) + if limit > 0 and total > limit: + out.hint(f"Showing {limit} of {total} repos. Use `--limit 0` to list all.") + + +@repos_cli.command( + "create", + examples=[ + "hf repos create my-model", + "hf repos create my-dataset --repo-type dataset --private", + "hf repos create my-space --type space --sdk gradio --flavor t4-medium --secrets HF_TOKEN -e THEME=dark --protected", + "hf repos create my-jupyterlab --type space --template SpacesExamples/jupyterlab", + "hf repos create my-space --type space --sdk gradio -v hf://org/my-model:/models -v hf://buckets/org/b:/data", + "hf repos create my-model --region us", + ], +) +def repo_create( + repo_id: RepoIdArg, + repo_type: RepoTypeOpt = RepoType.model, + sdk: Annotated[ + str | None, + Option( + "--sdk", + "--space-sdk", + help="Hugging Face Spaces SDK type. Required when --type is set to 'space'.", + ), + ] = None, + template: Annotated[ + str | None, + Option( + "--template", + help=( + "Create a Space from an official template. Pass a template repo id (e.g. " + "'SpacesExamples/jupyterlab') or its short name (e.g. 'JupyterLab'). List available templates with " + "`hf spaces templates`. Spaces only." + ), + ), + ] = None, + private: PrivateOpt = None, + public: PublicOpt = None, + protected: ProtectedOpt = None, + token: TokenOpt = None, + exist_ok: Annotated[ + bool, + Option( + help="Do not raise an error if repo already exists.", + ), + ] = False, + resource_group_id: Annotated[ + str | None, + Option( + help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.", + ), + ] = None, + region: Annotated[ + REPO_REGIONS | None, + Option( + "--region", + help="Cloud region in which to create the repo. Can be one of 'us' or 'eu'. Requires Team plan or above.", + ), + ] = None, + hardware: SpaceHardwareOpt = None, + storage: SpaceStorageOpt = None, + sleep_time: SpaceSleepTimeOpt = None, + secrets: SecretsOpt = None, + secrets_file: SecretsFileOpt = None, + env: EnvOpt = None, + env_file: EnvFileOpt = None, + volume: VolumesOpt = None, +) -> None: + """Create a new repo on the Hub.""" + api = get_hf_api(token=token) + repo_url = api.create_repo( + repo_id=repo_id, + repo_type=repo_type.value, + visibility="private" if private else "public" if public else "protected" if protected else None, # type: ignore [arg-type] + token=token, + exist_ok=exist_ok, + resource_group_id=resource_group_id, + region=region, + space_sdk=sdk, + space_hardware=hardware, + space_storage=storage, + space_sleep_time=sleep_time, + space_secrets=env_map_to_key_value_list(parse_env_map(secrets, secrets_file)), + space_variables=env_map_to_key_value_list(parse_env_map(env, env_file)), + space_volumes=parse_volumes(volume), + space_template=template, + ) + out.result("Repo created", repo_id=repo_url.repo_id, url=str(repo_url)) + + +@repos_cli.command( + "duplicate", + examples=[ + "hf repos duplicate openai/gdpval --type dataset", + "hf repos duplicate multimodalart/dreambooth-training my-dreambooth --type space --flavor l4x4 --secrets HF_TOKEN --private", + "hf repos duplicate org/my-space my-space --type space -v hf://org/my-model:/models -v hf://buckets/org/b:/data", + ], +) +def repo_duplicate( + from_id: RepoIdArg, + to_id: Annotated[ + str | None, + Argument( + help="Destination repo ID (e.g. `myorg/my-copy`). Defaults to your namespace with the same repo name.", + ), + ] = None, + repo_type: RepoTypeOpt = RepoType.model, + private: PrivateOpt = None, + public: PublicOpt = None, + protected: ProtectedOpt = None, + token: TokenOpt = None, + exist_ok: Annotated[ + bool, + Option( + help="Do not raise an error if repo already exists.", + ), + ] = False, + hardware: SpaceHardwareOpt = None, + storage: SpaceStorageOpt = None, + sleep_time: SpaceSleepTimeOpt = None, + secrets: SecretsOpt = None, + secrets_file: SecretsFileOpt = None, + env: EnvOpt = None, + env_file: EnvFileOpt = None, + volume: VolumesOpt = None, +) -> None: + """Duplicate a repo on the Hub (model, dataset, or Space).""" + api = get_hf_api(token=token) + repo_url = api.duplicate_repo( + from_id=from_id, + to_id=to_id, + repo_type=repo_type.value, + visibility="private" if private else "public" if public else "protected" if protected else None, # type: ignore [arg-type] + token=token, + exist_ok=exist_ok, + space_hardware=hardware, + space_storage=storage, + space_sleep_time=sleep_time, + space_secrets=env_map_to_key_value_list(parse_env_map(secrets, secrets_file)), + space_variables=env_map_to_key_value_list(parse_env_map(env, env_file)), + space_volumes=parse_volumes(volume), + ) + out.result("Repo duplicated", from_id=from_id, to_id=repo_url.repo_id, url=str(repo_url)) + + +@repos_cli.command("delete", examples=["hf repos delete my-model"]) +def repo_delete( + repo_id: RepoIdArg, + repo_type: RepoTypeOpt = RepoType.model, + token: TokenOpt = None, + missing_ok: Annotated[ + bool, + Option( + help="If set to True, do not raise an error if repo does not exist.", + ), + ] = False, + yes: Annotated[ + bool, + Option( + "-y", + "--yes", + help="Answer Yes to prompt automatically.", + ), + ] = False, +) -> None: + """Delete a repo from the Hub. This is an irreversible operation.""" + out.confirm(f"You are about to permanently delete {repo_type.value} '{repo_id}'. Proceed?", yes=yes) + api = get_hf_api(token=token) + api.delete_repo( + repo_id=repo_id, + repo_type=repo_type.value, + missing_ok=missing_ok, + ) + out.result("Repo deleted", repo_id=repo_id) + + +@repos_cli.command("move", examples=["hf repos move old-namespace/my-model new-namespace/my-model"]) +def repo_move( + from_id: RepoIdArg, + to_id: RepoIdArg, + token: TokenOpt = None, + repo_type: RepoTypeOpt = RepoType.model, +) -> None: + """Move a repository from a namespace to another namespace.""" + api = get_hf_api(token=token) + api.move_repo( + from_id=from_id, + to_id=to_id, + repo_type=repo_type.value, + ) + out.result("Repo moved", from_id=from_id, to_id=to_id) + + +@repos_cli.command( + "settings", + examples=[ + "hf repos settings my-model --private", + "hf repos settings my-model --gated auto", + "hf repos settings my-space --repo-type space --protected", + ], +) +def repo_settings( + repo_id: RepoIdArg, + gated: Annotated[ + GatedChoices | None, + Option( + help="The gated status for the repository.", + ), + ] = None, + private: PrivateOpt = None, + public: PublicOpt = None, + protected: ProtectedOpt = None, + token: TokenOpt = None, + repo_type: RepoTypeOpt = RepoType.model, +) -> None: + """Update the settings of a repository.""" + api = get_hf_api(token=token) + api.update_repo_settings( + repo_id=repo_id, + gated=(None if gated is None else False if gated is GatedChoices.false else gated.value), + visibility="private" if private else "public" if public else "protected" if protected else None, # type: ignore [arg-type] + repo_type=repo_type.value, + ) + out.result("Repo settings updated", repo_id=repo_id) + + +@repos_cli.command( + "delete-files", + examples=[ + "hf repos delete-files my-model file.txt", + 'hf repos delete-files my-model "*.json"', + "hf repos delete-files my-model folder/", + ], +) +def repo_delete_files( + repo_id: RepoIdArg, + patterns: Annotated[ + list[str], + Argument( + help="Glob patterns to match files to delete. Based on fnmatch, '*' matches files recursively.", + ), + ], + repo_type: RepoTypeOpt = RepoType.model, + revision: RevisionOpt = None, + commit_message: Annotated[ + str | None, + Option( + help="The summary / title / first line of the generated commit.", + ), + ] = None, + commit_description: Annotated[ + str | None, + Option( + help="The description of the generated commit.", + ), + ] = None, + create_pr: Annotated[ + bool, + Option( + help="Whether to create a new Pull Request for these changes.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + """Delete files from a repo on the Hub.""" + api = get_hf_api(token=token) + url = api.delete_files( + delete_patterns=patterns, + repo_id=repo_id, + repo_type=repo_type.value, + revision=revision, + commit_message=commit_message, + commit_description=commit_description, + create_pr=create_pr, + ) + out.result("Files deleted", repo_id=repo_id, commit_url=url) + + +# `hf repos cp` is an alias for the top-level `hf cp` command (see `cli/_cp.py`). +repos_cli.command( + name="cp", + examples=[ + # Download (repo or bucket -> local / stdout) + "hf repos cp hf://username/my-model/config.json config.json", + "hf repos cp hf://datasets/username/my-dataset/data.csv data/", + "hf repos cp hf://username/my-model/config.json -", + # Upload (local / stdin -> repo) + "hf repos cp model.safetensors hf://username/my-model/model.safetensors", + "hf repos cp config.json hf://username/my-model/logs/", + "hf repos cp - hf://username/my-model/config.json", + # Remote to remote (repo -> repo) + "hf repos cp hf://username/source-model/config.json hf://username/dest-model/config.json", + "hf repos cp hf://datasets/username/my-dataset/processed/ hf://datasets/username/dest-dataset/processed/", + "hf repos cp hf://username/my-model/logs/ hf://username/archive-model/logs/", + ], +)(make_cp("repos")) + + +@branch_cli.command( + "create", + examples=[ + "hf repos branch create my-model dev", + "hf repos branch create my-model dev --revision abc123", + ], +) +def branch_create( + repo_id: RepoIdArg, + branch: Annotated[ + str, + Argument( + help="The name of the branch to create.", + ), + ], + revision: RevisionOpt = None, + token: TokenOpt = None, + repo_type: RepoTypeOpt = RepoType.model, + exist_ok: Annotated[ + bool, + Option( + help="If set to True, do not raise an error if branch already exists.", + ), + ] = False, +) -> None: + """Create a new branch for a repo on the Hub.""" + api = get_hf_api(token=token) + api.create_branch( + repo_id=repo_id, + branch=branch, + revision=revision, + repo_type=repo_type.value, + exist_ok=exist_ok, + ) + out.result("Branch created", branch=branch, repo_type=repo_type.value, repo_id=repo_id) + + +@branch_cli.command("delete", examples=["hf repos branch delete my-model dev"]) +def branch_delete( + repo_id: RepoIdArg, + branch: Annotated[ + str, + Argument( + help="The name of the branch to delete.", + ), + ], + token: TokenOpt = None, + repo_type: RepoTypeOpt = RepoType.model, +) -> None: + """Delete a branch from a repo on the Hub.""" + api = get_hf_api(token=token) + api.delete_branch( + repo_id=repo_id, + branch=branch, + repo_type=repo_type.value, + ) + out.result("Branch deleted", branch=branch, repo_type=repo_type.value, repo_id=repo_id) + + +@tag_cli.command( + "create", + examples=[ + "hf repos tag create my-model v1.0", + 'hf repos tag create my-model v1.0 -m "First release"', + ], +) +def tag_create( + repo_id: RepoIdArg, + tag: Annotated[ + str, + Argument( + help="The name of the tag to create.", + ), + ], + message: Annotated[ + str | None, + Option( + "-m", + "--message", + help="The description of the tag to create.", + ), + ] = None, + revision: RevisionOpt = None, + token: TokenOpt = None, + repo_type: RepoTypeOpt = RepoType.model, +) -> None: + """Create a tag for a repo.""" + repo_type_str = repo_type.value + api = get_hf_api(token=token) + try: + api.create_tag(repo_id=repo_id, tag=tag, tag_message=message, revision=revision, repo_type=repo_type_str) + except RepositoryNotFoundError as e: + raise CLIError(f"{repo_type_str.capitalize()} '{repo_id}' not found.") from e + except RevisionNotFoundError as e: + raise CLIError(f"Revision '{revision}' not found.") from e + except HfHubHTTPError as e: + if e.response.status_code == 409: + raise CLIError(f"Tag '{tag}' already exists on '{repo_id}'.") from e + raise + out.result("Tag created", tag=tag, repo_type=repo_type_str, repo_id=repo_id) + + +@tag_cli.command("list | ls", examples=["hf repos tag list my-model"]) +def tag_list( + repo_id: RepoIdArg, + token: TokenOpt = None, + repo_type: RepoTypeOpt = RepoType.model, +) -> None: + """List tags for a repo.""" + repo_type_str = repo_type.value + api = get_hf_api(token=token) + try: + refs = api.list_repo_refs(repo_id=repo_id, repo_type=repo_type_str) + except RepositoryNotFoundError as e: + raise CLIError(f"{repo_type_str.capitalize()} '{repo_id}' not found.") from e + items = [{"name": t.name, "target_commit": t.target_commit, "ref": t.ref} for t in refs.tags] + out.table(items) + + +@tag_cli.command("delete", examples=["hf repos tag delete my-model v1.0"]) +def tag_delete( + repo_id: RepoIdArg, + tag: Annotated[ + str, + Argument( + help="The name of the tag to delete.", + ), + ], + yes: Annotated[ + bool, + Option( + "-y", + "--yes", + help="Answer Yes to prompt automatically", + ), + ] = False, + token: TokenOpt = None, + repo_type: RepoTypeOpt = RepoType.model, +) -> None: + """Delete a tag for a repo.""" + repo_type_str = repo_type.value + out.text(f"You are about to delete tag {tag} on {repo_type_str} {repo_id}") + out.confirm("Proceed?", yes=yes) + api = get_hf_api(token=token) + try: + api.delete_tag(repo_id=repo_id, tag=tag, repo_type=repo_type_str) + except RepositoryNotFoundError as e: + raise CLIError(f"{repo_type_str.capitalize()} '{repo_id}' not found.") from e + except RevisionNotFoundError as e: + raise CLIError(f"Tag '{tag}' not found on '{repo_id}'.") from e + out.result("Tag deleted", tag=tag, repo_type=repo_type_str, repo_id=repo_id) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/sandbox.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/sandbox.py new file mode 100644 index 0000000000000000000000000000000000000000..e147bf22820b85c8a9d7367ed0212fb065159ef8 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/sandbox.py @@ -0,0 +1,480 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to run and manage sandboxes on Hugging Face Jobs.""" + +import sys +import time +from contextlib import contextmanager +from typing import Annotated, Any, Iterator + +import click + +from huggingface_hub._sandbox import ( + DEFAULT_IDLE_TIMEOUT, + DEFAULT_IMAGE, + DEFAULT_SANDBOXES_PER_HOST, + MODE_LABEL, + MODE_POOL, + POOL_LABEL, + SANDBOX_LABEL, + SHARED_ID_SEP, + Sandbox, + SandboxPool, + SandboxProcess, + _split_sandbox_id, +) +from huggingface_hub._sandbox_cache import delete_pool_cache +from huggingface_hub.errors import CLIError, SandboxError + +from ._cli_utils import ( + EnvFileOpt, + EnvOpt, + SecretsFileOpt, + SecretsOpt, + TokenOpt, + VolumesOpt, + get_hf_api, + parse_env_map, + parse_volumes, + typer_factory, +) +from ._framework import Argument, Option +from ._output import out +from .jobs import FlavorOpt, NamespaceOpt + + +sandbox_cli = typer_factory(help="Run and manage sandboxes on Hugging Face Jobs.") +pool_cli = typer_factory(help="Warm pools of host VMs and spawn cheap shared sandboxes from them.") +sandbox_cli.add_group(pool_cli, name="pool") +process_cli = typer_factory(help="List and stop background processes running in a sandbox.") +sandbox_cli.add_group(process_cli, name="process") + +SandboxIdArg = Annotated[str, Argument(help="The sandbox id as printed by `hf sandbox create`.")] + + +@contextmanager +def _connect(sandbox_id: str, *, namespace: str | None, token: str | None) -> Iterator[Sandbox]: + """Reattach to a sandbox and close the HTTP client when the command is done.""" + sandbox = Sandbox.connect(sandbox_id, namespace=namespace, token=token) + try: + yield sandbox + finally: + sandbox.close() + + +@sandbox_cli.command( + "create", + examples=[ + "hf sandbox create", + "hf sandbox create ubuntu:24.04", + "hf sandbox create --flavor a10g-small", + "hf sandbox create --pool pool-ab12cd34ef56 --env LOG_LEVEL=debug", + ], +) +def sandbox_create( + image: Annotated[str | None, Argument(help="Docker image (needs /bin/sh).")] = None, + pool: Annotated[ + str | None, + Option("--pool", help="Spawn a cheap shared sandbox in this pool (from `hf sandbox pool create`)."), + ] = None, + flavor: FlavorOpt = None, + idle_timeout: Annotated[ + str | None, + Option(help="Auto-terminate the sandbox after this much inactivity (e.g. '10m'). Defaults to 10m."), + ] = None, + env: EnvOpt = None, + secrets: SecretsOpt = None, + env_file: EnvFileOpt = None, + secrets_file: SecretsFileOpt = None, + volume: VolumesOpt = None, + namespace: NamespaceOpt = None, + forward_hf_token: Annotated[ + bool, Option("--forward-hf-token", help="Inject your HF token as HF_TOKEN in the sandbox.") + ] = False, + token: TokenOpt = None, +) -> None: + """Create a sandbox: a dedicated VM by default, or a cheap shared one with `--pool`. + + Env and idle-timeout apply to the sandbox in both modes. With `--pool`, the image and + flavor come from the pool, so passing them here is an error; `--secrets` is also + rejected since pooled sandboxes have no encrypted-secrets channel (use `--env`). Define + a pool first with `hf sandbox pool create`. + """ + start = time.time() + idle = idle_timeout if idle_timeout is not None else DEFAULT_IDLE_TIMEOUT + + if pool is not None: + if image is not None or flavor is not None or volume: + raise CLIError("--pool fixes the image/flavor (and volumes aren't supported); drop those options.") + if secrets or secrets_file: + raise CLIError("--pool can't encrypt secrets; pass them with --env/--env-file instead.") + sbx = SandboxPool.connect(pool, namespace=namespace, token=token).create( + env=parse_env_map(env, env_file), + idle_timeout=idle, + forward_hf_token=forward_hf_token, + ) + out.result("Sandbox ready", id=sbx.id, host=sbx.host_id, pool=pool, elapsed=f"{time.time() - start:.1f}s") + out.hint(f"Run a command with `hf sandbox exec {sbx.id} -- echo hello`.") + out.hint(f"Terminate it with `hf sandbox kill {sbx.id}`.") + return + + sandbox = Sandbox.create( + image=image or DEFAULT_IMAGE, + flavor=flavor or "cpu-basic", + idle_timeout=idle, + env=parse_env_map(env, env_file), + secrets=parse_env_map(secrets, secrets_file), + volumes=parse_volumes(volume), + namespace=namespace, + forward_hf_token=forward_hf_token, + token=token, + ) + # Release the HTTP client (the sandbox keeps running) + sandbox.close() + out.result("Sandbox ready", id=sandbox.id, image=sandbox.image, elapsed=f"{time.time() - start:.1f}s") + out.hint(f"Run a command with `hf sandbox exec {sandbox.id} -- echo hello`.") + out.hint(f"Terminate it with `hf sandbox kill {sandbox.id}`.") + + +@sandbox_cli.command( + "exec", + context_settings={"ignore_unknown_options": True}, + examples=[ + 'hf sandbox exec -- python -c "print(42)"', + "hf sandbox exec -w /app -- pytest -x", + ], +) +def sandbox_exec( + sandbox_id: SandboxIdArg, + command: Annotated[list[str], Argument(help="The command to run.")], + workdir: Annotated[str | None, Option("-w", "--workdir", help="Working directory.")] = None, + env: EnvOpt = None, + env_file: EnvFileOpt = None, + exec_timeout: Annotated[ + float | None, Option("--timeout", help="Kill the command after this many seconds.") + ] = None, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Run a command in a sandbox, streaming output. Exits with the command's exit code. + + To start a long-running command in the background instead of waiting for it, use + `hf sandbox spawn`. + """ + + def write_stdout(data: str) -> None: + sys.stdout.write(data) + sys.stdout.flush() + + def write_stderr(data: str) -> None: + sys.stderr.write(data) + sys.stderr.flush() + + with _connect(sandbox_id, namespace=namespace, token=token) as sandbox: + result = sandbox.run( + list(command), + env=parse_env_map(env, env_file), + cwd=workdir, + timeout=exec_timeout, + on_stdout=write_stdout, + on_stderr=write_stderr, + check=False, + ) + if result.timed_out: + out.error(f"Command timed out after {exec_timeout}s.") + raise click.exceptions.Exit(code=result.exit_code or 124) # 124: conventional timeout exit code + if result.exit_code != 0: + raise click.exceptions.Exit(code=result.exit_code if result.exit_code is not None else 1) + + +@sandbox_cli.command( + "spawn", + context_settings={"ignore_unknown_options": True}, + examples=[ + "hf sandbox spawn -- python -m http.server 8000", + "hf sandbox spawn -w /app -- uvicorn app:app", + ], +) +def sandbox_spawn( + sandbox_id: SandboxIdArg, + command: Annotated[list[str], Argument(help="The command to run in the background.")], + workdir: Annotated[str | None, Option("-w", "--workdir", help="Working directory.")] = None, + env: EnvOpt = None, + env_file: EnvFileOpt = None, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Start a long-running command in the background and return its pid (don't wait). + + List a sandbox's processes with `hf sandbox process ls` and stop one with + `hf sandbox process kill`. + """ + with _connect(sandbox_id, namespace=namespace, token=token) as sandbox: + process = sandbox.run(list(command), env=parse_env_map(env, env_file), cwd=workdir, background=True) + out.result("Process started", sandbox=sandbox_id, pid=process.pid) + out.hint(f"List processes with `hf sandbox process ls {sandbox_id}`.") + out.hint(f"Stop it with `hf sandbox process kill {sandbox_id} {process.pid}`.") + + +@sandbox_cli.command( + "cp", + examples=[ + "hf sandbox cp data.csv :/data/data.csv", + "hf sandbox cp :/app/result.json result.json", + ], +) +def sandbox_cp( + src: Annotated[str, Argument(help="Source: a local path or :.")], + dst: Annotated[str, Argument(help="Destination: a local path or :.")], + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Copy a file between the local machine and a sandbox (docker-style).""" + + def parse(ref: str) -> tuple[str | None, str]: + # Only treat as a sandbox ref when the part before ':' looks like a sandbox id + # (more than one char): this leaves local paths and Windows drive letters like + # 'C:\data\file.csv' or 'C:/data/file.csv' (single-letter prefix) untouched. + if ":" in ref and not ref.startswith((".", "/", "~")): + sandbox_id, path = ref.split(":", 1) + if len(sandbox_id) > 1: + return sandbox_id, path + return None, ref + + src_sandbox, src_path = parse(src) + dst_sandbox, dst_path = parse(dst) + if (src_sandbox is None) == (dst_sandbox is None): + raise CLIError("Exactly one of SRC and DST must be a sandbox path (:).") + if src_sandbox is not None: + with _connect(src_sandbox, namespace=namespace, token=token) as sandbox: + sandbox.files.download(src_path, dst_path) + else: + assert dst_sandbox is not None + with _connect(dst_sandbox, namespace=namespace, token=token) as sandbox: + sandbox.files.upload(src_path, dst_path) + out.result("Copied", src=src, dst=dst) + + +@sandbox_cli.command( + "kill", + examples=[ + "hf sandbox kill ", + "hf sandbox kill # kills a whole shared host (all its sandboxes)", + "hf sandbox kill --all", + ], +) +def sandbox_kill( + sandbox_id: Annotated[str | None, Argument(help="The sandbox or host id to terminate.")] = None, + all_: Annotated[bool, Option("--all", help="Terminate every sandbox and host in the namespace.")] = False, + yes: Annotated[bool, Option("-y", "--yes", help="Answer Yes to prompts automatically.")] = False, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Terminate a sandbox, a whole shared host, or everything (--all).""" + api = get_hf_api(token=token) + + if all_: + jobs = list(api.list_jobs(status=["RUNNING", "SCHEDULING"], labels={SANDBOX_LABEL: "1"}, namespace=namespace)) + if not jobs: + out.text("No running sandboxes.") + return + out.confirm(f"Terminate {len(jobs)} sandbox job(s) (including shared hosts and all their sandboxes)?", yes=yes) + for job in jobs: + api.cancel_job(job_id=job.id, namespace=job.owner.name) + out.result("Terminated", jobs=len(jobs)) + return + + if sandbox_id is None: + raise CLIError("Provide a sandbox id, a host id, or --all.") + + sid, ns = _split_sandbox_id(sandbox_id, namespace) + if SHARED_ID_SEP in sid: + # One shared sandbox: remove it from its host (frees a slot; host keeps running). + try: + with _connect(sandbox_id, namespace=namespace, token=token) as sandbox: + sandbox.kill() + except SandboxError as e: + raise CLIError(str(e)) from e + out.result("Sandbox terminated", id=sandbox_id) + return + + # A bare id: either a dedicated sandbox job or a shared host job. + job = api.inspect_job(job_id=sid, namespace=ns) + if (job.labels or {}).get(MODE_LABEL) == MODE_POOL: + out.confirm(f"Terminate shared host {sid} and all of its sandboxes?", yes=yes) + api.cancel_job(job_id=job.id, namespace=job.owner.name) + out.result("Host terminated", id=sid) + return + try: + with _connect(sandbox_id, namespace=namespace, token=token) as sandbox: + sandbox.kill() + except SandboxError as e: + raise CLIError(str(e)) from e + out.result("Sandbox terminated", id=sandbox_id) + + +@pool_cli.command( + "create", + examples=[ + "hf sandbox pool create", + "hf sandbox pool create python:3.12 --flavor cpu-basic", + "hf sandbox pool create --per-host 50 --idle-timeout 30m", + ], +) +def pool_create( + image: Annotated[str | None, Argument(help="Docker image for the hosts (needs /bin/sh).")] = None, + flavor: FlavorOpt = None, + per_host: Annotated[ + int, + Option("--per-host", min=1, help=f"Sandboxes packed per host VM (default {DEFAULT_SANDBOXES_PER_HOST})."), + ] = DEFAULT_SANDBOXES_PER_HOST, + max_hosts: Annotated[ + int | None, Option("--max-hosts", min=1, help="Optional cap on the number of host VMs.") + ] = None, + idle_timeout: Annotated[ + str | None, + Option(help="Shut a host down once it has had no sandboxes for this long (e.g. '10m'). Defaults to 10m."), + ] = None, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Warm a pool: boot one host VM now, tagged so it can be found later by its pool id.""" + start = time.time() + image = image or DEFAULT_IMAGE + pool = SandboxPool( + image=image, + flavor=flavor or "cpu-basic", + sandboxes_per_host=per_host, + max_hosts=max_hosts, + idle_timeout=idle_timeout if idle_timeout is not None else DEFAULT_IDLE_TIMEOUT, + namespace=namespace, + token=token, + ) + pool_id = pool.name + host_ids = pool.host_ids + out.result( + "Pool created", + id=pool_id, + image=image, + flavor=flavor or "cpu-basic", + host=host_ids[0], + elapsed=f"{time.time() - start:.1f}s", + ) + out.hint(f"Spawn a sandbox with `hf sandbox create --pool {pool_id}`.") + out.hint(f"Delete the pool (and its hosts) with `hf sandbox pool delete {pool_id}`.") + + +@pool_cli.command("ls | list", examples=["hf sandbox pool ls"]) +def pool_ls( + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """List running sandbox pools (grouped from their host VMs).""" + api = get_hf_api(token=token) + pools: dict[str, dict[str, Any]] = {} + for job in api.list_jobs(status="RUNNING", labels={MODE_LABEL: MODE_POOL}, namespace=namespace): + pid = (job.labels or {}).get(POOL_LABEL) + if not pid: + continue + env = job.environment if isinstance(job.environment, dict) else {} + info = pools.setdefault( + pid, + { + "id": pid, + "image": job.docker_image or job.space_id, + "flavor": job.flavor, + "per_host": env.get("SBX_CAPACITY", ""), + "hosts": 0, + }, + ) + info["hosts"] += 1 + rows = list(pools.values()) + out.table(rows, id_key="id") + if not rows: + out.hint("Create one with `hf sandbox pool create`.") + else: + out.hint("Spawn a sandbox with `hf sandbox create --pool `.") + + +@pool_cli.command( + "delete | rm", + examples=["hf sandbox pool delete "], +) +def pool_delete( + pool_id: Annotated[str, Argument(help="Pool id to delete.")], + yes: Annotated[bool, Option("-y", "--yes", help="Answer Yes to prompts automatically.")] = False, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Terminate every host VM of a pool (and therefore all its sandboxes).""" + api = get_hf_api(token=token) + hosts = list( + api.list_jobs( + status=["RUNNING", "SCHEDULING"], + labels={MODE_LABEL: MODE_POOL, POOL_LABEL: pool_id}, + namespace=namespace, + ) + ) + if not hosts: + delete_pool_cache(pool_id) + out.text(f"No running hosts for pool '{pool_id}'.") + return + out.confirm(f"Terminate {len(hosts)} host(s) of pool '{pool_id}' (and all their sandboxes)?", yes=yes) + for job in hosts: + api.cancel_job(job_id=job.id, namespace=job.owner.name) + delete_pool_cache(pool_id) + out.result("Pool deleted", id=pool_id, hosts_terminated=len(hosts)) + + +def _fmt_cmd(cmd: str | list[str]) -> str: + return cmd if isinstance(cmd, str) else " ".join(cmd) + + +def _fmt_status(process: SandboxProcess) -> str: + if process.running: + return "running" + return "exited" if process.exit_code is None else f"exited ({process.exit_code})" + + +@process_cli.command("ls | list", examples=["hf sandbox process ls "]) +def process_ls( + sandbox_id: SandboxIdArg, + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """List the background processes running in a sandbox (started with `hf sandbox spawn`).""" + with _connect(sandbox_id, namespace=namespace, token=token) as sandbox: + processes = sandbox.processes() + rows = [{"pid": p.pid, "status": _fmt_status(p), "cmd": _fmt_cmd(p.cmd)} for p in processes] + out.table(rows, id_key="pid") + if not rows: + out.hint(f"Start one with `hf sandbox spawn {sandbox_id} -- `.") + else: + out.hint(f"Stop one with `hf sandbox process kill {sandbox_id} `.") + + +@process_cli.command("kill", examples=["hf sandbox process kill "]) +def process_kill( + sandbox_id: SandboxIdArg, + pid: Annotated[int, Argument(help="The pid as printed by `hf sandbox process ls`.")], + namespace: NamespaceOpt = None, + token: TokenOpt = None, +) -> None: + """Stop a background process running in a sandbox.""" + with _connect(sandbox_id, namespace=namespace, token=token) as sandbox: + process = next((p for p in sandbox.processes() if p.pid == pid), None) + if process is None: + raise CLIError(f"No process with pid {pid} in sandbox {sandbox_id}.") + process.kill() + out.result("Process stopped", sandbox=sandbox_id, pid=pid) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/skills.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/skills.py new file mode 100644 index 0000000000000000000000000000000000000000..df433dcdc3b35520da1e8a2f92501b8aa556c4dc --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/skills.py @@ -0,0 +1,511 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to manage skills for AI assistants.""" + +import os +import shutil +from pathlib import Path +from typing import Annotated + +from click import Command, Context, Group + +from huggingface_hub.errors import CLIError + +from ..utils import disable_progress_bars +from . import _skills +from ._cli_utils import TokenOpt, _has_local_formatting_option, get_hf_api, typer_factory +from ._framework import Argument, Option +from ._output import out +from ._skills import DEFAULT_SKILL_ID + + +_SKILL_DESCRIPTION = ( + "Hugging Face Hub CLI (`hf`) for downloading, uploading, and managing" + " models, datasets, spaces, buckets, repos, papers, jobs, and more on the Hugging Face Hub." + " Use when: handling authentication;" + " managing local cache;" + " managing Hugging Face Buckets;" + " running or scheduling jobs on Hugging Face infrastructure;" + " managing Hugging Face repos;" + " discussions and pull requests;" + " browsing models, datasets and spaces;" + " reading, searching, or browsing academic papers;" + " managing collections;" + " querying datasets;" + " configuring spaces;" + " setting up webhooks;" + " or deploying and managing HF Inference Endpoints." + " Make sure to use this skill whenever the user mentions" + " 'hf', 'huggingface', 'Hugging Face', 'huggingface-cli', or 'hugging face cli'," + " or wants to do anything related to the Hugging Face ecosystem and to AI and ML in general." + " Also use for cloud storage needs like training checkpoints, data pipelines, or agent traces." + " Use even if the user doesn't explicitly ask for a CLI command." + " Replaces the deprecated `huggingface-cli`." +) + +_SKILL_YAML_PREFIX = f"""\ +--- +name: hf-cli +description: "{_SKILL_DESCRIPTION}" +--- + +Install: `curl -LsSf https://hf.co/cli/install.sh | bash -s`. + +The Hugging Face Hub CLI tool `hf` is available. IMPORTANT: The `hf` command replaces the deprecated `huggingface-cli` command. + +Use `hf --help` to view available functions. Note that auth commands are now all under `hf auth` e.g. `hf auth whoami`. +""" + +_SKILL_TIPS = """ +## Mounting repos as local filesystems + +To mount Hub repositories or buckets as local filesystems — no download, no copy, no waiting — use `hf-mount`. Files are fetched on demand. GitHub: https://github.com/huggingface/hf-mount + +Install: `curl -fsSL https://raw.githubusercontent.com/huggingface/hf-mount/main/install.sh | sh` + +Some command examples: +- `hf-mount start repo openai-community/gpt2 /tmp/gpt2` — mount a repo (read-only) +- `hf-mount start --hf-token $HF_TOKEN bucket myuser/my-bucket /tmp/data` — mount a bucket (read-write) +- `hf-mount status` / `hf-mount stop /tmp/data` — list or unmount + +## Tips + +- Use `hf --help` for full options, descriptions, usage, and real-world examples +- Authenticate with `HF_TOKEN` env var (recommended) or with `--token` +- Update the CLI with `hf update` (uses the correct command for the detected install method) +""" + +CENTRAL_LOCAL = Path(".agents/skills") +CENTRAL_GLOBAL = Path("~/.agents/skills") +CLAUDE_LOCAL = Path(".claude/skills") +CLAUDE_GLOBAL = Path("~/.claude/skills") +# Flags worth explaining in the common-options glossary. Self-explanatory flags +# (--namespace, --yes, --private, …) are omitted even if they appear frequently. +_COMMON_FLAG_ALLOWLIST = {"--token", "--quiet", "--type", "--format", "--revision"} +# Keep token out of inline command signatures to encourage env based auth. +_INLINE_FLAG_EXCLUDE = {"--token"} + +_COMMON_FLAG_HELP_OVERRIDES: dict[str, str] = { + "--format": "Output format: `--format json` (or `--json`) or `--format table` (default).", + "--token": "Use a User Access Token. Prefer setting `HF_TOKEN` env var instead of passing `--token`.", +} + +# Global formatting flags injected into the skill markdown for commands that +# accept them. They aren't real click params on the command (they're consumed +# globally — see ``_consume_format_flags_for_leaf`` in ``_cli_utils.py``) so we +# add them synthetically here. +_GLOBAL_FORMAT_INLINE_FLAGS = ["--format [auto|human|agent|json|quiet]"] +_GLOBAL_COMMON_FLAGS: dict[str, tuple[str, str]] = { + "--format": ("--format", "Output format."), + "--quiet": ("-q / --quiet", "Quiet output (one ID per line)."), +} + +skills_cli = typer_factory(help="Manage skills for AI assistants.") + + +def _type_hint(param) -> str: + """Value hint for an option: enum choices inline as ``[a|b|c]``, otherwise the TYPE name. + + e.g. `--sort [downloads|likes|trending_score]` instead of `--sort CHOICE`. + """ + choices = getattr(param.type, "choices", None) + if choices: + return "[" + "|".join(str(c) for c in choices) + "]" + return getattr(param.type, "name", "").upper() or "VALUE" + + +def _format_params(cmd: Command) -> str: + """Format required params: positional as UPPER_CASE, options as ``--name TYPE``.""" + parts = [] + for p in cmd.params: + if not p.required or p.human_readable_name == "--help": + continue + if p.name and p.name.startswith("_"): + continue + long_name = next((o for o in getattr(p, "opts", []) if o.startswith("--")), None) + if long_name is not None: + type_name = _type_hint(p) + parts.append(f"{long_name} {type_name}") + elif p.name: + parts.append(p.human_readable_name) + return " ".join(parts) + + +def _collect_leaf_commands(group: Group, ctx: Context, path_parts: list[str]) -> list[tuple[list[str], Command]]: + """Recursively walk a Click Group, returning (full_path_parts, cmd) for every leaf command.""" + leaves: list[tuple[list[str], Command]] = [] + sub_ctx = Context(group, parent=ctx, info_name=path_parts[-1]) + for name in group.list_commands(sub_ctx): + cmd = group.get_command(sub_ctx, name) + if cmd is None or cmd.hidden: + continue + child_path = [*path_parts, name] + if isinstance(cmd, Group): + leaves.extend(_collect_leaf_commands(cmd, sub_ctx, child_path)) + else: + leaves.append((child_path, cmd)) + return leaves + + +def _iter_optional_params(cmd: Command): + """Yield (param, long_name, short_name) for each optional, non-internal param.""" + for p in cmd.params: + if p.required or p.human_readable_name == "--help": + continue + if p.name and p.name.startswith("_"): + continue + long_name = None + short_name = None + for opt in getattr(p, "opts", []): + if opt.startswith("--"): + long_name = long_name or opt + elif opt.startswith("-"): + short_name = opt + if long_name: + yield p, long_name, short_name + + +def _accepts_global_format_flags(cmd: Command) -> bool: + """Return True if the leaf command accepts the global '--format' / '--json' / '-q' flags.""" + if cmd.context_settings.get("ignore_unknown_options"): + return False + return not _has_local_formatting_option(cmd) + + +def _get_flag_names(cmd: Command, *, exclude: set[str] | None = None) -> list[str]: + """Return long-form flag names (--foo) for optional, non-internal params. + + Boolean flags are bare ('--dry-run'). Value-taking options include a type hint ('--include TEXT', '--max-workers INTEGER'). + Synthetic global formatting flags are appended for commands that accept them. + """ + flags: list[str] = [] + for p, long_name, _short in _iter_optional_params(cmd): + if exclude and long_name in exclude: + continue + if getattr(p, "is_flag", False): + flags.append(long_name) + else: + type_name = _type_hint(p) + flags.append(f"{long_name} {type_name}") + if _accepts_global_format_flags(cmd): + flags.extend(flag for flag in _GLOBAL_FORMAT_INLINE_FLAGS if not (exclude and flag.split()[0] in exclude)) + return flags + + +def _compute_common_flags( + leaf_commands: list[tuple[list[str], Command]], +) -> dict[str, tuple[str, str]]: + """Collect display info for flags in the allowlist.""" + flag_info: dict[str, tuple[str, str]] = {} + + for _path, cmd in leaf_commands: + for p, long_name, short_name in _iter_optional_params(cmd): + if long_name not in _COMMON_FLAG_ALLOWLIST: + continue + # Prefer the version with a short form (e.g. "-q / --quiet" over just "--quiet") + if long_name not in flag_info or (short_name and " / " not in flag_info[long_name][0]): + display = f"{short_name} / {long_name}" if short_name else long_name + help_text = (getattr(p, "help", None) or "").split("\n")[0].strip() + flag_info[long_name] = (display, help_text) + + # Inject the global formatting flags as common flags whenever any leaf + # command accepts them (the vast majority do). + if any(_accepts_global_format_flags(cmd) for _path, cmd in leaf_commands): + for long_name, entry in _GLOBAL_COMMON_FLAGS.items(): + flag_info.setdefault(long_name, entry) + + return flag_info + + +def _render_leaf(path_parts: list[str], cmd: Command) -> str: + """Render a single leaf command as a markdown list entry.""" + help_text = (cmd.help or "").split("\n")[0].strip() + params = _format_params(cmd) + parts = ["hf", *path_parts] + ([params] if params else []) + entry = f"- `{' '.join(parts)}` — {help_text}" + flags = _get_flag_names(cmd, exclude=_INLINE_FLAG_EXCLUDE) + if flags: + entry += f" `[{' '.join(flags)}]`" + return entry + + +def build_skill_md() -> str: + # Lazy import to avoid circular dependency (hf.py imports skills_cli from this module) + from huggingface_hub import __version__ + from huggingface_hub.cli.hf import app + + click_app = app # the app is already a click.Group + ctx = Context(click_app, info_name="hf") + + top_level: list[tuple[list[str], Command]] = [] + groups: list[tuple[str, Group]] = [] + for name in sorted(click_app.list_commands(ctx)): # type: ignore[attr-defined] + cmd = click_app.get_command(ctx, name) # type: ignore[attr-defined] + if cmd is None or cmd.hidden: + continue + if isinstance(cmd, Group): + groups.append((name, cmd)) + else: + top_level.append(([name], cmd)) + + group_leaves: list[tuple[str, list[tuple[list[str], Command]]]] = [] + all_leaf_commands: list[tuple[list[str], Command]] = list(top_level) + for name, group in groups: + leaves = _collect_leaf_commands(group, ctx, [name]) + group_leaves.append((name, leaves)) + all_leaf_commands.extend(leaves) + + common_flags = _compute_common_flags(all_leaf_commands) + + # wrap in list to widen list[LiteralString] -> list[str] for `ty` + lines: list[str] = list(_SKILL_YAML_PREFIX.splitlines()) + lines.append("") + lines.append(f"Generated with `huggingface_hub v{__version__}`. Run `hf skills add --force` to regenerate.") + lines.append("") + lines.append("## Commands") + lines.append("") + + for path_parts, cmd in top_level: + lines.append(_render_leaf(path_parts, cmd)) + + groups_dict = dict(groups) + for name, leaves in group_leaves: + group_cmd = groups_dict[name] + help_text = (group_cmd.help or "").split("\n")[0].strip() + lines.append("") + lines.append(f"### `hf {name}` — {help_text}") + lines.append("") + for path_parts, cmd in leaves: + lines.append(_render_leaf(path_parts, cmd)) + + if common_flags: + lines.append("") + lines.append("## Common options") + lines.append("") + for long_name, (display, help_text) in sorted(common_flags.items()): + help_text = _COMMON_FLAG_HELP_OVERRIDES.get(long_name, help_text) + if help_text: + lines.append(f"- `{display}` — {help_text}") + else: + lines.append(f"- `{display}`") + + lines.extend(_SKILL_TIPS.splitlines()) + + return "\n".join(lines) + + +def _remove_existing(path: Path, force: bool) -> None: + """Remove existing file/directory/symlink if force is True, otherwise raise an error.""" + if not (path.exists() or path.is_symlink()): + return + if not force: + raise CLIError(f"Skill already exists at {path}.\nRe-run with --force to overwrite.") + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + + +def _install_to(skills_dir: Path, skill_name: str, force: bool) -> Path: + """Install a marketplace skill into a skills directory. Returns the installed path.""" + try: + if skill_name.strip() == DEFAULT_SKILL_ID: + return _skills.install_generated_skill(build_skill_md(), skills_dir, force=force) + return _skills.add_skill(skill_name, skills_dir, force=force) + except FileExistsError as exc: + raise CLIError(f"{exc}\nRe-run with --force to overwrite.") from exc + + +def _create_symlink(agent_skills_dir: Path, skill_name: str, central_skill_path: Path, force: bool) -> Path: + """Create a relative symlink from agent directory to the central skill location.""" + agent_skills_dir = agent_skills_dir.expanduser().resolve() + agent_skills_dir.mkdir(parents=True, exist_ok=True) + link_path = agent_skills_dir / skill_name + + _remove_existing(link_path, force) + link_path.symlink_to(os.path.relpath(central_skill_path, agent_skills_dir)) + + return link_path + + +def _resolve_update_roots( + *, + claude: bool, + global_: bool, + dest: Path | None, +) -> list[Path]: + if dest is not None: + if claude or global_: + raise CLIError("--dest cannot be combined with --claude or --global.") + return [dest.expanduser().resolve()] + + roots: list[Path] = [CENTRAL_GLOBAL if global_ else CENTRAL_LOCAL] + if claude: + roots.append(CLAUDE_GLOBAL if global_ else CLAUDE_LOCAL) + return [root.expanduser().resolve() for root in roots] + + +@skills_cli.command("preview") +def skills_preview() -> None: + """Print the generated `hf-cli` SKILL.md to stdout.""" + print(build_skill_md()) + + +@skills_cli.command( + "list | ls", + examples=[ + "hf skills list", + "hf skills list --format json", + ], +) +def skills_list( + token: TokenOpt = None, +) -> None: + """List available skills from the Hugging Face marketplace.""" + install_locations: list[tuple[str, Path]] = [ + ("project", CENTRAL_LOCAL), + ("project (claude)", CLAUDE_LOCAL), + ("global", CENTRAL_GLOBAL), + ("global (claude)", CLAUDE_GLOBAL), + ] + installed: dict[str, set[str]] = {} + for label, root in install_locations: + for skill_dir in _skills._iter_unique_skill_dirs([root]): + installed.setdefault(skill_dir.name.lower(), set()).add(label) + + api = get_hf_api(token=token) + with disable_progress_bars(): + skills = _skills._load_marketplace_skills(api) + results = [ + { + "name": skill.name, + "description": skill.description or "", + **{ + label: "yes" if label in installed.get(skill.name.lower(), set()) else "" + for label, _ in install_locations + }, + } + for skill in skills + ] + out.table( + results, + id_key="name", + alignments={"project": "right", "global": "right", "project (claude)": "right", "global (claude)": "right"}, + ) + + +@skills_cli.command( + "add", + examples=[ + "hf skills add", + "hf skills add huggingface-gradio --dest=~/my-skills", + "hf skills add --global", + "hf skills add --claude", + "hf skills add huggingface-gradio --claude --global", + ], +) +def skills_add( + name: Annotated[ + str, + Argument(help="Marketplace skill name.", show_default=False), + ] = DEFAULT_SKILL_ID, + claude: Annotated[bool, Option("--claude", help="Install for Claude.")] = False, + global_: Annotated[ + bool, + Option( + "--global", + "-g", + help="Install globally (user-level) instead of in the current project directory.", + ), + ] = False, + dest: Annotated[ + Path | None, + Option( + help="Install into a custom destination (path to skills directory).", + ), + ] = None, + force: Annotated[ + bool, + Option( + "--force", + help="Overwrite existing skills in the destination.", + ), + ] = False, +) -> None: + """Install a Hugging Face skill for an AI assistant. + + The default `hf-cli` skill is generated locally from the installed CLI version; + other skills are downloaded from the Hugging Face marketplace. + Default location is in the current directory (.agents/skills) or user-level (~/.agents/skills). + If `--claude` is specified, the skill is also symlinked into Claude's legacy skills directory. + """ + if dest is not None: + if claude or global_: + raise CLIError("--dest cannot be combined with --claude or --global.") + skill_dest = _install_to(dest, name, force) + print(f"Installed '{name}' to {skill_dest}") + return + + # Install to central location + central_path = CENTRAL_GLOBAL if global_ else CENTRAL_LOCAL + central_skill_path = _install_to(central_path, name, force) + print(f"Installed '{name}' to central location: {central_skill_path}") + + if claude: + agent_target = CLAUDE_GLOBAL if global_ else CLAUDE_LOCAL + link_path = _create_symlink(agent_target, name, central_skill_path, force) + print(f"Created symlink: {link_path}") + + +@skills_cli.command( + "update", + examples=[ + "hf skills update", + "hf skills update hf-cli", + "hf skills update huggingface-gradio --dest=~/my-skills", + "hf skills update --claude", + ], +) +def skills_update( + name: Annotated[ + str | None, + Argument(help="Optional installed skill name to update.", show_default=False), + ] = None, + claude: Annotated[bool, Option("--claude", help="Update skills installed for Claude.")] = False, + global_: Annotated[ + bool, + Option( + "--global", + "-g", + help="Use global skills directories instead of the current project.", + ), + ] = False, + dest: Annotated[ + Path | None, + Option( + help="Update skills in a custom skills directory.", + ), + ] = None, +) -> None: + """Update installed Hugging Face marketplace skills.""" + roots = _resolve_update_roots(claude=claude, global_=global_, dest=dest) + + results = _skills.update_skills(roots, selector=name, hf_cli_content=build_skill_md()) + if not results: + print("No installed skills found.") + return + + for result in results: + detail = f" ({result.detail})" if result.detail else "" + print(f"{result.name}: {result.status}{detail}") diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/spaces.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/spaces.py new file mode 100644 index 0000000000000000000000000000000000000000..11bdeea97f60f66a7423b921fb1d376d1af9d7b7 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/spaces.py @@ -0,0 +1,1090 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with spaces on the Hugging Face Hub.""" + +import enum +import functools +import itertools +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +import time +from collections import deque +from pathlib import Path +from typing import TYPE_CHECKING, Annotated, Literal, get_args + +import click +from packaging import version +from typing_extensions import assert_never + +from huggingface_hub._hot_reload.client import multi_replica_reload_events +from huggingface_hub._hot_reload.types import ApiGetReloadEventSourceData, ReloadRegion +from huggingface_hub._space_api import SpaceHardware, SpaceStage +from huggingface_hub.cli._cli_utils import SoftChoice +from huggingface_hub.errors import CLIError, RemoteEntryNotFoundError, RepositoryNotFoundError, RevisionNotFoundError +from huggingface_hub.file_download import hf_hub_download +from huggingface_hub.hf_api import ExpandSpaceProperty_T, HfApi, SpaceSort_T +from huggingface_hub.repocard import SpaceCard +from huggingface_hub.utils import disable_progress_bars +from huggingface_hub.utils._parsing import parse_duration + +from ._cli_utils import ( + REPO_LIST_DEFAULT_LIMIT, + AuthorOpt, + EnvFileOpt, + EnvOpt, + FilterOpt, + LimitOpt, + RevisionOpt, + SearchOpt, + SecretsFileOpt, + SecretsOpt, + SshDryRunOpt, + SshIdentityFileOpt, + TokenOpt, + VolumesOpt, + exec_ssh, + get_hf_api, + make_expand_properties_parser, + parse_env_map, + parse_volumes, + typer_factory, +) +from ._file_listing import list_repo_files_cmd +from ._framework import Argument, Option +from ._output import _dataclass_to_dict, out + + +HOT_RELOADING_MIN_GRADIO = "6.1.0" + + +_EXPAND_PROPERTIES = sorted(get_args(ExpandSpaceProperty_T)) +_SORT_OPTIONS = get_args(SpaceSort_T) +SpaceSortEnum = enum.Enum("SpaceSortEnum", {s: s for s in _SORT_OPTIONS}, type=str) # type: ignore[misc] + + +ExpandOpt = Annotated[ + str | None, + Option( + help=f"Comma-separated properties to return. When used, only the listed properties (and id) are returned. Example: '--expand=likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.", + callback=make_expand_properties_parser(_EXPAND_PROPERTIES), + ), +] + +spaces_cli = typer_factory(help="Interact with spaces on the Hub.") +volumes_cli = typer_factory(help="Manage volumes for a Space on the Hub.") +secrets_cli = typer_factory(help="Manage secrets for a Space on the Hub.") +variables_cli = typer_factory(help="Manage environment variables for a Space on the Hub.") +spaces_cli.add_group(volumes_cli, name="volumes") +spaces_cli.add_group(secrets_cli, name="secrets") +spaces_cli.add_group(variables_cli, name="variables") + + +@spaces_cli.command( + "list | ls", + examples=[ + "hf spaces ls --limit 10", + 'hf spaces ls --search "chatbot" --author huggingface', + "hf spaces ls victor/deepsite", + "hf spaces ls victor/deepsite -R", + "hf spaces ls victor/deepsite --tree -h", + ], +) +def spaces_ls( + repo_id: Annotated[ + str | None, + Argument(help="Space ID (e.g. `username/repo-name`) to list files from. If omitted, lists spaces."), + ] = None, + search: SearchOpt = None, + author: AuthorOpt = None, + filter: FilterOpt = None, + sort: Annotated[ + SpaceSortEnum | None, + Option(help="Sort results."), + ] = None, + limit: LimitOpt = REPO_LIST_DEFAULT_LIMIT, + expand: ExpandOpt = None, + human_readable: Annotated[ + bool, + Option("--human-readable", "-h", help="Show sizes in human readable format (only for listing files)."), + ] = False, + as_tree: Annotated[ + bool, + Option("--tree", help="List files in tree format (only for listing files)."), + ] = False, + recursive: Annotated[ + bool, + Option("--recursive", "-R", help="List files recursively (only for listing files)."), + ] = False, + revision: RevisionOpt = None, + token: TokenOpt = None, +) -> None: + """List spaces on the Hub, or files in a space repo. + + When called with no argument, lists spaces on the Hub. + When called with a space ID, lists files in that space repo. + """ + if repo_id is not None: + if search is not None: + raise click.BadParameter("Cannot use --search when listing files.") + if author is not None: + raise click.BadParameter("Cannot use --author when listing files.") + if filter is not None: + raise click.BadParameter("Cannot use --filter when listing files.") + if sort is not None: + raise click.BadParameter("Cannot use --sort when listing files.") + if limit != REPO_LIST_DEFAULT_LIMIT: + raise click.BadParameter("Cannot use --limit when listing files.") + if expand is not None: + raise click.BadParameter("Cannot use --expand when listing files.") + return list_repo_files_cmd( + repo_id=repo_id, + repo_type="space", + human_readable=human_readable, + as_tree=as_tree, + recursive=recursive, + revision=revision, + token=token, + ) + + if as_tree: + raise click.BadParameter("Cannot use --tree when listing spaces.") + if recursive: + raise click.BadParameter("Cannot use --recursive when listing spaces.") + if human_readable: + raise click.BadParameter("Cannot use --human-readable when listing spaces.") + if revision is not None: + raise click.BadParameter("Cannot use --revision when listing spaces.") + api = get_hf_api(token=token) + sort_key = sort.value if sort else None + results = [ + _dataclass_to_dict(space_info) + for space_info in api.list_spaces( + filter=filter, + author=author, + search=search, + sort=sort_key, + limit=limit, + expand=expand, # type: ignore[arg-type] + ) + ] + out.table(results) + + +@spaces_cli.command( + "info", + examples=[ + "hf spaces info enzostvs/deepsite", + "hf spaces info gradio/theme_builder --expand sdk,runtime,likes", + ], +) +def spaces_info( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + revision: RevisionOpt = None, + expand: ExpandOpt = None, + token: TokenOpt = None, +) -> None: + """Get info about a space on the Hub.""" + api = get_hf_api(token=token) + try: + info = api.space_info(repo_id=space_id, revision=revision, expand=expand) # type: ignore[arg-type] + except RepositoryNotFoundError as e: + raise CLIError(f"Space '{space_id}' not found.") from e + except RevisionNotFoundError as e: + raise CLIError(f"Revision '{revision}' not found on '{space_id}'.") from e + out.dict(info) + + +@spaces_cli.command( + "card", + examples=[ + "hf spaces card mteb/leaderboard", + "hf spaces card mteb/leaderboard --metadata", + "hf spaces card mteb/leaderboard --metadata --format json", + "hf spaces card mteb/leaderboard --text", + ], +) +def spaces_card( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + metadata: Annotated[bool, Option("--metadata", help="Output only the metadata from the card.")] = False, + text: Annotated[bool, Option("--text", help="Output only the text body (no metadata).")] = False, + token: TokenOpt = None, +) -> None: + """Get the Space card (README) for a Space on the Hub.""" + if metadata and text: + raise CLIError("--metadata and --text are mutually exclusive.") + card = SpaceCard.load(space_id, token=token) + if metadata: + out.dict(card.data.to_dict()) + elif text: + out.text(card.text) + else: + out.text(card.content) + out.hint(f"Use `hf spaces card {space_id} --metadata` to extract only the card metadata.") + + +@spaces_cli.command( + "templates", + examples=["hf spaces templates"], +) +def spaces_templates( + token: TokenOpt = None, +) -> None: + """List the available Space templates. + + The `repo_id` (or `name`) of a template can be passed to `hf repos create --template ...` to + create a new Space from that template. + """ + api = get_hf_api(token=token) + templates = [_dataclass_to_dict(template) for template in api.list_space_templates()] + out.table(templates, id_key="name") + out.hint( + "Create a Space from a template with `hf repos create --type space --space-sdk --template `." + ) + + +@spaces_cli.command( + "search", + examples=[ + 'hf spaces search "generate image"', + 'hf spaces search "identify objects in pictures" --sdk gradio --limit 5', + 'hf spaces search "remove background from photo" --description --json', + ], +) +def spaces_search( + query: Annotated[str, Argument(help="Search query.")], + filter: FilterOpt = None, + sdk: Annotated[list[str] | None, Option(help="Filter by SDK (e.g. gradio, docker, static).")] = None, + include_non_running: Annotated[bool, Option(help="Include non-running spaces in results.")] = False, + description: Annotated[bool, Option(help="Show AI-generated descriptions.")] = False, + limit: LimitOpt = 10, + token: TokenOpt = None, +) -> None: + """Search spaces on the Hub using semantic search.""" + api = get_hf_api(token=token) + results = api.search_spaces( + query=query, + filter=filter, + sdk=sdk, + include_non_running=include_non_running, + token=token, + ) + items = [] + for r in itertools.islice(results, limit): + item: dict = { + "id": r.id, + "title": r.title, + "sdk": r.sdk, + "likes": r.likes, + "stage": r.runtime.stage if r.runtime else None, + "category": r.ai_category, + "score": round(r.semantic_relevancy_score, 2) if r.semantic_relevancy_score is not None else None, + } + if description: + item["description"] = r.ai_short_description + items.append(item) + out.table(items) + if not description: + out.hint("Use --description to show AI-generated descriptions.") + + +@spaces_cli.command( + "wait", + examples=[ + "hf spaces wait username/my-space", + "hf spaces wait username/my-space --timeout 5m", + ], +) +def spaces_wait( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + timeout: Annotated[ + str | None, + Option( + help="Max time to wait: int with s (seconds, default), m (minutes), h (hours) or d (days).", + ), + ] = None, + token: TokenOpt = None, +) -> None: + """Wait for a Space to finish building/starting. + + Blocks until the Space leaves an intermediate stage (BUILDING, APP_STARTING, etc.) + and reaches a settled stage. Exits with code 0 if the Space is RUNNING, + or a non-zero exit code otherwise (e.g. BUILD_ERROR, RUNTIME_ERROR). + """ + timeout_secs = parse_duration(timeout) if timeout is not None else None + api = get_hf_api(token=token) + status = out.status("Waiting for Space to be ready...") + try: + runtime = api.wait_for_space(space_id, timeout=timeout_secs) + except TimeoutError: + status.done("Timed out.") + raise CLIError(f"Timed out after {timeout} waiting for Space '{space_id}' to be ready.") from None + status.done(f"Space reached stage '{runtime.stage}'.") + if runtime.stage != SpaceStage.RUNNING: + raise CLIError(f"Space '{space_id}' is not running (stage='{runtime.stage}').") + out.result("Space ready", space_id=space_id, stage=str(runtime.stage)) + out.hint(f"Use `hf spaces logs {space_id}` to view run logs.") + + +@spaces_cli.command( + "dev-mode", + examples=[ + "hf spaces dev-mode my-user-name/deepsite", + ], +) +def dev_mode( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + stop: Annotated[bool, Option(help="Stop dev mode.")] = False, + token: TokenOpt = None, +): + """ + Enable or disable dev mode on a Space. + + Spaces Dev Mode eases the debugging of your application and makes iterating on Spaces faster by allowing you to + restart your application without stopping the Space container itself. This feature is available as part of a PRO + or Team & Enterprise plan. + + See docs: https://huggingface.co/docs/hub/spaces-dev-mode + """ + api = get_hf_api(token=token) + if stop: + api.disable_space_dev_mode(space_id) + print(f"Dev mode disabled for '{space_id}'") + return + api.enable_space_dev_mode(space_id) + runtime = api.wait_for_space(space_id) + if runtime.stage != SpaceStage.RUNNING: + out.warning(f"Dev mode is not ready (stage='{runtime.stage}')") + return + info = api.space_info(space_id) + folder = getattr(info.card_data, "dev-mode-folder", "" if info.sdk == "docker" else "/home/user/app") + folder_query_param = f"folder={folder}" if folder else "" + print("Connect to dev environment:") + print("") + print("Web:") + vscode_web_url = f"https://huggingface.co/spaces/{info.id}/dev-mode/vscode-web" + if folder_query_param: + vscode_web_url += f"?{folder_query_param}" + ssh_host = f"{info.subdomain}@ssh.hf.space" + print(f" * VSCode: {vscode_web_url}") + print("") + print("Local:") + print("1. Add your SSH key to https://huggingface.co/settings/keys") + print(f"2. SSH with `hf spaces ssh {space_id}` (or `ssh -i {ssh_host}`)") + print(" Or open") + print(f" * VSCode: vscode://vscode-remote/ssh-remote+{ssh_host}{folder}") + print(f" * Cursor: cursor://vscode-remote/ssh-remote+{ssh_host}{folder}") + print("") + print("PS: Dev mode stops after 48h of inactivity, don't forget to save your changes regularly.") + + +@spaces_cli.command( + "ssh", + examples=[ + "hf spaces ssh username/my-space", + "hf spaces ssh username/my-space --dry-run", + "hf spaces ssh username/my-space -i ~/.ssh/id_ed25519", + "hf spaces ssh username/my-space --auto", + ], +) +def spaces_ssh( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + identity_file: SshIdentityFileOpt = None, + dry_run: SshDryRunOpt = False, + auto: Annotated[ + bool, + Option("--auto", help="Enable Dev Mode without prompting if not already enabled."), + ] = False, + token: TokenOpt = None, +) -> None: + """SSH into a Space's Dev Mode container. + + Requires Dev Mode to be running on the Space and your SSH public key to be registered at https://huggingface.co/settings/keys. + + See: https://huggingface.co/docs/hub/spaces-dev-mode + """ + api = get_hf_api(token=token) + info = api.space_info(space_id) + if info.runtime is None or not info.runtime.dev_mode: + out.confirm( + f"Dev Mode is disabled on '{space_id}'. Enable it now?", yes=auto, default=True, confirm_param="--auto" + ) + api.enable_space_dev_mode(space_id) + runtime = api.wait_for_space(space_id) + if runtime.stage != SpaceStage.RUNNING: + raise CLIError(f"Space '{space_id}' is not running (stage='{runtime.stage}').") + info = api.space_info(space_id) + exec_ssh(f"{info.subdomain}@ssh.hf.space", identity_file=identity_file, dry_run=dry_run) + + +@spaces_cli.command( + "pause", + examples=[ + "hf spaces pause username/my-space", + ], +) +def spaces_pause( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + token: TokenOpt = None, +) -> None: + """Pause a Space.""" + api = get_hf_api(token=token) + runtime = api.pause_space(space_id) + out.result("Space paused", space_id=space_id, stage=runtime.stage) + out.hint(f"Use `hf spaces restart {space_id}` to restart it.") + out.hint( + f"Mount a Volume or bucket to persist data across restarts: `hf spaces volumes set {space_id} -v hf://...`" + ) + + +@spaces_cli.command( + "restart", + examples=[ + "hf spaces restart username/my-space", + "hf spaces restart username/my-space --factory-reboot", + ], +) +def spaces_restart( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + factory_reboot: Annotated[ + bool, + Option( + "--factory-reboot", + help="Rebuild the Space from scratch without using the build cache.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + """Restart a Space.""" + api = get_hf_api(token=token) + runtime = api.restart_space(space_id, factory_reboot=factory_reboot) + out.result( + "Space restart triggered", + space_id=space_id, + stage=runtime.stage, + factory_reboot=factory_reboot, + ) + out.hint(f"Use `hf spaces wait {space_id}` to wait until the Space is ready.") + out.hint( + f"Mount a Volume or bucket to persist data across restarts: `hf spaces volumes set {space_id} -v hf://...`" + ) + + +@spaces_cli.command( + "hardware", + examples=[ + "hf spaces hardware", + ], +) +def spaces_hardware(token: TokenOpt = None) -> None: + """List available hardware options for Spaces.""" + api = get_hf_api(token=token) + hardware_list = api.list_spaces_hardware() + items = [] + for hw in hardware_list: + accelerator = ( + f"{hw.accelerator.quantity}x {hw.accelerator.model} ({hw.accelerator.vram})" if hw.accelerator else None + ) + cost_min = f"${hw.unit_cost_usd:.4f}" if hw.unit_cost_usd else "free" + cost_hour = f"${hw.unit_cost_usd * 60:.2f}" if hw.unit_cost_usd else "free" + items.append( + { + "name": hw.name, + "pretty name": hw.pretty_name, + "cpu": hw.cpu, + "ram": hw.ram, + "accelerator": accelerator, + "cost/min": cost_min, + "cost/hour": cost_hour, + } + ) + out.table(items) + out.hint("Use `hf spaces settings --hardware ` to request hardware for a Space.") + + +@spaces_cli.command( + "settings", + examples=[ + "hf spaces settings username/my-space --sleep-time 300", + "hf spaces settings username/my-space --hardware t4-medium", + ], +) +def spaces_settings( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + sleep_time: Annotated[ + int | None, + Option( + "--sleep-time", + help="Idle time in seconds after which the Space goes to sleep. Use -1 to never sleep. Only available on upgraded hardware.", + ), + ] = None, + hardware: Annotated[ + str | None, + Option( + "--hardware", + help="Space hardware flavor (e.g. 'cpu-basic', 't4-medium', 'l4x4'). Run 'hf spaces hardware' to list available options.", + click_type=SoftChoice(SpaceHardware), + ), + ] = None, + token: TokenOpt = None, +) -> None: + """Update the settings of a Space.""" + api = get_hf_api(token=token) + if hardware is not None: + runtime = api.request_space_hardware(space_id, hardware=hardware, sleep_time=sleep_time) # type: ignore[arg-type] + elif sleep_time is not None: + runtime = api.set_space_sleep_time(space_id, sleep_time=sleep_time) + else: + raise CLIError("Specify at least one setting to update.") + out.result( + "Space settings updated", + space_id=space_id, + hardware=runtime.requested_hardware, + sleep_time=runtime.sleep_time, + ) + out.hint(f"Use `hf spaces info {space_id}` to verify the runtime configuration.") + + +@spaces_cli.command( + "logs", + examples=[ + "hf spaces logs username/my-space", + "hf spaces logs username/my-space --build", + "hf spaces logs -f username/my-space", + "hf spaces logs -n 50 username/my-space", + ], +) +def spaces_logs( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + build: Annotated[ + bool, + Option( + "--build", + help="Fetch the container build logs instead of the run logs. Useful when a Space is stuck in BUILD_ERROR.", + ), + ] = False, + follow: Annotated[ + bool, + Option( + "-f", + "--follow", + help="Follow log output (stream until the server closes the stream). Without this flag, only currently available logs are printed.", + ), + ] = False, + tail: Annotated[ + int | None, + Option( + "-n", + "--tail", + help="Number of lines to show from the end of the logs.", + ), + ] = None, + token: TokenOpt = None, +) -> None: + """Fetch the run or build logs of a Space. + + By default, prints currently available run logs and exits (non-blocking, like + `docker logs`). Use --follow/-f to stream until the server closes the stream. + Use --build to see the container build logs instead (useful when a Space is + stuck in BUILD_ERROR). + """ + if follow and tail is not None: + raise CLIError( + "Cannot use --follow and --tail together. Use --follow to stream logs or --tail to show recent logs." + ) + + api = get_hf_api(token=token) + logs = api.fetch_space_logs(space_id, build=build, follow=follow) + if tail is not None: + logs = deque(logs, maxlen=tail) + found_logs = False + for line in logs: + clean_line = line.strip() + out.text(clean_line) + if clean_line: + found_logs = True + if not found_logs and not build: + out.hint(f"No run logs found for space {space_id}. Try passing --build to fetch build logs instead.") + + +@spaces_cli.command( + "hot-reload", + examples=[ + "hf spaces hot-reload username/repo-name app.py # Open an interactive editor to the remote app.py file", + "hf spaces hot-reload username/repo-name -f app.py # Take local version from ./app.py and patch app.py remotely", + "hf spaces hot-reload username/repo-name app.py -f src/app.py # Take local version from ./src/app.py", + ], +) +def spaces_hot_reload( + space_id: Annotated[ + str, + Argument( + help="The space ID (e.g. `username/repo-name`).", + ), + ], + filename: Annotated[ + str | None, + Argument( + help="Path to the Python file in the Space repository. Can be omitted when --local-file is specified and path in repository matches." + ), + ] = None, + local_file: Annotated[ + Path | None, + Option( + "--local-file", + "-f", + help="Path of local file. Interactive editor mode if not specified", + ), + ] = None, + skip_checks: Annotated[bool, Option(help="Skip hot-reload compatibility checks.")] = False, + skip_summary: Annotated[bool, Option(help="Skip summary display after hot-reload is triggered")] = False, + token: TokenOpt = None, +) -> None: + """ + Hot-reload any Python file of a Space without a full rebuild + restart. + + ⚠ This feature is experimental ⚠ + + Only works with Gradio SDK (6.1+) + Opens an interactive editor unless --local-file/-f is specified. + + This command patches the live Python process using https://github.com/breuleux/jurigged + (AST-based diffing, in-place function updates, etc.), integrated with Gradio's native hot-reload support + (meaning that Gradio demo object changes are reflected in the UI) + + The command creates a remote commit. + If you are working from a local clone, run `git pull --autostash` afterwards + to bring the commit back and keep your local git state in sync. + """ + + click.secho("This feature is experimental and subject to change", fg="bright_black") + + api = get_hf_api(token=token) + + if not skip_checks: + space_info = api.space_info(space_id) + if space_info.sdk != "gradio": + raise CLIError(f"Hot-reloading is only available on Gradio SDK. Found {space_info.sdk} SDK") + if (card_data := space_info.card_data) is None: + raise CLIError(f"Unable to read cardData for Space {space_id}") + if (sdk_version := card_data.sdk_version) is None: + raise CLIError(f"Unable to read sdk_version from {space_id} cardData") + if version.parse(sdk_version) < version.Version(HOT_RELOADING_MIN_GRADIO): + raise CLIError(f"Hot-reloading requires Gradio >= {HOT_RELOADING_MIN_GRADIO} (found {sdk_version})") + if (current_sha := space_info.sha) is None: + raise CLIError(f"Unexpected `None` running SHA for Space {space_id}") + else: + current_sha = None + + if local_file: + local_path = str(local_file) + filename = local_file.as_posix() if filename is None else filename + elif filename: + if not skip_checks: + try: + api.auth_check( + repo_type="space", + repo_id=space_id, + write=True, + ) + except RepositoryNotFoundError as e: + raise CLIError( + f"Write access check to {space_id} repository failed. Make sure that you are authenticated" + ) from e + temp_dir = tempfile.TemporaryDirectory() + local_path = os.path.join(temp_dir.name, filename) + with disable_progress_bars(): + try: + hf_hub_download(repo_type="space", repo_id=space_id, filename=filename, local_dir=temp_dir.name) + except RemoteEntryNotFoundError: + click.secho(f"{filename} not found in remote repository. Assuming new file", fg="bright_black") + + editor_res = _editor_open(local_path) + if editor_res == "no-tty": + persistent_temp_dir = tempfile.mkdtemp() + shutil.copytree(temp_dir.name, persistent_temp_dir, dirs_exist_ok=True) + local_path = os.path.join(persistent_temp_dir, filename) + click.secho("No TTY detected. Non-interactive fallback:") + click.secho(f"- Edit {local_path}") + click.secho(f"- Run `hf spaces hot-reload {space_id} {filename} -f {local_path}`") + return + if editor_res == "no-editor": + raise CLIError("No editor found in local environment. Use -f flag to hot-reload from local path") + if editor_res != 0: + raise CLIError(f"Editor returned a non-zero exit code while attempting to edit {local_path}") + else: + raise CLIError("Either filename or --local-file/-f must be specified") + + commit_info = api.upload_file( + repo_type="space", + repo_id=space_id, + path_or_fileobj=local_path, + path_in_repo=filename, + parent_commit=current_sha, + _hot_reload=True, + ) + + if local_file is not None and local_file.resolve().is_relative_to(Path.cwd()): + click.secho(f"Created commit {commit_info.oid} in remote Space repository.") + click.secho("Consider running `git pull --autostash` to stay synced if you are working from a local clone.") + + if not skip_summary: + click.secho("Hot-reload summary:") + _spaces_hot_reload_summary( + api=api, + space_id=space_id, + current_sha=current_sha, + commit_sha=commit_info.oid, + local_path=local_path if local_file else filename, + filename=filename, + token=token, + ) + + +def _spaces_hot_reload_summary( + api: HfApi, + space_id: str, + current_sha: str | None, + commit_sha: str, + filename: str, + local_path: str, + token: str | None, +) -> None: + while (space_info := api.space_info(space_id)).sha == current_sha: + if current_sha is None or current_sha == commit_sha: + break + click.secho("Waiting for up-to-date Space infos", fg="bright_black", err=True) + time.sleep(2) + if space_info.sha != commit_sha: + raise CLIError(f"Expected SHA {commit_sha} after hot-reload but got {space_info.sha}") + if (runtime := space_info.runtime) is None: + raise CLIError(f"Unable to read SpaceRuntime from {space_id} infos") + if (hot_reloading := runtime.hot_reloading) is None: + raise CLIError(f"Space {space_id} current running version has not been hot-reloaded") + if hot_reloading.status != "created": + click.echo(f"Failed creating hot-reloaded commit. {hot_reloading.replica_statuses=}") + return + + if (space_host := space_info.host) is None: + raise CLIError("Unexpected None host on hotReloaded Space") + if (space_subdomain := space_info.subdomain) is None: + raise CLIError("Unexpected None subdomain on hotReloaded Space") + + def render_region(region: ReloadRegion) -> str: + res = f"{local_path}, " + if region["startLine"] == region["endLine"]: + res += f"line {region['startLine'] - 1}" + else: + res += f"lines {region['startLine'] - 1}-{region['endLine']}" + return res + + def display_event(event: ApiGetReloadEventSourceData) -> None: + if event["data"]["kind"] == "error": + click.secho("✘ Unexpected hot-reloading error", bold=True) + click.secho(event["data"]["traceback"], italic=True) + elif event["data"]["kind"] == "exception": + click.secho(f"✘ Exception at {render_region(event['data']['region'])}", bold=True) + click.secho(event["data"]["traceback"], italic=True) + elif event["data"]["kind"] == "add": + click.secho(f"✔︎ Created {event['data']['objectName']} {event['data']['objectType']}", bold=True) + elif event["data"]["kind"] == "delete": + click.secho(f"∅ Deleted {event['data']['objectName']} {event['data']['objectType']}", bold=True) + elif event["data"]["kind"] == "update": + click.secho(f"✔︎ Updated {event['data']['objectName']} {event['data']['objectType']}", bold=True) + elif event["data"]["kind"] == "run": + click.secho(f"▶ Run {render_region(event['data']['region'])}", bold=True) + click.secho(event["data"]["codeLines"], italic=True) + elif event["data"]["kind"] == "ui": + if event["data"]["updated"]: + click.secho("⟳ UI updated", bold=True) + else: + click.secho("∅ UI untouched", bold=True) + elif event["data"]["kind"] == "file": + if event["data"]["created"]: + click.secho(f"✔︎ {filename} created", bold=True) + else: + click.secho(f"✔︎ {filename} updated", bold=True) + else: + click.secho(f"❓ Unknown update event: {event=}") + if TYPE_CHECKING: + assert_never(event["data"]["kind"]) + + for replica_stream_event in multi_replica_reload_events( + commit_sha=commit_sha, + host=space_host, + subdomain=space_subdomain, + replica_hashes=[hash for hash, _ in hot_reloading.replica_statuses], + token=token, + ): + if replica_stream_event["kind"] == "event": + display_event(replica_stream_event["event"]) + elif replica_stream_event["kind"] == "replicaHash": + click.secho(f"---- Replica {replica_stream_event['hash']} ----") + elif replica_stream_event["kind"] == "fullMatch": + click.echo("✔︎ Same as first replica") + elif replica_stream_event["kind"] == "warning": + click.secho(f"⚠ {replica_stream_event['message']}", fg="bright_black") + else: + assert_never(replica_stream_event) + + +PREFERRED_EDITORS = ( + ("code", "code --wait"), + ("nvim", "nvim"), + ("nano", "nano"), + ("vim", "vim"), + ("vi", "vi"), +) + + +@functools.cache +def _get_editor_command() -> str | None: + for env in ("HF_EDITOR", "VISUAL", "EDITOR"): + if command := os.getenv(env, "").strip(): + return command + for binary_path, editor_command in PREFERRED_EDITORS: + if shutil.which(binary_path) is not None: + return editor_command + return None + + +def _editor_open(local_path: str) -> int | Literal["no-tty", "no-editor"]: + if not (sys.stdin.isatty() and sys.stdout.isatty()): + return "no-tty" + if (editor_command := _get_editor_command()) is None: + return "no-editor" + command = [*shlex.split(editor_command), local_path] + res = subprocess.run(command, start_new_session=True) + return res.returncode + + +@volumes_cli.command( + "list | ls", + examples=[ + "hf spaces volumes ls username/my-space", + ], +) +def volumes_ls( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + token: TokenOpt = None, +) -> None: + """List volumes mounted in a Space.""" + api = get_hf_api(token=token) + info = api.space_info(space_id) + if info.runtime is None: + raise CLIError(f"Runtime not available for Space '{space_id}'.") + volumes = info.runtime.volumes or [] + items = [_dataclass_to_dict(v) for v in volumes] + out.table(items) + out.hint( + f"Use `hf spaces volumes set {space_id} -v hf:///:/` to set volumes for a Space." + ) + + +@volumes_cli.command( + "set", + examples=[ + "hf spaces volumes set username/my-space -v hf://models/username/my-model:/models", + "hf spaces volumes set username/my-space -v hf://buckets/username/my-bucket:/data -v hf://datasets/username/my-dataset:/datasets:ro", + ], +) +def volumes_set( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + volume: VolumesOpt = None, + token: TokenOpt = None, +) -> None: + """Set (replace) volumes for a Space.""" + volumes = parse_volumes(volume) + if not volumes: + raise CLIError("At least one volume must be specified with -v/--volume.") + api = get_hf_api(token=token) + api.set_space_volumes(space_id, volumes=volumes) + out.result("Volumes set", space_id=space_id, volumes=[v.to_uri() for v in volumes]) + out.hint(f"Use `hf spaces volumes ls {space_id}` to list volumes for a Space.") + + +@volumes_cli.command( + "delete", + examples=[ + "hf spaces volumes delete username/my-space", + "hf spaces volumes delete username/my-space --yes", + ], +) +def volumes_delete( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + yes: Annotated[ + bool, + Option( + "-y", + "--yes", + help="Answer Yes to prompt automatically.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + """Remove all volumes from a Space.""" + out.confirm(f"You are about to remove all volumes from Space '{space_id}'. Proceed?", yes=yes) + api = get_hf_api(token=token) + api.delete_space_volumes(space_id) + out.result("Volumes deleted", space_id=space_id) + out.hint( + f"Use `hf spaces volumes set {space_id} -v hf:///:/` to set volumes for a Space." + ) + + +@secrets_cli.command( + "list | ls", + examples=["hf spaces secrets ls username/my-space"], +) +def secrets_ls( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + token: TokenOpt = None, +) -> None: + """List secrets for a Space. Secret values are write-only and not returned.""" + api = get_hf_api(token=token) + secrets = api.get_space_secrets(space_id) + items = [_dataclass_to_dict(s) for s in secrets.values()] + out.table(items) + out.hint(f"Use `hf spaces secrets add {space_id} -s KEY=VALUE` to add secrets to a Space.") + + +@secrets_cli.command( + "add", + examples=[ + "hf spaces secrets add username/my-space -s HF_TOKEN", + "hf spaces secrets add username/my-space -s OPENAI_API_KEY=sk-... -s ANTHROPIC_API_KEY=sk-...", + "hf spaces secrets add username/my-space --secrets-file .env.secrets", + ], +) +def secrets_add( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + secrets: SecretsOpt = None, + secrets_file: SecretsFileOpt = None, + token: TokenOpt = None, +) -> None: + """Add or update secrets for a Space.""" + secrets_map = parse_env_map(secrets, secrets_file) + if not secrets_map: + raise CLIError("At least one secret must be specified with -s/--secrets or --secrets-file.") + api = get_hf_api(token=token) + for key, value in secrets_map.items(): + api.add_space_secret(space_id, key=key, value=value or "") + out.result("Secrets added", space_id=space_id, keys=list(secrets_map)) + out.hint(f"Use `hf spaces secrets delete {space_id} ` to remove a secret from a Space.") + + +@secrets_cli.command( + "delete", + examples=[ + "hf spaces secrets delete username/my-space HF_TOKEN", + "hf spaces secrets delete username/my-space HF_TOKEN --yes", + ], +) +def secrets_delete( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + key: Annotated[str, Argument(help="Name of the secret to remove.")], + yes: Annotated[ + bool, + Option( + "-y", + "--yes", + help="Answer Yes to prompt automatically.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + """Remove a secret from a Space.""" + out.confirm( + f"You are about to remove secret '{key}' from Space '{space_id}'. The value cannot be recovered. Proceed?", + yes=yes, + ) + api = get_hf_api(token=token) + api.delete_space_secret(space_id, key=key) + out.result("Secret deleted", space_id=space_id, key=key) + out.hint(f"Use `hf spaces secrets add {space_id} -s {key}=` to re-add a secret to a Space.") + + +@variables_cli.command( + "list | ls", + examples=["hf spaces variables ls username/my-space"], +) +def variables_ls( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + token: TokenOpt = None, +) -> None: + """List environment variables for a Space.""" + api = get_hf_api(token=token) + variables = api.get_space_variables(space_id) + items = [_dataclass_to_dict(v) for v in variables.values()] + out.table(items) + out.hint(f"Use `hf spaces variables add {space_id} -e KEY=VALUE` to add variables to a Space.") + + +@variables_cli.command( + "add", + examples=[ + "hf spaces variables add username/my-space -e DEBUG=1", + "hf spaces variables add username/my-space -e MODEL_ID=gpt2 -e MAX_TOKENS=512", + "hf spaces variables add username/my-space --env-file .env", + ], +) +def variables_add( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + env: EnvOpt = None, + env_file: EnvFileOpt = None, + token: TokenOpt = None, +) -> None: + """Add or update environment variables for a Space.""" + env_map = parse_env_map(env, env_file) + if not env_map: + raise CLIError("At least one variable must be specified with -e/--env or --env-file.") + api = get_hf_api(token=token) + for key, value in env_map.items(): + api.add_space_variable(space_id, key=key, value=value or "") + out.result("Variables added", space_id=space_id, keys=list(env_map)) + out.hint(f"Use `hf spaces variables ls {space_id}` to list variables for a Space.") + + +@variables_cli.command( + "delete", + examples=[ + "hf spaces variables delete username/my-space DEBUG", + "hf spaces variables delete username/my-space DEBUG --yes", + ], +) +def variables_delete( + space_id: Annotated[str, Argument(help="The space ID (e.g. `username/repo-name`).")], + key: Annotated[str, Argument(help="Name of the variable to remove.")], + yes: Annotated[ + bool, + Option( + "-y", + "--yes", + help="Answer Yes to prompt automatically.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + """Remove an environment variable from a Space.""" + out.confirm( + f"You are about to remove variable '{key}' from Space '{space_id}'. Proceed?", + yes=yes, + ) + api = get_hf_api(token=token) + api.delete_space_variable(space_id, key=key) + out.result("Variable deleted", space_id=space_id, key=key) + out.hint(f"Use `hf spaces variables ls {space_id}` to list remaining variables for a Space.") diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/system.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/system.py new file mode 100644 index 0000000000000000000000000000000000000000..b2bf84a133044328b6b7632c5e1262e225c5e1e6 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/system.py @@ -0,0 +1,50 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to print information about the environment and version.""" + +import click + +from huggingface_hub import __version__ + +from ..utils import dump_environment_info +from ._cli_utils import _fetch_latest_pypi_version, run_update +from ._output import out + + +def env() -> None: + """Print information about the environment.""" + dump_environment_info() + + +def version() -> None: + """Print information about the hf version.""" + out.result("hf version", version=__version__) + + +def update() -> None: + """Update the `hf` CLI to the latest version.""" + out.text(f"Current version: {__version__}") + out.text("Checking for updates to latest version...") + latest_version = _fetch_latest_pypi_version("huggingface_hub") + if latest_version is not None and __version__ == latest_version: + out.text(f"hf is up to date ({__version__})") + return + + returncode = run_update() + if returncode != 0: + raise click.exceptions.Exit(code=returncode) + out.hint( + "You may also want to run `hf skills update` to refresh any installed skills " + "so your AI agent sees the latest command surface." + ) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/upload.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/upload.py new file mode 100644 index 0000000000000000000000000000000000000000..0eda15e098d153308197ac3fa0cb175203529d60 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/upload.py @@ -0,0 +1,279 @@ +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to upload a repo or file with the CLI.""" + +import os +import time +import warnings +from typing import Annotated + +import click + +from huggingface_hub import constants, logging +from huggingface_hub._commit_scheduler import CommitScheduler +from huggingface_hub.errors import CLIError, RevisionNotFoundError +from huggingface_hub.utils import parse_hf_uri + +from ._cli_utils import ( + PrivateOpt, + RepoIdArg, + RepoType, + RepoTypeOptionalOpt, + RevisionOpt, + TokenOpt, + get_hf_api, +) +from ._framework import Argument, Option +from ._output import out + + +logger = logging.get_logger(__name__) + + +UPLOAD_EXAMPLES = [ + "hf upload my-cool-model . .", + "hf upload Wauplin/my-cool-model ./models/model.safetensors", + "hf upload Wauplin/my-cool-dataset ./data /train --repo-type=dataset", + 'hf upload Wauplin/my-cool-model ./models . --commit-message="Epoch 34/50" --commit-description="Val accuracy: 68%"', + "hf upload bigcode/the-stack . . --repo-type dataset --create-pr", +] + + +def upload( + repo_id: RepoIdArg, + local_path: Annotated[ + str | None, + Argument( + help="Local path to the file or folder to upload. Wildcard patterns are supported. Defaults to current directory.", + ), + ] = None, + path_in_repo: Annotated[ + str | None, + Argument( + help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.", + ), + ] = None, + repo_type: RepoTypeOptionalOpt = None, + revision: RevisionOpt = None, + private: PrivateOpt = None, + include: Annotated[ + list[str] | None, + Option( + help="Glob patterns to match files to upload.", + ), + ] = None, + exclude: Annotated[ + list[str] | None, + Option( + help="Glob patterns to exclude from files to upload.", + ), + ] = None, + delete: Annotated[ + list[str] | None, + Option( + help="Glob patterns for file to be deleted from the repo while committing.", + ), + ] = None, + commit_message: Annotated[ + str | None, + Option( + help="The summary / title / first line of the generated commit.", + ), + ] = None, + commit_description: Annotated[ + str | None, + Option( + help="The description of the generated commit.", + ), + ] = None, + create_pr: Annotated[ + bool, + Option( + help="Whether to upload content as a new Pull Request.", + ), + ] = False, + every: Annotated[ + float | None, + Option( + help="If set, a background job is scheduled to create commits every `every` minutes.", + ), + ] = None, + token: TokenOpt = None, +) -> None: + """Upload a file or a folder to the Hub. Recommended for single-commit uploads.""" + + if every is not None and every <= 0: + raise click.BadParameter("--every must be a positive value", param_hint="every") + + # `repo_id` may be a plain repo id or an `hf://` URI (e.g. `hf://datasets/my-org/my-dataset@v1.0/data/`). + # When a URI is provided, it is authoritative for the repo type, revision and (optionally) path in repo, + # so explicit `--repo-type` / `--revision` options are forbidden alongside it. + # We branch on the `hf://` prefix (the user's *intent*) rather than on whether the string parses as a + # valid URI: a malformed URI then surfaces a precise `HfUriError` (formatted globally in `cli/_errors.py`) + # instead of silently falling through to the plain-repo-id path and failing later with an opaque error. + if repo_id.startswith(constants.HF_PROTOCOL): + if repo_type is not None: + raise CLIError(f"'--repo-type' cannot be used with an 'hf://' URI ('{repo_id}').") + if revision is not None: + raise CLIError(f"'--revision' cannot be used with an 'hf://' URI ('{repo_id}').") + uri = parse_hf_uri(repo_id) + if uri.is_bucket: + raise CLIError("Buckets are not supported by `hf upload`. Use `hf sync` instead.") + repo_id, repo_type_str, revision = uri.id, uri.type, uri.revision + if uri.path_in_repo: + if path_in_repo is not None: + raise CLIError( + f"Cannot combine a path in the hf:// URI ('{uri.path_in_repo}') with the `path_in_repo` argument ('{path_in_repo}')." + ) + path_in_repo = uri.path_in_repo + else: + repo_type_str = (repo_type or RepoType.model).value + + api = get_hf_api(token=token) + + # Resolve local_path and path_in_repo based on implicit/explicit rules + resolved_local_path, resolved_path_in_repo, resolved_include = _resolve_upload_paths( + repo_id=repo_id, local_path=local_path, path_in_repo=path_in_repo, include=include + ) + + def run_upload() -> str: + if os.path.isfile(resolved_local_path): + if resolved_include is not None and len(resolved_include) > 0 and isinstance(resolved_include, list): + warnings.warn("Ignoring --include since a single file is uploaded.") + if exclude is not None and len(exclude) > 0: + warnings.warn("Ignoring --exclude since a single file is uploaded.") + if delete is not None and len(delete) > 0: + warnings.warn("Ignoring --delete since a single file is uploaded.") + + # Schedule commits if `every` is set + if every is not None: + allow_patterns: list[str] | None + ignore_patterns: list[str] | None + if os.path.isfile(resolved_local_path): + # If file => watch entire folder + use allow_patterns + folder_path = os.path.dirname(resolved_local_path) + pi = ( + resolved_path_in_repo[: -len(resolved_local_path)] + if resolved_path_in_repo.endswith(resolved_local_path) + else resolved_path_in_repo + ) + allow_patterns = [resolved_local_path] + ignore_patterns = [] + else: + folder_path = resolved_local_path + pi = resolved_path_in_repo + allow_patterns = resolved_include + ignore_patterns = exclude + if delete is not None and len(delete) > 0: + warnings.warn("Ignoring --delete when uploading with scheduled commits.") + + scheduler = CommitScheduler( + folder_path=folder_path, + repo_id=repo_id, + repo_type=repo_type_str, + revision=revision, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + path_in_repo=pi, + private=private, + every=every, + hf_api=api, + ) + out.text(f"Scheduling commits every {every} minutes to {scheduler.repo_id}.") + try: + while True: + time.sleep(100) + except KeyboardInterrupt: + scheduler.stop() + return "Stopped scheduled commits." + + # Otherwise, create repo and proceed with the upload + if not os.path.isfile(resolved_local_path) and not os.path.isdir(resolved_local_path): + raise FileNotFoundError(f"No such file or directory: '{resolved_local_path}'.") + created = api.create_repo( + repo_id=repo_id, + repo_type=repo_type_str, + exist_ok=True, + private=private, + space_sdk="gradio" if repo_type_str == "space" else None, + # ^ We don't want it to fail when uploading to a Space => let's set Gradio by default. + # ^ I'd rather not add CLI args to set it explicitly as we already have `hf repos create` for that. + ).repo_id + + # Check if branch already exists and if not, create it + if revision is not None and not create_pr: + try: + api.repo_info(repo_id=created, repo_type=repo_type_str, revision=revision) + except RevisionNotFoundError: + logger.info(f"Branch '{revision}' not found. Creating it...") + api.create_branch(repo_id=created, repo_type=repo_type_str, branch=revision, exist_ok=True) + # ^ `exist_ok=True` to avoid race concurrency issues + + # File-based upload + if os.path.isfile(resolved_local_path): + return api.upload_file( + path_or_fileobj=resolved_local_path, + path_in_repo=resolved_path_in_repo, + repo_id=created, + repo_type=repo_type_str, + revision=revision, + commit_message=commit_message, + commit_description=commit_description, + create_pr=create_pr, + ) + + # Folder-based upload + return api.upload_folder( + folder_path=resolved_local_path, + path_in_repo=resolved_path_in_repo, + repo_id=created, + repo_type=repo_type_str, + revision=revision, + commit_message=commit_message, + commit_description=commit_description, + create_pr=create_pr, + allow_patterns=resolved_include, + ignore_patterns=exclude, + delete_patterns=delete, + ) + + result = run_upload() + out.result("Uploaded", url=result) + + +def _resolve_upload_paths( + *, repo_id: str, local_path: str | None, path_in_repo: str | None, include: list[str] | None +) -> tuple[str, str, list[str] | None]: + repo_name = repo_id.split("/")[-1] + resolved_include = include + + if local_path is not None and any(c in local_path for c in ["*", "?", "["]): + if include is not None: + raise ValueError("Cannot set --include when local_path contains a wildcard.") + if path_in_repo is not None and path_in_repo != ".": + raise ValueError("Cannot set path_in_repo when local_path contains a wildcard.") + return ".", local_path, ["."] # will be adjusted below; placeholder for type + + if local_path is None and os.path.isfile(repo_name): + return repo_name, repo_name, resolved_include + if local_path is None and os.path.isdir(repo_name): + return repo_name, ".", resolved_include + if local_path is None: + raise ValueError(f"'{repo_name}' is not a local file or folder. Please set local_path explicitly.") + + if path_in_repo is None and os.path.isfile(local_path): + return local_path, os.path.basename(local_path), resolved_include + if path_in_repo is None: + return local_path, ".", resolved_include + return local_path, path_in_repo, resolved_include diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/upload_large_folder.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/upload_large_folder.py new file mode 100644 index 0000000000000000000000000000000000000000..d9836754569cdd0b825ee69063625c0a963436ff --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/upload_large_folder.py @@ -0,0 +1,133 @@ +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to upload a large folder with the CLI.""" + +import os +import warnings +from typing import Annotated + +import click + +from huggingface_hub import logging +from huggingface_hub.utils import disable_progress_bars + +from ._cli_utils import ( + PrivateOpt, + RepoIdArg, + RepoType, + RepoTypeOpt, + RevisionOpt, + TokenOpt, + get_hf_api, +) +from ._framework import Argument, Option +from ._output import out + + +logger = logging.get_logger(__name__) + + +UPLOAD_LARGE_FOLDER_EXAMPLES = [ + "hf upload-large-folder Wauplin/my-cool-model ./large_model_dir", + "hf upload-large-folder Wauplin/my-cool-model ./large_model_dir --revision v1.0", +] + + +def upload_large_folder( + repo_id: RepoIdArg, + local_path: Annotated[ + str, + Argument( + help="Local path to the folder to upload.", + ), + ], + repo_type: RepoTypeOpt = RepoType.model, + revision: RevisionOpt = None, + private: PrivateOpt = None, + include: Annotated[ + list[str] | None, + Option( + help="Glob patterns to match files to upload.", + ), + ] = None, + exclude: Annotated[ + list[str] | None, + Option( + help="Glob patterns to exclude from files to upload.", + ), + ] = None, + token: TokenOpt = None, + num_workers: Annotated[ + int | None, + Option( + help="Number of workers to use to hash, upload and commit files.", + ), + ] = None, + no_report: Annotated[ + bool, + Option( + help="Whether to disable regular status report.", + ), + ] = False, + no_bars: Annotated[ + bool, + Option( + help="Whether to disable progress bars.", + ), + ] = False, +) -> None: + """[Deprecated] Upload a large folder to the Hub. Use `hf upload` instead.""" + if not os.path.isdir(local_path): + raise click.BadParameter("Large upload is only supported for folders.", param_hint="local_path") + + # Build the equivalent `hf upload` command to recommend to the user. + equivalent = [f"hf upload {repo_id} '{local_path}' --repo-type {repo_type.value}"] + if revision is not None: + equivalent.append(f"--revision '{revision}'") + if private: + equivalent.append("--private") + for pattern in include or []: + equivalent.append(f"--include '{pattern}'") + for pattern in exclude or []: + equivalent.append(f"--exclude '{pattern}'") + + out.warning( + "\n" + "================================================================================\n" + "`hf upload-large-folder` is DEPRECATED and will be removed in a future release.\n" + "\n" + "Use `hf upload` instead:\n" + "\n" + f" {' '.join(equivalent)}\n" + "================================================================================" + ) + + if no_bars: + disable_progress_bars() + + api = get_hf_api(token=token) + with warnings.catch_warnings(): + # Avoid printing the API-level deprecation warning on top of the CLI one above. + warnings.simplefilter("ignore", FutureWarning) + api.upload_large_folder( + repo_id=repo_id, + folder_path=local_path, + repo_type=repo_type.value, + revision=revision, + private=private, + allow_patterns=include, + ignore_patterns=exclude, + num_workers=num_workers, + print_report=not no_report, + ) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/cli/webhooks.py b/venv/lib/python3.12/site-packages/huggingface_hub/cli/webhooks.py new file mode 100644 index 0000000000000000000000000000000000000000..ddec817f1345edd86745d9a41470e22da4acfe4f --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/cli/webhooks.py @@ -0,0 +1,269 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to manage webhooks on the Hugging Face Hub.""" + +import enum +from typing import Annotated, get_args, get_type_hints + +import click + +from huggingface_hub.constants import WEBHOOK_DOMAIN_T +from huggingface_hub.hf_api import WebhookWatchedItem + +from ._cli_utils import ( + TokenOpt, + get_hf_api, + typer_factory, +) +from ._framework import Argument, Option +from ._output import out + + +# Build enums dynamically from Literal types to avoid duplication +_WATCHED_TYPES = get_args(get_type_hints(WebhookWatchedItem)["type"]) +WatchedItemType = enum.Enum("WatchedItemType", {t: t for t in _WATCHED_TYPES}, type=str) # type: ignore[misc] + +_DOMAIN_TYPES = get_args(WEBHOOK_DOMAIN_T) +WebhookDomain = enum.Enum("WebhookDomain", {d: d for d in _DOMAIN_TYPES}, type=str) # type: ignore[misc] + + +def _parse_watch(values: list[str]) -> list[WebhookWatchedItem]: + """Parse 'type:name' strings into WebhookWatchedItem objects. + + Args: + values: List of strings in the format 'type:name' + (e.g., 'model:bert-base-uncased', 'org:HuggingFace'). + + Returns: + List of WebhookWatchedItem objects. + + Raises: + click.BadParameter: If any value doesn't match the expected format. + """ + items = [] + valid_types = tuple(_WATCHED_TYPES) + for v in values: + if ":" not in v: + raise click.BadParameter( + f"Expected format 'type:name' (e.g. 'model:bert-base-uncased'), got '{v}'." + f" Valid types: {', '.join(valid_types)}." + ) + kind, name = v.split(":", 1) + if kind not in valid_types: + raise click.BadParameter(f"Invalid type '{kind}'. Valid types: {', '.join(valid_types)}.") + items.append(WebhookWatchedItem(type=kind, name=name)) # type: ignore + return items + + +webhooks_cli = typer_factory(help="Manage webhooks on the Hub.") + + +@webhooks_cli.command( + "list | ls", + examples=[ + "hf webhooks ls", + "hf webhooks ls --format json", + "hf webhooks ls --format quiet", + ], +) +def webhooks_ls( + token: TokenOpt = None, +) -> None: + """List all webhooks for the current user.""" + api = get_hf_api(token=token) + results = [ + { + "id": w.id, + "url": w.url or "(job)", + "disabled": w.disabled, + "domains": w.domains or [], + "watched": [f"{wi.type}:{wi.name}" for wi in (w.watched or [])], + } + for w in api.list_webhooks() + ] + out.table(results) + + +@webhooks_cli.command( + "info", + examples=[ + "hf webhooks info abc123", + ], +) +def webhooks_info( + webhook_id: Annotated[str, Argument(help="The ID of the webhook.")], + token: TokenOpt = None, +) -> None: + """Show full details for a single webhook.""" + api = get_hf_api(token=token) + webhook = api.get_webhook(webhook_id) + out.dict(webhook) + + +@webhooks_cli.command( + "create", + examples=[ + "hf webhooks create --url https://example.com/hook --watch model:bert-base-uncased", + "hf webhooks create --url https://example.com/hook --watch org:HuggingFace --watch model:gpt2 --domain repo", + "hf webhooks create --job-id 687f911eaea852de79c4a50a --watch user:julien-c", + ], +) +def webhooks_create( + watch: Annotated[ + list[str], + Option( + "--watch", + help="Item to watch, in 'type:name' format (e.g. 'model:bert-base-uncased'). Repeatable.", + ), + ], + url: Annotated[ + str | None, + Option(help="URL to send webhook payloads to. Mutually exclusive with --job-id."), + ] = None, + job_id: Annotated[ + str | None, + Option( + "--job-id", + help="ID of a Job to trigger (from job.id) instead of pinging a URL. Mutually exclusive with --url.", + ), + ] = None, + domain: Annotated[ + list[WebhookDomain] | None, + Option( + "--domain", + help="Domain to watch: 'repo' or 'discussions'. Repeatable. Defaults to all domains.", + ), + ] = None, + secret: Annotated[ + str | None, + Option(help="Optional secret used to sign webhook payloads."), + ] = None, + token: TokenOpt = None, +) -> None: + """Create a new webhook. + + Provide either --url (to ping a remote server) or --job-id (to trigger a Job), but not both. + """ + if url is not None and job_id is not None: + raise click.BadParameter("Provide either --url or --job-id, not both.") + if url is None and job_id is None: + raise click.BadParameter("Provide either --url or --job-id.") + api = get_hf_api(token=token) + watched_items = _parse_watch(watch) + domains = [d.value for d in domain] if domain else None + webhook = api.create_webhook(url=url, job_id=job_id, watched=watched_items, domains=domains, secret=secret) # type: ignore + out.result("Webhook created", id=webhook.id) + + +@webhooks_cli.command( + "update", + examples=[ + "hf webhooks update abc123 --url https://new-url.com/hook", + "hf webhooks update abc123 --watch model:gpt2 --domain repo", + "hf webhooks update abc123 --secret newsecret", + ], +) +def webhooks_update( + webhook_id: Annotated[str, Argument(help="The ID of the webhook to update.")], + url: Annotated[ + str | None, + Option(help="New URL to send webhook payloads to."), + ] = None, + watch: Annotated[ + list[str] | None, + Option( + "--watch", + help=( + "New list of items to watch, in 'type:name' format. " + "Repeatable. Replaces the entire existing watched list." + ), + ), + ] = None, + domain: Annotated[ + list[WebhookDomain] | None, + Option( + "--domain", + help="New list of domains to watch: 'repo' or 'discussions'. Repeatable.", + ), + ] = None, + secret: Annotated[ + str | None, + Option(help="New secret used to sign webhook payloads."), + ] = None, + token: TokenOpt = None, +) -> None: + """Update an existing webhook. Only provided options are changed.""" + api = get_hf_api(token=token) + watched_items = _parse_watch(watch) if watch else None + domains = [d.value for d in domain] if domain else None + webhook = api.update_webhook(webhook_id, url=url, watched=watched_items, domains=domains, secret=secret) # type: ignore + out.result("Webhook updated", id=webhook.id) + + +@webhooks_cli.command( + "enable", + examples=[ + "hf webhooks enable abc123", + ], +) +def webhooks_enable( + webhook_id: Annotated[str, Argument(help="The ID of the webhook to enable.")], + token: TokenOpt = None, +) -> None: + """Enable a disabled webhook.""" + api = get_hf_api(token=token) + webhook = api.enable_webhook(webhook_id) + out.result("Webhook enabled", id=webhook.id) + + +@webhooks_cli.command( + "disable", + examples=[ + "hf webhooks disable abc123", + ], +) +def webhooks_disable( + webhook_id: Annotated[str, Argument(help="The ID of the webhook to disable.")], + token: TokenOpt = None, +) -> None: + """Disable an active webhook.""" + api = get_hf_api(token=token) + webhook = api.disable_webhook(webhook_id) + out.result("Webhook disabled", id=webhook.id) + + +@webhooks_cli.command( + "delete", + examples=[ + "hf webhooks delete abc123", + "hf webhooks delete abc123 --yes", + ], +) +def webhooks_delete( + webhook_id: Annotated[str, Argument(help="The ID of the webhook to delete.")], + yes: Annotated[ + bool, + Option( + "--yes", + "-y", + help="Skip confirmation prompt.", + ), + ] = False, + token: TokenOpt = None, +) -> None: + """Delete a webhook permanently.""" + out.confirm(f"Are you sure you want to delete webhook '{webhook_id}'?", yes=yes) + api = get_hf_api(token=token) + api.delete_webhook(webhook_id) + out.result("Webhook deleted", id=webhook_id) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/community.py b/venv/lib/python3.12/site-packages/huggingface_hub/community.py new file mode 100644 index 0000000000000000000000000000000000000000..58e64100c48c461bb6d4dfb42230ffbf412bff57 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/community.py @@ -0,0 +1,363 @@ +""" +Data structures to interact with Discussions and Pull Requests on the Hub. + +See [the Discussions and Pull Requests guide](https://huggingface.co/docs/hub/repositories-pull-requests-discussions) +for more information on Pull Requests, Discussions, and the community tab. +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Literal, TypedDict + +from . import constants +from .utils import parse_datetime + + +DiscussionStatus = Literal["open", "closed", "merged", "draft"] + + +@dataclass +class Discussion: + """ + A Discussion or Pull Request on the Hub. + + This dataclass is not intended to be instantiated directly. + + Attributes: + title (`str`): + The title of the Discussion / Pull Request + status (`str`): + The status of the Discussion / Pull Request. + It must be one of: + * `"open"` + * `"closed"` + * `"merged"` (only for Pull Requests ) + * `"draft"` (only for Pull Requests ) + num (`int`): + The number of the Discussion / Pull Request. + repo_id (`str`): + The id (`"{namespace}/{repo_name}"`) of the repo on which + the Discussion / Pull Request was open. + repo_type (`str`): + The type of the repo on which the Discussion / Pull Request was open. + Possible values are: `"model"`, `"dataset"`, `"space"`. + author (`str`): + The username of the Discussion / Pull Request author. + Can be `"deleted"` if the user has been deleted since. + is_pull_request (`bool`): + Whether or not this is a Pull Request. + created_at (`datetime`): + The `datetime` of creation of the Discussion / Pull Request. + endpoint (`str`): + Endpoint of the Hub. Default is https://huggingface.co. + git_reference (`str`, *optional*): + (property) Git reference to which changes can be pushed if this is a Pull Request, `None` otherwise. + url (`str`): + (property) URL of the discussion on the Hub. + """ + + title: str + status: DiscussionStatus + num: int + repo_id: str + repo_type: str + author: str + is_pull_request: bool + created_at: datetime + endpoint: str + + @property + def git_reference(self) -> str | None: + """ + If this is a Pull Request , returns the git reference to which changes can be pushed. + Returns `None` otherwise. + """ + if self.is_pull_request: + return f"refs/pr/{self.num}" + return None + + @property + def url(self) -> str: + """Returns the URL of the discussion on the Hub.""" + if self.repo_type is None or self.repo_type == constants.REPO_TYPE_MODEL: + return f"{self.endpoint}/{self.repo_id}/discussions/{self.num}" + return f"{self.endpoint}/{self.repo_type}s/{self.repo_id}/discussions/{self.num}" + + +@dataclass +class DiscussionWithDetails(Discussion): + """ + Subclass of [`Discussion`]. + + Attributes: + title (`str`): + The title of the Discussion / Pull Request + status (`str`): + The status of the Discussion / Pull Request. + It can be one of: + * `"open"` + * `"closed"` + * `"merged"` (only for Pull Requests ) + * `"draft"` (only for Pull Requests ) + num (`int`): + The number of the Discussion / Pull Request. + repo_id (`str`): + The id (`"{namespace}/{repo_name}"`) of the repo on which + the Discussion / Pull Request was open. + repo_type (`str`): + The type of the repo on which the Discussion / Pull Request was open. + Possible values are: `"model"`, `"dataset"`, `"space"`. + author (`str`): + The username of the Discussion / Pull Request author. + Can be `"deleted"` if the user has been deleted since. + is_pull_request (`bool`): + Whether or not this is a Pull Request. + created_at (`datetime`): + The `datetime` of creation of the Discussion / Pull Request. + events (`list` of [`DiscussionEvent`]) + The list of [`DiscussionEvents`] in this Discussion or Pull Request. + conflicting_files (`Union[list[str], bool, None]`, *optional*): + A list of conflicting files if this is a Pull Request. + `None` if `self.is_pull_request` is `False`. + `True` if there are conflicting files but the list can't be retrieved. + target_branch (`str`, *optional*): + The branch into which changes are to be merged if this is a + Pull Request . `None` if `self.is_pull_request` is `False`. + merge_commit_oid (`str`, *optional*): + If this is a merged Pull Request , this is set to the OID / SHA of + the merge commit, `None` otherwise. + diff (`str`, *optional*): + The git diff if this is a Pull Request , `None` otherwise. + endpoint (`str`): + Endpoint of the Hub. Default is https://huggingface.co. + git_reference (`str`, *optional*): + (property) Git reference to which changes can be pushed if this is a Pull Request, `None` otherwise. + url (`str`): + (property) URL of the discussion on the Hub. + """ + + events: list["DiscussionEvent"] + conflicting_files: list[str] | bool | None + target_branch: str | None + merge_commit_oid: str | None + diff: str | None + + +class DiscussionEventArgs(TypedDict): + id: str + type: str + created_at: datetime + author: str + _event: dict + + +@dataclass +class DiscussionEvent: + """ + An event in a Discussion or Pull Request. + + Use concrete classes: + * [`DiscussionComment`] + * [`DiscussionStatusChange`] + * [`DiscussionCommit`] + * [`DiscussionTitleChange`] + + Attributes: + id (`str`): + The ID of the event. An hexadecimal string. + type (`str`): + The type of the event. + created_at (`datetime`): + A [`datetime`](https://docs.python.org/3/library/datetime.html?highlight=datetime#datetime.datetime) + object holding the creation timestamp for the event. + author (`str`): + The username of the Discussion / Pull Request author. + Can be `"deleted"` if the user has been deleted since. + """ + + id: str + type: str + created_at: datetime + author: str + + _event: dict + """Stores the original event data, in case we need to access it later.""" + + +@dataclass +class DiscussionComment(DiscussionEvent): + """A comment in a Discussion / Pull Request. + + Subclass of [`DiscussionEvent`]. + + + Attributes: + id (`str`): + The ID of the event. An hexadecimal string. + type (`str`): + The type of the event. + created_at (`datetime`): + A [`datetime`](https://docs.python.org/3/library/datetime.html?highlight=datetime#datetime.datetime) + object holding the creation timestamp for the event. + author (`str`): + The username of the Discussion / Pull Request author. + Can be `"deleted"` if the user has been deleted since. + content (`str`): + The raw markdown content of the comment. Mentions, links and images are not rendered. + edited (`bool`): + Whether or not this comment has been edited. + hidden (`bool`): + Whether or not this comment has been hidden. + """ + + content: str + edited: bool + hidden: bool + + @property + def rendered(self) -> str: + """The rendered comment, as a HTML string""" + return self._event["data"]["latest"]["html"] + + @property + def last_edited_at(self) -> datetime: + """The last edit time, as a `datetime` object.""" + return parse_datetime(self._event["data"]["latest"]["updatedAt"]) + + @property + def last_edited_by(self) -> str: + """The last edit time, as a `datetime` object.""" + return self._event["data"]["latest"].get("author", {}).get("name", "deleted") + + @property + def edit_history(self) -> list[dict]: + """The edit history of the comment""" + return self._event["data"]["history"] + + @property + def number_of_edits(self) -> int: + return len(self.edit_history) + + +@dataclass +class DiscussionStatusChange(DiscussionEvent): + """A change of status in a Discussion / Pull Request. + + Subclass of [`DiscussionEvent`]. + + Attributes: + id (`str`): + The ID of the event. An hexadecimal string. + type (`str`): + The type of the event. + created_at (`datetime`): + A [`datetime`](https://docs.python.org/3/library/datetime.html?highlight=datetime#datetime.datetime) + object holding the creation timestamp for the event. + author (`str`): + The username of the Discussion / Pull Request author. + Can be `"deleted"` if the user has been deleted since. + new_status (`str`): + The status of the Discussion / Pull Request after the change. + It can be one of: + * `"open"` + * `"closed"` + * `"merged"` (only for Pull Requests ) + """ + + new_status: str + + +@dataclass +class DiscussionCommit(DiscussionEvent): + """A commit in a Pull Request. + + Subclass of [`DiscussionEvent`]. + + Attributes: + id (`str`): + The ID of the event. An hexadecimal string. + type (`str`): + The type of the event. + created_at (`datetime`): + A [`datetime`](https://docs.python.org/3/library/datetime.html?highlight=datetime#datetime.datetime) + object holding the creation timestamp for the event. + author (`str`): + The username of the Discussion / Pull Request author. + Can be `"deleted"` if the user has been deleted since. + summary (`str`): + The summary of the commit. + oid (`str`): + The OID / SHA of the commit, as a hexadecimal string. + """ + + summary: str + oid: str + + +@dataclass +class DiscussionTitleChange(DiscussionEvent): + """A rename event in a Discussion / Pull Request. + + Subclass of [`DiscussionEvent`]. + + Attributes: + id (`str`): + The ID of the event. An hexadecimal string. + type (`str`): + The type of the event. + created_at (`datetime`): + A [`datetime`](https://docs.python.org/3/library/datetime.html?highlight=datetime#datetime.datetime) + object holding the creation timestamp for the event. + author (`str`): + The username of the Discussion / Pull Request author. + Can be `"deleted"` if the user has been deleted since. + old_title (`str`): + The previous title for the Discussion / Pull Request. + new_title (`str`): + The new title. + """ + + old_title: str + new_title: str + + +def deserialize_event(event: dict) -> DiscussionEvent: + """Instantiates a [`DiscussionEvent`] from a dict""" + event_id: str = event["id"] + event_type: str = event["type"] + created_at = parse_datetime(event["createdAt"]) + + common_args: DiscussionEventArgs = { + "id": event_id, + "type": event_type, + "created_at": created_at, + "author": event.get("author", {}).get("name", "deleted"), + "_event": event, + } + + if event_type == "comment": + return DiscussionComment( + **common_args, + edited=event["data"]["edited"], + hidden=event["data"]["hidden"], + content=event["data"]["latest"]["raw"], + ) + if event_type == "status-change": + return DiscussionStatusChange( + **common_args, + new_status=event["data"]["status"], + ) + if event_type == "commit": + return DiscussionCommit( + **common_args, + summary=event["data"]["subject"], + oid=event["data"]["oid"], + ) + if event_type == "title-change": + return DiscussionTitleChange( + **common_args, + old_title=event["data"]["from"], + new_title=event["data"]["to"], + ) + + return DiscussionEvent(**common_args) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/constants.py b/venv/lib/python3.12/site-packages/huggingface_hub/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..caed41d025bd7352dddc786df481dcb0c05a0cda --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/constants.py @@ -0,0 +1,342 @@ +import os +import re +import typing +from typing import Literal +from urllib.parse import urlsplit + + +# Possible values for env variables + + +ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"} +ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"}) + + +def _is_true(value: str | None) -> bool: + if value is None: + return False + return value.upper() in ENV_VARS_TRUE_VALUES + + +def _as_int(value: str | None) -> int | None: + if value is None: + return None + return int(value) + + +# Constants for file downloads + +PYTORCH_WEIGHTS_NAME = "pytorch_model.bin" +TF2_WEIGHTS_NAME = "tf_model.h5" +TF_WEIGHTS_NAME = "model.ckpt" +FLAX_WEIGHTS_NAME = "flax_model.msgpack" +CONFIG_NAME = "config.json" +REPOCARD_NAME = "README.md" +EVAL_RESULTS_FOLDER = ".eval_results" +DEFAULT_ETAG_TIMEOUT = 10 +DEFAULT_DOWNLOAD_TIMEOUT = 10 +DEFAULT_REQUEST_TIMEOUT = 10 +DOWNLOAD_CHUNK_SIZE = 10 * 1024 * 1024 +MAX_HTTP_DOWNLOAD_SIZE = 50 * 1000 * 1000 * 1000 # 50 GB + +# Constants for serialization + +PYTORCH_WEIGHTS_FILE_PATTERN = "pytorch_model{suffix}.bin" # Unsafe pickle: use safetensors instead +SAFETENSORS_WEIGHTS_FILE_PATTERN = "model{suffix}.safetensors" +TF2_WEIGHTS_FILE_PATTERN = "tf_model{suffix}.h5" + +# Constants for safetensors repos + +SAFETENSORS_SINGLE_FILE = "model.safetensors" +SAFETENSORS_INDEX_FILE = "model.safetensors.index.json" +SAFETENSORS_MAX_HEADER_LENGTH = 25_000_000 + +# Timeout of acquiring file lock and logging the attempt +FILELOCK_LOG_EVERY_SECONDS = 10 + +# Git-related constants + +DEFAULT_REVISION = "main" +REGEX_COMMIT_OID = re.compile(r"[A-Fa-f0-9]{5,40}") + +HUGGINGFACE_CO_URL_HOME = "https://huggingface.co/" + +_staging_mode = _is_true(os.environ.get("HUGGINGFACE_CO_STAGING")) + +_HF_DEFAULT_ENDPOINT = "https://huggingface.co" +_HF_DEFAULT_STAGING_ENDPOINT = "https://hub-ci.huggingface.co" +ENDPOINT = os.getenv("HF_ENDPOINT", _HF_DEFAULT_ENDPOINT).rstrip("/") +HUGGINGFACE_CO_URL_TEMPLATE = ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}" + +if _staging_mode: + ENDPOINT = _HF_DEFAULT_STAGING_ENDPOINT + HUGGINGFACE_CO_URL_TEMPLATE = _HF_DEFAULT_STAGING_ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}" + +# Hosts whose web URLs can be parsed into a ``hf://`` URI (see ``huggingface_hub/utils/_hf_uris.py``). +# Includes the public Hub host and its ``hf.co`` short domain, the staging host, and the host of the +# currently configured ``ENDPOINT`` so that self-hosted / staging endpoints work too. +HF_URL_HOSTS: frozenset[str] = frozenset( + {"hf.co"} + | { + host.lower() + for host in ( + urlsplit(_HF_DEFAULT_ENDPOINT).hostname, + urlsplit(_HF_DEFAULT_STAGING_ENDPOINT).hostname, + urlsplit(ENDPOINT).hostname, + ) + if host + } +) + +DATASETS_SERVER_ENDPOINT = "https://datasets-server.huggingface.co" + +HUGGINGFACE_HEADER_X_REPO_COMMIT = "X-Repo-Commit" +HUGGINGFACE_HEADER_X_LINKED_ETAG = "X-Linked-Etag" +HUGGINGFACE_HEADER_X_LINKED_SIZE = "X-Linked-Size" +HUGGINGFACE_HEADER_X_BILL_TO = "X-HF-Bill-To" + +INFERENCE_ENDPOINT = os.environ.get("HF_INFERENCE_ENDPOINT", "https://api-inference.huggingface.co") + +# See https://huggingface.co/docs/inference-endpoints/index +INFERENCE_ENDPOINTS_ENDPOINT = "https://api.endpoints.huggingface.cloud/v2" +INFERENCE_CATALOG_ENDPOINT = "https://endpoints.huggingface.co/api/catalog" + +# See https://api.endpoints.huggingface.cloud/#post-/v2/endpoint/-namespace- +INFERENCE_ENDPOINT_IMAGE_KEYS = [ + "custom", + "huggingface", + "huggingfaceNeuron", + "llamacpp", + "tei", + "tgi", + "tgiNeuron", +] + +# Proxy for third-party providers +INFERENCE_PROXY_TEMPLATE = "https://router.huggingface.co/{provider}" + +REPO_ID_SEPARATOR = "--" +# ^ this substring is not allowed in repo_ids on hf.co +# and is the canonical one we use for serialization of repo ids elsewhere. + + +REPO_TYPE_DATASET = "dataset" +REPO_TYPE_SPACE = "space" +REPO_TYPE_MODEL = "model" +REPO_TYPE_KERNEL = "kernel" +REPO_TYPES = [None, REPO_TYPE_MODEL, REPO_TYPE_DATASET, REPO_TYPE_SPACE] +REPO_TYPES_WITH_KERNEL = REPO_TYPES + [REPO_TYPE_KERNEL] +SPACES_SDK_TYPES = ["gradio", "streamlit", "docker", "static"] + +REPO_TYPES_URL_PREFIXES = { + REPO_TYPE_DATASET: "datasets/", + REPO_TYPE_SPACE: "spaces/", + REPO_TYPE_KERNEL: "kernels/", +} +REPO_TYPES_MAPPING = { + "datasets": REPO_TYPE_DATASET, + "spaces": REPO_TYPE_SPACE, + "models": REPO_TYPE_MODEL, + "kernels": REPO_TYPE_KERNEL, +} + +# HF Hub URIs (``hf://...``). See ``huggingface_hub/utils/_hf_uris.py`` +# and ``docs/source/en/package_reference/hf_uris.md`` for the full grammar. +HF_PROTOCOL = "hf://" +HfUriType = Literal["model", "dataset", "space", "kernel", "bucket"] +# Maps the plural URI prefix that may appear in a HF URI (e.g. ``datasets/``) +# to the canonical singular type name. Buckets are first-class HF URI types. +HF_URI_TYPE_PREFIXES: dict[str, HfUriType] = { + "models": "model", + "datasets": "dataset", + "spaces": "space", + "kernels": "kernel", + "buckets": "bucket", +} + + +DiscussionTypeFilter = Literal["all", "discussion", "pull_request"] +DISCUSSION_TYPES: tuple[DiscussionTypeFilter, ...] = typing.get_args(DiscussionTypeFilter) +DiscussionStatusFilter = Literal["all", "open", "closed"] +DISCUSSION_STATUS: tuple[DiscussionTypeFilter, ...] = typing.get_args(DiscussionStatusFilter) + +# Webhook subscription types +WEBHOOK_DOMAIN_T = Literal["repo", "discussions"] + +# default cache +default_home = os.path.join(os.path.expanduser("~"), ".cache") +HF_HOME = os.path.expandvars( + os.path.expanduser( + os.getenv( + "HF_HOME", + os.path.join(os.getenv("XDG_CACHE_HOME", default_home), "huggingface"), + ) + ) +) + +default_cache_path = os.path.join(HF_HOME, "hub") +default_assets_cache_path = os.path.join(HF_HOME, "assets") + +# Legacy env variables +HUGGINGFACE_HUB_CACHE = os.getenv("HUGGINGFACE_HUB_CACHE", default_cache_path) +HUGGINGFACE_ASSETS_CACHE = os.getenv("HUGGINGFACE_ASSETS_CACHE", default_assets_cache_path) + +# New env variables +HF_HUB_CACHE = os.path.expandvars( + os.path.expanduser( + os.getenv( + "HF_HUB_CACHE", + HUGGINGFACE_HUB_CACHE, + ) + ) +) +HF_ASSETS_CACHE = os.path.expandvars( + os.path.expanduser( + os.getenv( + "HF_ASSETS_CACHE", + HUGGINGFACE_ASSETS_CACHE, + ) + ) +) + +HF_HUB_OFFLINE = _is_true(os.environ.get("HF_HUB_OFFLINE") or os.environ.get("TRANSFORMERS_OFFLINE")) + + +def is_offline_mode() -> bool: + """Returns whether we are in offline mode for the Hub. + + When offline mode is enabled, all HTTP requests made with `get_session` will raise an `OfflineModeIsEnabled` exception. + + Example: + ```py + from huggingface_hub import is_offline_mode + + def list_files(repo_id: str): + if is_offline_mode(): + ... # list files from local cache (degraded experience but still functional) + else: + ... # list files from Hub (complete experience) + ``` + """ + return HF_HUB_OFFLINE + + +# File created to mark that the version check has been done. +# Check is performed once per 24 hours at most. +CHECK_FOR_UPDATE_DONE_PATH = os.path.join(HF_HOME, ".check_for_update_done") + +# File caching the AI agent harnesses registry fetched from `{ENDPOINT}/api/agent-harnesses`. +# Refreshed once per 24 hours at most (see `utils/_detect_agent.py`). +AGENT_HARNESSES_PATH = os.path.join(HF_HOME, ".agent_harnesses.json") + +# Set to skip the CLI update check (PyPI query + "new version available" warning at startup). +HF_HUB_DISABLE_UPDATE_CHECK = _is_true(os.environ.get("HF_HUB_DISABLE_UPDATE_CHECK")) + +# If set, log level will be set to DEBUG and all requests made to the Hub will be logged +# as curl commands for reproducibility. +HF_DEBUG = _is_true(os.environ.get("HF_DEBUG")) + +# Opt-out from telemetry requests +HF_HUB_DISABLE_TELEMETRY = ( + _is_true(os.environ.get("HF_HUB_DISABLE_TELEMETRY")) # HF-specific env variable + or _is_true(os.environ.get("DISABLE_TELEMETRY")) + or _is_true(os.environ.get("DO_NOT_TRACK")) # https://donottrack.sh/ +) + +HF_TOKEN_PATH = os.path.expandvars( + os.path.expanduser( + os.getenv( + "HF_TOKEN_PATH", + os.path.join(HF_HOME, "token"), + ) + ) +) +HF_STORED_TOKENS_PATH = os.path.join(os.path.dirname(HF_TOKEN_PATH), "stored_tokens") + +if _staging_mode: + # In staging mode, we use a different cache to ensure we don't mix up production and staging data or tokens + # In practice in `huggingface_hub` tests, we monkeypatch these values with temporary directories. The following + # lines are only used in third-party libraries tests (e.g. `transformers`, `diffusers`, etc.). + _staging_home = os.path.join(os.path.expanduser("~"), ".cache", "huggingface_staging") + HUGGINGFACE_HUB_CACHE = os.path.join(_staging_home, "hub") + HF_TOKEN_PATH = os.path.join(_staging_home, "token") + +# Here, `True` will disable progress bars globally without possibility of enabling it +# programmatically. `False` will enable them without possibility of disabling them. +# If environment variable is not set (None), then the user is free to enable/disable +# them programmatically. +# TL;DR: env variable has priority over code +__HF_HUB_DISABLE_PROGRESS_BARS = os.environ.get("HF_HUB_DISABLE_PROGRESS_BARS") +HF_HUB_DISABLE_PROGRESS_BARS: bool | None = ( + _is_true(__HF_HUB_DISABLE_PROGRESS_BARS) if __HF_HUB_DISABLE_PROGRESS_BARS is not None else None +) + +# Disable symlinks in the cache (files are copied instead of symlinked) +HF_HUB_DISABLE_SYMLINKS: bool = _is_true(os.environ.get("HF_HUB_DISABLE_SYMLINKS")) + +# Disable warning on machines that do not support symlinks (e.g. Windows non-developer) +HF_HUB_DISABLE_SYMLINKS_WARNING: bool = _is_true(os.environ.get("HF_HUB_DISABLE_SYMLINKS_WARNING")) + +# Disable warning when using experimental features +HF_HUB_DISABLE_EXPERIMENTAL_WARNING: bool = _is_true(os.environ.get("HF_HUB_DISABLE_EXPERIMENTAL_WARNING")) + +# Disable sending the cached token by default is all HTTP requests to the Hub +HF_HUB_DISABLE_IMPLICIT_TOKEN: bool = _is_true(os.environ.get("HF_HUB_DISABLE_IMPLICIT_TOKEN")) + +HF_XET_HIGH_PERFORMANCE: bool = _is_true(os.environ.get("HF_XET_HIGH_PERFORMANCE")) + +# Bucket and mount path used when launching Jobs +HF_JOBS_ARTIFACTS_BUCKET_NAME: str = "jobs-artifacts" +HF_JOBS_ARTIFACTS_MOUNT_PATH: str = "/data" + +# hf_transfer is not used anymore. Let's warn user is case they set the env variable. +# Note: we use FutureWarning (shown by default) instead of DeprecationWarning (silenced +# by default for end users) so users running standard `python` actually see the message. +if _is_true(os.environ.get("HF_HUB_ENABLE_HF_TRANSFER")) and not HF_XET_HIGH_PERFORMANCE: + import warnings + + warnings.warn( + "The `HF_HUB_ENABLE_HF_TRANSFER` environment variable is deprecated as 'hf_transfer' is not used anymore. " + "Please use `HF_XET_HIGH_PERFORMANCE` instead to enable high performance transfer with Xet. " + "Visit https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfxethighperformance for more details.", + FutureWarning, + ) + +# Used to override the etag timeout on a system level +HF_HUB_ETAG_TIMEOUT: int = _as_int(os.environ.get("HF_HUB_ETAG_TIMEOUT")) or DEFAULT_ETAG_TIMEOUT + +# Used to override the get request timeout on a system level +# Also used as a default timeout for other requests if not specified (kept the naming for legacy reasons) +HF_HUB_DOWNLOAD_TIMEOUT: int = _as_int(os.environ.get("HF_HUB_DOWNLOAD_TIMEOUT")) or DEFAULT_DOWNLOAD_TIMEOUT + +# Allows to add information about the requester in the user-agent (e.g. partner name) +HF_HUB_USER_AGENT_ORIGIN: str | None = os.environ.get("HF_HUB_USER_AGENT_ORIGIN") + +# If OAuth didn't work after 2 redirects, there's likely a third-party cookie issue in the Space iframe view. +# In this case, we redirect the user to the non-iframe view. +OAUTH_MAX_REDIRECTS = 2 + +# OAuth-related environment variables injected by the Space +OAUTH_CLIENT_ID = os.environ.get("OAUTH_CLIENT_ID") +OAUTH_CLIENT_SECRET = os.environ.get("OAUTH_CLIENT_SECRET") +OAUTH_SCOPES = os.environ.get("OAUTH_SCOPES") +OPENID_PROVIDER_URL = os.environ.get("OPENID_PROVIDER_URL") + +# OAuth client ID of the Device Code login flow (RFC 8628) used by `hf auth login` / `login()`. +# Overridable for Hub deployments (staging, Enterprise) where the default client ID is not provisioned. +DEVICE_CODE_OAUTH_CLIENT_ID = os.environ.get("HF_DEVICE_CODE_OAUTH_CLIENT_ID", "26be6b09-91c5-47da-9861-d2d2bb7a7e36") + +# Xet constants +HUGGINGFACE_HEADER_X_XET_ENDPOINT = "X-Xet-Cas-Url" +HUGGINGFACE_HEADER_X_XET_ACCESS_TOKEN = "X-Xet-Access-Token" +HUGGINGFACE_HEADER_X_XET_EXPIRATION = "X-Xet-Token-Expiration" +HUGGINGFACE_HEADER_X_XET_HASH = "X-Xet-Hash" +HUGGINGFACE_HEADER_X_XET_REFRESH_ROUTE = "X-Xet-Refresh-Route" +HUGGINGFACE_HEADER_LINK_XET_AUTH_KEY = "xet-auth" + +default_xet_cache_path = os.path.join(HF_HOME, "xet") +HF_XET_CACHE = os.getenv("HF_XET_CACHE", default_xet_cache_path) +HF_HUB_DISABLE_XET: bool = _is_true(os.environ.get("HF_HUB_DISABLE_XET")) + +# Bucket hosting the static sandbox server binary (see huggingface_hub.Sandbox) +SANDBOX_SERVER_BUCKET: str = "huggingface/sbx-server" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/dataclasses.py b/venv/lib/python3.12/site-packages/huggingface_hub/dataclasses.py new file mode 100644 index 0000000000000000000000000000000000000000..ff5170c3f82555a1656d27373dcdf799cc3d1646 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/dataclasses.py @@ -0,0 +1,684 @@ +import collections.abc +import inspect +import types +from collections.abc import Callable +from dataclasses import MISSING, Field, field, fields, make_dataclass +from functools import lru_cache, wraps +from typing import ( + Annotated, + Any, + ForwardRef, + Literal, + Type, + TypeVar, + Union, + get_args, + get_origin, + overload, +) + + +try: + # Python 3.11+ + from typing import NotRequired, Required # type: ignore +except ImportError: + try: + # In case typing_extensions is installed + from typing_extensions import NotRequired, Required # type: ignore + except ImportError: + # Fallback: create dummy types that will never match + Required = type("Required", (), {}) # type: ignore + NotRequired = type("NotRequired", (), {}) # type: ignore + +from .errors import ( + StrictDataclassClassValidationError, + StrictDataclassDefinitionError, + StrictDataclassFieldValidationError, +) + + +Validator_T = Callable[[Any], None] +T = TypeVar("T") +TypedDictType = TypeVar("TypedDictType", bound=dict[str, Any]) + +_TYPED_DICT_DEFAULT_VALUE = object() # used as default value in TypedDict fields (to distinguish from None) + + +# The overload decorator helps type checkers understand the different return types +@overload +def strict(cls: Type[T]) -> Type[T]: ... + + +@overload +def strict(*, accept_kwargs: bool = False) -> Callable[[Type[T]], Type[T]]: ... + + +def strict(cls: Type[T] | None = None, *, accept_kwargs: bool = False) -> Type[T] | Callable[[Type[T]], Type[T]]: + """ + Decorator to add strict validation to a dataclass. + + This decorator must be used on top of `@dataclass` to ensure IDEs and static typing tools + recognize the class as a dataclass. + + Can be used with or without arguments: + - `@strict` + - `@strict(accept_kwargs=True)` + + Args: + cls: + The class to convert to a strict dataclass. + accept_kwargs (`bool`, *optional*): + If True, allows arbitrary keyword arguments in `__init__`. Defaults to False. + + Returns: + The enhanced dataclass with strict validation on field assignment. + + Example: + ```py + >>> from dataclasses import dataclass + >>> from huggingface_hub.dataclasses import as_validated_field, strict, validated_field + + >>> @as_validated_field + >>> def positive_int(value: int): + ... if not value >= 0: + ... raise ValueError(f"Value must be positive, got {value}") + + >>> @strict(accept_kwargs=True) + ... @dataclass + ... class User: + ... name: str + ... age: int = positive_int(default=10) + + # Initialize + >>> User(name="John") + User(name='John', age=10) + + # Extra kwargs are accepted + >>> User(name="John", age=30, lastname="Doe") + User(name='John', age=30, *lastname='Doe') + + # Invalid type => raises + >>> User(name="John", age="30") + huggingface_hub.errors.StrictDataclassFieldValidationError: Validation error for field 'age': + TypeError: Field 'age' expected int, got str (value: '30') + + # Invalid value => raises + >>> User(name="John", age=-1) + huggingface_hub.errors.StrictDataclassFieldValidationError: Validation error for field 'age': + ValueError: Value must be positive, got -1 + ``` + """ + + def wrap(cls: Type[T]) -> Type[T]: + if not hasattr(cls, "__dataclass_fields__"): + raise StrictDataclassDefinitionError( + f"Class '{cls.__name__}' must be a dataclass before applying @strict." + ) + + # List and store validators + field_validators: dict[str, list[Validator_T]] = {} + for f in fields(cls): # type: ignore + validators = [] + validators.append(_create_type_validator(f)) + custom_validator = f.metadata.get("validator") + if custom_validator is not None: + if not isinstance(custom_validator, list): + custom_validator = [custom_validator] + for validator in custom_validator: + if not _is_validator(validator): + raise StrictDataclassDefinitionError( + f"Invalid validator for field '{f.name}': {validator}. Must be a callable taking a single argument." + ) + validators.extend(custom_validator) + field_validators[f.name] = validators + cls.__validators__ = field_validators # type: ignore + + # Override __setattr__ to validate fields on assignment + original_setattr = cls.__setattr__ + + def __strict_setattr__(self: Any, name: str, value: Any) -> None: + """Custom __setattr__ method for strict dataclasses.""" + # Run all validators + for validator in self.__validators__.get(name, []): + try: + validator(value) + except (ValueError, TypeError) as e: + raise StrictDataclassFieldValidationError(field=name, cause=e) from e + + # If validation passed, set the attribute + original_setattr(self, name, value) + + cls.__setattr__ = __strict_setattr__ # type: ignore + + if accept_kwargs: + # (optional) Override __init__ to accept arbitrary keyword arguments + original_init = cls.__init__ + + @wraps(original_init) + def __init__(self, *args, **kwargs: Any) -> None: + # Extract only the fields that are part of the dataclass + dataclass_fields = {f.name for f in fields(cls)} # type: ignore + standard_kwargs = {k: v for k, v in kwargs.items() if k in dataclass_fields} + + # User shouldn't define custom `__init__` when `accepts_kwargs`, and instead + # are advised to move field manipulation to `__post_init__` (e.g., derive new field from existing ones) + # We need to call bare `__init__` here without `__post_init__` but the``original_init`` would call + # post-init right away with no kwargs. + if len(args) > 0: + raise ValueError( + f"When `accept_kwargs=True`, {cls.__name__} accepts only keyword arguments, " + f"but found `{len(args)}` positional args." + ) + + for f in fields(cls): # type: ignore + if f.name in standard_kwargs: + setattr(self, f.name, standard_kwargs[f.name]) + elif f.default is not MISSING: + setattr(self, f.name, f.default) + elif f.default_factory is not MISSING: + setattr(self, f.name, f.default_factory()) + else: + raise TypeError(f"Missing required field - '{f.name}'") + + # Pass any additional kwargs to `__post_init__` and let the object + # decide whether to set the attr or use for different purposes (e.g. BC checks) + additional_kwargs = {} + for name, value in kwargs.items(): + if name not in dataclass_fields: + additional_kwargs[name] = value + + self.__post_init__(**additional_kwargs) + + cls.__init__ = __init__ # type: ignore + + # Define a default __post_init__ if not defined + if not hasattr(cls, "__post_init__"): + + def __post_init__(self, **kwargs: Any) -> None: + """Default __post_init__ to accept additional kwargs.""" + for name, value in kwargs.items(): + setattr(self, name, value) + + cls.__post_init__ = __post_init__ # type: ignore + + # (optional) Override __repr__ to include additional kwargs + original_repr = cls.__repr__ + + @wraps(original_repr) + def __repr__(self) -> str: + # Call the original __repr__ to get the standard fields + standard_repr = original_repr(self) + + # Get additional kwargs + additional_kwargs = [ + # add a '*' in front of additional kwargs to let the user know they are not part of the dataclass + f"*{k}={v!r}" + for k, v in self.__dict__.items() + if k not in cls.__dataclass_fields__ # type: ignore [attr-defined] + ] + additional_repr = ", ".join(additional_kwargs) + + # Combine both representations + return f"{standard_repr[:-1]}, {additional_repr})" if additional_kwargs else standard_repr + + if cls.__dataclass_params__.repr is True: # type: ignore [attr-defined] + cls.__repr__ = __repr__ # type: ignore + + # List all public methods starting with `validate_` => class validators. + class_validators = [] + + for name in dir(cls): + if not name.startswith("validate_"): + continue + method = getattr(cls, name) + if not callable(method): + continue + if len(inspect.signature(method).parameters) != 1: + raise StrictDataclassDefinitionError( + f"Class '{cls.__name__}' has a class validator '{name}' that takes more than one argument." + " Class validators must take only 'self' as an argument. Methods starting with 'validate_'" + " are considered to be class validators." + ) + class_validators.append(method) + + cls.__class_validators__ = class_validators # type: ignore + + # Add `validate` method to the class, but first check if it already exists + def validate(self: T) -> None: + """Run class validators on the instance.""" + for validator in cls.__class_validators__: # type: ignore [attr-defined] + try: + validator(self) + except (ValueError, TypeError) as e: + raise StrictDataclassClassValidationError(validator=validator.__name__, cause=e) from e + + # Hack to be able to raise if `.validate()` already exists except if it was created by this decorator on a parent class + # (in which case we just override it) + validate.__is_defined_by_strict_decorator__ = True # type: ignore [attr-defined] + + if hasattr(cls, "validate"): + if not getattr(cls.validate, "__is_defined_by_strict_decorator__", False): # type: ignore [attr-defined] + raise StrictDataclassDefinitionError( + f"Class '{cls.__name__}' already implements a method called 'validate'." + " This method name is reserved when using the @strict decorator on a dataclass." + " If you want to keep your own method, please rename it." + ) + + cls.validate = validate # type: ignore + + # Run class validators after initialization + initial_init = cls.__init__ + + @wraps(initial_init) + def init_with_validate(self, *args, **kwargs) -> None: + """Run class validators after initialization.""" + initial_init(self, *args, **kwargs) # type: ignore [call-arg] + cls.validate(self) # type: ignore [attr-defined] + + setattr(cls, "__init__", init_with_validate) + + return cls + + # Return wrapped class or the decorator itself + return wrap(cls) if cls is not None else wrap + + +def validate_typed_dict(schema: type[TypedDictType], data: dict) -> None: + """ + Validate that a dictionary conforms to the types defined in a TypedDict class. + + Under the hood, the typed dict is converted to a strict dataclass and validated using the `@strict` decorator. + + Args: + schema (`type[TypedDictType]`): + The TypedDict class defining the expected structure and types. + data (`dict`): + The dictionary to validate. + + Raises: + `StrictDataclassFieldValidationError`: + If any field in the dictionary does not conform to the expected type. + + Example: + ```py + >>> from typing import Annotated, TypedDict + >>> from huggingface_hub.dataclasses import validate_typed_dict + + >>> def positive_int(value: int): + ... if not value >= 0: + ... raise ValueError(f"Value must be positive, got {value}") + + >>> class User(TypedDict): + ... name: str + ... age: Annotated[int, positive_int] + + >>> # Valid data + >>> validate_typed_dict(User, {"name": "John", "age": 30}) + + >>> # Invalid type for age + >>> validate_typed_dict(User, {"name": "John", "age": "30"}) + huggingface_hub.errors.StrictDataclassFieldValidationError: Validation error for field 'age': + TypeError: Field 'age' expected int, got str (value: '30') + + >>> # Invalid value for age + >>> validate_typed_dict(User, {"name": "John", "age": -1}) + huggingface_hub.errors.StrictDataclassFieldValidationError: Validation error for field 'age': + ValueError: Value must be positive, got -1 + ``` + """ + # Convert typed dict to dataclass + strict_cls = _build_strict_cls_from_typed_dict(schema) + + # Validate the data by instantiating the strict dataclass + strict_cls(**data) # will raise if validation fails + + +@lru_cache +def _build_strict_cls_from_typed_dict(schema: type[TypedDictType]) -> Type: + # Extract type hints from the TypedDict class + type_hints = _get_typed_dict_annotations(schema) + + # If the TypedDict is not total, wrap fields as NotRequired (unless explicitly Required or NotRequired) + if not getattr(schema, "__total__", True): + for key, value in type_hints.items(): + origin = get_origin(value) + + if origin is Annotated: + base, *meta = get_args(value) + if not _is_required_or_notrequired(base): + base = NotRequired[base] + type_hints[key] = Annotated[tuple([base] + list(meta))] # type: ignore + elif not _is_required_or_notrequired(value): + type_hints[key] = NotRequired[value] + + # Convert type hints to dataclass fields + fields = [] + for key, value in type_hints.items(): + if get_origin(value) is Annotated: + base, *meta = get_args(value) + fields.append((key, base, field(default=_TYPED_DICT_DEFAULT_VALUE, metadata={"validator": meta[0]}))) + else: + fields.append((key, value, field(default=_TYPED_DICT_DEFAULT_VALUE))) + + # Create a strict dataclass from the TypedDict fields + return strict(make_dataclass(schema.__name__, fields)) + + +def _get_typed_dict_annotations(schema: type[TypedDictType]) -> dict[str, Any]: + """Extract type annotations from a TypedDict class.""" + try: + # Available in Python 3.14+ + import annotationlib + + return annotationlib.get_annotations(schema) + except ImportError: + return { + # We do not use `get_type_hints` here to avoid evaluating ForwardRefs (which might fail). + # ForwardRefs are not validated by @strict anyway. + name: value if value is not None else type(None) + for name, value in schema.__dict__.get("__annotations__", {}).items() + } + + +def validated_field( + validator: list[Validator_T] | Validator_T, + default: Any = MISSING, + default_factory: Any = MISSING, + init: bool = True, + repr: bool = True, + hash: bool | None = None, + compare: bool = True, + metadata: dict | None = None, + **kwargs: Any, +) -> Any: + """ + Create a dataclass field with a custom validator. + + Useful to apply several checks to a field. If only applying one rule, check out the [`as_validated_field`] decorator. + + Args: + validator (`Callable` or `list[Callable]`): + A method that takes a value as input and raises ValueError/TypeError if the value is invalid. + Can be a list of validators to apply multiple checks. + **kwargs: + Additional arguments to pass to `dataclasses.field()`. + + Returns: + A field with the validator attached in metadata + """ + if not isinstance(validator, list): + validator = [validator] + if metadata is None: + metadata = {} + metadata["validator"] = validator + return field( # type: ignore + default=default, # type: ignore + default_factory=default_factory, # type: ignore + init=init, + repr=repr, + hash=hash, + compare=compare, + metadata=metadata, + **kwargs, + ) + + +def as_validated_field(validator: Validator_T): + """ + Decorates a validator function as a [`validated_field`] (i.e. a dataclass field with a custom validator). + + Args: + validator (`Callable`): + A method that takes a value as input and raises ValueError/TypeError if the value is invalid. + """ + + def _inner( + default: Any = MISSING, + default_factory: Any = MISSING, + init: bool = True, + repr: bool = True, + hash: bool | None = None, + compare: bool = True, + metadata: dict | None = None, + **kwargs: Any, + ): + return validated_field( + validator, + default=default, + default_factory=default_factory, + init=init, + repr=repr, + hash=hash, + compare=compare, + metadata=metadata, + **kwargs, + ) + + return _inner + + +def type_validator(name: str, value: Any, expected_type: Any) -> None: + """Validate that 'value' matches 'expected_type'.""" + origin = get_origin(expected_type) + args = get_args(expected_type) + + if expected_type is Any: + return + elif expected_type is None: + _validate_none(name, value) + elif validator := _BASIC_TYPE_VALIDATORS.get(origin): + validator(name, value, args) + elif isinstance(expected_type, type): # simple types + _validate_simple_type(name, value, expected_type) + elif isinstance(expected_type, ForwardRef) or isinstance(expected_type, str): + return + elif origin is Required: + if value is _TYPED_DICT_DEFAULT_VALUE: + raise TypeError(f"Field '{name}' is required but missing.") + type_validator(name, value, args[0]) + elif origin is NotRequired: + if value is _TYPED_DICT_DEFAULT_VALUE: + return + type_validator(name, value, args[0]) + else: + raise TypeError(f"Unsupported type for field '{name}': {expected_type}") + + +def _validate_none(name: str, value: Any) -> None: + """Validate None type. + + 'None' is not a type, it's a special value. Type should be `NoneType` instead. + But in type annotations 'None' is accepted so we must support it. + """ + if value is not None: + raise TypeError(f"Field '{name}' expected None, got {type(value).__name__}") + + +def _validate_union(name: str, value: Any, args: tuple[Any, ...]) -> None: + """Validate that value matches one of the types in a Union.""" + errors = [] + for t in args: + try: + type_validator(name, value, t) + return # Valid if any type matches + except TypeError as e: + errors.append(str(e)) + + raise TypeError( + f"Field '{name}' with value {repr(value)} doesn't match any type in {args}. Errors: {'; '.join(errors)}" + ) + + +def _validate_literal(name: str, value: Any, args: tuple[Any, ...]) -> None: + """Validate Literal type.""" + if isinstance(value, bool): + if value not in [arg for arg in args if isinstance(arg, bool)]: + raise TypeError(f"Field '{name}' expected one of {args}, got {value}") + elif isinstance(value, int): + if value not in [arg for arg in args if isinstance(arg, int) and not isinstance(arg, bool)]: + raise TypeError(f"Field '{name}' expected one of {args}, got {value}") + elif value not in args: + raise TypeError(f"Field '{name}' expected one of {args}, got {value}") + + +def _validate_list(name: str, value: Any, args: tuple[Any, ...]) -> None: + """Validate list[T] type.""" + if not isinstance(value, list): + raise TypeError(f"Field '{name}' expected a list, got {type(value).__name__}") + + # Validate each item in the list + item_type = args[0] + for i, item in enumerate(value): + try: + type_validator(f"{name}[{i}]", item, item_type) + except TypeError as e: + raise TypeError(f"Invalid item at index {i} in list '{name}'") from e + + +def _validate_dict(name: str, value: Any, args: tuple[Any, ...]) -> None: + """Validate dict[K, V] type.""" + if not isinstance(value, dict): + raise TypeError(f"Field '{name}' expected a dict, got {type(value).__name__}") + + # Validate keys and values + key_type, value_type = args + for k, v in value.items(): + try: + type_validator(f"{name}.key", k, key_type) + type_validator(f"{name}[{k!r}]", v, value_type) + except TypeError as e: + raise TypeError(f"Invalid key or value in dict '{name}'") from e + + +def _validate_tuple(name: str, value: Any, args: tuple[Any, ...]) -> None: + """Validate Tuple type.""" + if not isinstance(value, tuple): + raise TypeError(f"Field '{name}' expected a tuple, got {type(value).__name__}") + + # Handle variable-length tuples: tuple[T, ...] + if len(args) == 2 and args[1] is Ellipsis: + for i, item in enumerate(value): + try: + type_validator(f"{name}[{i}]", item, args[0]) + except TypeError as e: + raise TypeError(f"Invalid item at index {i} in tuple '{name}'") from e + # Handle fixed-length tuples: tuple[T1, T2, ...] + elif len(args) != len(value): + raise TypeError(f"Field '{name}' expected a tuple of length {len(args)}, got {len(value)}") + else: + for i, (item, expected) in enumerate(zip(value, args)): + try: + type_validator(f"{name}[{i}]", item, expected) + except TypeError as e: + raise TypeError(f"Invalid item at index {i} in tuple '{name}'") from e + + +def _validate_set(name: str, value: Any, args: tuple[Any, ...]) -> None: + """Validate set[T] type.""" + if not isinstance(value, set): + raise TypeError(f"Field '{name}' expected a set, got {type(value).__name__}") + + # Validate each item in the set + item_type = args[0] + for i, item in enumerate(value): + try: + type_validator(f"{name} item", item, item_type) + except TypeError as e: + raise TypeError(f"Invalid item in set '{name}'") from e + + +def _validate_sequence(name: str, value: Any, args: tuple[Any, ...]) -> None: + """Validate Sequence or Sequence[T] type.""" + if not isinstance(value, collections.abc.Sequence): + raise TypeError(f"Field '{name}' expected a Sequence, got {type(value).__name__}") + + # If no type argument is provided (i.e., just `Sequence`), skip item validation + if not args: + return + + # Validate each item in the sequence + item_type = args[0] + for i, item in enumerate(value): + try: + type_validator(f"{name}[{i}]", item, item_type) + except TypeError as e: + raise TypeError(f"Invalid item at index {i} in sequence '{name}'") from e + + +def _validate_simple_type(name: str, value: Any, expected_type: type) -> None: + """Validate simple type (int, str, etc.).""" + if expected_type is int and isinstance(value, bool): + raise TypeError( + f"Field '{name}' expected {expected_type.__name__}, got {type(value).__name__} (value: {repr(value)})" + ) + if not isinstance(value, expected_type): + raise TypeError( + f"Field '{name}' expected {expected_type.__name__}, got {type(value).__name__} (value: {repr(value)})" + ) + + +def _create_type_validator(field: Field) -> Validator_T: + """Create a type validator function for a field.""" + # Hacky: we cannot use a lambda here because of reference issues + + def validator(value: Any) -> None: + type_validator(field.name, value, field.type) + + return validator + + +def _is_validator(validator: Any) -> bool: + """Check if a function is a validator. + + A validator is a Callable that can be called with a single positional argument. + The validator can have more arguments with default values. + + Basically, returns True if `validator(value)` is possible. + """ + if not callable(validator): + return False + + signature = inspect.signature(validator) + parameters = list(signature.parameters.values()) + if len(parameters) == 0: + return False + if parameters[0].kind not in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.VAR_POSITIONAL, + ): + return False + for parameter in parameters[1:]: + if parameter.default == inspect.Parameter.empty: + return False + return True + + +def _is_required_or_notrequired(type_hint: Any) -> bool: + """Helper to check if a type is Required/NotRequired.""" + return type_hint in (Required, NotRequired) or (get_origin(type_hint) in (Required, NotRequired)) + + +_BASIC_TYPE_VALIDATORS: dict[Any, Callable[[str, Any, tuple[Any, ...]], None]] = { + Union: _validate_union, + Literal: _validate_literal, + list: _validate_list, + dict: _validate_dict, + tuple: _validate_tuple, + set: _validate_set, + collections.abc.Sequence: _validate_sequence, +} + +# TODO: make it first class citizen when bumping to Python 3.10+ +_BASIC_TYPE_VALIDATORS[types.UnionType] = _validate_union # x | y syntax, available only Python 3.10+ + + +__all__ = [ + "strict", + "validate_typed_dict", + "validated_field", + "Validator_T", + "StrictDataclassClassValidationError", + "StrictDataclassDefinitionError", + "StrictDataclassFieldValidationError", +] diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/errors.py b/venv/lib/python3.12/site-packages/huggingface_hub/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..9a619b3d5b80a12c327ec9eba200be70a684a308 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/errors.py @@ -0,0 +1,609 @@ +"""Contains all custom errors.""" + +from enum import Enum +from pathlib import Path + +from httpx import HTTPError, Response + + +# CACHE ERRORS + + +class CacheNotFound(Exception): + """Exception thrown when the Huggingface cache is not found.""" + + cache_dir: str | Path + + def __init__(self, msg: str, cache_dir: str | Path, *args, **kwargs): + super().__init__(msg, *args, **kwargs) + self.cache_dir = cache_dir + + +class CorruptedCacheException(Exception): + """Exception for any unexpected structure in the Huggingface cache-system.""" + + +class CachedRepoTreeNotFoundError(Exception): + """Raised by [`get_cached_repo_tree`] when no tree listing is cached for the requested revision. + + The tree listing is populated as a side effect of [`snapshot_download`]. + """ + + +# HEADERS ERRORS + + +class LocalTokenNotFoundError(EnvironmentError): + """Raised if local token is required but not found.""" + + +# OIDC ERRORS + + +class OIDCError(Exception): + """Raised when keyless CI/CD auth via OIDC token exchange ("Trusted Publishers") cannot proceed. + + Typically because `HF_OIDC_RESOURCE` is set but no id token is available: not running in a + supported CI provider and `HF_OIDC_ID_TOKEN` is unset. + + See https://huggingface.co/docs/hub/trusted-publishers. + """ + + +# DEVICE CODE OAUTH ERRORS + + +class OAuthErrorCode(str, Enum): + """Known OAuth `error` codes returned by the Hub's token endpoint (RFC 6749 / RFC 8628).""" + + AUTHORIZATION_PENDING = "authorization_pending" + SLOW_DOWN = "slow_down" + EXPIRED_TOKEN = "expired_token" + ACCESS_DENIED = "access_denied" + INVALID_GRANT = "invalid_grant" + + +class DeviceCodeError(Exception): + """Raised when the Device Code OAuth login flow (RFC 8628) or an OAuth token refresh fails. + + Covers failures at any step: requesting the device code, polling for the token, + authorization denied/expired, or unexpected server responses. + + Attributes: + error_code (`str`, *optional*): + The OAuth `error` code returned by the server, if any. Known values are listed in + [`OAuthErrorCode`] but the server may return other codes. + """ + + def __init__(self, message: str, error_code: str | None = None): + super().__init__(message) + self.error_code = error_code + + +# HTTP ERRORS + + +class OfflineModeIsEnabled(ConnectionError): + """Raised when a request is made but `HF_HUB_OFFLINE=1` is set as environment variable.""" + + +class HfHubHTTPError(HTTPError, OSError): + """ + HTTPError to inherit from for any custom HTTP Error raised in HF Hub. + + Any HTTPError is converted at least into a `HfHubHTTPError`. If some information is + sent back by the server, it will be added to the error message. + + Added details: + - Request ID sourced from headers in order of precedence: "X-Request-Id", "X-Amzn-Trace-Id", "X-Amz-Cf-Id". + - Server error message from the header "X-Error-Message". + - Server error message if we can found one in the response body. + + Example: + ```py + import httpx + from huggingface_hub.utils import get_session, hf_raise_for_status, HfHubHTTPError + + response = get_session().post(...) + try: + hf_raise_for_status(response) + except HfHubHTTPError as e: + print(str(e)) # formatted message + e.request_id, e.server_message # details returned by server + + # Complete the error message with additional information once it's raised + e.append_to_message("\n`create_commit` expects the repository to exist.") + raise + ``` + """ + + def __init__( + self, + message: str, + *, + response: Response, + server_message: str | None = None, + ): + self.request_id = ( + response.headers.get("x-request-id") + or response.headers.get("X-Amzn-Trace-Id") + or response.headers.get("x-amz-cf-id") + ) + self.server_message = server_message + self.response = response + self.request = response.request + super().__init__(message) + + def append_to_message(self, additional_message: str) -> None: + """Append additional information to the `HfHubHTTPError` initial message.""" + self.args = (self.args[0] + additional_message,) + self.args[1:] + + @classmethod + def _reconstruct_hf_hub_http_error( + cls, message: str, response: Response, server_message: str | None + ) -> "HfHubHTTPError": + return cls(message, response=response, server_message=server_message) + + def __reduce_ex__(self, protocol): + """Fix pickling of Exception subclass with kwargs. We need to override __reduce_ex__ of the parent class""" + return (self.__class__._reconstruct_hf_hub_http_error, (str(self), self.response, self.server_message)) + + +# INFERENCE CLIENT ERRORS + + +class InferenceTimeoutError(HTTPError, TimeoutError): + """Error raised when a model is unavailable or the request times out.""" + + +# INFERENCE ENDPOINT ERRORS + + +class InferenceEndpointError(Exception): + """Generic exception when dealing with Inference Endpoints.""" + + +class InferenceEndpointTimeoutError(InferenceEndpointError, TimeoutError): + """Exception for timeouts while waiting for Inference Endpoint.""" + + +# SAFETENSORS ERRORS + + +class SafetensorsParsingError(Exception): + """Raised when failing to parse a safetensors file metadata. + + This can be the case if the file is not a safetensors file or does not respect the specification. + """ + + +class NotASafetensorsRepoError(Exception): + """Raised when a repo is not a Safetensors repo i.e. doesn't have either a `model.safetensors` or a + `model.safetensors.index.json` file. + """ + + +# TEXT GENERATION ERRORS + + +class TextGenerationError(HTTPError): + """Generic error raised if text-generation went wrong.""" + + +# Text Generation Inference Errors +class ValidationError(TextGenerationError): + """Server-side validation error.""" + + +class GenerationError(TextGenerationError): + pass + + +class OverloadedError(TextGenerationError): + pass + + +class IncompleteGenerationError(TextGenerationError): + pass + + +class UnknownError(TextGenerationError): + pass + + +# VALIDATION ERRORS + + +class HFValidationError(ValueError): + """Generic exception thrown by `huggingface_hub` validators. + + Inherits from [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError). + """ + + +class HfUriError(ValueError): + """Raised when an `hf://...` URI is malformed. + + See [`parse_hf_uri`] and the + [HF URIs reference](https://huggingface.co/docs/huggingface_hub/main/en/package_reference/hf_uris) + for the canonical syntax. + + Inherits from [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError). + """ + + def __init__(self, uri: str, msg: str): + self.uri = uri + self.msg = msg + full_msg = f"Invalid HF URI '{uri}'. {msg}" if uri else f"Invalid HF URI. {msg}" + super().__init__(full_msg) + + +# FILE METADATA ERRORS + + +class DryRunError(OSError): + """Error triggered when a dry run is requested but cannot be performed (e.g. invalid repo).""" + + +class FileMetadataError(OSError): + """Error triggered when the metadata of a file on the Hub cannot be retrieved (missing ETag or commit_hash). + + Inherits from `OSError` for backward compatibility. + """ + + +# BUCKET ERRORS + + +class BucketNotFoundError(HfHubHTTPError): + """ + Raised when trying to access a bucket that does not exist. + + Attributes: + bucket_id (`str` or `None`): + The bucket id (namespace/name) that was not found, if it could be determined from the request URL. + + Example: + + ```py + >>> from huggingface_hub import bucket_info + >>> bucket_info("") + (...) + huggingface_hub.errors.BucketNotFoundError: 404 Client Error. (Request ID: XXX) + + Bucket Not Found for url: https://huggingface.co/api/buckets/namespace/name. + Please make sure you specified the correct bucket id (namespace/name). + If the bucket is private, make sure you are authenticated and your token has the required permissions. + ``` + """ + + bucket_id: str | None = None + + +# JOB ERRORS + + +class JobNotFoundError(HfHubHTTPError): + """ + Raised when trying to access a Job that does not exist. + + Attributes: + job_id (`str`): + The job id that was not found. + """ + + job_id: str + + +# REPOSITORY ERRORS + + +class RepositoryNotFoundError(HfHubHTTPError): + """ + Raised when trying to access a hf.co URL with an invalid repository name, or + with a private repo name the user does not have access to. + + Attributes: + repo_id (`str` or `None`): + The repo id that was not found, if it could be determined from the request URL. + repo_type (`str` or `None`): + The repo type ("model", "dataset", or "space"), if it could be determined from the request URL. + + Example: + + ```py + >>> from huggingface_hub import model_info + >>> model_info("") + (...) + huggingface_hub.errors.RepositoryNotFoundError: 401 Client Error. (Request ID: PvMw_VjBMjVdMz53WKIzP) + + Repository Not Found for url: https://huggingface.co/api/models/%3Cnon_existent_repository%3E. + Please make sure you specified the correct `repo_id` and `repo_type`. + If the repo is private, make sure you are authenticated and your token has the required permissions. + Invalid username or password. + ``` + """ + + repo_id: str | None = None + repo_type: str | None = None + + +class GatedRepoError(RepositoryNotFoundError): + """ + Raised when trying to access a gated repository for which the user is not on the + authorized list. + + Note: derives from `RepositoryNotFoundError` to ensure backward compatibility. + + Example: + + ```py + >>> from huggingface_hub import model_info + >>> model_info("") + (...) + huggingface_hub.errors.GatedRepoError: 403 Client Error. (Request ID: ViT1Bf7O_026LGSQuVqfa) + + Cannot access gated repo for url https://huggingface.co/api/models/ardent-figment/gated-model. + Access to model ardent-figment/gated-model is restricted and you are not in the authorized list. + Visit https://huggingface.co/ardent-figment/gated-model to ask for access. + ``` + """ + + +class DisabledRepoError(HfHubHTTPError): + """ + Raised when trying to access a repository that has been disabled by its author. + + Example: + + ```py + >>> from huggingface_hub import dataset_info + >>> dataset_info("laion/laion-art") + (...) + huggingface_hub.errors.DisabledRepoError: 403 Client Error. (Request ID: Root=1-659fc3fa-3031673e0f92c71a2260dbe2;bc6f4dfb-b30a-4862-af0a-5cfe827610d8) + + Cannot access repository for url https://huggingface.co/api/datasets/laion/laion-art. + Access to this resource is disabled. + ``` + """ + + +# REVISION ERROR + + +class RevisionNotFoundError(HfHubHTTPError): + """ + Raised when trying to access a hf.co URL with a valid repository but an invalid + revision. + + Attributes: + repo_id (`str` or `None`): + The repo id, if it could be determined from the request URL. + repo_type (`str` or `None`): + The repo type ("model", "dataset", or "space"), if it could be determined from the request URL. + + Example: + + ```py + >>> from huggingface_hub import hf_hub_download + >>> hf_hub_download('bert-base-cased', 'config.json', revision='') + (...) + huggingface_hub.errors.RevisionNotFoundError: 404 Client Error. (Request ID: Mwhe_c3Kt650GcdKEFomX) + + Revision Not Found for url: https://huggingface.co/bert-base-cased/resolve/%3Cnon-existent-revision%3E/config.json. + ``` + """ + + repo_id: str | None = None + repo_type: str | None = None + + +# ENTRY ERRORS +class EntryNotFoundError(Exception): + """ + Raised when entry not found, either locally or remotely. + + Example: + + ```py + >>> from huggingface_hub import hf_hub_download + >>> hf_hub_download('bert-base-cased', '') + (...) + huggingface_hub.errors.RemoteEntryNotFoundError (...) + >>> hf_hub_download('bert-base-cased', '', local_files_only=True) + (...) + huggingface_hub.utils.errors.LocalEntryNotFoundError (...) + ``` + """ + + +class RemoteEntryNotFoundError(HfHubHTTPError, EntryNotFoundError): + """ + Raised when trying to access a hf.co URL with a valid repository and revision + but an invalid filename. + + Attributes: + repo_id (`str` or `None`): + The repo id, if it could be determined from the request URL. + repo_type (`str` or `None`): + The repo type ("model", "dataset", or "space"), if it could be determined from the request URL. + + Example: + + ```py + >>> from huggingface_hub import hf_hub_download + >>> hf_hub_download('bert-base-cased', '') + (...) + huggingface_hub.errors.EntryNotFoundError: 404 Client Error. (Request ID: 53pNl6M0MxsnG5Sw8JA6x) + + Entry Not Found for url: https://huggingface.co/bert-base-cased/resolve/main/%3Cnon-existent-file%3E. + ``` + """ + + repo_id: str | None = None + repo_type: str | None = None + + +class LocalEntryNotFoundError(FileNotFoundError, EntryNotFoundError): + """ + Raised when trying to access a file or snapshot that is not on the disk when network is + disabled or unavailable (connection issue). The entry may exist on the Hub. + + Example: + + ```py + >>> from huggingface_hub import hf_hub_download + >>> hf_hub_download('bert-base-cased', '', local_files_only=True) + (...) + huggingface_hub.errors.LocalEntryNotFoundError: Cannot find the requested files in the disk cache and outgoing traffic has been disabled. To enable hf.co look-ups and downloads online, set 'local_files_only' to False. + ``` + """ + + def __init__(self, message: str): + super().__init__(message) + + +class IncompleteSnapshotError(LocalEntryNotFoundError): + """ + Raised by [`snapshot_download`] when the Hub cannot be reached (offline, connection issue, or + `local_files_only=True`) and the cached snapshot is known to be incomplete: some files listed in + the repository's cached tree listing are missing from the local snapshot. + + This is a subclass of [`LocalEntryNotFoundError`] for backward compatibility. + + The `snapshot_path` attribute holds the path to the incomplete local snapshot, so a downstream library can locate + the latest cached files even though they are known to be incomplete. + """ + + def __init__(self, message: str, snapshot_path: str): + super().__init__(message) + self.snapshot_path = snapshot_path + + +# REQUEST ERROR +class BadRequestError(HfHubHTTPError, ValueError): + """ + Raised by `hf_raise_for_status` when the server returns a HTTP 400 error. + + Example: + + ```py + >>> resp = httpx.post("hf.co/api/check", ...) + >>> hf_raise_for_status(resp, endpoint_name="check") + huggingface_hub.errors.BadRequestError: Bad request for check endpoint: {details} (Request ID: XXX) + ``` + """ + + +# DDUF file format ERROR + + +class DDUFError(Exception): + """Base exception for errors related to the DDUF format.""" + + +class DDUFCorruptedFileError(DDUFError): + """Exception thrown when the DDUF file is corrupted.""" + + +class DDUFExportError(DDUFError): + """Base exception for errors during DDUF export.""" + + +class DDUFInvalidEntryNameError(DDUFExportError): + """Exception thrown when the entry name is invalid.""" + + +# STRICT DATACLASSES ERRORS + + +class StrictDataclassError(Exception): + """Base exception for strict dataclasses.""" + + +class StrictDataclassDefinitionError(StrictDataclassError): + """Exception thrown when a strict dataclass is defined incorrectly.""" + + +class StrictDataclassFieldValidationError(StrictDataclassError): + """Exception thrown when a strict dataclass fails validation for a given field.""" + + def __init__(self, field: str, cause: Exception): + error_message = f"Validation error for field '{field}':" + error_message += f"\n {cause.__class__.__name__}: {cause}" + super().__init__(error_message) + + +class StrictDataclassClassValidationError(StrictDataclassError): + """Exception thrown when a strict dataclass fails validation on a class validator.""" + + def __init__(self, validator: str, cause: Exception): + error_message = f"Class validation error for validator '{validator}':" + error_message += f"\n {cause.__class__.__name__}: {cause}" + super().__init__(error_message) + + +# XET ERRORS + + +class XetDownloadError(Exception): + """Exception thrown when the download from Xet Storage fails.""" + + +# LFS ERRORS + + +class FileDuplicationError(Exception): + """Raised when duplicating files across repos fails.""" + + +# CLI ERRORS + + +class CLIError(Exception): + """CLI error with clean message (no traceback by default).""" + + +class ConfirmationError(CLIError): + """Raised when a confirmation prompt is declined (non-interactive mode).""" + + +class CLIExtensionInstallError(CLIError): + """Error during CLI extension installation.""" + + +# SANDBOX ERRORS + + +class SandboxError(Exception): + """Base exception for sandbox operations (see `huggingface_hub.Sandbox`). + + Attributes: + status_code: The HTTP status returned by the in-sandbox server, if the error + originated from an API response (e.g. `404` for a missing file). `None` otherwise. + """ + + def __init__(self, message: str, *, status_code: int | None = None) -> None: + super().__init__(message) + self.status_code = status_code + + +class SandboxCommandError(SandboxError): + """Raised when a command run in a sandbox exits with a non-zero code. + + Attributes: + cmd: The command that failed. + result: The full `SandboxCommandResult` (exit_code, stdout, stderr, ...). + """ + + def __init__(self, cmd, result) -> None: + self.cmd = cmd + self.result = result + stderr_tail = result.stderr[-1000:] if result.stderr else "" + if result.timed_out: + reason = "timed out" + elif result.signal is not None: + reason = f"was killed by signal {result.signal}" + else: + reason = f"exited with code {result.exit_code}" + super().__init__(f"Command {cmd!r} {reason}. stderr:\n{stderr_tail}") diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/fastai_utils.py b/venv/lib/python3.12/site-packages/huggingface_hub/fastai_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2604e7b1e598d3d1c0b9dc8115e5e3c0a3fc3de2 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/fastai_utils.py @@ -0,0 +1,414 @@ +import json +import os +from pathlib import Path +from pickle import DEFAULT_PROTOCOL, PicklingError +from typing import Any + +from packaging import version + +from huggingface_hub import constants, snapshot_download +from huggingface_hub.hf_api import HfApi +from huggingface_hub.utils import ( + SoftTemporaryDirectory, + get_fastai_version, + get_fastcore_version, + get_python_version, +) + +from .utils import logging, validate_hf_hub_args + + +logger = logging.get_logger(__name__) + + +def _check_fastai_fastcore_versions( + fastai_min_version: str = "2.4", + fastcore_min_version: str = "1.3.27", +): + """ + Checks that the installed fastai and fastcore versions are compatible for pickle serialization. + + Args: + fastai_min_version (`str`, *optional*): + The minimum fastai version supported. + fastcore_min_version (`str`, *optional*): + The minimum fastcore version supported. + + > [!TIP] + > Raises the following error: + > + > - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) + > if the fastai or fastcore libraries are not available or are of an invalid version. + """ + + if (get_fastcore_version() or get_fastai_version()) == "N/A": + raise ImportError( + f"fastai>={fastai_min_version} and fastcore>={fastcore_min_version} are" + f" required. Currently using fastai=={get_fastai_version()} and" + f" fastcore=={get_fastcore_version()}." + ) + + current_fastai_version = version.Version(get_fastai_version()) + current_fastcore_version = version.Version(get_fastcore_version()) + + if current_fastai_version < version.Version(fastai_min_version): + raise ImportError( + "`push_to_hub_fastai` and `from_pretrained_fastai` require a" + f" fastai>={fastai_min_version} version, but you are using fastai version" + f" {get_fastai_version()} which is incompatible. Upgrade with `pip install" + " fastai==2.5.6`." + ) + + if current_fastcore_version < version.Version(fastcore_min_version): + raise ImportError( + "`push_to_hub_fastai` and `from_pretrained_fastai` require a" + f" fastcore>={fastcore_min_version} version, but you are using fastcore" + f" version {get_fastcore_version()} which is incompatible. Upgrade with" + " `pip install fastcore==1.3.27`." + ) + + +def _check_fastai_fastcore_pyproject_versions( + storage_folder: str, + fastai_min_version: str = "2.4", + fastcore_min_version: str = "1.3.27", +): + """ + Checks that the `pyproject.toml` file in the directory `storage_folder` has fastai and fastcore versions + that are compatible with `from_pretrained_fastai` and `push_to_hub_fastai`. If `pyproject.toml` does not exist + or does not contain versions for fastai and fastcore, then it logs a warning. + + Args: + storage_folder (`str`): + Folder to look for the `pyproject.toml` file. + fastai_min_version (`str`, *optional*): + The minimum fastai version supported. + fastcore_min_version (`str`, *optional*): + The minimum fastcore version supported. + + > [!TIP] + > Raises the following errors: + > + > - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) + > if the `toml` module is not installed. + > - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) + > if the `pyproject.toml` indicates a lower than minimum supported version of fastai or fastcore. + """ + + try: + import toml + except ModuleNotFoundError: + raise ImportError( + "`push_to_hub_fastai` and `from_pretrained_fastai` require the toml module." + " Install it with `pip install toml`." + ) + + # Checks that a `pyproject.toml`, with `build-system` and `requires` sections, exists in the repository. If so, get a list of required packages. + if not os.path.isfile(f"{storage_folder}/pyproject.toml"): + logger.warning( + "There is no `pyproject.toml` in the repository that contains the fastai" + " `Learner`. The `pyproject.toml` would allow us to verify that your fastai" + " and fastcore versions are compatible with those of the model you want to" + " load." + ) + return + pyproject_toml = toml.load(f"{storage_folder}/pyproject.toml") + + if "build-system" not in pyproject_toml.keys(): + logger.warning( + "There is no `build-system` section in the pyproject.toml of the repository" + " that contains the fastai `Learner`. The `build-system` would allow us to" + " verify that your fastai and fastcore versions are compatible with those" + " of the model you want to load." + ) + return + build_system_toml = pyproject_toml["build-system"] + + if "requires" not in build_system_toml.keys(): + logger.warning( + "There is no `requires` section in the pyproject.toml of the repository" + " that contains the fastai `Learner`. The `requires` would allow us to" + " verify that your fastai and fastcore versions are compatible with those" + " of the model you want to load." + ) + return + package_versions = build_system_toml["requires"] + + # Extracts contains fastai and fastcore versions from `pyproject.toml` if available. + # If the package is specified but not the version (e.g. "fastai" instead of "fastai=2.4"), the default versions are the highest. + fastai_packages = [pck for pck in package_versions if pck.startswith("fastai")] + if len(fastai_packages) == 0: + logger.warning("The repository does not have a fastai version specified in the `pyproject.toml`.") + # fastai_version is an empty string if not specified + else: + fastai_version = str(fastai_packages[0]).partition("=")[2] + if fastai_version != "" and version.Version(fastai_version) < version.Version(fastai_min_version): + raise ImportError( + "`from_pretrained_fastai` requires" + f" fastai>={fastai_min_version} version but the model to load uses" + f" {fastai_version} which is incompatible." + ) + + fastcore_packages = [pck for pck in package_versions if pck.startswith("fastcore")] + if len(fastcore_packages) == 0: + logger.warning("The repository does not have a fastcore version specified in the `pyproject.toml`.") + # fastcore_version is an empty string if not specified + else: + fastcore_version = str(fastcore_packages[0]).partition("=")[2] + if fastcore_version != "" and version.Version(fastcore_version) < version.Version(fastcore_min_version): + raise ImportError( + "`from_pretrained_fastai` requires" + f" fastcore>={fastcore_min_version} version, but you are using fastcore" + f" version {fastcore_version} which is incompatible." + ) + + +README_TEMPLATE = """--- +tags: +- fastai +--- + +# Amazing! + +🥳 Congratulations on hosting your fastai model on the Hugging Face Hub! + +# Some next steps +1. Fill out this model card with more information (see the template below and the [documentation here](https://huggingface.co/docs/hub/model-repos))! + +2. Create a demo in Gradio or Streamlit using 🤗 Spaces ([documentation here](https://huggingface.co/docs/hub/spaces)). + +3. Join the fastai community on the [Fastai Discord](https://discord.com/invite/YKrxeNn)! + +Greetings fellow fastlearner 🤝! Don't forget to delete this content from your model card. + + +--- + + +# Model card + +## Model description +More information needed + +## Intended uses & limitations +More information needed + +## Training and evaluation data +More information needed +""" + +PYPROJECT_TEMPLATE = f"""[build-system] +requires = ["setuptools>=40.8.0", "wheel", "python={get_python_version()}", "fastai={get_fastai_version()}", "fastcore={get_fastcore_version()}"] +build-backend = "setuptools.build_meta:__legacy__" +""" + + +def _create_model_card(repo_dir: Path): + """ + Creates a model card for the repository. + + Args: + repo_dir (`Path`): + Directory where model card is created. + """ + readme_path = repo_dir / "README.md" + + if not readme_path.exists(): + with readme_path.open("w", encoding="utf-8") as f: + f.write(README_TEMPLATE) + + +def _create_model_pyproject(repo_dir: Path): + """ + Creates a `pyproject.toml` for the repository. + + Args: + repo_dir (`Path`): + Directory where `pyproject.toml` is created. + """ + pyproject_path = repo_dir / "pyproject.toml" + + if not pyproject_path.exists(): + with pyproject_path.open("w", encoding="utf-8") as f: + f.write(PYPROJECT_TEMPLATE) + + +def _save_pretrained_fastai( + learner, + save_directory: str | Path, + config: dict[str, Any] | None = None, +): + """ + Saves a fastai learner to `save_directory` in pickle format using the default pickle protocol for the version of python used. + + Args: + learner (`Learner`): + The `fastai.Learner` you'd like to save. + save_directory (`str` or `Path`): + Specific directory in which you want to save the fastai learner. + config (`dict`, *optional*): + Configuration object. Will be uploaded as a .json file. Example: 'https://huggingface.co/espejelomar/fastai-pet-breeds-classification/blob/main/config.json'. + + > [!TIP] + > Raises the following error: + > + > - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError) + > if the config file provided is not a dictionary. + """ + _check_fastai_fastcore_versions() + + os.makedirs(save_directory, exist_ok=True) + + # if the user provides config then we update it with the fastai and fastcore versions in CONFIG_TEMPLATE. + if config is not None: + if not isinstance(config, dict): + raise RuntimeError(f"Provided config should be a dict. Got: '{type(config)}'") + path = os.path.join(save_directory, constants.CONFIG_NAME) + with open(path, "w") as f: + json.dump(config, f) + + _create_model_card(Path(save_directory)) + _create_model_pyproject(Path(save_directory)) + + # learner.export saves the model in `self.path`. + learner.path = Path(save_directory) + os.makedirs(save_directory, exist_ok=True) + try: + learner.export( + fname="model.pkl", + pickle_protocol=DEFAULT_PROTOCOL, + ) + except PicklingError: + raise PicklingError( + "You are using a lambda function, i.e., an anonymous function. `pickle`" + " cannot pickle function objects and requires that all functions have" + " names. One possible solution is to name the function." + ) + + +@validate_hf_hub_args +def from_pretrained_fastai( + repo_id: str, + revision: str | None = None, +): + """ + Load pretrained fastai model from the Hub or from a local directory. + + Args: + repo_id (`str`): + The location where the pickled fastai.Learner is. It can be either of the two: + - Hosted on the Hugging Face Hub. E.g.: 'espejelomar/fatai-pet-breeds-classification' or 'distilgpt2'. + You can add a `revision` by appending `@` at the end of `repo_id`. E.g.: `dbmdz/bert-base-german-cased@main`. + Revision is the specific model version to use. Since we use a git-based system for storing models and other + artifacts on the Hugging Face Hub, it can be a branch name, a tag name, or a commit id. + - Hosted locally. `repo_id` would be a directory containing the pickle and a pyproject.toml + indicating the fastai and fastcore versions used to build the `fastai.Learner`. E.g.: `./my_model_directory/`. + revision (`str`, *optional*): + Revision at which the repo's files are downloaded. See documentation of `snapshot_download`. + + Returns: + The `fastai.Learner` model in the `repo_id` repo. + """ + _check_fastai_fastcore_versions() + + # Load the `repo_id` repo. + # `snapshot_download` returns the folder where the model was stored. + # `cache_dir` will be the default '/root/.cache/huggingface/hub' + if not os.path.isdir(repo_id): + storage_folder = snapshot_download( + repo_id=repo_id, + revision=revision, + library_name="fastai", + library_version=get_fastai_version(), + ) + else: + storage_folder = repo_id + + _check_fastai_fastcore_pyproject_versions(storage_folder) + + from fastai.learner import load_learner # type: ignore + + return load_learner(os.path.join(storage_folder, "model.pkl")) + + +@validate_hf_hub_args +def push_to_hub_fastai( + learner, + *, + repo_id: str, + commit_message: str = "Push FastAI model using huggingface_hub.", + private: bool | None = None, + token: str | None = None, + config: dict | None = None, + branch: str | None = None, + create_pr: bool | None = None, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + delete_patterns: list[str] | str | None = None, + api_endpoint: str | None = None, +): + """ + Upload learner checkpoint files to the Hub. + + Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be pushed to the hub. Use + `delete_patterns` to delete existing remote files in the same commit. See [`upload_folder`] reference for more + details. + + Args: + learner (`Learner`): + The `fastai.Learner' you'd like to push to the Hub. + repo_id (`str`): + The repository id for your model in Hub in the format of "namespace/repo_name". The namespace can be your individual account or an organization to which you have write access (for example, 'stanfordnlp/stanza-de'). + commit_message (`str`, *optional*): + Message to commit while pushing. Will default to :obj:`"add model"`. + private (`bool`, *optional*): + Whether or not the repository created should be private. + If `None` (default), will default to been public except if the organization's default is private. + token (`str`, *optional*): + The Hugging Face account token to use as HTTP bearer authorization for remote files. If :obj:`None`, the token will be asked by a prompt. + config (`dict`, *optional*): + Configuration object to be saved alongside the model weights. + branch (`str`, *optional*): + The git branch on which to push the model. This defaults to + the default branch as specified in your repository, which + defaults to `"main"`. + create_pr (`boolean`, *optional*): + Whether or not to create a Pull Request from `branch` with that commit. + Defaults to `False`. + api_endpoint (`str`, *optional*): + The API endpoint to use when pushing the model to the hub. + allow_patterns (`list[str]` or `str`, *optional*): + If provided, only files matching at least one pattern are pushed. + ignore_patterns (`list[str]` or `str`, *optional*): + If provided, files matching any of the patterns are not pushed. + delete_patterns (`list[str]` or `str`, *optional*): + If provided, remote files matching any of the patterns will be deleted from the repo. + + Returns: + The url of the commit of your model in the given repository. + + > [!TIP] + > Raises the following error: + > + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if the user is not log on to the Hugging Face Hub. + """ + _check_fastai_fastcore_versions() + api = HfApi(endpoint=api_endpoint) + repo_id = api.create_repo(repo_id=repo_id, token=token, private=private, exist_ok=True).repo_id + + # Push the files to the repo in a single commit + with SoftTemporaryDirectory() as tmp: + saved_path = Path(tmp) / repo_id + _save_pretrained_fastai(learner, saved_path, config=config) + return api.upload_folder( + repo_id=repo_id, + token=token, + folder_path=saved_path, + commit_message=commit_message, + revision=branch, + create_pr=create_pr, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + delete_patterns=delete_patterns, + ) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/file_download.py b/venv/lib/python3.12/site-packages/huggingface_hub/file_download.py new file mode 100644 index 0000000000000000000000000000000000000000..b14bde47cf6d881274a9c23a3507dbf8bf3992b3 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/file_download.py @@ -0,0 +1,2026 @@ +import copy +import errno +import os +import re +import shutil +import stat +import time +import uuid +import warnings +from dataclasses import dataclass +from pathlib import Path +from typing import Any, BinaryIO, Literal, NoReturn, overload +from urllib.parse import quote, urlparse + +import httpx +from tqdm.auto import tqdm as base_tqdm + +from . import constants +from ._local_folder import ( + _create_cachedir_tag, + get_local_download_paths, + read_download_metadata, + write_download_metadata, +) +from ._tree_cache import read_tree_cache, tree_cache_folder_for_local_dir +from .errors import ( + FileMetadataError, + GatedRepoError, + HfHubHTTPError, + LocalEntryNotFoundError, + RemoteEntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, +) +from .utils import ( + OfflineModeIsEnabled, + SoftTemporaryDirectory, + WeakFileLock, + XetFileData, + build_hf_headers, + hf_raise_for_status, + logging, + parse_xet_file_data_from_response, + tqdm, + validate_hf_hub_args, +) +from .utils._http import ( + _DEFAULT_RETRY_ON_EXCEPTIONS, + _DEFAULT_RETRY_ON_STATUS_CODES, + _adjust_range_header, + _httpx_follow_relative_redirects_with_backoff, + http_stream_backoff, +) +from .utils._runtime import is_xet_available +from .utils._xet import XetTokenType, xet_connection_info_refresh_url +from .utils.sha import sha_fileobj +from .utils.tqdm import _get_progress_bar_context + + +logger = logging.get_logger(__name__) + +# Return value when trying to load a file from cache but the file does not exist in the distant repo. +_CACHED_NO_EXIST = object() +_CACHED_NO_EXIST_T = Any + +# Regex to get filename from a "Content-Disposition" header for CDN-served files +HEADER_FILENAME_PATTERN = re.compile(r'filename="(?P.*?)";') + +# Regex to check if the revision IS directly a commit_hash +REGEX_COMMIT_HASH = re.compile(r"^[0-9a-f]{40}$") + +# Regex to check if the file etag IS a valid sha256 +REGEX_SHA256 = re.compile(r"^[0-9a-f]{64}$") + +_are_symlinks_supported_in_dir: dict[str, bool] = {} + +# Internal retry timeout for metadata fetch when no local file exists +_ETAG_RETRY_TIMEOUT = 60 + + +def are_symlinks_supported(cache_dir: str | Path | None = None) -> bool: + """Return whether the symlinks are supported on the machine. + + Since symlinks support can change depending on the mounted disk, we need to check + on the precise cache folder. By default, the default HF cache directory is checked. + + Args: + cache_dir (`str`, `Path`, *optional*): + Path to the folder where cached files are stored. + + Returns: [bool] Whether symlinks are supported in the directory. + """ + # Defaults to HF cache + if cache_dir is None: + cache_dir = constants.HF_HUB_CACHE + cache_dir = str(Path(cache_dir).expanduser().resolve()) # make it unique + + # If symlinks are explicitly disabled by the user, always return False + if constants.HF_HUB_DISABLE_SYMLINKS: + return False + + # Check symlink compatibility only once (per cache directory) at first time use + if cache_dir not in _are_symlinks_supported_in_dir: + _are_symlinks_supported_in_dir[cache_dir] = True + + os.makedirs(cache_dir, exist_ok=True) + with SoftTemporaryDirectory(dir=cache_dir) as tmpdir: + src_path = Path(tmpdir) / "dummy_file_src" + src_path.touch() + dst_path = Path(tmpdir) / "dummy_file_dst" + + # Relative source path as in `_create_symlink`` + relative_src = os.path.relpath(src_path, start=os.path.dirname(dst_path)) + try: + os.symlink(relative_src, dst_path) + except OSError: + # Likely running on Windows + _are_symlinks_supported_in_dir[cache_dir] = False + + if not constants.HF_HUB_DISABLE_SYMLINKS_WARNING: + message = ( + "`huggingface_hub` cache-system uses symlinks by default to" + " efficiently store duplicated files but your machine does not" + f" support them in {cache_dir}. Caching files will still work" + " but in a degraded version that might require more space on" + " your disk. This warning can be disabled by setting the" + " `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For" + " more details, see" + " https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations." + ) + if os.name == "nt": + message += ( + "\nTo support symlinks on Windows, you either need to" + " activate Developer Mode or to run Python as an" + " administrator. In order to activate developer mode," + " see this article:" + " https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development" + ) + warnings.warn(message) + + return _are_symlinks_supported_in_dir[cache_dir] + + +@dataclass(frozen=True) +class HfFileMetadata: + """Data structure containing information about a file versioned on the Hub. + + Returned by [`get_hf_file_metadata`] based on a URL. + + Args: + commit_hash (`str`, *optional*): + The commit_hash related to the file. + etag (`str`, *optional*): + Etag of the file on the server. + location (`str`): + Location where to download the file. Can be a Hub url or not (CDN). + size (`size`): + Size of the file. In case of an LFS file, contains the size of the actual + LFS file, not the pointer. + xet_file_data (`XetFileData`, *optional*): + Xet information for the file. This is only set if the file is stored using Xet storage. + """ + + commit_hash: str | None + etag: str | None + location: str + size: int | None + xet_file_data: XetFileData | None + + +@dataclass +class DryRunFileInfo: + """Information returned when performing a dry run of a file download. + + Returned by [`hf_hub_download`] when `dry_run=True`. + + Args: + commit_hash (`str`): + The commit_hash related to the file. + file_size (`int`): + Size of the file. In case of an LFS file, contains the size of the actual LFS file, not the pointer. + filename (`str`): + Name of the file in the repo. + is_cached (`bool`): + Whether the file is already cached locally. + will_download (`bool`): + Whether the file will be downloaded if `hf_hub_download` is called with `dry_run=False`. + In practice, will_download is `True` if the file is not cached or if `force_download=True`. + """ + + commit_hash: str + file_size: int + filename: str + local_path: str + is_cached: bool + will_download: bool + + +@validate_hf_hub_args +def hf_hub_url( + repo_id: str, + filename: str, + *, + subfolder: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + endpoint: str | None = None, +) -> str: + """Construct the URL of a file from the given information. + + The resolved address can either be a huggingface.co-hosted url, or a link to + Cloudfront (a Content Delivery Network, or CDN) for large files which are + more than a few MBs. + + Args: + repo_id (`str`): + A namespace (user or an organization) name and a repo name separated + by a `/`. + filename (`str`): + The name of the file in the repo. + subfolder (`str`, *optional*): + An optional value corresponding to a folder inside the repo. + repo_type (`str`, *optional*): + Set to `"dataset"`, `"space"` or `"kernel"` if downloading from a dataset, space or kernel repo, + `None` or `"model"` if downloading from a model. Default is `None`. + revision (`str`, *optional*): + An optional Git revision id which can be a branch name, a tag, or a + commit hash. + endpoint (`str`, *optional*): + The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`. + + Example: + + ```python + >>> from huggingface_hub import hf_hub_url + + >>> hf_hub_url( + ... repo_id="julien-c/EsperBERTo-small", filename="pytorch_model.bin" + ... ) + 'https://huggingface.co/julien-c/EsperBERTo-small/resolve/main/pytorch_model.bin' + ``` + + > [!TIP] + > Notes: + > + > Cloudfront is replicated over the globe so downloads are way faster for + > the end user (and it also lowers our bandwidth costs). + > + > Cloudfront aggressively caches files by default (default TTL is 24 + > hours), however this is not an issue here because we implement a + > git-based versioning system on huggingface.co, which means that we store + > the files on S3/Cloudfront in a content-addressable way (i.e., the file + > name is its hash). Using content-addressable filenames means cache can't + > ever be stale. + > + > In terms of client-side caching from this library, we base our caching + > on the objects' entity tag (`ETag`), which is an identifier of a + > specific version of a resource [1]_. An object's ETag is: its git-sha1 + > if stored in git, or its sha256 if stored in git-lfs. + + References: + + - [1] https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag + """ + if subfolder == "": + subfolder = None + if subfolder is not None: + filename = f"{subfolder}/{filename}" + + if repo_type not in constants.REPO_TYPES_WITH_KERNEL: + raise ValueError("Invalid repo type") + + if repo_type in constants.REPO_TYPES_URL_PREFIXES: + repo_id = constants.REPO_TYPES_URL_PREFIXES[repo_type] + repo_id # type: ignore + + if revision is None: + revision = constants.DEFAULT_REVISION + url = constants.HUGGINGFACE_CO_URL_TEMPLATE.format( + repo_id=repo_id, revision=quote(revision, safe=""), filename=quote(filename) + ) + # Update endpoint if provided + if endpoint is not None and url.startswith(constants.ENDPOINT): + url = endpoint + url[len(constants.ENDPOINT) :] + return url + + +def _get_file_length_from_http_response(response: httpx.Response) -> int | None: + """ + Get the length of the file from the HTTP response headers. + + This function extracts the file size from the HTTP response headers, either from the + `Content-Range` or `Content-Length` header, if available (in that order). + + Args: + response (`httpx.Response`): + The HTTP response object. + + Returns: + `int` or `None`: The length of the file in bytes, or None if not available. + """ + + # If HTTP response contains compressed body (e.g. gzip), the `Content-Length` header will + # contain the length of the compressed body, not the uncompressed file size. + # And at the start of transmission there's no way to know the uncompressed file size for gzip, + # thus we return None in that case. + content_encoding = response.headers.get("Content-Encoding", "identity").lower() + if content_encoding != "identity": + # gzip/br/deflate/zstd etc + return None + + content_range = response.headers.get("Content-Range") + if content_range is not None: + return int(content_range.rsplit("/")[-1]) + + content_length = response.headers.get("Content-Length") + if content_length is not None: + return int(content_length) + + return None + + +@validate_hf_hub_args +def http_get( + url: str, + temp_file: BinaryIO, + *, + resume_size: int = 0, + headers: dict[str, Any] | None = None, + expected_size: int | None = None, + displayed_filename: str | None = None, + tqdm_class: type[base_tqdm] | None = None, + _nb_retries: int = 5, + _tqdm_bar: tqdm | None = None, +) -> None: + """ + Download a remote file. Do not gobble up errors, and will return errors tailored to the Hugging Face Hub. + + If ConnectionError (SSLError) or ReadTimeout happen while streaming data from the server, it is most likely a + transient error (network outage?). We log a warning message and try to resume the download a few times before + giving up. The method gives up after 5 attempts if no new data has being received from the server. + + Args: + url (`str`): + The URL of the file to download. + temp_file (`BinaryIO`): + The file-like object where to save the file. + resume_size (`int`, *optional*): + The number of bytes already downloaded. If set to 0 (default), the whole file is download. If set to a + positive number, the download will resume at the given position. + headers (`dict`, *optional*): + Dictionary of HTTP Headers to send with the request. + expected_size (`int`, *optional*): + The expected size of the file to download. If set, the download will raise an error if the size of the + received content is different from the expected one. + displayed_filename (`str`, *optional*): + The filename of the file that is being downloaded. Value is used only to display a nice progress bar. If + not set, the filename is guessed from the URL or the `Content-Disposition` header. + """ + if expected_size is not None and resume_size == expected_size: + # If the file is already fully downloaded, we don't need to download it again. + return + + initial_headers = headers + headers = copy.deepcopy(headers) or {} + if resume_size > 0: + headers["Range"] = _adjust_range_header(headers.get("Range"), resume_size) + elif expected_size and expected_size > constants.MAX_HTTP_DOWNLOAD_SIZE: + # Any files over 50GB will not be available through basic http requests. + raise ValueError( + "The file is too large to be downloaded using the regular download method. " + " Install `hf_xet` with `pip install hf_xet` for xet-powered downloads." + ) + + with http_stream_backoff( + method="GET", + url=url, + headers=headers, + timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT, + retry_on_exceptions=(), + retry_on_status_codes=(408, 429), + ) as response: + hf_raise_for_status(response) + + # If we requested a Range but got 200 back, the server ignored our Range header + # (e.g. CloudFront with Accept-Encoding: gzip). Reset file to avoid corruption. + if resume_size > 0 and response.status_code == 200: + temp_file.seek(0) + temp_file.truncate() + if _tqdm_bar is not None: + # When the progress bar is reused across retries, its counter has already been advanced by `resume_size` + # worth of chunks from earlier attempts. Those bytes are gone from disk now, so roll the counter back + # to keep the upcoming full re-download from double-counting (e.g. ending at 130/100 on a 100-byte file). + _tqdm_bar.update(-resume_size) + if callable(update_transfer := getattr(_tqdm_bar, "update_transfer", None)): + update_transfer(-resume_size) + resume_size = 0 + + total: int | None = _get_file_length_from_http_response(response) + if total is None: + # Hub serves compressible text files (e.g. vocab.json) with `Content-Encoding: gzip` and + # `Transfer-Encoding: chunked`, so the response carries no `Content-Length`. Fall back to the caller's + # `expected_size` (always known from the metadata HEAD on the hf_hub path) so the progress bar, and any + # aggregating wrapper such as snapshot_download's `_AggregatedTqdm` — still sees the file size. + total = expected_size + + if displayed_filename is None: + displayed_filename = url + content_disposition = response.headers.get("Content-Disposition") + if content_disposition is not None: + match = HEADER_FILENAME_PATTERN.search(content_disposition) + if match is not None: + # Means file is on CDN + displayed_filename = match.groupdict()["filename"] + + # Truncate filename if too long to display + if len(displayed_filename) > 40: + displayed_filename = f"(…){displayed_filename[-40:]}" + + consistency_error_message = ( + f"Consistency check failed: file should be of size {expected_size} but has size" + f" {{actual_size}} ({displayed_filename}).\nThis is usually due to network issues while downloading the file." + " Please retry with `force_download=True`." + ) + progress_cm = _get_progress_bar_context( + desc=displayed_filename, + log_level=logger.getEffectiveLevel(), + total=total, + initial=resume_size, + name="huggingface_hub.http_get", + tqdm_class=tqdm_class, + _tqdm_bar=_tqdm_bar, + ) + + with progress_cm as progress: + new_resume_size = resume_size + try: + for chunk in response.iter_bytes(chunk_size=constants.DOWNLOAD_CHUNK_SIZE): + if chunk: # filter out keep-alive new chunks + progress.update(len(chunk)) + if callable(update_transfer := getattr(progress, "update_transfer", None)): + update_transfer(len(chunk)) + temp_file.write(chunk) + new_resume_size += len(chunk) + # Some data has been downloaded from the server so we reset the number of retries. + _nb_retries = 5 + except (httpx.ConnectError, httpx.TimeoutException, httpx.RemoteProtocolError) as e: + # If ConnectionError (SSLError), ReadTimeout, or RemoteProtocolError (peer closed the connection before + # sending the complete body) happen while streaming data from the server, it is most likely a transient + # error (network outage?). We log a warning message and try to resume the download a few times before + # giving up. The retry mechanism is basic but should be enough in most cases. + if _nb_retries <= 0: + logger.warning("Error while downloading from %s: %s\nMax retries exceeded.", url, str(e)) + raise + logger.warning("Error while downloading from %s: %s\nTrying to resume download...", url, str(e)) + time.sleep(1) + return http_get( + url=url, + temp_file=temp_file, + resume_size=new_resume_size, + headers=initial_headers, + expected_size=expected_size, + tqdm_class=tqdm_class, + _nb_retries=_nb_retries - 1, + # Reuse the existing progress bar across retries so a custom `tqdm_class` (e.g. snapshot_download's `_AggregatedTqdm`, + # which mutates a shared parent bar in `__init__`) is not re-instantiated and does not double-count `total`/`initial`. + _tqdm_bar=progress, + ) + + if expected_size is not None and expected_size != temp_file.tell(): + raise OSError( + consistency_error_message.format( + actual_size=temp_file.tell(), + ) + ) + + +def xet_get( + *, + incomplete_path: Path, + xet_file_data: XetFileData, + headers: dict[str, str], + expected_size: int | None = None, + displayed_filename: str | None = None, + tqdm_class: type[base_tqdm] | None = None, + _tqdm_bar: tqdm | None = None, +) -> None: + """ + Download a file using Xet storage service. + + Args: + incomplete_path (`Path`): + The path to the file to download. + xet_file_data (`XetFileData`): + The file metadata needed to make the request to the xet storage service. + headers (`dict[str, str]`): + The headers to send to the xet storage service. + expected_size (`int`, *optional*): + The expected size of the file to download. If set, the download will raise an error if the size of the + received content is different from the expected one. + displayed_filename (`str`, *optional*): + The filename of the file that is being downloaded. Value is used only to display a nice progress bar. If + not set, the filename is guessed from the URL or the `Content-Disposition` header. + + **How it works:** + The file download system uses Xet storage, which is a content-addressable storage system that breaks files into chunks + for efficient storage and transfer. + + ``session.new_file_download_group()`` manages downloading files by: + - Registering download tasks (each with its unique content hash) and starting download immediately in the background + - Connecting to a storage server (CAS server) that knows how files are chunked + - Using authentication to ensure secure access + - Providing progress updates during download + + Authentication works transparently: the download group accepts a ``token_refresh_url`` + that is used to refresh the short-lived xet access token as needed. + + The download process works like this: + 1. Download tasks run in parallel: + 1.1. Prepare to write the file to disk or to a stream (e.g. truncate file, set up cache) + 1.2. Ask the server "how is this file split into chunks?" using the file's unique hash + The server responds with: + - Which chunks make up the complete file + - Where each chunk can be downloaded from + 1.3. For each needed chunk: + - Checks if we already have it in our local cache + - If not, download it from cloud storage (S3) + - Save it to cache for future use + - Assemble the chunks in order to recreate the original file + + """ + try: + from hf_xet import XetFileInfo # type: ignore[no-redef] + except ImportError: + raise ValueError( + "To use optimized download using Xet storage, you need to install the hf_xet package. " + 'Try `pip install "huggingface_hub[hf_xet]"` or `pip install hf_xet`.' + ) + + if not displayed_filename: + displayed_filename = incomplete_path.name + + # Truncate filename if too long to display + if len(displayed_filename) > 40: + displayed_filename = f"{displayed_filename[:40]}(…)" + + from .utils._xet import abort_xet_session, get_xet_session, xet_headers_without_auth + from .utils._xet_progress_reporting import XetDownloadProgressReporter + + xet_headers = xet_headers_without_auth(headers) + + session = get_xet_session() + + with XetDownloadProgressReporter( + reconstruction_desc=f"{displayed_filename}: reconstructing file", + transfer_desc=f"{displayed_filename}: downloading bytes", + total=expected_size, + log_level=logger.getEffectiveLevel(), + name="huggingface_hub.xet_get", + tqdm_class=tqdm_class, + external_reconstruction_bar=_tqdm_bar, + ) as progress: + try: + with session.new_file_download_group( + token_refresh_url=xet_file_data.refresh_route, + token_refresh_headers=headers, + custom_headers=xet_headers, + progress_callback=progress.update_progress, + ) as group: + group.start_download_file( + XetFileInfo(xet_file_data.file_hash, expected_size), str(incomplete_path.absolute()) + ) + except KeyboardInterrupt: + abort_xet_session() + raise + + +def _normalize_etag(etag: str | None) -> str | None: + """Normalize ETag HTTP header, so it can be used to create nice filepaths. + + The HTTP spec allows two forms of ETag: + ETag: W/"" + ETag: "" + + For now, we only expect the second form from the server, but we want to be future-proof so we support both. For + more context, see `TestNormalizeEtag` tests and https://github.com/huggingface/huggingface_hub/pull/1428. + + Args: + etag (`str`, *optional*): HTTP header + + Returns: + `str` or `None`: string that can be used as a nice directory name. + Returns `None` if input is None. + """ + if etag is None: + return None + return etag.lstrip("W/").strip('"') + + +def _create_relative_symlink(src: str, dst: str, new_blob: bool = False) -> None: + """Alias method used in `transformers` conversion script.""" + return _create_symlink(src=src, dst=dst, new_blob=new_blob) + + +def _create_symlink(src: str, dst: str, new_blob: bool = False) -> None: + """Create a symbolic link named dst pointing to src. + + By default, it will try to create a symlink using a relative path. Relative paths have 2 advantages: + - If the cache_folder is moved (example: back-up on a shared drive), relative paths within the cache folder will + not break. + - Relative paths seems to be better handled on Windows. Issue was reported 3 times in less than a week when + changing from relative to absolute paths. See https://github.com/huggingface/huggingface_hub/issues/1398, + https://github.com/huggingface/diffusers/issues/2729 and https://github.com/huggingface/transformers/pull/22228. + NOTE: The issue with absolute paths doesn't happen on admin mode. + When creating a symlink from the cache to a local folder, it is possible that a relative path cannot be created. + This happens when paths are not on the same volume. In that case, we use absolute paths. + + + The result layout looks something like + └── [ 128] snapshots + ├── [ 128] 2439f60ef33a0d46d85da5001d52aeda5b00ce9f + │ ├── [ 52] README.md -> ../../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812 + │ └── [ 76] pytorch_model.bin -> ../../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd + + If symlinks cannot be created on this platform (most likely to be Windows), the workaround is to avoid symlinks by + having the actual file in `dst`. If it is a new file (`new_blob=True`), we move it to `dst`. If it is not a new file + (`new_blob=False`), we don't know if the blob file is already referenced elsewhere. To avoid breaking existing + cache, the file is duplicated on the disk. + + In case symlinks are not supported, a warning message is displayed to the user once when loading `huggingface_hub`. + The warning message can be disabled with the `DISABLE_SYMLINKS_WARNING` environment variable. + """ + try: + os.remove(dst) + except OSError: + pass + + abs_src = os.path.abspath(os.path.expanduser(src)) + abs_dst = os.path.abspath(os.path.expanduser(dst)) + abs_dst_folder = os.path.dirname(abs_dst) + + # Use relative_dst in priority + try: + relative_src = os.path.relpath(abs_src, abs_dst_folder) + except ValueError: + # Raised on Windows if src and dst are not on the same volume. This is the case when creating a symlink to a + # local_dir instead of within the cache directory. + # See https://docs.python.org/3/library/os.path.html#os.path.relpath + relative_src = None + + try: + commonpath = os.path.commonpath([abs_src, abs_dst]) + _support_symlinks = are_symlinks_supported(commonpath) + except ValueError: + # Raised if src and dst are not on the same volume. Symlinks will still work on Linux/Macos. + # See https://docs.python.org/3/library/os.path.html#os.path.commonpath + _support_symlinks = os.name != "nt" and not constants.HF_HUB_DISABLE_SYMLINKS + except PermissionError: + # Permission error means src and dst are not in the same volume (e.g. destination path has been provided + # by the user via `local_dir`. Let's test symlink support there) + _support_symlinks = are_symlinks_supported(abs_dst_folder) + except OSError as e: + # OS error (errno=30) means that the commonpath is readonly on Linux/MacOS. + if e.errno == errno.EROFS: + _support_symlinks = are_symlinks_supported(abs_dst_folder) + else: + raise + + # Symlinks are supported => let's create a symlink. + if _support_symlinks: + src_rel_or_abs = relative_src or abs_src + logger.debug(f"Creating pointer from {src_rel_or_abs} to {abs_dst}") + try: + os.symlink(src_rel_or_abs, abs_dst) + return + except FileExistsError: + if os.path.islink(abs_dst) and os.path.realpath(abs_dst) == os.path.realpath(abs_src): + # `abs_dst` already exists and is a symlink to the `abs_src` blob. It is most likely that the file has + # been cached twice concurrently (exactly between `os.remove` and `os.symlink`). Do nothing. + return + else: + # Very unlikely to happen. Means a file `dst` has been created exactly between `os.remove` and + # `os.symlink` and is not a symlink to the `abs_src` blob file. Raise exception. + raise + except PermissionError: + # Permission error means src and dst are not in the same volume (e.g. download to local dir) and symlink + # is supported on both volumes but not between them. Let's just make a hard copy in that case. + pass + + # Symlinks are not supported => let's move or copy the file. + if new_blob: + logger.debug(f"Symlink not supported. Moving file from {abs_src} to {abs_dst}") + shutil.move(abs_src, abs_dst, copy_function=_copy_no_matter_what) + else: + logger.debug(f"Symlink not supported. Copying file from {abs_src} to {abs_dst}") + shutil.copyfile(abs_src, abs_dst) + + +def _cache_commit_hash_for_specific_revision(storage_folder: str, revision: str, commit_hash: str) -> None: + """Cache reference between a revision (tag, branch or truncated commit hash) and the corresponding commit hash. + + Does nothing if `revision` is already a proper `commit_hash` or reference is already cached. + """ + if revision != commit_hash: + ref_path = Path(storage_folder) / "refs" / revision + ref_path.parent.mkdir(parents=True, exist_ok=True) + if not ref_path.exists() or commit_hash != ref_path.read_text(): + # Update ref only if has been updated. Could cause useless error in case + # repo is already cached and user doesn't have write access to cache folder. + # See https://github.com/huggingface/huggingface_hub/issues/1216. + # Write atomically (tmp file + rename) so that concurrent readers never see + # a partially written ref. + tmp_path = ref_path.with_name(f"{ref_path.name}.{uuid.uuid4().hex[:8]}.tmp") + tmp_path.write_text(commit_hash) + os.replace(tmp_path, ref_path) + + +@validate_hf_hub_args +def repo_folder_name(*, repo_id: str, repo_type: str) -> str: + """Return a serialized version of a hf.co repo name and type, safe for disk storage + as a single non-nested folder. + + Example: models--julien-c--EsperBERTo-small + """ + # remove all `/` occurrences to correctly convert repo to directory name + parts = [f"{repo_type}s", *repo_id.split("/")] + return constants.REPO_ID_SEPARATOR.join(parts) + + +def _check_disk_space(expected_size: int, target_dir: str | Path) -> None: + """Check disk usage and log a warning if there is not enough disk space to download the file. + + Args: + expected_size (`int`): + The expected size of the file in bytes. + target_dir (`str`): + The directory where the file will be stored after downloading. + """ + + target_dir = Path(target_dir) # format as `Path` + for path in [target_dir] + list(target_dir.parents): # first check target_dir, then each parents one by one + try: + target_dir_free = shutil.disk_usage(path).free + if target_dir_free < expected_size: + warnings.warn( + "Not enough free disk space to download the file. " + f"The expected file size is: {expected_size / 1e6:.2f} MB. " + f"The target location {target_dir} only has {target_dir_free / 1e6:.2f} MB free disk space." + ) + return + except OSError: # raise on anything: file does not exist or space disk cannot be checked + pass + + +@overload +def hf_hub_download( + repo_id: str, + filename: str, + *, + subfolder: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + library_name: str | None = None, + library_version: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + user_agent: dict | str | None = None, + force_download: bool = False, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + token: bool | str | None = None, + local_files_only: bool = False, + headers: dict[str, str] | None = None, + endpoint: str | None = None, + tqdm_class: type[base_tqdm] | None = None, + dry_run: Literal[False] = False, +) -> str: ... + + +@overload +def hf_hub_download( + repo_id: str, + filename: str, + *, + subfolder: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + library_name: str | None = None, + library_version: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + user_agent: dict | str | None = None, + force_download: bool = False, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + token: bool | str | None = None, + local_files_only: bool = False, + headers: dict[str, str] | None = None, + endpoint: str | None = None, + tqdm_class: type[base_tqdm] | None = None, + dry_run: Literal[True] = True, +) -> DryRunFileInfo: ... + + +@overload +def hf_hub_download( + repo_id: str, + filename: str, + *, + subfolder: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + library_name: str | None = None, + library_version: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + user_agent: dict | str | None = None, + force_download: bool = False, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + token: bool | str | None = None, + local_files_only: bool = False, + headers: dict[str, str] | None = None, + endpoint: str | None = None, + tqdm_class: type[base_tqdm] | None = None, + dry_run: bool = False, +) -> str | DryRunFileInfo: ... + + +@validate_hf_hub_args +def hf_hub_download( + repo_id: str, + filename: str, + *, + subfolder: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + library_name: str | None = None, + library_version: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + user_agent: dict | str | None = None, + force_download: bool = False, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + token: bool | str | None = None, + local_files_only: bool = False, + headers: dict[str, str] | None = None, + endpoint: str | None = None, + tqdm_class: type[base_tqdm] | None = None, + dry_run: bool = False, +) -> str | DryRunFileInfo: + """Download a given file if it's not already present in the local cache. + + The new cache file layout looks like this: + - The cache directory contains one subfolder per repo_id (namespaced by repo type) + - inside each repo folder: + - refs is a list of the latest known revision => commit_hash pairs + - blobs contains the actual file blobs (identified by their git-sha or sha256, depending on + whether they're LFS files or not) + - snapshots contains one subfolder per commit, each "commit" contains the subset of the files + that have been resolved at that particular commit. Each filename is a symlink to the blob + at that particular commit. + + ``` + [ 96] . + └── [ 160] models--julien-c--EsperBERTo-small + ├── [ 160] blobs + │ ├── [321M] 403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd + │ ├── [ 398] 7cb18dc9bafbfcf74629a4b760af1b160957a83e + │ └── [1.4K] d7edf6bd2a681fb0175f7735299831ee1b22b812 + ├── [ 96] refs + │ └── [ 40] main + └── [ 128] snapshots + ├── [ 128] 2439f60ef33a0d46d85da5001d52aeda5b00ce9f + │ ├── [ 52] README.md -> ../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812 + │ └── [ 76] pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd + └── [ 128] bbc77c8132af1cc5cf678da3f1ddf2de43606d48 + ├── [ 52] README.md -> ../../blobs/7cb18dc9bafbfcf74629a4b760af1b160957a83e + └── [ 76] pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd + ``` + + If `local_dir` is provided, the file structure from the repo will be replicated in this location. When using this + option, the `cache_dir` will not be used and a `.cache/huggingface/` folder will be created at the root of `local_dir` + to store some metadata related to the downloaded files. While this mechanism is not as robust as the main + cache-system, it's optimized for regularly pulling the latest version of a repository. + + Args: + repo_id (`str`): + A user or an organization name and a repo name separated by a `/`. + filename (`str`): + The name of the file in the repo. + subfolder (`str`, *optional*): + An optional value corresponding to a folder inside the model repo. + repo_type (`str`, *optional*): + Set to `"dataset"`, `"space"` or `"kernel"` if downloading from a dataset, space or kernel repo, + `None` or `"model"` if downloading from a model. Default is `None`. + revision (`str`, *optional*): + An optional Git revision id which can be a branch name, a tag, or a + commit hash. + library_name (`str`, *optional*): + The name of the library to which the object corresponds. + library_version (`str`, *optional*): + The version of the library. + cache_dir (`str`, `Path`, *optional*): + Path to the folder where cached files are stored. + local_dir (`str` or `Path`, *optional*): + If provided, the downloaded file will be placed under this directory. + user_agent (`dict`, `str`, *optional*): + The user-agent info in the form of a dictionary or a string. + force_download (`bool`, *optional*, defaults to `False`): + Whether the file should be downloaded even if it already exists in + the local cache. + etag_timeout (`float`, *optional*, defaults to `10`): + When fetching ETag, how many seconds to wait for the server to send + data before giving up, which is passed to `httpx.request`. + token (`str`, `bool`, *optional*): + A token to be used for the download. + - If `True`, the token is read from the HuggingFace config + folder. + - If a string, it's used as the authentication token. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, avoid downloading the file and return the path to the + local cached file if it exists. + headers (`dict`, *optional*): + Additional headers to be sent with the request. + endpoint (`str`, *optional*): + The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`. + tqdm_class (`tqdm`, *optional*): + If provided, overwrites the default behavior for the progress bar. Passed + argument must inherit from `tqdm.auto.tqdm` or at least mimic its behavior. + Defaults to the custom HF progress bar that can be disabled by setting + `HF_HUB_DISABLE_PROGRESS_BARS` environment variable. + dry_run (`bool`, *optional*, defaults to `False`): + If `True`, perform a dry run without actually downloading the file. Returns a + [`DryRunFileInfo`] object containing information about what would be downloaded. + + Returns: + `str` or [`DryRunFileInfo`]: + - If `dry_run=False`: Local path of file or if networking is off, last version of file cached on disk. + - If `dry_run=True`: A [`DryRunFileInfo`] object containing download information. + + Raises: + [`~utils.RepositoryNotFoundError`] + If the repository to download from cannot be found. This may be because it doesn't exist, + or because it is set to `private` and you do not have access. + [`~utils.RevisionNotFoundError`] + If the revision to download from cannot be found. + [`~utils.RemoteEntryNotFoundError`] + If the file to download cannot be found. + [`~utils.LocalEntryNotFoundError`] + If network is disabled or unavailable and file is not found in cache. + [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) + If `token=True` but the token cannot be found. + [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) + If ETag cannot be determined. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If some parameter value is invalid. + + """ + if constants.HF_HUB_ETAG_TIMEOUT != constants.DEFAULT_ETAG_TIMEOUT: + # Respect environment variable above user value + etag_timeout = constants.HF_HUB_ETAG_TIMEOUT + + if revision is None: + revision = constants.DEFAULT_REVISION + + if cache_dir is None: + cache_dir = constants.HF_HUB_CACHE + cache_dir = str(Path(cache_dir).expanduser().resolve()) + + if local_dir is not None: + local_dir = str(Path(local_dir).expanduser().resolve()) + + if subfolder == "": + subfolder = None + if subfolder is not None: + # This is used to create a URL, and not a local path, hence the forward slash. + filename = f"{subfolder}/{filename}" + + if repo_type is None: + repo_type = "model" + if repo_type not in constants.REPO_TYPES_WITH_KERNEL: + raise ValueError( + f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES_WITH_KERNEL)}" + ) + + hf_headers = build_hf_headers( + token=token, + library_name=library_name, + library_version=library_version, + user_agent=user_agent, + headers=headers, + ) + + if local_dir is not None: + return _hf_hub_download_to_local_dir( + # Destination + local_dir=local_dir, + # File info + repo_id=repo_id, + repo_type=repo_type, + filename=filename, + revision=revision, + # HTTP info + endpoint=endpoint, + etag_timeout=etag_timeout, + headers=hf_headers, + token=token, + # Additional options + cache_dir=cache_dir, + force_download=force_download, + local_files_only=local_files_only, + tqdm_class=tqdm_class, + dry_run=dry_run, + ) + else: + return _hf_hub_download_to_cache_dir( + # Destination + cache_dir=cache_dir, + # File info + repo_id=repo_id, + filename=filename, + repo_type=repo_type, + revision=revision, + # HTTP info + endpoint=endpoint, + etag_timeout=etag_timeout, + headers=hf_headers, + token=token, + # Additional options + local_files_only=local_files_only, + force_download=force_download, + tqdm_class=tqdm_class, + dry_run=dry_run, + ) + + +def _hf_hub_download_to_cache_dir( + *, + # Destination + cache_dir: str, + # File info + repo_id: str, + filename: str, + repo_type: str, + revision: str, + # HTTP info + endpoint: str | None, + etag_timeout: float, + headers: dict[str, str], + token: bool | str | None, + # Additional options + local_files_only: bool, + force_download: bool, + tqdm_class: type[base_tqdm] | None, + dry_run: bool, +) -> str | DryRunFileInfo: + """Download a given file to a cache folder, if not already present. + + Method should not be called directly. Please use `hf_hub_download` instead. + """ + locks_dir = os.path.join(cache_dir, ".locks") + storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type)) + + # cross-platform transcription of filename, to be used as a local file path. + relative_filename = os.path.join(*filename.split("/")) + if os.name == "nt": + if relative_filename.startswith("..\\") or "\\..\\" in relative_filename: + raise ValueError( + f"Invalid filename: cannot handle filename '{relative_filename}' on Windows. Please ask the repository" + " owner to rename this file." + ) + + # if user provides a commit_hash and they already have the file on disk, shortcut everything. + if REGEX_COMMIT_HASH.match(revision): + pointer_path = _get_pointer_path(storage_folder, revision, relative_filename) + if os.path.exists(pointer_path): + if dry_run: + return DryRunFileInfo( + commit_hash=revision, + file_size=os.path.getsize(pointer_path), + filename=filename, + is_cached=True, + local_path=pointer_path, + will_download=force_download, + ) + if not force_download: + return pointer_path + + # Try to get metadata (etag, commit_hash, url, size) from the server. + # If we can't, a HEAD request error is returned. + (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_call_error) = _get_metadata_or_catch_error( + repo_id=repo_id, + filename=filename, + repo_type=repo_type, + revision=revision, + endpoint=endpoint, + etag_timeout=etag_timeout, + headers=headers, + token=token, + local_files_only=local_files_only, + storage_folder=storage_folder, + relative_filename=relative_filename, + tree_cache_folder=storage_folder, + ) + + # etag can be None for several reasons: + # 1. we passed local_files_only. + # 2. we don't have a connection + # 3. Hub is down (HTTP 500, 503, 504) + # 4. repo is not found -for example private or gated- and invalid/missing token sent + # 5. Hub is blocked by a firewall or proxy is not set correctly. + # => Try to get the last downloaded one from the specified revision. + # + # If the specified revision is a commit hash, look inside "snapshots". + # If the specified revision is a branch or tag, look inside "refs". + if head_call_error is not None: + # Couldn't make a HEAD call => let's try to find a local file + if not force_download: + commit_hash = None + if REGEX_COMMIT_HASH.match(revision): + commit_hash = revision + else: + ref_path = os.path.join(storage_folder, "refs", revision) + if os.path.isfile(ref_path): + with open(ref_path) as f: + commit_hash = f.read() + + # Return pointer file if exists + if commit_hash is not None: + pointer_path = _get_pointer_path(storage_folder, commit_hash, relative_filename) + if os.path.exists(pointer_path): + if dry_run: + return DryRunFileInfo( + commit_hash=commit_hash, + file_size=os.path.getsize(pointer_path), + filename=filename, + is_cached=True, + local_path=pointer_path, + will_download=force_download, + ) + if not force_download: + return pointer_path + + if isinstance(head_call_error, _DEFAULT_RETRY_ON_EXCEPTIONS) or ( + isinstance(head_call_error, HfHubHTTPError) + and head_call_error.response.status_code in _DEFAULT_RETRY_ON_STATUS_CODES + ): + logger.info("No local file found. Retrying..") + (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_call_error) = ( + _get_metadata_or_catch_error( + repo_id=repo_id, + filename=filename, + repo_type=repo_type, + revision=revision, + endpoint=endpoint, + etag_timeout=_ETAG_RETRY_TIMEOUT, + headers=headers, + token=token, + local_files_only=local_files_only, + storage_folder=storage_folder, + relative_filename=relative_filename, + retry_on_errors=True, + ) + ) + + # If still error, raise + if head_call_error is not None: + _raise_on_head_call_error(head_call_error, force_download, local_files_only) + + # From now on, etag, commit_hash, url and size are not None. + assert etag is not None, "etag must have been retrieved from server" + assert commit_hash is not None, "commit_hash must have been retrieved from server" + assert url_to_download is not None, "file location must have been retrieved from server" + assert expected_size is not None, "expected_size must have been retrieved from server" + blob_path = os.path.join(storage_folder, "blobs", etag) + pointer_path = _get_pointer_path(storage_folder, commit_hash, relative_filename) + + if dry_run: + is_cached = os.path.exists(pointer_path) or os.path.exists(blob_path) + return DryRunFileInfo( + commit_hash=commit_hash, + file_size=expected_size, + filename=filename, + is_cached=is_cached, + local_path=pointer_path, + will_download=force_download or not is_cached, + ) + + # Pointer already exists -> update the ref best-effort, then return without + # attempting to write to the cache (which may be mounted read-only). + if not force_download and os.path.exists(pointer_path): + try: + _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash) + except OSError: + pass + return pointer_path + + os.makedirs(os.path.dirname(blob_path), exist_ok=True) + os.makedirs(os.path.dirname(pointer_path), exist_ok=True) + + # Tag cache_dir so backup tools can skip it (CACHEDIR.TAG standard). + _create_cachedir_tag(Path(cache_dir)) + + # if passed revision is not identical to commit_hash + # then revision has to be a branch name or tag name. + # In that case store a ref. + _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash) + + # Prevent parallel downloads of the same file with a lock. + # etag could be duplicated across repos. + # Note: the lock is best-effort to avoid downloading the same file twice. Cache correctness + # does not depend on it: each download writes to a process-unique temporary file that is + # atomically renamed into place (see `_download_to_tmp_and_move`). + lock_path = os.path.join(locks_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type), f"{etag}.lock") + + # Some Windows versions do not allow for paths longer than 255 characters. + # In this case, we must specify it as an extended path by using the "\\?\" prefix. + if ( + os.name == "nt" + and len(os.path.abspath(lock_path)) > 255 + and not os.path.abspath(lock_path).startswith("\\\\?\\") + ): + lock_path = "\\\\?\\" + os.path.abspath(lock_path) + + if ( + os.name == "nt" + and len(os.path.abspath(blob_path)) > 255 + and not os.path.abspath(blob_path).startswith("\\\\?\\") + ): + blob_path = "\\\\?\\" + os.path.abspath(blob_path) + + Path(lock_path).parent.mkdir(parents=True, exist_ok=True) + + # Blob exists but pointer must be (safely) created -> take the lock + if not force_download and os.path.exists(blob_path): + with WeakFileLock(lock_path): + if not os.path.exists(pointer_path): + _create_symlink(blob_path, pointer_path, new_blob=False) + return pointer_path + + # Local file doesn't exist or etag isn't a match => retrieve file from remote (or cache) + + with WeakFileLock(lock_path): + _download_to_tmp_and_move( + incomplete_path=Path(blob_path + ".incomplete"), + destination_path=Path(blob_path), + url_to_download=url_to_download, + headers=headers, + expected_size=expected_size, + filename=filename, + force_download=force_download, + etag=etag, + xet_file_data=xet_file_data, + tqdm_class=tqdm_class, + ) + if not os.path.exists(pointer_path): + _create_symlink(blob_path, pointer_path, new_blob=True) + + return pointer_path + + +def _hf_hub_download_to_local_dir( + *, + # Destination + local_dir: str | Path, + # File info + repo_id: str, + repo_type: str, + filename: str, + revision: str, + # HTTP info + endpoint: str | None, + etag_timeout: float, + headers: dict[str, str], + token: bool | str | None, + # Additional options + cache_dir: str, + force_download: bool, + local_files_only: bool, + tqdm_class: type[base_tqdm] | None, + dry_run: bool, +) -> str | DryRunFileInfo: + """Download a given file to a local folder, if not already present. + + Method should not be called directly. Please use `hf_hub_download` instead. + """ + # Some Windows versions do not allow for paths longer than 255 characters. + # In this case, we must specify it as an extended path by using the "\\?\" prefix. + if os.name == "nt" and len(os.path.abspath(local_dir)) > 255: + local_dir = "\\\\?\\" + os.path.abspath(local_dir) + local_dir = Path(local_dir) + paths = get_local_download_paths(local_dir=local_dir, filename=filename) + local_metadata = read_download_metadata(local_dir=local_dir, filename=filename) + + # Local file exists + metadata exists + commit_hash matches => return file + if ( + REGEX_COMMIT_HASH.match(revision) + and paths.file_path.is_file() + and local_metadata is not None + and local_metadata.commit_hash == revision + ): + local_file = str(paths.file_path) + if dry_run: + return DryRunFileInfo( + commit_hash=revision, + file_size=os.path.getsize(local_file), + filename=filename, + is_cached=True, + local_path=local_file, + will_download=force_download, + ) + if not force_download: + return local_file + + # Local file doesn't exist or commit_hash doesn't match => we need the etag + # - Xet file => might be cached in /tree listing + # - otherwise => HEAD /resolve call + tree_cache_folder = tree_cache_folder_for_local_dir(str(local_dir)) + (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_call_error) = _get_metadata_or_catch_error( + repo_id=repo_id, + filename=filename, + repo_type=repo_type, + revision=revision, + endpoint=endpoint, + etag_timeout=etag_timeout, + headers=headers, + token=token, + local_files_only=local_files_only, + tree_cache_folder=tree_cache_folder, + ) + + if head_call_error is not None: + # No HEAD call but local file exists => default to local file + if paths.file_path.is_file(): + if dry_run or not force_download: + logger.warning( + f"Couldn't access the Hub to check for update but local file already exists. Defaulting to existing file. (error: {head_call_error})" + ) + local_path = str(paths.file_path) + if dry_run and local_metadata is not None: + return DryRunFileInfo( + commit_hash=local_metadata.commit_hash, + file_size=os.path.getsize(local_path), + filename=filename, + is_cached=True, + local_path=local_path, + will_download=force_download, + ) + if not force_download: + return local_path + elif not force_download: + if isinstance(head_call_error, _DEFAULT_RETRY_ON_EXCEPTIONS) or ( + isinstance(head_call_error, HfHubHTTPError) + and head_call_error.response.status_code in _DEFAULT_RETRY_ON_STATUS_CODES + ): + logger.info("No local file found. Retrying..") + (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_call_error) = ( + _get_metadata_or_catch_error( + repo_id=repo_id, + filename=filename, + repo_type=repo_type, + revision=revision, + endpoint=endpoint, + etag_timeout=_ETAG_RETRY_TIMEOUT, + headers=headers, + token=token, + local_files_only=local_files_only, + retry_on_errors=True, + ) + ) + + # If still error, raise + if head_call_error is not None: + _raise_on_head_call_error(head_call_error, force_download, local_files_only) + + # From now on, etag, commit_hash, url and size are not None. + assert etag is not None, "etag must have been retrieved from server" + assert commit_hash is not None, "commit_hash must have been retrieved from server" + assert url_to_download is not None, "file location must have been retrieved from server" + assert expected_size is not None, "expected_size must have been retrieved from server" + + # Local file exists => check if it's up-to-date + if not force_download and paths.file_path.is_file(): + # etag matches => update metadata and return file + if local_metadata is not None and local_metadata.etag == etag: + write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag) + if dry_run: + return DryRunFileInfo( + commit_hash=commit_hash, + file_size=expected_size, + filename=filename, + is_cached=True, + local_path=str(paths.file_path), + will_download=False, + ) + return str(paths.file_path) + + # metadata is outdated + etag is a sha256 + # => means it's an LFS file (large) + # => let's compute local hash and compare + # => if match, update metadata and return file + if local_metadata is None and REGEX_SHA256.match(etag) is not None: + with open(paths.file_path, "rb") as f: + file_hash = sha_fileobj(f).hex() + if file_hash == etag: + write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag) + if dry_run: + return DryRunFileInfo( + commit_hash=commit_hash, + file_size=expected_size, + filename=filename, + is_cached=True, + local_path=str(paths.file_path), + will_download=False, + ) + return str(paths.file_path) + + # Local file doesn't exist or etag isn't a match => retrieve file from remote (or cache) + + # If we are lucky enough, the file is already in the cache => copy it + if not force_download: + cached_path = try_to_load_from_cache( + repo_id=repo_id, + filename=filename, + cache_dir=cache_dir, + revision=commit_hash, + repo_type=repo_type, + ) + if isinstance(cached_path, str): + with WeakFileLock(paths.lock_path): + paths.file_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(cached_path, paths.file_path) + write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag) + if dry_run: + return DryRunFileInfo( + commit_hash=commit_hash, + file_size=expected_size, + filename=filename, + is_cached=True, + local_path=str(paths.file_path), + will_download=False, + ) + return str(paths.file_path) + + if dry_run: + is_cached = paths.file_path.is_file() + return DryRunFileInfo( + commit_hash=commit_hash, + file_size=expected_size, + filename=filename, + is_cached=is_cached, + local_path=str(paths.file_path), + will_download=force_download or not is_cached, + ) + + # Otherwise, let's download the file! + with WeakFileLock(paths.lock_path): + paths.file_path.unlink(missing_ok=True) # delete outdated file first + _download_to_tmp_and_move( + incomplete_path=paths.incomplete_path(etag), + destination_path=paths.file_path, + url_to_download=url_to_download, + headers=headers, + expected_size=expected_size, + filename=filename, + force_download=force_download, + etag=etag, + xet_file_data=xet_file_data, + tqdm_class=tqdm_class, + ) + + write_download_metadata(local_dir=local_dir, filename=filename, commit_hash=commit_hash, etag=etag) + return str(paths.file_path) + + +@validate_hf_hub_args +def try_to_load_from_cache( + repo_id: str, + filename: str, + cache_dir: str | Path | None = None, + revision: str | None = None, + repo_type: str | None = None, +) -> str | _CACHED_NO_EXIST_T | None: + """ + Explores the cache to return the latest cached file for a given revision if found. + + This function will not raise any exception if the file in not cached. + + Args: + cache_dir (`str` or `os.PathLike`): + The folder where the cached files lie. + repo_id (`str`): + The ID of the repo on huggingface.co. + filename (`str`): + The filename to look for inside `repo_id`. + revision (`str`, *optional*): + The specific model version to use. Will default to `"main"` if it's not provided and no `commit_hash` is + provided either. + repo_type (`str`, *optional*): + The type of the repository. Will default to `"model"`. + + Returns: + `Optional[str]` or `_CACHED_NO_EXIST`: + Will return `None` if the file was not cached. Otherwise: + - The exact path to the cached file if it's found in the cache + - A special value `_CACHED_NO_EXIST` if the file does not exist at the given commit hash and this fact was + cached. + + Example: + + ```python + from huggingface_hub import try_to_load_from_cache, _CACHED_NO_EXIST + + filepath = try_to_load_from_cache() + if isinstance(filepath, str): + # file exists and is cached + ... + elif filepath is _CACHED_NO_EXIST: + # non-existence of file is cached + ... + else: + # file is not cached + ... + ``` + """ + if revision is None: + revision = "main" + if repo_type is None: + repo_type = "model" + if repo_type not in constants.REPO_TYPES_WITH_KERNEL: + raise ValueError( + f"Invalid repo type: {repo_type}. Accepted repo types are: {str(constants.REPO_TYPES_WITH_KERNEL)}" + ) + if cache_dir is None: + cache_dir = constants.HF_HUB_CACHE + + repo_cache = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type)) + if not os.path.isdir(repo_cache): + # No cache for this model + return None + + refs_dir = os.path.join(repo_cache, "refs") + snapshots_dir = os.path.join(repo_cache, "snapshots") + no_exist_dir = os.path.join(repo_cache, ".no_exist") + + # Resolve refs (for instance to convert main to the associated commit sha) + if os.path.isdir(refs_dir): + revision_file = os.path.join(refs_dir, revision) + if os.path.isfile(revision_file): + with open(revision_file) as f: + revision = f.read() + + # Check if file is cached as "no_exist" + if os.path.isfile(os.path.join(no_exist_dir, revision, filename)): + return _CACHED_NO_EXIST + + # Check if revision folder exists + if not os.path.exists(snapshots_dir): + return None + cached_shas = os.listdir(snapshots_dir) + if revision not in cached_shas: + # No cache for this revision and we won't try to return a random revision + return None + + # Check if file exists in cache + cached_file = os.path.join(snapshots_dir, revision, filename) + return cached_file if os.path.isfile(cached_file) else None + + +@validate_hf_hub_args +def get_hf_file_metadata( + url: str, + token: bool | str | None = None, + timeout: float | None = constants.HF_HUB_ETAG_TIMEOUT, + library_name: str | None = None, + library_version: str | None = None, + user_agent: dict | str | None = None, + headers: dict[str, str] | None = None, + endpoint: str | None = None, + retry_on_errors: bool = False, +) -> HfFileMetadata: + """Fetch metadata of a file versioned on the Hub for a given url. + + Args: + url (`str`): + File url, for example returned by [`hf_hub_url`]. + token (`str` or `bool`, *optional*): + A token to be used for the download. + - If `True`, the token is read from the HuggingFace config + folder. + - If `False` or `None`, no token is provided. + - If a string, it's used as the authentication token. + timeout (`float`, *optional*, defaults to 10): + How many seconds to wait for the server to send metadata before giving up. + library_name (`str`, *optional*): + The name of the library to which the object corresponds. + library_version (`str`, *optional*): + The version of the library. + user_agent (`dict`, `str`, *optional*): + The user-agent info in the form of a dictionary or a string. + headers (`dict`, *optional*): + Additional headers to be sent with the request. + endpoint (`str`, *optional*): + Endpoint of the Hub. Defaults to . + retry_on_errors (`bool`, *optional*, defaults to `False`): + Whether to retry on errors (429, 5xx, timeout, network errors). + If False, no retry for fast fallback to local cache. + + Returns: + A [`HfFileMetadata`] object containing metadata such as location, etag, size and + commit_hash. + """ + hf_headers = build_hf_headers( + token=token, + library_name=library_name, + library_version=library_version, + user_agent=user_agent, + headers=headers, + ) + hf_headers["Accept-Encoding"] = "identity" # prevent any compression => we want to know the real size of the file + + # Retrieve metadata + response = _httpx_follow_relative_redirects_with_backoff( + method="HEAD", url=url, headers=hf_headers, timeout=timeout, retry_on_errors=retry_on_errors + ) + hf_raise_for_status(response) + + # Return + return HfFileMetadata( + commit_hash=response.headers.get(constants.HUGGINGFACE_HEADER_X_REPO_COMMIT), + # We favor a custom header indicating the etag of the linked resource, and we fall back to the regular etag header. + etag=_normalize_etag( + response.headers.get(constants.HUGGINGFACE_HEADER_X_LINKED_ETAG) or response.headers.get("ETag") + ), + # Either from response headers (if redirected) or defaults to request url + # Do not use directly `url` as we might have followed relative redirects. + location=response.headers.get("Location") or str(response.request.url), # type: ignore + size=_int_or_none( + response.headers.get(constants.HUGGINGFACE_HEADER_X_LINKED_SIZE) or response.headers.get("Content-Length") + ), + xet_file_data=parse_xet_file_data_from_response(response, endpoint=endpoint), # type: ignore + ) + + +def _get_metadata_or_catch_error( + *, + repo_id: str, + filename: str, + repo_type: str, + revision: str, + endpoint: str | None, + etag_timeout: float | None, + headers: dict[str, str], # mutated inplace! + token: bool | str | None, + local_files_only: bool, + relative_filename: str | None = None, # only used to store `.no_exists` in cache + storage_folder: str | None = None, # only used to store `.no_exists` in cache + retry_on_errors: bool = False, + tree_cache_folder: str | None = None, # if set, read the on-disk tree listing to skip the HEAD call +) -> ( + # Either an exception is caught and returned + tuple[None, None, None, None, None, Exception] + | + # Or the metadata is returned as + # `(url_to_download, etag, commit_hash, expected_size, xet_file_data, None)` + tuple[str, str, str, int, XetFileData | None, None] +): + """Get metadata for a file on the Hub, safely handling network issues. + + Returns either the etag, commit_hash and expected size of the file, or the error + raised while fetching the metadata. + + NOTE: This function mutates `headers` inplace! It removes the `authorization` header + if the file is a LFS blob and the domain of the url is different from the + domain of the location (typically an S3 bucket). + """ + if local_files_only: + return ( + None, + None, + None, + None, + None, + OfflineModeIsEnabled( + f"Cannot access file since 'local_files_only=True' as been set. (repo_id: {repo_id}, repo_type: {repo_type}, revision: {revision}, filename: {filename})" + ), + ) + + # Skip the per-file HEAD call when the file metadata can be rebuilt from a tree listing cached on disk. + if tree_cache_folder is not None and REGEX_COMMIT_HASH.match(revision): + tree_metadata = _xet_file_metadata_from_tree_cache( + tree_cache_folder=tree_cache_folder, + repo_id=repo_id, + repo_type=repo_type, + commit_hash=revision, + filename=filename, + endpoint=endpoint, + ) + if tree_metadata is not None: + return tree_metadata + + url = hf_hub_url(repo_id, filename, repo_type=repo_type, revision=revision, endpoint=endpoint) + url_to_download: str = url + etag: str | None = None + commit_hash: str | None = None + expected_size: int | None = None + head_error_call: Exception | None = None + xet_file_data: XetFileData | None = None + + # Try to get metadata from the server. + # Do not raise yet if the file is not found or not accessible. + if not local_files_only: + try: + try: + metadata = get_hf_file_metadata( + url=url, + timeout=etag_timeout, + headers=headers, + token=token, + endpoint=endpoint, + retry_on_errors=retry_on_errors, + ) + except RemoteEntryNotFoundError as http_error: + if storage_folder is not None and relative_filename is not None: + # Cache the non-existence of the file + commit_hash = http_error.response.headers.get(constants.HUGGINGFACE_HEADER_X_REPO_COMMIT) + if commit_hash is not None: + no_exist_file_path = Path(storage_folder) / ".no_exist" / commit_hash / relative_filename + if not no_exist_file_path.exists(): + try: + no_exist_file_path.parent.mkdir(parents=True, exist_ok=True) + no_exist_file_path.touch() + except OSError as e: + logger.error( + f"Could not cache non-existence of file. Will ignore error and continue. Error: {e}" + ) + _cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash) + raise + + # Commit hash must exist + commit_hash = metadata.commit_hash + if commit_hash is None: + raise FileMetadataError( + "Distant resource does not seem to be on huggingface.co. It is possible that a configuration issue" + " prevents you from downloading resources from https://huggingface.co. Please check your firewall" + " and proxy settings and make sure your SSL certificates are updated." + ) + + # Etag must exist + # If we don't have any of those, raise an error. + etag = metadata.etag + if etag is None: + raise FileMetadataError( + "Distant resource does not have an ETag, we won't be able to reliably ensure reproducibility." + ) + + # Size must exist + expected_size = metadata.size + if expected_size is None: + raise FileMetadataError("Distant resource does not have a Content-Length.") + + xet_file_data = metadata.xet_file_data + + # In case of a redirect, save an extra redirect on the request.get call, + # and ensure we download the exact atomic version even if it changed + # between the HEAD and the GET (unlikely, but hey). + # + # If url domain is different => we are downloading from a CDN => url is signed => don't send auth + # If url domain is the same => redirect due to repo rename AND downloading a regular file => keep auth + if xet_file_data is None and url != metadata.location: + url_to_download = metadata.location + if urlparse(url).netloc != urlparse(metadata.location).netloc: + # Remove authorization header when downloading a LFS blob + headers.pop("authorization", None) + except httpx.ProxyError: + # Actually raise on proxy error + raise + except (httpx.ConnectError, httpx.TimeoutException, OfflineModeIsEnabled) as error: + # Otherwise, our Internet connection is down. + # etag is None + head_error_call = error + except (RevisionNotFoundError, RemoteEntryNotFoundError): + # The repo was found but the revision or entry doesn't exist on the Hub (never existed or got deleted) + raise + except HfHubHTTPError as error: + # Multiple reasons for an http error: + # - Repository is private and invalid/missing token sent + # - Repository is gated and invalid/missing token sent + # - Hub is down (error 500 or 504) + # => let's switch to 'local_files_only=True' to check if the files are already cached. + # (if it's not the case, the error will be re-raised) + head_error_call = error + except FileMetadataError as error: + # Multiple reasons for a FileMetadataError: + # - Wrong network configuration (proxy, firewall, SSL certificates) + # - Inconsistency on the Hub + # => let's switch to 'local_files_only=True' to check if the files are already cached. + # (if it's not the case, the error will be re-raised) + head_error_call = error + + if not (local_files_only or etag is not None or head_error_call is not None): + raise RuntimeError("etag is empty due to uncovered problems") + + return (url_to_download, etag, commit_hash, expected_size, xet_file_data, head_error_call) # type: ignore + + +def _xet_file_metadata_from_tree_cache( + *, + tree_cache_folder: str, + repo_id: str, + repo_type: str, + commit_hash: str, + filename: str, + endpoint: str | None, +) -> tuple[str, str, str, int, XetFileData, None] | None: + """Rebuild the metadata a HEAD call would return, from the on-disk tree listing cache. + + The optimization is intentionally limited to Xet files when Xet is enabled: that's the only case where + skipping the HEAD call pays off, since Xet downloads don't rely on the `/resolve` redirect the HEAD call + would resolve. For regular files (or when Xet is unavailable) we return `None` and let the caller make the + HEAD call as usual. Also returns `None` when the commit's tree listing or the file is not cached. + """ + if not is_xet_available(): + return None + tree_entries = read_tree_cache(tree_cache_folder, commit_hash) + if tree_entries is None: + return None + entry = tree_entries.get(filename) + if entry is None or entry.xet_hash is None or entry.lfs_sha256 is None or entry.lfs_size is None: + return None + + xet_file_data = XetFileData( + file_hash=entry.xet_hash, + refresh_route=xet_connection_info_refresh_url( + token_type=XetTokenType.READ, + repo_id=repo_id, + repo_type=repo_type, + revision=commit_hash, + endpoint=endpoint, + ), + ) + # Mirror the server's HEAD response: ETag/size come from the LFS metadata for LFS-tracked files (which Xet + # files are). + etag = entry.lfs_sha256 + file_size = entry.lfs_size + location = hf_hub_url(repo_id, filename, repo_type=repo_type, revision=commit_hash, endpoint=endpoint) + return (location, etag, commit_hash, file_size, xet_file_data, None) + + +def _raise_on_head_call_error(head_call_error: Exception, force_download: bool, local_files_only: bool) -> NoReturn: + """Raise an appropriate error when the HEAD call failed and we cannot locate a local file.""" + # No head call => we cannot force download. + if force_download: + if local_files_only: + raise ValueError("Cannot pass 'force_download=True' and 'local_files_only=True' at the same time.") + elif isinstance(head_call_error, OfflineModeIsEnabled): + raise ValueError("Cannot pass 'force_download=True' when offline mode is enabled.") from head_call_error + else: + raise ValueError("Force download failed due to the above error.") from head_call_error + + # No head call + couldn't find an appropriate file on disk => raise an error. + if local_files_only: + raise LocalEntryNotFoundError( + "Cannot find the requested files in the disk cache and outgoing traffic has been disabled. To enable" + " hf.co look-ups and downloads online, set 'local_files_only' to False." + ) + elif isinstance(head_call_error, (RepositoryNotFoundError, GatedRepoError)) or ( + isinstance(head_call_error, HfHubHTTPError) and head_call_error.response.status_code == 401 + ): + # Repo not found or gated => let's raise the actual error + # Unauthorized => likely a token issue => let's raise the actual error + raise head_call_error + else: + # Otherwise: most likely a connection issue or Hub downtime => let's warn the user + raise LocalEntryNotFoundError( + "An error happened while trying to locate the file on the Hub and we cannot find the requested files" + " in the local cache. Please check your connection and try again or make sure your Internet connection" + " is on." + ) from head_call_error + + +def _download_to_tmp_and_move( + incomplete_path: Path, + destination_path: Path, + url_to_download: str, + headers: dict[str, str], + expected_size: int | None, + filename: str, + force_download: bool, + etag: str | None, + xet_file_data: XetFileData | None, + tqdm_class: type[base_tqdm] | None = None, +) -> None: + """Download content from a URL to a destination path. + + Internal logic: + - return early if file is already downloaded + - check disk space before downloading + - download content to a process-unique temporary file + - set correct permissions on temporary file + - move the temporary file to the destination path + + Both `incomplete_path` and `destination_path` must be on the same volume to avoid a local copy. + """ + if destination_path.exists() and not force_download: + # Do nothing if already exists (except if force_download=True) + return + + # Download to a process-unique temporary file before moving it in place. A shared + # `.incomplete` file corrupts the cache whenever the surrounding lock is not honored: + # on some filesystems (Lustre, GPFS, some NFS mounts) `flock(2)` silently succeeds for every + # caller and concurrent processes end up appending to the same file. With a unique file per + # process, a broken lock costs only duplicated bandwidth: each process downloads the full + # file and atomically renames it to the final destination. + # See https://github.com/huggingface/huggingface_hub/pull/4228. + tmp_path = incomplete_path.with_name(f"{incomplete_path.stem}.{uuid.uuid4().hex[:8]}.incomplete") + try: + with tmp_path.open("wb") as f: + logger.debug(f"Downloading '{filename}' to '{tmp_path}'") + + if expected_size is not None: # might be None if HTTP header not set correctly + # Check disk space in both tmp and destination path + _check_disk_space(expected_size, tmp_path.parent) + _check_disk_space(expected_size, destination_path.parent) + + if xet_file_data is not None and is_xet_available(): + logger.debug("Xet Storage is enabled for this repo. Downloading file from Xet Storage..") + xet_get( + incomplete_path=tmp_path, + xet_file_data=xet_file_data, + headers=headers, + expected_size=expected_size, + displayed_filename=filename, + tqdm_class=tqdm_class, + ) + else: + if xet_file_data is not None and not constants.HF_HUB_DISABLE_XET: + logger.warning( + "Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed. " + "Falling back to regular HTTP download. " + "For better performance, install the package with: `pip install huggingface_hub[hf_xet]` or `pip install hf_xet`" + ) + + http_get( + url_to_download, + f, + headers=headers, + expected_size=expected_size, + tqdm_class=tqdm_class, + ) + + logger.debug(f"Download complete. Moving file to {destination_path}") + _chmod_and_move(tmp_path, destination_path) + finally: + # No-op on success (file has been moved). On failure, do not keep a partial file around: + # it could not be reused anyway since the temporary name is unique to this download. + tmp_path.unlink(missing_ok=True) + + +def _int_or_none(value: str | None) -> int | None: + try: + return int(value) # type: ignore + except (TypeError, ValueError): + return None + + +def _chmod_and_move(src: Path, dst: Path) -> None: + """Set correct permission before moving a blob from tmp directory to cache dir. + + Do not take into account the `umask` from the process as there is no convenient way + to get it that is thread-safe. + + See: + - About umask: https://docs.python.org/3/library/os.html#os.umask + - Thread-safety: https://stackoverflow.com/a/70343066 + - About solution: https://github.com/huggingface/huggingface_hub/pull/1220#issuecomment-1326211591 + - Fix issue: https://github.com/huggingface/huggingface_hub/issues/1141 + - Fix issue: https://github.com/huggingface/huggingface_hub/issues/1215 + """ + # Get umask by creating a temporary file in the folder containing the incomplete file. + # We know this folder is writable since the incomplete file has just been written there. + # Probing next to `dst` is not always possible: when downloading to a local dir, `dst` is the + # final file location and its parents might not be writable (e.g. read-only root filesystem). + # See https://github.com/huggingface/huggingface_hub/issues/4304. + tmp_file = src.parent / f"tmp_{uuid.uuid4()}" + try: + tmp_file.touch() + cache_dir_mode = Path(tmp_file).stat().st_mode + os.chmod(str(src), stat.S_IMODE(cache_dir_mode)) + except OSError as e: + logger.warning( + f"Could not set the permissions on the file '{src}'. Error: {e}.\nContinuing without setting permissions." + ) + finally: + try: + tmp_file.unlink() + except OSError: + # fails if `tmp_file.touch()` failed => do nothing + # See https://github.com/huggingface/huggingface_hub/issues/2359 + pass + + shutil.move(str(src), str(dst), copy_function=_copy_no_matter_what) + + +def _copy_no_matter_what(src: str, dst: str) -> None: + """Copy file from src to dst. + + If `shutil.copy2` fails, fallback to `shutil.copyfile`. + """ + try: + # Copy file with metadata and permission + # Can fail e.g. if dst is an S3 mount + shutil.copy2(src, dst) + except OSError: + # Copy only file content + shutil.copyfile(src, dst) + + +def _get_pointer_path(storage_folder: str, revision: str, relative_filename: str) -> str: + # Using `os.path.abspath` instead of `Path.resolve()` to avoid resolving symlinks + snapshot_path = os.path.join(storage_folder, "snapshots") + pointer_path = os.path.join(snapshot_path, revision, relative_filename) + if Path(os.path.abspath(snapshot_path)) not in Path(os.path.abspath(pointer_path)).parents: + raise ValueError( + "Invalid pointer path: cannot create pointer path in snapshot folder if" + f" `storage_folder='{storage_folder}'`, `revision='{revision}'` and" + f" `relative_filename='{relative_filename}'`." + ) + return pointer_path diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/hf_api.py b/venv/lib/python3.12/site-packages/huggingface_hub/hf_api.py new file mode 100644 index 0000000000000000000000000000000000000000..9ce66fb1c43c90f886e161295d1fba06b3a94d2e --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/hf_api.py @@ -0,0 +1,15015 @@ +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import inspect +import itertools +import json +import re +import struct +import time +import warnings +from collections import defaultdict +from collections.abc import Callable, Iterable, Iterator +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from functools import wraps +from itertools import islice +from pathlib import Path +from secrets import token_hex +from typing import TYPE_CHECKING, Any, BinaryIO, Literal, TypeVar, overload +from urllib.parse import quote + +import httpcore +import httpx +from tqdm.auto import tqdm as base_tqdm + +from . import constants +from ._buckets import ( + BucketFile, + BucketFileMetadata, + BucketFolder, + BucketInfo, + BucketUrl, + SyncPlan, + _BucketAddFile, + _BucketCopyFile, + _BucketDeleteFile, + _parse_bucket_uri, + sync_bucket_internal, +) +from ._commit_api import ( + DUPLICATE_LFS_BATCH_SIZE, + CommitOperation, + CommitOperationAdd, + CommitOperationCopy, + CommitOperationDelete, + _CopySource, + _fetch_files_to_copy, + _fetch_upload_modes, + _send_commit, + _upload_files, + _warn_on_overwriting_operations, +) +from ._dataset_viewer import DatasetParquetEntry +from ._eval_results import EvalResultEntry, parse_eval_result_entries +from ._inference_endpoints import InferenceEndpoint, InferenceEndpointScalingMetric, InferenceEndpointType +from ._jobs_api import ( + TERMINAL_JOB_STAGES, + JobHardware, + JobHardwareInfo, + JobInfo, + JobSpec, + JobStage, + ScheduledJobInfo, + _create_job_spec, + _default_job_name_from_image, + _default_job_name_from_script, + _derive_job_volume_name, +) +from ._space_api import ( + INTERMEDIATE_SPACE_STAGES, + SpaceHardware, + SpaceRuntime, + SpaceSearchResult, + SpaceSecret, + SpaceStorage, + SpaceTemplate, + SpaceVariable, + Volume, +) +from ._upload_large_folder import upload_large_folder_internal +from ._upload_pipeline import pipelined_upload +from .community import ( + Discussion, + DiscussionComment, + DiscussionStatusChange, + DiscussionTitleChange, + DiscussionWithDetails, + deserialize_event, +) +from .errors import ( + BadRequestError, + EntryNotFoundError, + FileDuplicationError, + GatedRepoError, + HfHubHTTPError, + HfUriError, + LocalTokenNotFoundError, + RemoteEntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, +) +from .file_download import DryRunFileInfo, HfFileMetadata, get_hf_file_metadata, hf_hub_url +from .repocard_data import DatasetCardData, ModelCardData, SpaceCardData +from .utils import ( + DEFAULT_IGNORE_PATTERNS, + HfUri, + NotASafetensorsRepoError, + SafetensorsFileMetadata, + SafetensorsParsingError, + SafetensorsRepoMetadata, + TensorInfo, + are_progress_bars_disabled, + build_hf_headers, + chunk_iterable, + experimental, + filter_repo_objects, + fix_hf_endpoint_in_url, + get_session, + get_token, + hf_raise_for_status, + hf_thread_map, + http_backoff, + logging, + paginate, + parse_datetime, + parse_hf_uri, + parse_xet_file_data_from_response, + silent_tqdm, + validate_hf_hub_args, +) +from .utils import tqdm as hf_tqdm +from .utils._auth import _get_token_from_environment, _get_token_from_file, _get_token_from_google_colab +from .utils._deprecation import _deprecate_arguments, _deprecate_method +from .utils._http import _httpx_follow_relative_redirects_with_backoff +from .utils._runtime import is_xet_available +from .utils._typing import CallableT +from .utils._verification import collect_local_files, resolve_local_root, verify_maps +from .utils.endpoint_helpers import _is_emission_within_threshold + + +if TYPE_CHECKING: + from .inference._providers import PROVIDER_T + from .utils._verification import FolderVerification + from .utils._xet_progress_reporting import XetUploadProgressReporter + +R = TypeVar("R") # Return type +CollectionItemType_T = Literal["model", "dataset", "space", "paper", "collection", "bucket"] +CollectionSort_T = Literal["lastModified", "trending", "upvotes"] +RepoVisibility_T = Literal["public", "private", "protected"] + +ExpandModelProperty_T = Literal[ + "author", + "baseModels", + "cardData", + "childrenModelCount", + "config", + "createdAt", + "disabled", + "downloads", + "downloadsAllTime", + "evalResults", + "gated", + "gguf", + "inference", + "inferenceProviderMapping", + "lastModified", + "library_name", + "likes", + "mask_token", + "model-index", + "pipeline_tag", + "private", + "resourceGroup", + "safetensors", + "sha", + "siblings", + "spaces", + "tags", + "transformersInfo", + "trendingScore", + "usedStorage", + "widgetData", +] + +ExpandDatasetProperty_T = Literal[ + "author", + "cardData", + "citation", + "createdAt", + "description", + "disabled", + "downloads", + "downloadsAllTime", + "gated", + "lastModified", + "likes", + "mainSize", + "paperswithcode_id", + "private", + "resourceGroup", + "sha", + "siblings", + "tags", + "trendingScore", + "usedStorage", +] + +ExpandSpaceProperty_T = Literal[ + "author", + "cardData", + "createdAt", + "datasets", + "disabled", + "lastModified", + "likes", + "models", + "private", + "resourceGroup", + "runtime", + "sdk", + "sha", + "siblings", + "subdomain", + "tags", + "trendingScore", + "usedStorage", +] + +ModelSort_T = Literal["created_at", "downloads", "last_modified", "likes", "trending_score"] +DatasetSort_T = Literal["created_at", "downloads", "last_modified", "likes", "trending_score"] +SpaceSort_T = Literal["created_at", "last_modified", "likes", "trending_score"] +DailyPapersSort_T = Literal["publishedAt", "trending"] + +REPO_REGIONS = Literal["us", "eu"] + +USERNAME_PLACEHOLDER = "hf_user" +_REGEX_DISCUSSION_URL = re.compile(r".*/discussions/(\d+)$") +_REGEX_HTTP_PROTOCOL = re.compile(r"https?://") + +_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE = ( + "\nNote: Creating a commit assumes that the repo already exists on the" + " Huggingface Hub. Please use `create_repo` if it's not the case." +) +_AUTH_CHECK_NO_REPO_ERROR_MESSAGE = ( + "\nNote: The repository either does not exist or you do not have access rights." + " Please check the repository ID and your access permissions." + " If this is a private repository, ensure that your token is correct." +) +_BUCKET_PATHS_INFO_BATCH_SIZE = 1000 +_BUCKET_BATCH_ADD_CHUNK_SIZE = 1000 +_BUCKET_BATCH_DELETE_CHUNK_SIZE = 1000 + +# Regex used to match special revisions with "/" in them (see #1710) +SPECIAL_REFS_REVISION_REGEX = re.compile( + r""" + (^refs\/convert\/\w+) # `refs/convert/parquet` revisions + | + (^refs\/pr\/\d+) # PR revisions + """, + re.VERBOSE, +) + +logger = logging.get_logger(__name__) + + +def _resolve_repo_visibility( + *, + private: bool | None, + visibility: RepoVisibility_T | None, + repo_type: str | None, +) -> RepoVisibility_T | None: + if private is not None and visibility is not None: + raise ValueError("Received both `private` and `visibility` arguments. Please provide only one of them.") + + if visibility is None: + if private is None: + return None + return "private" if private else "public" + + if visibility == "protected" and repo_type != constants.REPO_TYPE_SPACE: + raise ValueError("Only Spaces can be 'protected'. Please set visibility to 'public' or 'private'.") + return visibility + + +def repo_type_and_id_from_hf_id(hf_id: str, hub_url: str | None = None) -> tuple[str | None, str | None, str]: + """ + Returns the repo type and ID from a huggingface.co URL linking to a + repository + + > [!WARNING] + > Deprecated: prefer [`parse_hf_uri`], which parses both `hf://` URIs and Hugging Face web URLs into a structured [`HfUri`]. + > See https://huggingface.co/docs/huggingface_hub/package_reference/hf_uris for more details. + + Args: + hf_id (`str`): + An URL or ID of a repository on the HF hub. Accepted values are: + + - https://huggingface.co/// + - https://huggingface.co// + - hf://// + - hf:/// + - // + - / + - + hub_url (`str`, *optional*): + The URL of the HuggingFace Hub, defaults to https://huggingface.co + + Returns: + A tuple with three items: repo_type (`str` or `None`), namespace (`str` or + `None`) and repo_id (`str`). + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If URL cannot be parsed. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If `repo_type` is unknown. + """ + input_hf_id = hf_id + + # Get the hub_url (with or without protocol) + full_hub_url = hub_url if hub_url is not None else constants.ENDPOINT + hub_url_without_protocol = _REGEX_HTTP_PROTOCOL.sub("", full_hub_url) + + # Check if hf_id is a URL containing the hub_url (check both with and without protocol) + hf_id_without_protocol = _REGEX_HTTP_PROTOCOL.sub("", hf_id) + is_hf_url = hub_url_without_protocol in hf_id_without_protocol and "@" not in hf_id + + HFFS_PREFIX = "hf://" + if hf_id.startswith(HFFS_PREFIX): # Remove "hf://" prefix if exists + hf_id = hf_id[len(HFFS_PREFIX) :] + + # If it's a URL, strip the endpoint prefix to get the path + if is_hf_url: + # Remove protocol if present + hf_id_normalized = _REGEX_HTTP_PROTOCOL.sub("", hf_id) + + # Remove the hub_url prefix to get the relative path + if hf_id_normalized.startswith(hub_url_without_protocol): + # Strip the hub URL and any leading slashes + hf_id = hf_id_normalized[len(hub_url_without_protocol) :].lstrip("/") + + url_segments = hf_id.split("/") + is_hf_id = len(url_segments) <= 3 + + namespace: str | None + if is_hf_url: + # For URLs, we need to extract repo_type, namespace, repo_id + # Expected format after stripping endpoint: [repo_type]/namespace/repo_id or namespace/repo_id + + if len(url_segments) >= 3: + # Check if first segment is a repo type + if url_segments[0] in constants.REPO_TYPES_MAPPING: + repo_type = constants.REPO_TYPES_MAPPING[url_segments[0]] + namespace = url_segments[1] + repo_id = url_segments[2] + elif url_segments[0] == "buckets": + # Special case for buckets + repo_type = "bucket" + namespace = url_segments[1] + repo_id = url_segments[2] + else: + # First segment is namespace + namespace = url_segments[0] + repo_id = url_segments[1] + repo_type = None + elif len(url_segments) == 2: + namespace = url_segments[0] + repo_id = url_segments[1] + + # Check if namespace is actually a repo type mapping + if namespace in constants.REPO_TYPES_MAPPING: + # Mean canonical dataset or model + repo_type = constants.REPO_TYPES_MAPPING[namespace] + namespace = None + elif namespace == "buckets": + # Special case for buckets + repo_type = "bucket" + namespace = None + else: + repo_type = None + else: + # Single segment + repo_id = url_segments[0] + namespace = None + repo_type = None + elif is_hf_id: + if len(url_segments) == 3: + # Passed // or // + repo_type, namespace, repo_id = url_segments[-3:] + elif len(url_segments) == 2: + if url_segments[0] in constants.REPO_TYPES_MAPPING: + # Passed '' or 'datasets/' for a canonical model or dataset + repo_type = constants.REPO_TYPES_MAPPING[url_segments[0]] + namespace = None + repo_id = hf_id.split("/")[-1] + elif url_segments[0] == "buckets": + # Special case for buckets + repo_type = "bucket" + namespace = None + repo_id = hf_id.split("/")[-1] + else: + # Passed / or / + namespace, repo_id = hf_id.split("/")[-2:] + repo_type = None + else: + # Passed + repo_id = url_segments[0] + namespace, repo_type = None, None + else: + raise ValueError(f"Unable to retrieve user and repo ID from the passed HF ID: {hf_id}") + + # Check if repo type is known (mapping "spaces" => "space" + empty value => `None`) + if repo_type in constants.REPO_TYPES_MAPPING: + repo_type = constants.REPO_TYPES_MAPPING[repo_type] # type: ignore + if repo_type == "": + repo_type = None + if repo_type not in constants.REPO_TYPES_WITH_KERNEL and repo_type != "bucket": + raise ValueError(f"Unknown `repo_type`: '{repo_type}' ('{input_hf_id}')") + + return repo_type, namespace, repo_id + + +@dataclass +class LastCommitInfo(dict): + oid: str + title: str + date: datetime + + def __post_init__(self): # hack to make LastCommitInfo backward compatible + self.update(asdict(self)) + + +@dataclass +class BlobLfsInfo(dict): + size: int + sha256: str + pointer_size: int + + def __post_init__(self): # hack to make BlobLfsInfo backward compatible + self.update(asdict(self)) + + +@dataclass +class BlobSecurityInfo(dict): + safe: bool # duplicate information with "status" field, keeping it for backward compatibility + status: str + av_scan: dict | None + pickle_import_scan: dict | None + + def __post_init__(self): # hack to make BlogSecurityInfo backward compatible + self.update(asdict(self)) + + +@dataclass +class TransformersInfo(dict): + auto_model: str + custom_class: str | None = None + # possible `pipeline_tag` values: https://github.com/huggingface/huggingface.js/blob/3ee32554b8620644a6287e786b2a83bf5caf559c/packages/tasks/src/pipelines.ts#L72 + pipeline_tag: str | None = None + processor: str | None = None + + def __post_init__(self): # hack to make TransformersInfo backward compatible + self.update(asdict(self)) + + +@dataclass +class SafeTensorsInfo(dict): + parameters: dict[str, int] + total: int + + def __post_init__(self): # hack to make SafeTensorsInfo backward compatible + self.update(asdict(self)) + + +@dataclass +class CommitInfo(str): + """Data structure containing information about a newly created commit. + + Returned by any method that creates a commit on the Hub: [`create_commit`], [`upload_file`], [`upload_folder`], + [`delete_file`], [`delete_folder`]. It inherits from `str` for backward compatibility but using methods specific + to `str` is deprecated. + + Attributes: + commit_url (`str`): + Url where to find the commit. + + commit_message (`str`): + The summary (first line) of the commit that has been created. + + commit_description (`str`): + Description of the commit that has been created. Can be empty. + + oid (`str`): + Commit hash id. Example: `"91c54ad1727ee830252e457677f467be0bfd8a57"`. + + pr_url (`str`, *optional*): + Url to the PR that has been created, if any. Populated when `create_pr=True` + is passed. + + pr_revision (`str`, *optional*): + Revision of the PR that has been created, if any. Populated when + `create_pr=True` is passed. Example: `"refs/pr/1"`. + + pr_num (`int`, *optional*): + Number of the PR discussion that has been created, if any. Populated when + `create_pr=True` is passed. Can be passed as `discussion_num` in + [`get_discussion_details`]. Example: `1`. + + repo_url (`RepoUrl`): + Repo URL of the commit containing info like repo_id, repo_type, etc. + """ + + commit_url: str + commit_message: str + commit_description: str + oid: str + _endpoint: str | None = field(default=None, repr=False) + pr_url: str | None = None + + # Computed from `commit_url` in `__post_init__` + repo_url: RepoUrl = field(init=False) + + # Computed from `pr_url` in `__post_init__` + pr_revision: str | None = field(init=False) + pr_num: int | None = field(init=False) + + def __new__(cls, *args, commit_url: str, **kwargs): + return str.__new__(cls, commit_url) + + def __post_init__(self): + """Populate pr-related fields after initialization. + + See https://docs.python.org/3.10/library/dataclasses.html#post-init-processing. + """ + # Repo info + self.repo_url = RepoUrl(self.commit_url.split("/commit/")[0], endpoint=self._endpoint) + + # PR info + if self.pr_url is not None: + self.pr_revision = _parse_revision_from_pr_url(self.pr_url) + self.pr_num = int(self.pr_revision.split("/")[-1]) + else: + self.pr_revision = None + self.pr_num = None + + +@dataclass +class AccessRequest: + """Data structure containing information about a user access request. + + Attributes: + username (`str`): + Username of the user who requested access. + fullname (`str`): + Fullname of the user who requested access. + email (`Optional[str]`): + Email of the user who requested access. + Can only be `None` in the /accepted list if the user was granted access manually. + timestamp (`datetime`): + Timestamp of the request. + status (`Literal["pending", "accepted", "rejected"]`): + Status of the request. Can be one of `["pending", "accepted", "rejected"]`. + fields (`dict[str, Any]`, *optional*): + Additional fields filled by the user in the gate form. + """ + + username: str + fullname: str + email: str | None + timestamp: datetime + status: Literal["pending", "accepted", "rejected"] + + # Additional fields filled by the user in the gate form + fields: dict[str, Any] | None = None + + +@dataclass +class WebhookWatchedItem: + """Data structure containing information about the items watched by a webhook. + + Attributes: + type (`Literal["dataset", "model", "org", "space", "user"]`): + Type of the item to be watched. Can be one of `["dataset", "model", "org", "space", "user"]`. + name (`str`): + Name of the item to be watched. Can be the username, organization name, model name, dataset name or space name. + """ + + type: Literal["dataset", "model", "org", "space", "user"] + name: str + + +@dataclass +class WebhookInfo: + """Data structure containing information about a webhook. + + One of `url` or `job` is specified, but not both. + + Attributes: + id (`str`): + ID of the webhook. + url (`str`, *optional*): + URL of the webhook. + job (`JobSpec`, *optional*): + Specifications of the Job to trigger. + watched (`list[WebhookWatchedItem]`): + List of items watched by the webhook, see [`WebhookWatchedItem`]. + domains (`list[WEBHOOK_DOMAIN_T]`): + List of domains the webhook is watching. Can be one of `["repo", "discussions"]`. + secret (`str`, *optional*): + Secret of the webhook. + disabled (`bool`): + Whether the webhook is disabled or not. + """ + + id: str + url: str | None + job: JobSpec | None + watched: list[WebhookWatchedItem] + domains: list[constants.WEBHOOK_DOMAIN_T] + secret: str | None + disabled: bool + + +class RepoUrl(str): + """Subclass of `str` describing a repo URL on the Hub. + + `RepoUrl` is returned by `HfApi.create_repo`. It inherits from `str` for backward + compatibility. At initialization, the URL is parsed to populate properties: + - endpoint (`str`) + - namespace (`str`) + - repo_name (`str`) + - repo_id (`str`) + - repo_type (`Literal["model", "dataset", "space"]`) + - url (`str`) + + Args: + url (`Any`): + String value of the repo url. + endpoint (`str`, *optional*): + Endpoint of the Hub. Defaults to . + + Example: + ```py + >>> RepoUrl('https://huggingface.co/openai-community/gpt2') + RepoUrl('https://huggingface.co/openai-community/gpt2', endpoint='https://huggingface.co', repo_type='model', repo_id='openai-community/gpt2') + + >>> RepoUrl('https://hub-ci.huggingface.co/datasets/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co') + RepoUrl('https://hub-ci.huggingface.co/datasets/dummy_user/dummy_dataset', endpoint='https://hub-ci.huggingface.co', repo_type='dataset', repo_id='dummy_user/dummy_dataset') + + >>> RepoUrl('hf://datasets/my-user/my-dataset') + RepoUrl('hf://datasets/my-user/my-dataset', endpoint='https://huggingface.co', repo_type='dataset', repo_id='user/dataset') + + >>> HfApi.create_repo("dummy_model") + RepoUrl('https://huggingface.co/Wauplin/dummy_model', endpoint='https://huggingface.co', repo_type='model', repo_id='Wauplin/dummy_model') + ``` + + Raises: + [`~errors.HfUriError`]: + If the URL cannot be parsed (e.g. canonical single-segment repo, or unknown `repo_type`). + """ + + def __new__(cls, url: Any, endpoint: str | None = None): + url = fix_hf_endpoint_in_url(url, endpoint=endpoint) + return super().__new__(cls, url) + + def __init__(self, url: Any, endpoint: str | None = None) -> None: + super().__init__() + self.endpoint = endpoint or constants.ENDPOINT + + # Parse with the shared 'parse_hf_uri' parser, which handles 'hf://' URIs as well as Hugging + # Face web URLs (including ones on this custom 'endpoint'). If that fails, the input is a bare + # '/' id, which we reparse as an 'hf://' URI. + raw = str(self) + try: + parsed = parse_hf_uri(raw, endpoint=self.endpoint) + except HfUriError: + if "://" in raw: + raise # it was a URL: the original error is authoritative, don't retry as a bare id + parsed = parse_hf_uri(f"{constants.HF_PROTOCOL}{raw}") + + # Populate fields ('parsed.id' is always '/'). + self.namespace, self.repo_name = parsed.id.split("/") + self.repo_id = parsed.id + self.repo_type = parsed.type + self.url = str(self) # just in case it's needed + + def __repr__(self) -> str: + return f"RepoUrl('{self}', endpoint='{self.endpoint}', repo_type='{self.repo_type}', repo_id='{self.repo_id}')" + + +def _resolve_copy_target_path( + src_file_path: str, + src_root_path: str | None, + is_single_file: bool, + destination_path: str, + destination_is_directory: bool, + destination_exists_as_directory: bool, + merge_contents: bool, +) -> str: + basename = src_file_path.rsplit("/", 1)[-1] + if is_single_file: + if destination_path == "": + return basename + if destination_is_directory: + return f"{destination_path.rstrip('/')}/{basename}" + return destination_path + + if src_root_path is None: + rel_path = src_file_path + elif src_file_path.startswith(src_root_path + "/"): + rel_path = src_file_path[len(src_root_path) + 1 :] + elif src_file_path == src_root_path: + rel_path = src_file_path.rsplit("/", 1)[-1] + else: + raise ValueError(f"Unexpected source path while copying folder: '{src_file_path}'.") + + if rel_path == "": + raise ValueError("Cannot copy an empty relative path.") + + # Rsync-style trailing slash on source means "copy contents of" — skip nesting. + # Without trailing slash, match `cp -r` behavior: nest source folder inside + # existing destination directory. Non-existing destination always uses rename semantics. + if destination_exists_as_directory and src_root_path is not None and not merge_contents: + src_dir_basename = src_root_path.rsplit("/", 1)[-1] + rel_path = f"{src_dir_basename}/{rel_path}" + + if destination_path == "": + return rel_path + return f"{destination_path.rstrip('/')}/{rel_path}" + + +@dataclass +class RepoSibling: + """ + Contains basic information about a repo file inside a repo on the Hub. + + > [!TIP] + > All attributes of this class are optional except `rfilename`. This is because only the file names are returned when + > listing repositories on the Hub (with [`list_models`], [`list_datasets`] or [`list_spaces`]). If you need more + > information like file size, blob id or lfs details, you must request them specifically from one repo at a time + > (using [`model_info`], [`dataset_info`] or [`space_info`]) as it adds more constraints on the backend server to + > retrieve these. + + Attributes: + rfilename (str): + file name, relative to the repo root. + size (`int`, *optional*): + The file's size, in bytes. This attribute is defined when `files_metadata` argument of [`repo_info`] is set + to `True`. It's `None` otherwise. + blob_id (`str`, *optional*): + The file's git OID. This attribute is defined when `files_metadata` argument of [`repo_info`] is set to + `True`. It's `None` otherwise. + lfs (`BlobLfsInfo`, *optional*): + The file's LFS metadata. This attribute is defined when`files_metadata` argument of [`repo_info`] is set to + `True` and the file is stored with Git LFS. It's `None` otherwise. + """ + + rfilename: str + size: int | None = None + blob_id: str | None = None + lfs: BlobLfsInfo | None = None + + +@dataclass +class RepoFile: + """ + Contains information about a file on the Hub. + + Attributes: + path (str): + file path relative to the repo root. + size (`int`): + The file's size, in bytes. + blob_id (`str`): + The file's git OID. + lfs (`BlobLfsInfo`, *optional*): + The file's LFS metadata. + xet_hash (`str`, *optional*): + The file's Xet hash. + last_commit (`LastCommitInfo`, *optional*): + The file's last commit metadata. Only defined if [`list_repo_tree`] and [`get_paths_info`] + are called with `expand=True`. + security (`BlobSecurityInfo`, *optional*): + The file's security scan metadata. Only defined if [`list_repo_tree`] and [`get_paths_info`] + are called with `expand=True`. + """ + + path: str + size: int + blob_id: str + lfs: BlobLfsInfo | None = None + xet_hash: str | None = None + last_commit: LastCommitInfo | None = None + security: BlobSecurityInfo | None = None + + def __init__(self, **kwargs): + self.path = kwargs.pop("path") + self.size = kwargs.pop("size") + self.blob_id = kwargs.pop("oid") + lfs = kwargs.pop("lfs", None) + if lfs is not None: + lfs = BlobLfsInfo(size=lfs["size"], sha256=lfs["oid"], pointer_size=lfs["pointerSize"]) + self.lfs = lfs + self.xet_hash = kwargs.pop("xetHash", None) + last_commit = kwargs.pop("lastCommit", None) or kwargs.pop("last_commit", None) + if last_commit is not None: + last_commit = LastCommitInfo( + oid=last_commit["id"], title=last_commit["title"], date=parse_datetime(last_commit["date"]) + ) + self.last_commit = last_commit + security = kwargs.pop("securityFileStatus", None) + if security is not None: + safe = security["status"] == "safe" + security = BlobSecurityInfo( + safe=safe, + status=security["status"], + av_scan=security["avScan"], + pickle_import_scan=security["pickleImportScan"], + ) + self.security = security + + # backwards compatibility + self.rfilename = self.path + self.lastCommit = self.last_commit + + +@dataclass +class RepoFolder: + """ + Contains information about a folder on the Hub. + + Attributes: + path (str): + folder path relative to the repo root. + tree_id (`str`): + The folder's git OID. + last_commit (`LastCommitInfo`, *optional*): + The folder's last commit metadata. Only defined if [`list_repo_tree`] and [`get_paths_info`] + are called with `expand=True`. + """ + + path: str + tree_id: str + last_commit: LastCommitInfo | None = None + + def __init__(self, **kwargs): + self.path = kwargs.pop("path") + self.tree_id = kwargs.pop("oid") + last_commit = kwargs.pop("lastCommit", None) or kwargs.pop("last_commit", None) + if last_commit is not None: + last_commit = LastCommitInfo( + oid=last_commit["id"], title=last_commit["title"], date=parse_datetime(last_commit["date"]) + ) + self.last_commit = last_commit + + +@dataclass +class InferenceProviderMapping: + provider: PROVIDER_T # Provider name + hf_model_id: str # ID of the model on the Hugging Face Hub + provider_id: str # ID of the model on the provider's side + status: Literal["error", "live", "staging"] + task: str + + adapter: str | None = None + adapter_weights_path: str | None = None + type: Literal["single-model", "tag-filter"] | None = None + + def __init__(self, **kwargs): + self.provider = kwargs.pop("provider") + self.hf_model_id = kwargs.pop("hf_model_id") + self.provider_id = kwargs.pop("providerId") + self.status = kwargs.pop("status") + self.task = kwargs.pop("task") + + self.adapter = kwargs.pop("adapter", None) + self.adapter_weights_path = kwargs.pop("adapterWeightsPath", None) + self.type = kwargs.pop("type", None) + self.__dict__.update(**kwargs) + + +@dataclass +class ModelInfo: + """ + Contains information about a model on the Hub. This object is returned by [`model_info`] and [`list_models`]. + + > [!TIP] + > Most attributes of this class are optional. This is because the data returned by the Hub depends on the query made. + > In general, the more specific the query, the more information is returned. On the contrary, when listing models + > using [`list_models`] only a subset of the attributes are returned. + + Attributes: + id (`str`): + ID of model. + author (`str`, *optional*): + Author of the model. + base_models (`list[str]`, *optional*): + List of base models this model is derived from. + card_data (`ModelCardData`, *optional*): + Model Card Metadata as a [`huggingface_hub.repocard_data.ModelCardData`] object. + children_model_count (`int`, *optional*): + Number of children models derived from this model. + config (`dict`, *optional*): + Model configuration. + created_at (`datetime`, *optional*): + Date of creation of the repo on the Hub. Note that the lowest value is `2022-03-02T23:29:04.000Z`, + corresponding to the date when we began to store creation dates. + disabled (`bool`, *optional*): + Is the repo disabled. + downloads (`int`): + Number of downloads of the model over the last 30 days. + downloads_all_time (`int`): + Cumulated number of downloads of the model since its creation. + eval_results (`list[EvalResultEntry]`, *optional*): + Model's evaluation results. + gated (`Literal["auto", "manual", False]`, *optional*): + Is the repo gated. + If so, whether there is manual or automatic approval. + gguf (`dict`, *optional*): + GGUF information of the model. + inference (`Literal["warm"]`, *optional*): + Status of the model on Inference Providers. Warm if the model is served by at least one provider. + inference_provider_mapping (`list[InferenceProviderMapping]`, *optional*): + A list of [`InferenceProviderMapping`] ordered after the user's provider order. + last_modified (`datetime`, *optional*): + Date of last commit to the repo. + library_name (`str`, *optional*): + Library associated with the model. + likes (`int`): + Number of likes of the model. + mask_token (`str`, *optional*): + Mask token used by the model. + model_index (`dict`, *optional*): + Model index for evaluation. + pipeline_tag (`str`, *optional*): + Pipeline tag associated with the model. + private (`bool`): + Is the repo private. + resource_group (`dict`, *optional*): + Resource group information for the model. + safetensors (`SafeTensorsInfo`, *optional*): + Model's safetensors information. + security_repo_status (`dict`, *optional*): + Model's security scan status. + sha (`str`, *optional*): + Repo SHA at this particular revision. + siblings (`list[RepoSibling]`): + List of [`huggingface_hub.hf_api.RepoSibling`] objects that constitute the model. + spaces (`list[str]`, *optional*): + List of spaces using the model. + tags (`list[str]`): + List of tags of the model. Compared to `card_data.tags`, contains extra tags computed by the Hub + (e.g. supported libraries, model's arXiv). + transformers_info (`TransformersInfo`, *optional*): + Transformers-specific info (auto class, processor, etc.) associated with the model. + trending_score (`int`, *optional*): + Trending score of the model. + used_storage (`int`, *optional*): + Size in bytes of the model on the Hub. + widget_data (`Any`, *optional*): + Widget data associated with the model. + """ + + id: str + author: str | None + base_models: list[str] | None + card_data: ModelCardData | None + children_model_count: int | None + config: dict | None + created_at: datetime | None + disabled: bool | None + downloads: int | None + downloads_all_time: int | None + eval_results: list[EvalResultEntry] | None + gated: Literal["auto", "manual", False] | None + gguf: dict | None + inference: Literal["warm"] | None + inference_provider_mapping: list[InferenceProviderMapping] | None + last_modified: datetime | None + library_name: str | None + likes: int | None + mask_token: str | None + model_index: dict | None + pipeline_tag: str | None + private: bool | None + resource_group: dict | None + safetensors: SafeTensorsInfo | None + security_repo_status: dict | None + sha: str | None + siblings: list[RepoSibling] | None + spaces: list[str] | None + tags: list[str] | None + transformers_info: TransformersInfo | None + trending_score: int | None + used_storage: int | None + widget_data: Any | None + + def __init__(self, **kwargs): + self.id = kwargs.pop("id") + self.author = kwargs.pop("author", None) + self.sha = kwargs.pop("sha", None) + last_modified = kwargs.pop("lastModified", None) or kwargs.pop("last_modified", None) + self.last_modified = parse_datetime(last_modified) if last_modified else None + created_at = kwargs.pop("createdAt", None) or kwargs.pop("created_at", None) + self.created_at = parse_datetime(created_at) if created_at else None + self.private = kwargs.pop("private", None) + self.gated = kwargs.pop("gated", None) + self.disabled = kwargs.pop("disabled", None) + self.downloads = kwargs.pop("downloads", None) + self.downloads_all_time = kwargs.pop("downloadsAllTime", None) + self.likes = kwargs.pop("likes", None) + self.library_name = kwargs.pop("library_name", None) + self.gguf = kwargs.pop("gguf", None) + + self.inference = kwargs.pop("inference", None) + + # little hack to simplify Inference Providers logic and make it backward and forward compatible + # right now, API returns a dict on model_info and a list on list_models. Let's harmonize to list. + mapping = kwargs.pop("inferenceProviderMapping", None) + if isinstance(mapping, list): + self.inference_provider_mapping = [ + InferenceProviderMapping(**{**value, "hf_model_id": self.id}) for value in mapping + ] + elif isinstance(mapping, dict): + self.inference_provider_mapping = [ + InferenceProviderMapping(**{**value, "hf_model_id": self.id, "provider": provider}) + for provider, value in mapping.items() + ] + elif mapping is None: + self.inference_provider_mapping = None + else: + raise ValueError( + f"Unexpected type for `inferenceProviderMapping`. Expecting `dict` or `list`. Got {mapping}." + ) + + self.tags = kwargs.pop("tags", None) + self.pipeline_tag = kwargs.pop("pipeline_tag", None) + self.mask_token = kwargs.pop("mask_token", None) + self.trending_score = kwargs.pop("trendingScore", None) + self.used_storage = kwargs.pop("usedStorage", None) + + card_data = kwargs.pop("cardData", None) or kwargs.pop("card_data", None) + self.card_data = ( + ModelCardData(**card_data, ignore_metadata_errors=True) if isinstance(card_data, dict) else card_data + ) + + self.widget_data = kwargs.pop("widgetData", None) + self.model_index = kwargs.pop("model-index", None) or kwargs.pop("model_index", None) + self.config = kwargs.pop("config", None) + transformers_info = kwargs.pop("transformersInfo", None) or kwargs.pop("transformers_info", None) + self.transformers_info = TransformersInfo(**transformers_info) if transformers_info else None + siblings = kwargs.pop("siblings", None) + self.siblings = ( + [ + RepoSibling( + rfilename=sibling["rfilename"], + size=sibling.get("size"), + blob_id=sibling.get("blobId"), + lfs=( + BlobLfsInfo( + size=sibling["lfs"]["size"], + sha256=sibling["lfs"]["sha256"], + pointer_size=sibling["lfs"]["pointerSize"], + ) + if sibling.get("lfs") + else None + ), + ) + for sibling in siblings + ] + if siblings is not None + else None + ) + self.spaces = kwargs.pop("spaces", None) + safetensors = kwargs.pop("safetensors", None) + self.safetensors = ( + SafeTensorsInfo( + parameters=safetensors["parameters"], + total=safetensors["total"], + ) + if safetensors + else None + ) + self.security_repo_status = kwargs.pop("securityRepoStatus", None) + eval_results = kwargs.pop("evalResults", None) + self.eval_results = parse_eval_result_entries(eval_results) if eval_results else None + self.base_models = kwargs.pop("baseModels", None) + self.children_model_count = kwargs.pop("childrenModelCount", None) + self.resource_group = kwargs.pop("resourceGroup", None) + # backwards compatibility + self.lastModified = self.last_modified + self.cardData = self.card_data + self.transformersInfo = self.transformers_info + self.__dict__.update(**kwargs) + + +@dataclass +class DatasetInfo: + """ + Contains information about a dataset on the Hub. This object is returned by [`dataset_info`] and [`list_datasets`]. + + > [!TIP] + > Most attributes of this class are optional. This is because the data returned by the Hub depends on the query made. + > In general, the more specific the query, the more information is returned. On the contrary, when listing datasets + > using [`list_datasets`] only a subset of the attributes are returned. + + Attributes: + id (`str`): + ID of dataset. + author (`str`): + Author of the dataset. + card_data (`DatasetCardData`, *optional*): + Dataset Card Metadata as a [`huggingface_hub.repocard_data.DatasetCardData`] object. + citation (`str`, *optional*): + Citation information for the dataset. + created_at (`datetime`, *optional*): + Date of creation of the repo on the Hub. Note that the lowest value is `2022-03-02T23:29:04.000Z`, + corresponding to the date when we began to store creation dates. + description (`str`, *optional*): + Description of the dataset. + disabled (`bool`, *optional*): + Is the repo disabled. + downloads (`int`): + Number of downloads of the dataset over the last 30 days. + downloads_all_time (`int`): + Cumulated number of downloads of the dataset since its creation. + gated (`Literal["auto", "manual", False]`, *optional*): + Is the repo gated. + If so, whether there is manual or automatic approval. + last_modified (`datetime`, *optional*): + Date of last commit to the repo. + likes (`int`): + Number of likes of the dataset. + main_size (`int`, *optional*): + Size in bytes of the main branch of the dataset. + paperswithcode_id (`str`, *optional*): + Papers with code ID of the dataset. + private (`bool`): + Is the repo private. + resource_group (`dict`, *optional*): + Resource group information for the dataset. + sha (`str`): + Repo SHA at this particular revision. + siblings (`list[RepoSibling]`): + List of [`huggingface_hub.hf_api.RepoSibling`] objects that constitute the dataset. + tags (`list[str]`): + List of tags of the dataset. + trending_score (`int`, *optional*): + Trending score of the dataset. + used_storage (`int`, *optional*): + Size in bytes of the dataset on the Hub. + """ + + id: str + author: str | None + card_data: DatasetCardData | None + citation: str | None + created_at: datetime | None + description: str | None + disabled: bool | None + downloads: int | None + downloads_all_time: int | None + gated: Literal["auto", "manual", False] | None + last_modified: datetime | None + likes: int | None + main_size: int | None + paperswithcode_id: str | None + private: bool | None + resource_group: dict | None + sha: str | None + siblings: list[RepoSibling] | None + tags: list[str] | None + trending_score: int | None + used_storage: int | None + + def __init__(self, **kwargs): + self.id = kwargs.pop("id") + self.author = kwargs.pop("author", None) + self.sha = kwargs.pop("sha", None) + created_at = kwargs.pop("createdAt", None) or kwargs.pop("created_at", None) + self.created_at = parse_datetime(created_at) if created_at else None + last_modified = kwargs.pop("lastModified", None) or kwargs.pop("last_modified", None) + self.last_modified = parse_datetime(last_modified) if last_modified else None + self.private = kwargs.pop("private", None) + self.gated = kwargs.pop("gated", None) + self.disabled = kwargs.pop("disabled", None) + self.downloads = kwargs.pop("downloads", None) + self.downloads_all_time = kwargs.pop("downloadsAllTime", None) + self.likes = kwargs.pop("likes", None) + self.main_size = kwargs.pop("mainSize", None) + self.paperswithcode_id = kwargs.pop("paperswithcode_id", None) + self.tags = kwargs.pop("tags", None) + self.trending_score = kwargs.pop("trendingScore", None) + self.used_storage = kwargs.pop("usedStorage", None) + + card_data = kwargs.pop("cardData", None) or kwargs.pop("card_data", None) + self.card_data = ( + DatasetCardData(**card_data, ignore_metadata_errors=True) if isinstance(card_data, dict) else card_data + ) + siblings = kwargs.pop("siblings", None) + self.siblings = ( + [ + RepoSibling( + rfilename=sibling["rfilename"], + size=sibling.get("size"), + blob_id=sibling.get("blobId"), + lfs=( + BlobLfsInfo( + size=sibling["lfs"]["size"], + sha256=sibling["lfs"]["sha256"], + pointer_size=sibling["lfs"]["pointerSize"], + ) + if sibling.get("lfs") + else None + ), + ) + for sibling in siblings + ] + if siblings is not None + else None + ) + self.citation = kwargs.pop("citation", None) + self.description = kwargs.pop("description", None) + self.resource_group = kwargs.pop("resourceGroup", None) + # backwards compatibility + self.lastModified = self.last_modified + self.cardData = self.card_data + self.__dict__.update(**kwargs) + + +@dataclass +class SpaceInfo: + """ + Contains information about a Space on the Hub. This object is returned by [`space_info`] and [`list_spaces`]. + + > [!TIP] + > Most attributes of this class are optional. This is because the data returned by the Hub depends on the query made. + > In general, the more specific the query, the more information is returned. On the contrary, when listing spaces + > using [`list_spaces`] only a subset of the attributes are returned. + + Attributes: + id (`str`): + ID of the Space. + author (`str`, *optional*): + Author of the Space. + card_data (`SpaceCardData`, *optional*): + Space Card Metadata as a [`huggingface_hub.repocard_data.SpaceCardData`] object. + created_at (`datetime`, *optional*): + Date of creation of the repo on the Hub. Note that the lowest value is `2022-03-02T23:29:04.000Z`, + corresponding to the date when we began to store creation dates. + datasets (`list[str]`, *optional*): + List of datasets used by the Space. + disabled (`bool`, *optional*): + Is the Space disabled. + gated (`Literal["auto", "manual", False]`, *optional*): + Is the repo gated. + If so, whether there is manual or automatic approval. + host (`str`, *optional*): + Host URL of the Space. + last_modified (`datetime`, *optional*): + Date of last commit to the repo. + likes (`int`): + Number of likes of the Space. + models (`list[str]`, *optional*): + List of models used by the Space. + private (`bool`): + Is the repo private. + resource_group (`dict`, *optional*): + Resource group information for the Space. + runtime (`SpaceRuntime`, *optional*): + Space runtime information as a [`huggingface_hub.hf_api.SpaceRuntime`] object. + sdk (`str`, *optional*): + SDK used by the Space. + sha (`str`, *optional*): + Repo SHA at this particular revision. + siblings (`list[RepoSibling]`): + List of [`huggingface_hub.hf_api.RepoSibling`] objects that constitute the Space. + subdomain (`str`, *optional*): + Subdomain of the Space. + tags (`list[str]`): + List of tags of the Space. + trending_score (`int`, *optional*): + Trending score of the Space. + used_storage (`int`, *optional*): + Size in bytes of the Space on the Hub. + """ + + id: str + author: str | None + card_data: SpaceCardData | None + created_at: datetime | None + datasets: list[str] | None + disabled: bool | None + gated: Literal["auto", "manual", False] | None + host: str | None + last_modified: datetime | None + likes: int | None + models: list[str] | None + private: bool | None + resource_group: dict | None + runtime: SpaceRuntime | None + sdk: str | None + sha: str | None + siblings: list[RepoSibling] | None + subdomain: str | None + tags: list[str] | None + trending_score: int | None + used_storage: int | None + + def __init__(self, **kwargs): + self.id = kwargs.pop("id") + self.author = kwargs.pop("author", None) + self.sha = kwargs.pop("sha", None) + created_at = kwargs.pop("createdAt", None) or kwargs.pop("created_at", None) + self.created_at = parse_datetime(created_at) if created_at else None + last_modified = kwargs.pop("lastModified", None) or kwargs.pop("last_modified", None) + self.last_modified = parse_datetime(last_modified) if last_modified else None + self.private = kwargs.pop("private", None) + self.gated = kwargs.pop("gated", None) + self.disabled = kwargs.pop("disabled", None) + self.host = kwargs.pop("host", None) + self.subdomain = kwargs.pop("subdomain", None) + self.likes = kwargs.pop("likes", None) + self.sdk = kwargs.pop("sdk", None) + self.tags = kwargs.pop("tags", None) + self.trending_score = kwargs.pop("trendingScore", None) + self.used_storage = kwargs.pop("usedStorage", None) + card_data = kwargs.pop("cardData", None) or kwargs.pop("card_data", None) + self.card_data = ( + SpaceCardData(**card_data, ignore_metadata_errors=True) if isinstance(card_data, dict) else card_data + ) + siblings = kwargs.pop("siblings", None) + self.siblings = ( + [ + RepoSibling( + rfilename=sibling["rfilename"], + size=sibling.get("size"), + blob_id=sibling.get("blobId"), + lfs=( + BlobLfsInfo( + size=sibling["lfs"]["size"], + sha256=sibling["lfs"]["sha256"], + pointer_size=sibling["lfs"]["pointerSize"], + ) + if sibling.get("lfs") + else None + ), + ) + for sibling in siblings + ] + if siblings is not None + else None + ) + runtime = kwargs.pop("runtime", None) + self.runtime = SpaceRuntime(runtime) if runtime else None + self.models = kwargs.pop("models", None) + self.datasets = kwargs.pop("datasets", None) + self.resource_group = kwargs.pop("resourceGroup", None) + # backwards compatibility + self.lastModified = self.last_modified + self.cardData = self.card_data + self.__dict__.update(**kwargs) + + +@dataclass +class KernelInfo: + """ + Contains information about a kernel repo on the Hub. This object is returned by [`kernel_info`]. + + Attributes: + id (`str`): + ID of the kernel repo. + author (`str`, *optional*): + Author of the kernel repo. + downloads (`int`, *optional*): + Number of downloads of the kernel repo over the last 30 days. + gated (`Literal["auto", "manual", False]`, *optional*): + Is the repo gated. If so, whether there is manual or automatic approval. + last_modified (`datetime`, *optional*): + Date of last commit to the repo. + likes (`int`, *optional*): + Number of likes of the kernel repo. + private (`bool`, *optional*): + Is the repo private. + sha (`str`, *optional*): + Repo SHA at this particular revision. + """ + + id: str + author: str | None + downloads: int | None + gated: Literal["auto", "manual", False] | None + last_modified: datetime | None + likes: int | None + private: bool | None + sha: str | None + + def __init__(self, **kwargs): + self.id = kwargs.pop("id") + self.author = kwargs.pop("author", None) + self.downloads = kwargs.pop("downloads", None) + self.gated = kwargs.pop("gated", None) + last_modified = kwargs.pop("lastModified", None) or kwargs.pop("last_modified", None) + self.last_modified = parse_datetime(last_modified) if last_modified else None + self.likes = kwargs.pop("likes", None) + self.private = kwargs.pop("private", None) + self.sha = kwargs.pop("sha", None) + + # future compatibility + self.__dict__.update(**kwargs) + + +@dataclass +class CollectionItem: + """ + Contains information about an item of a Collection (model, dataset, Space, paper, collection or bucket). + + Attributes: + item_object_id (`str`): + Unique ID of the item in the collection. + item_id (`str`): + ID of the underlying object on the Hub. Can be either a repo_id, a paper id, a collection slug + or a bucket id. + e.g. `"jbilcke-hf/ai-comic-factory"`, `"2307.09288"`, `"celinah/cerebras-function-calling-682607169c35fbfa98b30b9a"`. + item_type (`str`): + Type of the underlying object. Can be one of `"model"`, `"dataset"`, `"space"`, `"paper"`, `"collection"` + or `"bucket"`. + position (`int`): + Position of the item in the collection. + note (`str`, *optional*): + Note associated with the item, as plain text. + """ + + item_object_id: str # id in database + item_id: str # repo_id or paper id + item_type: str + position: int + note: str | None = None + + def __init__( + self, + _id: str, + id: str, + type: CollectionItemType_T, + position: int, + note: dict | None = None, + **kwargs, + ) -> None: + self.item_object_id: str = _id # id in database + self.item_id: str = id # repo_id or paper id + # if the item is a collection, override item_id with the slug + slug = kwargs.get("slug") + if slug is not None: + self.item_id = slug # collection slug + self.item_type: CollectionItemType_T = type + self.position: int = position + note_text = note.get("text") if note is not None else None + self.note = note_text if isinstance(note_text, str) else None + + +@dataclass +class Collection: + """ + Contains information about a Collection on the Hub. + + Attributes: + slug (`str`): + Slug of the collection. E.g. `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. + title (`str`): + Title of the collection. E.g. `"Recent models"`. + owner (`str`): + Owner of the collection. E.g. `"TheBloke"`. + items (`list[CollectionItem]`): + List of items in the collection. + last_updated (`datetime`): + Date of the last update of the collection. + position (`int`): + Position of the collection in the list of collections of the owner. + private (`bool`): + Whether the collection is private or not. + theme (`str`): + Theme of the collection. E.g. `"green"`. + upvotes (`int`): + Number of upvotes of the collection. + description (`str`, *optional*): + Description of the collection, as plain text. + url (`str`): + (property) URL of the collection on the Hub. + """ + + slug: str + title: str + owner: str + items: list[CollectionItem] + last_updated: datetime + position: int + private: bool + theme: str + upvotes: int + description: str | None = None + + def __init__(self, **kwargs) -> None: + self.slug = kwargs.pop("slug") + self.title = kwargs.pop("title") + self.owner = kwargs.pop("owner") + self.items = [CollectionItem(**item) for item in kwargs.pop("items")] + self.last_updated = parse_datetime(kwargs.pop("lastUpdated")) + self.position = kwargs.pop("position") + self.private = kwargs.pop("private") + self.theme = kwargs.pop("theme") + self.upvotes = kwargs.pop("upvotes") + self.description = kwargs.pop("description", None) + endpoint = kwargs.pop("endpoint", None) + if endpoint is None: + endpoint = constants.ENDPOINT + self._url = f"{endpoint}/collections/{self.slug}" + + @property + def url(self) -> str: + """Returns the URL of the collection on the Hub.""" + return self._url + + +@dataclass +class GitRefInfo: + """ + Contains information about a git reference for a repo on the Hub. + + Attributes: + name (`str`): + Name of the reference (e.g. tag name or branch name). + ref (`str`): + Full git ref on the Hub (e.g. `"refs/heads/main"` or `"refs/tags/v1.0"`). + target_commit (`str`): + OID of the target commit for the ref (e.g. `"e7da7f221d5bf496a48136c0cd264e630fe9fcc8"`) + """ + + name: str + ref: str + target_commit: str + + +@dataclass +class GitRefs: + """ + Contains information about all git references for a repo on the Hub. + + Object is returned by [`list_repo_refs`]. + + Attributes: + branches (`list[GitRefInfo]`): + A list of [`GitRefInfo`] containing information about branches on the repo. + converts (`list[GitRefInfo]`): + A list of [`GitRefInfo`] containing information about "convert" refs on the repo. + Converts are refs used (internally) to push preprocessed data in Dataset repos. + tags (`list[GitRefInfo]`): + A list of [`GitRefInfo`] containing information about tags on the repo. + pull_requests (`list[GitRefInfo]`, *optional*): + A list of [`GitRefInfo`] containing information about pull requests on the repo. + Only returned if `include_prs=True` is set. + """ + + branches: list[GitRefInfo] + converts: list[GitRefInfo] + tags: list[GitRefInfo] + pull_requests: list[GitRefInfo] | None = None + + +@dataclass +class GitCommitInfo: + """ + Contains information about a git commit for a repo on the Hub. Check out [`list_repo_commits`] for more details. + + Attributes: + commit_id (`str`): + OID of the commit (e.g. `"e7da7f221d5bf496a48136c0cd264e630fe9fcc8"`) + authors (`list[str]`): + List of authors of the commit. + created_at (`datetime`): + Datetime when the commit was created. + title (`str`): + Title of the commit. This is a free-text value entered by the authors. + message (`str`): + Description of the commit. This is a free-text value entered by the authors. + formatted_title (`str`): + Title of the commit formatted as HTML. Only returned if `formatted=True` is set. + formatted_message (`str`): + Description of the commit formatted as HTML. Only returned if `formatted=True` is set. + """ + + commit_id: str + + authors: list[str] + created_at: datetime + title: str + message: str + + formatted_title: str | None + formatted_message: str | None + + +@dataclass +class UserLikes: + """ + Contains information about a user likes on the Hub. + + Attributes: + user (`str`): + Name of the user for which we fetched the likes. + total (`int`): + Total number of likes. + datasets (`list[str]`): + List of datasets liked by the user (as repo_ids). + kernels (`list[str]`): + List of kernels liked by the user (as repo_ids). + models (`list[str]`): + List of models liked by the user (as repo_ids). + spaces (`list[str]`): + List of spaces liked by the user (as repo_ids). + """ + + # Metadata + user: str + total: int + + # User likes + datasets: list[str] + kernels: list[str] + models: list[str] + spaces: list[str] + + +@dataclass +class RepoStorageInfo: + """ + Contains storage information about a repository on the Hub. + + Returned by [`list_user_repos`]. + + Attributes: + id (`str`): + ID of the repo (e.g. `username/repo-name`). + type (`str`): + Type of the repo (`model`, `dataset`, `space`, or `bucket`). + updated_at (`datetime`): + Last update time of the repo. + visibility (`str`): + Visibility of the repo (`public` or `private`). + storage (`int`): + Storage used by the repo in bytes. + storage_percent (`float`): + Percentage of the namespace's total storage used by this repo. + """ + + id: str + type: str + updated_at: datetime + visibility: str + storage: int + storage_percent: float + + def __init__(self, **kwargs: Any) -> None: + self.id = kwargs["id"] + self.type = kwargs["type"] + self.updated_at = parse_datetime(kwargs["updatedAt"]) + self.visibility = kwargs["visibility"] + self.storage = kwargs["storage"] + self.storage_percent = kwargs.get("storagePercent") or 0 + + +@dataclass +class Organization: + """ + Contains information about an organization on the Hub. + + Attributes: + avatar_url (`str`): + URL of the organization's avatar. + name (`str`): + Name of the organization on the Hub (unique). + fullname (`str`): + Organization's full name. + details (`str`, *optional*): + Organization's description. + is_verified (`bool`, *optional*): + Whether the organization is verified. + is_following (`bool`, *optional*): + Whether the authenticated user follows this organization. + num_users (`int`, *optional*): + Number of members in the organization. + num_models (`int`, *optional*): + Number of models owned by the organization. + num_spaces (`int`, *optional*): + Number of Spaces owned by the organization. + num_datasets (`int`, *optional*): + Number of datasets owned by the organization. + num_followers (`int`, *optional*): + Number of followers of the organization. + num_papers (`int`, *optional*): + Number of papers authored by the organization. + plan (`str`, *optional*): + The organization's plan (e.g., "enterprise", "team"). + """ + + avatar_url: str + name: str + fullname: str + details: str | None = None + is_verified: bool | None = None + is_following: bool | None = None + num_users: int | None = None + num_models: int | None = None + num_spaces: int | None = None + num_datasets: int | None = None + num_followers: int | None = None + num_papers: int | None = None + plan: str | None = None + + def __init__(self, **kwargs) -> None: + self.avatar_url = kwargs.pop("avatarUrl", "") + self.name = kwargs.pop("name", "") + self.fullname = kwargs.pop("fullname", "") + self.details = kwargs.pop("details", None) + self.is_verified = kwargs.pop("isVerified", None) + self.is_following = kwargs.pop("isFollowing", None) + self.num_users = kwargs.pop("numUsers", None) + self.num_models = kwargs.pop("numModels", None) + self.num_spaces = kwargs.pop("numSpaces", None) + self.num_datasets = kwargs.pop("numDatasets", None) + self.num_followers = kwargs.pop("numFollowers", None) + self.num_papers = kwargs.pop("numPapers", None) + self.plan = kwargs.pop("plan", None) + + # forward compatibility + self.__dict__.update(**kwargs) + + +@dataclass +class User: + """ + Contains information about a user on the Hub. + + Attributes: + username (`str`): + Name of the user on the Hub (unique). + fullname (`str`): + User's full name. + avatar_url (`str`): + URL of the user's avatar. + details (`str`, *optional*): + User's details. + is_following (`bool`, *optional*): + Whether the authenticated user is following this user. + is_pro (`bool`, *optional*): + Whether the user is a pro user. + num_models (`int`, *optional*): + Number of models created by the user. + num_datasets (`int`, *optional*): + Number of datasets created by the user. + num_spaces (`int`, *optional*): + Number of spaces created by the user. + num_discussions (`int`, *optional*): + Number of discussions initiated by the user. + num_papers (`int`, *optional*): + Number of papers authored by the user. + num_upvotes (`int`, *optional*): + Number of upvotes received by the user. + num_likes (`int`, *optional*): + Number of likes given by the user. + num_following (`int`, *optional*): + Number of users this user is following. + num_followers (`int`, *optional*): + Number of users following this user. + orgs (list of [`Organization`]): + List of organizations the user is part of. + """ + + # Metadata + username: str + fullname: str + avatar_url: str + details: str | None = None + is_following: bool | None = None + is_pro: bool | None = None + num_models: int | None = None + num_datasets: int | None = None + num_spaces: int | None = None + num_discussions: int | None = None + num_papers: int | None = None + num_upvotes: int | None = None + num_likes: int | None = None + num_following: int | None = None + num_followers: int | None = None + orgs: list[Organization] = field(default_factory=list) + + def __init__(self, **kwargs) -> None: + self.username = kwargs.pop("user", "") + self.fullname = kwargs.pop("fullname", "") + self.avatar_url = kwargs.pop("avatarUrl", "") + self.is_following = kwargs.pop("isFollowing", None) + self.is_pro = kwargs.pop("isPro", None) + self.details = kwargs.pop("details", None) + self.num_models = kwargs.pop("numModels", None) + self.num_datasets = kwargs.pop("numDatasets", None) + self.num_spaces = kwargs.pop("numSpaces", None) + self.num_discussions = kwargs.pop("numDiscussions", None) + self.num_papers = kwargs.pop("numPapers", None) + self.num_upvotes = kwargs.pop("numUpvotes", None) + self.num_likes = kwargs.pop("numLikes", None) + self.num_following = kwargs.pop("numFollowing", None) + self.num_followers = kwargs.pop("numFollowers", None) + self.user_type = kwargs.pop("type", None) + self.orgs = [Organization(**org) for org in kwargs.pop("orgs", [])] + + # forward compatibility + self.__dict__.update(**kwargs) + + +@dataclass +class PaperAuthor: + """ + Contains information about a paper author on the Hub. + + Attributes: + name (`str`): + Name of the author. + user (`User`, *optional*): + Information about the author as a [`User`] object. + status (`str`, *optional*): + Status of the author on the Hub. + status_last_changed_at (`datetime`, *optional*): + Date when the status of the author changed. + hidden (`bool`, *optional*): + Whether the author is hidden on the Hub. + """ + + name: str + user: User | None + status: str | None + status_last_changed_at: datetime | None + hidden: bool | None + + def __init__(self, **kwargs) -> None: + self.name = kwargs.pop("name", "") + user = kwargs.pop("user", None) + self.user = User(**user) if user else None + self.status = kwargs.pop("status", None) + status_last_changed_at = kwargs.pop("statusLastChangedAt", None) + self.status_last_changed_at = parse_datetime(status_last_changed_at) if status_last_changed_at else None + self.hidden = kwargs.pop("hidden", None) + + self.__dict__.update(**kwargs) + + +@dataclass +class PaperInfo: + """ + Contains information about a paper on the Hub. + + Attributes: + id (`str`): + arXiv paper ID. + authors (`list[PaperAuthor]`, *optional*): + Authors of the paper. + published_at (`datetime`, *optional*): + Date paper published. + title (`str`, *optional*): + Title of the paper. + summary (`str`, *optional*): + Summary of the paper. + upvotes (`int`, *optional*): + Number of upvotes for the paper on the Hub. + discussion_id (`str`, *optional*): + Discussion ID for the paper on the Hub. + source (`str`, *optional*): + Source of the paper. + comments (`int`, *optional*): + Number of comments for the paper on the Hub. + submitted_at (`datetime`, *optional*): + Date paper appeared in daily papers on the Hub. + submitted_by (`User`, *optional*): + Information about who submitted the daily paper. + ai_summary (`str`, *optional*): + AI summary of the paper. + ai_keywords (`list[str]`, *optional*): + AI keywords of the paper. + organization (`Organization`, *optional*): + Information about the organization associated with the paper. + project_page (`str`, *optional*): + URL of the project page for the paper. + github_repo (`str`, *optional*): + URL of the GitHub repository for the paper. + github_stars (`int`, *optional*): + Number of stars of the GitHub repository for the paper. + linked_models (`list[ModelInfo]`, *optional*): + Models linked to the paper. Only returned by [`paper_info`]. + num_total_models (`int`, *optional*): + Total number of models linked to the paper. Only returned by [`paper_info`]. + linked_datasets (`list[DatasetInfo]`, *optional*): + Datasets linked to the paper. Only returned by [`paper_info`]. + num_total_datasets (`int`, *optional*): + Total number of datasets linked to the paper. Only returned by [`paper_info`]. + linked_spaces (`list[SpaceInfo]`, *optional*): + Spaces linked to the paper. Only returned by [`paper_info`]. + """ + + id: str + authors: list[PaperAuthor] | None + published_at: datetime | None + title: str | None + summary: str | None + upvotes: int | None + discussion_id: str | None + source: str | None + comments: int | None + submitted_at: datetime | None + submitted_by: User | None + ai_summary: str | None + ai_keywords: list[str] | None + organization: Organization | None + project_page: str | None + github_repo: str | None + github_stars: int | None + linked_models: list[ModelInfo] | None + num_total_models: int | None + linked_datasets: list[DatasetInfo] | None + num_total_datasets: int | None + linked_spaces: list[SpaceInfo] | None + + def __init__(self, **kwargs) -> None: + paper = kwargs.pop("paper", {}) + self.id = kwargs.pop("id", None) or paper.pop("id", None) + authors = paper.pop("authors", None) or kwargs.pop("authors", None) + self.authors = [PaperAuthor(**author) for author in authors] if authors else None + published_at = paper.pop("publishedAt", None) or kwargs.pop("publishedAt", None) + self.published_at = parse_datetime(published_at) if published_at else None + self.title = kwargs.pop("title", None) + self.source = kwargs.pop("source", None) + self.summary = paper.pop("summary", None) or kwargs.pop("summary", None) + self.upvotes = paper.pop("upvotes", None) or kwargs.pop("upvotes", None) + self.discussion_id = paper.pop("discussionId", None) or kwargs.pop("discussionId", None) + self.comments = kwargs.pop("numComments", 0) + submitted_at = kwargs.pop("publishedAt", None) or kwargs.pop("submittedOnDailyAt", None) + self.submitted_at = parse_datetime(submitted_at) if submitted_at else None + submitted_by = kwargs.pop("submittedBy", None) or kwargs.pop("submittedOnDailyBy", None) + self.submitted_by = User(**submitted_by) if submitted_by else None + self.ai_summary = kwargs.pop("ai_summary", None) + self.ai_keywords = kwargs.pop("ai_keywords", None) + organization = kwargs.pop("organization", None) + self.organization = Organization(**organization) if organization else None + self.project_page = kwargs.pop("projectPage", None) + self.github_repo = kwargs.pop("githubRepo", None) + self.github_stars = kwargs.pop("githubStars", None) + linked_models = kwargs.pop("linkedModels", None) + self.linked_models = [ModelInfo(**m) for m in linked_models] if linked_models is not None else None + self.num_total_models = kwargs.pop("numTotalModels", None) + linked_datasets = kwargs.pop("linkedDatasets", None) + self.linked_datasets = [DatasetInfo(**d) for d in linked_datasets] if linked_datasets is not None else None + self.num_total_datasets = kwargs.pop("numTotalDatasets", None) + linked_spaces = kwargs.pop("linkedSpaces", None) + self.linked_spaces = [SpaceInfo(**s) for s in linked_spaces] if linked_spaces is not None else None + + # forward compatibility + self.__dict__.update(**kwargs) + + +@dataclass +class LFSFileInfo: + """ + Contains information about a file stored as LFS on a repo on the Hub. + + Used in the context of listing and permanently deleting LFS files from a repo to free-up space. + See [`list_lfs_files`] and [`permanently_delete_lfs_files`] for more details. + + Git LFS files are tracked using SHA-256 object IDs, rather than file paths, to optimize performance + This approach is necessary because a single object can be referenced by multiple paths across different commits, + making it impractical to search and resolve these connections. Check out [our documentation](https://huggingface.co/docs/hub/storage-limits#advanced-track-lfs-file-references) + to learn how to know which filename(s) is(are) associated with each SHA. + + Attributes: + file_oid (`str`): + SHA-256 object ID of the file. This is the identifier to pass when permanently deleting the file. + filename (`str`): + Possible filename for the LFS object. See the note above for more information. + oid (`str`): + OID of the LFS object. + pushed_at (`datetime`): + Date the LFS object was pushed to the repo. + ref (`str`, *optional*): + Ref where the LFS object has been pushed (if any). + size (`int`): + Size of the LFS object. + + Example: + ```py + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> lfs_files = api.list_lfs_files("username/my-cool-repo") + + # Filter files files to delete based on a combination of `filename`, `pushed_at`, `ref` or `size`. + # e.g. select only LFS files in the "checkpoints" folder + >>> lfs_files_to_delete = (lfs_file for lfs_file in lfs_files if lfs_file.filename.startswith("checkpoints/")) + + # Permanently delete LFS files + >>> api.permanently_delete_lfs_files("username/my-cool-repo", lfs_files_to_delete) + ``` + """ + + file_oid: str + filename: str + oid: str + pushed_at: datetime + ref: str | None + size: int + + def __init__(self, **kwargs) -> None: + self.file_oid = kwargs.pop("fileOid") + self.filename = kwargs.pop("filename") + self.oid = kwargs.pop("oid") + self.pushed_at = parse_datetime(kwargs.pop("pushedAt")) + self.ref = kwargs.pop("ref", None) + self.size = kwargs.pop("size") + + # forward compatibility + self.__dict__.update(**kwargs) + + +@dataclass +class DatasetLeaderboardEntry: + """Contains information about a single entry in a dataset leaderboard on the Hub. + + A leaderboard ranks models based on their evaluation scores on a given benchmark dataset. + This object is returned by [`get_dataset_leaderboard`]. To get evaluation results for a + specific model across benchmarks, see [`ModelInfo.eval_results`] (via [`model_info`] with + `expand=["evalResults"]`) and [`EvalResultEntry`]. + + Attributes: + rank (`int`): + Rank of the model on the leaderboard (1-indexed). + model_id (`str`): + ID of the model (e.g. `"meta-llama/Llama-3-8b"`). + value (`float`): + Evaluation score value. + filename (`str`): + Name of the result file containing the evaluation data. + verified (`bool`): + Whether the result has been verified. + source (`dict[str, Any]`, *optional*): + Information about the source of the evaluation result. Contains keys like + `"url"`, `"name"`, and `"isExternal"`. Not all entries have a source. + author (`User` or `Organization`): + The model author, parsed based on the `"type"` field in the API response. + pull_request (`int`, *optional*): + Pull request number associated with the leaderboard entry, if any. + notes (`str`, *optional*): + Notes associated with the leaderboard entry, if any. + """ + + rank: int + model_id: str + value: float + filename: str + verified: bool + source: dict[str, Any] | None + author: User | Organization + pull_request: int | None = None + notes: str | None = None + + def __init__(self, **kwargs) -> None: + self.rank = kwargs.pop("rank") + self.model_id = kwargs.pop("modelId") + self.value = kwargs.pop("value") + self.filename = kwargs.pop("filename") + self.verified = kwargs.pop("verified") + self.source = kwargs.pop("source", None) + author_data = dict(kwargs.pop("author")) + author_type = author_data.get("type") + if author_type == "org": + self.author = Organization(**author_data) + else: + author_data["user"] = author_data.pop("name", "") + self.author = User(**author_data) + self.pull_request = kwargs.pop("pullRequest", None) + self.notes = kwargs.pop("notes", None) + + # forward compatibility + self.__dict__.update(**kwargs) + + +def future_compatible(fn: CallableT) -> CallableT: + """Wrap a method of `HfApi` to handle `run_as_future=True`. + + A method flagged as "future_compatible" will be called in a thread if `run_as_future=True` and return a + `concurrent.futures.Future` instance. Otherwise, it will be called normally and return the result. + """ + sig = inspect.signature(fn) + args_params = list(sig.parameters)[1:] # remove "self" from list + + @wraps(fn) + def _inner(self, *args, **kwargs): + # Get `run_as_future` value if provided (default to False) + if "run_as_future" in kwargs: + run_as_future = kwargs["run_as_future"] + kwargs["run_as_future"] = False # avoid recursion error + else: + run_as_future = False + for param, value in zip(args_params, args): + if param == "run_as_future": + run_as_future = value + break + + # Call the function in a thread if `run_as_future=True` + if run_as_future: + return self.run_as_future(fn, self, *args, **kwargs) + + # Otherwise, call the function normally + return fn(self, *args, **kwargs) + + _inner.is_future_compatible = True # type: ignore + return _inner # type: ignore + + +def _get_safetensors_metadata_size(size_bytes: bytes, filename: str, context_msg: str) -> int: + """ + Parse and validate safetensors metadata size from the first 8 bytes. + + This is a shared helper function used by both remote and local safetensors parsing. + + Args: + size_bytes: First 8 bytes of the safetensors file. + filename: Filename for error messages. + context_msg: Additional context for error messages. + + Returns: + The metadata size as an integer. + + Raises: + SafetensorsParsingError: If size_bytes is too short or metadata size exceeds limit. + """ + if len(size_bytes) < 8: + raise SafetensorsParsingError( + f"Failed to parse safetensors header for '{filename}' ({context_msg}): file is too small to be a valid " + "safetensors file." + ) + + metadata_size = struct.unpack(" constants.SAFETENSORS_MAX_HEADER_LENGTH: + raise SafetensorsParsingError( + f"Failed to parse safetensors header for '{filename}' ({context_msg}): safetensors header is too big. " + f"Maximum supported size is {constants.SAFETENSORS_MAX_HEADER_LENGTH} bytes (got {metadata_size})." + ) + + return metadata_size + + +def _parse_safetensors_header(metadata_as_bytes: bytes, filename: str, context_msg: str) -> SafetensorsFileMetadata: + """ + Parse safetensors metadata from raw header bytes. + + This is a shared helper function used by both remote and local safetensors parsing. + + Args: + metadata_as_bytes: Raw bytes of the JSON metadata header (without the 8-byte size prefix). + filename: Filename for error messages. + context_msg: Additional context for error messages (e.g., repo info or local path). + + Returns: + SafetensorsFileMetadata object. + + Raises: + SafetensorsParsingError: If the header cannot be parsed. + """ + # Parse json header + try: + metadata_as_dict = json.loads(metadata_as_bytes.decode(errors="ignore")) + except json.JSONDecodeError as e: + raise SafetensorsParsingError( + f"Failed to parse safetensors header for '{filename}' ({context_msg}): header is not json-encoded string. " + "Please make sure this is a correctly formatted safetensors file." + ) from e + + try: + return SafetensorsFileMetadata( + metadata=metadata_as_dict.get("__metadata__", {}), + tensors={ + key: TensorInfo( + dtype=tensor["dtype"], + shape=tensor["shape"], + data_offsets=tuple(tensor["data_offsets"]), # type: ignore + ) + for key, tensor in metadata_as_dict.items() + if key != "__metadata__" + }, + ) + except (KeyError, IndexError) as e: + raise SafetensorsParsingError( + f"Failed to parse safetensors header for '{filename}' ({context_msg}): header format not recognized. " + "Please make sure this is a correctly formatted safetensors file." + ) from e + + +class HfApi: + """ + Client to interact with the Hugging Face Hub via HTTP. + + The client is initialized with some high-level settings used in all requests + made to the Hub (HF endpoint, authentication, user agents...). Using the `HfApi` + client is preferred but not mandatory as all of its public methods are exposed + directly at the root of `huggingface_hub`. + + Args: + endpoint (`str`, *optional*): + Endpoint of the Hub. Defaults to . + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + library_name (`str`, *optional*): + The name of the library that is making the HTTP request. Will be added to + the user-agent header. Example: `"transformers"`. + library_version (`str`, *optional*): + The version of the library that is making the HTTP request. Will be added + to the user-agent header. Example: `"4.24.0"`. + user_agent (`str`, `dict`, *optional*): + The user agent info in the form of a dictionary or a single string. It will + be completed with information about the installed packages. + headers (`dict`, *optional*): + Additional headers to be sent with each request. Example: `{"X-My-Header": "value"}`. + Headers passed here are taking precedence over the default headers. + """ + + def __init__( + self, + endpoint: str | None = None, + token: str | bool | None = None, + library_name: str | None = None, + library_version: str | None = None, + user_agent: dict | str | None = None, + headers: dict[str, str] | None = None, + ) -> None: + self.endpoint = endpoint if endpoint is not None else constants.ENDPOINT + self.token = token + self.library_name = library_name + self.library_version = library_version + self.user_agent = user_agent + self.headers = headers + self._thread_pool: ThreadPoolExecutor | None = None + + # /whoami-v2 is the only endpoint for which we may want to cache results + self._whoami_cache: dict[str, dict] = {} + + def run_as_future(self, fn: Callable[..., R], *args, **kwargs) -> Future[R]: + """ + Run a method in the background and return a Future instance. + + The main goal is to run methods without blocking the main thread (e.g. to push data during a training). + Background jobs are queued to preserve order but are not ran in parallel. If you need to speed-up your scripts + by parallelizing lots of call to the API, you must setup and use your own [ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor). + + Note: Most-used methods like [`upload_file`], [`upload_folder`] and [`create_commit`] have a `run_as_future: bool` + argument to directly call them in the background. This is equivalent to calling `api.run_as_future(...)` on them + but less verbose. + + Args: + fn (`Callable`): + The method to run in the background. + *args, **kwargs: + Arguments with which the method will be called. + + Return: + `Future`: a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) instance to + get the result of the task. + + Example: + ```py + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> future = api.run_as_future(api.whoami) # instant + >>> future.done() + False + >>> future.result() # wait until complete and return result + (...) + >>> future.done() + True + ``` + """ + if self._thread_pool is None: + self._thread_pool = ThreadPoolExecutor(max_workers=1) + self._thread_pool + return self._thread_pool.submit(fn, *args, **kwargs) + + @validate_hf_hub_args + def whoami(self, token: bool | str | None = None, *, cache: bool = False) -> dict: + """ + Call HF API to know "whoami". + + If passing `cache=True`, the result will be cached for subsequent calls for the duration of the Python process. This is useful if you plan to call + `whoami` multiple times as this endpoint is heavily rate-limited for security reasons. + + Args: + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + cache (`bool`, *optional*): + Whether to cache the result of the `whoami` call for subsequent calls. + If an error occurs during the first call, it won't be cached. + Defaults to `False`. + """ + # Get the effective token using the helper function get_token + token = self.token if token is None else token + if token is False: + raise ValueError("Cannot use `token=False` with `whoami` method as it requires authentication.") + if token is True or token is None: + token = get_token() + if token is None: + raise LocalTokenNotFoundError( + "Token is required to call the /whoami-v2 endpoint, but no token found. You must provide a token or be logged in to " + "Hugging Face with `hf auth login` or `huggingface_hub.login`. See https://huggingface.co/settings/tokens." + ) + + if cache and (cached_token := self._whoami_cache.get(token)): + return cached_token + + # Call Hub + output = self._inner_whoami(token=token) + + # Cache result and return + if cache: + self._whoami_cache[token] = output + return output + + def _inner_whoami(self, token: str) -> dict: + r = get_session().get( + f"{self.endpoint}/api/whoami-v2", + headers=self._build_hf_headers(token=token), + ) + try: + hf_raise_for_status(r) + except HfHubHTTPError as e: + if e.response.status_code == 401: + error_message = "Invalid user token." + # Check which token is the effective one and generate the error message accordingly + if token == _get_token_from_google_colab(): + error_message += " The token from Google Colab vault is invalid. Please update it from the UI." + elif token == _get_token_from_environment(): + error_message += ( + " The token from HF_TOKEN environment variable is invalid. " + "Note that HF_TOKEN takes precedence over `hf auth login`." + ) + elif token == _get_token_from_file(): + error_message += ( + " The token stored is invalid. Please run `hf auth login --force` to set a new token." + ) + raise HfHubHTTPError(error_message, response=e.response) from e + if e.response.status_code == 429: + error_message = ( + "You've hit the rate limit for the /whoami-v2 endpoint, which is intentionally strict for security reasons." + " If you're calling it often, consider caching the response with `whoami(..., cache=True)`." + ) + raise HfHubHTTPError(error_message, response=e.response) from e + raise + return r.json() + + def get_model_tags(self) -> dict: + """ + List all valid model tags as a nested namespace object + """ + path = f"{self.endpoint}/api/models-tags-by-type" + r = get_session().get(path) + hf_raise_for_status(r) + return r.json() + + def get_dataset_tags(self) -> dict: + """ + List all valid dataset tags as a nested namespace object. + """ + path = f"{self.endpoint}/api/datasets-tags-by-type" + r = get_session().get(path) + hf_raise_for_status(r) + return r.json() + + @_deprecate_arguments(version="2.0", deprecated_args=["model_name"], custom_message="Use `search` instead.") + @validate_hf_hub_args + def list_models( + self, + *, + # Search-query parameter + filter: str | Iterable[str] | None = None, + author: str | None = None, + apps: str | list[str] | None = None, + gated: bool | None = None, + inference: Literal["warm"] | None = None, + inference_provider: Literal["all"] | PROVIDER_T | list[PROVIDER_T] | None = None, + model_name: str | None = None, + trained_dataset: str | list[str] | None = None, + search: str | None = None, + pipeline_tag: str | None = None, + num_parameters: str | None = None, + emissions_thresholds: tuple[float, float] | None = None, + # Sorting and pagination parameters + sort: ModelSort_T | None = None, + limit: int | None = None, + # Additional data to fetch + expand: list[ExpandModelProperty_T] | None = None, + full: bool | None = None, + cardData: bool = False, + fetch_config: bool = False, + token: bool | str | None = None, + ) -> Iterable[ModelInfo]: + """ + List models hosted on the Huggingface Hub, given some filters. + + Args: + filter (`str` or `Iterable[str]`, *optional*): + A string or list of string to filter models on the Hub. + Models can be filtered by library, language, task, tags, and more. + author (`str`, *optional*): + A string which identify the author (user or organization) of the + returned models. + apps (`str` or `List`, *optional*): + A string or list of strings to filter models on the Hub that + support the specified apps. Example values include `"ollama"` or `["ollama", "vllm"]`. + gated (`bool`, *optional*): + A boolean to filter models on the Hub that are gated or not. By default, all models are returned. + If `gated=True` is passed, only gated models are returned. + If `gated=False` is passed, only non-gated models are returned. + inference (`Literal["warm"]`, *optional*): + If "warm", filter models on the Hub currently served by at least one provider. + inference_provider (`Literal["all"]` or `str`, *optional*): + A string to filter models on the Hub that are served by a specific provider. + Pass `"all"` to get all models served by at least one provider. + trained_dataset (`str` or `List`, *optional*): + A string tag or a list of string tags of the trained dataset for a + model on the Hub. + search (`str`, *optional*): + A string that will be contained in the returned model ids. + pipeline_tag (`str`, *optional*): + A string pipeline tag to filter models on the Hub by, such as `summarization`. + num_parameters (`str`, *optional*): + Filter models by parameter count. Accepts the same range syntax as the Hub UI and API, for example + `"min:6B,max:128B"`, `"min:6B"` or `"max:128B"`. + emissions_thresholds (`Tuple`, *optional*): + A tuple of two ints or floats representing a minimum and maximum + carbon footprint to filter the resulting models with in grams. + sort (`ModelSort_T`, *optional*): + The key with which to sort the resulting models. Possible values are "created_at", "downloads", + "last_modified", "likes" and "trending_score". + limit (`int`, *optional*): + The limit on the number of models fetched. Leaving this option + to `None` fetches all models. + expand (`list[ExpandModelProperty_T]`, *optional*): + List properties to return in the response. When used, only the properties in the list will be returned. + This parameter cannot be used if `full`, `cardData` or `fetch_config` are passed. + Possible values are `"author"`, `"cardData"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"evalResults"`, `"gated"`, `"gguf"`, `"inference"`, `"inferenceProviderMapping"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"`, `"trendingScore"`, `"widgetData"`, and `"resourceGroup"`. + full (`bool`, *optional*): + Whether to fetch all model data, including the `last_modified`, + the `sha`, the files and the `tags`. This is set to `True` by + default when using a filter. + cardData (`bool`, *optional*): + Whether to grab the metadata for the model as well. Can contain + useful information such as carbon emissions, metrics, and + datasets trained on. + fetch_config (`bool`, *optional*): + Whether to fetch the model configs as well. This is not included + in `full` due to its size. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + model_name (`str`, *optional*): + (deprecated). Use `search` instead. + + Returns: + `Iterable[ModelInfo]`: an iterable of [`huggingface_hub.hf_api.ModelInfo`] objects. + + Example: + + ```python + >>> from huggingface_hub import HfApi + + >>> api = HfApi() + + # List all models + >>> api.list_models() + + # List text classification models + >>> api.list_models(filter="text-classification") + + # List models from the KerasHub library + >>> api.list_models(filter="keras-hub") + + # List models served by Cohere + >>> api.list_models(inference_provider="cohere") + + # List models with "bert" in their name + >>> api.list_models(search="bert") + + # List models with "bert" in their name and pushed by google + >>> api.list_models(search="bert", author="google") + + # List models with 6B to 128B parameters + >>> api.list_models(num_parameters="min:6B,max:128B", sort="likes") + ``` + """ + if expand and (full or cardData or fetch_config): + raise ValueError("`expand` cannot be used if `full`, `cardData` or `fetch_config` are passed.") + + if emissions_thresholds is not None and not cardData: + raise ValueError("`emissions_thresholds` were passed without setting `cardData=True`.") + + path = f"{self.endpoint}/api/models" + headers = self._build_hf_headers(token=token) + params: dict[str, Any] = {} + + # Build the filter list + filter_list: list[str] = [] + if filter: + filter_list.extend([filter] if isinstance(filter, str) else filter) + if trained_dataset: + datasets = [trained_dataset] if isinstance(trained_dataset, str) else trained_dataset + filter_list.extend(f"dataset:{d}" if not d.startswith("dataset:") else d for d in datasets) + if len(filter_list) > 0: + params["filter"] = filter_list + + # Handle other query params + if author: + params["author"] = author + if apps: + if isinstance(apps, str): + apps = [apps] + params["apps"] = apps + if gated is not None: + params["gated"] = gated + if inference is not None: + params["inference"] = inference + if inference_provider is not None: + params["inference_provider"] = inference_provider + if pipeline_tag: + params["pipeline_tag"] = pipeline_tag + if num_parameters is not None: + params["num_parameters"] = num_parameters + search_list = [] + if model_name: # deprecated + search_list.append(model_name) + if search: + search_list.append(search) + if len(search_list) > 0: + params["search"] = search_list + if sort is not None: + params["sort"] = ( + "lastModified" + if sort == "last_modified" + else "trendingScore" + if sort == "trending_score" + else "createdAt" + if sort == "created_at" + else sort + ) + if limit is not None: + params["limit"] = limit + + # Request additional data + if full: + params["full"] = True + if fetch_config: + params["config"] = True + if cardData: + params["cardData"] = True + if expand: + params["expand"] = expand + + # `items` is a generator + items = paginate(path, params=params, headers=headers) + if limit is not None: + items = islice(items, limit) # Do not iterate over all pages + for item in items: + if "siblings" not in item: + item["siblings"] = None + model_info = ModelInfo(**item) + if emissions_thresholds is None or _is_emission_within_threshold(model_info, *emissions_thresholds): + yield model_info + + @validate_hf_hub_args + def list_datasets( + self, + *, + # Search-query parameter + filter: str | Iterable[str] | None = None, + author: str | None = None, + benchmark: Literal[True] | Literal["official"] | str | None = None, + dataset_name: str | None = None, + gated: bool | None = None, + language_creators: str | list[str] | None = None, + language: str | list[str] | None = None, + multilinguality: str | list[str] | None = None, + size_categories: str | list[str] | None = None, + task_categories: str | list[str] | None = None, + task_ids: str | list[str] | None = None, + search: str | None = None, + # Sorting and pagination parameters + sort: DatasetSort_T | None = None, + limit: int | None = None, + # Additional data to fetch + expand: list[ExpandDatasetProperty_T] | None = None, + full: bool | None = None, + token: bool | str | None = None, + ) -> Iterable[DatasetInfo]: + """ + List datasets hosted on the Huggingface Hub, given some filters. + + Args: + filter (`str` or `Iterable[str]`, *optional*): + A string or list of string to filter datasets on the hub. + author (`str`, *optional*): + A string which identify the author of the returned datasets. + benchmark (`True`, `"official"`, `str`, *optional*): + Filter datasets by benchmark. Can be `True` or `"official"` to return official benchmark datasets. + For future-compatibility, can also be a string representing the benchmark name (currently only "official" is supported). + dataset_name (`str`, *optional*): + A string or list of strings that can be used to identify datasets on + the Hub by its name, such as `SQAC` or `wikineural` + gated (`bool`, *optional*): + A boolean to filter datasets on the Hub that are gated or not. By default, all datasets are returned. + If `gated=True` is passed, only gated datasets are returned. + If `gated=False` is passed, only non-gated datasets are returned. + language_creators (`str` or `List`, *optional*): + A string or list of strings that can be used to identify datasets on + the Hub with how the data was curated, such as `crowdsourced` or + `machine_generated`. + language (`str` or `List`, *optional*): + A string or list of strings representing a two-character language to + filter datasets by on the Hub. + multilinguality (`str` or `List`, *optional*): + A string or list of strings representing a filter for datasets that + contain multiple languages. + size_categories (`str` or `List`, *optional*): + A string or list of strings that can be used to identify datasets on + the Hub by the size of the dataset such as `100K>> from huggingface_hub import HfApi + + >>> api = HfApi() + + # List all datasets + >>> api.list_datasets() + + + # List only the text classification datasets + >>> api.list_datasets(filter="task_categories:text-classification") + + + # List only the datasets in russian for language modeling + >>> api.list_datasets( + ... filter=("language:ru", "task_ids:language-modeling") + ... ) + + # List FiftyOne datasets (identified by the tag "fiftyone" in dataset card) + >>> api.list_datasets(tags="fiftyone") + ``` + + Example usage with the `search` argument: + + ```python + >>> from huggingface_hub import HfApi + + >>> api = HfApi() + + # List all datasets with "text" in their name + >>> api.list_datasets(search="text") + + # List all datasets with "text" in their name made by google + >>> api.list_datasets(search="text", author="google") + ``` + """ + if expand and full: + raise ValueError("`expand` cannot be used if `full` is passed.") + + path = f"{self.endpoint}/api/datasets" + headers = self._build_hf_headers(token=token) + params: dict[str, Any] = {} + + # Build `filter` list + filter_list = [] + if filter is not None: + if isinstance(filter, str): + filter_list.append(filter) + else: + filter_list.extend(filter) + for key, value in ( + ("language_creators", language_creators), + ("language", language), + ("multilinguality", multilinguality), + ("size_categories", size_categories), + ("task_categories", task_categories), + ("task_ids", task_ids), + ): + if value: + if isinstance(value, str): + value = [value] + for value_item in value: + if not value_item.startswith(f"{key}:"): + data = f"{key}:{value_item}" + else: + data = value_item + filter_list.append(data) + if benchmark is not None: + if benchmark is True: # alias for official benchmark + benchmark = "official" + filter_list.append(f"benchmark:{benchmark}") + if len(filter_list) > 0: + params["filter"] = filter_list + + # Handle other query params + if author: + params["author"] = author + if gated is not None: + params["gated"] = gated + search_list = [] + if dataset_name: + search_list.append(dataset_name) + if search: + search_list.append(search) + if len(search_list) > 0: + params["search"] = search_list + if sort is not None: + params["sort"] = ( + "lastModified" + if sort == "last_modified" + else "trendingScore" + if sort == "trending_score" + else "createdAt" + if sort == "created_at" + else sort + ) + if limit is not None: + params["limit"] = limit + + # Request additional data + if expand: + params["expand"] = expand + if full: + params["full"] = True + + items = paginate(path, params=params, headers=headers) + if limit is not None: + items = islice(items, limit) # Do not iterate over all pages + for item in items: + if "siblings" not in item: + item["siblings"] = None + yield DatasetInfo(**item) + + @validate_hf_hub_args + def list_dataset_parquet_files( + self, + repo_id: str, + *, + config: str | None = None, + token: bool | str | None = None, + ) -> list[DatasetParquetEntry]: + """List parquet files available for a dataset on the Hub. + + All datasets hosted on the Hub are auto-converted to Parquet by the + [Dataset Viewer](https://huggingface.co/docs/dataset-viewer/parquet). + This method returns the list of parquet files with their URLs, configs, + splits and sizes. + + Args: + repo_id (`str`): + The dataset repository ID (e.g. `"username/dataset-name"`). + config (`str`, *optional*): + Filter by a specific config/subset name. When provided, only + parquet files for that config are returned. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `list[DatasetParquetEntry]`: a list of [`DatasetParquetEntry`] objects + containing config, split, url and size for each parquet file. + + Example: + ```python + >>> from huggingface_hub import list_dataset_parquet_files + >>> list_dataset_parquet_files("lhoestq/demo1") + >>> entries[0] + DatasetParquetEntry(config='default', split='train', url='https://huggingface.co/...', size=5038) + ``` + """ + if self.endpoint != constants._HF_DEFAULT_ENDPOINT: + raise ValueError( + "The Dataset Viewer is only available on the Hugging Face Hub" + f" (endpoint='{constants._HF_DEFAULT_ENDPOINT}'). It is not supported on" + f" third-party endpoints. (endpoint={self.endpoint})" + ) + + url = f"{constants.DATASETS_SERVER_ENDPOINT}/parquet" + params: dict[str, str] = {"dataset": repo_id} + if config is not None: + params["config"] = config + response = get_session().get(url, params=params, headers=self._build_hf_headers(token=token)) + hf_raise_for_status(response) + payload = response.json() + + return [ + DatasetParquetEntry( + config=file_info["config"], + split=file_info["split"], + url=file_info["url"], + size=file_info["size"], + ) + for file_info in payload.get("parquet_files", []) + ] + + @validate_hf_hub_args + def list_spaces( + self, + *, + # Search-query parameter + filter: str | Iterable[str] | None = None, + author: str | None = None, + search: str | None = None, + datasets: str | Iterable[str] | None = None, + models: str | Iterable[str] | None = None, + linked: bool = False, + # Sorting and pagination parameters + sort: SpaceSort_T | None = None, + limit: int | None = None, + # Additional data to fetch + expand: list[ExpandSpaceProperty_T] | None = None, + full: bool | None = None, + token: bool | str | None = None, + ) -> Iterable[SpaceInfo]: + """ + List spaces hosted on the Huggingface Hub, given some filters. + + Args: + filter (`str` or `Iterable`, *optional*): + A string tag or list of tags that can be used to identify Spaces on the Hub. + author (`str`, *optional*): + A string which identify the author of the returned Spaces. + search (`str`, *optional*): + A string that will be contained in the returned Spaces. + datasets (`str` or `Iterable`, *optional*): + Whether to return Spaces that make use of a dataset. + The name of a specific dataset can be passed as a string. + models (`str` or `Iterable`, *optional*): + Whether to return Spaces that make use of a model. + The name of a specific model can be passed as a string. + linked (`bool`, *optional*): + Whether to return Spaces that make use of either a model or a dataset. + sort (`SpaceSort_T`, *optional*): + The key with which to sort the resulting spaces. Possible values are "created_at", "last_modified", + "likes" and "trending_score". + limit (`int`, *optional*): + The limit on the number of Spaces fetched. Leaving this option + to `None` fetches all Spaces. + expand (`list[ExpandSpaceProperty_T]`, *optional*): + List properties to return in the response. When used, only the properties in the list will be returned. + This parameter cannot be used if `full` is passed. + Possible values are `"author"`, `"cardData"`, `"datasets"`, `"disabled"`, `"lastModified"`, `"createdAt"`, `"likes"`, `"models"`, `"private"`, `"runtime"`, `"sdk"`, `"siblings"`, `"sha"`, `"subdomain"`, `"tags"`, `"trendingScore"`, `"usedStorage"`, and `"resourceGroup"`. + full (`bool`, *optional*): + Whether to fetch all Spaces data, including the `last_modified`, `siblings` + and `card_data` fields. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[SpaceInfo]`: an iterable of [`huggingface_hub.hf_api.SpaceInfo`] objects. + """ + if expand and full: + raise ValueError("`expand` cannot be used if `full` is passed.") + + path = f"{self.endpoint}/api/spaces" + headers = self._build_hf_headers(token=token) + params: dict[str, Any] = {} + if filter is not None: + params["filter"] = filter + if author is not None: + params["author"] = author + if search is not None: + params["search"] = search + if sort is not None: + params["sort"] = ( + "lastModified" + if sort == "last_modified" + else "trendingScore" + if sort == "trending_score" + else "createdAt" + if sort == "created_at" + else sort + ) + if limit is not None: + params["limit"] = limit + if linked: + params["linked"] = True + if datasets is not None: + params["datasets"] = datasets + if models is not None: + params["models"] = models + + # Request additional data + if expand: + params["expand"] = expand + if full: + params["full"] = True + + items = paginate(path, params=params, headers=headers) + if limit is not None: + items = islice(items, limit) # Do not iterate over all pages + for item in items: + if "siblings" not in item: + item["siblings"] = None + yield SpaceInfo(**item) + + @validate_hf_hub_args + def search_spaces( + self, + query: str, + *, + filter: str | Iterable[str] | None = None, + sdk: str | list[str] | None = None, + include_non_running: bool = False, + token: bool | str | None = None, + ) -> Iterable[SpaceSearchResult]: + """Search Spaces on the Hub using semantic search. + + This endpoint uses semantic search (embedding-based) for multi-word queries + and full-text search for single-word queries. + + Args: + query (`str`): + The search query string. + filter (`str` or `Iterable[str]`, *optional*): + A string tag or list of tags to filter by. + sdk (`str` or `list[str]`, *optional*): + Filter by SDK (e.g. `"gradio"`, `"docker"`, `"static"`). + include_non_running (`bool`, *optional*): + Whether to include non-running Spaces in results. Defaults to `False`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[SpaceSearchResult]`: an iterable of [`SpaceSearchResult`] objects. + + Example: + ```python + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> results = list(api.search_spaces("generate image")) + >>> results[0].id + 'mrfakename/Z-Image-Turbo' + >>> results[0].ai_category + 'Image Generation' + ``` + """ + path = f"{self.endpoint}/api/spaces/semantic-search" + headers = self._build_hf_headers(token=token) + params: dict[str, Any] = {"q": query} + if filter is not None: + params["filter"] = filter + if sdk is not None: + params["sdk"] = sdk + if include_non_running: + params["includeNonRunning"] = True + + r = get_session().get(path, headers=headers, params=params) + hf_raise_for_status(r) + for item in r.json(): + yield SpaceSearchResult(item) + + @validate_hf_hub_args + def unlike( + self, + repo_id: str, + *, + token: bool | str | None = None, + repo_type: str | None = None, + ) -> None: + """ + Unlike a given repo on the Hub (e.g. remove from favorite list). + + To prevent spam usage, it is not possible to `like` a repository from a script. + + See also [`list_liked_repos`]. + + Args: + repo_id (`str`): + The repository to unlike. Example: `"user/my-cool-model"`. + + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if unliking a dataset or space, `None` or + `"model"` if unliking a model. Default is `None`. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If repository is not found (error 404): wrong repo_id/repo_type, private + but not authenticated or repo does not exist. + + Example: + ```python + >>> from huggingface_hub import list_liked_repos, unlike + >>> "gpt2" in list_liked_repos().models # we assume you have already liked gpt2 + True + >>> unlike("gpt2") + >>> "gpt2" in list_liked_repos().models + False + ``` + """ + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + response = get_session().delete( + url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/like", headers=self._build_hf_headers(token=token) + ) + hf_raise_for_status(response) + + @validate_hf_hub_args + def list_liked_repos( + self, + user: str | None = None, + *, + token: bool | str | None = None, + ) -> UserLikes: + """ + List all public repos liked by a user on huggingface.co. + + This list is public so token is optional. If `user` is not passed, it defaults to + the logged in user. + + See also [`unlike`]. + + Args: + user (`str`, *optional*): + Name of the user for which you want to fetch the likes. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`UserLikes`]: object containing the user name and 3 lists of repo ids (1 for + models, 1 for datasets and 1 for Spaces). + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If `user` is not passed and no token found (either from argument or from machine). + + Example: + ```python + >>> from huggingface_hub import list_liked_repos + + >>> likes = list_liked_repos("julien-c") + + >>> likes.user + "julien-c" + + >>> likes.models + ["osanseviero/streamlit_1.15", "Xhaheen/ChatGPT_HF", ...] + ``` + """ + # User is either provided explicitly or retrieved from current token. + if user is None: + me = self.whoami(token=token) + if me["type"] == "user": + user = me["name"] + else: + raise ValueError( + "Cannot list liked repos. You must provide a 'user' as input or be logged in as a user." + ) + + path = f"{self.endpoint}/api/users/{user}/likes" + headers = self._build_hf_headers(token=token) + + likes = list(paginate(path, params={}, headers=headers)) + # Looping over a list of items similar to: + # { + # 'createdAt': '2021-09-09T21:53:27.000Z', + # 'repo': { + # 'name': 'PaddlePaddle/PaddleOCR', + # 'type': 'space' + # } + # } + # Let's loop 3 times over the received list. Less efficient but more straightforward to read. + return UserLikes( + user=user, + total=len(likes), + kernels=[like["repo"]["name"] for like in likes if like["repo"]["type"] == "kernel"], + models=[like["repo"]["name"] for like in likes if like["repo"]["type"] == "model"], + datasets=[like["repo"]["name"] for like in likes if like["repo"]["type"] == "dataset"], + spaces=[like["repo"]["name"] for like in likes if like["repo"]["type"] == "space"], + ) + + def list_user_repos( + self, + namespace: str | None = None, + *, + token: bool | str | None = None, + ) -> Iterable[RepoStorageInfo]: + """List all repositories (models, datasets, spaces, buckets) for a user or organization with storage info. + + Uses the `/api/settings/repositories` endpoint for the authenticated user or + `/api/organizations/{namespace}/settings/repositories` for an organization. + + Args: + namespace (`str`, *optional*): + Organization name. If not provided, lists repos for the authenticated user. + token (`bool` or `str`, *optional*): + A valid user access token. Defaults to the locally saved token. + + Returns: + `Iterable[RepoStorageInfo]`: An iterable of [`RepoStorageInfo`] objects. + + Example: + ```python + >>> from huggingface_hub import list_user_repos + + >>> repos = list(list_user_repos()) + >>> repos[0] + RepoStorageInfo(id='username/my-model', type='model', ...) + + >>> # List repos from an organization + >>> repos = list(list_user_repos(namespace="my-org")) + ``` + """ + if namespace is not None: + path = f"{self.endpoint}/api/organizations/{namespace}/settings/repositories" + else: + path = f"{self.endpoint}/api/settings/repositories" + headers = self._build_hf_headers(token=token) + for item in paginate(path, params={}, headers=headers): + yield RepoStorageInfo(**item) + + @validate_hf_hub_args + def list_repo_likers( + self, + repo_id: str, + *, + repo_type: str | None = None, + token: bool | str | None = None, + ) -> Iterable[User]: + """ + List all users who liked a given repo on the hugging Face Hub. + + See also [`list_liked_repos`]. + + Args: + repo_id (`str`): + The repository to retrieve . Example: `"user/my-cool-model"`. + + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + + Returns: + `Iterable[User]`: an iterable of [`huggingface_hub.hf_api.User`] objects. + """ + + # Construct the API endpoint + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/likers" + for liker in paginate(path, params={}, headers=self._build_hf_headers(token=token)): + yield User(username=liker["user"], fullname=liker["fullname"], avatar_url=liker["avatarUrl"]) + + @validate_hf_hub_args + def model_info( + self, + repo_id: str, + *, + revision: str | None = None, + timeout: float | None = None, + securityStatus: bool | None = None, + files_metadata: bool = False, + expand: list[ExpandModelProperty_T] | None = None, + token: bool | str | None = None, + ) -> ModelInfo: + """ + Get info on one specific model on huggingface.co + + Model can be private if you pass an acceptable token or are logged in. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + revision (`str`, *optional*): + The revision of the model repository from which to get the + information. + timeout (`float`, *optional*): + Whether to set a timeout for the request to the Hub. + securityStatus (`bool`, *optional*): + Whether to retrieve the security status from the model + repository as well. The security status will be returned in the `security_repo_status` field. + files_metadata (`bool`, *optional*): + Whether or not to retrieve metadata for files in the repository + (size, LFS metadata, etc). Defaults to `False`. + expand (`list[ExpandModelProperty_T]`, *optional*): + List properties to return in the response. When used, only the properties in the list will be returned. + This parameter cannot be used if `securityStatus` or `files_metadata` are passed. + Possible values are `"author"`, `"baseModels"`, `"cardData"`, `"childrenModelCount"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"evalResults"`, `"gated"`, `"gguf"`, `"inference"`, `"inferenceProviderMapping"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"`, `"trendingScore"`, `"widgetData"`, `"usedStorage"`, and `"resourceGroup"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`huggingface_hub.hf_api.ModelInfo`]: The model repository information. + + > [!TIP] + > Raises the following errors: + > + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + > - [`~utils.RevisionNotFoundError`] + > If the revision to download from cannot be found. + """ + if expand and (securityStatus or files_metadata): + raise ValueError("`expand` cannot be used if `securityStatus` or `files_metadata` are set.") + + headers = self._build_hf_headers(token=token) + path = ( + f"{self.endpoint}/api/models/{repo_id}" + if revision is None + else (f"{self.endpoint}/api/models/{repo_id}/revision/{quote(revision, safe='')}") + ) + params: dict = {} + if securityStatus: + params["securityStatus"] = True + if files_metadata: + params["blobs"] = True + if expand: + params["expand"] = expand + r = get_session().get(path, headers=headers, timeout=timeout, params=params) + hf_raise_for_status(r) + data = r.json() + return ModelInfo(**data) + + @validate_hf_hub_args + def dataset_info( + self, + repo_id: str, + *, + revision: str | None = None, + timeout: float | None = None, + files_metadata: bool = False, + expand: list[ExpandDatasetProperty_T] | None = None, + token: bool | str | None = None, + ) -> DatasetInfo: + """ + Get info on one specific dataset on huggingface.co. + + Dataset can be private if you pass an acceptable token. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + revision (`str`, *optional*): + The revision of the dataset repository from which to get the + information. + timeout (`float`, *optional*): + Whether to set a timeout for the request to the Hub. + files_metadata (`bool`, *optional*): + Whether or not to retrieve metadata for files in the repository + (size, LFS metadata, etc). Defaults to `False`. + expand (`list[ExpandDatasetProperty_T]`, *optional*): + List properties to return in the response. When used, only the properties in the list will be returned. + This parameter cannot be used if `files_metadata` is passed. + Possible values are `"author"`, `"cardData"`, `"citation"`, `"createdAt"`, `"disabled"`, `"description"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"lastModified"`, `"likes"`, `"mainSize"`, `"paperswithcode_id"`, `"private"`, `"siblings"`, `"sha"`, `"tags"`, `"trendingScore"`, `"usedStorage"`, and `"resourceGroup"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`hf_api.DatasetInfo`]: The dataset repository information. + + > [!TIP] + > Raises the following errors: + > + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + > - [`~utils.RevisionNotFoundError`] + > If the revision to download from cannot be found. + """ + if expand and files_metadata: + raise ValueError("`expand` cannot be used if `files_metadata` is set.") + + headers = self._build_hf_headers(token=token) + path = ( + f"{self.endpoint}/api/datasets/{repo_id}" + if revision is None + else (f"{self.endpoint}/api/datasets/{repo_id}/revision/{quote(revision, safe='')}") + ) + params: dict = {} + if files_metadata: + params["blobs"] = True + if expand: + params["expand"] = expand + + r = get_session().get(path, headers=headers, timeout=timeout, params=params) + hf_raise_for_status(r) + data = r.json() + return DatasetInfo(**data) + + @validate_hf_hub_args + def get_dataset_leaderboard( + self, + repo_id: str, + *, + base_model_only: bool | None = None, + token: bool | str | None = None, + timeout: float | None = None, + ) -> list[DatasetLeaderboardEntry]: + """Get the leaderboard for a dataset on the Hub. + + The leaderboard ranks models based on their evaluation scores on the given benchmark + dataset. Not all datasets have leaderboards — only benchmark datasets with evaluation + results submitted to them. This gives a dataset-centric view of scores; for a model-centric + view, use [`model_info`] with `expand=["evalResults"]`. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. For example: `"allenai/olmOCR-bench"`. + base_model_only (`bool` or `None`, *optional*): + By default, the leaderboard only includes models that have no declared `base_model` relation + (i.e. canonical/root repos), matching the Hub's default leaderboard view. Fine-tuned or derivative + repos that declare a parent model are excluded. Pass `base_model_only=False` to disable this filter and + include every submitted result, regardless of whether the model declares a base model relation. + token (`bool` or `str`, *optional*): + A valid user access token. Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + timeout (`float`, *optional*): + Whether to set a timeout for the request to the Hub. + + Returns: + `list[DatasetLeaderboardEntry]`: A list of [`DatasetLeaderboardEntry`] objects representing + the leaderboard entries, sorted by rank. + + > [!TIP] + > Raises the following errors: + > + > - [`~utils.RepositoryNotFoundError`] + > If the repository cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + > - [`~utils.HfHubHTTPError`] + > If the dataset does not have a leaderboard. + + Example: + ```python + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> leaderboard = api.get_dataset_leaderboard("allenai/olmOCR-bench") + >>> leaderboard[0].model_id + 'datalab-to/chandra-ocr-2' + >>> leaderboard[0].rank + 1 + + # Include fine-tuned / derivative models too + >>> full_leaderboard = api.get_dataset_leaderboard("allenai/olmOCR-bench", base_model_only=False) + ``` + """ + headers = self._build_hf_headers(token=token) + path = f"{self.endpoint}/api/datasets/{repo_id}/leaderboard" + params = {} + if base_model_only is not None: + params["base_model"] = base_model_only + r = get_session().get(path, headers=headers, params=params, timeout=timeout) + hf_raise_for_status(r) + data = r.json() + return [DatasetLeaderboardEntry(**entry) for entry in data] + + @validate_hf_hub_args + def space_info( + self, + repo_id: str, + *, + revision: str | None = None, + timeout: float | None = None, + files_metadata: bool = False, + expand: list[ExpandSpaceProperty_T] | None = None, + token: bool | str | None = None, + ) -> SpaceInfo: + """ + Get info on one specific Space on huggingface.co. + + Space can be private if you pass an acceptable token. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + revision (`str`, *optional*): + The revision of the space repository from which to get the + information. + timeout (`float`, *optional*): + Whether to set a timeout for the request to the Hub. + files_metadata (`bool`, *optional*): + Whether or not to retrieve metadata for files in the repository + (size, LFS metadata, etc). Defaults to `False`. + expand (`list[ExpandSpaceProperty_T]`, *optional*): + List properties to return in the response. When used, only the properties in the list will be returned. + This parameter cannot be used if `full` is passed. + Possible values are `"author"`, `"cardData"`, `"createdAt"`, `"datasets"`, `"disabled"`, `"lastModified"`, `"likes"`, `"models"`, `"private"`, `"runtime"`, `"sdk"`, `"siblings"`, `"sha"`, `"subdomain"`, `"tags"`, `"trendingScore"`, `"usedStorage"`, and `"resourceGroup"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`~hf_api.SpaceInfo`]: The space repository information. + + > [!TIP] + > Raises the following errors: + > + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + > - [`~utils.RevisionNotFoundError`] + > If the revision to download from cannot be found. + """ + if expand and files_metadata: + raise ValueError("`expand` cannot be used if `files_metadata` is set.") + + headers = self._build_hf_headers(token=token) + path = ( + f"{self.endpoint}/api/spaces/{repo_id}" + if revision is None + else (f"{self.endpoint}/api/spaces/{repo_id}/revision/{quote(revision, safe='')}") + ) + params: dict = {} + if files_metadata: + params["blobs"] = True + if expand: + params["expand"] = expand + + r = get_session().get(path, headers=headers, timeout=timeout, params=params) + hf_raise_for_status(r) + data = r.json() + return SpaceInfo(**data) + + @validate_hf_hub_args + def kernel_info( + self, + repo_id: str, + *, + revision: str | None = None, + timeout: float | None = None, + token: bool | str | None = None, + ) -> KernelInfo: + """ + Get info on one specific kernel on huggingface.co. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated by a `/`. + revision (`str`, *optional*): + The revision of the kernel repository from which to get the + information. + timeout (`float`, *optional*): + Whether to set a timeout for the request to the Hub. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`~hf_api.ModelInfo`]: The kernel repository information. + """ + headers = self._build_hf_headers(token=token) + path = ( + f"{self.endpoint}/api/kernels/{repo_id}" + if revision is None + else (f"{self.endpoint}/api/kernels/{repo_id}/revision/{quote(revision, safe='')}") + ) + r = get_session().get(path, headers=headers, timeout=timeout) + hf_raise_for_status(r) + data = r.json() + return KernelInfo(**data) + + @validate_hf_hub_args + def repo_info( + self, + repo_id: str, + *, + revision: str | None = None, + repo_type: str | None = None, + timeout: float | None = None, + files_metadata: bool = False, + expand: ExpandModelProperty_T | ExpandDatasetProperty_T | ExpandSpaceProperty_T | None = None, + token: bool | str | None = None, + ) -> ModelInfo | DatasetInfo | SpaceInfo | KernelInfo: + """ + Get the info object for a given repo of a given type. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + revision (`str`, *optional*): + The revision of the repository from which to get the + information. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if getting repository info from a dataset or a space, + `None` or `"model"` if getting repository info from a model. Default is `None`. + timeout (`float`, *optional*): + Whether to set a timeout for the request to the Hub. + expand (`ExpandModelProperty_T` or `ExpandDatasetProperty_T` or `ExpandSpaceProperty_T`, *optional*): + List properties to return in the response. When used, only the properties in the list will be returned. + This parameter cannot be used if `files_metadata` is passed. + For an exhaustive list of available properties, check out [`model_info`], [`dataset_info`] or [`space_info`]. + files_metadata (`bool`, *optional*): + Whether or not to retrieve metadata for files in the repository + (size, LFS metadata, etc). Defaults to `False`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Union[SpaceInfo, DatasetInfo, ModelInfo]`: The repository information, as a + [`huggingface_hub.hf_api.DatasetInfo`], [`huggingface_hub.hf_api.ModelInfo`] + or [`huggingface_hub.hf_api.SpaceInfo`] object. + + > [!TIP] + > Raises the following errors: + > + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + > - [`~utils.RevisionNotFoundError`] + > If the revision to download from cannot be found. + """ + match repo_type: + case None | "model": + method = self.model_info + case "dataset": + method = self.dataset_info # type: ignore + case "space": + method = self.space_info # type: ignore + case "kernel": + # No expand/files_metadata for kernels + return self.kernel_info(repo_id, revision=revision, token=token, timeout=timeout) + case _: + raise ValueError("Unsupported repo type.") + return method( + repo_id, + revision=revision, + token=token, + timeout=timeout, + expand=expand, # type: ignore + files_metadata=files_metadata, + ) + + @validate_hf_hub_args + def repo_exists( + self, + repo_id: str, + *, + repo_type: str | None = None, + token: str | bool | None = None, + ) -> bool: + """ + Checks if a repository exists on the Hugging Face Hub. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if getting repository info from a dataset or a space, + `None` or `"model"` if getting repository info from a model. Default is `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + True if the repository exists, False otherwise. + + Examples: + ```py + >>> from huggingface_hub import repo_exists + >>> repo_exists("google/gemma-7b") + True + >>> repo_exists("google/not-a-repo") + False + ``` + """ + try: + self.repo_info(repo_id=repo_id, repo_type=repo_type, token=token) + return True + except GatedRepoError: + return True # we don't have access but it exists + except RepositoryNotFoundError: + return False + + @validate_hf_hub_args + def revision_exists( + self, + repo_id: str, + revision: str, + *, + repo_type: str | None = None, + token: str | bool | None = None, + ) -> bool: + """ + Checks if a specific revision exists on a repo on the Hugging Face Hub. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + revision (`str`): + The revision of the repository to check. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if getting repository info from a dataset or a space, + `None` or `"model"` if getting repository info from a model. Default is `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + True if the repository and the revision exists, False otherwise. + + Examples: + ```py + >>> from huggingface_hub import revision_exists + >>> revision_exists("google/gemma-7b", "float16") + True + >>> revision_exists("google/gemma-7b", "not-a-revision") + False + ``` + """ + try: + self.repo_info(repo_id=repo_id, revision=revision, repo_type=repo_type, token=token) + return True + except RevisionNotFoundError: + return False + except RepositoryNotFoundError: + return False + + @validate_hf_hub_args + def file_exists( + self, + repo_id: str, + filename: str, + *, + repo_type: str | None = None, + revision: str | None = None, + token: str | bool | None = None, + ) -> bool: + """ + Checks if a file exists in a repository on the Hugging Face Hub. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + filename (`str`): + The name of the file to check, for example: + `"config.json"` + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if getting repository info from a dataset or a space, + `None` or `"model"` if getting repository info from a model. Default is `None`. + revision (`str`, *optional*): + The revision of the repository from which to get the information. Defaults to `"main"` branch. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + True if the file exists, False otherwise. + + Examples: + ```py + >>> from huggingface_hub import file_exists + >>> file_exists("bigcode/starcoder", "config.json") + True + >>> file_exists("bigcode/starcoder", "not-a-file") + False + >>> file_exists("bigcode/not-a-repo", "config.json") + False + ``` + """ + url = hf_hub_url( + repo_id=repo_id, repo_type=repo_type, revision=revision, filename=filename, endpoint=self.endpoint + ) + try: + if token is None: + token = self.token + get_hf_file_metadata(url, token=token) + return True + except GatedRepoError: # raise specifically on gated repo + raise + except (RepositoryNotFoundError, RemoteEntryNotFoundError, RevisionNotFoundError): + return False + + @validate_hf_hub_args + def list_repo_files( + self, + repo_id: str, + *, + revision: str | None = None, + repo_type: str | None = None, + token: str | bool | None = None, + ) -> list[str]: + """ + Get the list of files in a given repo. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated by a `/`. + revision (`str`, *optional*): + The revision of the repository from which to get the information. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or space, `None` or `"model"` if uploading to + a model. Default is `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `list[str]`: the list of files in a given repository. + """ + return [ + f.rfilename + for f in self.list_repo_tree( + repo_id=repo_id, recursive=True, revision=revision, repo_type=repo_type, token=token + ) + if isinstance(f, RepoFile) + ] + + @validate_hf_hub_args + def list_repo_tree( + self, + repo_id: str, + path_in_repo: str | None = None, + *, + recursive: bool = False, + expand: bool = False, + revision: str | None = None, + repo_type: str | None = None, + token: str | bool | None = None, + ) -> Iterable[RepoFile | RepoFolder]: + """ + List a repo tree's files and folders and get information about them. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated by a `/`. + path_in_repo (`str`, *optional*): + Relative path of the tree (folder) in the repo, for example: + `"checkpoints/1fec34a/results"`. Will default to the root tree (folder) of the repository. + recursive (`bool`, *optional*, defaults to `False`): + Whether to list tree's files and folders recursively. + expand (`bool`, *optional*, defaults to `False`): + Whether to fetch more information about the tree's files and folders (e.g. last commit and files' security scan results). This + operation is more expensive for the server so only 50 results are returned per page (instead of 1000). + As pagination is implemented in `huggingface_hub`, this is transparent for you except for the time it + takes to get the results. + revision (`str`, *optional*): + The revision of the repository from which to get the tree. Defaults to `"main"` branch. + repo_type (`str`, *optional*): + The type of the repository from which to get the tree (`"model"`, `"dataset"`, `"space"` or `"kernel"`). + Defaults to `"model"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[Union[RepoFile, RepoFolder]]`: + The information about the tree's files and folders, as an iterable of [`RepoFile`] and [`RepoFolder`] objects. The order of the files and folders is + not guaranteed. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo + does not exist. + [`~utils.RevisionNotFoundError`]: + If revision is not found (error 404) on the repo. + [`~utils.RemoteEntryNotFoundError`]: + If the tree (folder) does not exist (error 404) on the repo. + + Examples: + + Get information about a repo's tree. + ```py + >>> from huggingface_hub import list_repo_tree + >>> repo_tree = list_repo_tree("lysandre/arxiv-nlp") + >>> repo_tree + + >>> list(repo_tree) + [ + RepoFile(path='.gitattributes', size=391, blob_id='ae8c63daedbd4206d7d40126955d4e6ab1c80f8f', lfs=None, last_commit=None, security=None), + RepoFile(path='README.md', size=391, blob_id='43bd404b159de6fba7c2f4d3264347668d43af25', lfs=None, last_commit=None, security=None), + RepoFile(path='config.json', size=554, blob_id='2f9618c3a19b9a61add74f70bfb121335aeef666', lfs=None, last_commit=None, security=None), + RepoFile( + path='flax_model.msgpack', size=497764107, blob_id='8095a62ccb4d806da7666fcda07467e2d150218e', + lfs={'size': 497764107, 'sha256': 'd88b0d6a6ff9c3f8151f9d3228f57092aaea997f09af009eefd7373a77b5abb9', 'pointer_size': 134}, last_commit=None, security=None + ), + RepoFile(path='merges.txt', size=456318, blob_id='226b0752cac7789c48f0cb3ec53eda48b7be36cc', lfs=None, last_commit=None, security=None), + RepoFile( + path='pytorch_model.bin', size=548123560, blob_id='64eaa9c526867e404b68f2c5d66fd78e27026523', + lfs={'size': 548123560, 'sha256': '9be78edb5b928eba33aa88f431551348f7466ba9f5ef3daf1d552398722a5436', 'pointer_size': 134}, last_commit=None, security=None + ), + RepoFile(path='vocab.json', size=898669, blob_id='b00361fece0387ca34b4b8b8539ed830d644dbeb', lfs=None, last_commit=None, security=None)] + ] + ``` + + Get even more information about a repo's tree (last commit and files' security scan results) + ```py + >>> from huggingface_hub import list_repo_tree + >>> repo_tree = list_repo_tree("prompthero/openjourney-v4", expand=True) + >>> list(repo_tree) + [ + RepoFolder( + path='feature_extractor', + tree_id='aa536c4ea18073388b5b0bc791057a7296a00398', + last_commit={ + 'oid': '47b62b20b20e06b9de610e840282b7e6c3d51190', + 'title': 'Upload diffusers weights (#48)', + 'date': datetime.datetime(2023, 3, 21, 9, 5, 27, tzinfo=datetime.timezone.utc) + } + ), + RepoFolder( + path='safety_checker', + tree_id='65aef9d787e5557373fdf714d6c34d4fcdd70440', + last_commit={ + 'oid': '47b62b20b20e06b9de610e840282b7e6c3d51190', + 'title': 'Upload diffusers weights (#48)', + 'date': datetime.datetime(2023, 3, 21, 9, 5, 27, tzinfo=datetime.timezone.utc) + } + ), + RepoFile( + path='model_index.json', + size=582, + blob_id='d3d7c1e8c3e78eeb1640b8e2041ee256e24c9ee1', + lfs=None, + last_commit={ + 'oid': 'b195ed2d503f3eb29637050a886d77bd81d35f0e', + 'title': 'Fix deprecation warning by changing `CLIPFeatureExtractor` to `CLIPImageProcessor`. (#54)', + 'date': datetime.datetime(2023, 5, 15, 21, 41, 59, tzinfo=datetime.timezone.utc) + }, + security={ + 'safe': True, + 'av_scan': {'virusFound': False, 'virusNames': None}, + 'pickle_import_scan': None + } + ) + ... + ] + ``` + """ + repo_type = repo_type or constants.REPO_TYPE_MODEL + revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION + headers = self._build_hf_headers(token=token) + + encoded_path_in_repo = "/" + quote(path_in_repo, safe="") if path_in_repo else "" + tree_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tree/{revision}{encoded_path_in_repo}" + for path_info in paginate(path=tree_url, headers=headers, params={"recursive": recursive, "expand": expand}): + yield (RepoFile(**path_info) if path_info["type"] == "file" else RepoFolder(**path_info)) + + @validate_hf_hub_args + def verify_repo_checksums( + self, + repo_id: str, + *, + repo_type: str | None = None, + revision: str | None = None, + local_dir: str | Path | None = None, + cache_dir: str | Path | None = None, + token: str | bool | None = None, + ) -> FolderVerification: + """ + Verify local files for a repo against Hub checksums. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated by a `/`. + repo_type (`str`, *optional*): + The type of the repository from which to get the tree (`"model"`, `"dataset"` or `"space"`. + Defaults to `"model"`. + revision (`str`, *optional*): + The revision of the repository from which to get the tree. Defaults to `"main"` branch. + local_dir (`str` or `Path`, *optional*): + The local directory to verify. + cache_dir (`str` or `Path`, *optional*): + The cache directory to verify. + token (Union[bool, str, None], optional): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`FolderVerification`]: a structured result containing the verification details. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo + does not exist. + [`~utils.RevisionNotFoundError`]: + If revision is not found (error 404) on the repo. + + """ + + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + + if local_dir is not None and cache_dir is not None: + raise ValueError("Pass either `local_dir` or `cache_dir`, not both.") + + root, remote_revision = resolve_local_root( + repo_id=repo_id, + repo_type=repo_type, + revision=revision, + cache_dir=Path(cache_dir) if cache_dir is not None else None, + local_dir=Path(local_dir) if local_dir is not None else None, + ) + local_by_path = collect_local_files(root) + + # get remote entries (only files, not folders) + remote_by_path: dict[str, RepoFile] = {} + for entry in self.list_repo_tree( + repo_id=repo_id, recursive=True, revision=remote_revision, repo_type=repo_type, token=token + ): + if isinstance(entry, RepoFile): + remote_by_path[entry.path] = entry + + return verify_maps( + remote_by_path=remote_by_path, + local_by_path=local_by_path, + revision=remote_revision, + verified_path=root, + ) + + @validate_hf_hub_args + def list_repo_refs( + self, + repo_id: str, + *, + repo_type: str | None = None, + include_pull_requests: bool = False, + token: str | bool | None = None, + ) -> GitRefs: + """ + Get the list of refs of a given repo (both tags and branches). + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + repo_type (`str`, *optional*): + Set to `"dataset"`, `"space"` or `"kernel"` if listing refs from a dataset, a Space or a Kernel, + `None` or `"model"` if listing from a model. Default is `None`. + include_pull_requests (`bool`, *optional*): + Whether to include refs from pull requests in the list. Defaults to `False`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Example: + ```py + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> api.list_repo_refs("gpt2") + GitRefs(branches=[GitRefInfo(name='main', ref='refs/heads/main', target_commit='e7da7f221d5bf496a48136c0cd264e630fe9fcc8')], converts=[], tags=[]) + + >>> api.list_repo_refs("bigcode/the-stack", repo_type='dataset') + GitRefs( + branches=[ + GitRefInfo(name='main', ref='refs/heads/main', target_commit='18edc1591d9ce72aa82f56c4431b3c969b210ae3'), + GitRefInfo(name='v1.1.a1', ref='refs/heads/v1.1.a1', target_commit='f9826b862d1567f3822d3d25649b0d6d22ace714') + ], + converts=[], + tags=[ + GitRefInfo(name='v1.0', ref='refs/tags/v1.0', target_commit='c37a8cd1e382064d8aced5e05543c5f7753834da') + ] + ) + ``` + + Returns: + [`GitRefs`]: object containing all information about branches and tags for a + repo on the Hub. + """ + repo_type = repo_type or constants.REPO_TYPE_MODEL + response = get_session().get( + f"{self.endpoint}/api/{repo_type}s/{repo_id}/refs", + headers=self._build_hf_headers(token=token), + params={"include_prs": 1} if include_pull_requests else {}, + ) + hf_raise_for_status(response) + data = response.json() + + def _format_as_git_ref_info(item: dict) -> GitRefInfo: + return GitRefInfo(name=item["name"], ref=item["ref"], target_commit=item["targetCommit"]) + + return GitRefs( + branches=[_format_as_git_ref_info(item) for item in data["branches"]], + converts=[_format_as_git_ref_info(item) for item in data["converts"]], + tags=[_format_as_git_ref_info(item) for item in data["tags"]], + pull_requests=[_format_as_git_ref_info(item) for item in data["pullRequests"]] + if include_pull_requests + else None, + ) + + @validate_hf_hub_args + def list_repo_commits( + self, + repo_id: str, + *, + repo_type: str | None = None, + token: bool | str | None = None, + revision: str | None = None, + formatted: bool = False, + ) -> list[GitCommitInfo]: + """ + Get the list of commits of a given revision for a repo on the Hub. + + Commits are sorted by date (last commit first). + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated by a `/`. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if listing commits from a dataset or a Space, `None` or `"model"` if + listing from a model. Default is `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the `"main"` branch. + formatted (`bool`): + Whether to return the HTML-formatted title and description of the commits. Defaults to False. + + Example: + ```py + >>> from huggingface_hub import HfApi + >>> api = HfApi() + + # Commits are sorted by date (last commit first) + >>> initial_commit = api.list_repo_commits("gpt2")[-1] + + # Initial commit is always a system commit containing the `.gitattributes` file. + >>> initial_commit + GitCommitInfo( + commit_id='9b865efde13a30c13e0a33e536cf3e4a5a9d71d8', + authors=['system'], + created_at=datetime.datetime(2019, 2, 18, 10, 36, 15, tzinfo=datetime.timezone.utc), + title='initial commit', + message='', + formatted_title=None, + formatted_message=None + ) + + # Create an empty branch by deriving from initial commit + >>> api.create_branch("gpt2", "new_empty_branch", revision=initial_commit.commit_id) + ``` + + Returns: + list[[`GitCommitInfo`]]: list of objects containing information about the commits for a repo on the Hub. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo + does not exist. + [`~utils.RevisionNotFoundError`]: + If revision is not found (error 404) on the repo. + """ + repo_type = repo_type or constants.REPO_TYPE_MODEL + revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION + + # Paginate over results and return the list of commits. + return [ + GitCommitInfo( + commit_id=item["id"], + authors=[author["user"] for author in item["authors"]], + created_at=parse_datetime(item["date"]), + title=item["title"], + message=item["message"], + formatted_title=item.get("formatted", {}).get("title"), + formatted_message=item.get("formatted", {}).get("message"), + ) + for item in paginate( + f"{self.endpoint}/api/{repo_type}s/{repo_id}/commits/{revision}", + headers=self._build_hf_headers(token=token), + params={"expand[]": "formatted"} if formatted else {}, + ) + ] + + @validate_hf_hub_args + def get_paths_info( + self, + repo_id: str, + paths: list[str] | str, + *, + expand: bool = False, + revision: str | None = None, + repo_type: str | None = None, + token: str | bool | None = None, + ) -> list[RepoFile | RepoFolder]: + """ + Get information about a repo's paths. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated by a `/`. + paths (`Union[list[str], str]`, *optional*): + The paths to get information about. If a path do not exist, it is ignored without raising + an exception. + expand (`bool`, *optional*, defaults to `False`): + Whether to fetch more information about the paths (e.g. last commit and files' security scan results). This + operation is more expensive for the server so only 50 results are returned per page (instead of 1000). + As pagination is implemented in `huggingface_hub`, this is transparent for you except for the time it + takes to get the results. + revision (`str`, *optional*): + The revision of the repository from which to get the information. Defaults to `"main"` branch. + repo_type (`str`, *optional*): + The type of the repository from which to get the information (`"model"`, `"dataset"` or `"space"`. + Defaults to `"model"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `list[Union[RepoFile, RepoFolder]]`: + The information about the paths, as a list of [`RepoFile`] and [`RepoFolder`] objects. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo + does not exist. + [`~utils.RevisionNotFoundError`]: + If revision is not found (error 404) on the repo. + + Example: + ```py + >>> from huggingface_hub import get_paths_info + >>> paths_info = get_paths_info("allenai/c4", ["README.md", "en"], repo_type="dataset") + >>> paths_info + [ + RepoFile(path='README.md', size=2379, blob_id='f84cb4c97182890fc1dbdeaf1a6a468fd27b4fff', lfs=None, last_commit=None, security=None), + RepoFolder(path='en', tree_id='dc943c4c40f53d02b31ced1defa7e5f438d5862e', last_commit=None) + ] + ``` + """ + repo_type = repo_type or constants.REPO_TYPE_MODEL + revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION + headers = self._build_hf_headers(token=token) + + response = get_session().post( + f"{self.endpoint}/api/{repo_type}s/{repo_id}/paths-info/{revision}", + data={ + "paths": paths if isinstance(paths, list) else [paths], + "expand": expand, + }, + headers=headers, + ) + hf_raise_for_status(response) + paths_info = response.json() + return [ + RepoFile(**path_info) if path_info["type"] == "file" else RepoFolder(**path_info) + for path_info in paths_info + ] + + @validate_hf_hub_args + def super_squash_history( + self, + repo_id: str, + *, + branch: str | None = None, + commit_message: str | None = None, + repo_type: str | None = None, + token: str | bool | None = None, + ) -> None: + """Squash commit history on a branch for a repo on the Hub. + + Squashing the repo history is useful when you know you'll make hundreds of commits and you don't want to + clutter the history. Squashing commits can only be performed from the head of a branch. + + > [!WARNING] + > Once squashed, the commit history cannot be retrieved. This is a non-revertible operation. + + > [!WARNING] + > Once the history of a branch has been squashed, it is not possible to merge it back into another branch since + > their history will have diverged. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated by a `/`. + branch (`str`, *optional*): + The branch to squash. Defaults to the head of the `"main"` branch. + commit_message (`str`, *optional*): + The commit message to use for the squashed commit. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if listing commits from a dataset or a Space, `None` or `"model"` if + listing from a model. Default is `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If repository is not found (error 404): wrong repo_id/repo_type, private but not authenticated or repo + does not exist. + [`~utils.RevisionNotFoundError`]: + If the branch to squash cannot be found. + [`~utils.BadRequestError`]: + If invalid reference for a branch. You cannot squash history on tags. + + Example: + ```py + >>> from huggingface_hub import HfApi + >>> api = HfApi() + + # Create repo + >>> repo_id = api.create_repo("test-squash").repo_id + + # Make a lot of commits. + >>> api.upload_file(repo_id=repo_id, path_in_repo="file.txt", path_or_fileobj=b"content") + >>> api.upload_file(repo_id=repo_id, path_in_repo="lfs.bin", path_or_fileobj=b"content") + >>> api.upload_file(repo_id=repo_id, path_in_repo="file.txt", path_or_fileobj=b"another_content") + + # Squash history + >>> api.super_squash_history(repo_id=repo_id) + ``` + """ + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + if repo_type not in constants.REPO_TYPES: + raise ValueError("Invalid repo type") + if branch is None: + branch = constants.DEFAULT_REVISION + + # Prepare request + url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/super-squash/{quote(branch, safe='')}" + headers = self._build_hf_headers(token=token) + commit_message = commit_message or f"Super-squash branch '{branch}' using huggingface_hub" + + # Super-squash + response = get_session().post(url=url, headers=headers, json={"message": commit_message}) + hf_raise_for_status(response) + + @validate_hf_hub_args + def list_lfs_files( + self, + repo_id: str, + *, + repo_type: str | None = None, + token: bool | str | None = None, + ) -> Iterable[LFSFileInfo]: + """ + List all LFS files in a repo on the Hub. + + This is primarily useful to count how much storage a repo is using and to eventually clean up large files + with [`permanently_delete_lfs_files`]. Note that this would be a permanent action that will affect all commits + referencing this deleted files and that cannot be undone. + + Args: + repo_id (`str`): + The repository for which you are listing LFS files. + repo_type (`str`, *optional*): + Type of repository. Set to `"dataset"` or `"space"` if listing from a dataset or space, `None` or + `"model"` if listing from a model. Default is `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[LFSFileInfo]`: An iterator of [`LFSFileInfo`] objects. + + Example: + ```py + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> lfs_files = api.list_lfs_files("username/my-cool-repo") + + # Filter files files to delete based on a combination of `filename`, `pushed_at`, `ref` or `size`. + # e.g. select only LFS files in the "checkpoints" folder + >>> lfs_files_to_delete = (lfs_file for lfs_file in lfs_files if lfs_file.filename.startswith("checkpoints/")) + + # Permanently delete LFS files + >>> api.permanently_delete_lfs_files("username/my-cool-repo", lfs_files_to_delete) + ``` + """ + # Prepare request + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/lfs-files" + headers = self._build_hf_headers(token=token) + + # Paginate over LFS items + for item in paginate(url, params={}, headers=headers): + yield LFSFileInfo(**item) + + @validate_hf_hub_args + def permanently_delete_lfs_files( + self, + repo_id: str, + lfs_files: Iterable[LFSFileInfo], + *, + rewrite_history: bool = True, + repo_type: str | None = None, + token: bool | str | None = None, + ) -> None: + """ + Permanently delete LFS files from a repo on the Hub. + + > [!WARNING] + > This is a permanent action that will affect all commits referencing the deleted files and might corrupt your + > repository. This is a non-revertible operation. Use it only if you know what you are doing. + + Args: + repo_id (`str`): + The repository for which you are listing LFS files. + lfs_files (`Iterable[LFSFileInfo]`): + An iterable of [`LFSFileInfo`] items to permanently delete from the repo. Use [`list_lfs_files`] to list + all LFS files from a repo. + rewrite_history (`bool`, *optional*, default to `True`): + Whether to rewrite repository history to remove file pointers referencing the deleted LFS files (recommended). + repo_type (`str`, *optional*): + Type of repository. Set to `"dataset"` or `"space"` if listing from a dataset or space, `None` or + `"model"` if listing from a model. Default is `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Example: + ```py + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> lfs_files = api.list_lfs_files("username/my-cool-repo") + + # Filter files files to delete based on a combination of `filename`, `pushed_at`, `ref` or `size`. + # e.g. select only LFS files in the "checkpoints" folder + >>> lfs_files_to_delete = (lfs_file for lfs_file in lfs_files if lfs_file.filename.startswith("checkpoints/")) + + # Permanently delete LFS files + >>> api.permanently_delete_lfs_files("username/my-cool-repo", lfs_files_to_delete) + ``` + """ + # Prepare request + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/lfs-files/batch" + headers = self._build_hf_headers(token=token) + + # Delete LFS items by batches of 1000 + for batch in chunk_iterable(lfs_files, 1000): + shas = [item.file_oid for item in batch] + if len(shas) == 0: + return + payload = { + "deletions": { + "sha": shas, + "rewriteHistory": rewrite_history, + } + } + response = get_session().post(url, headers=headers, json=payload) + hf_raise_for_status(response) + + def list_space_templates(self, *, token: str | bool | None = None) -> list[SpaceTemplate]: + """List the official Space templates available on the Hub. + + The `repo_id` of a returned template (or its short `name`) can be passed as `space_template` + to [`HfApi.create_repo`] to seed a new Space from that template. + + Args: + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `list[SpaceTemplate]`: The list of available Space templates. + + Example: + ```py + >>> from huggingface_hub import list_space_templates + >>> templates = list_space_templates() + >>> templates[0] + SpaceTemplate(name='Streamlit', repo_id='streamlit/streamlit-template-space', sdk='docker', preferred_private=False) + ``` + """ + r = get_session().get(f"{self.endpoint}/api/spaces/templates", headers=self._build_hf_headers(token=token)) + hf_raise_for_status(r) + return [SpaceTemplate(item) for item in r.json()["templates"]] + + @_deprecate_arguments( + version="2.0", + deprecated_args={"space_storage"}, + custom_message="Use `space_volumes` to mount volumes on a Space.", + ) + @validate_hf_hub_args + def create_repo( + self, + repo_id: str, + *, + token: str | bool | None = None, + private: bool | None = None, + visibility: RepoVisibility_T | None = None, + repo_type: str | None = None, + exist_ok: bool = False, + resource_group_id: str | None = None, + region: REPO_REGIONS | None = None, + space_sdk: str | None = None, + space_hardware: SpaceHardware | None = None, + space_storage: SpaceStorage | None = None, + space_sleep_time: int | None = None, + space_secrets: list[dict[str, str]] | None = None, + space_variables: list[dict[str, str]] | None = None, + space_volumes: list[Volume] | None = None, + space_template: str | None = None, + ) -> RepoUrl: + """Create an empty repo on the HuggingFace Hub. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + private (`bool`, *optional*): + Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists. Cannot be passed together with `visibility`. + visibility (`Literal["public", "private", "protected"]`, *optional*): + Visibility of the repo. Can be `"public"` or `"private"`, or `"protected"` for Spaces. If `None` + (default), the repo will be public unless the organization's default is private. This value is ignored + if the repo already exists. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + exist_ok (`bool`, *optional*, defaults to `False`): + If `True`, do not raise an error if repo already exists. + resource_group_id (`str`, *optional*): + Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations and + allow to define which members of the organization can access the resource. The ID of a resource group + can be found in the URL of the resource's page on the Hub (e.g. `"66670e5163145ca562cb1988"`). + To learn more about resource groups, see https://huggingface.co/docs/hub/en/security-resource-groups. + region (`Literal["us", "eu"]`, *optional*): + Cloud region in which to create the repo. Can be one of `"us"` or `"eu"`. If not specified, the repo will be + created in the default region. Requires Team plan or above. + space_sdk (`str`, *optional*): + Choice of SDK to use if repo_type is "space". Can be "streamlit", "gradio", "docker", or "static". + space_hardware (`SpaceHardware` or `str`, *optional*): + Choice of Hardware if repo_type is "space". See [`SpaceHardware`] for a complete list. + space_storage (`SpaceStorage` or `str`, *optional*): + Choice of persistent storage tier. Example: `"small"`. See [`SpaceStorage`] for a complete list. + space_sleep_time (`int`, *optional*): + Number of seconds of inactivity to wait before a Space is put to sleep. Set to `-1` if you don't want + your Space to sleep (default behavior for upgraded hardware). For free hardware, you can't configure + the sleep time (value is fixed to 48 hours of inactivity). + See https://huggingface.co/docs/hub/spaces-gpus#sleep-time for more details. + space_secrets (`list[dict[str, str]]`, *optional*): + A list of secret keys to set in your Space. Each item is in the form `{"key": ..., "value": ..., "description": ...}` where description is optional. + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets. + space_variables (`list[dict[str, str]]`, *optional*): + A list of public environment variables to set in your Space. Each item is in the form `{"key": ..., "value": ..., "description": ...}` where description is optional. + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets-and-environment-variables. + space_volumes (`list[Volume]`, *optional*): + A list of [`Volume`] objects to mount in the Space at creation time. Each volume has a `type` + (`"bucket"`, `"model"`, `"dataset"`, or `"space"`), a `source` (repo or bucket ID), a `mount_path` + (path inside the container), and optional `revision`, `read_only`, and `path` fields. + Only applicable if repo_type is "space". + space_template (`str`, *optional*): + Seed the new Space from an official template. Can be either the template repo id + (e.g. `"SpacesExamples/jupyterlab"`) or its short name (e.g. `"JupyterLab"`). Use + [`HfApi.list_space_templates`] to list available templates. Only applicable if repo_type is + "space". If the template is recommended to be private and visibility is not explicitly set, the Space + is created as private. + + Returns: + [`RepoUrl`]: URL to the newly created repo. Value is a subclass of `str` containing + attributes like `endpoint`, `repo_type` and `repo_id`. + """ + organization, name = repo_id.split("/") if "/" in repo_id else (None, repo_id) + + path = f"{self.endpoint}/api/repos/create" + + if repo_type not in constants.REPO_TYPES_WITH_KERNEL: + raise ValueError("Invalid repo type") + + resolved_visibility = _resolve_repo_visibility(private=private, visibility=visibility, repo_type=repo_type) + + resolved_space_template: str | None = None + if space_template is not None: + if repo_type != constants.REPO_TYPE_SPACE: + raise ValueError(f"space_template can only be used with repo_type 'space'. Got repo_type={repo_type}.") + + # space_template passed => resolve it + all_templates = self.list_space_templates(token=token) + for candidate in all_templates: + if candidate.repo_id == space_template: + template = candidate + break + if candidate.name.lower() == space_template.lower(): + template = candidate + break + else: + raise ValueError( + f"Unknown Space template '{space_template}'. Expected one of {', '.join(sorted(t.name for t in all_templates))}. Use `HfApi.list_space_templates()` to list templates." + ) + + resolved_space_template = template.repo_id + # If the chosen template is recommended to be private and the user did not explicitly set a + # visibility, default to private (mirrors the recommendation shown in the web UI). + if template.preferred_private and private is None and visibility is None: + resolved_visibility = "private" + # space_sdk can be omitted by the user, we set it following server info + if space_sdk is not None and space_sdk != template.sdk: + raise ValueError( + f"space_sdk must match the SDK of the chosen space_template. Got {space_sdk}, expected {template.sdk}." + ) + space_sdk = template.sdk + + payload: dict[str, Any] = {"name": name, "organization": organization} + if resolved_visibility is not None: + payload["visibility"] = resolved_visibility + if repo_type is not None: + payload["type"] = repo_type + if repo_type == "space": + if space_sdk is None: + raise ValueError( + "No space_sdk provided. `create_repo` expects space_sdk to be one" + f" of {constants.SPACES_SDK_TYPES} when repo_type is 'space'`" + ) + if space_sdk not in constants.SPACES_SDK_TYPES: + raise ValueError(f"Invalid space_sdk. Please choose one of {constants.SPACES_SDK_TYPES}.") + payload["sdk"] = space_sdk + if resolved_space_template is not None: + payload["template"] = resolved_space_template + + if space_sdk is not None and repo_type != "space": + warnings.warn("Ignoring provided space_sdk because repo_type is not 'space'.") + + space_args: list[tuple[str, str, Any]] = [ + # input arg, payload key, value + ("space_hardware", "hardware", space_hardware), + ("space_storage", "storageTier", space_storage), + ("space_sleep_time", "sleepTimeSeconds", space_sleep_time), + ("space_secrets", "secrets", space_secrets), + ("space_variables", "variables", space_variables), + ("space_volumes", "volumes", [v.to_dict() for v in space_volumes] if space_volumes else None), + ] + + if repo_type == constants.REPO_TYPE_SPACE: + for _, key, value in space_args: + if value is not None: + payload[key] = value + if space_sleep_time is not None and space_hardware == SpaceHardware.CPU_BASIC: + warnings.warn( + "If your Space runs on the default 'cpu-basic' hardware, it will go to sleep if inactive for more" + " than 48 hours. This value is not configurable. If you don't want your Space to deactivate or if" + " you want to set a custom sleep time, you need to upgrade to a paid Hardware.", + UserWarning, + ) + else: + if provided_space_args := [arg for arg, _, value in space_args if value is not None]: + warnings.warn(f"Ignoring provided {', '.join(provided_space_args)} because repo_type is not 'space'.") + + if resource_group_id is not None: + payload["resourceGroupId"] = resource_group_id + if region is not None: + payload["region"] = region + + headers = self._build_hf_headers(token=token) + while True: + r = get_session().post(path, headers=headers, json=payload) + if r.status_code == 409 and "Cannot create repo: another conflicting operation is in progress" in r.text: + # Since https://github.com/huggingface/moon-landing/pull/7272 (private repo), it is not possible to + # concurrently create repos on the Hub for a same user. This is rarely an issue, except when running + # tests. To avoid any inconvenience, we retry to create the repo for this specific error. + # NOTE: This could have been fixed directly in the tests, but adding it here should fix CIs for all + # dependent libraries. + # NOTE: If a fix is implemented server-side, we should be able to remove this retry mechanism. + logger.debug("Create repo failed due to a concurrency issue. Retrying...") + continue + break + + try: + hf_raise_for_status(r) + except HfHubHTTPError as err: + if exist_ok and err.response.status_code == 409: + # Repo already exists and `exist_ok=True` + pass + elif exist_ok and err.response.status_code in (401, 402, 403): + # 401 -> if JWT token without create repo scope + # 402 -> if payment required (e.g. if Gradio/Docker Space and free user) + # 403 -> if no write permission on the namespace + # In all 3 cases, repo might already exist + try: + self.repo_info(repo_id=repo_id, repo_type=repo_type, token=token) + if repo_type is None or repo_type == constants.REPO_TYPE_MODEL: + return RepoUrl(f"{self.endpoint}/{repo_id}") + return RepoUrl(f"{self.endpoint}/{constants.REPO_TYPES_URL_PREFIXES[repo_type]}{repo_id}") + except HfHubHTTPError: + raise err + else: + raise + + d = r.json() + return RepoUrl(d["url"], endpoint=self.endpoint) + + @validate_hf_hub_args + def delete_repo( + self, + repo_id: str, + *, + token: str | bool | None = None, + repo_type: str | None = None, + missing_ok: bool = False, + ) -> None: + """ + Delete a repo from the HuggingFace Hub. CAUTION: this is irreversible. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. + missing_ok (`bool`, *optional*, defaults to `False`): + If `True`, do not raise an error if repo does not exist. + + Raises: + [`~utils.RepositoryNotFoundError`] + If the repository to delete from cannot be found and `missing_ok` is set to False (default). + """ + organization, name = repo_id.split("/") if "/" in repo_id else (None, repo_id) + + path = f"{self.endpoint}/api/repos/delete" + + if repo_type not in constants.REPO_TYPES_WITH_KERNEL: + raise ValueError("Invalid repo type") + + json = {"name": name, "organization": organization} + if repo_type is not None: + json["type"] = repo_type + + headers = self._build_hf_headers(token=token) + r = get_session().request("DELETE", path, headers=headers, json=json) + try: + hf_raise_for_status(r) + except RepositoryNotFoundError: + if not missing_ok: + raise + + @validate_hf_hub_args + def update_repo_settings( + self, + repo_id: str, + *, + gated: Literal["auto", "manual", False] | None = None, + private: bool | None = None, + visibility: RepoVisibility_T | None = None, + token: str | bool | None = None, + repo_type: str | None = None, + ) -> None: + """ + Update the settings of a repository, including gated access and visibility. + + To give more control over how repos are used, the Hub allows repo authors to enable + access requests for their repos, and also to change the visibility of the repo. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated by a /. + gated (`Literal["auto", "manual", False]`, *optional*): + The gated status for the repository. If set to `None` (default), the `gated` setting of the repository won't be updated. + * "auto": The repository is gated, and access requests are automatically approved or denied based on predefined criteria. + * "manual": The repository is gated, and access requests require manual approval. + * False : The repository is not gated, and anyone can access it. + private (`bool`, *optional*): + Whether the repository should be private. Cannot be passed together with `visibility`. + visibility (`Literal["public", "private", "protected"]`, *optional*): + Visibility of the repository. Can be `"public"` or `"private"`, or `"protected"` for Spaces. + token (`Union[str, bool, None]`, *optional*): + A valid user access token (string). Defaults to the locally saved token, + which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass False. + repo_type (`str`, *optional*): + The type of the repository to update settings from (`"model"`, `"dataset"` or `"space"`). + Defaults to `"model"`. + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If gated is not one of "auto", "manual", or False. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If repo_type is not one of the values in constants.REPO_TYPES. + [`~utils.HfHubHTTPError`]: + If the request to the Hugging Face Hub API fails. + [`~utils.RepositoryNotFoundError`] + If the repository to download from cannot be found. This may be because it doesn't exist, + or because it is set to `private` and you do not have access. + """ + + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL # default repo type + + resolved_visibility = _resolve_repo_visibility(private=private, visibility=visibility, repo_type=repo_type) + + # Prepare the JSON payload for the PUT request + payload: dict = {} + + if gated is not None: + if gated not in ["auto", "manual", False]: + raise ValueError(f"Invalid gated status, must be one of 'auto', 'manual', or False. Got '{gated}'.") + payload["gated"] = gated + + if resolved_visibility is not None: + payload["visibility"] = resolved_visibility + + if len(payload) == 0: + raise ValueError("At least one setting must be updated.") + + # Build headers + headers = self._build_hf_headers(token=token) + + r = get_session().put( + url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/settings", + headers=headers, + json=payload, + ) + hf_raise_for_status(r) + + def move_repo( + self, + from_id: str, + to_id: str, + *, + repo_type: str | None = None, + token: str | bool | None = None, + ): + """ + Moving a repository from namespace1/repo_name1 to namespace2/repo_name2 + + Note there are certain limitations. For more information about moving + repositories, please see + https://hf.co/docs/hub/repositories-settings#renaming-or-transferring-a-repo. + + Args: + from_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. Original repository identifier. + to_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. Final repository identifier. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + > [!TIP] + > Raises the following errors: + > + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + """ + if len(from_id.split("/")) != 2: + raise ValueError(f"Invalid repo_id: {from_id}. It should have a namespace (:namespace:/:repo_name:)") + + if len(to_id.split("/")) != 2: + raise ValueError(f"Invalid repo_id: {to_id}. It should have a namespace (:namespace:/:repo_name:)") + + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL # Hub won't accept `None`. + + json = {"fromRepo": from_id, "toRepo": to_id, "type": repo_type} + + path = f"{self.endpoint}/api/repos/move" + headers = self._build_hf_headers(token=token) + r = get_session().post(path, headers=headers, json=json) + try: + hf_raise_for_status(r) + except HfHubHTTPError as e: + e.append_to_message( + "\nFor additional documentation please see" + " https://hf.co/docs/hub/repositories-settings#renaming-or-transferring-a-repo." + ) + raise + + @overload + def create_commit( # type: ignore + self, + repo_id: str, + operations: Iterable[CommitOperation], + *, + commit_message: str, + commit_description: str | None = None, + token: str | bool | None = None, + repo_type: str | None = None, + revision: str | None = None, + create_pr: bool | None = None, + num_threads: int = 5, + parent_commit: str | None = None, + run_as_future: Literal[False] = ..., + _hot_reload: bool | None = None, + ) -> CommitInfo: ... + + @overload + def create_commit( + self, + repo_id: str, + operations: Iterable[CommitOperation], + *, + commit_message: str, + commit_description: str | None = None, + token: str | bool | None = None, + repo_type: str | None = None, + revision: str | None = None, + create_pr: bool | None = None, + num_threads: int = 5, + parent_commit: str | None = None, + run_as_future: Literal[True] = ..., + _hot_reload: bool | None = None, + ) -> Future[CommitInfo]: ... + + @validate_hf_hub_args + @future_compatible + def create_commit( + self, + repo_id: str, + operations: Iterable[CommitOperation], + *, + commit_message: str, + commit_description: str | None = None, + token: str | bool | None = None, + repo_type: str | None = None, + revision: str | None = None, + create_pr: bool | None = None, + num_threads: int = 5, + parent_commit: str | None = None, + run_as_future: bool = False, + _hot_reload: bool | None = None, + ) -> CommitInfo | Future[CommitInfo]: + """ + Creates a commit in the given repo, deleting & uploading files as needed. + + > [!WARNING] + > The input list of `CommitOperation` will be mutated during the commit process. Do not reuse the same objects + > for multiple commits. + + > [!WARNING] + > `create_commit` assumes that the repo already exists on the Hub. If you get a + > Client error 404, please make sure you are authenticated, that your token has the required permissions, + > and that `repo_id` and `repo_type` are set correctly. If repo does not exist, + > create it first using [`~hf_api.create_repo`]. + + > [!WARNING] + > `create_commit` is limited to 25k LFS files and a 1GB payload for regular files. + + Args: + repo_id (`str`): + The repository in which the commit will be created, for example: + `"username/custom_transformers"` + + operations (`Iterable` of [`~hf_api.CommitOperation`]): + An iterable of operations to include in the commit, either: + + - [`~hf_api.CommitOperationAdd`] to upload a file + - [`~hf_api.CommitOperationDelete`] to delete a file + - [`~hf_api.CommitOperationCopy`] to copy a file + + Operation objects will be mutated to include information relative to the upload. Do not reuse the + same objects for multiple commits. + + commit_message (`str`): + The summary (first line) of the commit that will be created. + + commit_description (`str`, *optional*): + The description of the commit that will be created + + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the `"main"` branch. + + create_pr (`boolean`, *optional*): + Whether or not to create a Pull Request with that commit. Defaults to `False`. + If `revision` is not set, PR is opened against the `"main"` branch. If + `revision` is set and is a branch, PR is opened against this branch. If + `revision` is set and is not a branch name (example: a commit oid), an + `RevisionNotFoundError` is returned by the server. + + num_threads (`int`, *optional*): + Number of concurrent threads for uploading files. Defaults to 5. + Setting it to 2 means at most 2 files will be uploaded concurrently. + + parent_commit (`str`, *optional*): + The OID / SHA of the parent commit, as a hexadecimal string. + Shorthands (7 first characters) are also supported. If specified and `create_pr` is `False`, + the commit will fail if `revision` does not point to `parent_commit`. If specified and `create_pr` + is `True`, the pull request will be created from `parent_commit`. Specifying `parent_commit` + ensures the repo has not changed before committing the changes, and can be especially useful + if the repo is updated / committed to concurrently. + run_as_future (`bool`, *optional*): + Whether or not to run this method in the background. Background jobs are run sequentially without + blocking the main thread. Passing `run_as_future=True` will return a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) + object. Defaults to `False`. + + Returns: + [`CommitInfo`] or `Future`: + Instance of [`CommitInfo`] containing information about the newly created commit (commit hash, commit + url, pr url, commit message,...). If `run_as_future=True` is passed, returns a Future object which will + contain the result when executed. + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If commit message is empty. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If parent commit is not a valid commit OID. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If a README.md file with an invalid metadata section is committed. In this case, the commit will fail + early, before trying to upload any file. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If `create_pr` is `True` and revision is neither `None` nor `"main"`. + [`~utils.RepositoryNotFoundError`]: + If repository is not found (error 404): wrong repo_id/repo_type, private + but not authenticated or repo does not exist. + """ + if parent_commit is not None and not constants.REGEX_COMMIT_OID.fullmatch(parent_commit): + raise ValueError( + f"`parent_commit` is not a valid commit OID. It must match the following regex: {constants.REGEX_COMMIT_OID}" + ) + + if commit_message is None or len(commit_message) == 0: + raise ValueError("`commit_message` can't be empty, please pass a value.") + + commit_description = commit_description if commit_description is not None else "" + repo_type = repo_type if repo_type is not None else constants.REPO_TYPE_MODEL + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + unquoted_revision = revision or constants.DEFAULT_REVISION + revision = quote(unquoted_revision, safe="") + create_pr = create_pr if create_pr is not None else False + _hot_reload = _hot_reload if _hot_reload is not None else False + + headers = self._build_hf_headers(token=token) + + operations = list(operations) + additions = [op for op in operations if isinstance(op, CommitOperationAdd)] + copies = [op for op in operations if isinstance(op, CommitOperationCopy)] + nb_additions = len(additions) + nb_copies = len(copies) + nb_deletions = len(operations) - nb_additions - nb_copies + + for addition in additions: + if addition._is_committed: + raise ValueError( + f"CommitOperationAdd {addition} has already being committed and cannot be reused. Please create a" + " new CommitOperationAdd object if you want to create a new commit." + ) + + if repo_type != "dataset": + for addition in additions: + if addition.path_in_repo.endswith((".arrow", ".parquet")): + warnings.warn( + f"It seems that you are about to commit a data file ({addition.path_in_repo}) to a {repo_type}" + " repository. You are sure this is intended? If you are trying to upload a dataset, please" + " set `repo_type='dataset'` or `--repo-type=dataset` in a CLI." + ) + + logger.debug( + f"About to commit to the hub: {len(additions)} addition(s), {len(copies)} copy(ies) and" + f" {nb_deletions} deletion(s)." + ) + + # If updating a README.md file, make sure the metadata format is valid + # It's better to fail early than to fail after all the files have been uploaded. + for addition in additions: + if addition.path_in_repo == "README.md": + with addition.as_file() as file: + content = file.read().decode() + self._validate_yaml(content, repo_type=repo_type, token=token) + # Skip other additions after `README.md` has been processed + break + + # If updating twice the same file or update then delete a file in a single commit + _warn_on_overwriting_operations(operations) + + self.preupload_lfs_files( + repo_id=repo_id, + additions=additions, + token=token, + repo_type=repo_type, + revision=unquoted_revision, # first-class methods take unquoted revision + create_pr=create_pr, + num_threads=num_threads, + free_memory=False, # do not remove `CommitOperationAdd.path_or_fileobj` on LFS files for "normal" users + ) + + files_to_copy = _fetch_files_to_copy( + copies=copies, + repo_type=repo_type, + repo_id=repo_id, + headers=headers, + revision=unquoted_revision, + endpoint=self.endpoint, + ) + + self._duplicate_lfs_files( + repo_id=repo_id, copies=copies, files_to_copy=files_to_copy, token=token, repo_type=repo_type + ) + + # Remove no-op operations (files that have not changed) + operations_without_no_op = [] + for operation in operations: + if ( + isinstance(operation, CommitOperationAdd) + and operation._remote_oid is not None + and operation._remote_oid == operation._local_oid + ): + # File already exists on the Hub and has not changed: we can skip it. + logger.debug(f"Skipping upload for '{operation.path_in_repo}' as the file has not changed.") + continue + if ( + isinstance(operation, CommitOperationCopy) + and operation._dest_oid is not None + and operation._dest_oid == operation._src_oid + ): + # Source and destination files are identical - skip + logger.debug( + f"Skipping copy for '{operation.src_path_in_repo}' -> '{operation.path_in_repo}' as the content of the source file is the same as the destination file." + ) + continue + operations_without_no_op.append(operation) + if len(operations) != len(operations_without_no_op): + logger.info( + f"Removing {len(operations) - len(operations_without_no_op)} file(s) from commit that have not changed." + ) + + # Return early if empty commit + if len(operations_without_no_op) == 0: + logger.warning("No files have been modified since last commit. Skipping to prevent empty commit.") + + # Get latest commit info + try: + info = self.repo_info(repo_id=repo_id, repo_type=repo_type, revision=unquoted_revision, token=token) + except RepositoryNotFoundError as e: + e.append_to_message(_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE) + raise + + # Return commit info based on latest commit + url_prefix = self.endpoint + if repo_type is not None and repo_type != constants.REPO_TYPE_MODEL: + url_prefix = f"{url_prefix}/{repo_type}s" + return CommitInfo( + commit_url=f"{url_prefix}/{repo_id}/commit/{info.sha}", + commit_message=commit_message, + commit_description=commit_description, + oid=info.sha, # type: ignore + _endpoint=self.endpoint, + ) + + try: + commit_info = _send_commit( + operations=operations_without_no_op, + files_to_copy=files_to_copy, + commit_message=commit_message, + commit_description=commit_description, + repo_type=repo_type, + repo_id=repo_id, + headers=headers, + revision=revision, + endpoint=self.endpoint, + parent_commit=parent_commit, + create_pr=create_pr, + hot_reload=_hot_reload, + ) + except RepositoryNotFoundError as e: + e.append_to_message(_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE) + raise + except RemoteEntryNotFoundError as e: + if nb_deletions > 0 and "A file with this name doesn't exist" in str(e): + e.append_to_message( + "\nMake sure to differentiate file and folder paths in delete" + " operations with a trailing '/' or using `is_folder=True/False`." + ) + raise + + # Mark additions as committed (cannot be reused in another commit) + for addition in additions: + addition._is_committed = True + + return commit_info + + def preupload_lfs_files( + self, + repo_id: str, + additions: Iterable[CommitOperationAdd], + *, + token: str | bool | None = None, + repo_type: str | None = None, + revision: str | None = None, + create_pr: bool | None = None, + num_threads: int = 5, + free_memory: bool = True, + gitignore_content: str | None = None, + ): + """Pre-upload LFS files to S3 in preparation on a future commit. + + This method is useful if you are generating the files to upload on-the-fly and you don't want to store them + in memory before uploading them all at once. + + > [!WARNING] + > This is a power-user method. You shouldn't need to call it directly to make a normal commit. + > Use [`create_commit`] directly instead. + + > [!WARNING] + > Commit operations will be mutated during the process. In particular, the attached `path_or_fileobj` will be + > removed after the upload to save memory (and replaced by an empty `bytes` object). Do not reuse the same + > objects except to pass them to [`create_commit`]. If you don't want to remove the attached content from the + > commit operation object, pass `free_memory=False`. + + Args: + repo_id (`str`): + The repository in which you will commit the files, for example: `"username/custom_transformers"`. + + additions (`Iterable` of [`CommitOperationAdd`]): + The list of files to upload. Warning: the objects in this list will be mutated to include information + relative to the upload. Do not reuse the same objects for multiple commits. + + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + repo_type (`str`, *optional*): + The type of repository to upload to (e.g. `"model"` -default-, `"dataset"` or `"space"`). + + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the `"main"` branch. + + create_pr (`boolean`, *optional*): + Whether or not you plan to create a Pull Request with that commit. Defaults to `False`. + + num_threads (`int`, *optional*): + Number of concurrent threads for uploading files. Defaults to 5. + Setting it to 2 means at most 2 files will be uploaded concurrently. + + free_memory (`bool`, *optional*, defaults to `True`): + If `True`, the `path_or_fileobj` attribute of each `CommitOperationAdd` is replaced by an empty + `bytes` object after upload to save memory. Set to `False` if you need to reuse the operation + objects outside of a subsequent [`create_commit`] call. + + gitignore_content (`str`, *optional*): + The content of the `.gitignore` file to know which files should be ignored. The order of priority + is to first check if `gitignore_content` is passed, then check if the `.gitignore` file is present + in the list of files to commit and finally default to the `.gitignore` file already hosted on the Hub + (if any). + + Example: + ```py + >>> from huggingface_hub import CommitOperationAdd, preupload_lfs_files, create_commit, create_repo + + >>> repo_id = create_repo("test_preupload").repo_id + + # Generate and preupload LFS files one by one + >>> operations = [] # List of all `CommitOperationAdd` objects that will be generated + >>> for i in range(5): + ... content = ... # generate binary content + ... addition = CommitOperationAdd(path_in_repo=f"shard_{i}_of_5.bin", path_or_fileobj=content) + ... preupload_lfs_files(repo_id, additions=[addition]) # upload + free memory + ... operations.append(addition) + + # Create commit + >>> create_commit(repo_id, operations=operations, commit_message="Commit all shards") + ``` + """ + repo_type = repo_type if repo_type is not None else constants.REPO_TYPE_MODEL + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION + create_pr = create_pr if create_pr is not None else False + headers = self._build_hf_headers(token=token) + + # Check if a `gitignore` file is being committed to the Hub. + additions = list(additions) + if gitignore_content is None: + for addition in additions: + if addition.path_in_repo == ".gitignore": + with addition.as_file() as f: + gitignore_content = f.read().decode() + break + + # Filter out already uploaded files + new_additions = [addition for addition in additions if not addition._is_uploaded] + + # Check which new files are LFS + # For some items, we might have already fetched the upload mode (in case of upload_large_folder) + additions_no_upload_mode = [addition for addition in new_additions if addition._upload_mode is None] + if len(additions_no_upload_mode) > 0: + try: + _fetch_upload_modes( + additions=additions_no_upload_mode, + repo_type=repo_type, + repo_id=repo_id, + headers=headers, + revision=revision, + endpoint=self.endpoint, + create_pr=create_pr or False, + gitignore_content=gitignore_content, + ) + except RepositoryNotFoundError as e: + e.append_to_message(_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE) + raise + + # Filter out regular files + new_lfs_additions = [addition for addition in new_additions if addition._upload_mode == "lfs"] + + # Filter out files listed in .gitignore + new_lfs_additions_to_upload = [] + for addition in new_lfs_additions: + if addition._should_ignore: + logger.debug(f"Skipping upload for LFS file '{addition.path_in_repo}' (ignored by gitignore file).") + else: + new_lfs_additions_to_upload.append(addition) + if len(new_lfs_additions) != len(new_lfs_additions_to_upload): + logger.info( + f"Skipped upload for {len(new_lfs_additions) - len(new_lfs_additions_to_upload)} LFS file(s) " + "(ignored by gitignore file)." + ) + # If no LFS files remain to upload, keep previous behavior and log explicitly + if len(new_lfs_additions_to_upload) == 0: + logger.debug("No LFS files to upload.") + return + # Prepare upload parameters + upload_kwargs = { + "additions": new_lfs_additions_to_upload, + "repo_type": repo_type, + "repo_id": repo_id, + "headers": headers, + "endpoint": self.endpoint, + # If `create_pr`, we don't want to check user permission on the revision as users with read permission + # should still be able to create PRs even if they don't have write permission on the target branch of the + # PR (i.e. `revision`). + "revision": revision if not create_pr else None, + } + _upload_files( + **upload_kwargs, # type: ignore[arg-type] + num_threads=num_threads, + create_pr=create_pr, + ) + for addition in new_lfs_additions_to_upload: + addition._is_uploaded = True + if free_memory: + addition.path_or_fileobj = b"" + + @validate_hf_hub_args + def _duplicate_lfs_files( + self, + repo_id: str, + copies: Iterable[CommitOperationCopy], + *, + files_to_copy: dict, + token: str | bool | None = None, + repo_type: str | None = None, + ) -> None: + """Duplicate LFS files from source repositories to the destination repository. + + This method is the equivalent of [`preupload_lfs_files`] for cross-repo copy operations. It must be called + before [`create_commit`] to ensure that LFS files from the source repositories are available in the destination + repository before the commit is created. + + Args: + repo_id (`str`): + The destination repository in which you will commit the files, for example: + `"username/custom_transformers"`. + + copies (`Iterable` of [`CommitOperationCopy`]): + The list of copy operations describing which files to duplicate. Only cross-repo copies (where + `src_repo_id` is set) with LFS files will be processed. Warning: the objects in this list will be + mutated to include information relative to the duplication. Do not reuse the same objects for multiple + commits. + + files_to_copy (`dict`): + Pre-fetched file info from [`_fetch_files_to_copy`]. LFS metadata is extracted from this dict instead + of making additional API calls. Keys are `_CopySource` tuples, values are `RepoFile` (for LFS files) + or `bytes` (for regular files). + + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + repo_type (`str`, *optional*): + The type of the destination repository (e.g. `"model"` -default-, `"dataset"` or `"space"`). + """ + repo_type = repo_type if repo_type is not None else constants.REPO_TYPE_MODEL + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + headers = self._build_hf_headers(token=token) + + copies = list(copies) + # Filter to cross-repo copies that haven't been duplicated yet + cross_repo_copies = [op for op in copies if op.src_repo_id is not None and not op._is_duplicated] + if not cross_repo_copies: + logger.debug("No cross-repo LFS files to duplicate.") + return + + # The /lfs-files/duplicate endpoint lives on the *source* repo and takes the destination as `target`. + cross_repo_copies.sort(key=lambda op: (op.src_repo_id or "", op.src_repo_type or "", op.src_revision or "")) + + for (src_repo_id, src_repo_type, src_revision), group in itertools.groupby( + cross_repo_copies, key=lambda op: (op.src_repo_id, op.src_repo_type, op.src_revision) + ): + operations = list(group) + + lfs_files: list[dict] = [] + seen_oids: set[str] = set() + + for op in operations: + key = _CopySource(op.src_repo_id, op.src_repo_type, op.src_path_in_repo, op.src_revision) + src_file = files_to_copy.get(key) + if src_file is None or isinstance(src_file, bytes): + continue + if not src_file.lfs: + continue + if not src_file.xet_hash: + raise ValueError( + f"Cannot duplicate LFS file '{src_file.path}' from {src_repo_type}s/{src_repo_id}: file has no xet hash." + f" (file: {src_file})" + ) + oid = src_file.lfs.sha256 + if oid not in seen_oids: + seen_oids.add(oid) + lfs_files.append({"xetHash": src_file.xet_hash, "sha256": oid, "filename": src_file.path}) + + if not lfs_files: + continue + + # Call the duplicate endpoint on the *source* repo, in batches + duplicate_url = f"{self.endpoint}/api/{src_repo_type}s/{src_repo_id}/lfs-files/duplicate" + for batch in chunk_iterable(lfs_files, DUPLICATE_LFS_BATCH_SIZE): + response = get_session().post( + duplicate_url, + headers=headers, + json={"target": {"type": repo_type, "name": repo_id}, "files": list(batch)}, + ) + hf_raise_for_status(response) + data = response.json() + failures = data.get("failed", []) + if failures: + messages = [f" - {f['sha256']}: {f['error']}" for f in failures] + raise FileDuplicationError( + f"Failed to duplicate files from {src_repo_type}s/{src_repo_id} " + f"to {repo_type}s/{repo_id}:\n" + "\n".join(messages) + ) + + for op in cross_repo_copies: + op._is_duplicated = True + + @overload + def upload_file( # type: ignore + self, + *, + path_or_fileobj: str | Path | bytes | BinaryIO, + path_in_repo: str, + repo_id: str, + token: str | bool | None = None, + repo_type: str | None = None, + revision: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + create_pr: bool | None = None, + parent_commit: str | None = None, + run_as_future: Literal[False] = ..., + _hot_reload: bool | None = None, + ) -> CommitInfo: ... + + @overload + def upload_file( + self, + *, + path_or_fileobj: str | Path | bytes | BinaryIO, + path_in_repo: str, + repo_id: str, + token: str | bool | None = None, + repo_type: str | None = None, + revision: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + create_pr: bool | None = None, + parent_commit: str | None = None, + run_as_future: Literal[True] = ..., + _hot_reload: bool | None = None, + ) -> Future[CommitInfo]: ... + + @validate_hf_hub_args + @future_compatible + def upload_file( + self, + *, + path_or_fileobj: str | Path | bytes | BinaryIO, + path_in_repo: str, + repo_id: str, + token: str | bool | None = None, + repo_type: str | None = None, + revision: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + create_pr: bool | None = None, + parent_commit: str | None = None, + run_as_future: bool = False, + _hot_reload: bool | None = None, + ) -> CommitInfo | Future[CommitInfo]: + """ + Upload a local file (up to 50 GB) to the given repo. The upload is done + through a HTTP post request, and doesn't require git or git-lfs to be + installed. + + Args: + path_or_fileobj (`str`, `Path`, `bytes`, or `IO`): + Path to a file on the local machine or binary data stream / + fileobj / buffer. + path_in_repo (`str`): + Relative filepath in the repo, for example: + `"checkpoints/1fec34a/weights.bin"` + repo_id (`str`): + The repository to which the file will be uploaded, for example: + `"username/custom_transformers"` + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the `"main"` branch. + commit_message (`str`, *optional*): + The summary / title / first line of the generated commit + commit_description (`str` *optional*) + The description of the generated commit + create_pr (`boolean`, *optional*): + Whether or not to create a Pull Request with that commit. Defaults to `False`. + If `revision` is not set, PR is opened against the `"main"` branch. If + `revision` is set and is a branch, PR is opened against this branch. If + `revision` is set and is not a branch name (example: a commit oid), an + `RevisionNotFoundError` is returned by the server. + parent_commit (`str`, *optional*): + The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. + If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. + If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. + Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be + especially useful if the repo is updated / committed to concurrently. + run_as_future (`bool`, *optional*): + Whether or not to run this method in the background. Background jobs are run sequentially without + blocking the main thread. Passing `run_as_future=True` will return a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) + object. Defaults to `False`. + + + Returns: + [`CommitInfo`] or `Future`: + Instance of [`CommitInfo`] containing information about the newly created commit (commit hash, commit + url, pr url, commit message,...). If `run_as_future=True` is passed, returns a Future object which will + contain the result when executed. + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + > - [`~utils.RevisionNotFoundError`] + > If the revision to download from cannot be found. + + > [!WARNING] + > `upload_file` assumes that the repo already exists on the Hub. If you get a + > Client error 404, please make sure you are authenticated, that your token has the required permissions, + > and that `repo_id` and `repo_type` are set correctly. If repo does not exist, + > create it first using [`~hf_api.create_repo`]. + + Example: + + ```python + >>> from huggingface_hub import upload_file + + >>> with open("./local/filepath", "rb") as fobj: + ... upload_file( + ... path_or_fileobj=fileobj, + ... path_in_repo="remote/file/path.h5", + ... repo_id="username/my-dataset", + ... repo_type="dataset", + ... token="my_token", + ... ) + + >>> upload_file( + ... path_or_fileobj=".\\\\local\\\\file\\\\path", + ... path_in_repo="remote/file/path.h5", + ... repo_id="username/my-model", + ... token="my_token", + ... ) + + >>> upload_file( + ... path_or_fileobj=".\\\\local\\\\file\\\\path", + ... path_in_repo="remote/file/path.h5", + ... repo_id="username/my-model", + ... token="my_token", + ... create_pr=True, + ... ) + ``` + """ + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + + commit_message = ( + commit_message if commit_message is not None else f"Upload {path_in_repo} with huggingface_hub" + ) + operation = CommitOperationAdd( + path_or_fileobj=path_or_fileobj, + path_in_repo=path_in_repo, + ) + + return self.create_commit( + repo_id=repo_id, + repo_type=repo_type, + operations=[operation], + commit_message=commit_message, + commit_description=commit_description, + token=token, + revision=revision, + create_pr=create_pr, + _hot_reload=_hot_reload, + parent_commit=parent_commit, + ) + + @overload + def upload_folder( # type: ignore + self, + *, + repo_id: str, + folder_path: str | Path, + path_in_repo: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + token: str | bool | None = None, + repo_type: str | None = None, + revision: str | None = None, + create_pr: bool | None = None, + parent_commit: str | None = None, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + delete_patterns: list[str] | str | None = None, + run_as_future: Literal[False] = ..., + ) -> CommitInfo: ... + + @overload + def upload_folder( # type: ignore + self, + *, + repo_id: str, + folder_path: str | Path, + path_in_repo: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + token: str | bool | None = None, + repo_type: str | None = None, + revision: str | None = None, + create_pr: bool | None = None, + parent_commit: str | None = None, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + delete_patterns: list[str] | str | None = None, + run_as_future: Literal[True] = ..., + ) -> Future[CommitInfo]: ... + + @validate_hf_hub_args + @future_compatible + def upload_folder( + self, + *, + repo_id: str, + folder_path: str | Path, + path_in_repo: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + token: str | bool | None = None, + repo_type: str | None = None, + revision: str | None = None, + create_pr: bool | None = None, + parent_commit: str | None = None, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + delete_patterns: list[str] | str | None = None, + run_as_future: bool = False, + ) -> CommitInfo | Future[CommitInfo]: + """ + Upload a local folder to the given repo. The upload is done through a HTTP requests, and doesn't require git or + git-lfs to be installed. + + The structure of the folder will be preserved. Files with the same name already present in the repository will + be overwritten. Others will be left untouched. + + Use the `allow_patterns` and `ignore_patterns` arguments to specify which files to upload. These parameters + accept either a single pattern or a list of patterns. Patterns are Standard Wildcards (globbing patterns) as + documented [here](https://tldp.org/LDP/GNU-Linux-Tools-Summary/html/x11655.htm). If both `allow_patterns` and + `ignore_patterns` are provided, both constraints apply. By default, all files from the folder are uploaded. + + Use the `delete_patterns` argument to specify remote files you want to delete. Input type is the same as for + `allow_patterns` (see above). If `path_in_repo` is also provided, the patterns are matched against paths + relative to this folder. For example, `upload_folder(..., path_in_repo="experiment", delete_patterns="logs/*")` + will delete any remote file under `./experiment/logs/`. Note that the `.gitattributes` file will not be deleted + even if it matches the patterns. + + Any `.git/` folder present in any subdirectory will be ignored. However, please be aware that the `.gitignore` + file is not taken into account. + + When `hf_xet` is installed (the default), files are uploaded through a streamed pipeline: uploads start while + the folder is still being checked against the Hub, files are hashed while being chunked for upload (single + read pass), and large folders are automatically committed in several batches to stay below server limits + (follow-up commits get a ` (part N)` suffix on the commit message). If the upload is interrupted, re-running + the same call resumes it: already-committed files are skipped and already-uploaded data is deduplicated. When + `hf_xet` is not installed, falls back to a single commit created with [`create_commit`]. + + Args: + repo_id (`str`): + The repository to which the file will be uploaded, for example: + `"username/custom_transformers"` + folder_path (`str` or `Path`): + Path to the folder to upload on the local file system + path_in_repo (`str`, *optional*): + Relative path of the directory in the repo, for example: + `"checkpoints/1fec34a/results"`. Will default to the root folder of the repository. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the `"main"` branch. + commit_message (`str`, *optional*): + The summary / title / first line of the generated commit. Defaults to: + `f"Upload {path_in_repo} with huggingface_hub"` + commit_description (`str` *optional*): + The description of the generated commit + create_pr (`boolean`, *optional*): + Whether or not to create a Pull Request with that commit. Defaults to `False`. The PR is always + opened against the default branch: setting both `create_pr=True` and `revision` raises a + `ValueError`. Note that each call with `create_pr=True` opens a new pull request: to resume an + interrupted upload into the existing PR, re-run with `revision="refs/pr/N"` instead. + parent_commit (`str`, *optional*): + The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. + If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. + Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be + especially useful if the repo is updated / committed to concurrently. If the upload is split into + several commits (large folders), `parent_commit` only applies to the first one. + allow_patterns (`list[str]` or `str`, *optional*): + If provided, only files matching at least one pattern are uploaded. + ignore_patterns (`list[str]` or `str`, *optional*): + If provided, files matching any of the patterns are not uploaded. + delete_patterns (`list[str]` or `str`, *optional*): + If provided, remote files matching any of the patterns will be deleted from the repo while committing + new files. This is useful if you don't know which files have already been uploaded. + Note: to avoid discrepancies the `.gitattributes` file is not deleted even if it matches the pattern. + run_as_future (`bool`, *optional*): + Whether or not to run this method in the background. Background jobs are run sequentially without + blocking the main thread. Passing `run_as_future=True` will return a [Future](https://docs.python.org/3/library/concurrent.futures.html#future-objects) + object. Defaults to `False`. + + Returns: + [`CommitInfo`] or `Future`: + Instance of [`CommitInfo`] containing information about the newly created commit (commit hash, commit + url, pr url, commit message,...). If `run_as_future=True` is passed, returns a Future object which will + contain the result when executed. + + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + + > [!WARNING] + > `upload_folder` assumes that the repo already exists on the Hub. If you get a Client error 404, please make + > sure you are authenticated, that your token has the required permissions, and that `repo_id` and `repo_type` + > are set correctly. If repo does not exist, create it first using [`~hf_api.create_repo`]. + + Example: + + ```python + # Upload checkpoints folder except the log files + >>> upload_folder( + ... folder_path="local/checkpoints", + ... path_in_repo="remote/experiment/checkpoints", + ... repo_id="username/my-dataset", + ... repo_type="datasets", + ... token="my_token", + ... ignore_patterns="**/logs/*.txt", + ... ) + + # Upload checkpoints folder including logs while deleting existing logs from the repo + # Useful if you don't know exactly which log files have already being pushed + >>> upload_folder( + ... folder_path="local/checkpoints", + ... path_in_repo="remote/experiment/checkpoints", + ... repo_id="username/my-dataset", + ... repo_type="datasets", + ... token="my_token", + ... delete_patterns="**/logs/*.txt", + ... ) + + # Upload checkpoints folder while creating a PR + >>> upload_folder( + ... folder_path="local/checkpoints", + ... path_in_repo="remote/experiment/checkpoints", + ... repo_id="username/my-dataset", + ... repo_type="datasets", + ... token="my_token", + ... create_pr=True, + ... ) + ``` + """ + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + if create_pr and revision is not None and revision != constants.DEFAULT_REVISION: + raise ValueError( + f"Cannot use `create_pr=True` with `revision='{revision}'`: pull requests created by" + " `upload_folder` are always opened against the default branch. Don't set `revision` when" + " `create_pr=True`." + ) + + # By default, upload folder to the root directory in repo. + if path_in_repo is None: + path_in_repo = "" + + # Do not upload .git folder + if ignore_patterns is None: + ignore_patterns = [] + elif isinstance(ignore_patterns, str): + ignore_patterns = [ignore_patterns] + ignore_patterns += DEFAULT_IGNORE_PATTERNS + + delete_operations = self._prepare_folder_deletions( + repo_id=repo_id, + repo_type=repo_type, + revision=constants.DEFAULT_REVISION if create_pr else revision, + token=token, + path_in_repo=path_in_repo, + delete_patterns=delete_patterns, + ) + add_operations = self._prepare_upload_folder_additions( + folder_path, + path_in_repo, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + token=token, + repo_type=repo_type, + ) + + # Optimize operations: if some files will be overwritten, we don't need to delete them first + if len(add_operations) > 0: + added_paths = {op.path_in_repo for op in add_operations} + delete_operations = [ + delete_op for delete_op in delete_operations if delete_op.path_in_repo not in added_paths + ] + commit_operations = delete_operations + add_operations + + commit_message = commit_message or "Upload folder using huggingface_hub" + + if is_xet_available(): + # Streamed multi-commit pipeline: uploads and commits overlap, large folders are + # committed in adaptive batches, interrupted uploads resume by re-running. + return pipelined_upload( + self, + repo_id=repo_id, + repo_type=repo_type or constants.REPO_TYPE_MODEL, + add_operations=add_operations, + delete_operations=delete_operations, + commit_message=commit_message, + commit_description=commit_description, + token=token, + revision=revision, + create_pr=create_pr or False, + parent_commit=parent_commit, + ) + + # Legacy single-commit path (hf_xet not installed). + if len(add_operations) > 30: + log = logger.warning if len(add_operations) > 200 else logger.info + log( + "It seems you are trying to upload a large folder at once. This might take some time and then fail if " + "the folder is too large. Installing `hf_xet` (`pip install hf_xet`) enables a more robust, resumable " + "upload that handles large folders in multiple commits." + ) + return self.create_commit( + repo_type=repo_type, + repo_id=repo_id, + operations=commit_operations, + commit_message=commit_message, + commit_description=commit_description, + token=token, + revision=revision, + create_pr=create_pr, + parent_commit=parent_commit, + ) + + @validate_hf_hub_args + def delete_file( + self, + path_in_repo: str, + repo_id: str, + *, + token: str | bool | None = None, + repo_type: str | None = None, + revision: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + create_pr: bool | None = None, + parent_commit: str | None = None, + ) -> CommitInfo: + """ + Deletes a file in the given repo. + + Args: + path_in_repo (`str`): + Relative filepath in the repo, for example: + `"checkpoints/1fec34a/weights.bin"` + repo_id (`str`): + The repository from which the file will be deleted, for example: + `"username/custom_transformers"` + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if the file is in a dataset or + space, `None` or `"model"` if in a model. Default is `None`. + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the `"main"` branch. + commit_message (`str`, *optional*): + The summary / title / first line of the generated commit. Defaults to + `f"Delete {path_in_repo} with huggingface_hub"`. + commit_description (`str` *optional*) + The description of the generated commit + create_pr (`boolean`, *optional*): + Whether or not to create a Pull Request with that commit. Defaults to `False`. + If `revision` is not set, PR is opened against the `"main"` branch. If + `revision` is set and is a branch, PR is opened against this branch. If + `revision` is set and is not a branch name (example: a commit oid), an + `RevisionNotFoundError` is returned by the server. + parent_commit (`str`, *optional*): + The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. + If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. + If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. + Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be + especially useful if the repo is updated / committed to concurrently. + + + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + > - [`~utils.RevisionNotFoundError`] + > If the revision to download from cannot be found. + > - [`~utils.EntryNotFoundError`] + > If the file to download cannot be found. + + """ + commit_message = ( + commit_message if commit_message is not None else f"Delete {path_in_repo} with huggingface_hub" + ) + + operations = [CommitOperationDelete(path_in_repo=path_in_repo)] + + return self.create_commit( + repo_id=repo_id, + repo_type=repo_type, + token=token, + operations=operations, + revision=revision, + commit_message=commit_message, + commit_description=commit_description, + create_pr=create_pr, + parent_commit=parent_commit, + ) + + @validate_hf_hub_args + def delete_files( + self, + repo_id: str, + delete_patterns: list[str], + *, + token: bool | str | None = None, + repo_type: str | None = None, + revision: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + create_pr: bool | None = None, + parent_commit: str | None = None, + ) -> CommitInfo: + """ + Delete files from a repository on the Hub. + + If a folder path is provided, the entire folder is deleted as well as + all files it contained. + + Args: + repo_id (`str`): + The repository from which the folder will be deleted, for example: + `"username/custom_transformers"` + delete_patterns (`list[str]`): + List of files or folders to delete. Each string can either be + a file path, a folder path, or a wildcard pattern. Patterns are Standard + Wildcards (globbing patterns) as documented [here](https://tldp.org/LDP/GNU-Linux-Tools-Summary/html/x11655.htm). + The pattern matching is based on [`fnmatch`](https://docs.python.org/3/library/fnmatch.html). + Note that `fnmatch` matches `*` across path boundaries, unlike traditional Unix shell globbing. + E.g. `["file.txt", "folder/", "data/*.parquet"]` + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + to the stored token. + repo_type (`str`, *optional*): + Type of the repo to delete files from. Can be `"model"`, + `"dataset"` or `"space"`. Defaults to `"model"`. + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the `"main"` branch. + commit_message (`str`, *optional*): + The summary (first line) of the generated commit. Defaults to + `f"Delete files using huggingface_hub"`. + commit_description (`str` *optional*) + The description of the generated commit. + create_pr (`boolean`, *optional*): + Whether or not to create a Pull Request with that commit. Defaults to `False`. + If `revision` is not set, PR is opened against the `"main"` branch. If + `revision` is set and is a branch, PR is opened against this branch. If + `revision` is set and is not a branch name (example: a commit oid), an + `RevisionNotFoundError` is returned by the server. + parent_commit (`str`, *optional*): + The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. + If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. + If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. + Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be + especially useful if the repo is updated / committed to concurrently. + """ + operations = self._prepare_folder_deletions( + repo_id=repo_id, repo_type=repo_type, delete_patterns=delete_patterns, path_in_repo="", revision=revision + ) + + if commit_message is None: + commit_message = f"Delete files {' '.join(delete_patterns)} with huggingface_hub" + + return self.create_commit( + repo_id=repo_id, + repo_type=repo_type, + token=token, + operations=operations, + revision=revision, + commit_message=commit_message, + commit_description=commit_description, + create_pr=create_pr, + parent_commit=parent_commit, + ) + + @validate_hf_hub_args + def delete_folder( + self, + path_in_repo: str, + repo_id: str, + *, + token: bool | str | None = None, + repo_type: str | None = None, + revision: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + create_pr: bool | None = None, + parent_commit: str | None = None, + ) -> CommitInfo: + """ + Deletes a folder in the given repo. + + Simple wrapper around [`create_commit`] method. + + Args: + path_in_repo (`str`): + Relative folder path in the repo, for example: `"checkpoints/1fec34a"`. + repo_id (`str`): + The repository from which the folder will be deleted, for example: + `"username/custom_transformers"` + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + to the stored token. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if the folder is in a dataset or + space, `None` or `"model"` if in a model. Default is `None`. + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the `"main"` branch. + commit_message (`str`, *optional*): + The summary / title / first line of the generated commit. Defaults to + `f"Delete folder {path_in_repo} with huggingface_hub"`. + commit_description (`str` *optional*) + The description of the generated commit. + create_pr (`boolean`, *optional*): + Whether or not to create a Pull Request with that commit. Defaults to `False`. + If `revision` is not set, PR is opened against the `"main"` branch. If + `revision` is set and is a branch, PR is opened against this branch. If + `revision` is set and is not a branch name (example: a commit oid), an + `RevisionNotFoundError` is returned by the server. + parent_commit (`str`, *optional*): + The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. + If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. + If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. + Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be + especially useful if the repo is updated / committed to concurrently. + """ + return self.create_commit( + repo_id=repo_id, + repo_type=repo_type, + token=token, + operations=[CommitOperationDelete(path_in_repo=path_in_repo, is_folder=True)], + revision=revision, + commit_message=( + commit_message if commit_message is not None else f"Delete folder {path_in_repo} with huggingface_hub" + ), + commit_description=commit_description, + create_pr=create_pr, + parent_commit=parent_commit, + ) + + def upload_large_folder( + self, + repo_id: str, + folder_path: str | Path, + *, + repo_type: str, # Repo type is required! + revision: str | None = None, + private: bool | None = None, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + num_workers: int | None = None, + print_report: bool = True, + print_report_every: int = 60, + ) -> None: + """Upload a large folder to the Hub in the most resilient way possible. + + > [!WARNING] + > `upload_large_folder` is deprecated and will be removed in a future release. [`upload_folder`] is now multi-commits + > by default and resilient to interruptions so it is the recommended way to upload large folders. + + Several workers are started to upload files in an optimized way. Before being committed to a repo, files must be + hashed and be pre-uploaded if they are LFS files. Workers will perform these tasks for each file in the folder. + At each step, some metadata information about the upload process is saved in the folder under `.cache/.huggingface/` + to be able to resume the process if interrupted. The whole process might result in several commits. + + Args: + repo_id (`str`): + The repository to which the file will be uploaded. + E.g. `"HuggingFaceTB/smollm-corpus"`. + folder_path (`str` or `Path`): + Path to the folder to upload on the local file system. + repo_type (`str`): + Type of the repository. Must be one of `"model"`, `"dataset"` or `"space"`. + Unlike in all other `HfApi` methods, `repo_type` is explicitly required here. This is to avoid + any mistake when uploading a large folder to the Hub, and therefore prevent from having to re-upload + everything. + revision (`str`, `optional`): + The branch to commit to. If not provided, the `main` branch will be used. + private (`bool`, `optional`): + Whether the repository should be private. + If `None` (default), the repo will be public unless the organization's default is private. + allow_patterns (`list[str]` or `str`, *optional*): + If provided, only files matching at least one pattern are uploaded. + ignore_patterns (`list[str]` or `str`, *optional*): + If provided, files matching any of the patterns are not uploaded. + num_workers (`int`, *optional*): + Number of workers to start. Defaults to half of CPU cores (minimum 1). + A higher number of workers may speed up the process if your machine allows it. However, on machines with a + slower connection, it is recommended to keep the number of workers low to ensure better resumability. + Indeed, partially uploaded files will have to be completely re-uploaded if the process is interrupted. + print_report (`bool`, *optional*): + Whether to print a report of the upload progress. Defaults to True. + Report is printed to `sys.stdout` every X seconds (60 by defaults) and overwrites the previous report. + print_report_every (`int`, *optional*): + Frequency at which the report is printed. Defaults to 60 seconds. + + > [!TIP] + > A few things to keep in mind: + > - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations + > - Do not start several processes in parallel. + > - You can interrupt and resume the process at any time. + > - Do not upload the same folder to several repositories. If you need to do so, you must delete the local `.cache/.huggingface/` folder first. + + > [!WARNING] + > While being much more robust to upload large folders, `upload_large_folder` is more limited than [`upload_folder`] feature-wise. In practice: + > - you cannot set a custom `path_in_repo`. If you want to upload to a subfolder, you need to set the proper structure locally. + > - you cannot set a custom `commit_message` and `commit_description` since multiple commits are created. + > - you cannot delete from the repo while uploading. Please make a separate commit first. + > - you cannot create a PR directly. Please create a PR first (from the UI or using [`create_pull_request`]) and then commit to it by passing `revision`. + + **Technical details:** + + `upload_large_folder` process is as follow: + 1. (Check parameters and setup.) + 2. Create repo if missing. + 3. List local files to upload. + 4. Run validation checks and display warnings if repository limits might be exceeded: + - Warns if the total number of files exceeds 100k (recommended limit). + - Warns if any folder contains more than 10k files (recommended limit). + - Warns about files larger than 20GB (recommended) or 50GB (hard limit). + 5. Start workers. Workers can perform the following tasks: + - Hash a file. + - Get upload mode (regular or LFS) for a list of files. + - Pre-upload an LFS file. + - Commit a bunch of files. + Once a worker finishes a task, it will move on to the next task based on the priority list (see below) until + all files are uploaded and committed. + 6. While workers are up, regularly print a report to sys.stdout. + + Order of priority: + 1. Commit if more than 5 minutes since last commit attempt (and at least 1 file). + 2. Commit if at least 150 files are ready to commit. + 3. Get upload mode if at least 10 files have been hashed. + 4. Pre-upload LFS file if at least 1 file and no worker is pre-uploading. + 5. Hash file if at least 1 file and no worker is hashing. + 6. Get upload mode if at least 1 file and no worker is getting upload mode. + 7. Pre-upload LFS file if at least 1 file. + 8. Hash file if at least 1 file to hash. + 9. Get upload mode if at least 1 file to get upload mode. + 10. Commit if at least 1 file to commit and at least 1 min since last commit attempt. + 11. Commit if at least 1 file to commit and all other queues are empty. + + Special rules: + - Only one worker can commit at a time. + - If no tasks are available, the worker waits for 10 seconds before checking again. + """ + warnings.warn( + "\n" + "================================================================================\n" + "`upload_large_folder` is DEPRECATED and will be removed in a future release.\n" + "\n" + "Use `upload_folder` instead:\n" + "\n" + f' api.upload_folder(repo_id="{repo_id}", repo_type="{repo_type}", folder_path="{folder_path}")\n' + "================================================================================\n", + FutureWarning, + stacklevel=2, + ) + return upload_large_folder_internal( + self, + repo_id=repo_id, + folder_path=folder_path, + repo_type=repo_type, + revision=revision, + private=private, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + num_workers=num_workers, + print_report=print_report, + print_report_every=print_report_every, + ) + + @validate_hf_hub_args + def get_hf_file_metadata( + self, + *, + url: str, + token: bool | str | None = None, + timeout: float | None = constants.HF_HUB_ETAG_TIMEOUT, + ) -> HfFileMetadata: + """Fetch metadata of a file versioned on the Hub for a given url. + + Args: + url (`str`): + File url, for example returned by [`hf_hub_url`]. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + timeout (`float`, *optional*, defaults to 10): + How many seconds to wait for the server to send metadata before giving up. + + Returns: + A [`HfFileMetadata`] object containing metadata such as location, etag, size and commit_hash. + """ + if token is None: + # Cannot do `token = token or self.token` as token can be `False`. + token = self.token + + return get_hf_file_metadata( + url=url, + token=token, + timeout=timeout, + library_name=self.library_name, + library_version=self.library_version, + user_agent=self.user_agent, + endpoint=self.endpoint, + ) + + @overload + def hf_hub_download( + self, + repo_id: str, + filename: str, + *, + subfolder: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + force_download: bool = False, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + token: bool | str | None = None, + local_files_only: bool = False, + tqdm_class: type[base_tqdm] | None = None, + dry_run: Literal[False] = False, + ) -> str: ... + + @overload + def hf_hub_download( + self, + repo_id: str, + filename: str, + *, + subfolder: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + force_download: bool = False, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + token: bool | str | None = None, + local_files_only: bool = False, + tqdm_class: type[base_tqdm] | None = None, + dry_run: Literal[True], + ) -> DryRunFileInfo: ... + + @validate_hf_hub_args + def hf_hub_download( + self, + repo_id: str, + filename: str, + *, + subfolder: str | None = None, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + force_download: bool = False, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + token: bool | str | None = None, + local_files_only: bool = False, + tqdm_class: type[base_tqdm] | None = None, + dry_run: bool = False, + ) -> str | DryRunFileInfo: + """Download a given file if it's not already present in the local cache. + + The new cache file layout looks like this: + - The cache directory contains one subfolder per repo_id (namespaced by repo type) + - inside each repo folder: + - refs is a list of the latest known revision => commit_hash pairs + - blobs contains the actual file blobs (identified by their git-sha or sha256, depending on + whether they're LFS files or not) + - snapshots contains one subfolder per commit, each "commit" contains the subset of the files + that have been resolved at that particular commit. Each filename is a symlink to the blob + at that particular commit. + + ``` + [ 96] . + └── [ 160] models--julien-c--EsperBERTo-small + ├── [ 160] blobs + │ ├── [321M] 403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd + │ ├── [ 398] 7cb18dc9bafbfcf74629a4b760af1b160957a83e + │ └── [1.4K] d7edf6bd2a681fb0175f7735299831ee1b22b812 + ├── [ 96] refs + │ └── [ 40] main + └── [ 128] snapshots + ├── [ 128] 2439f60ef33a0d46d85da5001d52aeda5b00ce9f + │ ├── [ 52] README.md -> ../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812 + │ └── [ 76] pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd + └── [ 128] bbc77c8132af1cc5cf678da3f1ddf2de43606d48 + ├── [ 52] README.md -> ../../blobs/7cb18dc9bafbfcf74629a4b760af1b160957a83e + └── [ 76] pytorch_model.bin -> ../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd + ``` + + If `local_dir` is provided, the file structure from the repo will be replicated in this location. When using this + option, the `cache_dir` will not be used and a `.cache/huggingface/` folder will be created at the root of `local_dir` + to store some metadata related to the downloaded files. While this mechanism is not as robust as the main + cache-system, it's optimized for regularly pulling the latest version of a repository. + + Args: + repo_id (`str`): + A user or an organization name and a repo name separated by a `/`. + filename (`str`): + The name of the file in the repo. + subfolder (`str`, *optional*): + An optional value corresponding to a folder inside the repository. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if downloading from a dataset or space, + `None` or `"model"` if downloading from a model. Default is `None`. + revision (`str`, *optional*): + An optional Git revision id which can be a branch name, a tag, or a + commit hash. + cache_dir (`str`, `Path`, *optional*): + Path to the folder where cached files are stored. + local_dir (`str` or `Path`, *optional*): + If provided, the downloaded file will be placed under this directory. + force_download (`bool`, *optional*, defaults to `False`): + Whether the file should be downloaded even if it already exists in + the local cache. + etag_timeout (`float`, *optional*, defaults to `10`): + When fetching ETag, how many seconds to wait for the server to send + data before giving up which is passed to `httpx.request`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, avoid downloading the file and return the path to the + local cached file if it exists. + tqdm_class (`tqdm`, *optional*): + If provided, overwrites the default behavior for the progress bar. Passed + argument must inherit from `tqdm.auto.tqdm` or at least mimic its behavior. + Defaults to the custom HF progress bar that can be disabled by setting + `HF_HUB_DISABLE_PROGRESS_BARS` environment variable. + dry_run (`bool`, *optional*, defaults to `False`): + If `True`, perform a dry run without actually downloading the file. Returns a + [`DryRunFileInfo`] object containing information about what would be downloaded. + + Returns: + `str` or [`DryRunFileInfo`]: + - If `dry_run=False`: Local path of file or if networking is off, last version of file cached on disk. + - If `dry_run=True`: A [`DryRunFileInfo`] object containing download information. + + Raises: + [`~utils.RepositoryNotFoundError`] + If the repository to download from cannot be found. This may be because it doesn't exist, + or because it is set to `private` and you do not have access. + [`~utils.RevisionNotFoundError`] + If the revision to download from cannot be found. + [`~utils.RemoteEntryNotFoundError`] + If the file to download cannot be found. + [`~utils.LocalEntryNotFoundError`] + If network is disabled or unavailable and file is not found in cache. + [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) + If `token=True` but the token cannot be found. + [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) + If ETag cannot be determined. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If some parameter value is invalid. + """ + from .file_download import hf_hub_download + + if token is None: + # Cannot do `token = token or self.token` as token can be `False`. + token = self.token + + return hf_hub_download( + repo_id=repo_id, + filename=filename, + subfolder=subfolder, + repo_type=repo_type, + revision=revision, + endpoint=self.endpoint, + library_name=self.library_name, + library_version=self.library_version, + cache_dir=cache_dir, + local_dir=local_dir, + user_agent=self.user_agent, + force_download=force_download, + etag_timeout=etag_timeout, + token=token, + headers=self.headers, + local_files_only=local_files_only, + tqdm_class=tqdm_class, + dry_run=dry_run, + ) + + @overload + def snapshot_download( + self, + repo_id: str, + *, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + force_download: bool = False, + token: bool | str | None = None, + local_files_only: bool = False, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + max_workers: int = 8, + tqdm_class: type[base_tqdm] | None = None, + dry_run: Literal[False] = False, + ) -> str: ... + + @overload + def snapshot_download( + self, + repo_id: str, + *, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + force_download: bool = False, + token: bool | str | None = None, + local_files_only: bool = False, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + max_workers: int = 8, + tqdm_class: type[base_tqdm] | None = None, + dry_run: Literal[True], + ) -> list[DryRunFileInfo]: ... + + @validate_hf_hub_args + def snapshot_download( + self, + repo_id: str, + *, + repo_type: str | None = None, + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT, + force_download: bool = False, + token: bool | str | None = None, + local_files_only: bool = False, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + max_workers: int = 8, + tqdm_class: type[base_tqdm] | None = None, + dry_run: bool = False, + ) -> str | list[DryRunFileInfo]: + """Download repo files. + + Download a whole snapshot of a repo's files at the specified revision. This is useful when you want all files from + a repo, because you don't know which ones you will need a priori. All files are nested inside a folder in order + to keep their actual filename relative to that folder. You can also filter which files to download using + `allow_patterns` and `ignore_patterns`. + + If `local_dir` is provided, the file structure from the repo will be replicated in this location. When using this + option, the `cache_dir` will not be used and a `.cache/huggingface/` folder will be created at the root of `local_dir` + to store some metadata related to the downloaded files.While this mechanism is not as robust as the main + cache-system, it's optimized for regularly pulling the latest version of a repository. + + An alternative would be to clone the repo but this requires git and git-lfs to be installed and properly + configured. It is also not possible to filter which files to download when cloning a repository using git. + + Args: + repo_id (`str`): + A user or an organization name and a repo name separated by a `/`. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if downloading from a dataset or space, + `None` or `"model"` if downloading from a model. Default is `None`. + revision (`str`, *optional*): + An optional Git revision id which can be a branch name, a tag, or a + commit hash. + cache_dir (`str`, `Path`, *optional*): + Path to the folder where cached files are stored. + local_dir (`str` or `Path`, *optional*): + If provided, the downloaded files will be placed under this directory. + etag_timeout (`float`, *optional*, defaults to `10`): + When fetching ETag, how many seconds to wait for the server to send + data before giving up which is passed to `httpx.request`. + force_download (`bool`, *optional*, defaults to `False`): + Whether the file should be downloaded even if it already exists in the local cache. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, avoid downloading the file and return the path to the + local cached file if it exists. + allow_patterns (`list[str]` or `str`, *optional*): + If provided, only files matching at least one pattern are downloaded. + ignore_patterns (`list[str]` or `str`, *optional*): + If provided, files matching any of the patterns are not downloaded. + max_workers (`int`, *optional*): + Number of concurrent threads to download files (1 thread = 1 file download). + Defaults to 8. + tqdm_class (`tqdm`, *optional*): + If provided, overwrites the default behavior for the progress bar. Passed + argument must inherit from `tqdm.auto.tqdm` or at least mimic its behavior. + Note that the `tqdm_class` is not passed to each individual download. + Defaults to the custom HF progress bar that can be disabled by setting + `HF_HUB_DISABLE_PROGRESS_BARS` environment variable. + dry_run (`bool`, *optional*, defaults to `False`): + If `True`, perform a dry run without actually downloading the files. Returns a list of + [`DryRunFileInfo`] objects containing information about what would be downloaded. + + Returns: + `str` or list of [`DryRunFileInfo`]: + - If `dry_run=False`: Folder path of the repo snapshot. + - If `dry_run=True`: A list of [`DryRunFileInfo`] objects containing download information. + + Raises: + [`~utils.RepositoryNotFoundError`] + If the repository to download from cannot be found. This may be because it doesn't exist, + or because it is set to `private` and you do not have access. + [`~utils.RevisionNotFoundError`] + If the revision to download from cannot be found. + [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) + If `token=True` and the token cannot be found. + [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if + ETag cannot be determined. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + if some parameter value is invalid. + """ + from ._snapshot_download import snapshot_download + + if token is None: + # Cannot do `token = token or self.token` as token can be `False`. + token = self.token + + return snapshot_download( + repo_id=repo_id, + repo_type=repo_type, + revision=revision, + endpoint=self.endpoint, + cache_dir=cache_dir, + local_dir=local_dir, + library_name=self.library_name, + library_version=self.library_version, + user_agent=self.user_agent, + etag_timeout=etag_timeout, + force_download=force_download, + token=token, + local_files_only=local_files_only, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + max_workers=max_workers, + tqdm_class=tqdm_class, + headers=self.headers, + dry_run=dry_run, + ) + + def get_safetensors_metadata( + self, + repo_id: str, + *, + repo_type: str | None = None, + revision: str | None = None, + token: bool | str | None = None, + timeout: float | None = constants.HF_HUB_DOWNLOAD_TIMEOUT, + ) -> SafetensorsRepoMetadata: + """ + Parse metadata for a safetensors repo on the Hub. + + We first check if the repo has a single safetensors file or a sharded safetensors repo. If it's a single + safetensors file, we parse the metadata from this file. If it's a sharded safetensors repo, we parse the + metadata from the index file and then parse the metadata from each shard. + + To parse metadata from a single safetensors file, use [`parse_safetensors_file_metadata`]. + + For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. + + Args: + repo_id (`str`): + A user or an organization name and a repo name separated by a `/`. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if the file is in a dataset or space, `None` or `"model"` if in a + model. Default is `None`. + revision (`str`, *optional*): + The git revision to fetch the file from. Can be a branch name, a tag, or a commit hash. Defaults to the + head of the `"main"` branch. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + timeout (`float`, *optional*, defaults to 10): + How many seconds to wait for the server to send data before giving up, passed to each request that + fetches a safetensors file header. Set to `None` to disable the timeout (not recommended, as a stalled + connection can hang the call indefinitely). + + Returns: + [`SafetensorsRepoMetadata`]: information related to safetensors repo. + + Raises: + [`NotASafetensorsRepoError`] + If the repo is not a safetensors repo i.e. doesn't have either a + `model.safetensors` or a `model.safetensors.index.json` file. + [`SafetensorsParsingError`] + If a safetensors file header couldn't be parsed correctly. + + Example: + ```py + # Parse repo with single weights file + >>> metadata = get_safetensors_metadata("bigscience/bloomz-560m") + >>> metadata + SafetensorsRepoMetadata( + metadata=None, + sharded=False, + weight_map={'h.0.input_layernorm.bias': 'model.safetensors', ...}, + files_metadata={'model.safetensors': SafetensorsFileMetadata(...)} + ) + >>> metadata.files_metadata["model.safetensors"].metadata + {'format': 'pt'} + + # Parse repo with sharded model + >>> metadata = get_safetensors_metadata("bigscience/bloom") + Parse safetensors files: 100%|██████████████████████████████████████████| 72/72 [00:12<00:00, 5.78it/s] + >>> metadata + SafetensorsRepoMetadata(metadata={'total_size': 352494542848}, sharded=True, weight_map={...}, files_metadata={...}) + >>> len(metadata.files_metadata) + 72 # All safetensors files have been fetched + + # Parse repo with sharded model + >>> get_safetensors_metadata("runwayml/stable-diffusion-v1-5") + NotASafetensorsRepoError: 'runwayml/stable-diffusion-v1-5' is not a safetensors repo. Couldn't find 'model.safetensors.index.json' or 'model.safetensors' files. + ``` + """ + if self.file_exists( # Single safetensors file => non-sharded model + repo_id=repo_id, + filename=constants.SAFETENSORS_SINGLE_FILE, + repo_type=repo_type, + revision=revision, + token=token, + ): + file_metadata = self.parse_safetensors_file_metadata( + repo_id=repo_id, + filename=constants.SAFETENSORS_SINGLE_FILE, + repo_type=repo_type, + revision=revision, + token=token, + timeout=timeout, + ) + return SafetensorsRepoMetadata( + metadata=None, + sharded=False, + weight_map={ + tensor_name: constants.SAFETENSORS_SINGLE_FILE for tensor_name in file_metadata.tensors.keys() + }, + files_metadata={constants.SAFETENSORS_SINGLE_FILE: file_metadata}, + ) + elif self.file_exists( # Multiple safetensors files => sharded with index + repo_id=repo_id, + filename=constants.SAFETENSORS_INDEX_FILE, + repo_type=repo_type, + revision=revision, + token=token, + ): + # Fetch index + index_file = self.hf_hub_download( + repo_id=repo_id, + filename=constants.SAFETENSORS_INDEX_FILE, + repo_type=repo_type, + revision=revision, + token=token, + ) + with open(index_file) as f: + index = json.load(f) + + weight_map = index.get("weight_map", {}) + + # Fetch metadata per shard + files_metadata = {} + + def _parse(filename: str) -> None: + files_metadata[filename] = self.parse_safetensors_file_metadata( + repo_id=repo_id, + filename=filename, + repo_type=repo_type, + revision=revision, + token=token, + timeout=timeout, + ) + + hf_thread_map( + _parse, + set(weight_map.values()), + desc="Parse safetensors files", + tqdm_class=hf_tqdm, + ) + + return SafetensorsRepoMetadata( + metadata=index.get("metadata", None), + sharded=True, + weight_map=weight_map, + files_metadata=files_metadata, + ) + else: + # Not a safetensors repo + raise NotASafetensorsRepoError( + f"'{repo_id}' is not a safetensors repo. Couldn't find '{constants.SAFETENSORS_INDEX_FILE}' or '{constants.SAFETENSORS_SINGLE_FILE}' files." + ) + + def parse_safetensors_file_metadata( + self, + repo_id: str, + filename: str, + *, + repo_type: str | None = None, + revision: str | None = None, + token: bool | str | None = None, + timeout: float | None = constants.HF_HUB_DOWNLOAD_TIMEOUT, + ) -> SafetensorsFileMetadata: + """ + Parse metadata from a safetensors file on the Hub. + + To parse metadata from all safetensors files in a repo at once, use [`get_safetensors_metadata`]. + + For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. + + Args: + repo_id (`str`): + A user or an organization name and a repo name separated by a `/`. + filename (`str`): + The name of the file in the repo. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if the file is in a dataset or space, `None` or `"model"` if in a + model. Default is `None`. + revision (`str`, *optional*): + The git revision to fetch the file from. Can be a branch name, a tag, or a commit hash. Defaults to the + head of the `"main"` branch. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + timeout (`float`, *optional*, defaults to 10): + How many seconds to wait for the server to send data before giving up. Set to `None` to disable the + timeout (not recommended, as a stalled connection can hang the call indefinitely). + + Returns: + [`SafetensorsFileMetadata`]: information related to a safetensors file. + + Raises: + [`NotASafetensorsRepoError`]: + If the repo is not a safetensors repo i.e. doesn't have either a + `model.safetensors` or a `model.safetensors.index.json` file. + [`SafetensorsParsingError`]: + If a safetensors file header couldn't be parsed correctly. + """ + url = hf_hub_url( + repo_id=repo_id, filename=filename, repo_type=repo_type, revision=revision, endpoint=self.endpoint + ) + _headers = self._build_hf_headers(token=token) + + context_msg = f"repo '{repo_id}', revision '{revision or constants.DEFAULT_REVISION}'" + + # 1. Fetch first 100kb + # Empirically, 97% of safetensors files have a metadata size < 100kb (over the top 1000 models on the Hub). + # We assume fetching 100kb is faster than making 2 GET requests. Therefore we always fetch the first 100kb to + # avoid the 2nd GET in most cases. + # See https://github.com/huggingface/huggingface_hub/pull/1855#discussion_r1404286419. + response = get_session().get(url, headers={**_headers, "range": "bytes=0-100000"}, timeout=timeout) + hf_raise_for_status(response) + + # 2. Parse and validate metadata size using shared helper + metadata_size = _get_safetensors_metadata_size(response.content[:8], filename, context_msg) + + # 3.a. Get metadata from payload + if metadata_size <= 100000: + metadata_as_bytes = response.content[8 : 8 + metadata_size] + else: # 3.b. Request full metadata + response = get_session().get( + url, headers={**_headers, "range": f"bytes=8-{metadata_size + 7}"}, timeout=timeout + ) + hf_raise_for_status(response) + metadata_as_bytes = response.content + + # 4. Parse json header using shared helper + return _parse_safetensors_header(metadata_as_bytes, filename, context_msg) + + @validate_hf_hub_args + def create_branch( + self, + repo_id: str, + *, + branch: str, + revision: str | None = None, + token: bool | str | None = None, + repo_type: str | None = None, + exist_ok: bool = False, + ) -> None: + """ + Create a new branch for a repo on the Hub, starting from the specified revision (defaults to `main`). + To find a revision suiting your needs, you can use [`list_repo_refs`] or [`list_repo_commits`]. + + Args: + repo_id (`str`): + The repository in which the branch will be created. + Example: `"user/my-cool-model"`. + + branch (`str`): + The name of the branch to create. + + revision (`str`, *optional*): + The git revision to create the branch from. It can be a branch name or + the OID/SHA of a commit, as a hexadecimal string. Defaults to the head + of the `"main"` branch. + + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if creating a branch on a dataset or + space, `None` or `"model"` if tagging a model. Default is `None`. + + exist_ok (`bool`, *optional*, defaults to `False`): + If `True`, do not raise an error if branch already exists. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If repository is not found (error 404): wrong repo_id/repo_type, private + but not authenticated or repo does not exist. + [`~utils.BadRequestError`]: + If invalid reference for a branch. Ex: `refs/pr/5` or 'refs/foo/bar'. + [`~utils.HfHubHTTPError`]: + If the branch already exists on the repo (error 409) and `exist_ok` is + set to `False`. + """ + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + branch = quote(branch, safe="") + + # Prepare request + branch_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/branch/{branch}" + headers = self._build_hf_headers(token=token) + payload = {} + if revision is not None: + payload["startingPoint"] = revision + + # Create branch + response = get_session().post(url=branch_url, headers=headers, json=payload) + try: + hf_raise_for_status(response) + except HfHubHTTPError as e: + if exist_ok and e.response.status_code == 409: + return + elif exist_ok and e.response.status_code == 403: + # No write permission on the namespace but branch might already exist + try: + refs = self.list_repo_refs(repo_id=repo_id, repo_type=repo_type, token=token) + for branch_ref in refs.branches: + if branch_ref.name == branch: + return # Branch already exists => do not raise + except HfHubHTTPError: + pass # We raise the original error if the branch does not exist + raise + + @validate_hf_hub_args + def delete_branch( + self, + repo_id: str, + *, + branch: str, + token: bool | str | None = None, + repo_type: str | None = None, + ) -> None: + """ + Delete a branch from a repo on the Hub. + + Args: + repo_id (`str`): + The repository in which a branch will be deleted. + Example: `"user/my-cool-model"`. + + branch (`str`): + The name of the branch to delete. + + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if creating a branch on a dataset or + space, `None` or `"model"` if tagging a model. Default is `None`. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If repository is not found (error 404): wrong repo_id/repo_type, private + but not authenticated or repo does not exist. + [`~utils.HfHubHTTPError`]: + If trying to delete a protected branch. Ex: `main` cannot be deleted. + [`~utils.HfHubHTTPError`]: + If trying to delete a branch that does not exist. + + """ + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + branch = quote(branch, safe="") + + # Prepare request + branch_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/branch/{branch}" + headers = self._build_hf_headers(token=token) + + # Delete branch + response = get_session().delete(url=branch_url, headers=headers) + hf_raise_for_status(response) + + @validate_hf_hub_args + def create_tag( + self, + repo_id: str, + *, + tag: str, + tag_message: str | None = None, + revision: str | None = None, + token: bool | str | None = None, + repo_type: str | None = None, + exist_ok: bool = False, + ) -> None: + """ + Tag a given commit of a repo on the Hub. + + Args: + repo_id (`str`): + The repository in which a commit will be tagged. + Example: `"user/my-cool-model"`. + + tag (`str`): + The name of the tag to create. + + tag_message (`str`, *optional*): + The description of the tag to create. + + revision (`str`, *optional*): + The git revision to tag. It can be a branch name or the OID/SHA of a + commit, as a hexadecimal string. Shorthands (7 first characters) are + also supported. Defaults to the head of the `"main"` branch. + + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if tagging a dataset or + space, `None` or `"model"` if tagging a model. Default is + `None`. + + exist_ok (`bool`, *optional*, defaults to `False`): + If `True`, do not raise an error if tag already exists. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If repository is not found (error 404): wrong repo_id/repo_type, private + but not authenticated or repo does not exist. + [`~utils.RevisionNotFoundError`]: + If revision is not found (error 404) on the repo. + [`~utils.HfHubHTTPError`]: + If the branch already exists on the repo (error 409) and `exist_ok` is + set to `False`. + """ + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION + + # Prepare request + tag_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tag/{revision}" + headers = self._build_hf_headers(token=token) + payload = {"tag": tag} + if tag_message is not None: + payload["message"] = tag_message + + # Tag + response = get_session().post(url=tag_url, headers=headers, json=payload) + try: + hf_raise_for_status(response) + except HfHubHTTPError as e: + if not (e.response.status_code == 409 and exist_ok): + raise + + @validate_hf_hub_args + def delete_tag( + self, + repo_id: str, + *, + tag: str, + token: bool | str | None = None, + repo_type: str | None = None, + ) -> None: + """ + Delete a tag from a repo on the Hub. + + Args: + repo_id (`str`): + The repository in which a tag will be deleted. + Example: `"user/my-cool-model"`. + + tag (`str`): + The name of the tag to delete. + + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if tagging a dataset or space, `None` or + `"model"` if tagging a model. Default is `None`. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If repository is not found (error 404): wrong repo_id/repo_type, private + but not authenticated or repo does not exist. + [`~utils.RevisionNotFoundError`]: + If tag is not found. + """ + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + tag = quote(tag, safe="") + + # Prepare request + tag_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tag/{tag}" + headers = self._build_hf_headers(token=token) + + # Un-tag + response = get_session().delete(url=tag_url, headers=headers) + hf_raise_for_status(response) + + @validate_hf_hub_args + def get_full_repo_name( + self, + model_id: str, + *, + organization: str | None = None, + token: bool | str | None = None, + ): + """ + Returns the repository name for a given model ID and optional + organization. + + Args: + model_id (`str`): + The name of the model. + organization (`str`, *optional*): + If passed, the repository name will be in the organization + namespace instead of the user namespace. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `str`: The repository name in the user's namespace + ({username}/{model_id}) if no organization is passed, and under the + organization namespace ({organization}/{model_id}) otherwise. + """ + if organization is None: + if "/" in model_id: + username = model_id.split("/")[0] + else: + username = self.whoami(token=token)["name"] # type: ignore + return f"{username}/{model_id}" + else: + return f"{organization}/{model_id}" + + @validate_hf_hub_args + def get_repo_discussions( + self, + repo_id: str, + *, + author: str | None = None, + discussion_type: constants.DiscussionTypeFilter | None = None, + discussion_status: constants.DiscussionStatusFilter | None = None, + repo_type: str | None = None, + token: bool | str | None = None, + ) -> Iterator[Discussion]: + """ + Fetches Discussions and Pull Requests for the given repo. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + author (`str`, *optional*): + Pass a value to filter by discussion author. `None` means no filter. + Default is `None`. + discussion_type (`str`, *optional*): + Set to `"pull_request"` to fetch only pull requests, `"discussion"` + to fetch only discussions. Set to `"all"` or `None` to fetch both. + Default is `None`. + discussion_status (`str`, *optional*): + Set to `"open"` (respectively `"closed"`) to fetch only open + (respectively closed) discussions. Set to `"all"` or `None` + to fetch both. + Default is `None`. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if fetching from a dataset or + space, `None` or `"model"` if fetching from a model. Default is + `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterator[Discussion]`: An iterator of [`Discussion`] objects. + + Example: + Collecting all discussions of a repo in a list: + + ```python + >>> from huggingface_hub import get_repo_discussions + >>> discussions_list = list(get_repo_discussions(repo_id="bert-base-uncased")) + ``` + + Iterating over discussions of a repo: + + ```python + >>> from huggingface_hub import get_repo_discussions + >>> for discussion in get_repo_discussions(repo_id="bert-base-uncased"): + ... print(discussion.num, discussion.title) + ``` + """ + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + if discussion_type is not None and discussion_type not in constants.DISCUSSION_TYPES: + raise ValueError(f"Invalid discussion_type, must be one of {constants.DISCUSSION_TYPES}") + if discussion_status is not None and discussion_status not in constants.DISCUSSION_STATUS: + raise ValueError(f"Invalid discussion_status, must be one of {constants.DISCUSSION_STATUS}") + + headers = self._build_hf_headers(token=token) + path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions" + + params: dict[str, str | int] = {} + if discussion_type is not None: + params["type"] = discussion_type + if discussion_status is not None: + params["status"] = discussion_status + if author is not None: + params["author"] = author + + def _fetch_discussion_page(page_index: int): + params["p"] = page_index + resp = get_session().get(path, headers=headers, params=params) + hf_raise_for_status(resp) + paginated_discussions = resp.json() + total = paginated_discussions["count"] + start = paginated_discussions["start"] + discussions = paginated_discussions["discussions"] + has_next = (start + len(discussions)) < total + return discussions, has_next + + has_next, page_index = True, 0 + + while has_next: + discussions, has_next = _fetch_discussion_page(page_index=page_index) + for discussion in discussions: + yield Discussion( + title=discussion["title"], + num=discussion["num"], + author=discussion.get("author", {}).get("name", "deleted"), + created_at=parse_datetime(discussion["createdAt"]), + status=discussion["status"], + repo_id=discussion["repo"]["name"], + repo_type=discussion["repo"]["type"], + is_pull_request=discussion["isPullRequest"], + endpoint=self.endpoint, + ) + page_index = page_index + 1 + + @validate_hf_hub_args + def get_discussion_details( + self, + repo_id: str, + discussion_num: int, + *, + repo_type: str | None = None, + token: bool | str | None = None, + ) -> DiscussionWithDetails: + """Fetches a Discussion's / Pull Request 's details from the Hub. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + discussion_num (`int`): + The number of the Discussion or Pull Request . Must be a strictly positive integer. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: [`DiscussionWithDetails`] + + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + """ + if not isinstance(discussion_num, int) or discussion_num <= 0: + raise ValueError("Invalid discussion_num, must be a positive integer") + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + + path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions/{discussion_num}" + headers = self._build_hf_headers(token=token) + resp = get_session().get(path, params={"diff": "1"}, headers=headers) + hf_raise_for_status(resp) + + discussion_details = resp.json() + is_pull_request = discussion_details["isPullRequest"] + + target_branch = discussion_details["changes"]["base"] if is_pull_request else None + conflicting_files = discussion_details["filesWithConflicts"] if is_pull_request else None + merge_commit_oid = discussion_details["changes"].get("mergeCommitId", None) if is_pull_request else None + + return DiscussionWithDetails( + title=discussion_details["title"], + num=discussion_details["num"], + author=discussion_details.get("author", {}).get("name", "deleted"), + created_at=parse_datetime(discussion_details["createdAt"]), + status=discussion_details["status"], + repo_id=discussion_details["repo"]["name"], + repo_type=discussion_details["repo"]["type"], + is_pull_request=discussion_details["isPullRequest"], + events=[deserialize_event(evt) for evt in discussion_details["events"]], + conflicting_files=conflicting_files, + target_branch=target_branch, + merge_commit_oid=merge_commit_oid, + diff=discussion_details.get("diff"), + endpoint=self.endpoint, + ) + + @validate_hf_hub_args + def create_discussion( + self, + repo_id: str, + title: str, + *, + token: bool | str | None = None, + description: str | None = None, + repo_type: str | None = None, + pull_request: bool = False, + ) -> DiscussionWithDetails: + """Creates a Discussion or Pull Request. + + Pull Requests created programmatically will be in `"draft"` status. + + Creating a Pull Request with changes can also be done at once with [`HfApi.create_commit`]. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + title (`str`): + The title of the discussion. It can be up to 200 characters long, + and must be at least 3 characters long. Leading and trailing whitespaces + will be stripped. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + description (`str`, *optional*): + An optional description for the Pull Request. + Defaults to `"Discussion opened with the huggingface_hub Python library"` + pull_request (`bool`, *optional*): + Whether to create a Pull Request or discussion. If `True`, creates a Pull Request. + If `False`, creates a discussion. Defaults to `False`. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + + Returns: [`DiscussionWithDetails`] + + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access.""" + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + + if description is not None: + description = description.strip() + description = ( + description + if description + else ( + f"{'Pull Request' if pull_request else 'Discussion'} opened with the" + " [huggingface_hub Python" + " library](https://huggingface.co/docs/huggingface_hub)" + ) + ) + + headers = self._build_hf_headers(token=token) + resp = get_session().post( + f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions", + json={ + "title": title.strip(), + "description": description, + "pullRequest": pull_request, + }, + headers=headers, + ) + hf_raise_for_status(resp) + num = resp.json()["num"] + return self.get_discussion_details( + repo_id=repo_id, + repo_type=repo_type, + discussion_num=num, + token=token, + ) + + @validate_hf_hub_args + def create_pull_request( + self, + repo_id: str, + title: str, + *, + token: bool | str | None = None, + description: str | None = None, + repo_type: str | None = None, + ) -> DiscussionWithDetails: + """Creates a Pull Request . Pull Requests created programmatically will be in `"draft"` status. + + Creating a Pull Request with changes can also be done at once with [`HfApi.create_commit`]; + + This is a wrapper around [`HfApi.create_discussion`]. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + title (`str`): + The title of the discussion. It can be up to 200 characters long, + and must be at least 3 characters long. Leading and trailing whitespaces + will be stripped. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + description (`str`, *optional*): + An optional description for the Pull Request. + Defaults to `"Discussion opened with the huggingface_hub Python library"` + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + + Returns: [`DiscussionWithDetails`] + + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access.""" + return self.create_discussion( + repo_id=repo_id, + title=title, + token=token, + description=description, + repo_type=repo_type, + pull_request=True, + ) + + def _post_discussion_changes( + self, + *, + repo_id: str, + discussion_num: int, + resource: str, + body: dict | None = None, + token: bool | str | None = None, + repo_type: str | None = None, + ) -> httpx.Response: + """Internal utility to POST changes to a Discussion or Pull Request""" + if not isinstance(discussion_num, int) or discussion_num <= 0: + raise ValueError("Invalid discussion_num, must be a positive integer") + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + repo_id = f"{repo_type}s/{repo_id}" + + path = f"{self.endpoint}/api/{repo_id}/discussions/{discussion_num}/{resource}" + + headers = self._build_hf_headers(token=token) + resp = get_session().post(path, headers=headers, json=body) + hf_raise_for_status(resp) + return resp + + @validate_hf_hub_args + def comment_discussion( + self, + repo_id: str, + discussion_num: int, + comment: str, + *, + token: bool | str | None = None, + repo_type: str | None = None, + ) -> DiscussionComment: + """Creates a new comment on the given Discussion. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + discussion_num (`int`): + The number of the Discussion or Pull Request . Must be a strictly positive integer. + comment (`str`): + The content of the comment to create. Comments support markdown formatting. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`DiscussionComment`]: the newly created comment + + + Examples: + ```python + + >>> comment = \"\"\" + ... Hello @otheruser! + ... + ... # This is a title + ... + ... **This is bold**, *this is italic* and ~this is strikethrough~ + ... And [this](http://url) is a link + ... \"\"\" + + >>> HfApi().comment_discussion( + ... repo_id="username/repo_name", + ... discussion_num=34 + ... comment=comment + ... ) + # DiscussionComment(id='deadbeef0000000', type='comment', ...) + + ``` + + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + """ + resp = self._post_discussion_changes( + repo_id=repo_id, + repo_type=repo_type, + discussion_num=discussion_num, + token=token, + resource="comment", + body={"comment": comment}, + ) + return deserialize_event(resp.json()["newMessage"]) # type: ignore + + @validate_hf_hub_args + def rename_discussion( + self, + repo_id: str, + discussion_num: int, + new_title: str, + *, + token: bool | str | None = None, + repo_type: str | None = None, + ) -> DiscussionTitleChange: + """Renames a Discussion. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + discussion_num (`int`): + The number of the Discussion or Pull Request . Must be a strictly positive integer. + new_title (`str`): + The new title for the discussion + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`DiscussionTitleChange`]: the title change event + + + Examples: + ```python + >>> new_title = "New title, fixing a typo" + >>> HfApi().rename_discussion( + ... repo_id="username/repo_name", + ... discussion_num=34 + ... new_title=new_title + ... ) + # DiscussionTitleChange(id='deadbeef0000000', type='title-change', ...) + + ``` + + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + """ + resp = self._post_discussion_changes( + repo_id=repo_id, + repo_type=repo_type, + discussion_num=discussion_num, + token=token, + resource="title", + body={"title": new_title}, + ) + return deserialize_event(resp.json()["newTitle"]) # type: ignore + + @validate_hf_hub_args + def change_discussion_status( + self, + repo_id: str, + discussion_num: int, + new_status: Literal["open", "closed"], + *, + token: bool | str | None = None, + comment: str | None = None, + repo_type: str | None = None, + ) -> DiscussionStatusChange: + """Closes or re-opens a Discussion or Pull Request. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + discussion_num (`int`): + The number of the Discussion or Pull Request . Must be a strictly positive integer. + new_status (`str`): + The new status for the discussion, either `"open"` or `"closed"`. + comment (`str`, *optional*): + An optional comment to post with the status change. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`DiscussionStatusChange`]: the status change event + + + Examples: + ```python + >>> new_title = "New title, fixing a typo" + >>> HfApi().rename_discussion( + ... repo_id="username/repo_name", + ... discussion_num=34 + ... new_title=new_title + ... ) + # DiscussionStatusChange(id='deadbeef0000000', type='status-change', ...) + + ``` + + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + """ + if new_status not in ["open", "closed"]: + raise ValueError("Invalid status, valid statuses are: 'open' and 'closed'") + body: dict[str, str] = {"status": new_status} + if comment and comment.strip(): + body["comment"] = comment.strip() + resp = self._post_discussion_changes( + repo_id=repo_id, + repo_type=repo_type, + discussion_num=discussion_num, + token=token, + resource="status", + body=body, + ) + return deserialize_event(resp.json()["newStatus"]) # type: ignore + + @validate_hf_hub_args + def merge_pull_request( + self, + repo_id: str, + discussion_num: int, + *, + token: bool | str | None = None, + comment: str | None = None, + repo_type: str | None = None, + ): + """Merges a Pull Request. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + discussion_num (`int`): + The number of the Discussion or Pull Request . Must be a strictly positive integer. + comment (`str`, *optional*): + An optional comment to post with the status change. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`DiscussionStatusChange`]: the status change event + + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + """ + self._post_discussion_changes( + repo_id=repo_id, + repo_type=repo_type, + discussion_num=discussion_num, + token=token, + resource="merge", + body={"comment": comment.strip()} if comment and comment.strip() else None, + ) + + @validate_hf_hub_args + def edit_discussion_comment( + self, + repo_id: str, + discussion_num: int, + comment_id: str, + new_content: str, + *, + token: bool | str | None = None, + repo_type: str | None = None, + ) -> DiscussionComment: + """Edits a comment on a Discussion / Pull Request. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + discussion_num (`int`): + The number of the Discussion or Pull Request . Must be a strictly positive integer. + comment_id (`str`): + The ID of the comment to edit. + new_content (`str`): + The new content of the comment. Comments support markdown formatting. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`DiscussionComment`]: the edited comment + + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + """ + resp = self._post_discussion_changes( + repo_id=repo_id, + repo_type=repo_type, + discussion_num=discussion_num, + token=token, + resource=f"comment/{comment_id.lower()}/edit", + body={"content": new_content}, + ) + return deserialize_event(resp.json()["updatedComment"]) # type: ignore + + @validate_hf_hub_args + def hide_discussion_comment( + self, + repo_id: str, + discussion_num: int, + comment_id: str, + *, + token: bool | str | None = None, + repo_type: str | None = None, + ) -> DiscussionComment: + """Hides a comment on a Discussion / Pull Request. + + > [!WARNING] + > Hidden comments' content cannot be retrieved anymore. Hiding a comment is irreversible. + + Args: + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + discussion_num (`int`): + The number of the Discussion or Pull Request . Must be a strictly positive integer. + comment_id (`str`): + The ID of the comment to edit. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if uploading to a dataset or + space, `None` or `"model"` if uploading to a model. Default is + `None`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`DiscussionComment`]: the hidden comment + + > [!TIP] + > Raises the following errors: + > + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the HuggingFace API returned an error + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if some parameter value is invalid + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it doesn't exist, + > or because it is set to `private` and you do not have access. + """ + warnings.warn( + "Hidden comments' content cannot be retrieved anymore. Hiding a comment is irreversible.", + UserWarning, + ) + resp = self._post_discussion_changes( + repo_id=repo_id, + repo_type=repo_type, + discussion_num=discussion_num, + token=token, + resource=f"comment/{comment_id.lower()}/hide", + ) + return deserialize_event(resp.json()["updatedComment"]) # type: ignore + + @validate_hf_hub_args + def add_space_secret( + self, + repo_id: str, + key: str, + value: str, + *, + description: str | None = None, + token: bool | str | None = None, + ) -> None: + """Adds or updates a secret in a Space. + + Secrets allow to set secret keys or tokens to a Space without hardcoding them. + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets. + + Args: + repo_id (`str`): + ID of the repo to update. Example: `"bigcode/in-the-stack"`. + key (`str`): + Secret key. Example: `"GITHUB_API_KEY"` + value (`str`): + Secret value. Example: `"your_github_api_key"`. + description (`str`, *optional*): + Secret description. Example: `"Github API key to access the Github API"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + """ + payload = {"key": key, "value": value} + if description is not None: + payload["description"] = description + r = get_session().post( + f"{self.endpoint}/api/spaces/{repo_id}/secrets", + headers=self._build_hf_headers(token=token), + json=payload, + ) + hf_raise_for_status(r) + + @validate_hf_hub_args + def delete_space_secret(self, repo_id: str, key: str, *, token: bool | str | None = None) -> None: + """Deletes a secret from a Space. + + Secrets allow to set secret keys or tokens to a Space without hardcoding them. + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets. + + Args: + repo_id (`str`): + ID of the repo to update. Example: `"bigcode/in-the-stack"`. + key (`str`): + Secret key. Example: `"GITHUB_API_KEY"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + """ + r = get_session().request( + "DELETE", + f"{self.endpoint}/api/spaces/{repo_id}/secrets", + headers=self._build_hf_headers(token=token), + json={"key": key}, + ) + hf_raise_for_status(r) + + @validate_hf_hub_args + def get_space_secrets(self, repo_id: str, *, token: bool | str | None = None) -> dict[str, SpaceSecret]: + """Gets all secrets from a Space. + + Secret values are write-only and cannot be read back. Only the key, description, and last update time + are returned. + + Secrets allow to set secret keys or tokens to a Space without hardcoding them. + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets. + + Args: + repo_id (`str`): + ID of the repo to query. Example: `"bigcode/in-the-stack"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `dict[str, SpaceSecret]`: Dictionary of [`SpaceSecret`] objects keyed by secret name. + + Example: + ```python + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> api.get_space_secrets("username/my-space") + {'HF_TOKEN': SpaceSecret(key='HF_TOKEN', description='...', updated_at=datetime.datetime(...))} + ``` + """ + r = get_session().get( + f"{self.endpoint}/api/spaces/{repo_id}/secrets", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(r) + return {k: SpaceSecret(k, v) for k, v in r.json().items()} + + @validate_hf_hub_args + def get_space_variables(self, repo_id: str, *, token: bool | str | None = None) -> dict[str, SpaceVariable]: + """Gets all variables from a Space. + + Variables allow to set environment variables to a Space without hardcoding them. + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets-and-environment-variables + + Args: + repo_id (`str`): + ID of the repo to query. Example: `"bigcode/in-the-stack"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + """ + r = get_session().get( + f"{self.endpoint}/api/spaces/{repo_id}/variables", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(r) + return {k: SpaceVariable(k, v) for k, v in r.json().items()} + + @validate_hf_hub_args + def add_space_variable( + self, + repo_id: str, + key: str, + value: str, + *, + description: str | None = None, + token: bool | str | None = None, + ) -> dict[str, SpaceVariable]: + """Adds or updates a variable in a Space. + + Variables allow to set environment variables to a Space without hardcoding them. + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets-and-environment-variables + + Args: + repo_id (`str`): + ID of the repo to update. Example: `"bigcode/in-the-stack"`. + key (`str`): + Variable key. Example: `"MODEL_REPO_ID"` + value (`str`): + Variable value. Example: `"the_model_repo_id"`. + description (`str`): + Description of the variable. Example: `"Model Repo ID of the implemented model"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + """ + payload = {"key": key, "value": value} + if description is not None: + payload["description"] = description + r = get_session().post( + f"{self.endpoint}/api/spaces/{repo_id}/variables", + headers=self._build_hf_headers(token=token), + json=payload, + ) + hf_raise_for_status(r) + return {k: SpaceVariable(k, v) for k, v in r.json().items()} + + @validate_hf_hub_args + def delete_space_variable( + self, repo_id: str, key: str, *, token: bool | str | None = None + ) -> dict[str, SpaceVariable]: + """Deletes a variable from a Space. + + Variables allow to set environment variables to a Space without hardcoding them. + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets-and-environment-variables + + Args: + repo_id (`str`): + ID of the repo to update. Example: `"bigcode/in-the-stack"`. + key (`str`): + Variable key. Example: `"MODEL_REPO_ID"` + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + """ + r = get_session().request( + "DELETE", + f"{self.endpoint}/api/spaces/{repo_id}/variables", + headers=self._build_hf_headers(token=token), + json={"key": key}, + ) + hf_raise_for_status(r) + return {k: SpaceVariable(k, v) for k, v in r.json().items()} + + @validate_hf_hub_args + def get_space_runtime(self, repo_id: str, *, token: bool | str | None = None) -> SpaceRuntime: + """Gets runtime information about a Space. + + Args: + repo_id (`str`): + ID of the repo to update. Example: `"bigcode/in-the-stack"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + Returns: + [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware. + """ + r = get_session().get( + f"{self.endpoint}/api/spaces/{repo_id}/runtime", headers=self._build_hf_headers(token=token) + ) + hf_raise_for_status(r) + return SpaceRuntime(r.json()) + + def list_spaces_hardware(self, token: bool | str | None = None) -> list[JobHardwareInfo]: + """List available hardware options for Spaces. + + Returns: + `list[JobHardwareInfo]`: A list of available hardware configurations. + + Example: + + ```python + >>> from huggingface_hub import list_spaces_hardware + >>> hardware_list = list_spaces_hardware() + >>> hardware_list[0] + JobHardwareInfo(name='cpu-basic', pretty_name='CPU Basic', cpu='2 vCPU', ram='16 GB', ...) + >>> hardware_list[0].name + 'cpu-basic' + ``` + """ + response = get_session().get( + f"{self.endpoint}/api/spaces/hardware", headers=self._build_hf_headers(token=token) + ) + hf_raise_for_status(response) + return [JobHardwareInfo(**hardware) for hardware in response.json()] + + @validate_hf_hub_args + def request_space_hardware( + self, + repo_id: str, + hardware: SpaceHardware, + *, + token: bool | str | None = None, + sleep_time: int | None = None, + ) -> SpaceRuntime: + """Request new hardware for a Space. + + Args: + repo_id (`str`): + ID of the repo to update. Example: `"bigcode/in-the-stack"`. + hardware (`str` or [`SpaceHardware`]): + Hardware on which to run the Space. Example: `"t4-medium"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + sleep_time (`int`, *optional*): + Number of seconds of inactivity to wait before a Space is put to sleep. Set to `-1` if you don't want + your Space to sleep (default behavior for upgraded hardware). For free hardware, you can't configure + the sleep time (value is fixed to 48 hours of inactivity). + See https://huggingface.co/docs/hub/spaces-gpus#sleep-time for more details. + Returns: + [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware. + + > [!TIP] + > It is also possible to request hardware directly when creating the Space repo! See [`create_repo`] for details. + """ + if sleep_time is not None and hardware == SpaceHardware.CPU_BASIC: + warnings.warn( + "If your Space runs on the default 'cpu-basic' hardware, it will go to sleep if inactive for more" + " than 48 hours. This value is not configurable. If you don't want your Space to deactivate or if" + " you want to set a custom sleep time, you need to upgrade to a paid Hardware.", + UserWarning, + ) + payload: dict[str, Any] = {"flavor": hardware} + if sleep_time is not None: + payload["sleepTimeSeconds"] = sleep_time + r = get_session().post( + f"{self.endpoint}/api/spaces/{repo_id}/hardware", + headers=self._build_hf_headers(token=token), + json=payload, + ) + hf_raise_for_status(r) + return SpaceRuntime(r.json()) + + @validate_hf_hub_args + def set_space_sleep_time(self, repo_id: str, sleep_time: int, *, token: bool | str | None = None) -> SpaceRuntime: + """Set a custom sleep time for a Space running on upgraded hardware.. + + Your Space will go to sleep after X seconds of inactivity. You are not billed when your Space is in "sleep" + mode. If a new visitor lands on your Space, it will "wake it up". Only upgraded hardware can have a + configurable sleep time. To know more about the sleep stage, please refer to + https://huggingface.co/docs/hub/spaces-gpus#sleep-time. + + Args: + repo_id (`str`): + ID of the repo to update. Example: `"bigcode/in-the-stack"`. + sleep_time (`int`, *optional*): + Number of seconds of inactivity to wait before a Space is put to sleep. Set to `-1` if you don't want + your Space to pause (default behavior for upgraded hardware). For free hardware, you can't configure + the sleep time (value is fixed to 48 hours of inactivity). + See https://huggingface.co/docs/hub/spaces-gpus#sleep-time for more details. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + Returns: + [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware. + + > [!TIP] + > It is also possible to set a custom sleep time when requesting hardware with [`request_space_hardware`]. + """ + r = get_session().post( + f"{self.endpoint}/api/spaces/{repo_id}/sleeptime", + headers=self._build_hf_headers(token=token), + json={"seconds": sleep_time}, + ) + hf_raise_for_status(r) + runtime = SpaceRuntime(r.json()) + + hardware = runtime.requested_hardware or runtime.hardware + if hardware == SpaceHardware.CPU_BASIC: + warnings.warn( + "If your Space runs on the default 'cpu-basic' hardware, it will go to sleep if inactive for more" + " than 48 hours. This value is not configurable. If you don't want your Space to deactivate or if" + " you want to set a custom sleep time, you need to upgrade to a paid Hardware.", + UserWarning, + ) + return runtime + + @validate_hf_hub_args + def pause_space(self, repo_id: str, *, token: bool | str | None = None) -> SpaceRuntime: + """Pause your Space. + + A paused Space stops executing until manually restarted by its owner. This is different from the sleeping + state in which free Spaces go after 48h of inactivity. Paused time is not billed to your account, no matter the + hardware you've selected. To restart your Space, use [`restart_space`] and go to your Space settings page. + + For more details, please visit [the docs](https://huggingface.co/docs/hub/spaces-gpus#pause). + + Args: + repo_id (`str`): + ID of the Space to pause. Example: `"Salesforce/BLIP2"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`SpaceRuntime`]: Runtime information about your Space including `stage=PAUSED` and requested hardware. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If your Space is not found (error 404). Most probably wrong repo_id or your space is private but you + are not authenticated. + [`~utils.HfHubHTTPError`]: + 403 Forbidden: only the owner of a Space can pause it. If you want to manage a Space that you don't + own, either ask the owner by opening a Discussion or duplicate the Space. + [`~utils.BadRequestError`]: + If your Space is a static Space. Static Spaces are always running and never billed. If you want to hide + a static Space, you can set it to private. + """ + r = get_session().post( + f"{self.endpoint}/api/spaces/{repo_id}/pause", headers=self._build_hf_headers(token=token) + ) + hf_raise_for_status(r) + return SpaceRuntime(r.json()) + + @validate_hf_hub_args + def enable_space_dev_mode(self, repo_id: str, *, token: bool | str | None = None) -> SpaceRuntime: + """Enable dev mode on a Space. + + Spaces Dev Mode eases the debugging of your application and makes iterating on Spaces faster by allowing you + to restart your application without stopping the Space container itself. This feature is available as part of + a PRO or Team & Enterprise plan. See https://huggingface.co/docs/hub/spaces-dev-mode for more details. + + Args: + repo_id (`str`): + ID of the Space to enable dev mode. Example: `"Salesforce/BLIP2"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`SpaceRuntime`]: Runtime information about your Space. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If your Space is not found (error 404). Most probably wrong repo_id or your space is private but you + are not authenticated. + [`~utils.HfHubHTTPError`]: + 403 Forbidden: only the owner of a Space can set dev mode. If you want to handle a Space that you don't + own, either ask the owner by opening a Discussion or duplicate the Space. + [`~utils.BadRequestError`]: + If your Space is a static Space. Static Spaces are always running and never billed. If you want to hide + a static Space, you can set it to private. + """ + r = get_session().post( + f"{self.endpoint}/api/spaces/{repo_id}/dev-mode", + headers=self._build_hf_headers(token=token), + json={"enabled": True}, + ) + hf_raise_for_status(r) + return SpaceRuntime(r.json()) + + @validate_hf_hub_args + def disable_space_dev_mode( + self, + repo_id: str, + *, + token: bool | str | None = None, + ) -> SpaceRuntime: + """Disable dev mode on a Space. + + Spaces Dev Mode eases the debugging of your application and makes iterating on Spaces faster by allowing you + to restart your application without stopping the Space container itself. This feature is available as part of + a PRO or Team & Enterprise plan. See https://huggingface.co/docs/hub/spaces-dev-mode for more details. + + Args: + repo_id (`str`): + ID of the Space to disable dev mode. Example: `"Salesforce/BLIP2"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`SpaceRuntime`]: Runtime information about your Space. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If your Space is not found (error 404). Most probably wrong repo_id or your space is private but you + are not authenticated. + [`~utils.HfHubHTTPError`]: + 403 Forbidden: only the owner of a Space can set dev mode. If you want to handle a Space that you don't + own, either ask the owner by opening a Discussion or duplicate the Space. + [`~utils.BadRequestError`]: + If your Space is a static Space. Static Spaces are always running and never billed. If you want to hide + a static Space, you can set it to private. + """ + r = get_session().post( + f"{self.endpoint}/api/spaces/{repo_id}/dev-mode", + headers=self._build_hf_headers(token=token), + json={"enabled": False}, + ) + hf_raise_for_status(r) + return SpaceRuntime(r.json()) + + @validate_hf_hub_args + def restart_space( + self, repo_id: str, *, token: bool | str | None = None, factory_reboot: bool = False + ) -> SpaceRuntime: + """Restart your Space. + + This is the only way to programmatically restart a Space if you've put it on Pause (see [`pause_space`]). You + must be the owner of the Space to restart it. If you are using an upgraded hardware, your account will be + billed as soon as the Space is restarted. You can trigger a restart no matter the current state of a Space. + + For more details, please visit [the docs](https://huggingface.co/docs/hub/spaces-gpus#pause). + + Args: + repo_id (`str`): + ID of the Space to restart. Example: `"Salesforce/BLIP2"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + factory_reboot (`bool`, *optional*): + If `True`, the Space will be rebuilt from scratch without caching any requirements. + + Returns: + [`SpaceRuntime`]: Runtime information about your Space. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If your Space is not found (error 404). Most probably wrong repo_id or your space is private but you + are not authenticated. + [`~utils.HfHubHTTPError`]: + 403 Forbidden: only the owner of a Space can restart it. If you want to restart a Space that you don't + own, either ask the owner by opening a Discussion or duplicate the Space. + [`~utils.BadRequestError`]: + If your Space is a static Space. Static Spaces are always running and never billed. If you want to hide + a static Space, you can set it to private. + """ + params = {} + if factory_reboot: + params["factory"] = "true" + r = get_session().post( + f"{self.endpoint}/api/spaces/{repo_id}/restart", headers=self._build_hf_headers(token=token), params=params + ) + hf_raise_for_status(r) + return SpaceRuntime(r.json()) + + def _stream_sse_events( + self, + *, + url: str, + log_label: str, + timeout: int, + follow: bool, + token: bool | str | None = None, + skip_previous_events_on_retry: bool = True, + tolerated_status_codes: tuple[int, ...] = (), + tolerated_exception_types: tuple[type[Exception], ...] = (), + on_iteration_end: Callable[[], bool] | None = None, + params: dict[str, Any] | None = None, + ) -> Iterable[dict[str, Any]]: + # Shared SSE streaming loop with retry/backoff and event-index dedup. + # Used by Spaces logs and Jobs logs/metrics. Two retry styles: + # - on_iteration_end is None: retries are the only backstop (Spaces). + # - on_iteration_end is set: it polls authoritative state after every + # failed iteration; ReadTimeouts/tolerated errors fall through to it + # instead of consuming retries (Jobs). + nb_tries = 0 + max_retries = 5 if follow else 0 + min_wait_time = 1 + max_wait_time = 10 + sleep_time = 0 + start_event_idx = 0 + error_to_retry: Exception | None = None + while True: + if error_to_retry is not None: + logger.warning(f"'{error_to_retry}' thrown while requesting {log_label}") + logger.warning(f"Retrying in {sleep_time}s [Retry {nb_tries}/{max_retries}].") + error_to_retry = None + time.sleep(sleep_time) + try: + with get_session().stream( + "GET", + url, + headers=self._build_hf_headers(token=token), + timeout=timeout, + params=params, + ) as response: + if response.status_code == 200: + event_idx = -1 + for line in response.iter_lines(): + if line and line.startswith("data: {"): + event_idx += 1 + if event_idx >= start_event_idx: + if skip_previous_events_on_retry: + start_event_idx += 1 + yield json.loads(line[len("data: ") :]) + break + elif response.status_code not in tolerated_status_codes: + hf_raise_for_status(response) + except HfHubHTTPError: + # Permanent HTTP error (404/403/...). Never retry — fail fast. + raise + except httpx.DecodingError: + # Response ended prematurely. + break + except KeyboardInterrupt: + break + except (httpx.HTTPError, httpcore.TimeoutException) as err: + is_no_new_line_timeout = isinstance(err, (httpx.ReadTimeout, httpcore.ReadTimeout)) + if is_no_new_line_timeout and not follow: + break # no-follow: timeout means the buffer is drained + if on_iteration_end is not None: + # Authoritative-state mode: ReadTimeouts and tolerated errors + # fall through to the post-iteration check without consuming + # retries. Note: ReadTimeout is handled here regardless of + # `tolerated_exception_types` — entries in that tuple only + # fire for non-timeout errors. + if is_no_new_line_timeout or type(err) in tolerated_exception_types: + pass + elif nb_tries >= max_retries: + raise + else: + nb_tries += 1 + sleep_time = min(max_wait_time, max(min_wait_time, sleep_time * 2)) + error_to_retry = err + else: + # Retry-only mode: every error in follow mode burns a retry. + if nb_tries >= max_retries: + if is_no_new_line_timeout: + break # follow mode, silent stream, retries exhausted: give up + raise + nb_tries += 1 + sleep_time = min(max_wait_time, max(min_wait_time, sleep_time * 2)) + error_to_retry = err + # Drop params after the first attempt: the start_event_idx dedup + # requires a stable replay prefix, which `tail` would break. + params = None + if on_iteration_end is not None and on_iteration_end(): + break + + def _fetch_space_logs_sse( + self, + *, + repo_id: str, + build: bool, + timeout: int, + follow: bool, + token: bool | str | None = None, + ) -> Iterable[dict[str, Any]]: + log_type = "build" if build else "run" + yield from self._stream_sse_events( + url=f"{self.endpoint}/api/spaces/{repo_id}/logs/{log_type}", + log_label=f"spaces /logs/{log_type} for repo_id={repo_id!r}", + timeout=timeout, + follow=follow, + token=token, + ) + + @validate_hf_hub_args + def fetch_space_logs( + self, + repo_id: str, + *, + build: bool = False, + follow: bool = False, + token: bool | str | None = None, + ) -> Iterable[str]: + """Fetch the run or build logs of a Space on the Hub. + + Useful for debugging a Space that is failing to build or crashing at runtime, + especially from a script or agentic workflow where reading logs in a browser + is not an option. + + Args: + repo_id (`str`): + ID of the Space. Example: `"bigcode/in-the-stack"`. + build (`bool`, *optional*, defaults to `False`): + If `True`, fetch the container build logs (useful when a Space is stuck + in `BUILD_ERROR`). If `False` (default), fetch the run logs, i.e. the + stdout/stderr of the running application. + follow (`bool`, *optional*, defaults to `False`): + If `True`, stream logs in real-time (blocking) until the server closes + the stream or `KeyboardInterrupt` is raised. If `False` (default), fetch + only the currently buffered logs and return immediately (non-blocking, + like `docker logs`). + token (`bool` or `str`, *optional*): + A valid user access token. Defaults to the locally saved token, which is + the recommended authentication method. Set to `False` to disable + authentication. See + https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Returns: + `Iterable[str]`: A generator yielding log lines as they become available. + + Example: + + ```python + >>> from huggingface_hub import fetch_space_logs + >>> # Non-blocking: print currently available run logs and exit. + >>> for line in fetch_space_logs("username/my-space"): + ... print(line, end="") + + >>> # Debug a build failure: + >>> for line in fetch_space_logs("username/my-space", build=True): + ... print(line, end="") + + >>> # Stream run logs until the server closes the stream. + >>> for line in fetch_space_logs("username/my-space", follow=True): + ... print(line, end="") + ``` + """ + # - Spaces /logs/{run|build} is SSE with `data: {"data": "...", "timestamp": "..."}` events. + # - Keep-alive messages are sent as empty `data:` events (skipped by the `data: {` filter). + # - In no-follow mode we use a short read timeout to drain the buffer and return. + timeout = 120 if follow else 5 + for event in self._fetch_space_logs_sse( + repo_id=repo_id, + build=build, + timeout=timeout, + follow=follow, + token=token, + ): + yield event["data"] + + @validate_hf_hub_args + def wait_for_space( + self, + repo_id: str, + *, + timeout: float | None = None, + poll_interval: float = 1.0, + token: bool | str | None = None, + ) -> SpaceRuntime: + """Wait until a Space reaches a terminal stage (not building/starting). + + Polls [`get_space_runtime`] every `poll_interval` seconds until the Space's stage + is no longer intermediate (`BUILDING`, `RUNNING_BUILDING`, `APP_STARTING`, + `RUNNING_APP_STARTING`). Returns the final [`SpaceRuntime`] in all cases — check + `runtime.stage` to act on the outcome (e.g. `RUNNING` vs `BUILD_ERROR`). + + Args: + repo_id (`str`): + ID of the Space to wait for. Example: `"username/my-space"`. + timeout (`float`, *optional*): + Maximum time to wait in seconds. If `None`, waits indefinitely. + poll_interval (`float`, *optional*): + Seconds between status checks. Defaults to 1s. + token (`bool` or `str`, *optional*): + A valid user access token. Defaults to the locally saved token, which is the + recommended authentication method. Set to `False` to disable authentication. + See https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Returns: + [`SpaceRuntime`]: The final runtime information once the Space reaches a terminal stage. + + Raises: + `TimeoutError`: + If the Space has not reached a terminal stage after `timeout` seconds. + + Example: + + ```python + >>> from huggingface_hub import restart_space, wait_for_space + >>> restart_space("username/my-space") + >>> runtime = wait_for_space("username/my-space") + >>> runtime.stage + 'RUNNING' + ``` + """ + if timeout is not None and timeout < 0: + raise ValueError("`timeout` cannot be negative.") + if poll_interval <= 0: + raise ValueError("`poll_interval` must be positive.") + + deadline = None if timeout is None else time.monotonic() + timeout + while True: + runtime = self.get_space_runtime(repo_id, token=token) + if runtime.stage not in INTERMEDIATE_SPACE_STAGES: + return runtime + remaining = None if deadline is None else deadline - time.monotonic() + if remaining is not None and remaining <= 0: + raise TimeoutError(f"Space '{repo_id}' is still in stage '{runtime.stage}' after {timeout} seconds.") + time.sleep(poll_interval if remaining is None else min(poll_interval, remaining)) + + @_deprecate_arguments( + version="2.0", + deprecated_args={"space_storage"}, + custom_message="Use `space_volumes` to mount volumes on a Space.", + ) + @validate_hf_hub_args + def duplicate_repo( + self, + from_id: str, + to_id: str | None = None, + *, + repo_type: str | None = None, + private: bool | None = None, + visibility: RepoVisibility_T | None = None, + token: bool | str | None = None, + exist_ok: bool = False, + space_hardware: SpaceHardware | None = None, + space_storage: SpaceStorage | None = None, + space_sleep_time: int | None = None, + space_secrets: list[dict[str, str]] | None = None, + space_variables: list[dict[str, str]] | None = None, + space_volumes: list[Volume] | None = None, + ) -> RepoUrl: + """Duplicate a repo on the Hub (model, dataset, or Space). + + This performs a server-side copy that preserves full git history and LFS objects + without requiring a local download/upload round-trip. + + Args: + from_id (`str`): + ID of the repo to duplicate. Example: `"openai/gdpval"`. + to_id (`str`, *optional*): + ID of the new repo. Example: `"myorg/my-gdpval"`. If not provided, the new + repo will have the same name as the original repo, but in your account. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if duplicating a dataset or Space, + `None` or `"model"` if duplicating a model. Default is `None`. + private (`bool`, *optional*): + Whether the new repo should be private or not. Defaults to the same + privacy as the original repo. Cannot be passed together with `visibility`. + visibility (`Literal["public", "private", "protected"]`, *optional*): + Visibility of the new repo. Can be `"public"` or `"private"`, or `"protected"` for Spaces. Defaults + to the same visibility as the original repo. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + exist_ok (`bool`, *optional*, defaults to `False`): + If `True`, do not raise an error if repo already exists. + space_hardware (`SpaceHardware` or `str`, *optional*): + Choice of Hardware if repo_type is "space". Example: `"t4-medium"`. See + [`SpaceHardware`] for a complete list. + space_storage (`SpaceStorage` or `str`, *optional*): + Choice of persistent storage tier if repo_type is "space". Example: + `"small"`. See [`SpaceStorage`] for a complete list. + space_sleep_time (`int`, *optional*): + Number of seconds of inactivity to wait before a Space is put to sleep. + Set to `-1` if you don't want your Space to sleep (default behavior for + upgraded hardware). For free hardware, you can't configure the sleep time + (value is fixed to 48 hours of inactivity). Only applicable if repo_type is "space". + See https://huggingface.co/docs/hub/spaces-gpus#sleep-time for more details. + space_secrets (`list[dict[str, str]]`, *optional*): + A list of secret keys to set in your Space. Each item is in the form + `{"key": ..., "value": ..., "description": ...}` where description is optional. + Only applicable if repo_type is "space". + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets. + space_variables (`list[dict[str, str]]`, *optional*): + A list of public environment variables to set in your Space. Each item is in + the form `{"key": ..., "value": ..., "description": ...}` where description + is optional. Only applicable if repo_type is "space". + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets-and-environment-variables. + space_volumes (`list[Volume]`, *optional*): + A list of [`Volume`] objects to mount in the Space at duplication time. Each volume has a `type` + (`"bucket"`, `"model"`, `"dataset"`, or `"space"`), a `source` (repo or bucket ID), a `mount_path` + (path inside the container), and optional `revision`, `read_only`, and `path` fields. + Only applicable if repo_type is "space". + + Returns: + [`RepoUrl`]: URL to the newly created repo. Value is a subclass of `str` containing + attributes like `endpoint`, `repo_type` and `repo_id`. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If one of `from_id` or `to_id` cannot be found. This may be because it doesn't exist, + or because it is set to `private` and you do not have access. + [`HfHubHTTPError`]: + If the HuggingFace API returned an error + + Example: + ```python + >>> from huggingface_hub import duplicate_repo + + # Duplicate a model to your account + >>> duplicate_repo("google/gemma-7b") + RepoUrl('https://huggingface.co/nateraw/gemma-7b',...) + + # Duplicate a dataset with a custom name + >>> duplicate_repo("openai/gdpval", to_id="myorg/my-gdpval", repo_type="dataset") + RepoUrl('https://huggingface.co/datasets/myorg/my-gdpval',...) + + # Duplicate a Space with custom hardware + >>> duplicate_repo("multimodalart/dreambooth-training", repo_type="space", space_hardware="t4-medium") + RepoUrl('https://huggingface.co/spaces/nateraw/dreambooth-training',...) + ``` + """ + if repo_type not in constants.REPO_TYPES: + raise ValueError("Invalid repo type") + + resolved_visibility = _resolve_repo_visibility(private=private, visibility=visibility, repo_type=repo_type) + + # Map repo_type to API path segment + api_prefix = { + None: "models", + constants.REPO_TYPE_MODEL: "models", + constants.REPO_TYPE_DATASET: "datasets", + constants.REPO_TYPE_SPACE: "spaces", + }[repo_type] + + # Resolve the target namespace + name. When 'to_id' is provided we take the name from it, + # otherwise we reuse the source name. Both may be a URL, an 'hf://' URI, or a bare id, so we + # try 'parse_hf_uri' first and fall back to a plain split ('/', or just + # '' for 'to_id'). A missing namespace defaults to the caller's account. + to_namespace: str | None = None + if to_id is not None: + try: + to_namespace, to_repo_name = parse_hf_uri(to_id).id.split("/") + except HfUriError: + namespace_and_name = to_id.rsplit("/", 1) + to_namespace = namespace_and_name[0] if len(namespace_and_name) == 2 else None + to_repo_name = namespace_and_name[-1] + else: + try: + to_repo_name = parse_hf_uri(from_id).id.split("/")[1] + except HfUriError: + to_repo_name = from_id.rsplit("/", 1)[-1] + if to_namespace is None: + to_namespace = self.whoami(token)["name"] + + payload: dict[str, Any] = {"repository": f"{to_namespace}/{to_repo_name}"} + + if resolved_visibility is not None: + payload["visibility"] = resolved_visibility + + # Space-specific options + space_args: list[tuple[str, str, Any]] = [ + # input arg, payload key, value + ("space_hardware", "hardware", space_hardware), + ("space_storage", "storageTier", space_storage), + ("space_sleep_time", "sleepTimeSeconds", space_sleep_time), + ("space_secrets", "secrets", space_secrets), + ("space_variables", "variables", space_variables), + ("space_volumes", "volumes", [v.to_dict() for v in space_volumes] if space_volumes else None), + ] + + if repo_type == "space": + for _, key, value in space_args: + if value is not None: + payload[key] = value + if space_sleep_time is not None and space_hardware == SpaceHardware.CPU_BASIC: + warnings.warn( + "If your Space runs on the default 'cpu-basic' hardware, it will go to sleep if inactive for more" + " than 48 hours. This value is not configurable. If you don't want your Space to deactivate or if" + " you want to set a custom sleep time, you need to upgrade to a paid Hardware.", + UserWarning, + ) + else: + if provided_space_args := [arg for arg, _, value in space_args if value is not None]: + warnings.warn(f"Ignoring provided {', '.join(provided_space_args)} because repo_type is not 'space'.") + + r = get_session().post( + f"{self.endpoint}/api/{api_prefix}/{from_id}/duplicate", + headers=self._build_hf_headers(token=token), + json=payload, + ) + + try: + hf_raise_for_status(r) + except HfHubHTTPError as err: + if exist_ok and err.response.status_code == 409: + pass + else: + raise + + return RepoUrl(r.json()["url"], endpoint=self.endpoint) + + @_deprecate_method(version="2.0", message="Use `duplicate_repo` instead.") + @validate_hf_hub_args + def duplicate_space( + self, + from_id: str, + to_id: str | None = None, + *, + private: bool | None = None, + visibility: RepoVisibility_T | None = None, + token: bool | str | None = None, + exist_ok: bool = False, + hardware: SpaceHardware | None = None, + storage: SpaceStorage | None = None, + sleep_time: int | None = None, + secrets: list[dict[str, str]] | None = None, + variables: list[dict[str, str]] | None = None, + ) -> RepoUrl: + """Duplicate a Space. + + Programmatically duplicate a Space. The new Space will be created in your account and will be in the same state + as the original Space (running or paused). You can duplicate a Space no matter the current state of a Space. + + Args: + from_id (`str`): + ID of the Space to duplicate. Example: `"pharma/CLIP-Interrogator"`. + to_id (`str`, *optional*): + ID of the new Space. Example: `"dog/CLIP-Interrogator"`. If not provided, the new Space will have the same + name as the original Space, but in your account. + private (`bool`, *optional*): + Whether the new Space should be private or not. Defaults to the same privacy as the original Space. Cannot be passed together with `visibility`. + visibility (`Literal["public", "private", "protected"]`, *optional*): + Visibility of the new Space. Can be `"public"`, `"private"`, or `"protected"`. Defaults to the same + visibility as the original Space. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + exist_ok (`bool`, *optional*, defaults to `False`): + If `True`, do not raise an error if repo already exists. + hardware (`SpaceHardware` or `str`, *optional*): + Choice of Hardware. Example: `"t4-medium"`. See [`SpaceHardware`] for a complete list. + storage (`SpaceStorage` or `str`, *optional*): + Choice of persistent storage tier. Example: `"small"`. See [`SpaceStorage`] for a complete list. + sleep_time (`int`, *optional*): + Number of seconds of inactivity to wait before a Space is put to sleep. Set to `-1` if you don't want + your Space to sleep (default behavior for upgraded hardware). For free hardware, you can't configure + the sleep time (value is fixed to 48 hours of inactivity). + See https://huggingface.co/docs/hub/spaces-gpus#sleep-time for more details. + secrets (`list[dict[str, str]]`, *optional*): + A list of secret keys to set in your Space. Each item is in the form `{"key": ..., "value": ..., "description": ...}` where description is optional. + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets. + variables (`list[dict[str, str]]`, *optional*): + A list of public environment variables to set in your Space. Each item is in the form `{"key": ..., "value": ..., "description": ...}` where description is optional. + For more details, see https://huggingface.co/docs/hub/spaces-overview#managing-secrets-and-environment-variables. + + Returns: + [`RepoUrl`]: URL to the newly created repo. Value is a subclass of `str` containing + attributes like `endpoint`, `repo_type` and `repo_id`. + + Raises: + [`~utils.RepositoryNotFoundError`]: + If one of `from_id` or `to_id` cannot be found. This may be because it doesn't exist, + or because it is set to `private` and you do not have access. + [`HfHubHTTPError`]: + If the HuggingFace API returned an error + + Example: + ```python + >>> from huggingface_hub import duplicate_space + + # Duplicate a Space to your account + >>> duplicate_space("multimodalart/dreambooth-training") + RepoUrl('https://huggingface.co/spaces/nateraw/dreambooth-training',...) + + # Can set custom destination id and visibility flag. + >>> duplicate_space("multimodalart/dreambooth-training", to_id="my-dreambooth", visibility="private") + RepoUrl('https://huggingface.co/spaces/nateraw/my-dreambooth',...) + ``` + + > [!WARNING] + > `duplicate_space` is deprecated and will be removed in version 2.0. Use [`~HfApi.duplicate_repo`] instead. + """ + kwargs: dict[str, Any] = {} + if to_id is not None: + kwargs["to_id"] = to_id + return self.duplicate_repo( + from_id=from_id, + repo_type="space", + private=private, + visibility=visibility, + token=token, + exist_ok=exist_ok, + space_hardware=hardware, + space_storage=storage, + space_sleep_time=sleep_time, + space_secrets=secrets, + space_variables=variables, + **kwargs, + ) + + @_deprecate_method(version="2.0", message="Use `set_space_volumes` instead.") + @validate_hf_hub_args + def request_space_storage( + self, + repo_id: str, + storage: SpaceStorage, + *, + token: bool | str | None = None, + ) -> SpaceRuntime: + """Request persistent storage for a Space. + + > [!WARNING] + > `request_space_storage` is deprecated and will be removed in version 2.0. Use [`set_space_volumes`] instead. + + Args: + repo_id (`str`): + ID of the Space to update. Example: `"open-llm-leaderboard/open_llm_leaderboard"`. + storage (`str` or [`SpaceStorage`]): + Storage tier. Either 'small', 'medium', or 'large'. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + Returns: + [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware. + """ + payload: dict[str, SpaceStorage] = {"tier": storage} + r = get_session().post( + f"{self.endpoint}/api/spaces/{repo_id}/storage", + headers=self._build_hf_headers(token=token), + json=payload, + ) + hf_raise_for_status(r) + return SpaceRuntime(r.json()) + + @_deprecate_method(version="2.0", message="Use `delete_space_volumes` instead.") + @validate_hf_hub_args + def delete_space_storage( + self, + repo_id: str, + *, + token: bool | str | None = None, + ) -> SpaceRuntime: + """Delete persistent storage for a Space. + + > [!WARNING] + > `delete_space_storage` is deprecated and will be removed in version 2.0. Use [`delete_space_volumes`] instead. + + Args: + repo_id (`str`): + ID of the Space to update. Example: `"open-llm-leaderboard/open_llm_leaderboard"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + Returns: + [`SpaceRuntime`]: Runtime information about a Space including Space stage and hardware. + Raises: + [`BadRequestError`] + If space has no persistent storage. + """ + r = get_session().delete( + f"{self.endpoint}/api/spaces/{repo_id}/storage", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(r) + return SpaceRuntime(r.json()) + + @validate_hf_hub_args + def set_space_volumes( + self, + repo_id: str, + volumes: list[Volume], + *, + token: bool | str | None = None, + ) -> None: + """Set volumes for a Space. + + Sets (or replaces) the list of volumes mounted in the Space. Each volume gives the Space's container access + to a Hub resource (model, dataset, or storage bucket). + + Args: + repo_id (`str`): + ID of the Space to update. Example: `"username/my-space"`. + volumes (`list[Volume]`): + List of [`Volume`] objects to mount. Each volume has a `type` (`"bucket"`, `"model"`, `"dataset"`, or + `"space"`), a `source` (repo or bucket ID), a `mount_path` (path inside the container), and optional + `revision`, `read_only`, and `path` fields. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Raises: + [`BadRequestError`]: + If the Space is a static Space (volumes are not supported on static Spaces). + + Example: + ```python + >>> from huggingface_hub import HfApi, Volume + >>> api = HfApi() + >>> api.set_space_volumes( + ... "username/my-space", + ... volumes=[ + ... Volume(type="model", source="username/my-model", mount_path="/models", read_only=True), + ... Volume(type="bucket", source="username/my-bucket", mount_path="/data"), + ... ], + ... ) + ``` + """ + payload = {"volumes": [vol.to_dict() for vol in volumes]} + r = get_session().put( + f"{self.endpoint}/api/spaces/{repo_id}/volumes", + headers=self._build_hf_headers(token=token), + json=payload, + ) + hf_raise_for_status(r) + + @validate_hf_hub_args + def delete_space_volumes( + self, + repo_id: str, + *, + token: bool | str | None = None, + ) -> None: + """Remove all volumes from a Space. + + Args: + repo_id (`str`): + ID of the Space to update. Example: `"username/my-space"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Raises: + [`BadRequestError`]: + If the Space has no volumes attached. + + Example: + ```python + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> api.delete_space_volumes("username/my-space") + ``` + """ + r = get_session().delete( + f"{self.endpoint}/api/spaces/{repo_id}/volumes", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(r) + + ####################### + # Inference Endpoints # + ####################### + + def list_inference_endpoints( + self, namespace: str | None = None, *, token: bool | str | None = None + ) -> list[InferenceEndpoint]: + """Lists all inference endpoints for the given namespace. + + Args: + namespace (`str`, *optional*): + The namespace to list endpoints for. Defaults to the current user. Set to `"*"` to list all endpoints + from all namespaces (i.e. personal namespace and all orgs the user belongs to). + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + list[`InferenceEndpoint`]: A list of all inference endpoints for the given namespace. + + Example: + ```python + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> api.list_inference_endpoints() + [InferenceEndpoint(name='my-endpoint', ...), ...] + ``` + """ + # Special case: list all endpoints for all namespaces the user has access to + if namespace == "*": + user = self.whoami(token=token) + + # List personal endpoints first + endpoints: list[InferenceEndpoint] = list_inference_endpoints(namespace=self._get_namespace(token=token)) + + # Then list endpoints for all orgs the user belongs to and ignore 401 errors (no billing or no access) + for org in user.get("orgs", []): + try: + endpoints += list_inference_endpoints(namespace=org["name"], token=token) + except HfHubHTTPError as error: + if error.response.status_code == 401: # Either no billing or user don't have access) + logger.debug("Cannot list Inference Endpoints for org '%s': %s", org["name"], error) + pass + + return endpoints + + # Normal case: list endpoints for a specific namespace + namespace = namespace or self._get_namespace(token=token) + + response = get_session().get( + f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + + return [ + InferenceEndpoint.from_raw(endpoint, namespace=namespace, token=token) + for endpoint in response.json()["items"] + ] + + def create_inference_endpoint( + self, + name: str, + *, + repository: str, + framework: str, + accelerator: str, + instance_size: str, + instance_type: str, + region: str, + vendor: str, + account_id: str | None = None, + min_replica: int = 1, + max_replica: int = 1, + scaling_metric: InferenceEndpointScalingMetric | None = None, + scaling_threshold: float | None = None, + scale_to_zero_timeout: int | None = None, + revision: str | None = None, + task: str | None = None, + custom_image: dict | None = None, + container_command: list[str] | None = None, + container_args: list[str] | None = None, + env: dict[str, str] | None = None, + secrets: dict[str, str] | None = None, + type: InferenceEndpointType | str = InferenceEndpointType.AUTHENTICATED, + domain: str | None = None, + path: str | None = None, + cache_http_responses: bool | None = None, + tags: list[str] | None = None, + namespace: str | None = None, + token: bool | str | None = None, + ) -> InferenceEndpoint: + """Create a new Inference Endpoint. + + Args: + name (`str`): + The unique name for the new Inference Endpoint. + repository (`str`): + The name of the model repository associated with the Inference Endpoint (e.g. `"gpt2"`). + framework (`str`): + The machine learning framework used for the model (e.g. `"custom"`). + accelerator (`str`): + The hardware accelerator to be used for inference (e.g. `"cpu"`). + instance_size (`str`): + The size or type of the instance to be used for hosting the model (e.g. `"x4"`). + instance_type (`str`): + The cloud instance type where the Inference Endpoint will be deployed (e.g. `"intel-icl"`). + region (`str`): + The cloud region in which the Inference Endpoint will be created (e.g. `"us-east-1"`). + vendor (`str`): + The cloud provider or vendor where the Inference Endpoint will be hosted (e.g. `"aws"`). + account_id (`str`, *optional*): + The account ID used to link a VPC to a private Inference Endpoint (if applicable). + min_replica (`int`, *optional*): + The minimum number of replicas (instances) to keep running for the Inference Endpoint. To enable + scaling to zero, set this value to 0 and adjust `scale_to_zero_timeout` accordingly. Defaults to 1. + max_replica (`int`, *optional*): + The maximum number of replicas (instances) to scale to for the Inference Endpoint. Defaults to 1. + scaling_metric (`str` or [`InferenceEndpointScalingMetric `], *optional*): + The metric reference for scaling. Either "pendingRequests" or "hardwareUsage" when provided. Defaults to + None (meaning: let the HF Endpoints service specify the metric). + scaling_threshold (`float`, *optional*): + The scaling metric threshold used to trigger a scale up. Ignored when scaling metric is not provided. + Defaults to None (meaning: let the HF Endpoints service specify the threshold). + scale_to_zero_timeout (`int`, *optional*): + The duration in minutes before an inactive endpoint is scaled to zero, or no scaling to zero if + set to None and `min_replica` is not 0. Defaults to None. + revision (`str`, *optional*): + The specific model revision to deploy on the Inference Endpoint (e.g. `"6c0e6080953db56375760c0471a8c5f2929baf11"`). + task (`str`, *optional*): + The task on which to deploy the model (e.g. `"text-classification"`). + custom_image (`dict`, *optional*): + A custom Docker image to use for the Inference Endpoint. This is useful if you want to deploy an + Inference Endpoint running on the `text-generation-inference` (TGI) framework or a custom container + (see examples). + container_command (`list[str]`, *optional*): + Override the container entrypoint command (maps to `model.command` in the API payload). Typically + used together with `custom_image`. + container_args (`list[str]`, *optional*): + Arguments appended to the container entrypoint (maps to `model.args` in the API payload). Typically + used together with `custom_image` to pass runtime flags to the container. + env (`dict[str, str]`, *optional*): + Non-secret environment variables to inject in the container environment. + secrets (`dict[str, str]`, *optional*): + Secret values to inject in the container environment. + type ([`InferenceEndpointType]`, *optional*): + The type of the Inference Endpoint, which can be `"authenticated"` (default), `"public"` or + `"private"`. `"protected"` is deprecated in favor of `"authenticated"` and will be removed in a + future release. + domain (`str`, *optional*): + The custom domain for the Inference Endpoint deployment, if setup the inference endpoint will be available at this domain (e.g. `"my-new-domain.cool-website.woof"`). + path (`str`, *optional*): + The custom path to the deployed model, should start with a `/` (e.g. `"/models/google-bert/bert-base-uncased"`). + cache_http_responses (`bool`, *optional*): + Whether to cache HTTP responses from the Inference Endpoint. Defaults to `False`. + tags (`list[str]`, *optional*): + A list of tags to associate with the Inference Endpoint. + namespace (`str`, *optional*): + The namespace where the Inference Endpoint will be created. Defaults to the current user's namespace. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`InferenceEndpoint`]: information about the updated Inference Endpoint. + + Example: + ```python + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> endpoint = api.create_inference_endpoint( + ... "my-endpoint-name", + ... repository="gpt2", + ... framework="pytorch", + ... task="text-generation", + ... accelerator="cpu", + ... vendor="aws", + ... region="us-east-1", + ... type="authenticated", + ... instance_size="x2", + ... instance_type="intel-icl", + ... ) + >>> endpoint + InferenceEndpoint(name='my-endpoint-name', status="pending",...) + + # Run inference on the endpoint + >>> endpoint.client.text_generation(...) + "..." + ``` + + ```python + # Start an Inference Endpoint running Zephyr-7b-beta on TGI + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> endpoint = api.create_inference_endpoint( + ... "aws-zephyr-7b-beta-0486", + ... repository="HuggingFaceH4/zephyr-7b-beta", + ... framework="pytorch", + ... task="text-generation", + ... accelerator="gpu", + ... vendor="aws", + ... region="us-east-1", + ... type="authenticated", + ... instance_size="x1", + ... instance_type="nvidia-a10g", + ... env={ + ... "MAX_BATCH_PREFILL_TOKENS": "2048", + ... "MAX_INPUT_LENGTH": "1024", + ... "MAX_TOTAL_TOKENS": "1512", + ... "MODEL_ID": "/repository" + ... }, + ... custom_image={ + ... "healthRoute": "/health", + ... "url": "ghcr.io/huggingface/text-generation-inference:1.1.0", + ... }, + ... secrets={"MY_SECRET_KEY": "secret_value"}, + ... tags=["dev", "text-generation"], + ... ) + ``` + + ```python + # Start an Inference Endpoint running ProsusAI/finbert while scaling to zero in 15 minutes + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> endpoint = api.create_inference_endpoint( + ... "finbert-classifier", + ... repository="ProsusAI/finbert", + ... framework="pytorch", + ... task="text-classification", + ... min_replica=0, + ... scale_to_zero_timeout=15, + ... accelerator="cpu", + ... vendor="aws", + ... region="us-east-1", + ... type="authenticated", + ... instance_size="x2", + ... instance_type="intel-icl", + ... ) + >>> endpoint.wait(timeout=300) + # Run inference on the endpoint + >>> endpoint.client.text_generation(...) + TextClassificationOutputElement(label='positive', score=0.8983615040779114) + ``` + + """ + namespace = namespace or self._get_namespace(token=token) + + if type == InferenceEndpointType.PROTECTED: + warnings.warn( + "`type='protected'` is deprecated and will be removed in a future release. " + "Use `type='authenticated'` instead.", + FutureWarning, + ) + + if custom_image is not None: + image = ( + custom_image + if next(iter(custom_image)) in constants.INFERENCE_ENDPOINT_IMAGE_KEYS + else {"custom": custom_image} + ) + else: + image = {"huggingface": {}} + + payload: dict = { + "accountId": account_id, + "compute": { + "accelerator": accelerator, + "instanceSize": instance_size, + "instanceType": instance_type, + "scaling": { + "maxReplica": max_replica, + "minReplica": min_replica, + "scaleToZeroTimeout": scale_to_zero_timeout, + }, + }, + "model": { + "framework": framework, + "repository": repository, + "revision": revision, + "task": task, + "image": image, + }, + "name": name, + "provider": { + "region": region, + "vendor": vendor, + }, + "type": type, + } + if scaling_metric: + payload["compute"]["scaling"]["measure"] = {scaling_metric: scaling_threshold} # type: ignore + model_payload: dict[str, Any] = payload["model"] + if container_command is not None: + model_payload["command"] = container_command + if container_args is not None: + model_payload["args"] = container_args + if env: + payload["model"]["env"] = env + if secrets: + payload["model"]["secrets"] = secrets + if domain is not None or path is not None: + payload["route"] = {} + if domain is not None: + payload["route"]["domain"] = domain + if path is not None: + payload["route"]["path"] = path + if cache_http_responses is not None: + payload["cacheHttpResponses"] = cache_http_responses + if tags is not None: + payload["tags"] = tags + + response = get_session().post( + f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}", + headers=self._build_hf_headers(token=token), + json=payload, + ) + hf_raise_for_status(response) + + return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) + + @experimental + @validate_hf_hub_args + def create_inference_endpoint_from_catalog( + self, + repo_id: str, + *, + name: str | None = None, + accelerator: Literal["cpu", "gpu", "neuron"] | str | None = None, + token: bool | str | None = None, + namespace: str | None = None, + ) -> InferenceEndpoint: + """Create a new Inference Endpoint from a model in the Hugging Face Inference Catalog. + + The goal of the Inference Catalog is to provide a curated list of models that are optimized for inference + and for which default configurations have been tested. See https://endpoints.huggingface.co/catalog for a list + of available models in the catalog. + + Args: + repo_id (`str`): + The ID of the model in the catalog to deploy as an Inference Endpoint. + name (`str`, *optional*): + The unique name for the new Inference Endpoint. If not provided, a random name will be generated. + accelerator (`str`, *optional*): + The hardware accelerator to be used for inference. Possible values include `"cpu"`, `"gpu"`, and + `"neuron"`. If not provided, the server will use a default appropriate for the model. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + namespace (`str`, *optional*): + The namespace where the Inference Endpoint will be created. Defaults to the current user's namespace. + + Returns: + [`InferenceEndpoint`]: information about the new Inference Endpoint. + + > [!WARNING] + > `create_inference_endpoint_from_catalog` is experimental. Its API is subject to change in the future. Please provide feedback + > if you have any suggestions or requests. + """ + token = token or self.token or get_token() + payload: dict = { + "namespace": namespace or self._get_namespace(token=token), + "repoId": repo_id, + } + if name is not None: + payload["endpointName"] = name + if accelerator is not None: + payload["accelerator"] = accelerator + + response = get_session().post( + f"{constants.INFERENCE_CATALOG_ENDPOINT}/deploy", + headers=self._build_hf_headers(token=token), + json=payload, + ) + hf_raise_for_status(response) + data = response.json()["endpoint"] + return InferenceEndpoint.from_raw(data, namespace=data["name"], token=token) + + @experimental + @validate_hf_hub_args + def list_inference_catalog(self, *, token: bool | str | None = None) -> list[str]: + """List models available in the Hugging Face Inference Catalog. + + The goal of the Inference Catalog is to provide a curated list of models that are optimized for inference + and for which default configurations have been tested. See https://endpoints.huggingface.co/catalog for a list + of available models in the catalog. + + Use [`create_inference_endpoint_from_catalog`] to deploy a model from the catalog. + + Args: + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + + Returns: + List[`str`]: A list of model IDs available in the catalog. + > [!WARNING] + > `list_inference_catalog` is experimental. Its API is subject to change in the future. Please provide feedback + > if you have any suggestions or requests. + """ + response = get_session().get( + f"{constants.INFERENCE_CATALOG_ENDPOINT}/repo-list", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + return response.json()["models"] + + def get_inference_endpoint( + self, name: str, *, namespace: str | None = None, token: bool | str | None = None + ) -> InferenceEndpoint: + """Get information about an Inference Endpoint. + + Args: + name (`str`): + The name of the Inference Endpoint to retrieve information about. + namespace (`str`, *optional*): + The namespace in which the Inference Endpoint is located. Defaults to the current user. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`InferenceEndpoint`]: information about the requested Inference Endpoint. + + Example: + ```python + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> endpoint = api.get_inference_endpoint("my-text-to-image") + >>> endpoint + InferenceEndpoint(name='my-text-to-image', ...) + + # Get status + >>> endpoint.status + 'running' + >>> endpoint.url + 'https://my-text-to-image.region.vendor.endpoints.huggingface.cloud' + + # Run inference + >>> endpoint.client.text_to_image(...) + ``` + """ + namespace = namespace or self._get_namespace(token=token) + + response = get_session().get( + f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + + return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) + + def update_inference_endpoint( + self, + name: str, + *, + # Compute update + accelerator: str | None = None, + instance_size: str | None = None, + instance_type: str | None = None, + min_replica: int | None = None, + max_replica: int | None = None, + scale_to_zero_timeout: int | None = None, + scaling_metric: InferenceEndpointScalingMetric | None = None, + scaling_threshold: float | None = None, + # Model update + repository: str | None = None, + framework: str | None = None, + revision: str | None = None, + task: str | None = None, + custom_image: dict | None = None, + env: dict[str, str] | None = None, + secrets: dict[str, str] | None = None, + # Route update + domain: str | None = None, + path: str | None = None, + # Other + cache_http_responses: bool | None = None, + tags: list[str] | None = None, + namespace: str | None = None, + token: bool | str | None = None, + ) -> InferenceEndpoint: + """Update an Inference Endpoint. + + This method allows the update of either the compute configuration, the deployed model, the route, or any combination. + All arguments are optional but at least one must be provided. + + For convenience, you can also update an Inference Endpoint using [`InferenceEndpoint.update`]. + + Args: + name (`str`): + The name of the Inference Endpoint to update. + + accelerator (`str`, *optional*): + The hardware accelerator to be used for inference (e.g. `"cpu"`). + instance_size (`str`, *optional*): + The size or type of the instance to be used for hosting the model (e.g. `"x4"`). + instance_type (`str`, *optional*): + The cloud instance type where the Inference Endpoint will be deployed (e.g. `"intel-icl"`). + min_replica (`int`, *optional*): + The minimum number of replicas (instances) to keep running for the Inference Endpoint. + max_replica (`int`, *optional*): + The maximum number of replicas (instances) to scale to for the Inference Endpoint. + scale_to_zero_timeout (`int`, *optional*): + The duration in minutes before an inactive endpoint is scaled to zero. + scaling_metric (`str` or [`InferenceEndpointScalingMetric `], *optional*): + The metric reference for scaling. Either "pendingRequests" or "hardwareUsage" when provided. + Defaults to None. + scaling_threshold (`float`, *optional*): + The scaling metric threshold used to trigger a scale up. Ignored when scaling metric is not provided. + Defaults to None. + repository (`str`, *optional*): + The name of the model repository associated with the Inference Endpoint (e.g. `"gpt2"`). + framework (`str`, *optional*): + The machine learning framework used for the model (e.g. `"custom"`). + revision (`str`, *optional*): + The specific model revision to deploy on the Inference Endpoint (e.g. `"6c0e6080953db56375760c0471a8c5f2929baf11"`). + task (`str`, *optional*): + The task on which to deploy the model (e.g. `"text-classification"`). + custom_image (`dict`, *optional*): + A custom Docker image to use for the Inference Endpoint. This is useful if you want to deploy an + Inference Endpoint running on the `text-generation-inference` (TGI) framework (see examples). + env (`dict[str, str]`, *optional*): + Non-secret environment variables to inject in the container environment + secrets (`dict[str, str]`, *optional*): + Secret values to inject in the container environment. + + domain (`str`, *optional*): + The custom domain for the Inference Endpoint deployment, if setup the inference endpoint will be available at this domain (e.g. `"my-new-domain.cool-website.woof"`). + path (`str`, *optional*): + The custom path to the deployed model, should start with a `/` (e.g. `"/models/google-bert/bert-base-uncased"`). + + cache_http_responses (`bool`, *optional*): + Whether to cache HTTP responses from the Inference Endpoint. + tags (`list[str]`, *optional*): + A list of tags to associate with the Inference Endpoint. + + namespace (`str`, *optional*): + The namespace where the Inference Endpoint will be updated. Defaults to the current user's namespace. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`InferenceEndpoint`]: information about the updated Inference Endpoint. + """ + namespace = namespace or self._get_namespace(token=token) + + # Populate only the fields that are not None + payload: dict = defaultdict(lambda: defaultdict(dict)) + if accelerator is not None: + payload["compute"]["accelerator"] = accelerator + if instance_size is not None: + payload["compute"]["instanceSize"] = instance_size + if instance_type is not None: + payload["compute"]["instanceType"] = instance_type + if max_replica is not None: + payload["compute"]["scaling"]["maxReplica"] = max_replica + if min_replica is not None: + payload["compute"]["scaling"]["minReplica"] = min_replica + if scale_to_zero_timeout is not None: + payload["compute"]["scaling"]["scaleToZeroTimeout"] = scale_to_zero_timeout + if scaling_metric: + payload["compute"]["scaling"]["measure"] = {scaling_metric: scaling_threshold} + if repository is not None: + payload["model"]["repository"] = repository + if framework is not None: + payload["model"]["framework"] = framework + if revision is not None: + payload["model"]["revision"] = revision + if task is not None: + payload["model"]["task"] = task + if custom_image is not None: + payload["model"]["image"] = {"custom": custom_image} + if env is not None: + payload["model"]["env"] = env + if secrets is not None: + payload["model"]["secrets"] = secrets + if domain is not None: + payload["route"]["domain"] = domain + if path is not None: + payload["route"]["path"] = path + if cache_http_responses is not None: + payload["cacheHttpResponses"] = cache_http_responses + if tags is not None: + payload["tags"] = tags + + response = get_session().put( + f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}", + headers=self._build_hf_headers(token=token), + json=payload, + ) + hf_raise_for_status(response) + + return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) + + def delete_inference_endpoint( + self, name: str, *, namespace: str | None = None, token: bool | str | None = None + ) -> None: + """Delete an Inference Endpoint. + + This operation is not reversible. If you don't want to be charged for an Inference Endpoint, it is preferable + to pause it with [`pause_inference_endpoint`] or scale it to zero with [`scale_to_zero_inference_endpoint`]. + + For convenience, you can also delete an Inference Endpoint using [`InferenceEndpoint.delete`]. + + Args: + name (`str`): + The name of the Inference Endpoint to delete. + namespace (`str`, *optional*): + The namespace in which the Inference Endpoint is located. Defaults to the current user. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + """ + namespace = namespace or self._get_namespace(token=token) + response = get_session().delete( + f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + + def pause_inference_endpoint( + self, name: str, *, namespace: str | None = None, token: bool | str | None = None + ) -> InferenceEndpoint: + """Pause an Inference Endpoint. + + A paused Inference Endpoint will not be charged. It can be resumed at any time using [`resume_inference_endpoint`]. + This is different than scaling the Inference Endpoint to zero with [`scale_to_zero_inference_endpoint`], which + would be automatically restarted when a request is made to it. + + For convenience, you can also pause an Inference Endpoint using [`pause_inference_endpoint`]. + + Args: + name (`str`): + The name of the Inference Endpoint to pause. + namespace (`str`, *optional*): + The namespace in which the Inference Endpoint is located. Defaults to the current user. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`InferenceEndpoint`]: information about the paused Inference Endpoint. + """ + namespace = namespace or self._get_namespace(token=token) + + response = get_session().post( + f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/pause", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + + return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) + + def resume_inference_endpoint( + self, + name: str, + *, + namespace: str | None = None, + running_ok: bool = True, + token: bool | str | None = None, + ) -> InferenceEndpoint: + """Resume an Inference Endpoint. + + For convenience, you can also resume an Inference Endpoint using [`InferenceEndpoint.resume`]. + + Args: + name (`str`): + The name of the Inference Endpoint to resume. + namespace (`str`, *optional*): + The namespace in which the Inference Endpoint is located. Defaults to the current user. + running_ok (`bool`, *optional*): + If `True`, the method will not raise an error if the Inference Endpoint is already running. Defaults to + `True`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`InferenceEndpoint`]: information about the resumed Inference Endpoint. + """ + namespace = namespace or self._get_namespace(token=token) + + response = get_session().post( + f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/resume", + headers=self._build_hf_headers(token=token), + ) + try: + hf_raise_for_status(response) + except HfHubHTTPError as error: + # If already running (and it's ok), then fetch current status and return + if running_ok and error.response.status_code == 400 and "already running" in error.response.text: + return self.get_inference_endpoint(name, namespace=namespace, token=token) + # Otherwise, raise the error + raise + + return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) + + def scale_to_zero_inference_endpoint( + self, name: str, *, namespace: str | None = None, token: bool | str | None = None + ) -> InferenceEndpoint: + """Scale Inference Endpoint to zero. + + An Inference Endpoint scaled to zero will not be charged. It will be resume on the next request to it, with a + cold start delay. This is different than pausing the Inference Endpoint with [`pause_inference_endpoint`], which + would require a manual resume with [`resume_inference_endpoint`]. + + For convenience, you can also scale an Inference Endpoint to zero using [`InferenceEndpoint.scale_to_zero`]. + + Args: + name (`str`): + The name of the Inference Endpoint to scale to zero. + namespace (`str`, *optional*): + The namespace in which the Inference Endpoint is located. Defaults to the current user. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`InferenceEndpoint`]: information about the scaled-to-zero Inference Endpoint. + """ + namespace = namespace or self._get_namespace(token=token) + + response = get_session().post( + f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/scale-to-zero", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + + return InferenceEndpoint.from_raw(response.json(), namespace=namespace, token=token) + + def _get_namespace(self, token: bool | str | None = None) -> str: + """Get the default namespace for the current user.""" + me = self.whoami(token=token) + if me["type"] == "user": + return me["name"] + else: + raise ValueError( + "Cannot determine default namespace. You must provide a 'namespace' as input or be logged in as a" + " user." + ) + + ######################## + # Collection Endpoints # + ######################## + @validate_hf_hub_args + def list_collections( + self, + *, + owner: list[str] | str | None = None, + item: list[str] | str | None = None, + sort: CollectionSort_T | None = None, + limit: int | None = None, + token: bool | str | None = None, + ) -> Iterable[Collection]: + """List collections on the Huggingface Hub, given some filters. + + > [!WARNING] + > When listing collections, the item list per collection is truncated to 4 items maximum. To retrieve all items + > from a collection, you must use [`get_collection`]. + + Args: + owner (`list[str]` or `str`, *optional*): + Filter by owner's username. + item (`list[str]` or `str`, *optional*): + Filter collections containing a particular items. Example: `"models/teknium/OpenHermes-2.5-Mistral-7B"`, `"datasets/squad"` or `"papers/2311.12983"`. + sort (`Literal["lastModified", "trending", "upvotes"]`, *optional*): + Sort collections by last modified, trending or upvotes. + limit (`int`, *optional*): + Maximum number of collections to be returned. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[Collection]`: an iterable of [`Collection`] objects. + """ + # Construct the API endpoint + path = f"{self.endpoint}/api/collections" + headers = self._build_hf_headers(token=token) + params: dict = {} + if owner is not None: + params.update({"owner": owner}) + if item is not None: + params.update({"item": item}) + if sort is not None: + params.update({"sort": sort}) + if limit is not None: + params.update({"limit": limit}) + + # Paginate over the results until limit is reached + items = paginate(path, headers=headers, params=params) + if limit is not None: + items = islice(items, limit) # Do not iterate over all pages + + # Parse as Collection and return + for position, collection_data in enumerate(items): + yield Collection(position=position, **collection_data) + + def get_collection(self, collection_slug: str, *, token: bool | str | None = None) -> Collection: + """Gets information about a Collection on the Hub. + + Args: + collection_slug (`str`): + Slug of the collection of the Hub. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: [`Collection`] + + Example: + + ```py + >>> from huggingface_hub import get_collection + >>> collection = get_collection("TheBloke/recent-models-64f9a55bb3115b4f513ec026") + >>> collection.title + 'Recent models' + >>> len(collection.items) + 37 + >>> collection.items[0] + CollectionItem( + item_object_id='651446103cd773a050bf64c2', + item_id='TheBloke/U-Amethyst-20B-AWQ', + item_type='model', + position=88, + note=None + ) + ``` + """ + r = get_session().get( + f"{self.endpoint}/api/collections/{collection_slug}", headers=self._build_hf_headers(token=token) + ) + hf_raise_for_status(r) + return Collection(**{**r.json(), "endpoint": self.endpoint}) + + def create_collection( + self, + title: str, + *, + namespace: str | None = None, + description: str | None = None, + private: bool = False, + exists_ok: bool = False, + token: bool | str | None = None, + ) -> Collection: + """Create a new Collection on the Hub. + + Args: + title (`str`): + Title of the collection to create. Example: `"Recent models"`. + namespace (`str`, *optional*): + Namespace of the collection to create (username or org). Will default to the owner name. + description (`str`, *optional*): + Description of the collection to create. The maximum size for a description is 150 characters. + private (`bool`, *optional*): + Whether the collection should be private or not. Defaults to `False` (i.e. public collection). + exists_ok (`bool`, *optional*): + If `True`, do not raise an error if collection already exists. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: [`Collection`] + + Example: + + ```py + >>> from huggingface_hub import create_collection + >>> collection = create_collection( + ... title="ICCV 2023", + ... description="Portfolio of models, papers and demos I presented at ICCV 2023", + ... ) + >>> collection.slug + "username/iccv-2023-64f9a55bb3115b4f513ec026" + ``` + """ + if namespace is None: + namespace = self.whoami(token)["name"] + + payload = { + "title": title, + "namespace": namespace, + "private": private, + } + if description is not None: + payload["description"] = description + + r = get_session().post( + f"{self.endpoint}/api/collections", headers=self._build_hf_headers(token=token), json=payload + ) + try: + hf_raise_for_status(r) + except HfHubHTTPError as err: + if exists_ok and err.response.status_code == 409: + # Collection already exists and `exists_ok=True` + slug = r.json()["slug"] + return self.get_collection(slug, token=token) + else: + raise + return Collection(**{**r.json(), "endpoint": self.endpoint}) + + def update_collection_metadata( + self, + collection_slug: str, + *, + title: str | None = None, + description: str | None = None, + position: int | None = None, + private: bool | None = None, + theme: str | None = None, + token: bool | str | None = None, + ) -> Collection: + """Update metadata of a collection on the Hub. + + All arguments are optional. Only provided metadata will be updated. + + Args: + collection_slug (`str`): + Slug of the collection to update. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. + title (`str`): + Title of the collection to update. + description (`str`, *optional*): + Description of the collection to update. The maximum size for a description is 150 characters. + position (`int`, *optional*): + New position of the collection in the list of collections of the user. + private (`bool`, *optional*): + Whether the collection should be private or not. + theme (`str`, *optional*): + Theme of the collection on the Hub. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: [`Collection`] + + Example: + + ```py + >>> from huggingface_hub import update_collection_metadata + >>> collection = update_collection_metadata( + ... collection_slug="username/iccv-2023-64f9a55bb3115b4f513ec026", + ... title="ICCV Oct. 2023" + ... description="Portfolio of models, datasets, papers and demos I presented at ICCV Oct. 2023", + ... private=False, + ... theme="pink", + ... ) + >>> collection.slug + "username/iccv-oct-2023-64f9a55bb3115b4f513ec026" + # ^collection slug got updated but not the trailing ID + ``` + """ + payload = { + "position": position, + "private": private, + "theme": theme, + "title": title, + "description": description, + } + r = get_session().patch( + f"{self.endpoint}/api/collections/{collection_slug}", + headers=self._build_hf_headers(token=token), + # Only send not-none values to the API + json={key: value for key, value in payload.items() if value is not None}, + ) + hf_raise_for_status(r) + return Collection(**{**r.json()["data"], "endpoint": self.endpoint}) + + def delete_collection( + self, collection_slug: str, *, missing_ok: bool = False, token: bool | str | None = None + ) -> None: + """Delete a collection on the Hub. + + Args: + collection_slug (`str`): + Slug of the collection to delete. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. + missing_ok (`bool`, *optional*): + If `True`, do not raise an error if the collection doesn't exist. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Example: + + ```py + >>> from huggingface_hub import delete_collection + >>> collection = delete_collection("username/useless-collection-64f9a55bb3115b4f513ec026", missing_ok=True) + ``` + + > [!WARNING] + > This is a non-revertible action. A deleted collection cannot be restored. + """ + r = get_session().delete( + f"{self.endpoint}/api/collections/{collection_slug}", headers=self._build_hf_headers(token=token) + ) + try: + hf_raise_for_status(r) + except HfHubHTTPError as err: + if missing_ok and err.response.status_code == 404: + # Collection doesn't exist and `missing_ok=True` + return + else: + raise + + def add_collection_item( + self, + collection_slug: str, + item_id: str, + item_type: CollectionItemType_T, + *, + note: str | None = None, + exists_ok: bool = False, + token: bool | str | None = None, + ) -> Collection: + """Add an item to a collection on the Hub. + + Args: + collection_slug (`str`): + Slug of the collection to update. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. + item_id (`str`): + Id of the item to add to the collection. Use the repo_id for repos/spaces/datasets, + the paper id for papers, the slug of another collection (e.g. `"moonshotai/kimi-k2"`) + or a bucket id (e.g. `"namespace/bucket-name"`). + item_type (`str`): + Type of the item to add. Can be one of `"model"`, `"dataset"`, `"space"`, `"paper"`, `"collection"` + or `"bucket"`. + note (`str`, *optional*): + A note to attach to the item in the collection. The maximum size for a note is 500 characters. + exists_ok (`bool`, *optional*): + If `True`, do not raise an error if item already exists. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: [`Collection`] + + Raises: + [`HfHubHTTPError`]: + HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` + or `admin` role in the organization the repo belongs to or if you passed a `read` token. + [`HfHubHTTPError`]: + HTTP 404 if the item you try to add to the collection does not exist on the Hub. + [`HfHubHTTPError`]: + HTTP 409 if the item you try to add to the collection is already in the collection (and exists_ok=False) + + Example: + + ```py + >>> from huggingface_hub import add_collection_item + >>> collection = add_collection_item( + ... collection_slug="davanstrien/climate-64f99dc2a5067f6b65531bab", + ... item_id="pierre-loic/climate-news-articles", + ... item_type="dataset" + ... ) + >>> collection.items[-1].item_id + "pierre-loic/climate-news-articles" + # ^item got added to the collection on last position + + # Add item with a note + >>> add_collection_item( + ... collection_slug="davanstrien/climate-64f99dc2a5067f6b65531bab", + ... item_id="datasets/climate_fever", + ... item_type="dataset" + ... note="This dataset adopts the FEVER methodology that consists of 1,535 real-world claims regarding climate-change collected on the internet." + ... ) + (...) + ``` + """ + payload: dict[str, Any] = {"item": {"id": item_id, "type": item_type}} + if note is not None: + payload["note"] = note + r = get_session().post( + f"{self.endpoint}/api/collections/{collection_slug}/items", + headers=self._build_hf_headers(token=token), + json=payload, + ) + try: + hf_raise_for_status(r) + except HfHubHTTPError as err: + if exists_ok and err.response.status_code == 409: + # Item already exists and `exists_ok=True` + return self.get_collection(collection_slug, token=token) + else: + raise + return Collection(**{**r.json(), "endpoint": self.endpoint}) + + def update_collection_item( + self, + collection_slug: str, + item_object_id: str, + *, + note: str | None = None, + position: int | None = None, + token: bool | str | None = None, + ) -> None: + """Update an item in a collection. + + Args: + collection_slug (`str`): + Slug of the collection to update. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. + item_object_id (`str`): + ID of the item in the collection. This is not the id of the item on the Hub (repo_id or paper id). + It must be retrieved from a [`CollectionItem`] object. Example: `collection.items[0].item_object_id`. + note (`str`, *optional*): + A note to attach to the item in the collection. The maximum size for a note is 500 characters. + position (`int`, *optional*): + New position of the item in the collection. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Example: + + ```py + >>> from huggingface_hub import get_collection, update_collection_item + + # Get collection first + >>> collection = get_collection("TheBloke/recent-models-64f9a55bb3115b4f513ec026") + + # Update item based on its ID (add note + update position) + >>> update_collection_item( + ... collection_slug="TheBloke/recent-models-64f9a55bb3115b4f513ec026", + ... item_object_id=collection.items[-1].item_object_id, + ... note="Newly updated model!" + ... position=0, + ... ) + ``` + """ + payload = {"position": position, "note": note} + r = get_session().patch( + f"{self.endpoint}/api/collections/{collection_slug}/items/{item_object_id}", + headers=self._build_hf_headers(token=token), + # Only send not-none values to the API + json={key: value for key, value in payload.items() if value is not None}, + ) + hf_raise_for_status(r) + + def delete_collection_item( + self, + collection_slug: str, + item_object_id: str, + *, + missing_ok: bool = False, + token: bool | str | None = None, + ) -> None: + """Delete an item from a collection. + + Args: + collection_slug (`str`): + Slug of the collection to update. Example: `"TheBloke/recent-models-64f9a55bb3115b4f513ec026"`. + item_object_id (`str`): + ID of the item in the collection. This is not the id of the item on the Hub (repo_id or paper id). + It must be retrieved from a [`CollectionItem`] object. Example: `collection.items[0].item_object_id`. + missing_ok (`bool`, *optional*): + If `True`, do not raise an error if the item doesn't exist. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Example: + + ```py + >>> from huggingface_hub import get_collection, delete_collection_item + + # Get collection first + >>> collection = get_collection("TheBloke/recent-models-64f9a55bb3115b4f513ec026") + + # Delete item based on its ID + >>> delete_collection_item( + ... collection_slug="TheBloke/recent-models-64f9a55bb3115b4f513ec026", + ... item_object_id=collection.items[-1].item_object_id, + ... ) + ``` + """ + r = get_session().delete( + f"{self.endpoint}/api/collections/{collection_slug}/items/{item_object_id}", + headers=self._build_hf_headers(token=token), + ) + try: + hf_raise_for_status(r) + except HfHubHTTPError as err: + if missing_ok and err.response.status_code == 404: + # Item already deleted and `missing_ok=True` + return + else: + raise + + ########################## + # Manage access requests # + ########################## + + @validate_hf_hub_args + def list_pending_access_requests( + self, repo_id: str, *, repo_type: str | None = None, token: bool | str | None = None + ) -> Iterable[AccessRequest]: + """ + Get pending access requests for a given gated repo. + + A pending request means the user has requested access to the repo but the request has not been processed yet. + If the approval mode is automatic, this list should be empty. Pending requests can be accepted or rejected + using [`accept_access_request`] and [`reject_access_request`]. + + For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. + + Args: + repo_id (`str`): + The id of the repo to get access requests for. + repo_type (`str`, *optional*): + The type of the repo to get access requests for. Must be one of `model`, `dataset` or `space`. + Defaults to `model`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[AccessRequest]`: An iterable of [`AccessRequest`] objects. Each time contains a `username`, `email`, + `status` and `timestamp` attribute. If the gated repo has a custom form, the `fields` attribute will + be populated with user's answers. + + Raises: + [`HfHubHTTPError`]: + HTTP 400 if the repo is not gated. + [`HfHubHTTPError`]: + HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` + or `admin` role in the organization the repo belongs to or if you passed a `read` token. + + Example: + ```py + >>> from huggingface_hub import list_pending_access_requests, accept_access_request + + # List pending requests + >>> requests = list(list_pending_access_requests("meta-llama/Llama-2-7b")) + >>> len(requests) + 411 + >>> requests[0] + [ + AccessRequest( + username='clem', + fullname='Clem 🤗', + email='***', + timestamp=datetime.datetime(2023, 11, 23, 18, 4, 53, 828000, tzinfo=datetime.timezone.utc), + status='pending', + fields=None, + ), + ... + ] + + # Accept Clem's request + >>> accept_access_request("meta-llama/Llama-2-7b", "clem") + ``` + """ + yield from self._list_access_requests(repo_id, "pending", repo_type=repo_type, token=token) + + @validate_hf_hub_args + def list_accepted_access_requests( + self, repo_id: str, *, repo_type: str | None = None, token: bool | str | None = None + ) -> Iterable[AccessRequest]: + """ + Get accepted access requests for a given gated repo. + + An accepted request means the user has requested access to the repo and the request has been accepted. The user + can download any file of the repo. If the approval mode is automatic, this list should contains by default all + requests. Accepted requests can be cancelled or rejected at any time using [`cancel_access_request`] and + [`reject_access_request`]. A cancelled request will go back to the pending list while a rejected request will + go to the rejected list. In both cases, the user will lose access to the repo. + + For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. + + Args: + repo_id (`str`): + The id of the repo to get access requests for. + repo_type (`str`, *optional*): + The type of the repo to get access requests for. Must be one of `model`, `dataset` or `space`. + Defaults to `model`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[AccessRequest]`: An iterable of [`AccessRequest`] objects. Each time contains a `username`, `email`, + `status` and `timestamp` attribute. If the gated repo has a custom form, the `fields` attribute will + be populated with user's answers. + + Raises: + [`HfHubHTTPError`]: + HTTP 400 if the repo is not gated. + [`HfHubHTTPError`]: + HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` + or `admin` role in the organization the repo belongs to or if you passed a `read` token. + + Example: + ```py + >>> from huggingface_hub import list_accepted_access_requests + + >>> requests = list(list_accepted_access_requests("meta-llama/Llama-2-7b")) + >>> len(requests) + 411 + >>> requests[0] + [ + AccessRequest( + username='clem', + fullname='Clem 🤗', + email='***', + timestamp=datetime.datetime(2023, 11, 23, 18, 4, 53, 828000, tzinfo=datetime.timezone.utc), + status='accepted', + fields=None, + ), + ... + ] + ``` + """ + yield from self._list_access_requests(repo_id, "accepted", repo_type=repo_type, token=token) + + @validate_hf_hub_args + def list_rejected_access_requests( + self, repo_id: str, *, repo_type: str | None = None, token: bool | str | None = None + ) -> Iterable[AccessRequest]: + """ + Get rejected access requests for a given gated repo. + + A rejected request means the user has requested access to the repo and the request has been explicitly rejected + by a repo owner (either you or another user from your organization). The user cannot download any file of the + repo. Rejected requests can be accepted or cancelled at any time using [`accept_access_request`] and + [`cancel_access_request`]. A cancelled request will go back to the pending list while an accepted request will + go to the accepted list. + + For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. + + Args: + repo_id (`str`): + The id of the repo to get access requests for. + repo_type (`str`, *optional*): + The type of the repo to get access requests for. Must be one of `model`, `dataset` or `space`. + Defaults to `model`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[AccessRequest]`: An iterable of [`AccessRequest`] objects. Each time contains a `username`, `email`, + `status` and `timestamp` attribute. If the gated repo has a custom form, the `fields` attribute will + be populated with user's answers. + + Raises: + [`HfHubHTTPError`]: + HTTP 400 if the repo is not gated. + [`HfHubHTTPError`]: + HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` + or `admin` role in the organization the repo belongs to or if you passed a `read` token. + + Example: + ```py + >>> from huggingface_hub import list_rejected_access_requests + + >>> requests = list(list_rejected_access_requests("meta-llama/Llama-2-7b")) + >>> len(requests) + 411 + >>> requests[0] + [ + AccessRequest( + username='clem', + fullname='Clem 🤗', + email='***', + timestamp=datetime.datetime(2023, 11, 23, 18, 4, 53, 828000, tzinfo=datetime.timezone.utc), + status='rejected', + fields=None, + ), + ... + ] + ``` + """ + yield from self._list_access_requests(repo_id, "rejected", repo_type=repo_type, token=token) + + def _list_access_requests( + self, + repo_id: str, + status: Literal["accepted", "rejected", "pending"], + repo_type: str | None = None, + token: bool | str | None = None, + ) -> Iterable[AccessRequest]: + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + + for request in paginate( + f"{constants.ENDPOINT}/api/{repo_type}s/{repo_id}/user-access-request/{status}", + params={}, + headers=self._build_hf_headers(token=token), + ): + yield AccessRequest( + username=request["user"]["user"], + fullname=request["user"]["fullname"], + email=request["user"].get("email"), + status=request["status"], + timestamp=parse_datetime(request["timestamp"]), + fields=request.get("fields"), # only if custom fields in form + ) + + @validate_hf_hub_args + def cancel_access_request( + self, repo_id: str, user: str, *, repo_type: str | None = None, token: bool | str | None = None + ) -> None: + """ + Cancel an access request from a user for a given gated repo. + + A cancelled request will go back to the pending list and the user will lose access to the repo. + + For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. + + Args: + repo_id (`str`): + The id of the repo to cancel access request for. + user (`str`): + The username of the user which access request should be cancelled. + repo_type (`str`, *optional*): + The type of the repo to cancel access request for. Must be one of `model`, `dataset` or `space`. + Defaults to `model`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Raises: + [`HfHubHTTPError`]: + HTTP 400 if the repo is not gated. + [`HfHubHTTPError`]: + HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` + or `admin` role in the organization the repo belongs to or if you passed a `read` token. + [`HfHubHTTPError`]: + HTTP 404 if the user does not exist on the Hub. + [`HfHubHTTPError`]: + HTTP 404 if the user access request cannot be found. + [`HfHubHTTPError`]: + HTTP 404 if the user access request is already in the pending list. + """ + self._handle_access_request(repo_id, user, "pending", repo_type=repo_type, token=token) + + @validate_hf_hub_args + def accept_access_request( + self, repo_id: str, user: str, *, repo_type: str | None = None, token: bool | str | None = None + ) -> None: + """ + Accept an access request from a user for a given gated repo. + + Once the request is accepted, the user will be able to download any file of the repo and access the community + tab. If the approval mode is automatic, you don't have to accept requests manually. An accepted request can be + cancelled or rejected at any time using [`cancel_access_request`] and [`reject_access_request`]. + + For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. + + Args: + repo_id (`str`): + The id of the repo to accept access request for. + user (`str`): + The username of the user which access request should be accepted. + repo_type (`str`, *optional*): + The type of the repo to accept access request for. Must be one of `model`, `dataset` or `space`. + Defaults to `model`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Raises: + [`HfHubHTTPError`]: + HTTP 400 if the repo is not gated. + [`HfHubHTTPError`]: + HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` + or `admin` role in the organization the repo belongs to or if you passed a `read` token. + [`HfHubHTTPError`]: + HTTP 404 if the user does not exist on the Hub. + [`HfHubHTTPError`]: + HTTP 404 if the user access request cannot be found. + [`HfHubHTTPError`]: + HTTP 404 if the user access request is already in the accepted list. + """ + self._handle_access_request(repo_id, user, "accepted", repo_type=repo_type, token=token) + + @validate_hf_hub_args + def reject_access_request( + self, + repo_id: str, + user: str, + *, + repo_type: str | None = None, + rejection_reason: str | None, + token: bool | str | None = None, + ) -> None: + """ + Reject an access request from a user for a given gated repo. + + A rejected request will go to the rejected list. The user cannot download any file of the repo. Rejected + requests can be accepted or cancelled at any time using [`accept_access_request`] and [`cancel_access_request`]. + A cancelled request will go back to the pending list while an accepted request will go to the accepted list. + + For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. + + Args: + repo_id (`str`): + The id of the repo to reject access request for. + user (`str`): + The username of the user which access request should be rejected. + repo_type (`str`, *optional*): + The type of the repo to reject access request for. Must be one of `model`, `dataset` or `space`. + Defaults to `model`. + rejection_reason (`str`, *optional*): + Optional rejection reason that will be visible to the user (max 200 characters). + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Raises: + [`HfHubHTTPError`]: + HTTP 400 if the repo is not gated. + [`HfHubHTTPError`]: + HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` + or `admin` role in the organization the repo belongs to or if you passed a `read` token. + [`HfHubHTTPError`]: + HTTP 404 if the user does not exist on the Hub. + [`HfHubHTTPError`]: + HTTP 404 if the user access request cannot be found. + [`HfHubHTTPError`]: + HTTP 404 if the user access request is already in the rejected list. + """ + self._handle_access_request( + repo_id, user, "rejected", repo_type=repo_type, rejection_reason=rejection_reason, token=token + ) + + @validate_hf_hub_args + def _handle_access_request( + self, + repo_id: str, + user: str, + status: Literal["accepted", "rejected", "pending"], + repo_type: str | None = None, + rejection_reason: str | None = None, + token: bool | str | None = None, + ) -> None: + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + + payload = {"user": user, "status": status} + + if rejection_reason is not None: + if status != "rejected": + raise ValueError("`rejection_reason` can only be passed when rejecting an access request.") + payload["rejectionReason"] = rejection_reason + + response = get_session().post( + f"{constants.ENDPOINT}/api/{repo_type}s/{repo_id}/user-access-request/handle", + headers=self._build_hf_headers(token=token), + json=payload, + ) + hf_raise_for_status(response) + + @validate_hf_hub_args + def grant_access( + self, repo_id: str, user: str, *, repo_type: str | None = None, token: bool | str | None = None + ) -> None: + """ + Grant access to a user for a given gated repo. + + Granting access don't require for the user to send an access request by themselves. The user is automatically + added to the accepted list meaning they can download the files You can revoke the granted access at any time + using [`cancel_access_request`] or [`reject_access_request`]. + + For more info about gated repos, see https://huggingface.co/docs/hub/models-gated. + + Args: + repo_id (`str`): + The id of the repo to grant access to. + user (`str`): + The username of the user to grant access. + repo_type (`str`, *optional*): + The type of the repo to grant access to. Must be one of `model`, `dataset` or `space`. + Defaults to `model`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Raises: + [`HfHubHTTPError`]: + HTTP 400 if the repo is not gated. + [`HfHubHTTPError`]: + HTTP 400 if the user already has access to the repo. + [`HfHubHTTPError`]: + HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write` + or `admin` role in the organization the repo belongs to or if you passed a `read` token. + [`HfHubHTTPError`]: + HTTP 404 if the user does not exist on the Hub. + """ + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + + response = get_session().post( + f"{constants.ENDPOINT}/api/{repo_type}s/{repo_id}/user-access-request/grant", + headers=self._build_hf_headers(token=token), + json={"user": user}, + ) + hf_raise_for_status(response) + return response.json() + + ################### + # Manage webhooks # + ################### + + @validate_hf_hub_args + def get_webhook(self, webhook_id: str, *, token: bool | str | None = None) -> WebhookInfo: + """Get a webhook by its id. + + Args: + webhook_id (`str`): + The unique identifier of the webhook to get. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved token, which is the recommended + method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`WebhookInfo`]: + Info about the webhook. + + Example: + ```python + >>> from huggingface_hub import get_webhook + >>> webhook = get_webhook("654bbbc16f2ec14d77f109cc") + >>> print(webhook) + WebhookInfo( + id="654bbbc16f2ec14d77f109cc", + job=None, + watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], + url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", + secret="my-secret", + domains=["repo", "discussion"], + disabled=False, + ) + ``` + """ + response = get_session().get( + f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + webhook_data = response.json()["webhook"] + + watched_items = [WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook_data["watched"]] + + webhook = WebhookInfo( + id=webhook_data["id"], + url=webhook_data.get("url"), + job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None, + watched=watched_items, + domains=webhook_data["domains"], + secret=webhook_data.get("secret"), + disabled=webhook_data["disabled"], + ) + + return webhook + + @validate_hf_hub_args + def list_webhooks(self, *, token: bool | str | None = None) -> list[WebhookInfo]: + """List all configured webhooks. + + Args: + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved token, which is the recommended + method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `list[WebhookInfo]`: + List of webhook info objects. + + Example: + ```python + >>> from huggingface_hub import list_webhooks + >>> webhooks = list_webhooks() + >>> len(webhooks) + 2 + >>> webhooks[0] + WebhookInfo( + id="654bbbc16f2ec14d77f109cc", + watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], + url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", + secret="my-secret", + domains=["repo", "discussion"], + disabled=False, + ) + ``` + """ + response = get_session().get( + f"{constants.ENDPOINT}/api/settings/webhooks", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + webhooks_data = response.json() + + return [ + WebhookInfo( + id=webhook["id"], + url=webhook.get("url"), + job=JobSpec(**webhook["job"]) if webhook.get("job") else None, + watched=[WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook["watched"]], + domains=webhook["domains"], + secret=webhook.get("secret"), + disabled=webhook["disabled"], + ) + for webhook in webhooks_data + ] + + @validate_hf_hub_args + def create_webhook( + self, + *, + url: str | None = None, + job_id: str | None = None, + watched: list[dict | WebhookWatchedItem], + domains: list[constants.WEBHOOK_DOMAIN_T] | None = None, + secret: str | None = None, + token: bool | str | None = None, + ) -> WebhookInfo: + """Create a new webhook. + + The webhook can either send a payload to a URL, or trigger a Job to run on Hugging Face infrastructure. + This function should be called with one of `url` or `job_id`, but not both. + + Args: + url (`str`): + URL to send the payload to. + job_id (`str`): + ID of the source Job to trigger with the webhook payload in the environment variable WEBHOOK_PAYLOAD. + Additional environment variables are available for convenience: WEBHOOK_REPO_ID, WEBHOOK_REPO_TYPE and WEBHOOK_SECRET. + watched (`list[WebhookWatchedItem]`): + List of [`WebhookWatchedItem`] to be watched by the webhook. It can be users, orgs, models, datasets or spaces. + Watched items can also be provided as plain dictionaries. + domains (`list[Literal["repo", "discussion"]]`, optional): + List of domains to watch. It can be "repo", "discussion" or both. + secret (`str`, optional): + A secret to sign the payload with. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved token, which is the recommended + method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`WebhookInfo`]: + Info about the newly created webhook. + + Example: + + Create a webhook that sends a payload to a URL + ```python + >>> from huggingface_hub import create_webhook + >>> payload = create_webhook( + ... watched=[{"type": "user", "name": "julien-c"}, {"type": "org", "name": "HuggingFaceH4"}], + ... url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", + ... domains=["repo", "discussion"], + ... secret="my-secret", + ... ) + >>> print(payload) + WebhookInfo( + id="654bbbc16f2ec14d77f109cc", + url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", + job=None, + watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], + domains=["repo", "discussion"], + secret="my-secret", + disabled=False, + ) + ``` + + Run a Job and then create a webhook that triggers this Job + ```python + >>> from huggingface_hub import create_webhook, run_job + >>> job = run_job( + ... image="ubuntu", + ... command=["bash", "-c", r"echo An event occurred in $WEBHOOK_REPO_ID: $WEBHOOK_PAYLOAD"], + ... ) + >>> payload = create_webhook( + ... watched=[{"type": "user", "name": "julien-c"}, {"type": "org", "name": "HuggingFaceH4"}], + ... job_id=job.id, + ... domains=["repo", "discussion"], + ... secret="my-secret", + ... ) + >>> print(payload) + WebhookInfo( + id="654bbbc16f2ec14d77f109cc", + url=None, + job=JobSpec( + docker_image='ubuntu', + space_id=None, + command=['bash', '-c', 'echo An event occurred in $WEBHOOK_REPO_ID: $WEBHOOK_PAYLOAD'], + arguments=[], + environment={}, + secrets=[], + flavor='cpu-basic', + timeout=None, + tags=None, + arch=None + ), + watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], + domains=["repo", "discussion"], + secret="my-secret", + disabled=False, + ) + ``` + """ + watched_dicts = [asdict(item) if isinstance(item, WebhookWatchedItem) else item for item in watched] + + post_webhooks_json: dict = {"watched": watched_dicts} + if domains is not None: + post_webhooks_json["domains"] = domains + if secret is not None: + post_webhooks_json["secret"] = secret + if url is not None and job_id is not None: + raise ValueError("Set `url` or `job_id` but not both.") + elif url is not None: + post_webhooks_json["url"] = url + elif job_id is not None: + post_webhooks_json["jobSourceId"] = job_id + else: + raise ValueError("Missing argument for webhook: `url` or `job_id`.") + + response = get_session().post( + f"{constants.ENDPOINT}/api/settings/webhooks", + json=post_webhooks_json, + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + webhook_data = response.json()["webhook"] + watched_items = [WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook_data["watched"]] + + webhook = WebhookInfo( + id=webhook_data["id"], + url=webhook_data.get("url"), + job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None, + watched=watched_items, + domains=webhook_data["domains"], + secret=webhook_data.get("secret"), + disabled=webhook_data["disabled"], + ) + + return webhook + + @validate_hf_hub_args + def update_webhook( + self, + webhook_id: str, + *, + url: str | None = None, + watched: list[dict | WebhookWatchedItem] | None = None, + domains: list[constants.WEBHOOK_DOMAIN_T] | None = None, + secret: str | None = None, + token: bool | str | None = None, + ) -> WebhookInfo: + """Update an existing webhook. + + Args: + webhook_id (`str`): + The unique identifier of the webhook to be updated. + url (`str`, optional): + The URL to which the payload will be sent. + watched (`list[WebhookWatchedItem]`, optional): + List of items to watch. It can be users, orgs, models, datasets, or spaces. + Refer to [`WebhookWatchedItem`] for more details. Watched items can also be provided as plain dictionaries. + domains (`list[Literal["repo", "discussion"]]`, optional): + The domains to watch. This can include "repo", "discussion", or both. + secret (`str`, optional): + A secret to sign the payload with, providing an additional layer of security. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved token, which is the recommended + method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`WebhookInfo`]: + Info about the updated webhook. + + Example: + ```python + >>> from huggingface_hub import update_webhook + >>> updated_payload = update_webhook( + ... webhook_id="654bbbc16f2ec14d77f109cc", + ... url="https://new.webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", + ... watched=[{"type": "user", "name": "julien-c"}, {"type": "org", "name": "HuggingFaceH4"}], + ... domains=["repo"], + ... secret="my-secret", + ... ) + >>> print(updated_payload) + WebhookInfo( + id="654bbbc16f2ec14d77f109cc", + job=None, + url="https://new.webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", + watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], + domains=["repo"], + secret="my-secret", + disabled=False, + ``` + """ + if watched is None: + watched = [] + watched_dicts = [asdict(item) if isinstance(item, WebhookWatchedItem) else item for item in watched] + + update_json: dict = {"watched": watched_dicts} + if url is not None: + update_json["url"] = url + if domains is not None: + update_json["domains"] = domains + if secret is not None: + update_json["secret"] = secret + + response = get_session().post( + f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}", + json=update_json, + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + webhook_data = response.json()["webhook"] + + watched_items = [WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook_data["watched"]] + + webhook = WebhookInfo( + id=webhook_data["id"], + url=webhook_data.get("url"), + job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None, + watched=watched_items, + domains=webhook_data["domains"], + secret=webhook_data.get("secret"), + disabled=webhook_data["disabled"], + ) + + return webhook + + @validate_hf_hub_args + def enable_webhook(self, webhook_id: str, *, token: bool | str | None = None) -> WebhookInfo: + """Enable a webhook (makes it "active"). + + Args: + webhook_id (`str`): + The unique identifier of the webhook to enable. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved token, which is the recommended + method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`WebhookInfo`]: + Info about the enabled webhook. + + Example: + ```python + >>> from huggingface_hub import enable_webhook + >>> enabled_webhook = enable_webhook("654bbbc16f2ec14d77f109cc") + >>> enabled_webhook + WebhookInfo( + id="654bbbc16f2ec14d77f109cc", + job=None, + url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", + watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], + domains=["repo", "discussion"], + secret="my-secret", + disabled=False, + ) + ``` + """ + response = get_session().post( + f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}/enable", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + webhook_data = response.json()["webhook"] + + watched_items = [WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook_data["watched"]] + + webhook = WebhookInfo( + id=webhook_data["id"], + url=webhook_data.get("url"), + job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None, + watched=watched_items, + domains=webhook_data["domains"], + secret=webhook_data.get("secret"), + disabled=webhook_data["disabled"], + ) + + return webhook + + @validate_hf_hub_args + def disable_webhook(self, webhook_id: str, *, token: bool | str | None = None) -> WebhookInfo: + """Disable a webhook (makes it "disabled"). + + Args: + webhook_id (`str`): + The unique identifier of the webhook to disable. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved token, which is the recommended + method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`WebhookInfo`]: + Info about the disabled webhook. + + Example: + ```python + >>> from huggingface_hub import disable_webhook + >>> disabled_webhook = disable_webhook("654bbbc16f2ec14d77f109cc") + >>> disabled_webhook + WebhookInfo( + id="654bbbc16f2ec14d77f109cc", + url="https://webhook.site/a2176e82-5720-43ee-9e06-f91cb4c91548", + jon=None, + watched=[WebhookWatchedItem(type="user", name="julien-c"), WebhookWatchedItem(type="org", name="HuggingFaceH4")], + domains=["repo", "discussion"], + secret="my-secret", + disabled=True, + ) + ``` + """ + response = get_session().post( + f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}/disable", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + webhook_data = response.json()["webhook"] + + watched_items = [WebhookWatchedItem(type=item["type"], name=item["name"]) for item in webhook_data["watched"]] + + webhook = WebhookInfo( + id=webhook_data["id"], + url=webhook_data.get("url"), + job=JobSpec(**webhook_data["job"]) if webhook_data.get("job") else None, + watched=watched_items, + domains=webhook_data["domains"], + secret=webhook_data.get("secret"), + disabled=webhook_data["disabled"], + ) + + return webhook + + @validate_hf_hub_args + def delete_webhook(self, webhook_id: str, *, token: bool | str | None = None) -> None: + """Delete a webhook. + + Args: + webhook_id (`str`): + The unique identifier of the webhook to delete. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved token, which is the recommended + method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `None` + + Example: + ```python + >>> from huggingface_hub import delete_webhook + >>> delete_webhook("654bbbc16f2ec14d77f109cc") + ``` + """ + response = get_session().delete( + f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + + ############# + # Internals # + ############# + + def _build_hf_headers( + self, + token: bool | str | None = None, + library_name: str | None = None, + library_version: str | None = None, + user_agent: dict | str | None = None, + ) -> dict[str, str]: + """ + Alias for [`build_hf_headers`] that uses the token from [`HfApi`] client + when `token` is not provided. + """ + if token is None: + # Cannot do `token = token or self.token` as token can be `False`. + token = self.token + return build_hf_headers( + token=token, + library_name=library_name or self.library_name, + library_version=library_version or self.library_version, + user_agent=user_agent or self.user_agent, + headers=self.headers, + ) + + def _prepare_folder_deletions( + self, + repo_id: str, + repo_type: str | None, + revision: str | None, + path_in_repo: str, + delete_patterns: list[str] | str | None, + token: bool | str | None = None, + ) -> list[CommitOperationDelete]: + """Generate the list of Delete operations for a commit to delete files from a repo. + + List remote files and match them against the `delete_patterns` constraints. Returns a list of [`CommitOperationDelete`] + with the matching items. + + Note: `.gitattributes` file is essential to make a repo work properly on the Hub. This file will always be + kept even if it matches the `delete_patterns` constraints. + """ + if delete_patterns is None: + # If no delete patterns, no need to list and filter remote files + return [] + + # List remote files + filenames = self.list_repo_files(repo_id=repo_id, revision=revision, repo_type=repo_type, token=token) + + # Compute relative path in repo + if path_in_repo and path_in_repo not in (".", "./"): + path_in_repo = path_in_repo.strip("/") + "/" # harmonize + relpath_to_abspath = { + file[len(path_in_repo) :]: file for file in filenames if file.startswith(path_in_repo) + } + else: + relpath_to_abspath = {file: file for file in filenames} + + # Apply filter on relative paths and return + return [ + CommitOperationDelete(path_in_repo=relpath_to_abspath[relpath], is_folder=False) + for relpath in filter_repo_objects(relpath_to_abspath.keys(), allow_patterns=delete_patterns) + if relpath_to_abspath[relpath] != ".gitattributes" + ] + + def _prepare_upload_folder_additions( + self, + folder_path: str | Path, + path_in_repo: str, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + repo_type: str | None = None, + token: bool | str | None = None, + ) -> list[CommitOperationAdd]: + """Generate the list of Add operations for a commit to upload a folder. + + Files not matching the `allow_patterns` (allowlist) and `ignore_patterns` (denylist) + constraints are discarded. + """ + + folder_path = Path(folder_path).expanduser().resolve() + if not folder_path.is_dir(): + raise ValueError(f"Provided path: '{folder_path}' is not a directory") + + # List files from folder + relpath_to_abspath = { + path.relative_to(folder_path).as_posix(): path + for path in sorted(folder_path.glob("**/*")) # sorted to be deterministic + if path.is_file() + } + + # Filter files + # Patterns are applied on the path relative to `folder_path`. `path_in_repo` is prefixed after the filtering. + filtered_repo_objects = list( + filter_repo_objects( + relpath_to_abspath.keys(), allow_patterns=allow_patterns, ignore_patterns=ignore_patterns + ) + ) + + prefix = f"{path_in_repo.strip('/')}/" if path_in_repo else "" + + # If updating a README.md file, make sure the metadata format is valid + # It's better to fail early than to fail after all the files have been hashed. + if "README.md" in filtered_repo_objects: + self._validate_yaml( + content=relpath_to_abspath["README.md"].read_text(encoding="utf8"), + repo_type=repo_type, + token=token, + ) + return [ + CommitOperationAdd( + path_or_fileobj=relpath_to_abspath[relpath], # absolute path on disk + path_in_repo=prefix + relpath, # "absolute" path in repo + ) + for relpath in filtered_repo_objects + ] + + def _validate_yaml(self, content: str, *, repo_type: str | None = None, token: bool | str | None = None): + """ + Validate YAML from `README.md`, used before file hashing and upload. + + Args: + content (`str`): + Content of `README.md` to validate. + repo_type (`str`, *optional*): + The type of the repo to grant access to. Must be one of `model`, `dataset` or `space`. + Defaults to `model`. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Raises: + - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + if YAML is invalid + """ + repo_type = repo_type if repo_type is not None else constants.REPO_TYPE_MODEL + headers = self._build_hf_headers(token=token) + + response = get_session().post( + f"{self.endpoint}/api/validate-yaml", + json={"content": content, "repoType": repo_type}, + headers=headers, + ) + # Handle warnings (example: empty metadata) + response_content = response.json() + message = "\n".join([f"- {warning.get('message')}" for warning in response_content.get("warnings", [])]) + if message: + warnings.warn(f"Warnings while validating metadata in README.md:\n{message}") + + # Raise on errors + try: + hf_raise_for_status(response) + except BadRequestError as e: + errors = response_content.get("errors", []) + message = "\n".join([f"- {error.get('message')}" for error in errors]) + raise ValueError(f"Invalid metadata in README.md.\n{message}") from e + + def get_user_overview(self, username: str, token: bool | str | None = None) -> User: + """ + Get an overview of a user on the Hub. + + Args: + username (`str`): + Username of the user to get an overview of. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `User`: A [`User`] object with the user's overview. + + Raises: + [`HfHubHTTPError`]: + HTTP 404 If the user does not exist on the Hub. + """ + r = get_session().get( + f"{constants.ENDPOINT}/api/users/{username}/overview", headers=self._build_hf_headers(token=token) + ) + hf_raise_for_status(r) + return User(**r.json()) + + @validate_hf_hub_args + def get_organization_overview(self, organization: str, token: bool | str | None = None) -> Organization: + """ + Get an overview of an organization on the Hub. + + Args: + organization (`str`): + Name of the organization to get an overview of. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved token, which is the recommended method + for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Organization`: An [`Organization`] object with the organization's overview. + + Raises: + [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError): + HTTP 404 If the organization does not exist on the Hub. + """ + r = get_session().get( + f"{constants.ENDPOINT}/api/organizations/{organization}/overview", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(r) + return Organization(**r.json()) + + @validate_hf_hub_args + def list_organization_followers(self, organization: str, token: bool | str | None = None) -> Iterable[User]: + """ + List followers of an organization on the Hub. + + Args: + organization (`str`): + Name of the organization to get the followers of. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[User]`: A list of [`User`] objects with the followers of the organization. + + Raises: + [`HfHubHTTPError`]: + HTTP 404 If the organization does not exist on the Hub. + + """ + for follower in paginate( + path=f"{constants.ENDPOINT}/api/organizations/{organization}/followers", + params={}, + headers=self._build_hf_headers(token=token), + ): + yield User(**follower) + + def list_organization_members(self, organization: str, token: bool | str | None = None) -> Iterable[User]: + """ + List of members of an organization on the Hub. + + Args: + organization (`str`): + Name of the organization to get the members of. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[User]`: A list of [`User`] objects with the members of the organization. + + Raises: + [`HfHubHTTPError`]: + HTTP 404 If the organization does not exist on the Hub. + + """ + for member in paginate( + path=f"{constants.ENDPOINT}/api/organizations/{organization}/members", + params={}, + headers=self._build_hf_headers(token=token), + ): + yield User(**member) + + def list_user_followers(self, username: str, token: bool | str | None = None) -> Iterable[User]: + """ + Get the list of followers of a user on the Hub. + + Args: + username (`str`): + Username of the user to get the followers of. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[User]`: A list of [`User`] objects with the followers of the user. + + Raises: + [`HfHubHTTPError`]: + HTTP 404 If the user does not exist on the Hub. + + """ + for follower in paginate( + path=f"{constants.ENDPOINT}/api/users/{username}/followers", + params={}, + headers=self._build_hf_headers(token=token), + ): + yield User(**follower) + + def list_user_following(self, username: str, token: bool | str | None = None) -> Iterable[User]: + """ + Get the list of users followed by a user on the Hub. + + Args: + username (`str`): + Username of the user to get the users followed by. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[User]`: A list of [`User`] objects with the users followed by the user. + + Raises: + [`HfHubHTTPError`]: + HTTP 404 If the user does not exist on the Hub. + + """ + for followed_user in paginate( + path=f"{constants.ENDPOINT}/api/users/{username}/following", + params={}, + headers=self._build_hf_headers(token=token), + ): + yield User(**followed_user) + + def list_papers( + self, + *, + query: str | None = None, + limit: int | None = None, + token: bool | str | None = None, + ) -> Iterable[PaperInfo]: + """ + List daily papers on the Hugging Face Hub given a search query. + + Args: + query (`str`, *optional*): + A search query string to find papers. + If provided, returns papers that match the query. + limit (`int`, *optional*): + The maximum number of papers to return. + token (Union[bool, str, None], *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[PaperInfo]`: an iterable of [`huggingface_hub.hf_api.PaperInfo`] objects. + + Example: + + ```python + >>> from huggingface_hub import HfApi + + >>> api = HfApi() + + # List all papers with "attention" in their title + >>> api.list_papers(query="attention") + ``` + """ + path = f"{self.endpoint}/api/papers/search" + params: dict[str, Any] = {} + if query: + params["q"] = query + if limit is not None: + params["limit"] = limit + r = get_session().get( + path, + params=params, + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(r) + for paper in r.json(): + yield PaperInfo(**paper) + + def paper_info(self, id: str) -> PaperInfo: + """ + Get information for a paper on the Hub. + + Args: + id (`str`, **optional**): + ArXiv id of the paper. + + Returns: + `PaperInfo`: A `PaperInfo` object. + + Raises: + [`HfHubHTTPError`]: + HTTP 404 If the paper does not exist on the Hub. + """ + path = f"{self.endpoint}/api/papers/{id}" + r = get_session().get(path) + hf_raise_for_status(r) + return PaperInfo(**r.json()) + + def read_paper(self, id: str) -> str: + """ + Get the markdown content of a paper page on the Hub. + + Args: + id (`str`): + ArXiv id of the paper. + + Returns: + `str`: The paper page content as markdown. + + Raises: + [`HfHubHTTPError`]: + HTTP 404 If the paper does not exist on the Hub. + """ + path = f"{self.endpoint}/papers/{id}.md" + r = get_session().get(path) + hf_raise_for_status(r) + return r.text + + def list_daily_papers( + self, + *, + date: str | None = None, + token: bool | str | None = None, + week: str | None = None, + month: str | None = None, + submitter: str | None = None, + sort: DailyPapersSort_T | None = None, + p: int | None = None, + limit: int | None = None, + ) -> Iterable[PaperInfo]: + """ + List the daily papers published on a given date on the Hugging Face Hub. + + Args: + date (`str`, *optional*): + Date in ISO format (YYYY-MM-DD) for which to fetch daily papers. + Defaults to most recent ones. + token (Union[bool, str, None], *optional*): + A valid user access token (string). Defaults to the locally saved + token. To disable authentication, pass `False`. + week (`str`, *optional*): + Week in ISO format (YYYY-Www) for which to fetch daily papers. Example, `2025-W09`. + month (`str`, *optional*): + Month in ISO format (YYYY-MM) for which to fetch daily papers. Example, `2025-02`. + submitter (`str`, *optional*): + Username of the submitter to filter daily papers. + sort (`Literal["publishedAt", "trending"]`, *optional*): + Sort order for the daily papers. Can be either by `publishedAt` or by `trending`. + Defaults to `"publishedAt"` + p (`int`, *optional*): + Page number for pagination. Defaults to 0. + limit (`int`, *optional*): + Limit of papers to fetch. Defaults to 50. + + Returns: + `Iterable[PaperInfo]`: an iterable of [`huggingface_hub.hf_api.PaperInfo`] objects. + + Example: + + ```python + >>> from huggingface_hub import HfApi + + >>> api = HfApi() + >>> list(api.list_daily_papers(date="2025-10-29")) + ``` + """ + path = f"{self.endpoint}/api/daily_papers" + + params = { + k: v + for k, v in { + "p": p, + "limit": limit, + "sort": sort, + "date": date, + "week": week, + "month": month, + "submitter": submitter, + }.items() + if v is not None + } + + r = get_session().get(path, params=params, headers=self._build_hf_headers(token=token)) + hf_raise_for_status(r) + for paper in r.json(): + yield PaperInfo(**paper) + + def auth_check( + self, + repo_id: str, + *, + repo_type: str | None = None, + token: bool | str | None = None, + write: bool = False, + ) -> None: + """ + Check if the provided user token has access to a specific repository on the Hugging Face Hub. + + This method verifies whether the user, authenticated via the provided token, has access to the specified + repository. If the repository is not found or if the user lacks the required permissions to access it, + the method raises an appropriate exception. + + Args: + repo_id (`str`): + The repository to check for access. Format should be `"user/repo_name"`. + Example: `"user/my-cool-model"`. + + repo_type (`str`, *optional*): + The type of the repository. Should be one of `"model"`, `"dataset"`, or `"space"`. + If not specified, the default is `"model"`. + + token (`Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + write (`bool`, *optional*): + If `True`, checks whether the user has content write permission on the repository. + If `False` (default), only checks for read access. + + Raises: + [`~utils.RepositoryNotFoundError`]: + Raised if the repository does not exist, is private, or the user does not have access. This can + occur if the `repo_id` or `repo_type` is incorrect or if the repository is private but the user + is not authenticated. + + [`~utils.GatedRepoError`]: + Raised if the repository exists but is gated and the user is not authorized to access it. + + Example: + Check if the user has access to a repository: + + ```python + >>> from huggingface_hub import auth_check + >>> from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError + + try: + auth_check("user/my-cool-model") + except GatedRepoError: + # Handle gated repository error + print("You do not have permission to access this gated repository.") + except RepositoryNotFoundError: + # Handle repository not found error + print("The repository was not found or you do not have access.") + ``` + + In this example: + - If the user has access, the method completes successfully. + - If the repository is gated or does not exist, appropriate exceptions are raised, allowing the user + to handle them accordingly. + """ + headers = self._build_hf_headers(token=token) + if repo_type is None: + repo_type = constants.REPO_TYPE_MODEL + if repo_type not in constants.REPO_TYPES: + raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}") + path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/auth-check" + if write: + path = f"{path}/write" + r = get_session().get(path, headers=headers) + hf_raise_for_status(r) + + def run_job( + self, + *, + image: str, + command: list[str], + env: dict[str, Any] | None = None, + secrets: dict[str, Any] | None = None, + flavor: JobHardware | str | None = None, + timeout: int | float | str | None = None, + name: str | None = None, + labels: dict[str, str] | None = None, + volumes: list[Volume] | None = None, + expose: list[int] | None = None, + ssh: bool = False, + namespace: str | None = None, + token: bool | str | None = None, + ) -> JobInfo: + """ + Run compute Jobs on Hugging Face infrastructure. + + Args: + image (`str`): + The Docker image to use. + Examples: `"ubuntu"`, `"python:3.12"`, `"pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel"`. + Example with an image from a Space: `"hf.co/spaces/lhoestq/duckdb"`. + + command (`list[str]`): + The command to run. Example: `["echo", "hello"]`. + + env (`dict[str, Any]`, *optional*): + Defines the environment variables for the Job. + + secrets (`dict[str, Any]`, *optional*): + Defines the secret environment variables for the Job. + + flavor (`str`, *optional*): + Flavor for the hardware. See [`JobHardware`] for possible values. + Defaults to `"cpu-basic"`. + + timeout (`Union[int, float, str]`, *optional*): + Max duration for the Job: int with s (seconds, default), m (minutes), h (hours) or d (days). + Example: `300` or `"5m"` for 5 minutes. + + name (`str`, *optional*): + A name for the Job. Stored as the `name` label. Cannot be passed together with a `name` key in + `labels`. Names do not have to be unique. Defaults to a name derived from image and command (with a short hash suffix). + + labels (`dict[str, str]`, *optional*): + Labels to attach to the job (key-value pairs). + + volumes (`list[Volume]`, *optional*): + Hugging Face Buckets or Repos to mount as volumes in the job container. + Each volume is a [`Volume`] with `type` (`"bucket"`, `"model"`, `"dataset"`, or `"space"`), + `source` (e.g. `"username/my-bucket"`), and `mount_path` (e.g. `"/data"`). + + expose (`list[int]`, *optional*): + Container ports to expose through the jobs proxy. Each listed port is reachable + on the public jobs domain (e.g. `https://--8000.hf.jobs`). Access always + requires an HF token with read access to the job's namespace. + + ssh (`bool`, *optional*): + If True, the job's container is reachable over SSH at the URL given by `job.status.ssh_url` + (e.g. `ssh @ssh.hf.jobs`, or `hf jobs ssh ` from the CLI). Connecting requires + write access to the job's namespace and an SSH public key registered on the Hub + (https://huggingface.co/settings/keys). Defaults to False. + + namespace (`str`, *optional*): + The namespace where the Job will be created. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Example: + Run your first Job: + + ```python + >>> from huggingface_hub import run_job + >>> run_job(image="python:3.12", command=["python", "-c" ,"print('Hello from HF compute!')"]) + ``` + + Run a GPU Job: + + ```python + >>> from huggingface_hub import run_job + >>> image = "pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel" + >>> command = ["python", "-c", "import torch; print(f"This code ran with the following GPU: {torch.cuda.get_device_name()}")"] + >>> run_job(image=image, command=command, flavor="a10g-small") + ``` + + Run a Job with volumes: + + ```python + >>> from huggingface_hub import Volume, run_job + >>> dataset_volume = Volume(type="dataset", source="HuggingFaceFW/fineweb", mount_path="/data") + >>> output_bucket_volume = Volume(type="bucket", source="username/my-bucket", mount_path="/output") + >>> image = "duckdb/duckdb" + >>> command = ["duckdb", "-c", "COPY (SELECT * FROM '/data/**/*.parquet' LIMIT 5) TO '/output/first-rows.parquet'"] + >>> run_job(image=image, command=command, volumes=[dataset_volume, output_bucket_volume]) + ``` + + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + if name is None and not (labels and "name" in labels): + name = _default_job_name_from_image(image, command) + job_spec = _create_job_spec( + image=image, + command=command, + env=env, + secrets=secrets, + flavor=flavor, + timeout=timeout, + name=name, + labels=labels, + volumes=volumes, + expose=expose, + ssh=ssh, + ) + response = get_session().post( + f"{self.endpoint}/api/jobs/{namespace}", + json=job_spec, + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + job_info = response.json() + return JobInfo(**job_info, endpoint=self.endpoint) + + def _fetch_running_job_sse( + self, + *, + job_id: str, + route: str, + timeout: int, + skip_previous_events_on_retry: bool, + tolerated_status_codes: tuple[int, ...] = (), + tolerated_exception_types: tuple[type[Exception], ...] = (), + follow: bool = True, + namespace: str | None = None, + token: bool | str | None = None, + params: dict[str, Any] | None = None, + ) -> Iterable[dict[str, Any]]: + if namespace is None: + namespace = self.whoami(token=token)["name"] + + def has_job_finished() -> bool: + # We don't use http_backoff: this is the authoritative check that + # decides whether to keep streaming. + job_status_response = get_session().get( + f"{self.endpoint}/api/jobs/{namespace}/{job_id}", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(job_status_response) + job_status = job_status_response.json() + return "status" in job_status and job_status["status"]["stage"] not in ("RUNNING", "UPDATING") + + yield from self._stream_sse_events( + url=f"{self.endpoint}/api/jobs/{namespace}/{job_id}/{route}", + log_label=f"jobs /{route} for {job_id=}", + timeout=timeout, + follow=follow, + token=token, + skip_previous_events_on_retry=skip_previous_events_on_retry, + tolerated_status_codes=tolerated_status_codes, + tolerated_exception_types=tolerated_exception_types, + on_iteration_end=has_job_finished, + params=params, + ) + + def fetch_job_logs( + self, + *, + job_id: str, + namespace: str | None = None, + follow: bool = False, + tail: int | None = None, + token: bool | str | None = None, + ) -> Iterable[str]: + """ + Fetch all the logs from a compute Job on Hugging Face infrastructure. + + Args: + job_id (`str`): + ID of the Job. + + namespace (`str`, *optional*): + The namespace where the Job is running. Defaults to the current user's namespace. + + follow (`bool`, *optional*): + If `True`, stream logs in real-time until the job completes (blocking). + If `False` (default), fetch only the currently available logs and return immediately (non-blocking). + + tail (`int`, *optional*): + Maximum number of lines to return from the logs. When combined with `follow=True`, + starts from the last N lines and continues streaming new logs. When `follow=False`, + returns only the last N lines from currently available logs. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Example: + + ```python + >>> from huggingface_hub import fetch_job_logs, run_job + >>> job = run_job(image="python:3.12", command=["python", "-c" ,"print('Hello from HF compute!')"]) + >>> for log in fetch_job_logs(job_id=job.id): + ... print(log) + Hello from HF compute! + + >>> # Non-blocking: fetch only currently available logs + >>> for log in fetch_job_logs(job_id=job.id, follow=False): + ... print(log) + + >>> # Stream logs starting from the last 100 lines + >>> for log in fetch_job_logs(job_id=job.id, follow=True, tail=100): + ... print(log) + ``` + """ + # - We need to retry because sometimes the /logs doesn't return logs when the job just started. + # (for example it can return only two lines: one for "Job started" and one empty line) + # - Timeouts can happen in case of build errors + # - ChunkedEncodingError can happen in case of stopped logging in the middle of streaming + # - Infinite empty log stream can happen in case of build error + # (the logs stream is infinite and empty except for the Job started message) + # - there is a ": keep-alive" every 30 seconds + + seconds_between_keep_alive = 30 + # When not following, use a short timeout: the server replays historical logs + # quickly, then pauses waiting for new events (~30s keep-alive). 5 seconds is + # enough to receive all buffered logs. + timeout = 4 * seconds_between_keep_alive if follow else 5 + params = {"tail": tail} if tail is not None else None + for event in self._fetch_running_job_sse( + job_id=job_id, + route="logs", + timeout=timeout, + skip_previous_events_on_retry=True, + follow=follow, + namespace=namespace, + token=token, + params=params, + ): + # timestamp = event["timestamp"] + if not event["data"].startswith("===== Job started"): + log = event["data"] + yield log + + def fetch_job_metrics( + self, + *, + job_id: str, + namespace: str | None = None, + token: bool | str | None = None, + ) -> Iterable[dict[str, Any]]: + """ + Fetch all the live metrics from a compute Job on Hugging Face infrastructure. + + Args: + job_id (`str`): + ID of the Job. + + namespace (`str`, *optional*): + The namespace where the Job is running. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Example: + + ```python + >>> from huggingface_hub import fetch_job_metrics, run_job + >>> job = run_job(image="python:3.12", command=["python", "-c" ,"print('Hello from HF compute!')"], flavor="a10g-small") + >>> for metrics in fetch_job_metrics(job_id=job.id): + ... print(metrics) + { + "cpu_usage_pct": 0, + "cpu_millicores": 3500, + "memory_used_bytes": 1306624, + "memory_total_bytes": 15032385536, + "rx_bps": 0, + "tx_bps": 0, + "gpus": { + "882fa930": { + "utilization": 0, + "memory_used_bytes": 0, + "memory_total_bytes": 22836000000 + } + }, + "replica": "57vr7" + } + ``` + """ + # - there is one "metric" event every second, like this: + # event: metric + # data: {"cpu_usage_pct":0,"cpu_millicores":3500,"memory_used_bytes":1417216,"memory_total_bytes":15032385536,"rx_bps":0,"tx_bps":0,"gpus":{"d901cd7f":{"utilization":0,"memory_used_bytes":0,"memory_total_bytes":22836000000}},"replica":"j6qz9"} + # - the stream doesn't end when the job finishes, so we rely on timeouts (httpx.NetworkError with Timeout as cause) + # - httpx.ReadTimeout can happen if the job is marked as running but the hardware is not available yet, that we can ignore + # - it returns an internal error 500 if the job has already finished, we simply ignore it + # - ChunkedEncodingError can happen in case of stopped logging in the middle of streaming + # - there is a ": keep-alive" every 30 seconds + seconds_between_events = 1 + yield from self._fetch_running_job_sse( + job_id=job_id, + route="metrics", + timeout=10 * seconds_between_events, + skip_previous_events_on_retry=False, + tolerated_status_codes=(500,), + namespace=namespace, + token=token, + ) + + def list_jobs( + self, + *, + status: list[JobStage | str] | JobStage | str | None = None, + labels: dict[str, str] | None = None, + timeout: int | None = None, + namespace: str | None = None, + token: bool | str | None = None, + ) -> Iterable[JobInfo]: + """ + List compute Jobs on Hugging Face infrastructure. + + Args: + status (`JobStage`, `str` or `list`, *optional*): + Only return Jobs with the given status(es), e.g. `"RUNNING"` or `[JobStage.RUNNING, JobStage.SCHEDULING]`. + See [`JobStage`] for possible values. + + labels (`dict[str, str]`, *optional*): + Only return Jobs that have all the given `key=value` labels, e.g. `{"env": "prod", "team": "ml"}`. + + timeout (`float`, *optional*): + Whether to set a timeout for the request to the Hub. + + namespace (`str`, *optional*): + The namespace from where it lists the jobs. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Returns: + `Iterable[JobInfo]`: an iterable of [`JobInfo`] objects. + """ + if namespace is None: + namespace = whoami(token=token)["name"] + params: list[tuple[str, Any]] = [] + if status is not None: + statuses = [status] if isinstance(status, (str, JobStage)) else status + params.extend(("stage", (s.value if isinstance(s, JobStage) else str(s)).upper()) for s in statuses) + if labels is not None: + params.extend(("label", f"{key}={value}") for key, value in labels.items()) + + path = f"{self.endpoint}/api/jobs/{namespace}" + headers = self._build_hf_headers(token=token) + for job_info in paginate(path, params=params, headers=headers, timeout=timeout): + yield JobInfo(**job_info, endpoint=self.endpoint) + + def list_jobs_hardware(self, token: bool | str | None = None) -> list[JobHardwareInfo]: + """ + List available hardware options for Jobs on Hugging Face infrastructure. + + Returns: + `list[JobHardwareInfo]`: A list of available hardware configurations. + + Example: + + ```python + >>> from huggingface_hub import HfApi + >>> api = HfApi() + >>> hardware_list = api.list_jobs_hardware() + >>> hardware_list[0] + JobHardwareInfo(name='cpu-basic', pretty_name='CPU Basic', cpu='2 vCPU', ram='16 GB', ephemeral_storage='20 GB', accelerator=None, unit_cost_micro_usd=167, unit_cost_usd=0.000167, unit_label='minute') + >>> hardware_list[0].name + 'cpu-basic' + + # Filter GPU options + >>> gpu_hardware = [hw for hw in hardware_list if hw.accelerator is not None] + >>> gpu_hardware[0].accelerator.model + 'T4' + ``` + """ + response = get_session().get(f"{self.endpoint}/api/jobs/hardware", headers=self._build_hf_headers(token=token)) + hf_raise_for_status(response) + return [JobHardwareInfo(**hardware) for hardware in response.json()] + + def inspect_job( + self, + *, + job_id: str, + namespace: str | None = None, + token: bool | str | None = None, + ) -> JobInfo: + """ + Inspect a compute Job on Hugging Face infrastructure. + + Args: + job_id (`str`): + ID of the Job. + + namespace (`str`, *optional*): + The namespace where the Job is running. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Example: + + ```python + >>> from huggingface_hub import inspect_job, run_job + >>> job = run_job(image="python:3.12", command=["python", "-c" ,"print('Hello from HF compute!')"]) + >>> inspect_job(job.id) + JobInfo( + id='68780d00bbe36d38803f645f', + created_at=datetime.datetime(2025, 7, 16, 20, 35, 12, 808000, tzinfo=datetime.timezone.utc), + docker_image='python:3.12', + space_id=None, + command=['python', '-c', "print('Hello from HF compute!')"], + arguments=[], + environment={}, + secrets={}, + flavor='cpu-basic', + status=JobStatus(stage='RUNNING', message=None) + ) + ``` + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + response = get_session().get( + f"{self.endpoint}/api/jobs/{namespace}/{job_id}", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + return JobInfo(**response.json(), endpoint=self.endpoint) + + @overload + def wait_for_job( + self, + job_id: str, + *, + timeout: float | None = None, + poll_interval: float = 1.0, + stages: list[JobStage] | None = None, + namespace: str | None = None, + token: bool | str | None = None, + ) -> JobInfo: ... + + @overload + def wait_for_job( + self, + job_id: list[str], + *, + timeout: float | None = None, + poll_interval: float = 1.0, + stages: list[JobStage] | None = None, + namespace: str | None = None, + token: bool | str | None = None, + ) -> list[JobInfo]: ... + + def wait_for_job( + self, + job_id: str | list[str], + *, + timeout: float | None = None, + poll_interval: float = 1.0, + stages: list[JobStage] | None = None, + namespace: str | None = None, + token: bool | str | None = None, + ) -> JobInfo | list[JobInfo]: + """ + Wait until one or more compute Jobs on Hugging Face infrastructure reach a given stage. + + Each Job status is polled (with [`inspect_job`]) every `poll_interval` seconds until its stage is one + of `stages` (terminal stages by default: `"COMPLETED"`, `"CANCELED"`, `"ERROR"` or `"DELETED"`). The + final [`JobInfo`] is returned in all cases: a failed or canceled Job does **not** raise an exception — + check `job.status.stage` to act on the outcome. + + Terminal stages always stop the wait, even when not listed in `stages`. This avoids waiting forever for + a stage the Job will never reach (e.g. waiting for `"RUNNING"` on a Job that fails during scheduling). + + Args: + job_id (`str` or `list[str]`): + ID of the Job, or a list of Job IDs to wait for. If a list is passed, a list of [`JobInfo`] + is returned (in the same order). + + timeout (`float`, *optional*): + The maximum time to wait for the Job(s) to finish, in seconds. If `None`, will wait + indefinitely. + + poll_interval (`float`, *optional*): + The time to wait between each status check, in seconds. Defaults to 1s. + + stages (`list[JobStage]`, *optional*): + The stages to wait for. Defaults to the terminal stages (`"COMPLETED"`, `"CANCELED"`, + `"ERROR"`, `"DELETED"`). Pass e.g. `[JobStage.RUNNING]` to wait for the Job to start running. + Terminal stages always stop the wait regardless of this value. + + namespace (`str`, *optional*): + The namespace where the Job(s) are running. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Returns: + [`JobInfo`] or `list[JobInfo]`: the final Job info(s). + + Raises: + `TimeoutError`: + If at least one Job has not reached one of the target stages after `timeout` seconds. + + Example: + + ```python + >>> from huggingface_hub import run_job, wait_for_job + >>> job = run_job(image="python:3.12", command=["python", "-c", "print('Hello from HF compute!')"]) + >>> wait_for_job(job_id=job.id).status.stage + 'COMPLETED' + ``` + """ + if timeout is not None and timeout < 0: + raise ValueError("`timeout` cannot be negative.") + if poll_interval <= 0: + raise ValueError("`poll_interval` must be positive.") + + # Terminal stages always stop the wait, so a Job that never reaches the target stage doesn't hang. + target_stages = set(stages) | set(TERMINAL_JOB_STAGES) if stages else set(TERMINAL_JOB_STAGES) + + if namespace is None: + namespace = self.whoami(token=token)["name"] + deadline = None if timeout is None else time.monotonic() + timeout + + def _wait_single(single_job_id: str) -> JobInfo: + while True: + job = self.inspect_job(job_id=single_job_id, namespace=namespace, token=token) + if job.status.stage in target_stages: + return job + remaining = None if deadline is None else deadline - time.monotonic() + if remaining is not None and remaining <= 0: + raise TimeoutError( + f"Job '{single_job_id}' is still in stage '{job.status.stage}' after {timeout} seconds." + ) + time.sleep(poll_interval if remaining is None else min(poll_interval, remaining)) + + if isinstance(job_id, str): + return _wait_single(job_id) + return [_wait_single(single_job_id) for single_job_id in job_id] + + def cancel_job( + self, + *, + job_id: str, + namespace: str | None = None, + token: bool | str | None = None, + ) -> None: + """ + Cancel a compute Job on Hugging Face infrastructure. + + Args: + job_id (`str`): + ID of the Job. + + namespace (`str`, *optional*): + The namespace where the Job is running. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + response = get_session().post( + f"{self.endpoint}/api/jobs/{namespace}/{job_id}/cancel", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + + def update_job_labels( + self, + *, + job_id: str, + labels: dict[str, str], + namespace: str | None = None, + token: bool | str | None = None, + ) -> JobInfo: + """ + Update labels of an existing Job. + + Replaces all existing user-provided labels with the new labels. + + Args: + job_id (`str`): + ID of the Job. + + labels (`dict[str, str]`): + New labels to set on the job. Replaces all existing labels. + Both keys and values must be max 100 characters and contain only + alphanumeric characters, dots, dashes, and underscores. + + namespace (`str`, *optional*): + The namespace where the Job is running. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Returns: + [`JobInfo`]: The updated Job info. + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + response = get_session().put( + f"{self.endpoint}/api/jobs/{namespace}/{job_id}/labels", + headers=self._build_hf_headers(token=token), + json={"labels": labels}, + ) + hf_raise_for_status(response) + return JobInfo(**response.json(), endpoint=self.endpoint) + + @experimental + def run_uv_job( + self, + script: str, + *, + script_args: list[str] | None = None, + dependencies: list[str] | None = None, + python: str | None = None, + image: str | None = None, + env: dict[str, Any] | None = None, + secrets: dict[str, Any] | None = None, + flavor: JobHardware | str | None = None, + timeout: int | float | str | None = None, + name: str | None = None, + labels: dict[str, str] | None = None, + volumes: list[Volume] | None = None, + expose: list[int] | None = None, + ssh: bool = False, + namespace: str | None = None, + token: bool | str | None = None, + ) -> JobInfo: + """ + Run a UV script Job on Hugging Face infrastructure. + + Args: + script (`str`): + Path or URL of the UV script, or a command. + + script_args (`list[str]`, *optional*) + Arguments to pass to the script or command. + + dependencies (`list[str]`, *optional*) + Dependencies to use to run the UV script. + + python (`str`, *optional*) + Use a specific Python version. Default is 3.12. + + image (`str`, *optional*, defaults to "ghcr.io/astral-sh/uv:python3.12-bookworm"): + Use a custom Docker image with `uv` installed. + + env (`dict[str, Any]`, *optional*): + Defines the environment variables for the Job. + + secrets (`dict[str, Any]`, *optional*): + Defines the secret environment variables for the Job. + + flavor (`str`, *optional*): + Flavor for the hardware. See [`JobHardware`] for possible values. + Defaults to `"cpu-basic"`. + + timeout (`Union[int, float, str]`, *optional*): + Max duration for the Job: int with s (seconds, default), m (minutes), h (hours) or d (days). + Example: `300` or `"5m"` for 5 minutes. + + name (`str`, *optional*): + A name for the Job. Stored as the `name` label. Cannot be passed together with a `name` key in + `labels`. Names do not have to be unique. Defaults to a name derived from script and its arguments (with a short hash suffix). + + labels (`dict[str, str]`, *optional*): + Labels to attach to the job (key-value pairs). + + volumes (`list[Volume]`, *optional*): + Hugging Face Buckets or Repos to mount as volumes in the job container. + Each volume is a [`Volume`] with `type` (`"bucket"`, `"model"`, `"dataset"`, or `"space"`), + `source` (e.g. `"username/my-bucket"`), and `mount_path` (e.g. `"/data"`). + + expose (`list[int]`, *optional*): + Container ports to expose through the jobs proxy. Each listed port is reachable + on the public jobs domain (e.g. `https://--8000.hf.jobs`). Access always + requires an HF token with read access to the job's namespace. + + ssh (`bool`, *optional*): + If True, the job's container is reachable over SSH at the URL given by `job.status.ssh_url` + (e.g. `ssh @ssh.hf.jobs`, or `hf jobs ssh ` from the CLI). Connecting requires + write access to the job's namespace and an SSH public key registered on the Hub + (https://huggingface.co/settings/keys). Defaults to False. + + namespace (`str`, *optional*): + The namespace where the Job will be created. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Example: + + Run a script from a URL: + + ```python + >>> from huggingface_hub import run_uv_job + >>> script = "https://raw.githubusercontent.com/huggingface/trl/refs/heads/main/trl/scripts/sft.py" + >>> script_args = ["--model_name_or_path", "Qwen/Qwen2-0.5B", "--dataset_name", "trl-lib/Capybara", "--push_to_hub"] + >>> run_uv_job(script, script_args=script_args, dependencies=["trl"], flavor="a10g-small") + ``` + + Run a local script: + + ```python + >>> from huggingface_hub import run_uv_job + >>> script = "my_sft.py" + >>> script_args = ["--model_name_or_path", "Qwen/Qwen2-0.5B", "--dataset_name", "trl-lib/Capybara", "--push_to_hub"] + >>> run_uv_job(script, script_args=script_args, dependencies=["trl"], flavor="a10g-small") + ``` + + Run a command: + + ```python + >>> from huggingface_hub import run_uv_job + >>> script = "lighteval" + >>> script_args= ["endpoint", "inference-providers", "model_name=openai/gpt-oss-20b,provider=auto", "lighteval|gsm8k|0|0"] + >>> run_uv_job(script, script_args=script_args, dependencies=["lighteval"], flavor="a10g-small") + ``` + + Mount volumes, e.g. to save model checkpoints during training: + + ```python + >>> from huggingface_hub import Volume, run_uv_job + >>> script = "my_sft.py" + >>> script_args = ["--output_dir", "/training-outputs/training-v3-final", ...] + >>> checkpoints_bucket = Volume(type="bucket", source="username/my-bucket", mount_path="/training-outputs") + >>> run_uv_job(script, script_args=script_args, volumes=[checkpoints_bucket]) + ``` + """ + image = image or "ghcr.io/astral-sh/uv:python3.12-bookworm" + env = env or {} + secrets = secrets or {} + + if name is None and not (labels and "name" in labels): + name = _default_job_name_from_script(script, script_args or []) + + # Build command + command, env, secrets, extra_volumes = self._create_uv_command_env_and_secrets( + script=script, + script_args=script_args, + dependencies=dependencies, + python=python, + env=env, + secrets=secrets, + namespace=namespace, + token=token, + volumes=volumes, + ) + if extra_volumes: + volumes = (volumes or []) + extra_volumes + # Create RunCommand args + return self.run_job( + image=image, + command=command, + env=env, + secrets=secrets, + flavor=flavor, + timeout=timeout, + name=name, + labels=labels, + volumes=volumes, + expose=expose, + ssh=ssh, + namespace=namespace, + token=token, + ) + + def create_scheduled_job( + self, + *, + image: str, + command: list[str], + schedule: str, + suspend: bool | None = None, + concurrency: bool | None = None, + env: dict[str, Any] | None = None, + secrets: dict[str, Any] | None = None, + flavor: JobHardware | str | None = None, + timeout: int | float | str | None = None, + name: str | None = None, + labels: dict[str, str] | None = None, + volumes: list[Volume] | None = None, + expose: list[int] | None = None, + namespace: str | None = None, + token: bool | str | None = None, + ) -> ScheduledJobInfo: + """ + Create scheduled compute Jobs on Hugging Face infrastructure. + + Args: + image (`str`): + The Docker image to use. + Examples: `"ubuntu"`, `"python:3.12"`, `"pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel"`. + Example with an image from a Space: `"hf.co/spaces/lhoestq/duckdb"`. + + command (`list[str]`): + The command to run. Example: `["echo", "hello"]`. + + schedule (`str`): + One of "@annually", "@yearly", "@monthly", "@weekly", "@daily", "@hourly", or a + CRON schedule expression (e.g., '0 9 * * 1' for 9 AM every Monday). + + suspend (`bool`, *optional*): + If True, the scheduled Job is suspended (paused). Defaults to False. + + concurrency (`bool`, *optional*): + If True, multiple instances of this Job can run concurrently. Defaults to False. + + env (`dict[str, Any]`, *optional*): + Defines the environment variables for the Job. + + secrets (`dict[str, Any]`, *optional*): + Defines the secret environment variables for the Job. + + flavor (`str`, *optional*): + Flavor for the hardware. See [`JobHardware`] for possible values. + Defaults to `"cpu-basic"`. + + timeout (`Union[int, float, str]`, *optional*): + Max duration for the Job: int with s (seconds, default), m (minutes), h (hours) or d (days). + Example: `300` or `"5m"` for 5 minutes. + + name (`str`, *optional*): + A name for the scheduled Job. Stored as the `name` label. Cannot be passed together with a `name` + key in `labels`. Names do not have to be unique. Defaults to a name derived from image and command (with a short hash suffix). + + labels (`dict[str, str]`, *optional*): + Labels to attach to the job (key-value pairs). + + volumes (`list[Volume]`, *optional*): + Hugging Face Buckets or Repos to mount as volumes in the job container. + Each volume is a [`Volume`] with `type` (`"bucket"`, `"model"`, `"dataset"`, or `"space"`), + `source` (e.g. `"username/my-bucket"`), and `mount_path` (e.g. `"/data"`). + + expose (`list[int]`, *optional*): + Container ports to expose through the jobs proxy. Each listed port is reachable + on the public jobs domain (e.g. `https://--8000.hf.jobs`). Access always + requires an HF token with read access to the job's namespace. + + namespace (`str`, *optional*): + The namespace where the Job will be created. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Example: + Create your first scheduled Job: + + ```python + >>> from huggingface_hub import create_scheduled_job + >>> create_scheduled_job(image="python:3.12", command=["python", "-c" ,"print('Hello from HF compute!')"], schedule="@hourly") + ``` + + Use a CRON schedule expression: + + ```python + >>> from huggingface_hub import create_scheduled_job + >>> create_scheduled_job(image="python:3.12", command=["python", "-c" ,"print('this runs every 5min')"], schedule="*/5 * * * *") + ``` + + Create a scheduled GPU Job: + + ```python + >>> from huggingface_hub import create_scheduled_job + >>> image = "pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel" + >>> command = ["python", "-c", "import torch; print(f"This code ran with the following GPU: {torch.cuda.get_device_name()}")"] + >>> create_scheduled_job(image, command, flavor="a10g-small", schedule="@hourly") + ``` + + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + if name is None and not (labels and "name" in labels): + name = _default_job_name_from_image(image, command) + + # prepare payload to send to HF Jobs API + job_spec = _create_job_spec( + image=image, + command=command, + env=env, + secrets=secrets, + flavor=flavor, + timeout=timeout, + name=name, + labels=labels, + volumes=volumes, + expose=expose, + ) + input_json: dict[str, Any] = { + "jobSpec": job_spec, + "schedule": schedule, + } + if concurrency is not None: + input_json["concurrency"] = concurrency + if suspend is not None: + input_json["suspend"] = suspend + response = get_session().post( + f"{self.endpoint}/api/scheduled-jobs/{namespace}", + json=input_json, + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + scheduled_job_info = response.json() + return ScheduledJobInfo(**scheduled_job_info) + + def list_scheduled_jobs( + self, + *, + timeout: int | None = None, + namespace: str | None = None, + token: bool | str | None = None, + ) -> list[ScheduledJobInfo]: + """ + List scheduled compute Jobs on Hugging Face infrastructure. + + Args: + timeout (`float`, *optional*): + Whether to set a timeout for the request to the Hub. + + namespace (`str`, *optional*): + The namespace from where it lists the jobs. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + response = get_session().get( + f"{self.endpoint}/api/scheduled-jobs/{namespace}", + headers=self._build_hf_headers(token=token), + timeout=timeout, + ) + hf_raise_for_status(response) + return [ScheduledJobInfo(**scheduled_job_info) for scheduled_job_info in response.json()] + + def inspect_scheduled_job( + self, + *, + scheduled_job_id: str, + namespace: str | None = None, + token: bool | str | None = None, + ) -> ScheduledJobInfo: + """ + Inspect a scheduled compute Job on Hugging Face infrastructure. + + Args: + scheduled_job_id (`str`): + ID of the scheduled Job. + + namespace (`str`, *optional*): + The namespace where the scheduled Job is. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Example: + + ```python + >>> from huggingface_hub import inspect_job, create_scheduled_job + >>> scheduled_job = create_scheduled_job(image="python:3.12", command=["python", "-c" ,"print('Hello from HF compute!')"], schedule="@hourly") + >>> inspect_scheduled_job(scheduled_job.id) + ``` + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + response = get_session().get( + f"{self.endpoint}/api/scheduled-jobs/{namespace}/{scheduled_job_id}", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + return ScheduledJobInfo(**response.json()) + + def delete_scheduled_job( + self, + *, + scheduled_job_id: str, + namespace: str | None = None, + token: bool | str | None = None, + ) -> None: + """ + Delete a scheduled compute Job on Hugging Face infrastructure. + + Args: + scheduled_job_id (`str`): + ID of the scheduled Job. + + namespace (`str`, *optional*): + The namespace where the scheduled Job is. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + response = get_session().delete( + f"{self.endpoint}/api/scheduled-jobs/{namespace}/{scheduled_job_id}", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + + def suspend_scheduled_job( + self, + *, + scheduled_job_id: str, + namespace: str | None = None, + token: bool | str | None = None, + ) -> None: + """ + Suspend (pause) a scheduled compute Job on Hugging Face infrastructure. + + Args: + scheduled_job_id (`str`): + ID of the scheduled Job. + + namespace (`str`, *optional*): + The namespace where the scheduled Job is. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + response = get_session().post( + f"{self.endpoint}/api/scheduled-jobs/{namespace}/{scheduled_job_id}/suspend", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + + def resume_scheduled_job( + self, + *, + scheduled_job_id: str, + namespace: str | None = None, + token: bool | str | None = None, + ) -> None: + """ + Resume (unpause) a scheduled compute Job on Hugging Face infrastructure. + + Args: + scheduled_job_id (`str`): + ID of the scheduled Job. + + namespace (`str`, *optional*): + The namespace where the scheduled Job is. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + response = get_session().post( + f"{self.endpoint}/api/scheduled-jobs/{namespace}/{scheduled_job_id}/resume", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + + def trigger_scheduled_job( + self, + *, + scheduled_job_id: str, + namespace: str | None = None, + token: bool | str | None = None, + ) -> JobInfo: + """ + Trigger a scheduled Job to run immediately. + + This triggers one immediate run of the scheduled job's spec. It does **not** modify the schedule + and does **not** affect the next scheduled run. If an instance is already running and the scheduled + job does not allow concurrent runs, the request is rejected (HTTP 409). + + Args: + scheduled_job_id (`str`): + ID of the scheduled Job. + + namespace (`str`, *optional*): + The namespace where the scheduled Job is. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Returns: + [`JobInfo`]: Info about the triggered run. + + Raises: + [`~utils.HfHubHTTPError`]: + HTTP 409 if another instance is already running and `concurrency` is disabled on the scheduled job. + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + response = get_session().post( + f"{self.endpoint}/api/scheduled-jobs/{namespace}/{scheduled_job_id}/run", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + return JobInfo(**response.json(), endpoint=self.endpoint) + + def update_scheduled_job_labels( + self, + *, + scheduled_job_id: str, + labels: dict[str, str], + namespace: str | None = None, + token: bool | str | None = None, + ) -> ScheduledJobInfo: + """ + Update labels of an existing scheduled Job. + + Replaces all existing user-provided labels with the new labels. + + Args: + scheduled_job_id (`str`): + ID of the scheduled Job. + + labels (`dict[str, str]`): + New labels to set on the scheduled job. Replaces all existing labels. + Both keys and values must be max 100 characters and contain only + alphanumeric characters, dots, dashes, and underscores. + + namespace (`str`, *optional*): + The namespace where the scheduled Job is. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Returns: + [`ScheduledJobInfo`]: The updated scheduled Job info. + """ + if namespace is None: + namespace = self.whoami(token=token)["name"] + response = get_session().put( + f"{self.endpoint}/api/scheduled-jobs/{namespace}/{scheduled_job_id}/labels", + headers=self._build_hf_headers(token=token), + json={"labels": labels}, + ) + hf_raise_for_status(response) + return ScheduledJobInfo(**response.json()) + + @experimental + def create_scheduled_uv_job( + self, + script: str, + *, + script_args: list[str] | None = None, + schedule: str, + suspend: bool | None = None, + concurrency: bool | None = None, + dependencies: list[str] | None = None, + python: str | None = None, + image: str | None = None, + env: dict[str, Any] | None = None, + secrets: dict[str, Any] | None = None, + flavor: JobHardware | str | None = None, + timeout: int | float | str | None = None, + name: str | None = None, + labels: dict[str, str] | None = None, + volumes: list[Volume] | None = None, + expose: list[int] | None = None, + namespace: str | None = None, + token: bool | str | None = None, + ) -> ScheduledJobInfo: + """ + Run a UV script Job on Hugging Face infrastructure. + + Args: + script (`str`): + Path or URL of the UV script, or a command. + + script_args (`list[str]`, *optional*) + Arguments to pass to the script, or a command. + + schedule (`str`): + One of "@annually", "@yearly", "@monthly", "@weekly", "@daily", "@hourly", or a + CRON schedule expression (e.g., '0 9 * * 1' for 9 AM every Monday). + + suspend (`bool`, *optional*): + If True, the scheduled Job is suspended (paused). Defaults to False. + + concurrency (`bool`, *optional*): + If True, multiple instances of this Job can run concurrently. Defaults to False. + + dependencies (`list[str]`, *optional*) + Dependencies to use to run the UV script. + + python (`str`, *optional*) + Use a specific Python version. Default is 3.12. + + image (`str`, *optional*, defaults to "ghcr.io/astral-sh/uv:python3.12-bookworm"): + Use a custom Docker image with `uv` installed. + + env (`dict[str, Any]`, *optional*): + Defines the environment variables for the Job. + + secrets (`dict[str, Any]`, *optional*): + Defines the secret environment variables for the Job. + + flavor (`str`, *optional*): + Flavor for the hardware. See [`JobHardware`] for possible values. + Defaults to `"cpu-basic"`. + + timeout (`Union[int, float, str]`, *optional*): + Max duration for the Job: int with s (seconds, default), m (minutes), h (hours) or d (days). + Example: `300` or `"5m"` for 5 minutes. + + name (`str`, *optional*): + A name for the scheduled Job. Stored as the `name` label. Cannot be passed together with a `name` + key in `labels`. Names do not have to be unique. Defaults to a name derived from script and its arguments (with a short hash suffix). + + labels (`dict[str, str]`, *optional*): + Labels to attach to the job (key-value pairs). + + volumes (`list[Volume]`, *optional*): + Hugging Face Buckets or Repos to mount as volumes in the job container. + Each volume is a [`Volume`] with `type` (`"bucket"`, `"model"`, `"dataset"`, or `"space"`), + `source` (e.g. `"username/my-bucket"`), and `mount_path` (e.g. `"/data"`). + + expose (`list[int]`, *optional*): + Container ports to expose through the jobs proxy. Each listed port is reachable + on the public jobs domain (e.g. `https://--8000.hf.jobs`). Access always + requires an HF token with read access to the job's namespace. + + namespace (`str`, *optional*): + The namespace where the Job will be created. Defaults to the current user's namespace. + + token `(Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Example: + + Schedule a script from a URL: + + ```python + >>> from huggingface_hub import create_scheduled_uv_job + >>> script = "https://raw.githubusercontent.com/huggingface/trl/refs/heads/main/trl/scripts/sft.py" + >>> script_args = ["--model_name_or_path", "Qwen/Qwen2-0.5B", "--dataset_name", "trl-lib/Capybara", "--push_to_hub"] + >>> create_scheduled_uv_job(script, script_args=script_args, dependencies=["trl"], flavor="a10g-small", schedule="@weekly") + ``` + + Schedule a local script: + + ```python + >>> from huggingface_hub import create_scheduled_uv_job + >>> script = "my_sft.py" + >>> script_args = ["--model_name_or_path", "Qwen/Qwen2-0.5B", "--dataset_name", "trl-lib/Capybara", "--push_to_hub"] + >>> create_scheduled_uv_job(script, script_args=script_args, dependencies=["trl"], flavor="a10g-small", schedule="@weekly") + ``` + + Schedule a command: + + ```python + >>> from huggingface_hub import create_scheduled_uv_job + >>> script = "lighteval" + >>> script_args= ["endpoint", "inference-providers", "model_name=openai/gpt-oss-20b,provider=auto", "lighteval|gsm8k|0|0"] + >>> create_scheduled_uv_job(script, script_args=script_args, dependencies=["lighteval"], flavor="a10g-small", schedule="@weekly") + ``` + """ + image = image or "ghcr.io/astral-sh/uv:python3.12-bookworm" + if name is None and not (labels and "name" in labels): + name = _default_job_name_from_script(script, script_args or []) + + # Build command + command, env, secrets, extra_volumes = self._create_uv_command_env_and_secrets( + script=script, + script_args=script_args, + dependencies=dependencies, + python=python, + env=env, + secrets=secrets, + namespace=namespace, + token=token, + volumes=volumes, + ) + if extra_volumes: + volumes = (volumes or []) + extra_volumes + # Create RunCommand args + return self.create_scheduled_job( + image=image, + command=command, + schedule=schedule, + suspend=suspend, + concurrency=concurrency, + env=env, + secrets=secrets, + flavor=flavor, + timeout=timeout, + name=name, + labels=labels, + volumes=volumes, + expose=expose, + namespace=namespace, + token=token, + ) + + def _create_uv_command_env_and_secrets( + self, + *, + script: str, + script_args: list[str] | None, + dependencies: list[str] | None, + python: str | None, + env: dict[str, Any] | None, + secrets: dict[str, Any] | None, + namespace: str | None, + token: bool | str | None, + volumes: list[Volume] | None = None, + ) -> tuple[list[str], dict[str, Any], dict[str, Any], list[Volume]]: + env = env or {} + secrets = secrets or {} + + # Build command + uv_args = [] + if dependencies: + for dependency in dependencies: + uv_args += ["--with", dependency] + if python: + uv_args += ["--python", python] + script_args = script_args or [] + + if namespace is None: + namespace = self.whoami(token=token)["name"] + + # Find the local files to pass to the job + local_files_to_include = {candidate for candidate in [script] + script_args if Path(candidate).is_file()} + # Fail early for missing scripts or config files + missing_local_files = { + candidate + for candidate in [script] + script_args + if not Path(candidate).is_file() + and Path(candidate).suffix in [".py", ".sh", ".yaml", ".yml", ".toml"] + and not candidate.startswith("https://") + and not candidate.startswith("http://") + } + if missing_local_files: + raise FileNotFoundError(", ".join(missing_local_files)) + + if len(local_files_to_include) == 0: + # Direct URL execution or command - no upload needed + command = ["uv", "run"] + uv_args + [script] + script_args + return command, env, secrets, [] + + # Find appropriate remote file names + remote_to_local_file_names: dict[str, str] = {} + for local_file_to_include in local_files_to_include: + local_file_path = Path(local_file_to_include) + # Sanitize spaces for predictable remote paths + remote_file_path = Path(local_file_path.name.replace(" ", "_")) + if remote_file_path.name in remote_to_local_file_names: + for i in itertools.count(): + remote_file_name = remote_file_path.with_stem(remote_file_path.stem + f"({i})").name + if remote_file_name not in remote_to_local_file_names: + remote_to_local_file_names[remote_file_name] = local_file_to_include + break + else: + remote_to_local_file_names[remote_file_path.name] = local_file_to_include + local_to_remote_file_names = { + local_file_to_include: remote_file_name + for remote_file_name, local_file_to_include in remote_to_local_file_names.items() + } + + # Local files are shipped to the job via a bucket mounted at /data. + existing_mount_paths = {v.mount_path for v in (volumes or [])} + if constants.HF_JOBS_ARTIFACTS_MOUNT_PATH in existing_mount_paths: + raise ValueError( + f"Mount path {constants.HF_JOBS_ARTIFACTS_MOUNT_PATH!r} is reserved for Jobs artifacts when running local scripts. Mount your volume at a different path." + ) + + extra_volumes = self._upload_scripts_to_bucket( + namespace=namespace, + remote_to_local_file_names=remote_to_local_file_names, + token=token, + ) + # Rewrite script and script_args to reference the mounted path. The bucket + # volume is scoped to the per-job subfolder (via `Volume.path`), so the job + # container sees the uploaded files directly at the mount root. + mount_path = constants.HF_JOBS_ARTIFACTS_MOUNT_PATH + if script in local_to_remote_file_names: + script = f"{mount_path}/{local_to_remote_file_names[script]}" + script_args = [ + f"{mount_path}/{local_to_remote_file_names[arg]}" if arg in local_to_remote_file_names else arg + for arg in script_args + ] + command = ["uv", "run"] + uv_args + [script] + script_args + return command, env, secrets, extra_volumes + + def _upload_scripts_to_bucket( + self, + *, + namespace: str, + remote_to_local_file_names: dict[str, str], + token: bool | str | None, + ) -> list[Volume]: + """Upload script files to a per-job subfolder in the artifacts bucket. + + Creates a bucket `/jobs-artifacts` (if it doesn't exist) and uploads + each script to `{timestamp}-{random}/{remote_name}` inside it. Returns a + [`Volume`] scoped to that bucket subfolder. Volume is in read-write mode so the Job can save data back to this bucket. + """ + bucket_id = f"{namespace}/{constants.HF_JOBS_ARTIFACTS_BUCKET_NAME}" + subfolder_id = f"{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S')}-{token_hex(3)}" + + bucket_url = self.create_bucket(bucket_id=bucket_id, exist_ok=True, token=token, private=True) + + add_ops: list[tuple[str | Path | bytes, str]] = [ + (Path(local_path), f"{subfolder_id}/{remote_name}") + for remote_name, local_path in remote_to_local_file_names.items() + ] + self.batch_bucket_files(bucket_id=bucket_id, add=add_ops, token=token) + print(f"Your script and Job artifacts will be saved in this bucket: {bucket_url.url}") + + volume = Volume( + type="bucket", + source=bucket_id, + mount_path=constants.HF_JOBS_ARTIFACTS_MOUNT_PATH, + path=subfolder_id, + read_only=False, + ) + return [volume] + + def sync_job_volume( + self, + source: str | Path, + mount_path: str, + *, + remote_name: str | None = None, + read_only: bool = True, + namespace: str | None = None, + token: bool | str | None = None, + ) -> Volume: + """Sync a local directory to a bucket and return a [`Volume`] ready to mount in a Job. + + Files are uploaded to a subfolder of the `{namespace}/jobs-artifacts` bucket (auto-created as + private; a warning is emitted if it already exists and is public) using the same sync logic + as [`sync_bucket`]: re-syncing the same directory only + uploads new or modified files. By default the subfolder name is derived from the directory + path and the machine's hostname, so repeated calls from the same directory reuse the same + remote folder. Pass `remote_name` to use a fixed name instead. + + Note that the data is *copied* to the bucket, not mounted live: changes made locally after + the sync are not visible to the Job (re-run `sync_job_volume` to update), and the volume is + mounted read-only by default. To retrieve data written by a Job to a read-write volume, sync + the bucket folder back with [`sync_bucket`]. If the source directory is empty (e.g. an output + directory), a placeholder `.keep` file is uploaded so the volume can still be mounted. + + Args: + source (`str` or `Path`): + Path to a local directory to sync. + mount_path (`str`): + Mount path inside the Job container, e.g. `"/inputs"`. Must start with `/`. + remote_name (`str`, *optional*): + Name of the bucket subfolder to sync to. Defaults to a `{dirname}-{hash}` name derived + from the source path and the machine's hostname. + read_only (`bool`, *optional*, defaults to `True`): + Mount the volume read-only in the Job. Pass `False` to let the Job write back to the + bucket folder (e.g. to retrieve outputs with [`sync_bucket`] afterwards). + namespace (`str`, *optional*): + The namespace owning the `jobs-artifacts` bucket. Defaults to the current user's + namespace. Use the same namespace as the Job that will mount the volume. + token (`Union[bool, str, None]`, *optional*): + A valid user access token. If not provided, the locally saved token will be used, which is the + recommended authentication method. Set to `False` to disable authentication. + Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication. + + Returns: + [`Volume`]: A bucket volume scoped to the synced subfolder, to pass in the `volumes` list + of [`run_job`], [`run_uv_job`], [`create_scheduled_job`] or [`create_scheduled_uv_job`]. + + Example: + ```python + >>> from huggingface_hub import run_uv_job, sync_job_volume + + # Upload ./training-data once, then run multiple jobs against it + >>> volume = sync_job_volume("./training-data", "/data") + >>> run_uv_job("train.py", script_args=["--learning-rate", "0.01"], volumes=[volume]) + >>> run_uv_job("train.py", script_args=["--learning-rate", "0.05"], volumes=[volume]) + + # Read-write volume to retrieve outputs after the job completes + >>> volume = sync_job_volume("./outputs", "/outputs", read_only=False) + >>> job = run_uv_job("process.py", volumes=[volume]) + ``` + """ + if not mount_path.startswith("/"): + raise ValueError( + f"Mount path must be an absolute path inside the container (e.g. '/data'), got {mount_path!r}." + ) + source_path = Path(source).expanduser() + if not source_path.is_dir(): + raise ValueError(f"Source must be an existing local directory: '{source}'.") + + if namespace is None: + namespace = self.whoami(token=token)["name"] + bucket_id = f"{namespace}/{constants.HF_JOBS_ARTIFACTS_BUCKET_NAME}" + folder = remote_name or _derive_job_volume_name(source_path) + + # The jobs-artifacts bucket holds user scripts and data, so it must be private. A new bucket + # is created private here; if it already exists we cannot change its visibility, so warn when + # it is public instead of silently uploading the data to a publicly accessible bucket. + self.create_bucket(bucket_id=bucket_id, exist_ok=True, private=True, token=token) + if not self.bucket_info(bucket_id=bucket_id, token=token).private: + warnings.warn( + f"Bucket '{bucket_id}' already exists and is public: data synced for the Job will be " + f"publicly accessible. Make it private from {self.endpoint}/buckets/{bucket_id}.", + UserWarning, + ) + self.sync_bucket(str(source_path), f"hf://buckets/{bucket_id}/{folder}", token=token) + if not any(path.is_file() for path in source_path.rglob("*")): + # A folder cannot be empty in a bucket, and mounting a non-existent folder fails the Job. + # Upload a placeholder file so an empty directory (e.g. an output dir) can still be mounted. + self.batch_bucket_files(bucket_id, add=[(b"", f"{folder}/.keep")], token=token) + + return Volume(type="bucket", source=bucket_id, mount_path=mount_path, path=folder, read_only=read_only) + + @validate_hf_hub_args + def create_bucket( + self, + bucket_id: str, + *, + private: bool | None = None, + resource_group_id: str | None = None, + region: REPO_REGIONS | None = None, + exist_ok: bool = False, + token: bool | str | None = None, + ) -> BucketUrl: + """Create a bucket on the Hub. + + Args: + bucket_id (`str`): + A namespace (user or an organization) and a bucket name separated by a `/`. + If no namespace is provided, the bucket will be created in the current user's namespace. + private (`bool`, *optional*): + Whether to make the bucket private. If `None` (default), the bucket will be public unless the + organization's default is private. + resource_group_id (`str`, *optional*): + Resource group in which to create the bucket. Resource groups are only available for Enterprise Hub + organizations and allow to define which members of the organization can access the resource. The ID + of a resource group can be found in the URL of the resource's page on the Hub + (e.g. `"66670e5163145ca562cb1988"`). To learn more about resource groups, see + https://huggingface.co/docs/hub/en/security-resource-groups. + region (`Literal["us", "eu"]`, *optional*): + Cloud region in which to create the bucket. Can be one of `"us"` or `"eu"`. If not specified, the bucket will be + created in the default region. Requires Team plan or above. + exist_ok (`bool`, *optional*, defaults to `False`): + If `True`, do not raise an error if the bucket already exists. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`BucketUrl`]: URL to the newly created bucket containing + attributes like `endpoint`, `namespace`, and `bucket_id`. + + Example: + ```python + >>> from huggingface_hub import create_bucket + + >>> url = create_bucket(bucket_id="my-bucket") + >>> url.bucket_id + 'user/my-bucket' + >>> url.url + 'https://huggingface.co/buckets/user/my-bucket' + >>> url.uri.to_uri() + 'hf://buckets/user/my-bucket' + + >>> create_bucket(bucket_id="my-bucket", private=True, exist_ok=True) + BucketUrl(...) + + >>> create_bucket(bucket_id="my-bucket", region="us") + BucketUrl(...) + ``` + """ + payload: dict[str, Any] = {} + if private is not None: + payload["private"] = private + if resource_group_id is not None: + payload["resourceGroupId"] = resource_group_id + if region is not None: + payload["region"] = region + + if "/" not in bucket_id: + namespace, name = "me", bucket_id # "me" namespace refers to the current user + else: + parsed = _parse_bucket_uri(bucket_id) + if parsed.path_in_repo: + raise ValueError(f"Invalid bucket ID: {bucket_id}") + namespace, name = parsed.id.split("/") + + response = get_session().post( + f"{self.endpoint}/api/buckets/{namespace}/{name}", + headers=self._build_hf_headers(token=token), + json=payload, + ) + try: + hf_raise_for_status(response) + except HfHubHTTPError as err: + if exist_ok and err.response.status_code == 409: + # Bucket already exists and `exist_ok=True` + pass + elif exist_ok and err.response.status_code in (401, 403): + # 401 -> if JWT token without create bucket scope + # 403 -> if no write permission on the namespace + # In both cases, bucket might already exist + try: + self.bucket_info(bucket_id=bucket_id, token=token) + return BucketUrl(f"{self.endpoint}/buckets/{bucket_id}", endpoint=self.endpoint) + except HfHubHTTPError: + raise err + else: + raise + return BucketUrl(response.json()["url"], endpoint=self.endpoint) + + @validate_hf_hub_args + def bucket_info( + self, + bucket_id: str, + *, + token: bool | str | None = None, + ) -> BucketInfo: + """Get information about a specific bucket on the Hub. + + Args: + bucket_id (`str`): + The ID of the bucket (e.g. `"username/my-bucket"`). + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`BucketInfo`]: The bucket information. + + Raises: + [`~errors.BucketNotFoundError`]: If the bucket cannot be found. This may be because it doesn't exist, + or because it is set to `private` and you do not have access. + + Example: + ```python + >>> from huggingface_hub import bucket_info + >>> info = bucket_info(bucket_id="Wauplin/first-bucket") + >>> info.id + 'Wauplin/first-bucket' + >>> info.private + False + >>> info.created_at + datetime.datetime(2026, 2, 6, 17, 37, 57, tzinfo=datetime.timezone.utc) + >>> info.size + 551879671 + >>> info.total_files + 12 + ``` + """ + response = get_session().get( + f"{self.endpoint}/api/buckets/{bucket_id}", + headers=self._build_hf_headers(token=token), + ) + hf_raise_for_status(response) + return BucketInfo(**response.json()) + + @validate_hf_hub_args + def list_buckets( + self, + namespace: str | None = None, + *, + search: str | None = None, + token: bool | str | None = None, + ) -> Iterable[BucketInfo]: + """List buckets on the Hub under a certain namespace. + + Args: + namespace (`str`, *optional*): + List buckets under this namespace (user or organization). Defaults to listing user's buckets. + search (`str`, *optional*): + A search string to filter bucket names. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[BucketInfo]`: An iterable of [`BucketInfo`] objects. + + Example: + ```python + >>> from huggingface_hub import list_buckets + >>> for bucket in list_buckets(): # lists buckets in the user's namespace + ... print(bucket) + + >>> for bucket in list_buckets(namespace="huggingface"): # lists buckets in the "huggingface" organization + ... print(bucket) + + >>> for bucket in list_buckets(search="my-prefix"): # filter buckets by name + ... print(bucket) + ``` + """ + if namespace is None: + namespace = "me" + params: dict[str, Any] = {} + if search is not None: + params["search"] = search + for item in paginate( + f"{self.endpoint}/api/buckets/{namespace}", params=params, headers=self._build_hf_headers(token=token) + ): + yield BucketInfo(**item) + + @validate_hf_hub_args + def delete_bucket( + self, + bucket_id: str, + *, + missing_ok: bool = False, + token: bool | str | None = None, + ) -> None: + """Delete a bucket from the Hub. + + Args: + bucket_id (`str`): + The ID of the bucket (e.g. `"username/my-bucket"`). + missing_ok (`bool`, *optional*, defaults to `False`): + If `True`, do not raise an error if the bucket does not exist. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Raises: + [`~errors.BucketNotFoundError`]: If the bucket cannot be found and `missing_ok` is set to `False` (default). + + Example: + ```python + >>> from huggingface_hub import delete_bucket + >>> delete_bucket(bucket_id="Wauplin/first-bucket") + >>> delete_bucket(bucket_id="Wauplin/first-bucket", missing_ok=True) + ``` + """ + response = get_session().delete( + f"{self.endpoint}/api/buckets/{bucket_id}", + headers=self._build_hf_headers(token=token), + ) + + try: + hf_raise_for_status(response) + except HfHubHTTPError as e: + if e.response.status_code != 404 or not missing_ok: + raise + + @validate_hf_hub_args + def move_bucket( + self, + from_id: str, + to_id: str, + *, + token: bool | str | None = None, + ) -> None: + """Move a bucket from "namespace1/repo_name1" to "namespace2/repo_name2" + + Note there are certain limitations. For more information about moving + repositories, please see + https://hf.co/docs/hub/repositories-settings#renaming-or-transferring-a-repo. + + Args: + from_id (`str`): + A namespace (user or an organization) and a bucket name separated + by a `/`. Original bucket identifier (e.g. `"username/my-bucket"`). + to_id (`str`): + A namespace (user or an organization) and a bucket name separated + by a `/`. Final bucket identifier (e.g. `"username/new-bucket-name"` + or `"organization/my-bucket"`). + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Raises: + [`~errors.BucketNotFoundError`]: + If the source bucket cannot be found. This may be because it doesn't exist, + or because it is set to `private` and you do not have access. + + Example: + ```python + >>> from huggingface_hub import move_bucket + + >>> # Rename a bucket within the same namespace + >>> move_bucket(from_id="username/old-name", to_id="username/new-name") + + >>> # Transfer a bucket to an organization + >>> move_bucket(from_id="username/my-bucket", to_id="my-org/my-bucket") + ``` + """ + if len(from_id.split("/")) != 2: + raise ValueError(f"Invalid repo_id: {from_id}. It should have a namespace (:namespace:/:repo_name:)") + + if len(to_id.split("/")) != 2: + raise ValueError(f"Invalid repo_id: {to_id}. It should have a namespace (:namespace:/:repo_name:)") + + json_payload = {"fromRepo": from_id, "toRepo": to_id, "type": "bucket"} + + path = f"{self.endpoint}/api/repos/move" + headers = self._build_hf_headers(token=token) + response = get_session().post(path, headers=headers, json=json_payload) + hf_raise_for_status(response) + + @validate_hf_hub_args + def list_bucket_tree( + self, + bucket_id: str, + prefix: str | None = None, + *, + recursive: bool | None = None, + token: str | bool | None = None, + ) -> Iterable[BucketFile | BucketFolder]: + """List files in a bucket. + + Args: + bucket_id (`str`): + The ID of the bucket (e.g. `"username/my-bucket"`). + prefix (`str`, *optional*): + Filter results to files whose path starts with this prefix. + recursive (`bool`, *optional*): + If `True`, list files recursively. If `False` (default), list files and directories only at root. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[Union[BucketFile, BucketFolder]]`: An iterable of [`BucketFile`] and [`BucketFolder`] objects + containing file and directory information (path, etc.). + + Example: + ```python + >>> from huggingface_hub import list_bucket_tree + >>> for file_info in list_bucket_tree(bucket_id="username/my-bucket"): + ... print(file_info.path) + + >>> # Filter by prefix + >>> for file_info in list_bucket_tree(bucket_id="username/my-bucket", prefix="models/"): + ... print(file_info.path) + ``` + """ + encoded_prefix = "/" + quote(prefix, safe="") if prefix else "" + params = {} + if recursive is not None: + params["recursive"] = recursive + for item in paginate( + path=f"{self.endpoint}/api/buckets/{bucket_id}/tree{encoded_prefix}", + headers=self._build_hf_headers(token=token), + params=params, + ): + if item["type"] == "file": + yield BucketFile(**item) + elif item["type"] == "directory": + yield BucketFolder(**item) + + @validate_hf_hub_args + def get_bucket_paths_info( + self, + bucket_id: str, + paths: Iterable[str], + *, + token: str | bool | None = None, + ) -> Iterable[BucketFile]: + """ + Get information about a bucket's paths. + + Calls are made in batches of 1000 paths. Results are yielded as they are received. + + Args: + bucket_id (`str`): + The ID of the bucket (e.g. `"username/my-bucket"`). + paths (`Iterable[str]`): + The paths to get information about. If a path does not exist, it is ignored without raising an exception. + Only file paths are supported. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + `Iterable[BucketFile]`: + The information about the paths, as an iterable of [`BucketFile`] objects. + + Example: + ```py + >>> from huggingface_hub import get_bucket_paths_info + >>> paths_info = get_bucket_paths_info("username/my-bucket", ["file.txt", "checkpoints/model.safetensors"]) + >>> for info in paths_info: + ... print(info) + BucketFile(type='file', path='file.txt', size=2379, xet_hash='96e637d9665bd35477b1908a23f2e254edfba0618dbd2d62f90a6baee7d139cf', mtime=datetime.datetime(2024, 9, 25, 15, 31, 2, 346000, tzinfo=datetime.timezone.utc)) + BucketFile(type='file', path='checkpoints/model.safetensors', size=2408828, xet_hash='3ed0e9fefe788ddd61d1e26eba67057e9740a064b009256fbafadf6bb95785ca', mtime=datetime.datetime(2024, 9, 25, 15, 31, 2, 346000, tzinfo=datetime.timezone.utc)) + ``` + """ + headers = self._build_hf_headers(token=token) + + for batch in chunk_iterable(paths, chunk_size=_BUCKET_PATHS_INFO_BATCH_SIZE): + response = http_backoff( + "POST", + f"{self.endpoint}/api/buckets/{bucket_id}/paths-info", + json={"paths": list(batch)}, + headers=headers, + ) + hf_raise_for_status(response) + for path_info in response.json(): + yield BucketFile(**path_info) + + @validate_hf_hub_args + def copy_files(self, source: str, destination: str, *, token: str | bool | None = None) -> None: + """Copy files between locations on the Hub. + + Copy files from a bucket or repository (model, dataset, space) to a bucket or another repository. + Both individual files and entire folders are supported. + + When copying folders, a trailing `/` on the source path uses rsync-style semantics: copy the *contents* + of the folder into the destination, without nesting the source folder itself. Without a trailing `/`, + the source folder is nested inside the destination (like `cp -r`). + + When copying from a repository to a bucket, `.gitattributes` files are automatically excluded since they + are git-specific metadata and not relevant in a bucket context. + + Repo-to-repo copies use [`CommitOperationCopy`] under the hood and create a commit on the destination + repository. Bucket-to-repo copies are not supported. + + > [!WARNING] + > Server-side copies only work within the same [storage region](https://huggingface.co/docs/hub/storage-regions). + + Args: + source (`str`): + Source location as an `hf://` URI. Can be a bucket path (e.g. `"hf://buckets/my-bucket/path/to/file"`) + or a repo path (e.g. `"hf://username/my-model/weights.bin"`, `"hf://datasets/username/my-dataset/data/"`). + destination (`str`): + Destination location as an `hf://` URI pointing to a bucket (e.g. `"hf://buckets/my-bucket/target/path"`) + or a repository (e.g. `"hf://username/my-model/target/path"`). + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError): + If source/destination URIs are invalid or if copying from a bucket to a repo. + + Example: + ```python + >>> from huggingface_hub import copy_files + + # Copy a single file between buckets + >>> copy_files("hf://buckets/my-bucket/data.bin", "hf://buckets/other-bucket/data.bin") + + # Copy a folder into another bucket (nests: backup/models/...) + >>> copy_files("hf://buckets/my-bucket/models", "hf://buckets/other-bucket/backup/") + + # Copy folder contents (trailing /): files go directly into backup/ + >>> copy_files("hf://buckets/my-bucket/models/", "hf://buckets/other-bucket/backup/") + + # Copy a file from a model repo to a bucket + >>> copy_files("hf://username/my-model/model.safetensors", "hf://buckets/my-bucket/") + + # Copy an entire dataset to a bucket + >>> copy_files("hf://datasets/username/my-dataset/", "hf://buckets/my-bucket/datasets/") + + # Copy files between repositories + >>> copy_files("hf://username/source-model/", "hf://username/dest-model/") + + # Copy a file from one repo to another + >>> copy_files("hf://username/source-model/config.json", "hf://username/dest-model/config.json") + ``` + """ + source_uri = parse_hf_uri(source) + destination_uri = parse_hf_uri(destination) + + # Rsync-style trailing slash on source: "copy contents of" instead of "copy directory into". + # Check before parsing strips the slash. + merge_contents = source.endswith("/") + + if destination_uri.is_repo: + if source_uri.is_bucket: + raise ValueError("Bucket-to-repo copy is not supported.") + self._copy_to_repo(source_uri, destination_uri, merge_contents, source, destination, token=token) + else: + self._copy_to_bucket(source_uri, destination_uri, merge_contents, source, destination, token=token) + + def _copy_to_bucket( + self, + source: HfUri, + destination: HfUri, + merge_contents: bool, + source_str: str, + destination_str: str, + *, + token: str | bool | None = None, + ) -> None: + destination_bucket_id = destination.id + destination_path = destination.path_in_repo + destination_is_directory = False + destination_exists_as_directory = False + + if destination_path == "": + destination_is_directory = True + destination_exists_as_directory = True + else: + dest_path_info = list(self.get_bucket_paths_info(destination_bucket_id, [destination_path], token=token)) + if dest_path_info: + destination_is_directory = False + else: + destination_exists_as_directory = any( + self.list_bucket_tree(destination_bucket_id, prefix=destination_path, recursive=False, token=token) + ) + destination_is_directory = destination_exists_as_directory or destination_str.endswith("/") + + all_adds: list[tuple[str, str]] = [] + all_copies: list[_BucketCopyFile] = [] + pending_downloads: list[tuple[str, str]] = [] + + def _resolve_target_path(src_file_path: str, src_root_path: str | None, is_single_file: bool) -> str: + return _resolve_copy_target_path( + src_file_path, + src_root_path, + is_single_file, + destination_path, + destination_is_directory, + destination_exists_as_directory, + merge_contents, + ) + + def _build_copy_op( + target_path: str, xet_hash: str, size: int, source_repo_type: str, source_repo_id: str + ) -> _BucketCopyFile: + """Server-side copy by xet hash — no data transfer needed.""" + return _BucketCopyFile( + destination=target_path, + xet_hash=xet_hash, + source_repo_type=source_repo_type, + source_repo_id=source_repo_id, + size=size, + ) + + def _add_repo_file(file: RepoFile, target_path: str) -> None: + if file.xet_hash is not None: + all_copies.append(_build_copy_op(target_path, file.xet_hash, file.size, source.type, source.id)) + else: + pending_downloads.append((file.path, target_path)) + + if source.is_bucket: + source_path = source.path_in_repo + source_path_info = list(self.get_bucket_paths_info(source.id, [source_path], token=token)) + + if source_path_info: + source_file = source_path_info[0] + target_path = _resolve_target_path(source_file.path, None, is_single_file=True) + all_copies.append( + _build_copy_op(target_path, source_file.xet_hash, source_file.size, "bucket", source.id) + ) + else: + for item in self.list_bucket_tree(source.id, prefix=source_path or None, recursive=True, token=token): + if not isinstance(item, BucketFile): + continue + if source_path and not (item.path == source_path or item.path.startswith(source_path + "/")): + continue + target_path = _resolve_target_path(item.path, source_path or None, is_single_file=False) + all_copies.append(_build_copy_op(target_path, item.xet_hash, item.size, "bucket", source.id)) + else: + for file, target_path in self._iter_repo_files_for_copy( + source, + destination_path, + destination_is_directory, + destination_exists_as_directory, + merge_contents, + token=token, + ): + # Skip .gitattributes files (git-specific metadata, not relevant in a bucket) + if file.path.rsplit("/", 1)[-1] == ".gitattributes": + continue + _add_repo_file(file, target_path) + + if not all_copies and not all_adds and not pending_downloads: + if source.is_bucket: + raise EntryNotFoundError(f"No files found at '{source_str}' in bucket '{source.id}'.") + else: + raise EntryNotFoundError(f"No files found at '{source_str}' in {source.type} '{source.id}'.") + + if pending_downloads: + + def _download_and_collect(item: tuple[str, str]) -> None: + file_path, target_path = item + local_path = self.hf_hub_download( + repo_id=source.id, + repo_type=source.type, + filename=file_path, + revision=source.revision, + token=token, + tqdm_class=silent_tqdm, # type: ignore + ) + all_adds.append((local_path, target_path)) + + hf_thread_map(_download_and_collect, pending_downloads, desc="Downloading text files for copy") + + # Send copies first (no upload needed), then adds (may need upload) + if all_copies: + for copy_chunk in chunk_iterable(all_copies, chunk_size=_BUCKET_BATCH_ADD_CHUNK_SIZE): + self._batch_bucket_files(destination_bucket_id, copy=list(copy_chunk), token=token) + if all_adds: + for add_chunk in chunk_iterable(all_adds, chunk_size=_BUCKET_BATCH_ADD_CHUNK_SIZE): + self._batch_bucket_files(destination_bucket_id, add=list(add_chunk), token=token) + + def _iter_repo_files_for_copy( + self, + source: HfUri, + destination_path: str, + destination_is_directory: bool, + destination_exists_as_directory: bool, + merge_contents: bool, + *, + token: str | bool | None = None, + ) -> Iterable[tuple[RepoFile, str]]: + """Yield (file, target_path) pairs from a repo source, with target paths resolved.""" + source_path = source.path_in_repo + source_repo_path_info: list[RepoFile | RepoFolder] = [] + if source_path != "": + source_repo_path_info = self.get_paths_info( + repo_id=source.id, + paths=[source_path], + repo_type=source.type, + revision=source.revision, + token=token, + ) + + def _resolve(src_file_path: str, src_root_path: str | None, is_single_file: bool) -> str: + return _resolve_copy_target_path( + src_file_path, + src_root_path, + is_single_file, + destination_path, + destination_is_directory, + destination_exists_as_directory, + merge_contents, + ) + + if len(source_repo_path_info) == 1 and isinstance(source_repo_path_info[0], RepoFile): + file = source_repo_path_info[0] + yield file, _resolve(file.path, None, is_single_file=True) + else: + for repo_item in self.list_repo_tree( + repo_id=source.id, + path_in_repo=source_path, + recursive=True, + repo_type=source.type, + revision=source.revision, + token=token, + ): + if not isinstance(repo_item, RepoFile): + continue + yield repo_item, _resolve(repo_item.path, source_path or None, is_single_file=False) + + def _copy_to_repo( + self, + source: HfUri, + destination: HfUri, + merge_contents: bool, + source_str: str, + destination_str: str, + *, + token: str | bool | None = None, + ) -> None: + destination_path = destination.path_in_repo + destination_is_directory = False + destination_exists_as_directory = False + + if destination_path == "": + destination_is_directory = True + destination_exists_as_directory = True + else: + dest_path_info = self.get_paths_info( + repo_id=destination.id, + paths=[destination_path], + repo_type=destination.type, + revision=destination.revision, + token=token, + ) + if len(dest_path_info) == 1 and isinstance(dest_path_info[0], RepoFile): + destination_is_directory = False + elif len(dest_path_info) == 1 and isinstance(dest_path_info[0], RepoFolder): + destination_is_directory = True + destination_exists_as_directory = True + else: + try: + destination_exists_as_directory = any( + self.list_repo_tree( + repo_id=destination.id, + path_in_repo=destination_path, + repo_type=destination.type, + revision=destination.revision, + token=token, + ) + ) + except RemoteEntryNotFoundError: + destination_exists_as_directory = False + destination_is_directory = destination_exists_as_directory or destination_str.endswith("/") + + is_same_repo = source.id == destination.id and source.type == destination.type + + commit_ops: list[CommitOperationCopy] = [ + CommitOperationCopy( + src_path_in_repo=file.path, + path_in_repo=target, + src_revision=source.revision, + src_repo_id=None if is_same_repo else source.id, + src_repo_type=None if is_same_repo else source.type, + ) + for file, target in self._iter_repo_files_for_copy( + source, + destination_path, + destination_is_directory, + destination_exists_as_directory, + merge_contents, + token=token, + ) + ] + + if not commit_ops: + raise EntryNotFoundError(f"No files found at '{source_str}' in {source.type} '{source.id}'.") + + self.create_commit( + repo_id=destination.id, + repo_type=destination.type, + revision=destination.revision, + operations=commit_ops, + commit_message=f"Copy files from {source.type}s/{source.id}", + token=token, + ) + + @validate_hf_hub_args + def batch_bucket_files( + self, + bucket_id: str, + *, + add: list[tuple[str | Path | bytes, str]] | None = None, + copy: list[tuple[str, str, str, str]] | None = None, + delete: list[str] | None = None, + token: str | bool | None = None, + ): + """Add, copy, and/or delete files in a bucket. + + This is a non-transactional operation. If an error occurs in the process, some files may have been uploaded, + copied, or deleted while others haven't. + + Args: + bucket_id (`str`): + The ID of the bucket (e.g. `"username/my-bucket"`). + add (`list` of `tuple`, *optional*): + Files to upload. Each element is a `(source, destination)` tuple where `source` is a path to a local + file (`str` or `Path`) or raw `bytes` content, and `destination` is the path in the bucket. + copy (`list` of `tuple`, *optional*): + Files to copy by xet hash. Each element is a `(source_repo_type, source_repo_id, xet_hash, + destination)` tuple where: + - `source_repo_type` is the type of the source repository: `"model"`, `"dataset"`, `"space"`, or + `"bucket"`. + - `source_repo_id` is the ID of the source repository or bucket (e.g. `"username/my-model"`). + - `xet_hash` is the xet hash of the file to copy. + - `destination` is the destination path in the bucket. + This is a server-side operation — no data is downloaded or re-uploaded. + delete (`list` of `str`, *optional*): + Paths of files to delete from the bucket. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Example: + ```python + >>> from huggingface_hub import batch_bucket_files + + # Upload files + >>> batch_bucket_files( + ... "username/my-bucket", + ... add=[ + ... ("./model.safetensors", "models/model.safetensors"), + ... (b'{{"key": "value"}}', "config.json"), + ... ], + ... ) + + # Copy xet files from another bucket or repo (server-side, no data transfer) + >>> batch_bucket_files( + ... "username/my-bucket", + ... copy=[ + ... ("bucket", "username/source-bucket", "", "models/model.safetensors"), + ... ("model", "username/my-model", "", "models/config.safetensors"), + ... ], + ... ) + + # Delete files + >>> batch_bucket_files("username/my-bucket", delete=["old-model.bin"]) + + # Upload and delete in one batch + >>> batch_bucket_files( + ... "username/my-bucket", + ... add=[("./new.txt", "new.txt")], + ... delete=["old.txt"], + ... ) + ``` + """ + add = add or [] + copy = copy or [] + delete = delete or [] + + # Small batch: do everything in one call + if len(add) + len(copy) + len(delete) <= _BUCKET_BATCH_ADD_CHUNK_SIZE: + self._batch_bucket_files(bucket_id, add=add, copy=copy, delete=delete, token=token) # type: ignore + return + + # Large batch: chunk copies first (no upload), then adds, then deletes + from .utils._xet_progress_reporting import XetUploadProgressReporter + + if add and not are_progress_bars_disabled(): + progress = XetUploadProgressReporter(total_files=len(add)) + else: + progress = None + + try: + for copy_chunk in chunk_iterable(copy, chunk_size=_BUCKET_BATCH_ADD_CHUNK_SIZE): + self._batch_bucket_files(bucket_id, copy=list(copy_chunk), token=token) + + for add_chunk in chunk_iterable(add, chunk_size=_BUCKET_BATCH_ADD_CHUNK_SIZE): + self._batch_bucket_files(bucket_id, add=list(add_chunk), token=token, _progress=progress) + + for delete_chunk in chunk_iterable(delete, chunk_size=_BUCKET_BATCH_DELETE_CHUNK_SIZE): + self._batch_bucket_files(bucket_id, delete=list(delete_chunk), token=token) + finally: + if progress is not None: + progress.close() + + return + + def _batch_bucket_files( + self, + bucket_id: str, + *, + add: list[tuple[str | Path | bytes, str] | _BucketAddFile] | None = None, + copy: list[tuple[str, str, str, str] | _BucketCopyFile] | None = None, + delete: list[str | _BucketDeleteFile] | None = None, + token: str | bool | None = None, + _progress: XetUploadProgressReporter | None = None, + ): + """Internal method: process a single batch of bucket file operations (upload to XET + call /batch).""" + # Convert public API inputs to internal operation objects + operations: list[_BucketAddFile | _BucketCopyFile | _BucketDeleteFile] = [] + if add: + for add_item in add: + if isinstance(add_item, _BucketAddFile): + operations.append(add_item) + else: + source, destination = add_item + operations.append(_BucketAddFile(source=source, destination=destination)) + if copy: + for copy_item in copy: + if isinstance(copy_item, _BucketCopyFile): + operations.append(copy_item) + else: + source_repo_type, source_repo_id, xet_hash, destination = copy_item + operations.append( + _BucketCopyFile( + destination=destination, + xet_hash=xet_hash, + source_repo_type=source_repo_type, + source_repo_id=source_repo_id, + ) + ) + if delete: + for delete_item in delete: + if isinstance(delete_item, _BucketDeleteFile): + operations.append(delete_item) + else: + operations.append(_BucketDeleteFile(path=delete_item)) + + if not operations: + return + + from hf_xet import SKIP_SHA256 + + from .utils._xet import ( + XetTokenType, + abort_xet_session, + get_xet_session, + xet_connection_info_refresh_url, + xet_headers_without_auth, + ) + from .utils._xet_progress_reporting import XetUploadProgressReporter + + headers = self._build_hf_headers(token=token) + + add_operations = [op for op in operations if isinstance(op, _BucketAddFile)] + add_operations_to_upload = [op for op in add_operations if op.xet_hash is None] + add_bytes_operations = [op for op in add_operations if isinstance(op.source, bytes)] + add_path_operations = [op for op in add_operations if not isinstance(op.source, bytes)] + + if len(add_operations_to_upload) > 0: + refresh_url = xet_connection_info_refresh_url( + token_type=XetTokenType.WRITE, + repo_id=bucket_id, + repo_type="bucket", + endpoint=self.endpoint, + ) + xet_headers = xet_headers_without_auth(headers) + + owns_progress = _progress is None + if _progress is not None: + progress = _progress + progress.reset_for_next_commit() + progress_callback = progress.update_progress + elif not are_progress_bars_disabled(): + progress = XetUploadProgressReporter() + progress_callback = progress.update_progress + else: + progress, progress_callback = None, None + + session = get_xet_session() + + try: + with session.new_upload_commit( + token_refresh_url=refresh_url, + token_refresh_headers=headers, + custom_headers=xet_headers, + progress_callback=progress_callback, + ) as commit: + handles = [] + for op in add_path_operations: + if op.xet_hash is None: + handles.append((commit.start_upload_file(str(op.source), sha256=SKIP_SHA256), op)) + for op in add_bytes_operations: + if op.xet_hash is None: + handles.append((commit.start_upload_bytes(op.source, sha256=SKIP_SHA256), op)) + for handle, op in handles: + result = handle.result() + op.xet_hash = result.xet_info.hash + op.size = result.xet_info.file_size + except KeyboardInterrupt: + abort_xet_session() + raise + finally: + if owns_progress and progress is not None: + progress.close() + + def _payload_as_ndjson() -> Iterable[bytes]: + for op in operations: + if isinstance(op, _BucketAddFile): + payload = { + "type": "addFile", + "path": op.destination, + "xetHash": op.xet_hash, + "mtime": op.mtime, + } + if op.content_type is not None: + payload["contentType"] = op.content_type + elif isinstance(op, _BucketCopyFile): + payload = { + "type": "copyFile", + "path": op.destination, + "xetHash": op.xet_hash, + "sourceRepoType": op.source_repo_type, + "sourceRepoId": op.source_repo_id, + } + else: + payload = { + "type": "deleteFile", + "path": op.path, + } + yield json.dumps(payload).encode() + yield b"\n" + + headers = { + "Content-Type": "application/x-ndjson", + **headers, + } + data = b"".join(_payload_as_ndjson()) + + response = http_backoff( + "POST", f"{self.endpoint}/api/buckets/{bucket_id}/batch", headers=headers, content=data + ) + hf_raise_for_status(response) + + @validate_hf_hub_args + def get_bucket_file_metadata( + self, + bucket_id: str, + remote_path: str, + *, + token: str | bool | None = None, + ) -> BucketFileMetadata: + """Fetch metadata of a file in a bucket. + + Args: + bucket_id (`str`): + The ID of the bucket (e.g. `"username/my-bucket"`). + remote_path (`str`): + The path of the file in the bucket. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Returns: + [`BucketFileMetadata`]: The file metadata containing size and xet information. + + Example: + ```python + >>> from huggingface_hub import get_bucket_file_metadata + >>> metadata = get_bucket_file_metadata( + ... bucket_id="username/my-bucket", + ... remote_path="models/model.safetensors", + ... ) + >>> metadata.size + 42000 + ``` + """ + response = _httpx_follow_relative_redirects_with_backoff( + "HEAD", + f"{self.endpoint}/buckets/{bucket_id}/resolve/{quote(remote_path, safe='')}", + headers=self._build_hf_headers(token=token), + retry_on_errors=True, + ) + + xet_file_data = parse_xet_file_data_from_response(response) + if xet_file_data is None: + raise ValueError(f"Could not parse xet file data for '{remote_path}' in bucket '{bucket_id}'.") + + size = response.headers.get("Content-Length") + if size is None: + raise ValueError(f"Could not get size for '{remote_path}' in bucket '{bucket_id}'.") + + return BucketFileMetadata(size=int(size), xet_file_data=xet_file_data) + + @validate_hf_hub_args + def download_bucket_files( + self, + bucket_id: str, + files: list[tuple[str | BucketFile, str | Path]], + *, + raise_on_missing_files: bool = False, + token: str | bool | None = None, + ) -> None: + """Download files from a bucket. + + Files input is a list of `(remote file, local file)` tuples where `remote file` is either the path of the file + in the bucket or a [`BucketFile`] object, and `local file` is the destination path on the local filesystem. + When passing a [`BucketFile`] object (obtained from [`list_bucket_tree`]), the method will skip the metadata + fetching step and directly download the files. + + Args: + bucket_id (`str`): + The ID of the bucket (e.g. `"username/my-bucket"`). + files (`list[tuple[Union[str, BucketFile], Union[str, Path]]]`): + Files to download as a list of tuple (source, destination). See description above for format details. + raise_on_missing_files (`bool`, *optional*): + If `True`, raise an [`EntryNotFoundError`] when a requested file does not exist in the bucket. If + `False` (default), missing files are skipped with a warning. + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + + Example: + ```python + >>> from huggingface_hub import download_bucket_files + + >>> download_bucket_files( + ... bucket_id="username/my-bucket", + ... files=[ + ... ("models/model.safetensors", "./local/model.safetensors"), + ... ("config.json", "./local/config.json"), + ... ], + ... ) + ``` + + ```python + >>> from huggingface_hub import download_bucket_files + + >>> parquet_files = [file for file in list_bucket_tree(bucket_id="username/my-bucket") if file.path.endswith(".parquet")] + >>> download_bucket_files( + ... bucket_id="username/my-bucket", + ... files=[(file, f"./local/{file.path}") for file in parquet_files], + ... ) + ``` + """ + from hf_xet import XetFileInfo # type: ignore[no-redef] + + from .utils._xet import abort_xet_session, get_xet_session, xet_headers_without_auth + + headers = self._build_hf_headers(token=token) + + if len(files) == 0: + return + + # Resolve all string paths to BucketFile objects in a single batch request + str_paths = [path for path, _ in files if not isinstance(path, BucketFile)] + bucket_files_by_path: dict[str, BucketFile] = {} + if str_paths: + bucket_files_by_path = { + info.path: info for info in self.get_bucket_paths_info(bucket_id, str_paths, token=token) + } + + # Check for missing files + missing_paths = [path for path in str_paths if path not in bucket_files_by_path] + if missing_paths: + if raise_on_missing_files: + raise EntryNotFoundError( + f"{len(missing_paths)} file(s) not found in bucket '{bucket_id}': {', '.join(missing_paths)}" + ) + for path in missing_paths: + warnings.warn(f"File '{path}' not found in bucket '{bucket_id}'. Skipping.") + + non_zero_download_items: list[tuple[XetFileInfo, str]] = [] + first_valid_bucket_file: BucketFile | None = None + for remote_file, local_path in files: + if not isinstance(remote_file, BucketFile): + if remote_file not in bucket_files_by_path: + continue # skip missing files (already warned above) + remote_file = bucket_files_by_path[remote_file] + if first_valid_bucket_file is None: + first_valid_bucket_file = remote_file + dest_path = Path(local_path).absolute() + if remote_file.size == 0: + # Create empty file without downloading + if dest_path.exists(): + if dest_path.is_dir(): + raise IsADirectoryError(f"Expected file but found directory at '{dest_path}'") + if dest_path.stat().st_size != 0: + dest_path.write_bytes(b"") + else: + dest_path.parent.mkdir(parents=True, exist_ok=True) + dest_path.touch() + else: + non_zero_download_items.append((XetFileInfo(remote_file.xet_hash, remote_file.size), str(dest_path))) + + if len(non_zero_download_items) == 0 or first_valid_bucket_file is None: + return + + # Fetch refresh route (same for all files in this bucket) + remote_path = first_valid_bucket_file.path + metadata = self.get_bucket_file_metadata(bucket_id, remote_path, token=token) + + xet_headers = xet_headers_without_auth(headers) + + # Download files + from .utils._xet_progress_reporting import XetDownloadProgressReporter + + session = get_xet_session() + + with XetDownloadProgressReporter( + reconstruction_desc="Downloading bucket files", + transfer_desc="Downloading bytes", + total=sum(xet_info.file_size for xet_info, _ in non_zero_download_items), + log_level=logger.getEffectiveLevel(), + name="huggingface_hub.download_bucket_files", + ) as progress: + try: + with session.new_file_download_group( + token_refresh_url=metadata.xet_file_data.refresh_route, + token_refresh_headers=headers, + custom_headers=xet_headers, + progress_callback=progress.update_progress, + ) as group: + for xet_info, dest in non_zero_download_items: + group.start_download_file(xet_info, dest) + except KeyboardInterrupt: + abort_xet_session() + raise + + @validate_hf_hub_args + def sync_bucket( + self, + source: str | None = None, + dest: str | None = None, + *, + delete: bool = False, + ignore_times: bool = False, + ignore_sizes: bool = False, + existing: bool = False, + ignore_existing: bool = False, + include: list[str] | None = None, + exclude: list[str] | None = None, + filter_from: str | None = None, + plan: str | None = None, + apply: str | None = None, + dry_run: bool = False, + verbose: bool = False, + quiet: bool = False, + token: bool | str | None = None, + ) -> SyncPlan: + """Sync files between a local directory and a bucket. + + This is equivalent to the ``hf buckets sync`` CLI command. One of ``source`` or ``dest`` must be a bucket path + (``hf://buckets/...``) and the other must be a local directory path. + + Args: + source (`str`, *optional*): + Source path: local directory or ``hf://buckets/namespace/bucket_name(/prefix)``. + Required unless using ``apply``. + dest (`str`, *optional*): + Destination path: local directory or ``hf://buckets/namespace/bucket_name(/prefix)``. + Required unless using ``apply``. + delete (`bool`, *optional*, defaults to `False`): + Delete destination files not present in source. + ignore_times (`bool`, *optional*, defaults to `False`): + Skip files only based on size, ignoring modification times. + ignore_sizes (`bool`, *optional*, defaults to `False`): + Skip files only based on modification times, ignoring sizes. + existing (`bool`, *optional*, defaults to `False`): + Skip creating new files on receiver (only update existing files). + ignore_existing (`bool`, *optional*, defaults to `False`): + Skip updating files that exist on receiver (only create new files). + include (`list[str]`, *optional*): + Include files matching patterns (fnmatch-style). + exclude (`list[str]`, *optional*): + Exclude files matching patterns (fnmatch-style). + filter_from (`str`, *optional*): + Path to a filter file with include/exclude rules. + plan (`str`, *optional*): + Save sync plan to this JSONL file instead of executing. + apply (`str`, *optional*): + Apply a previously saved plan file. When set, ``source`` and ``dest`` are not needed. + dry_run (`bool`, *optional*, defaults to `False`): + Print sync plan to stdout as JSONL without executing. + verbose (`bool`, *optional*, defaults to `False`): + Show detailed per-file operations. + quiet (`bool`, *optional*, defaults to `False`): + Suppress all output and progress bars. + token (Union[bool, str, None], optional): + A valid user access token. If not provided, the locally saved token will be used. + + Returns: + [`SyncPlan`]: The computed (or loaded) sync plan. + + Example: + ```python + >>> from huggingface_hub import HfApi + >>> api = HfApi() + + # Upload local directory to bucket + >>> api.sync_bucket("./data", "hf://buckets/username/my-bucket") + + # Download bucket to local directory + >>> api.sync_bucket("hf://buckets/username/my-bucket", "./data") + + # Sync with delete and filtering + >>> api.sync_bucket( + ... "./data", + ... "hf://buckets/username/my-bucket", + ... delete=True, + ... include=["*.safetensors"], + ... ) + + # Dry run: preview what would be synced + >>> plan = api.sync_bucket("./data", "hf://buckets/username/my-bucket", dry_run=True) + >>> plan.summary() + {'uploads': 3, 'downloads': 0, 'deletes': 0, 'skips': 1, 'total_size': 4096} + + # Save plan for review, then apply + >>> api.sync_bucket("./data", "hf://buckets/username/my-bucket", plan="sync-plan.jsonl") + >>> api.sync_bucket(apply="sync-plan.jsonl") + ``` + """ + return sync_bucket_internal( + source=source, + dest=dest, + api=self, + delete=delete, + ignore_times=ignore_times, + ignore_sizes=ignore_sizes, + existing=existing, + ignore_existing=ignore_existing, + include=include, + exclude=exclude, + filter_from=filter_from, + plan=plan, + apply=apply, + dry_run=dry_run, + verbose=verbose, + quiet=quiet, + token=token, + ) + + +def _parse_revision_from_pr_url(pr_url: str) -> str: + """Safely parse revision number from a PR url. + + Example: + ```py + >>> _parse_revision_from_pr_url("https://huggingface.co/bigscience/bloom/discussions/2") + "refs/pr/2" + ``` + """ + re_match = re.match(_REGEX_DISCUSSION_URL, pr_url) + if re_match is None: + raise RuntimeError(f"Unexpected response from the hub, expected a Pull Request URL but got: '{pr_url}'") + return f"refs/pr/{re_match[1]}" + + +def parse_local_safetensors_file_metadata(path: str | Path) -> SafetensorsFileMetadata: + """ + Parse metadata from a local safetensors file. + + For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. + + Args: + path (`str` or `Path`): + Path to the safetensors file. + + Returns: + [`SafetensorsFileMetadata`]: information related to the safetensors file. + + Raises: + [`SafetensorsParsingError`]: + If the safetensors file header couldn't be parsed correctly. + `FileNotFoundError`: + If the file does not exist. + + Example: + ```py + >>> metadata = parse_local_safetensors_file_metadata("path/to/model.safetensors") + >>> metadata + SafetensorsFileMetadata( + metadata={'format': 'pt'}, + tensors={'layer.weight': TensorInfo(dtype='F32', shape=[512, 512], ...}, ...} + ) + >>> metadata.parameter_count + {'F32': 262144} + ``` + """ + path = Path(path) + filename = path.name + context_msg = f"path '{path}'" + + with open(path, "rb") as f: + # 1. Read first 8 bytes and parse/validate metadata size using shared helper + size_bytes = f.read(8) + metadata_size = _get_safetensors_metadata_size(size_bytes, filename, context_msg) + + # 2. Read metadata bytes + metadata_as_bytes = f.read(metadata_size) + if len(metadata_as_bytes) < metadata_size: + raise SafetensorsParsingError( + f"Failed to parse safetensors header for '{filename}' ({context_msg}): file is truncated. Expected " + f"{metadata_size} bytes of metadata but got {len(metadata_as_bytes)}." + ) + + # 3. Parse using shared helper + return _parse_safetensors_header(metadata_as_bytes, filename, context_msg) + + +def get_local_safetensors_metadata(path: str | Path) -> SafetensorsRepoMetadata: + """ + Parse metadata for a local safetensors file or folder. + + Supports: + - Single safetensors file (e.g., `model.safetensors`) + - Directory with non-sharded model (contains `model.safetensors`) + - Directory with sharded model (contains `model.safetensors.index.json`) + + For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format. + + Args: + path (`str` or `Path`): + Path to a safetensors file or directory containing safetensors files. + + Returns: + [`SafetensorsRepoMetadata`]: information related to the safetensors repo. + + Raises: + [`NotASafetensorsRepoError`]: + If the path is not a valid safetensors file or folder (i.e., doesn't have either a + `model.safetensors` or a `model.safetensors.index.json` file). + [`SafetensorsParsingError`]: + If a safetensors file header couldn't be parsed correctly. + `FileNotFoundError`: + If the path does not exist. + + Example: + ```py + # Parse single safetensors file + >>> metadata = get_local_safetensors_metadata("path/to/model.safetensors") + >>> metadata + SafetensorsRepoMetadata(metadata=None, sharded=False, weight_map={...}, files_metadata={...}) + + # Parse directory with sharded model + >>> metadata = get_local_safetensors_metadata("path/to/model_folder") + >>> metadata + SafetensorsRepoMetadata(metadata={'total_size': ...}, sharded=True, weight_map={...}, files_metadata={...}) + >>> len(metadata.files_metadata) + 3 # Number of safetensors shards + ``` + """ + path = Path(path) + + # Case 1: Direct path to a safetensors file + if path.is_file(): + file_metadata = parse_local_safetensors_file_metadata(path) + return SafetensorsRepoMetadata( + metadata=None, + sharded=False, + weight_map={tensor_name: path.name for tensor_name in file_metadata.tensors.keys()}, + files_metadata={path.name: file_metadata}, + ) + + # Case 2: Directory + if not path.is_dir(): + raise FileNotFoundError(f"Path '{path}' does not exist.") + + single_file_path = path / constants.SAFETENSORS_SINGLE_FILE + index_file_path = path / constants.SAFETENSORS_INDEX_FILE + + # Case 2a: Non-sharded model (single model.safetensors file) + if single_file_path.exists(): + file_metadata = parse_local_safetensors_file_metadata(single_file_path) + return SafetensorsRepoMetadata( + metadata=None, + sharded=False, + weight_map={ + tensor_name: constants.SAFETENSORS_SINGLE_FILE for tensor_name in file_metadata.tensors.keys() + }, + files_metadata={constants.SAFETENSORS_SINGLE_FILE: file_metadata}, + ) + + # Case 2b: Sharded model (model.safetensors.index.json) + if index_file_path.exists(): + with open(index_file_path) as f: + index = json.load(f) + + weight_map = index.get("weight_map", {}) + + # Parse metadata from each shard + files_metadata = {} + for shard_filename in set(weight_map.values()): + shard_path = path / shard_filename + files_metadata[shard_filename] = parse_local_safetensors_file_metadata(shard_path) + + return SafetensorsRepoMetadata( + metadata=index.get("metadata", None), + sharded=True, + weight_map=weight_map, + files_metadata=files_metadata, + ) + + # Not a valid safetensors folder + raise NotASafetensorsRepoError( + f"'{path}' is not a valid safetensors folder. Couldn't find '{constants.SAFETENSORS_INDEX_FILE}' or " + f"'{constants.SAFETENSORS_SINGLE_FILE}' files." + ) + + +api = HfApi() + +whoami = api.whoami +auth_check = api.auth_check + +list_models = api.list_models +model_info = api.model_info + +list_datasets = api.list_datasets +list_dataset_parquet_files = api.list_dataset_parquet_files +dataset_info = api.dataset_info +get_dataset_leaderboard = api.get_dataset_leaderboard + +list_spaces = api.list_spaces +list_space_templates = api.list_space_templates +search_spaces = api.search_spaces +space_info = api.space_info + +kernel_info = api.kernel_info + +list_papers = api.list_papers +paper_info = api.paper_info +read_paper = api.read_paper +list_daily_papers = api.list_daily_papers + +repo_exists = api.repo_exists +revision_exists = api.revision_exists +file_exists = api.file_exists +repo_info = api.repo_info +list_repo_files = api.list_repo_files +list_repo_refs = api.list_repo_refs +list_repo_commits = api.list_repo_commits +list_repo_tree = api.list_repo_tree +get_paths_info = api.get_paths_info +verify_repo_checksums = api.verify_repo_checksums + +get_model_tags = api.get_model_tags +get_dataset_tags = api.get_dataset_tags + +create_commit = api.create_commit +create_repo = api.create_repo +delete_repo = api.delete_repo +update_repo_settings = api.update_repo_settings +move_repo = api.move_repo +upload_file = api.upload_file +upload_folder = api.upload_folder +delete_file = api.delete_file +delete_folder = api.delete_folder +delete_files = api.delete_files +upload_large_folder = api.upload_large_folder +preupload_lfs_files = api.preupload_lfs_files +create_branch = api.create_branch +delete_branch = api.delete_branch +create_tag = api.create_tag +delete_tag = api.delete_tag +get_full_repo_name = api.get_full_repo_name + +# Danger-zone API +super_squash_history = api.super_squash_history +list_lfs_files = api.list_lfs_files +permanently_delete_lfs_files = api.permanently_delete_lfs_files + +# Safetensors helpers +get_safetensors_metadata = api.get_safetensors_metadata +parse_safetensors_file_metadata = api.parse_safetensors_file_metadata + +# Background jobs +run_as_future = api.run_as_future + +# Activity API +list_liked_repos = api.list_liked_repos +list_repo_likers = api.list_repo_likers +list_user_repos = api.list_user_repos +unlike = api.unlike + +# Community API +get_discussion_details = api.get_discussion_details +get_repo_discussions = api.get_repo_discussions +create_discussion = api.create_discussion +create_pull_request = api.create_pull_request +change_discussion_status = api.change_discussion_status +comment_discussion = api.comment_discussion +edit_discussion_comment = api.edit_discussion_comment +rename_discussion = api.rename_discussion +merge_pull_request = api.merge_pull_request + +# Space API +get_space_secrets = api.get_space_secrets +add_space_secret = api.add_space_secret +delete_space_secret = api.delete_space_secret +get_space_variables = api.get_space_variables +add_space_variable = api.add_space_variable +delete_space_variable = api.delete_space_variable +get_space_runtime = api.get_space_runtime +list_spaces_hardware = api.list_spaces_hardware +request_space_hardware = api.request_space_hardware +set_space_sleep_time = api.set_space_sleep_time +pause_space = api.pause_space +restart_space = api.restart_space +duplicate_repo = api.duplicate_repo +duplicate_space = api.duplicate_space +request_space_storage = api.request_space_storage +delete_space_storage = api.delete_space_storage +set_space_volumes = api.set_space_volumes +delete_space_volumes = api.delete_space_volumes +enable_space_dev_mode = api.enable_space_dev_mode +disable_space_dev_mode = api.disable_space_dev_mode +fetch_space_logs = api.fetch_space_logs +wait_for_space = api.wait_for_space + +# Inference Endpoint API +list_inference_endpoints = api.list_inference_endpoints +create_inference_endpoint = api.create_inference_endpoint +get_inference_endpoint = api.get_inference_endpoint +update_inference_endpoint = api.update_inference_endpoint +delete_inference_endpoint = api.delete_inference_endpoint +pause_inference_endpoint = api.pause_inference_endpoint +resume_inference_endpoint = api.resume_inference_endpoint +scale_to_zero_inference_endpoint = api.scale_to_zero_inference_endpoint +create_inference_endpoint_from_catalog = api.create_inference_endpoint_from_catalog +list_inference_catalog = api.list_inference_catalog + +# Collections API +get_collection = api.get_collection +list_collections = api.list_collections +create_collection = api.create_collection +update_collection_metadata = api.update_collection_metadata +delete_collection = api.delete_collection +add_collection_item = api.add_collection_item +update_collection_item = api.update_collection_item +delete_collection_item = api.delete_collection_item +delete_collection_item = api.delete_collection_item + +# Access requests API +list_pending_access_requests = api.list_pending_access_requests +list_accepted_access_requests = api.list_accepted_access_requests +list_rejected_access_requests = api.list_rejected_access_requests +cancel_access_request = api.cancel_access_request +accept_access_request = api.accept_access_request +reject_access_request = api.reject_access_request +grant_access = api.grant_access + +# Webhooks API +create_webhook = api.create_webhook +disable_webhook = api.disable_webhook +delete_webhook = api.delete_webhook +enable_webhook = api.enable_webhook +get_webhook = api.get_webhook +list_webhooks = api.list_webhooks +update_webhook = api.update_webhook + + +# User API +get_user_overview = api.get_user_overview +get_organization_overview = api.get_organization_overview +list_organization_followers = api.list_organization_followers +list_organization_members = api.list_organization_members +list_user_followers = api.list_user_followers +list_user_following = api.list_user_following + +# Jobs API +run_job = api.run_job +fetch_job_logs = api.fetch_job_logs +fetch_job_metrics = api.fetch_job_metrics +list_jobs = api.list_jobs +list_jobs_hardware = api.list_jobs_hardware +inspect_job = api.inspect_job +wait_for_job = api.wait_for_job +cancel_job = api.cancel_job +update_job_labels = api.update_job_labels +run_uv_job = api.run_uv_job +create_scheduled_job = api.create_scheduled_job +list_scheduled_jobs = api.list_scheduled_jobs +inspect_scheduled_job = api.inspect_scheduled_job +delete_scheduled_job = api.delete_scheduled_job +suspend_scheduled_job = api.suspend_scheduled_job +resume_scheduled_job = api.resume_scheduled_job +trigger_scheduled_job = api.trigger_scheduled_job +update_scheduled_job_labels = api.update_scheduled_job_labels +create_scheduled_uv_job = api.create_scheduled_uv_job +sync_job_volume = api.sync_job_volume + +# Buckets API +create_bucket = api.create_bucket +bucket_info = api.bucket_info +list_buckets = api.list_buckets +delete_bucket = api.delete_bucket +move_bucket = api.move_bucket +list_bucket_tree = api.list_bucket_tree +get_bucket_paths_info = api.get_bucket_paths_info +copy_files = api.copy_files +batch_bucket_files = api.batch_bucket_files +get_bucket_file_metadata = api.get_bucket_file_metadata +download_bucket_files = api.download_bucket_files +sync_bucket = api.sync_bucket diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/hf_file_system.py b/venv/lib/python3.12/site-packages/huggingface_hub/hf_file_system.py new file mode 100644 index 0000000000000000000000000000000000000000..00fada7aafaf61ee0bbdcc40dc480bdee1efdc65 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/hf_file_system.py @@ -0,0 +1,1419 @@ +import os +import tempfile +import threading +from collections import deque +from collections.abc import Iterable, Iterator +from contextlib import ExitStack +from copy import deepcopy +from dataclasses import dataclass, field +from datetime import datetime +from itertools import chain +from pathlib import Path, PurePosixPath +from typing import Any, NoReturn, Union +from urllib.parse import quote, unquote + +import fsspec +import httpx +from fsspec.callbacks import _DEFAULT_CALLBACK, NoOpCallback, TqdmCallback +from fsspec.config import apply_config +from fsspec.utils import isfilelike + +from . import constants +from ._commit_api import CommitOperationCopy, CommitOperationDelete +from .errors import ( + BucketNotFoundError, + EntryNotFoundError, + HfHubHTTPError, + RepositoryNotFoundError, + RevisionNotFoundError, +) +from .file_download import hf_hub_url, http_get +from .hf_api import SPECIAL_REFS_REVISION_REGEX, BucketFile, BucketFolder, HfApi, LastCommitInfo, RepoFile, RepoFolder +from .utils import HFValidationError, hf_raise_for_status, http_backoff, http_stream_backoff, parse_hf_uri +from .utils.insecure_hashlib import md5 + + +@dataclass +class HfFileSystemResolvedPath: + """Top level Data structure containing information about a resolved Hugging Face file system path.""" + + root: str + path: str + + def unresolve(self) -> str: + return f"{self.root}/{self.path}".rstrip("/") + + +@dataclass +class HfFileSystemResolvedRepositoryPath(HfFileSystemResolvedPath): + """Data structure containing information about a resolved path in a repository.""" + + repo_type: str + repo_id: str + revision: str + path_in_repo: str + root: str = field(init=False) + path: str = field(init=False) + # The part placed after '@' in the initial path. It can be a quoted or unquoted refs revision. + # Used to reconstruct the unresolved path to return to the user. + _raw_revision: str | None = field(default=None, repr=False) + + def __post_init__(self): + repo_path = constants.REPO_TYPES_URL_PREFIXES.get(self.repo_type, "") + self.repo_id + if self._raw_revision: + self.root = f"{repo_path}@{self._raw_revision}" + elif self.revision != constants.DEFAULT_REVISION: + self.root = f"{repo_path}@{safe_revision(self.revision)}" + else: + self.root = repo_path + self.path = self.path_in_repo + + +@dataclass +class HfFileSystemResolvedBucketPath(HfFileSystemResolvedPath): + """Data structure containing information about a resolved path in a bucket.""" + + bucket_id: str + root: str = field(init=False) + + def __post_init__(self): + self.root = "buckets/" + self.bucket_id + + +# We need to improve fsspec.spec._Cached which is AbstractFileSystem's metaclass +_cached_base: Any = type(fsspec.AbstractFileSystem) + + +class _Cached(_cached_base): + """ + Metaclass for caching HfFileSystem instances according to the args. + + This creates an additional reference to the filesystem, which prevents the + filesystem from being garbage collected when all *user* references go away. + A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also* + be made for a filesystem instance to be garbage collected. + + This is a slightly modified version of `fsspec.spec._Cached` to improve it. + In particular in `_tokenize` the pid isn't taken into account for the + `fs_token` used to identify cached instances. The `fs_token` logic is also + robust to defaults values and the order of the args. Finally new instances + reuse the states from sister instances in the main thread. + """ + + def __init__(cls, *args, **kwargs): + # Hack: override https://github.com/fsspec/filesystem_spec/blob/dcb167e8f50e6273d4cfdfc4cab8fc5aa4c958bf/fsspec/spec.py#L53 + super().__init__(*args, **kwargs) + # Note: we intentionally create a reference here, to avoid garbage + # collecting instances when all other references are gone. To really + # delete a FileSystem, the cache must be cleared. + cls._cache = {} + + def __call__(cls, *args, **kwargs): + # Hack: override https://github.com/fsspec/filesystem_spec/blob/dcb167e8f50e6273d4cfdfc4cab8fc5aa4c958bf/fsspec/spec.py#L65 + # Apply fsspec config (env vars / config files) before tokenizing so that + # HfFileSystem picks up defaults the same way other fsspec filesystems do. + kwargs = apply_config(cls, kwargs) + skip = kwargs.pop("skip_instance_cache", False) + fs_token = cls._tokenize(cls, threading.get_ident(), *args, **kwargs) + fs_token_main_thread = cls._tokenize(cls, threading.main_thread().ident, *args, **kwargs) + if not skip and cls.cachable and fs_token in cls._cache: + # reuse cached instance + cls._latest = fs_token + return cls._cache[fs_token] + else: + # create new instance + obj = type.__call__(cls, *args, **kwargs) + if not skip and cls.cachable and fs_token_main_thread in cls._cache: + # reuse the cache from the main thread instance in the new instance + instance_state = cls._cache[fs_token_main_thread]._get_instance_state() + for attr, state_value in instance_state.items(): + setattr(obj, attr, state_value) + obj._fs_token_ = fs_token + obj.storage_args = args + obj.storage_options = kwargs + if cls.cachable and not skip: + cls._latest = fs_token + cls._cache[fs_token] = obj + return obj + + +class HfFileSystem(fsspec.AbstractFileSystem, metaclass=_Cached): # ty: ignore[conflicting-metaclass] + """ + Access a remote Hugging Face Hub repository as if were a local file system. + + > [!WARNING] + > [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading + > Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility + > layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. + + The file system supports paths for the `hf://` protocol, which follows those URL schemes: + + * Models, Datasets and Spaces repositories: + + ``` + hf://[@]/ + hf://datasets/[@]/ + hf://spaces/[@]/ + ``` + + * Buckets (generic storage): + + ``` + hf://buckets// + ``` + + Note: when using the [`HfFileSystem`] directly, passing the `hf://` protocol prefix is optional in paths. + + Args: + endpoint (`str`, *optional*): + Endpoint of the Hub. Defaults to . + token (`bool` or `str`, *optional*): + A valid user access token (string). Defaults to the locally saved + token, which is the recommended method for authentication (see + https://huggingface.co/docs/huggingface_hub/quick-start#authentication). + To disable authentication, pass `False`. + block_size (`int`, *optional*): + Block size for reading and writing files. + expand_info (`bool`, *optional*): + Whether to expand the information of the files. + **storage_options (`dict`, *optional*): + Additional options for the filesystem. See [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.__init__). + + Usage: + + ```python + >>> from huggingface_hub import hffs + + >>> # List files + >>> hffs.glob("my-username/my-model/*.bin") + ['my-username/my-model/pytorch_model.bin'] + >>> hffs.ls("datasets/my-username/my-dataset", detail=False) + ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] + + >>> # Read/write files + >>> with hffs.open("my-username/my-model/pytorch_model.bin") as f: + ... data = f.read() + >>> with hffs.open("my-username/my-model/pytorch_model.bin", "wb") as f: + ... f.write(data) + ``` + + Specify a token for authentication: + ```python + >>> from huggingface_hub import HfFileSystem + >>> hffs = HfFileSystem(token=token) + ``` + """ + + root_marker = "" + protocol = "hf" + + def __init__( + self, + *args, + endpoint: str | None = None, + token: bool | str | None = None, + block_size: int | None = None, + expand_info: bool | None = None, + **storage_options, + ): + super().__init__(*args, **storage_options) + self.endpoint = endpoint or constants.ENDPOINT + self.token = token + self._api = HfApi(endpoint=endpoint, token=token) + self.block_size = block_size + self.expand_info = expand_info + # Maps (repo_type, repo_id, revision) to a 2-tuple with: + # * the 1st element indicating whether the repository and the revision exist + # * the 2nd element being the exception raised if the repository or revision doesn't exist + self._repo_and_revision_exists_cache: dict[tuple[str, str, str | None], tuple[bool, Exception | None]] = {} + # Same for buckets + self._bucket_exists_cache: dict[str, tuple[bool, Exception | None]] = {} + # Note: special case for buckets: revision is always None + # Maps parent directory path to path infos + self.dircache: dict[str, list[dict[str, Any]]] = {} + + @classmethod + def _tokenize(cls, threading_ident: int, *args, **kwargs) -> str: + """Deterministic token for caching""" + # make fs_token robust to default values and to kwargs order + kwargs["endpoint"] = kwargs.get("endpoint") or constants.ENDPOINT + kwargs["token"] = kwargs.get("token") + kwargs = {key: kwargs[key] for key in sorted(kwargs)} + # contrary to fsspec, we don't include pid here + tokenize_args = (cls, threading_ident, args, kwargs) + h = md5(str(tokenize_args).encode()) + return h.hexdigest() + + def _repo_and_revision_exist( + self, repo_type: str, repo_id: str, revision: str | None + ) -> tuple[bool, Exception | None]: + if (repo_type, repo_id, revision) not in self._repo_and_revision_exists_cache: + try: + self._api.repo_info( + repo_id, revision=revision, repo_type=repo_type, timeout=constants.HF_HUB_ETAG_TIMEOUT + ) + except (RepositoryNotFoundError, HFValidationError) as e: + self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)] = False, e + self._repo_and_revision_exists_cache[(repo_type, repo_id, None)] = False, e + except RevisionNotFoundError as e: + self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)] = False, e + self._repo_and_revision_exists_cache[(repo_type, repo_id, None)] = True, None + else: + self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)] = True, None + self._repo_and_revision_exists_cache[(repo_type, repo_id, None)] = True, None + return self._repo_and_revision_exists_cache[(repo_type, repo_id, revision)] + + def _bucket_exists(self, bucket_id: str) -> tuple[bool, Exception | None]: + if bucket_id not in self._bucket_exists_cache: + try: + self._api.bucket_info(bucket_id) + except BucketNotFoundError as e: + self._bucket_exists_cache[bucket_id] = False, e + else: + self._bucket_exists_cache[bucket_id] = True, None + return self._bucket_exists_cache[bucket_id] + + def resolve_path( + self, path: str, revision: str | None = None + ) -> HfFileSystemResolvedRepositoryPath | HfFileSystemResolvedBucketPath: + """ + Resolve a Hugging Face file system path into its components. + + Args: + path (`str`): + Path to resolve. + revision (`str`, *optional*): + The revision of the repo to resolve. Defaults to the revision specified in the path. + + Returns: + [`HfFileSystemResolvedPath`]: Resolved path information containing `repo_type`, `repo_id`, `revision` and `path_in_repo`. + + Raises: + `ValueError`: + If path contains conflicting revision information. + `NotImplementedError`: + If trying to list repositories. + """ + path = self._strip_protocol(path) + if not path: + raise NotImplementedError("Access to buckets and repositories lists is not implemented.") + if path.count("/") == 0: + raise ValueError( + f"Repository id must be 'namespace/name', got '{path}'. Single-segment ids (e.g. 'gpt2') are no longer supported." + ) + + parsed = parse_hf_uri(f"{constants.HF_PROTOCOL}{path}") + + # --- Buckets --- + if parsed.is_bucket: + bucket_exists, err = self._bucket_exists(parsed.id) + if not bucket_exists: + _raise_file_not_found(path, err) + return HfFileSystemResolvedBucketPath(bucket_id=parsed.id, path=parsed.path_in_repo) + + # --- Repositories --- + # Align revision from path with explicit revision argument + if revision is not None and parsed.revision is not None and parsed.revision != revision: + # The caller provided an explicit revision that conflicts with what parse_hf_uri + # parsed. This can happen when a user has a branch literally named "refs" and a + # file at "pr/10" — parse_hf_uri would greedily match "refs/pr/10" as a special + # ref. Fall back to simple '@' splitting so the caller's revision wins. + path_without_type = path.split("/", 1)[1] if path.split("/")[0] in constants.HF_URI_TYPE_PREFIXES else path + repo_id, after_at = path_without_type.split("@", 1) + revision_in_path, path_in_repo = after_at.split("/", 1) if "/" in after_at else (after_at, "") + revision_in_path_decoded = unquote(revision_in_path) + if revision_in_path_decoded != revision: + raise ValueError( + f'Revision specified in path ("{revision_in_path_decoded}") and in `revision` argument ("{revision}") are not the same.' + ) + repo_and_revision_exist, err = self._repo_and_revision_exist(parsed.type, repo_id, revision) + if not repo_and_revision_exist: + _raise_file_not_found(path, err) + return HfFileSystemResolvedRepositoryPath( + parsed.type, repo_id, revision, path_in_repo, _raw_revision=revision_in_path + ) + + if parsed.revision is not None and revision is None: + revision = parsed.revision + + repo_and_revision_exist, err = self._repo_and_revision_exist(parsed.type, parsed.id, revision) + if not repo_and_revision_exist: + _raise_file_not_found(path, err) + + # Extract raw revision from original path for unresolve() fidelity + raw_revision: str | None = None + if "@" in path and parsed.revision is not None: + path_without_type = path.split("/", 1)[1] if path.split("/")[0] in constants.HF_URI_TYPE_PREFIXES else path + raw_after_at = path_without_type.split("@", 1)[1] + raw_revision = raw_after_at[: -(len(parsed.path_in_repo) + 1)] if parsed.path_in_repo else raw_after_at + + revision = revision if revision is not None else constants.DEFAULT_REVISION + return HfFileSystemResolvedRepositoryPath( + parsed.type, parsed.id, revision, parsed.path_in_repo, _raw_revision=raw_revision + ) + + def invalidate_cache(self, path: str | None = None) -> None: + """ + Clear the cache for a given path. + + For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.invalidate_cache). + + Args: + path (`str`, *optional*): + Path to clear from cache. If not provided, clear the entire cache. + + """ + if not path: + self.dircache.clear() + self._repo_and_revision_exists_cache.clear() + else: + resolved_path = self.resolve_path(path) + path = resolved_path.unresolve() + while path: + self.dircache.pop(path, None) + path = self._parent(path) + + # Only clear repo cache if path is to repo root + if not resolved_path.path: + if isinstance(resolved_path, HfFileSystemResolvedRepositoryPath): + self._repo_and_revision_exists_cache.pop( + (resolved_path.repo_type, resolved_path.repo_id, None), None + ) + self._repo_and_revision_exists_cache.pop( + (resolved_path.repo_type, resolved_path.repo_id, resolved_path.revision), None + ) + else: + self._bucket_exists_cache.pop(resolved_path.bucket_id, None) + + def _open( # type: ignore + self, + path: str, + mode: str = "rb", + block_size: int | None = None, + revision: str | None = None, + **kwargs, + ) -> Union["HfFileSystemFile", "HfFileSystemStreamFile"]: + block_size = block_size if block_size is not None else self.block_size + if block_size is not None: + kwargs["block_size"] = block_size + if "a" in mode: + raise NotImplementedError("Appending to remote files is not yet supported.") + if block_size == 0: + return HfFileSystemStreamFile(self, path, mode=mode, revision=revision, **kwargs) + else: + return HfFileSystemFile(self, path, mode=mode, revision=revision, **kwargs) + + def _rm(self, path: str, revision: str | None = None, **kwargs) -> None: + resolved_path = self.resolve_path(path, revision=revision) + if isinstance(resolved_path, HfFileSystemResolvedBucketPath): + self._api.batch_bucket_files(resolved_path.bucket_id, delete=[resolved_path.path]) + else: + self._api.delete_file( + path_in_repo=resolved_path.path_in_repo, + repo_id=resolved_path.repo_id, + token=self.token, + repo_type=resolved_path.repo_type, + revision=resolved_path.revision, + commit_message=kwargs.get("commit_message"), + commit_description=kwargs.get("commit_description"), + ) + self.invalidate_cache(path=resolved_path.unresolve()) + + def rm( + self, + path: str, + recursive: bool = False, + maxdepth: int | None = None, + revision: str | None = None, + **kwargs, + ) -> None: + """ + Delete files from a repository. + + For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm). + + > [!WARNING] + > Note: When possible, use `HfApi.delete_file()` for better performance. + + Args: + path (`str`): + Path to delete. + recursive (`bool`, *optional*): + If True, delete directory and all its contents. Defaults to False. + maxdepth (`int`, *optional*): + Maximum number of subdirectories to visit when deleting recursively. + revision (`str`, *optional*): + The git revision to delete from. + + """ + resolved_path = self.resolve_path(path, revision=revision) + paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth, revision=revision) + if isinstance(resolved_path, HfFileSystemResolvedBucketPath): + delete = [self.resolve_path(path).path for path in paths if not self.isdir(path)] + self._api.batch_bucket_files(resolved_path.bucket_id, delete=delete) + else: + paths_in_repo = [self.resolve_path(path).path for path in paths if not self.isdir(path)] + operations = [CommitOperationDelete(path_in_repo=path_in_repo) for path_in_repo in paths_in_repo] + commit_message = f"Delete {path} " + commit_message += "recursively " if recursive else "" + commit_message += f"up to depth {maxdepth} " if maxdepth is not None else "" + # TODO: use `commit_description` to list all the deleted paths? + self._api.create_commit( + repo_id=resolved_path.repo_id, + repo_type=resolved_path.repo_type, + token=self.token, + operations=operations, + revision=resolved_path.revision, + commit_message=kwargs.get("commit_message", commit_message), + commit_description=kwargs.get("commit_description"), + ) + self.invalidate_cache(path=resolved_path.unresolve()) + + def ls( + self, path: str, detail: bool = True, refresh: bool = False, revision: str | None = None, **kwargs + ) -> list[str | dict[str, Any]]: + """ + List the contents of a directory. + + For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.ls). + + > [!WARNING] + > Note: When possible, use `HfApi.list_repo_tree()` for better performance. + + Args: + path (`str`): + Path to the directory. + detail (`bool`, *optional*): + If True, returns a list of dictionaries containing file information. If False, + returns a list of file paths. Defaults to True. + refresh (`bool`, *optional*): + If True, bypass the cache and fetch the latest data. Defaults to False. + revision (`str`, *optional*): + The git revision to list from. + + Returns: + `list[Union[str, dict[str, Any]]]`: List of file paths (if detail=False) or list of file information + dictionaries (if detail=True). + """ + resolved_path = self.resolve_path(path, revision=revision) + path = resolved_path.unresolve() + try: + out = self._ls_tree(path, refresh=refresh, revision=revision, **kwargs) + except EntryNotFoundError: + # Path could be a file + if not resolved_path.path: + _raise_file_not_found(path, None) + try: + out = self._ls_tree(self._parent(path), refresh=refresh, revision=revision, **kwargs) + except EntryNotFoundError: + out = [] + out = [o for o in out if o["name"] == path] + if len(out) == 0: + _raise_file_not_found(path, None) + return out if detail else [o["name"] for o in out] + + def _ls_tree( + self, + path: str, + recursive: bool = False, + refresh: bool = False, + revision: str | None = None, + expand_info: bool | None = None, + maxdepth: int | None = None, + ): + expand_info = ( + expand_info if expand_info is not None else (self.expand_info if self.expand_info is not None else False) + ) + resolved_path = self.resolve_path(path, revision=revision) + path = resolved_path.unresolve() + root_path = resolved_path.root + maxdepth = maxdepth if recursive else 1 + + out = [] + if path in self.dircache and not refresh: + cached_path_infos = self.dircache[path] + out.extend(cached_path_infos) + dirs_not_in_dircache = [] + if recursive: + # Use BFS to traverse the cache and build the "recursive "output + # (The Hub uses a so-called "tree first" strategy for the tree endpoint but we sort the output to follow the spec so the result is (eventually) the same) + depth = 2 + dirs_to_visit = deque( + [(depth, path_info) for path_info in cached_path_infos if path_info["type"] == "directory"] + ) + while dirs_to_visit: + depth, dir_info = dirs_to_visit.popleft() + if maxdepth is None or depth <= maxdepth: + if dir_info["name"] not in self.dircache: + dirs_not_in_dircache.append(dir_info["name"]) + else: + cached_path_infos = self.dircache[dir_info["name"]] + out.extend(cached_path_infos) + dirs_to_visit.extend( + [ + (depth + 1, path_info) + for path_info in cached_path_infos + if path_info["type"] == "directory" + ] + ) + + dirs_not_expanded = [] + if expand_info and isinstance(resolved_path, HfFileSystemResolvedRepositoryPath): + # Check if there are directories in repos with non-expanded entries + dirs_not_expanded = [self._parent(o["name"]) for o in out if o["last_commit"] is None] + + if (recursive and dirs_not_in_dircache) or (expand_info and dirs_not_expanded): + # If the dircache is incomplete, find the common path of the missing and non-expanded entries + # and extend the output with the result of `_ls_tree(common_path, recursive=True)` + common_prefix = os.path.commonprefix(dirs_not_in_dircache + dirs_not_expanded) + # Get the parent directory if the common prefix itself is not a directory + common_path = ( + common_prefix.rstrip("/") + if common_prefix.endswith("/") + or common_prefix == root_path + or common_prefix in chain(dirs_not_in_dircache, dirs_not_expanded) + else self._parent(common_prefix) + ) + if maxdepth is not None: + common_path_depth = common_path[len(path) :].count("/") + maxdepth -= common_path_depth + out = [o for o in out if not o["name"].startswith(common_path + "/")] + for cached_path in list(self.dircache): + if cached_path.startswith(common_path + "/"): + self.dircache.pop(cached_path, None) + self.dircache.pop(common_path, None) + out.extend( + self._ls_tree( + common_path, + recursive=recursive, + refresh=True, + revision=revision, + expand_info=expand_info, + maxdepth=maxdepth, + ) + ) + else: + tree: Iterable[RepoFile | RepoFolder | BucketFile | BucketFolder] + if isinstance(resolved_path, HfFileSystemResolvedBucketPath): + tree = self._list_bucket_tree_with_folders( + resolved_path.bucket_id, + prefix=resolved_path.path, + recursive=recursive, + ) + else: + tree = self._api.list_repo_tree( + resolved_path.repo_id, + resolved_path.path, + recursive=recursive, + expand=expand_info, + revision=resolved_path.revision, + repo_type=resolved_path.repo_type, + ) + for path_info in tree: + cache_path = root_path + "/" + path_info.path + if isinstance(path_info, RepoFile): + cache_path_info = { + "name": cache_path, + "size": path_info.size, + "type": "file", + "blob_id": path_info.blob_id, + "lfs": path_info.lfs, + "xet_hash": path_info.xet_hash, + "last_commit": path_info.last_commit, + "security": path_info.security, + } + elif isinstance(path_info, BucketFile): + cache_path_info = { + "name": cache_path, + "size": path_info.size, + "type": "file", + "xet_hash": path_info.xet_hash, + "mtime": path_info.mtime, + "uploaded_at": path_info.uploaded_at, + } + elif isinstance(path_info, RepoFolder): + cache_path_info = { + "name": cache_path, + "size": 0, + "type": "directory", + "tree_id": path_info.tree_id, + "last_commit": path_info.last_commit, + } + else: + cache_path_info = { + "name": cache_path, + "size": 0, + "type": "directory", + "uploaded_at": path_info.uploaded_at, + } + parent_path = self._parent(cache_path_info["name"]) + self.dircache.setdefault(parent_path, []).append(cache_path_info) + depth = cache_path[len(path) :].count("/") + if maxdepth is None or depth <= maxdepth: + out.append(cache_path_info) + return out + + def _list_bucket_tree_with_folders( + self, bucket_id: str, prefix: str, recursive: bool + ) -> Iterable[BucketFile | BucketFolder]: + """Same as `HfApi.list_bucket_tree` but always includes folders""" + bucket_files = self._api.list_bucket_tree(bucket_id, prefix, recursive=recursive) + bucket_folders: dict[str, BucketFolder] = {} + min_depth = 1 + prefix.count("/") if prefix else 0 + out: list[BucketFile | BucketFolder] = [] + + for bucket_entry in bucket_files: + out.append(bucket_entry) + + # If recursive=False, both files and folders are returned by the server => nothing to do + if not recursive: + continue + + # Otherwise, let's rebuild BucketFolders manually + for parent_bucket_folder_str in list(PurePosixPath(bucket_entry.path).parents)[: -min_depth - 1]: + parent_bucket_folder = BucketFolder( + type="directory", path=str(parent_bucket_folder_str), uploaded_at=bucket_entry.uploaded_at + ) + + # If folder not visited yet, add it + if parent_bucket_folder.path not in bucket_folders: + out.append(parent_bucket_folder) + bucket_folders[parent_bucket_folder.path] = parent_bucket_folder + continue + + # Otherwise, get back BucketFolder object and update its 'uploaded_at' + if parent_bucket_folder.uploaded_at is not None: + bucket_folder = bucket_folders[parent_bucket_folder.path] + if bucket_folder.uploaded_at is None or ( + bucket_folder.uploaded_at < parent_bucket_folder.uploaded_at + ): + bucket_folder.uploaded_at = parent_bucket_folder.uploaded_at + + if not out: + raise EntryNotFoundError(f"File not found in bucket '{bucket_id}': '{prefix}'") + return out + + def walk(self, path: str, *args, **kwargs) -> Iterator[tuple[str, list[str], list[str]]]: + """ + Return all files below the given path. + + For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.walk). + + Args: + path (`str`): + Root path to list files from. + + Returns: + `Iterator[tuple[str, list[str], list[str]]]`: An iterator of (path, list of directory names, list of file names) tuples. + """ + path = self.resolve_path(path, revision=kwargs.get("revision")).unresolve() + yield from super().walk(path, *args, **kwargs) + + def glob(self, path: str, maxdepth: int | None = None, **kwargs) -> list[str]: + """ + Find files by glob-matching. + + For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob). + + Args: + path (`str`): + Path pattern to match. + maxdepth (`int`, *optional*): + Maximum depth to descend into directories. By default, no limit. + + Returns: + `list[str]`: List of paths matching the pattern. + """ + path = self.resolve_path(path, revision=kwargs.get("revision")).unresolve() + return super().glob(path, maxdepth=maxdepth, **kwargs) + + def find( + self, + path: str, + maxdepth: int | None = None, + withdirs: bool = False, + detail: bool = False, + refresh: bool = False, + revision: str | None = None, + **kwargs, + ) -> list[str] | dict[str, dict[str, Any]]: + """ + List all files below path. + + For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). + + Args: + path (`str`): + Root path to list files from. + maxdepth (`int`, *optional*): + Maximum depth to descend into subdirectories. + withdirs (`bool`, *optional*): + Include directory paths in the output. Defaults to False. + detail (`bool`, *optional*): + If True, returns a dict mapping paths to file information. Defaults to False. + refresh (`bool`, *optional*): + If True, bypass the cache and fetch the latest data. Defaults to False. + revision (`str`, *optional*): + The git revision to list from. + + Returns: + `Union[list[str], dict[str, dict[str, Any]]]`: List of paths or dict of file information. + """ + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + resolved_path = self.resolve_path(path, revision=revision) + path = resolved_path.unresolve() + try: + out = self._ls_tree(path, recursive=True, refresh=refresh, maxdepth=maxdepth, **kwargs) + except EntryNotFoundError: + # Path could be a file + try: + if self.info(path, revision=revision, **kwargs)["type"] == "file": + out = {path: {}} + else: + out = {} + except FileNotFoundError: + out = {} + else: + if not withdirs: + out = [o for o in out if o["type"] != "directory"] + else: + # If `withdirs=True`, include the directory itself to be consistent with the spec + path_info = self.info(path, **kwargs) + out = [path_info] + out if path_info["type"] == "directory" else out + out = {o["name"]: o for o in out} + names = sorted(out) + if not detail: + return names + else: + return {name: out[name] for name in names} + + def cp_file(self, path1: str, path2: str, revision: str | None = None, **kwargs) -> None: + """ + Copy a file within or between repositories. + + > [!WARNING] + > Note: When possible, use `HfApi.upload_file()` for better performance. + + Args: + path1 (`str`): + Source path to copy from. + path2 (`str`): + Destination path to copy to. + revision (`str`, *optional*): + The git revision to copy from. + + """ + resolved_path1 = self.resolve_path(path1, revision=revision) + resolved_path2 = self.resolve_path(path2, revision=revision) + if isinstance(resolved_path1, HfFileSystemResolvedBucketPath) or isinstance( + resolved_path2, HfFileSystemResolvedBucketPath + ): + raise NotImplementedError("Copy from/to buckets is not available yet") + + same_repo = ( + resolved_path1.repo_type == resolved_path2.repo_type and resolved_path1.repo_id == resolved_path2.repo_id + ) + + if same_repo: + commit_message = f"Copy {path1} to {path2}" + self._api.create_commit( + repo_id=resolved_path1.repo_id, + repo_type=resolved_path1.repo_type, + revision=resolved_path2.revision, + commit_message=kwargs.get("commit_message", commit_message), + commit_description=kwargs.get("commit_description", ""), + operations=[ + CommitOperationCopy( + src_path_in_repo=resolved_path1.path_in_repo, + path_in_repo=resolved_path2.path_in_repo, + src_revision=resolved_path1.revision, + ) + ], + ) + else: + with self.open(path1, "rb", revision=resolved_path1.revision) as f: + content = f.read() + commit_message = f"Copy {path1} to {path2}" + self._api.upload_file( + path_or_fileobj=content, + path_in_repo=resolved_path2.path_in_repo, + repo_id=resolved_path2.repo_id, + token=self.token, + repo_type=resolved_path2.repo_type, + revision=resolved_path2.revision, + commit_message=kwargs.get("commit_message", commit_message), + commit_description=kwargs.get("commit_description"), + ) + self.invalidate_cache(path=resolved_path1.unresolve()) + self.invalidate_cache(path=resolved_path2.unresolve()) + + def modified(self, path: str, **kwargs) -> datetime: + """ + Get the last modified time of a file. + + For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). + + Args: + path (`str`): + Path to the file. + + Returns: + `datetime`: Last modified time of the file. + """ + info = self.info(path, **{**kwargs, "expand_info": True}) # type: ignore + if "last_commit" in info: + if info["last_commit"] is None: + raise NotImplementedError(f"'modified' is not implemented for repository paths like '{path}'") + return info["last_commit"].date + elif "mtime" in info and info["mtime"]: + return info["mtime"] + elif "uploaded_at" in info and info["uploaded_at"]: + return info["uploaded_at"] + else: + raise NotImplementedError(f"Cannot determined 'modified' for path '{path}' (info: {info})") + + def info(self, path: str, refresh: bool = False, revision: str | None = None, **kwargs) -> dict[str, Any]: + """ + Get information about a file or directory. + + For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). + + > [!WARNING] + > Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance + > (or `HfApi.get_bucket_paths_info()` or `HfApi.bucket_info()` for buckets) + + Args: + path (`str`): + Path to get info for. + refresh (`bool`, *optional*): + If True, bypass the cache and fetch the latest data. Defaults to False. + revision (`str`, *optional*): + The git revision to get info from. + + Returns: + `dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). + + """ + resolved_path = self.resolve_path(path, revision=revision) + path = resolved_path.unresolve() + expand_info = kwargs.get( + "expand_info", self.expand_info if self.expand_info is not None else False + ) # don't expose it as a parameter in the public API to follow the spec + out: dict[str, Any] | None + if not resolved_path.path: + # Path is the root directory + out = { + "name": path, + "size": 0, + "type": "directory", + } + if isinstance(resolved_path, HfFileSystemResolvedRepositoryPath): + out["last_commit"] = None + if isinstance(resolved_path, HfFileSystemResolvedRepositoryPath) and expand_info: + last_commit = self._api.list_repo_commits( + resolved_path.repo_id, repo_type=resolved_path.repo_type, revision=resolved_path.revision + )[-1] + out = { + **out, + "tree_id": None, # TODO: tree_id of the root directory? + "last_commit": LastCommitInfo( + oid=last_commit.commit_id, title=last_commit.title, date=last_commit.created_at + ), + } + elif isinstance(resolved_path, HfFileSystemResolvedBucketPath): + parent_path = self._parent(path) + # Fill the cache with cheap call + self.ls(parent_path, refresh=refresh) + out1 = [o for o in self.dircache[parent_path] if o["name"] == path] + if not out1: + _raise_file_not_found(path, None) + out = out1[0] + else: + out = None + parent_path = self._parent(path) + if not expand_info and parent_path not in self.dircache: + # Fill the cache with cheap call + self.ls(parent_path) + if parent_path in self.dircache: + # Check if the path is in the cache + out1 = [o for o in self.dircache[parent_path] if o["name"] == path] + if not out1: + _raise_file_not_found(path, None) + out = out1[0] + if refresh or out is None or (expand_info and out and out["last_commit"] is None): + paths_info = self._api.get_paths_info( + resolved_path.repo_id, + resolved_path.path_in_repo, + expand=expand_info, + revision=resolved_path.revision, + repo_type=resolved_path.repo_type, + ) + if not paths_info: + _raise_file_not_found(path, None) + path_info = paths_info[0] + root_path = HfFileSystemResolvedRepositoryPath( + resolved_path.repo_type, + resolved_path.repo_id, + resolved_path.revision, + path_in_repo="", + _raw_revision=resolved_path._raw_revision, + ).unresolve() + if isinstance(path_info, RepoFile): + out = { + "name": root_path + "/" + path_info.path, + "size": path_info.size, + "type": "file", + "blob_id": path_info.blob_id, + "lfs": path_info.lfs, + "xet_hash": path_info.xet_hash, + "last_commit": path_info.last_commit, + "security": path_info.security, + } + else: + out = { + "name": root_path + "/" + path_info.path, + "size": 0, + "type": "directory", + "tree_id": path_info.tree_id, + "last_commit": path_info.last_commit, + } + if not expand_info: + out = {k: out[k] for k in ["name", "size", "type"]} + assert out is not None + return out + + def exists(self, path, **kwargs): + """ + Check if a file exists. + + For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). + + > [!WARNING] + > Note: When possible, use `HfApi.file_exists()` for better performance. + + Args: + path (`str`): + Path to check. + + Returns: + `bool`: True if file exists, False otherwise. + """ + try: + if kwargs.get("refresh", False): + self.invalidate_cache(path) + + self.info(path, **kwargs) + return True + except OSError: + return False + + def isdir(self, path): + """ + Check if a path is a directory. + + For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). + + Args: + path (`str`): + Path to check. + + Returns: + `bool`: True if path is a directory, False otherwise. + """ + try: + return self.info(path)["type"] == "directory" + except OSError: + return False + + def isfile(self, path): + """ + Check if a path is a file. + + For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). + + Args: + path (`str`): + Path to check. + + Returns: + `bool`: True if path is a file, False otherwise. + """ + try: + return self.info(path)["type"] == "file" + except OSError: + return False + + def url(self, path: str) -> str: + """ + Get the HTTP URL of the given path. + + Args: + path (`str`): + Path to get URL for. + + Returns: + `str`: HTTP URL to access the file or directory on the Hub. + """ + resolved_path = self.resolve_path(path) + if isinstance(resolved_path, HfFileSystemResolvedBucketPath): + url = f"{self.endpoint}/buckets/{resolved_path.bucket_id}/resolve/{quote(resolved_path.path)}" + else: + url = hf_hub_url( + resolved_path.repo_id, + resolved_path.path_in_repo, + repo_type=resolved_path.repo_type, + revision=resolved_path.revision, + endpoint=self.endpoint, + ) + if self.isdir(path): + url = url.replace("/resolve/", "/tree/", 1) + return url + + def get_file(self, rpath, lpath, callback=_DEFAULT_CALLBACK, outfile=None, **kwargs) -> None: + """ + Copy single remote file to local. + + > [!WARNING] + > Note: When possible, use `HfApi.hf_hub_download()` or `HfApi.download_bucket_files` for better performance. + + Args: + rpath (`str`): + Remote path to download from. + lpath (`str`): + Local path to download to. + callback (`Callback`, *optional*): + Optional callback to track download progress. Defaults to no callback. + outfile (`IO`, *optional*): + Optional file-like object to write to. If provided, `lpath` is ignored. + + """ + revision = kwargs.get("revision") + unhandled_kwargs = set(kwargs.keys()) - {"revision"} + if not isinstance(callback, (NoOpCallback, TqdmCallback)) or len(unhandled_kwargs) > 0: + # for now, let's not handle custom callbacks + # and let's not handle custom kwargs + return super().get_file(rpath, lpath, callback=callback, outfile=outfile, **kwargs) + + # Taken from https://github.com/fsspec/filesystem_spec/blob/47b445ae4c284a82dd15e0287b1ffc410e8fc470/fsspec/spec.py#L883 + if isfilelike(lpath): + outfile = lpath + elif self.isdir(rpath): + os.makedirs(lpath, exist_ok=True) + return None + + if isinstance(lpath, (str, Path)): # otherwise, let's assume it's a file-like object + os.makedirs(os.path.dirname(lpath), exist_ok=True) + + # Open file if not already open + close_file = False + if outfile is None: + outfile = open(lpath, "wb") + close_file = True + initial_pos = outfile.tell() + + # Custom implementation of `get_file` to use `http_get`. + resolve_remote_path = self.resolve_path(rpath, revision=revision) + expected_size = self.info(rpath, revision=revision)["size"] + callback.set_size(expected_size) + try: + http_get( + url=self.url(resolve_remote_path.unresolve()), + temp_file=outfile, # type: ignore + displayed_filename=rpath, + expected_size=expected_size, + resume_size=0, + headers=self._api._build_hf_headers(), + _tqdm_bar=callback.tqdm if isinstance(callback, TqdmCallback) else None, + ) + outfile.seek(initial_pos) + finally: + # Close file only if we opened it ourselves + if close_file: + outfile.close() + + @property + def transaction(self): + """A context within which files are committed together upon exit + + Requires the file class to implement `.commit()` and `.discard()` + for the normal and exception cases. + """ + # Taken from https://github.com/fsspec/filesystem_spec/blob/3fbb6fee33b46cccb015607630843dea049d3243/fsspec/spec.py#L231 + # See https://github.com/huggingface/huggingface_hub/issues/1733 + raise NotImplementedError("Transactional commits are not supported.") + + def start_transaction(self): + """Begin write transaction for deferring files, non-context version""" + # Taken from https://github.com/fsspec/filesystem_spec/blob/3fbb6fee33b46cccb015607630843dea049d3243/fsspec/spec.py#L241 + # See https://github.com/huggingface/huggingface_hub/issues/1733 + raise NotImplementedError("Transactional commits are not supported.") + + def __reduce__(self): + # re-populate the instance cache at HfFileSystem._cache and re-populate the state of every instance + return make_instance, ( + type(self), + self.storage_args, + self.storage_options, + self._get_instance_state(), + ) + + def _get_instance_state(self): + return { + "dircache": deepcopy(self.dircache), + "_repo_and_revision_exists_cache": deepcopy(self._repo_and_revision_exists_cache), + "_bucket_exists_cache": deepcopy(self._bucket_exists_cache), + } + + +class HfFileSystemFile(fsspec.spec.AbstractBufferedFile): + def __init__(self, fs: HfFileSystem, path: str, revision: str | None = None, **kwargs): + try: + self.resolved_path = fs.resolve_path(path, revision=revision) + except FileNotFoundError as e: + if "w" in kwargs.get("mode", ""): + raise FileNotFoundError( + f"{e}.\nMake sure the repository and revision exist before writing data." + ) from e + raise + super().__init__(fs, self.resolved_path.unresolve(), **kwargs) + self.fs: HfFileSystem + + def __del__(self): + if not hasattr(self, "resolved_path"): + # Means that the constructor failed. Nothing to do. + return + return super().__del__() + + def _fetch_range(self, start: int, end: int) -> bytes: + headers = { + "range": f"bytes={start}-{end - 1}", + **self.fs._api._build_hf_headers(), + } + url = self.url() + r = http_backoff("GET", url, headers=headers, timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT) + hf_raise_for_status(r) + return r.content + + def _initiate_upload(self) -> None: + self.temp_file = tempfile.NamedTemporaryFile(prefix="hffs-", delete=False) + + def _upload_chunk(self, final: bool = False) -> None: + self.buffer.seek(0) + block = self.buffer.read() + self.temp_file.write(block) + if final: + self.temp_file.close() + if isinstance(self.resolved_path, HfFileSystemResolvedBucketPath): + self.fs._api.batch_bucket_files( + self.resolved_path.bucket_id, add=[(self.temp_file.name, self.resolved_path.path)] + ) + else: + self.fs._api.upload_file( + path_or_fileobj=self.temp_file.name, + path_in_repo=self.resolved_path.path_in_repo, + repo_id=self.resolved_path.repo_id, + token=self.fs.token, + repo_type=self.resolved_path.repo_type, + revision=self.resolved_path.revision, + commit_message=self.kwargs.get("commit_message"), + commit_description=self.kwargs.get("commit_description"), + ) + os.remove(self.temp_file.name) + self.fs.invalidate_cache( + path=self.resolved_path.unresolve(), + ) + + def read(self, length=-1): + """Read remote file. + + If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems the file is + loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. + """ + if self.mode == "rb" and (length is None or length == -1) and self.loc == 0: + with self.fs.open(self.path, "rb", block_size=0) as f: # block_size=0 enables fast streaming + out = f.read() + self.loc += len(out) + return out + return super().read(length) + + def url(self) -> str: + return self.fs.url(self.path) + + +class HfFileSystemStreamFile(fsspec.spec.AbstractBufferedFile): + def __init__( + self, + fs: HfFileSystem, + path: str, + mode: str = "rb", + revision: str | None = None, + block_size: int = 0, + cache_type: str = "none", + **kwargs, + ): + if block_size != 0: + raise ValueError(f"HfFileSystemStreamFile only supports block_size=0 but got {block_size}") + if cache_type != "none": + raise ValueError(f"HfFileSystemStreamFile only supports cache_type='none' but got {cache_type}") + if "w" in mode: + raise ValueError(f"HfFileSystemStreamFile only supports reading but got mode='{mode}'") + # Let a missing repo/revision/file surface as FileNotFoundError (as the buffered + # `HfFileSystemFile` does) instead of an AttributeError on `self.resolved_path` below. + self.resolved_path = fs.resolve_path(path, revision=revision) + # avoid an unnecessary .info() call to instantiate .details + self.details = {"name": self.resolved_path.unresolve(), "size": None} + super().__init__( + fs, self.resolved_path.unresolve(), mode=mode, block_size=block_size, cache_type=cache_type, **kwargs + ) + self.response: httpx.Response | None = None + self.fs: HfFileSystem + self._exit_stack = ExitStack() + # streaming state + self._stream_iterator: Iterator[bytes] | None = None + self._stream_buffer = bytearray() + + def seek(self, loc: int, whence: int = 0): + if loc == 0 and whence == 1: + return + if loc == self.loc and whence == 0: + return + raise ValueError("Cannot seek streaming HF file") + + def read(self, length: int = -1): + """Read the remote file. + + If the file is already open, we reuse the connection. + Otherwise, open a new connection and read from it. + + If reading the stream fails, we retry with a new connection. + """ + if self.response is None: + self._open_connection() + + retried_once = False + while True: + try: + if self.response is None or self._stream_iterator is None: + return b"" # Already read the entire file + out = self._read_from_stream(self._stream_iterator, length) + self.loc += len(out) + return out + except Exception: + if self.response is not None: + self.response.close() + if retried_once: # Already retried once, give up + raise + # First failure, retry with range header + self._open_connection() + retried_once = True + + def _read_from_stream(self, iterator: Iterator[bytes], length: int = -1) -> bytes: + """Read up to `length` bytes from stream buffer and stream. + + If length < 0, read until EOF. + + If EOF is reached before length, fewer bytes may be returned. + """ + if length == 0: + return b"" + + if length < 0: + buf = bytearray(self._stream_buffer) + self._stream_buffer.clear() + for chunk in iterator: + buf.extend(chunk) + return bytes(buf) + + if length <= len(self._stream_buffer): + result = bytes(self._stream_buffer[:length]) + del self._stream_buffer[:length] + return result + + buf = bytearray(self._stream_buffer) + self._stream_buffer.clear() + for chunk in iterator: + need = length - len(buf) + if need > len(chunk): + buf.extend(chunk) + else: + buf.extend(chunk[:need]) + self._stream_buffer.extend(chunk[need:]) + break + return bytes(buf) + + def url(self) -> str: + return self.fs.url(self.path) + + def __del__(self): + if not hasattr(self, "resolved_path"): + # Means that the constructor failed. Nothing to do. + return + self._exit_stack.close() + return super().__del__() + + def __reduce__(self): + return reopen, (self.fs, self.path, self.mode, self.blocksize, self.cache.name) + + def _open_connection(self): + """Open a connection to the remote file.""" + # reset streaming state + self._stream_buffer.clear() + self._stream_iterator = None + + url = self.url() + headers = self.fs._api._build_hf_headers() + if self.loc > 0: + headers["Range"] = f"bytes={self.loc}-" + self.response = self._exit_stack.enter_context( + http_stream_backoff( + "GET", + url, + headers=headers, + timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT, + ) + ) + + try: + hf_raise_for_status(self.response) + except HfHubHTTPError as e: + if e.response.status_code == 416: + # Range not satisfiable => means that we have already read the entire file + self.response = None + return + raise + + self._stream_iterator = self.response.iter_bytes() + + +def safe_revision(revision: str) -> str: + return revision if SPECIAL_REFS_REVISION_REGEX.match(revision) else safe_quote(revision) + + +def safe_quote(s: str) -> str: + return quote(s, safe="") + + +def _raise_file_not_found(path: str, err: Exception | None) -> NoReturn: + msg = path + if isinstance(err, RepositoryNotFoundError): + msg = f"{path} (repository not found)" + elif isinstance(err, RevisionNotFoundError): + msg = f"{path} (revision not found)" + elif isinstance(err, HFValidationError): + msg = f"{path} (invalid repository id)" + raise FileNotFoundError(msg) from err + + +def reopen(fs: HfFileSystem, path: str, mode: str, block_size: int, cache_type: str): + return fs.open(path, mode=mode, block_size=block_size, cache_type=cache_type) + + +def make_instance(cls, args, kwargs, instance_state): + fs = cls(*args, **kwargs) + for attr, state_value in instance_state.items(): + setattr(fs, attr, state_value) + return fs + + +hffs = HfFileSystem() diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/hub_mixin.py b/venv/lib/python3.12/site-packages/huggingface_hub/hub_mixin.py new file mode 100644 index 0000000000000000000000000000000000000000..937cfe1851538aa8389222885015212c80514871 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/hub_mixin.py @@ -0,0 +1,834 @@ +import inspect +import json +import os +from collections.abc import Callable +from dataclasses import Field, asdict, dataclass, is_dataclass +from pathlib import Path +from typing import Any, ClassVar, Protocol, TypeVar + +import packaging.version + +from . import constants +from .errors import EntryNotFoundError, HfHubHTTPError +from .file_download import hf_hub_download +from .hf_api import HfApi +from .repocard import ModelCard, ModelCardData +from .utils import ( + SoftTemporaryDirectory, + is_jsonable, + is_safetensors_available, + is_simple_optional_type, + is_torch_available, + logging, + unwrap_simple_optional_type, + validate_hf_hub_args, +) + + +if is_torch_available(): + import torch # type: ignore + +if is_safetensors_available(): + import safetensors + from safetensors.torch import load_model as load_model_as_safetensor + from safetensors.torch import save_model as save_model_as_safetensor + + +logger = logging.get_logger(__name__) + + +# Type alias for dataclass instances, copied from https://github.com/python/typeshed/blob/9f28171658b9ca6c32a7cb93fbb99fc92b17858b/stdlib/_typeshed/__init__.pyi#L349 +class DataclassInstance(Protocol): + __dataclass_fields__: ClassVar[dict[str, Field]] + + +# Generic variable that is either ModelHubMixin or a subclass thereof +T = TypeVar("T", bound="ModelHubMixin") +# Generic variable to represent an args type +ARGS_T = TypeVar("ARGS_T") +ENCODER_T = Callable[[ARGS_T], Any] +DECODER_T = Callable[[Any], ARGS_T] +CODER_T = tuple[ENCODER_T, DECODER_T] + + +DEFAULT_MODEL_CARD = """ +--- +# For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 +# Doc / guide: https://huggingface.co/docs/hub/model-cards +{{ card_data }} +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Code: {{ repo_url | default("[More Information Needed]", true) }} +- Paper: {{ paper_url | default("[More Information Needed]", true) }} +- Docs: {{ docs_url | default("[More Information Needed]", true) }} +""" + + +@dataclass +class MixinInfo: + model_card_template: str + model_card_data: ModelCardData + docs_url: str | None = None + paper_url: str | None = None + repo_url: str | None = None + + +class ModelHubMixin: + """ + A generic mixin to integrate ANY machine learning framework with the Hub. + + To integrate your framework, your model class must inherit from this class. Custom logic for saving/loading models + have to be overwritten in [`_from_pretrained`] and [`_save_pretrained`]. [`PyTorchModelHubMixin`] is a good example + of mixin integration with the Hub. Check out our [integration guide](../guides/integrations) for more instructions. + + When inheriting from [`ModelHubMixin`], you can define class-level attributes. These attributes are not passed to + `__init__` but to the class definition itself. This is useful to define metadata about the library integrating + [`ModelHubMixin`]. + + For more details on how to integrate the mixin with your library, checkout the [integration guide](../guides/integrations). + + Args: + repo_url (`str`, *optional*): + URL of the library repository. Used to generate model card. + paper_url (`str`, *optional*): + URL of the library paper. Used to generate model card. + docs_url (`str`, *optional*): + URL of the library documentation. Used to generate model card. + model_card_template (`str`, *optional*): + Template of the model card. Used to generate model card. Defaults to a generic template. + language (`str` or `list[str]`, *optional*): + Language supported by the library. Used to generate model card. + library_name (`str`, *optional*): + Name of the library integrating ModelHubMixin. Used to generate model card. + license (`str`, *optional*): + License of the library integrating ModelHubMixin. Used to generate model card. + E.g: "apache-2.0" + license_name (`str`, *optional*): + Name of the library integrating ModelHubMixin. Used to generate model card. + Only used if `license` is set to `other`. + E.g: "coqui-public-model-license". + license_link (`str`, *optional*): + URL to the license of the library integrating ModelHubMixin. Used to generate model card. + Only used if `license` is set to `other` and `license_name` is set. + E.g: "https://coqui.ai/cpml". + pipeline_tag (`str`, *optional*): + Tag of the pipeline. Used to generate model card. E.g. "text-classification". + tags (`list[str]`, *optional*): + Tags to be added to the model card. Used to generate model card. E.g. ["computer-vision"] + coders (`dict[Type, tuple[Callable, Callable]]`, *optional*): + Dictionary of custom types and their encoders/decoders. Used to encode/decode arguments that are not + jsonable by default. E.g. dataclasses, argparse.Namespace, OmegaConf, etc. + + Example: + + ```python + >>> from huggingface_hub import ModelHubMixin + + # Inherit from ModelHubMixin + >>> class MyCustomModel( + ... ModelHubMixin, + ... library_name="my-library", + ... tags=["computer-vision"], + ... repo_url="https://github.com/huggingface/my-cool-library", + ... paper_url="https://arxiv.org/abs/2304.12244", + ... docs_url="https://huggingface.co/docs/my-cool-library", + ... # ^ optional metadata to generate model card + ... ): + ... def __init__(self, size: int = 512, device: str = "cpu"): + ... # define how to initialize your model + ... super().__init__() + ... ... + ... + ... def _save_pretrained(self, save_directory: Path) -> None: + ... # define how to serialize your model + ... ... + ... + ... @classmethod + ... def from_pretrained( + ... cls: type[T], + ... pretrained_model_name_or_path: Union[str, Path], + ... *, + ... force_download: bool = False, + ... token: Optional[Union[str, bool]] = None, + ... cache_dir: Optional[Union[str, Path]] = None, + ... local_files_only: bool = False, + ... revision: Optional[str] = None, + ... **model_kwargs, + ... ) -> T: + ... # define how to deserialize your model + ... ... + + >>> model = MyCustomModel(size=256, device="gpu") + + # Save model weights to local directory + >>> model.save_pretrained("my-awesome-model") + + # Push model weights to the Hub + >>> model.push_to_hub("my-awesome-model") + + # Download and initialize weights from the Hub + >>> reloaded_model = MyCustomModel.from_pretrained("username/my-awesome-model") + >>> reloaded_model.size + 256 + + # Model card has been correctly populated + >>> from huggingface_hub import ModelCard + >>> card = ModelCard.load("username/my-awesome-model") + >>> card.data.tags + ["x-custom-tag", "pytorch_model_hub_mixin", "model_hub_mixin"] + >>> card.data.library_name + "my-library" + ``` + """ + + _hub_mixin_config: dict | DataclassInstance | None = None + # ^ optional config attribute automatically set in `from_pretrained` + _hub_mixin_info: MixinInfo + # ^ information about the library integrating ModelHubMixin (used to generate model card) + _hub_mixin_inject_config: bool # whether `_from_pretrained` expects `config` or not + _hub_mixin_init_parameters: dict[str, inspect.Parameter] # __init__ parameters + _hub_mixin_jsonable_default_values: dict[str, Any] # default values for __init__ parameters + _hub_mixin_jsonable_custom_types: tuple[type, ...] # custom types that can be encoded/decoded + _hub_mixin_coders: dict[type, CODER_T] # encoders/decoders for custom types + # ^ internal values to handle config + + def __init_subclass__( + cls, + *, + # Generic info for model card + repo_url: str | None = None, + paper_url: str | None = None, + docs_url: str | None = None, + # Model card template + model_card_template: str = DEFAULT_MODEL_CARD, + # Model card metadata + language: list[str] | None = None, + library_name: str | None = None, + license: str | None = None, + license_name: str | None = None, + license_link: str | None = None, + pipeline_tag: str | None = None, + tags: list[str] | None = None, + # How to encode/decode arguments with custom type into a JSON config? + coders: None + | ( + dict[type, CODER_T] + # Key is a type. + # Value is a tuple (encoder, decoder). + # Example: {MyCustomType: (lambda x: x.value, lambda data: MyCustomType(data))} + ) = None, + ) -> None: + """Inspect __init__ signature only once when subclassing + handle modelcard.""" + super().__init_subclass__() + + # Will be reused when creating modelcard + tags = tags or [] + tags.append("model_hub_mixin") + + # Initialize MixinInfo if not existent + info = MixinInfo(model_card_template=model_card_template, model_card_data=ModelCardData()) + + # If parent class has a MixinInfo, inherit from it as a copy + if hasattr(cls, "_hub_mixin_info"): + # Inherit model card template from parent class if not explicitly set + if model_card_template == DEFAULT_MODEL_CARD: + info.model_card_template = cls._hub_mixin_info.model_card_template + + # Inherit from parent model card data + info.model_card_data = ModelCardData(**cls._hub_mixin_info.model_card_data.to_dict()) + + # Inherit other info + info.docs_url = cls._hub_mixin_info.docs_url + info.paper_url = cls._hub_mixin_info.paper_url + info.repo_url = cls._hub_mixin_info.repo_url + cls._hub_mixin_info = info + + # Update MixinInfo with metadata + if model_card_template is not None and model_card_template != DEFAULT_MODEL_CARD: + info.model_card_template = model_card_template + if repo_url is not None: + info.repo_url = repo_url + if paper_url is not None: + info.paper_url = paper_url + if docs_url is not None: + info.docs_url = docs_url + if language is not None: + info.model_card_data.language = language + if library_name is not None: + info.model_card_data.library_name = library_name + if license is not None: + info.model_card_data.license = license + if license_name is not None: + info.model_card_data.license_name = license_name + if license_link is not None: + info.model_card_data.license_link = license_link + if pipeline_tag is not None: + info.model_card_data.pipeline_tag = pipeline_tag + if tags is not None: + normalized_tags = list(tags) + if info.model_card_data.tags is not None: + info.model_card_data.tags.extend(normalized_tags) + else: + info.model_card_data.tags = normalized_tags + + if info.model_card_data.tags is not None: + info.model_card_data.tags = sorted(set(info.model_card_data.tags)) + + # Handle encoders/decoders for args + cls._hub_mixin_coders = coders or {} + cls._hub_mixin_jsonable_custom_types = tuple(cls._hub_mixin_coders.keys()) + + # Inspect __init__ signature to handle config + cls._hub_mixin_init_parameters = dict(inspect.signature(cls.__init__).parameters) + cls._hub_mixin_jsonable_default_values = { + param.name: cls._encode_arg(param.default) + for param in cls._hub_mixin_init_parameters.values() + if param.default is not inspect.Parameter.empty and cls._is_jsonable(param.default) + } + cls._hub_mixin_inject_config = "config" in inspect.signature(cls._from_pretrained).parameters + + def __new__(cls: type[T], *args, **kwargs) -> T: + """Create a new instance of the class and handle config. + + 3 cases: + - If `self._hub_mixin_config` is already set, do nothing. + - If `config` is passed as a dataclass, set it as `self._hub_mixin_config`. + - Otherwise, build `self._hub_mixin_config` from default values and passed values. + """ + instance = super().__new__(cls) + + # If `config` is already set, return early + if instance._hub_mixin_config is not None: + return instance + + # Infer passed values + passed_values = { + **{ + key: value + for key, value in zip( + # [1:] to skip `self` parameter + list(cls._hub_mixin_init_parameters)[1:], + args, + ) + }, + **kwargs, + } + + # If config passed as dataclass => set it and return early + if is_dataclass(passed_values.get("config")): + instance._hub_mixin_config = passed_values["config"] + return instance + + # Otherwise, build config from default + passed values + init_config = { + # default values + **cls._hub_mixin_jsonable_default_values, + # passed values + **{ + key: cls._encode_arg(value) # Encode custom types as jsonable value + for key, value in passed_values.items() + if instance._is_jsonable(value) # Only if jsonable or we have a custom encoder + }, + } + passed_config = init_config.pop("config", {}) + + # Populate `init_config` with provided config + if isinstance(passed_config, dict): + init_config.update(passed_config) + + # Set `config` attribute and return + if init_config != {}: + instance._hub_mixin_config = init_config + return instance + + @classmethod + def _is_jsonable(cls, value: Any) -> bool: + """Check if a value is JSON serializable.""" + if is_dataclass(value): + return True + if isinstance(value, cls._hub_mixin_jsonable_custom_types): + return True + return is_jsonable(value) + + @classmethod + def _encode_arg(cls, arg: Any) -> Any: + """Encode an argument into a JSON serializable format.""" + if is_dataclass(arg): + return asdict(arg) # type: ignore[arg-type] + for type_, (encoder, _) in cls._hub_mixin_coders.items(): + if isinstance(arg, type_): + if arg is None: + return None + return encoder(arg) + return arg + + @classmethod + def _decode_arg(cls, expected_type: type[ARGS_T], value: Any) -> ARGS_T | None: + """Decode a JSON serializable value into an argument.""" + if is_simple_optional_type(expected_type): + if value is None: + return None + expected_type = unwrap_simple_optional_type(expected_type) # type: ignore + # Dataclass => handle it + if is_dataclass(expected_type): + return _load_dataclass(expected_type, value) # type: ignore + # Otherwise => check custom decoders + for type_, (_, decoder) in cls._hub_mixin_coders.items(): + if inspect.isclass(expected_type) and issubclass(expected_type, type_): + return decoder(value) + # Otherwise => don't decode + return value + + def save_pretrained( + self, + save_directory: str | Path, + *, + config: dict | DataclassInstance | None = None, + repo_id: str | None = None, + push_to_hub: bool = False, + model_card_kwargs: dict[str, Any] | None = None, + **push_to_hub_kwargs, + ) -> str | None: + """ + Save weights in local directory. + + Args: + save_directory (`str` or `Path`): + Path to directory in which the model weights and configuration will be saved. + config (`dict` or `DataclassInstance`, *optional*): + Model configuration specified as a key/value dictionary or a dataclass instance. + push_to_hub (`bool`, *optional*, defaults to `False`): + Whether or not to push your model to the Huggingface Hub after saving it. + repo_id (`str`, *optional*): + ID of your repository on the Hub. Used only if `push_to_hub=True`. Will default to the folder name if + not provided. + model_card_kwargs (`dict[str, Any]`, *optional*): + Additional arguments passed to the model card template to customize the model card. + push_to_hub_kwargs: + Additional key word arguments passed along to the [`~ModelHubMixin.push_to_hub`] method. + Returns: + `str` or `None`: url of the commit on the Hub if `push_to_hub=True`, `None` otherwise. + """ + save_directory = Path(save_directory) + save_directory.mkdir(parents=True, exist_ok=True) + + # Remove config.json if already exists. After `_save_pretrained` we don't want to overwrite config.json + # as it might have been saved by the custom `_save_pretrained` already. However we do want to overwrite + # an existing config.json if it was not saved by `_save_pretrained`. + config_path = save_directory / constants.CONFIG_NAME + config_path.unlink(missing_ok=True) + + # save model weights/files (framework-specific) + self._save_pretrained(save_directory) + + # save config (if provided and if not serialized yet in `_save_pretrained`) + if config is None: + config = self._hub_mixin_config + if config is not None: + if is_dataclass(config): + config = asdict(config) # type: ignore[arg-type] + if not config_path.exists(): + config_str = json.dumps(config, sort_keys=True, indent=2) + config_path.write_text(config_str) + + # save model card + model_card_path = save_directory / "README.md" + model_card_kwargs = model_card_kwargs if model_card_kwargs is not None else {} + if not model_card_path.exists(): # do not overwrite if already exists + self.generate_model_card(**model_card_kwargs).save(save_directory / "README.md") + + # push to the Hub if required + if push_to_hub: + kwargs = push_to_hub_kwargs.copy() # soft-copy to avoid mutating input + if config is not None: # kwarg for `push_to_hub` + kwargs["config"] = config + if repo_id is None: + repo_id = save_directory.name # Defaults to `save_directory` name + return self.push_to_hub(repo_id=repo_id, model_card_kwargs=model_card_kwargs, **kwargs) + return None + + def _save_pretrained(self, save_directory: Path) -> None: + """ + Overwrite this method in subclass to define how to save your model. + Check out our [integration guide](../guides/integrations) for instructions. + + Args: + save_directory (`str` or `Path`): + Path to directory in which the model weights and configuration will be saved. + """ + raise NotImplementedError + + @classmethod + @validate_hf_hub_args + def from_pretrained( + cls: type[T], + pretrained_model_name_or_path: str | Path, + *, + force_download: bool = False, + token: str | bool | None = None, + cache_dir: str | Path | None = None, + local_files_only: bool = False, + revision: str | None = None, + **model_kwargs, + ) -> T: + """ + Download a model from the Huggingface Hub and instantiate it. + + Args: + pretrained_model_name_or_path (`str`, `Path`): + - Either the `model_id` (string) of a model hosted on the Hub, e.g. `bigscience/bloom`. + - Or a path to a `directory` containing model weights saved using + [`~transformers.PreTrainedModel.save_pretrained`], e.g., `../path/to/my_model_directory/`. + revision (`str`, *optional*): + Revision of the model on the Hub. Can be a branch name, a git tag or any commit id. + Defaults to the latest commit on `main` branch. + force_download (`bool`, *optional*, defaults to `False`): + Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding + the existing cache. + token (`str` or `bool`, *optional*): + The token to use as HTTP bearer authorization for remote files. By default, it will use the token + cached when running `hf auth login`. + cache_dir (`str`, `Path`, *optional*): + Path to the folder where cached files are stored. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, avoid downloading the file and return the path to the local cached file if it exists. + model_kwargs (`dict`, *optional*): + Additional kwargs to pass to the model during initialization. + """ + model_id = str(pretrained_model_name_or_path) + config_file: str | None = None + if os.path.isdir(model_id): + if constants.CONFIG_NAME in os.listdir(model_id): + config_file = os.path.join(model_id, constants.CONFIG_NAME) + else: + logger.warning(f"{constants.CONFIG_NAME} not found in {Path(model_id).resolve()}") + else: + try: + config_file = hf_hub_download( + repo_id=model_id, + filename=constants.CONFIG_NAME, + revision=revision, + cache_dir=cache_dir, + force_download=force_download, + token=token, + local_files_only=local_files_only, + ) + except HfHubHTTPError as e: + logger.info(f"{constants.CONFIG_NAME} not found on the HuggingFace Hub: {str(e)}") + + # Read config + config = None + if config_file is not None: + with open(config_file, encoding="utf-8") as f: + config = json.load(f) + + # Decode custom types in config + for key, value in config.items(): + if key in cls._hub_mixin_init_parameters: + expected_type = cls._hub_mixin_init_parameters[key].annotation + if expected_type is not inspect.Parameter.empty: + config[key] = cls._decode_arg(expected_type, value) + + # Populate model_kwargs from config + for param in cls._hub_mixin_init_parameters.values(): + if param.name not in model_kwargs and param.name in config: + model_kwargs[param.name] = config[param.name] + + # Check if `config` argument was passed at init + if "config" in cls._hub_mixin_init_parameters and "config" not in model_kwargs: + # Decode `config` argument if it was passed + config_annotation = cls._hub_mixin_init_parameters["config"].annotation + config = cls._decode_arg(config_annotation, config) + + # Forward config to model initialization + model_kwargs["config"] = config + + # Inject config if `**kwargs` are expected + if is_dataclass(cls): + for key in cls.__dataclass_fields__: + if key not in model_kwargs and key in config: + model_kwargs[key] = config[key] + elif any(param.kind == inspect.Parameter.VAR_KEYWORD for param in cls._hub_mixin_init_parameters.values()): + for key, value in config.items(): # type: ignore[union-attr] + if key not in model_kwargs: + model_kwargs[key] = value + + # Finally, also inject if `_from_pretrained` expects it + if cls._hub_mixin_inject_config and "config" not in model_kwargs: + model_kwargs["config"] = config + + instance = cls._from_pretrained( + model_id=str(model_id), + revision=revision, + cache_dir=cache_dir, + force_download=force_download, + local_files_only=local_files_only, + token=token, + **model_kwargs, + ) + + # Implicitly set the config as instance attribute if not already set by the class + # This way `config` will be available when calling `save_pretrained` or `push_to_hub`. + if config is not None and (getattr(instance, "_hub_mixin_config", None) in (None, {})): + instance._hub_mixin_config = config + + return instance + + @classmethod + def _from_pretrained( + cls: type[T], + *, + model_id: str, + revision: str | None, + cache_dir: str | Path | None, + force_download: bool, + local_files_only: bool, + token: str | bool | None, + **model_kwargs, + ) -> T: + """Overwrite this method in subclass to define how to load your model from pretrained. + + Use [`hf_hub_download`] or [`snapshot_download`] to download files from the Hub before loading them. Most + args taken as input can be directly passed to those 2 methods. If needed, you can add more arguments to this + method using "model_kwargs". For example [`PyTorchModelHubMixin._from_pretrained`] takes as input a `map_location` + parameter to set on which device the model should be loaded. + + Check out our [integration guide](../guides/integrations) for more instructions. + + Args: + model_id (`str`): + ID of the model to load from the Huggingface Hub (e.g. `bigscience/bloom`). + revision (`str`, *optional*): + Revision of the model on the Hub. Can be a branch name, a git tag or any commit id. Defaults to the + latest commit on `main` branch. + force_download (`bool`, *optional*, defaults to `False`): + Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding + the existing cache. + token (`str` or `bool`, *optional*): + The token to use as HTTP bearer authorization for remote files. By default, it will use the token + cached when running `hf auth login`. + cache_dir (`str`, `Path`, *optional*): + Path to the folder where cached files are stored. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, avoid downloading the file and return the path to the local cached file if it exists. + model_kwargs: + Additional keyword arguments passed along to the [`~ModelHubMixin._from_pretrained`] method. + """ + raise NotImplementedError + + @validate_hf_hub_args + def push_to_hub( + self, + repo_id: str, + *, + config: dict | DataclassInstance | None = None, + commit_message: str = "Push model using huggingface_hub.", + private: bool | None = None, + token: str | None = None, + branch: str | None = None, + create_pr: bool | None = None, + allow_patterns: list[str] | str | None = None, + ignore_patterns: list[str] | str | None = None, + delete_patterns: list[str] | str | None = None, + model_card_kwargs: dict[str, Any] | None = None, + ) -> str: + """ + Upload model checkpoint to the Hub. + + Use `allow_patterns` and `ignore_patterns` to precisely filter which files should be pushed to the hub. Use + `delete_patterns` to delete existing remote files in the same commit. See [`upload_folder`] reference for more + details. + + Args: + repo_id (`str`): + ID of the repository to push to (example: `"username/my-model"`). + config (`dict` or `DataclassInstance`, *optional*): + Model configuration specified as a key/value dictionary or a dataclass instance. + commit_message (`str`, *optional*): + Message to commit while pushing. + private (`bool`, *optional*): + Whether the repository created should be private. + If `None` (default), the repo will be public unless the organization's default is private. + token (`str`, *optional*): + The token to use as HTTP bearer authorization for remote files. By default, it will use the token + cached when running `hf auth login`. + branch (`str`, *optional*): + The git branch on which to push the model. This defaults to `"main"`. + create_pr (`boolean`, *optional*): + Whether or not to create a Pull Request from `branch` with that commit. Defaults to `False`. + allow_patterns (`list[str]` or `str`, *optional*): + If provided, only files matching at least one pattern are pushed. + ignore_patterns (`list[str]` or `str`, *optional*): + If provided, files matching any of the patterns are not pushed. + delete_patterns (`list[str]` or `str`, *optional*): + If provided, remote files matching any of the patterns will be deleted from the repo. + model_card_kwargs (`dict[str, Any]`, *optional*): + Additional arguments passed to the model card template to customize the model card. + + Returns: + The url of the commit of your model in the given repository. + """ + api = HfApi(token=token) + repo_id = api.create_repo(repo_id=repo_id, private=private, exist_ok=True).repo_id + + # Push the files to the repo in a single commit + with SoftTemporaryDirectory() as tmp: + saved_path = Path(tmp) / repo_id + self.save_pretrained(saved_path, config=config, model_card_kwargs=model_card_kwargs) + return api.upload_folder( + repo_id=repo_id, + repo_type="model", + folder_path=saved_path, + commit_message=commit_message, + revision=branch, + create_pr=create_pr, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + delete_patterns=delete_patterns, + ) + + def generate_model_card(self, *args, **kwargs) -> ModelCard: + card = ModelCard.from_template( + card_data=self._hub_mixin_info.model_card_data, + template_str=self._hub_mixin_info.model_card_template, + repo_url=self._hub_mixin_info.repo_url, + paper_url=self._hub_mixin_info.paper_url, + docs_url=self._hub_mixin_info.docs_url, + **kwargs, + ) + return card + + +class PyTorchModelHubMixin(ModelHubMixin): + """ + Implementation of [`ModelHubMixin`] to provide model Hub upload/download capabilities to PyTorch models. The model + is set in evaluation mode by default using `model.eval()` (dropout modules are deactivated). To train the model, + you should first set it back in training mode with `model.train()`. + + See [`ModelHubMixin`] for more details on how to use the mixin. + + Example: + + ```python + >>> import torch + >>> import torch.nn as nn + >>> from huggingface_hub import PyTorchModelHubMixin + + >>> class MyModel( + ... nn.Module, + ... PyTorchModelHubMixin, + ... library_name="keras-nlp", + ... repo_url="https://github.com/keras-team/keras-nlp", + ... paper_url="https://arxiv.org/abs/2304.12244", + ... docs_url="https://keras.io/keras_nlp/", + ... # ^ optional metadata to generate model card + ... ): + ... def __init__(self, hidden_size: int = 512, vocab_size: int = 30000, output_size: int = 4): + ... super().__init__() + ... self.param = nn.Parameter(torch.rand(hidden_size, vocab_size)) + ... self.linear = nn.Linear(output_size, vocab_size) + + ... def forward(self, x): + ... return self.linear(x + self.param) + >>> model = MyModel(hidden_size=256) + + # Save model weights to local directory + >>> model.save_pretrained("my-awesome-model") + + # Push model weights to the Hub + >>> model.push_to_hub("my-awesome-model") + + # Download and initialize weights from the Hub + >>> model = MyModel.from_pretrained("username/my-awesome-model") + >>> model.hidden_size + 256 + ``` + """ + + def __init_subclass__(cls, *args, tags: list[str] | None = None, **kwargs) -> None: + tags = tags or [] + tags.append("pytorch_model_hub_mixin") + kwargs["tags"] = tags + return super().__init_subclass__(*args, **kwargs) + + def _save_pretrained(self, save_directory: Path) -> None: + """Save weights from a Pytorch model to a local directory.""" + model_to_save = self.module if hasattr(self, "module") else self # type: ignore + save_model_as_safetensor(model_to_save, str(save_directory / constants.SAFETENSORS_SINGLE_FILE)) # type: ignore [arg-type] + + @classmethod + def _from_pretrained( + cls, + *, + model_id: str, + revision: str | None, + cache_dir: str | Path | None, + force_download: bool, + local_files_only: bool, + token: str | bool | None, + map_location: str = "cpu", + strict: bool = False, + **model_kwargs, + ): + """Load Pytorch pretrained weights and return the loaded model.""" + model = cls(**model_kwargs) + if os.path.isdir(model_id): + print("Loading weights from local directory") + model_file = os.path.join(model_id, constants.SAFETENSORS_SINGLE_FILE) + return cls._load_as_safetensor(model, model_file, map_location, strict) + else: + try: + model_file = hf_hub_download( + repo_id=model_id, + filename=constants.SAFETENSORS_SINGLE_FILE, + revision=revision, + cache_dir=cache_dir, + force_download=force_download, + token=token, + local_files_only=local_files_only, + ) + return cls._load_as_safetensor(model, model_file, map_location, strict) + except EntryNotFoundError: + model_file = hf_hub_download( + repo_id=model_id, + filename=constants.PYTORCH_WEIGHTS_NAME, + revision=revision, + cache_dir=cache_dir, + force_download=force_download, + token=token, + local_files_only=local_files_only, + ) + return cls._load_as_pickle(model, model_file, map_location, strict) + + @classmethod + def _load_as_pickle(cls, model: T, model_file: str, map_location: str, strict: bool) -> T: + state_dict = torch.load(model_file, map_location=torch.device(map_location), weights_only=True) + model.load_state_dict(state_dict, strict=strict) # type: ignore + model.eval() # type: ignore + return model + + @classmethod + def _load_as_safetensor(cls, model: T, model_file: str, map_location: str, strict: bool) -> T: + if packaging.version.parse(safetensors.__version__) < packaging.version.parse("0.4.3"): # type: ignore [attr-defined] + load_model_as_safetensor(model, model_file, strict=strict) # type: ignore [arg-type] + if map_location != "cpu": + logger.warning( + "Loading model weights on other devices than 'cpu' is not supported natively in your version of safetensors." + " This means that the model is loaded on 'cpu' first and then copied to the device." + " This leads to a slower loading time." + " Please update safetensors to version 0.4.3 or above for improved performance." + ) + model.to(map_location) # type: ignore [attr-defined] + else: + safetensors.torch.load_model(model, model_file, strict=strict, device=map_location) # type: ignore [arg-type] + model.eval() # type: ignore + return model + + +def _load_dataclass(datacls: type[DataclassInstance], data: dict) -> DataclassInstance: + """Load a dataclass instance from a dictionary. + + Fields not expected by the dataclass are ignored. + """ + return datacls(**{k: v for k, v in data.items() if k in datacls.__dataclass_fields__}) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/__init__.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4c9605db6e6855e4f83d51e36ddbbeb452f353bb Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/__pycache__/_client.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/__pycache__/_client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e19a5932614f28f700dbdac1ccc7692ed40f69b2 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/__pycache__/_client.cpython-312.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a265c2ca66d3eb1820894d4d6010c0d2522557d68682de8d4215b7fe96dda921 +size 153220 diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/__pycache__/_common.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/__pycache__/_common.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8691478dd0f4fa3b1b28cedf13248f56dd46801d Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/__pycache__/_common.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_client.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_client.py new file mode 100644 index 0000000000000000000000000000000000000000..8cec83e2ab45f632d14c5c4d61f9852e3b113a66 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_client.py @@ -0,0 +1,3394 @@ +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Related resources: +# https://huggingface.co/tasks +# https://huggingface.co/docs/huggingface.js/inference/README +# https://github.com/huggingface/huggingface.js/tree/main/packages/inference/src +# https://github.com/huggingface/text-generation-inference/tree/main/clients/python +# https://github.com/huggingface/text-generation-inference/blob/main/clients/python/text_generation/client.py +# https://huggingface.slack.com/archives/C03E4DQ9LAJ/p1680169099087869 +# https://github.com/huggingface/unity-api#tasks +# +# Some TODO: +# - add all tasks +# +# NOTE: the philosophy of this client is "let's make it as easy as possible to use it, even if less optimized". Some +# examples of how it translates: +# - Timeout / Server unavailable is handled by the client in a single "timeout" parameter. +# - Files can be provided as bytes, file paths, or URLs and the client will try to "guess" the type. +# - Images are parsed as PIL.Image for easier manipulation. +# - Provides a "recommended model" for each task => suboptimal but user-wise quicker to get a first script running. +# - Only the main parameters are publicly exposed. Power users can always read the docs for more options. +import base64 +import logging +import os +import re +import warnings +from collections.abc import Iterable +from contextlib import ExitStack +from typing import TYPE_CHECKING, Any, Literal, Optional, Union, overload + +from huggingface_hub import constants +from huggingface_hub.errors import BadRequestError, HfHubHTTPError, InferenceTimeoutError +from huggingface_hub.inference._common import ( + TASKS_EXPECTING_IMAGES, + ContentT, + RequestParameters, + _b64_encode, + _b64_to_image, + _bytes_to_dict, + _bytes_to_image, + _bytes_to_list, + _get_unsupported_text_generation_kwargs, + _import_numpy, + _set_unsupported_text_generation_kwargs, + _stream_chat_completion_response, + _stream_text_generation_response, + raise_text_generation_error, +) +from huggingface_hub.inference._generated.types import ( + AudioClassificationOutputElement, + AudioClassificationOutputTransform, + AudioToAudioOutputElement, + AutomaticSpeechRecognitionOutput, + ChatCompletionInputGrammarType, + ChatCompletionInputMessage, + ChatCompletionInputStreamOptions, + ChatCompletionInputTool, + ChatCompletionInputToolChoiceClass, + ChatCompletionInputToolChoiceEnum, + ChatCompletionOutput, + ChatCompletionStreamOutput, + DocumentQuestionAnsweringOutputElement, + FillMaskOutputElement, + ImageClassificationOutputElement, + ImageClassificationOutputTransform, + ImageSegmentationOutputElement, + ImageSegmentationSubtask, + ImageToImageTargetSize, + ImageToTextOutput, + ImageToVideoTargetSize, + ObjectDetectionOutputElement, + Padding, + QuestionAnsweringOutputElement, + SummarizationOutput, + SummarizationTruncationStrategy, + TableQuestionAnsweringOutputElement, + TextClassificationOutputElement, + TextClassificationOutputTransform, + TextGenerationInputGrammarType, + TextGenerationOutput, + TextGenerationStreamOutput, + TextToSpeechEarlyStoppingEnum, + TokenClassificationAggregationStrategy, + TokenClassificationOutputElement, + TranslationOutput, + TranslationTruncationStrategy, + VisualQuestionAnsweringOutputElement, + ZeroShotClassificationOutputElement, + ZeroShotImageClassificationOutputElement, +) +from huggingface_hub.inference._providers import PROVIDER_OR_POLICY_T, get_provider_helper +from huggingface_hub.utils import ( + build_hf_headers, + get_session, + hf_raise_for_status, + validate_hf_hub_args, +) +from huggingface_hub.utils._auth import get_token + + +if TYPE_CHECKING: + import numpy as np + from PIL.Image import Image + +logger = logging.getLogger(__name__) + + +MODEL_KWARGS_NOT_USED_REGEX = re.compile(r"The following `model_kwargs` are not used by the model: \[(.*?)\]") + + +class InferenceClient: + """ + Initialize a new Inference Client. + + [`InferenceClient`] aims to provide a unified experience to perform inference. The client can be used + seamlessly with either the (free) Inference API, self-hosted Inference Endpoints, or third-party Inference Providers. + + Args: + model (`str`, `optional`): + The model to run inference with. Can be a model id hosted on the Hugging Face Hub, e.g. `meta-llama/Meta-Llama-3-8B-Instruct` + or a URL to a deployed Inference Endpoint. Defaults to None, in which case a recommended model is + automatically selected for the task. + Note: for better compatibility with OpenAI's client, `model` has been aliased as `base_url`. Those 2 + arguments are mutually exclusive. If a URL is passed as `model` or `base_url` for chat completion, the `(/v1)/chat/completions` suffix path will be appended to the URL. + provider (`str`, *optional*): + Name of the provider to use for inference. Can be `"cerebras"`, `"cohere"`, `"deepinfra"`, `"fal-ai"`, `"featherless-ai"`, `"fireworks-ai"`, `"groq"`, `"hf-inference"`, `"novita"`, `"nscale"`, `"openai"`, `"ovhcloud"`, `"publicai"`, `"replicate"`, `"scaleway"`, `"together"`, `"wavespeed"` or `"zai-org"`. + Defaults to "auto": automatic routing, which defaults to "fastest" provider; you can + switch to "cheapest" or "preferred" provider order at https://hf.co/settings/inference-providers. + If model is a URL or `base_url` is passed, then `provider` is not used. + token (`str`, *optional*): + Hugging Face token. Will default to the locally saved token if not provided. + Note: for better compatibility with OpenAI's client, `token` has been aliased as `api_key`. Those 2 + arguments are mutually exclusive and have the exact same behavior. + timeout (`float`, `optional`): + The maximum number of seconds to wait for a response from the server. Defaults to None, meaning it will loop until the server is available. + headers (`dict[str, str]`, `optional`): + Additional headers to send to the server. By default only the authorization and user-agent headers are sent. + Values in this dictionary will override the default values. + bill_to (`str`, `optional`): + The billing account to use for the requests. By default the requests are billed on the user's account. + Requests can only be billed to an organization the user is a member of, and which has subscribed to Enterprise Hub. + cookies (`dict[str, str]`, `optional`): + Additional cookies to send to the server. + base_url (`str`, `optional`): + Base URL to run inference. This is a duplicated argument from `model` to make [`InferenceClient`] + follow the same pattern as `openai.OpenAI` client. Cannot be used if `model` is set. Defaults to None. + api_key (`str`, `optional`): + Token to use for authentication. This is a duplicated argument from `token` to make [`InferenceClient`] + follow the same pattern as `openai.OpenAI` client. Cannot be used if `token` is set. Defaults to None. + """ + + provider: PROVIDER_OR_POLICY_T | None + + @validate_hf_hub_args + def __init__( + self, + model: str | None = None, + *, + provider: PROVIDER_OR_POLICY_T | None = None, + token: str | None = None, + timeout: float | None = None, + headers: dict[str, str] | None = None, + cookies: dict[str, str] | None = None, + bill_to: str | None = None, + # OpenAI compatibility + base_url: str | None = None, + api_key: str | None = None, + ) -> None: + if model is not None and base_url is not None: + raise ValueError( + "Received both `model` and `base_url` arguments. Please provide only one of them." + " `base_url` is an alias for `model` to make the API compatible with OpenAI's client." + " If using `base_url` for chat completion, the `/chat/completions` suffix path will be appended to the base url." + " When passing a URL as `model`, the client will not append any suffix path to it." + ) + if token is not None and api_key is not None: + raise ValueError( + "Received both `token` and `api_key` arguments. Please provide only one of them." + " `api_key` is an alias for `token` to make the API compatible with OpenAI's client." + " It has the exact same behavior as `token`." + ) + token = token if token is not None else api_key + if isinstance(token, bool): + # Legacy behavior: previously it was possible to pass `token=False` to disable authentication. This is not + # supported anymore as authentication is required. Better to explicitly raise here rather than risking + # sending the locally saved token without the user knowing about it. + if token is False: + raise ValueError( + "Cannot use `token=False` to disable authentication as authentication is required to run Inference." + ) + warnings.warn( + "Using `token=True` to automatically use the locally saved token is deprecated and will be removed in a future release. " + "Please use `token=None` instead (default).", + DeprecationWarning, + ) + token = get_token() + + self.model: str | None = base_url or model + self.token: str | None = token + + self.headers = {**headers} if headers is not None else {} + if bill_to is not None: + if ( + constants.HUGGINGFACE_HEADER_X_BILL_TO in self.headers + and self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO] != bill_to + ): + warnings.warn( + f"Overriding existing '{self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO]}' value in headers with '{bill_to}'.", + UserWarning, + ) + self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO] = bill_to + + if token is not None and not token.startswith("hf_"): + warnings.warn( + "You've provided an external provider's API key, so requests will be billed directly by the provider. " + "The `bill_to` parameter is only applicable for Hugging Face billing and will be ignored.", + UserWarning, + ) + + # Configure provider + self.provider = provider # type: ignore[assignment] + + self.cookies = cookies + self.timeout = timeout + + self.exit_stack = ExitStack() + + def __repr__(self): + return f"" + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.exit_stack.close() + + def close(self): + self.exit_stack.close() + + @overload + def _inner_post( # type: ignore[misc] + self, request_parameters: RequestParameters, *, stream: Literal[False] = ... + ) -> bytes: ... + + @overload + def _inner_post( # type: ignore[misc] + self, request_parameters: RequestParameters, *, stream: Literal[True] = ... + ) -> Iterable[str]: ... + + @overload + def _inner_post(self, request_parameters: RequestParameters, *, stream: bool = False) -> bytes | Iterable[str]: ... + + def _inner_post(self, request_parameters: RequestParameters, *, stream: bool = False) -> bytes | Iterable[str]: + """Make a request to the inference server.""" + # TODO: this should be handled in provider helpers directly + if request_parameters.task in TASKS_EXPECTING_IMAGES and "Accept" not in request_parameters.headers: + request_parameters.headers["Accept"] = "image/png" + + try: + response = self.exit_stack.enter_context( + get_session().stream( + "POST", + request_parameters.url, + json=request_parameters.json, + content=request_parameters.data, + headers=request_parameters.headers, + cookies=self.cookies, + timeout=self.timeout, + ) + ) + hf_raise_for_status(response) + if stream: + return response.iter_lines() + else: + return response.read() + except TimeoutError as error: + # Convert any `TimeoutError` to a `InferenceTimeoutError` + raise InferenceTimeoutError(f"Inference call timed out: {request_parameters.url}") from error # type: ignore + except HfHubHTTPError as error: + if error.response.status_code == 422 and request_parameters.task != "unknown": + msg = str(error.args[0]) + if len(error.response.text) > 0: + msg += f"{os.linesep}{error.response.text}{os.linesep}" + error.args = (msg,) + error.args[1:] + raise + + def audio_classification( + self, + audio: ContentT, + *, + model: str | None = None, + top_k: int | None = None, + function_to_apply: Optional["AudioClassificationOutputTransform"] = None, + ) -> list[AudioClassificationOutputElement]: + """ + Perform audio classification on the provided audio content. + + Args: + audio (Union[str, Path, bytes, BinaryIO]): + The audio content to classify. It can be raw audio bytes, a local audio file, or a URL pointing to an + audio file. + model (`str`, *optional*): + The model to use for audio classification. Can be a model ID hosted on the Hugging Face Hub + or a URL to a deployed Inference Endpoint. If not provided, the default recommended model for + audio classification will be used. + top_k (`int`, *optional*): + When specified, limits the output to the top K most probable classes. + function_to_apply (`"AudioClassificationOutputTransform"`, *optional*): + The function to apply to the model outputs in order to retrieve the scores. + + Returns: + `list[AudioClassificationOutputElement]`: List of [`AudioClassificationOutputElement`] items containing the predicted labels and their confidence. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.audio_classification("audio.flac") + [ + AudioClassificationOutputElement(score=0.4976358711719513, label='hap'), + AudioClassificationOutputElement(score=0.3677836060523987, label='neu'), + ... + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="audio-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=audio, + parameters={"function_to_apply": function_to_apply, "top_k": top_k}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return AudioClassificationOutputElement.parse_obj_as_list(response) + + def audio_to_audio( + self, + audio: ContentT, + *, + model: str | None = None, + ) -> list[AudioToAudioOutputElement]: + """ + Performs multiple tasks related to audio-to-audio depending on the model (eg: speech enhancement, source separation). + + Args: + audio (Union[str, Path, bytes, BinaryIO]): + The audio content for the model. It can be raw audio bytes, a local audio file, or a URL pointing to an + audio file. + model (`str`, *optional*): + The model can be any model which takes an audio file and returns another audio file. Can be a model ID hosted on the Hugging Face Hub + or a URL to a deployed Inference Endpoint. If not provided, the default recommended model for + audio_to_audio will be used. + + Returns: + `list[AudioToAudioOutputElement]`: A list of [`AudioToAudioOutputElement`] items containing audios label, content-type, and audio content in blob. + + Raises: + `InferenceTimeoutError`: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> audio_output = client.audio_to_audio("audio.flac") + >>> for i, item in enumerate(audio_output): + >>> with open(f"output_{i}.flac", "wb") as f: + f.write(item.blob) + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="audio-to-audio", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=audio, + parameters={}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + audio_output = AudioToAudioOutputElement.parse_obj_as_list(response) + for item in audio_output: + item.blob = base64.b64decode(item.blob) + return audio_output + + def automatic_speech_recognition( + self, + audio: ContentT, + *, + model: str | None = None, + extra_body: dict | None = None, + ) -> AutomaticSpeechRecognitionOutput: + """ + Perform automatic speech recognition (ASR or audio-to-text) on the given audio content. + + Args: + audio (Union[str, Path, bytes, BinaryIO]): + The content to transcribe. It can be raw audio bytes, local audio file, or a URL to an audio file. + model (`str`, *optional*): + The model to use for ASR. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended model for ASR will be used. + extra_body (`dict`, *optional*): + Additional provider-specific parameters to pass to the model. Refer to the provider's documentation + for supported parameters. + Returns: + [`AutomaticSpeechRecognitionOutput`]: An item containing the transcribed text and optionally the timestamp chunks. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.automatic_speech_recognition("hello_world.flac").text + "hello world" + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="automatic-speech-recognition", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=audio, + parameters={**(extra_body or {})}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_params=request_parameters) + return AutomaticSpeechRecognitionOutput.parse_obj_as_instance(response) + + @overload + def chat_completion( # type: ignore + self, + messages: list[dict | ChatCompletionInputMessage], + *, + model: str | None = None, + stream: Literal[False] = False, + frequency_penalty: float | None = None, + logit_bias: list[float] | None = None, + logprobs: bool | None = None, + max_tokens: int | None = None, + n: int | None = None, + presence_penalty: float | None = None, + response_format: ChatCompletionInputGrammarType | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stream_options: ChatCompletionInputStreamOptions | None = None, + temperature: float | None = None, + tool_choice: Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"] | None = None, + tool_prompt: str | None = None, + tools: list[ChatCompletionInputTool] | None = None, + top_logprobs: int | None = None, + top_p: float | None = None, + extra_body: dict | None = None, + ) -> ChatCompletionOutput: ... + + @overload + def chat_completion( # type: ignore + self, + messages: list[dict | ChatCompletionInputMessage], + *, + model: str | None = None, + stream: Literal[True] = True, + frequency_penalty: float | None = None, + logit_bias: list[float] | None = None, + logprobs: bool | None = None, + max_tokens: int | None = None, + n: int | None = None, + presence_penalty: float | None = None, + response_format: ChatCompletionInputGrammarType | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stream_options: ChatCompletionInputStreamOptions | None = None, + temperature: float | None = None, + tool_choice: Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"] | None = None, + tool_prompt: str | None = None, + tools: list[ChatCompletionInputTool] | None = None, + top_logprobs: int | None = None, + top_p: float | None = None, + extra_body: dict | None = None, + ) -> Iterable[ChatCompletionStreamOutput]: ... + + @overload + def chat_completion( + self, + messages: list[dict | ChatCompletionInputMessage], + *, + model: str | None = None, + stream: bool = False, + frequency_penalty: float | None = None, + logit_bias: list[float] | None = None, + logprobs: bool | None = None, + max_tokens: int | None = None, + n: int | None = None, + presence_penalty: float | None = None, + response_format: ChatCompletionInputGrammarType | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stream_options: ChatCompletionInputStreamOptions | None = None, + temperature: float | None = None, + tool_choice: Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"] | None = None, + tool_prompt: str | None = None, + tools: list[ChatCompletionInputTool] | None = None, + top_logprobs: int | None = None, + top_p: float | None = None, + extra_body: dict | None = None, + ) -> ChatCompletionOutput | Iterable[ChatCompletionStreamOutput]: ... + + def chat_completion( + self, + messages: list[dict | ChatCompletionInputMessage], + *, + model: str | None = None, + stream: bool = False, + # Parameters from ChatCompletionInput (handled manually) + frequency_penalty: float | None = None, + logit_bias: list[float] | None = None, + logprobs: bool | None = None, + max_tokens: int | None = None, + n: int | None = None, + presence_penalty: float | None = None, + response_format: ChatCompletionInputGrammarType | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stream_options: ChatCompletionInputStreamOptions | None = None, + temperature: float | None = None, + tool_choice: Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"] | None = None, + tool_prompt: str | None = None, + tools: list[ChatCompletionInputTool] | None = None, + top_logprobs: int | None = None, + top_p: float | None = None, + extra_body: dict | None = None, + ) -> ChatCompletionOutput | Iterable[ChatCompletionStreamOutput]: + """ + A method for completing conversations using a specified language model. + + > [!TIP] + > The `client.chat_completion` method is aliased as `client.chat.completions.create` for compatibility with OpenAI's client. + > Inputs and outputs are strictly the same and using either syntax will yield the same results. + > Check out the [Inference guide](https://huggingface.co/docs/huggingface_hub/guides/inference#openai-compatibility) + > for more details about OpenAI's compatibility. + + > [!TIP] + > You can pass provider-specific parameters to the model by using the `extra_body` argument. + + Args: + messages (List of [`ChatCompletionInputMessage`]): + Conversation history consisting of roles and content pairs. + model (`str`, *optional*): + The model to use for chat-completion. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended model for chat-based text-generation will be used. + See https://huggingface.co/tasks/text-generation for more details. + If `model` is a model ID, it is passed to the server as the `model` parameter. If you want to define a + custom URL while setting `model` in the request payload, you must set `base_url` when initializing [`InferenceClient`]. + frequency_penalty (`float`, *optional*): + Penalizes new tokens based on their existing frequency + in the text so far. Range: [-2.0, 2.0]. Defaults to 0.0. + logit_bias (`list[float]`, *optional*): + Adjusts the likelihood of specific tokens appearing in the generated output. + logprobs (`bool`, *optional*): + Whether to return log probabilities of the output tokens or not. If true, returns the log + probabilities of each output token returned in the content of message. + max_tokens (`int`, *optional*): + Maximum number of tokens allowed in the response. Defaults to 100. + n (`int`, *optional*): + The number of completions to generate for each prompt. + presence_penalty (`float`, *optional*): + Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the + text so far, increasing the model's likelihood to talk about new topics. + response_format ([`ChatCompletionInputGrammarType`], *optional*): + Grammar constraints. Can be either a JSONSchema or a regex. + seed (Optional[`int`], *optional*): + Seed for reproducible control flow. Defaults to None. + stop (`list[str]`, *optional*): + Up to four strings which trigger the end of the response. + Defaults to None. + stream (`bool`, *optional*): + Enable realtime streaming of responses. Defaults to False. + stream_options ([`ChatCompletionInputStreamOptions`], *optional*): + Options for streaming completions. + temperature (`float`, *optional*): + Controls randomness of the generations. Lower values ensure + less random completions. Range: [0, 2]. Defaults to 1.0. + top_logprobs (`int`, *optional*): + An integer between 0 and 5 specifying the number of most likely tokens to return at each token + position, each with an associated log probability. logprobs must be set to true if this parameter is + used. + top_p (`float`, *optional*): + Fraction of the most likely next words to sample from. + Must be between 0 and 1. Defaults to 1.0. + tool_choice ([`ChatCompletionInputToolChoiceClass`] or [`ChatCompletionInputToolChoiceEnum`], *optional*): + The tool to use for the completion. Defaults to "auto". + tool_prompt (`str`, *optional*): + A prompt to be appended before the tools. + tools (List of [`ChatCompletionInputTool`], *optional*): + A list of tools the model may call. Currently, only functions are supported as a tool. Use this to + provide a list of functions the model may generate JSON inputs for. + extra_body (`dict`, *optional*): + Additional provider-specific parameters to pass to the model. Refer to the provider's documentation + for supported parameters. + Returns: + [`ChatCompletionOutput`] or Iterable of [`ChatCompletionStreamOutput`]: + Generated text returned from the server: + - if `stream=False`, the generated text is returned as a [`ChatCompletionOutput`] (default). + - if `stream=True`, the generated text is returned token by token as a sequence of [`ChatCompletionStreamOutput`]. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + + ```py + >>> from huggingface_hub import InferenceClient + >>> messages = [{"role": "user", "content": "What is the capital of France?"}] + >>> client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") + >>> client.chat_completion(messages, max_tokens=100) + ChatCompletionOutput( + choices=[ + ChatCompletionOutputComplete( + finish_reason='eos_token', + index=0, + message=ChatCompletionOutputMessage( + role='assistant', + content='The capital of France is Paris.', + name=None, + tool_calls=None + ), + logprobs=None + ) + ], + created=1719907176, + id='', + model='meta-llama/Meta-Llama-3-8B-Instruct', + object='text_completion', + system_fingerprint='2.0.4-sha-f426a33', + usage=ChatCompletionOutputUsage( + completion_tokens=8, + prompt_tokens=17, + total_tokens=25 + ) + ) + ``` + + Example using streaming: + ```py + >>> from huggingface_hub import InferenceClient + >>> messages = [{"role": "user", "content": "What is the capital of France?"}] + >>> client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") + >>> for token in client.chat_completion(messages, max_tokens=10, stream=True): + ... print(token) + ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content='The', role='assistant'), index=0, finish_reason=None)], created=1710498504) + ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content=' capital', role='assistant'), index=0, finish_reason=None)], created=1710498504) + (...) + ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content=' may', role='assistant'), index=0, finish_reason=None)], created=1710498504) + ``` + + Example using OpenAI's syntax: + ```py + # instead of `from openai import OpenAI` + from huggingface_hub import InferenceClient + + # instead of `client = OpenAI(...)` + client = InferenceClient( + base_url=..., + api_key=..., + ) + + output = client.chat.completions.create( + model="meta-llama/Meta-Llama-3-8B-Instruct", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Count to 10"}, + ], + stream=True, + max_tokens=1024, + ) + + for chunk in output: + print(chunk.choices[0].delta.content) + ``` + + Example using a third-party provider directly with extra (provider-specific) parameters. Usage will be billed on your Together AI account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="together", # Use Together AI provider + ... api_key="", # Pass your Together API key directly + ... ) + >>> client.chat_completion( + ... model="meta-llama/Meta-Llama-3-8B-Instruct", + ... messages=[{"role": "user", "content": "What is the capital of France?"}], + ... extra_body={"safety_model": "Meta-Llama/Llama-Guard-7b"}, + ... ) + ``` + + Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="novita", # Use Novita provider + ... api_key="hf_...", # Pass your HF token + ... ) + >>> client.chat_completion( + ... model="meta-llama/Meta-Llama-3-8B-Instruct", + ... messages=[{"role": "user", "content": "What is the capital of France?"}], + ... ) + ``` + + Example using Image + Text as input: + ```py + >>> from huggingface_hub import InferenceClient + + # provide a remote URL + >>> image_url ="https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" + # or a base64-encoded image + >>> image_path = "/path/to/image.jpeg" + >>> with open(image_path, "rb") as f: + ... base64_image = base64.b64encode(f.read()).decode("utf-8") + >>> image_url = f"data:image/jpeg;base64,{base64_image}" + + >>> client = InferenceClient("meta-llama/Llama-3.2-11B-Vision-Instruct") + >>> output = client.chat.completions.create( + ... messages=[ + ... { + ... "role": "user", + ... "content": [ + ... { + ... "type": "image_url", + ... "image_url": {"url": image_url}, + ... }, + ... { + ... "type": "text", + ... "text": "Describe this image in one sentence.", + ... }, + ... ], + ... }, + ... ], + ... ) + >>> output + The image depicts the iconic Statue of Liberty situated in New York Harbor, New York, on a clear day. + ``` + + Example using tools: + ```py + >>> client = InferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") + >>> messages = [ + ... { + ... "role": "system", + ... "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous.", + ... }, + ... { + ... "role": "user", + ... "content": "What's the weather like the next 3 days in San Francisco, CA?", + ... }, + ... ] + >>> tools = [ + ... { + ... "type": "function", + ... "function": { + ... "name": "get_current_weather", + ... "description": "Get the current weather", + ... "parameters": { + ... "type": "object", + ... "properties": { + ... "location": { + ... "type": "string", + ... "description": "The city and state, e.g. San Francisco, CA", + ... }, + ... "format": { + ... "type": "string", + ... "enum": ["celsius", "fahrenheit"], + ... "description": "The temperature unit to use. Infer this from the users location.", + ... }, + ... }, + ... "required": ["location", "format"], + ... }, + ... }, + ... }, + ... { + ... "type": "function", + ... "function": { + ... "name": "get_n_day_weather_forecast", + ... "description": "Get an N-day weather forecast", + ... "parameters": { + ... "type": "object", + ... "properties": { + ... "location": { + ... "type": "string", + ... "description": "The city and state, e.g. San Francisco, CA", + ... }, + ... "format": { + ... "type": "string", + ... "enum": ["celsius", "fahrenheit"], + ... "description": "The temperature unit to use. Infer this from the users location.", + ... }, + ... "num_days": { + ... "type": "integer", + ... "description": "The number of days to forecast", + ... }, + ... }, + ... "required": ["location", "format", "num_days"], + ... }, + ... }, + ... }, + ... ] + + >>> response = client.chat_completion( + ... model="meta-llama/Meta-Llama-3-70B-Instruct", + ... messages=messages, + ... tools=tools, + ... tool_choice="auto", + ... max_tokens=500, + ... ) + >>> response.choices[0].message.tool_calls[0].function + ChatCompletionOutputFunctionDefinition( + arguments={ + 'location': 'San Francisco, CA', + 'format': 'fahrenheit', + 'num_days': 3 + }, + name='get_n_day_weather_forecast', + description=None + ) + ``` + + Example using response_format: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") + >>> messages = [ + ... { + ... "role": "user", + ... "content": "I saw a puppy a cat and a raccoon during my bike ride in the park. What did I see and when?", + ... }, + ... ] + >>> response_format = { + ... "type": "json", + ... "value": { + ... "properties": { + ... "location": {"type": "string"}, + ... "activity": {"type": "string"}, + ... "animals_seen": {"type": "integer", "minimum": 1, "maximum": 5}, + ... "animals": {"type": "array", "items": {"type": "string"}}, + ... }, + ... "required": ["location", "activity", "animals_seen", "animals"], + ... }, + ... } + >>> response = client.chat_completion( + ... messages=messages, + ... response_format=response_format, + ... max_tokens=500, + ... ) + >>> response.choices[0].message.content + '{\n\n"activity": "bike ride",\n"animals": ["puppy", "cat", "raccoon"],\n"animals_seen": 3,\n"location": "park"}' + ``` + """ + # Since `chat_completion(..., model=xxx)` is also a payload parameter for the server, we need to handle 'model' differently. + # `self.model` takes precedence over 'model' argument for building URL. + # `model` takes precedence for payload value. + model_id_or_url = self.model or model + payload_model = model or self.model + + # Get the provider helper + provider_helper = get_provider_helper( + self.provider, + task="conversational", + model=model_id_or_url + if model_id_or_url is not None and model_id_or_url.startswith(("http://", "https://")) + else payload_model, + ) + + # Prepare the payload + parameters = { + "model": payload_model, + "frequency_penalty": frequency_penalty, + "logit_bias": logit_bias, + "logprobs": logprobs, + "max_tokens": max_tokens, + "n": n, + "presence_penalty": presence_penalty, + "response_format": response_format, + "seed": seed, + "stop": stop, + "temperature": temperature, + "tool_choice": tool_choice, + "tool_prompt": tool_prompt, + "tools": tools, + "top_logprobs": top_logprobs, + "top_p": top_p, + "stream": stream, + "stream_options": stream_options, + **(extra_body or {}), + } + request_parameters = provider_helper.prepare_request( + inputs=messages, + parameters=parameters, + headers=self.headers, + model=model_id_or_url, + api_key=self.token, + ) + data = self._inner_post(request_parameters, stream=stream) + + if stream: + return _stream_chat_completion_response(data) # type: ignore + + return ChatCompletionOutput.parse_obj_as_instance(data) # type: ignore + + def document_question_answering( + self, + image: ContentT, + question: str, + *, + model: str | None = None, + doc_stride: int | None = None, + handle_impossible_answer: bool | None = None, + lang: str | None = None, + max_answer_len: int | None = None, + max_question_len: int | None = None, + max_seq_len: int | None = None, + top_k: int | None = None, + word_boxes: list[list[float] | str] | None = None, + ) -> list[DocumentQuestionAnsweringOutputElement]: + """ + Answer questions on document images. + + Args: + image (`Union[str, Path, bytes, BinaryIO]`): + The input image for the context. It can be raw bytes, an image file, or a URL to an online image. + question (`str`): + Question to be answered. + model (`str`, *optional*): + The model to use for the document question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended document question answering model will be used. + Defaults to None. + doc_stride (`int`, *optional*): + If the words in the document are too long to fit with the question for the model, it will be split in + several chunks with some overlap. This argument controls the size of that overlap. + handle_impossible_answer (`bool`, *optional*): + Whether to accept impossible as an answer + lang (`str`, *optional*): + Language to use while running OCR. Defaults to english. + max_answer_len (`int`, *optional*): + The maximum length of predicted answers (e.g., only answers with a shorter length are considered). + max_question_len (`int`, *optional*): + The maximum length of the question after tokenization. It will be truncated if needed. + max_seq_len (`int`, *optional*): + The maximum length of the total sentence (context + question) in tokens of each chunk passed to the + model. The context will be split in several chunks (using doc_stride as overlap) if needed. + top_k (`int`, *optional*): + The number of answers to return (will be chosen by order of likelihood). Can return less than top_k + answers if there are not enough options available within the context. + word_boxes (`list[Union[list[float], str`, *optional*): + A list of words and bounding boxes (normalized 0->1000). If provided, the inference will skip the OCR + step and use the provided bounding boxes instead. + Returns: + `list[DocumentQuestionAnsweringOutputElement]`: a list of [`DocumentQuestionAnsweringOutputElement`] items containing the predicted label, associated probability, word ids, and page number. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.document_question_answering(image="https://huggingface.co/spaces/impira/docquery/resolve/2359223c1837a7587402bda0f2643382a6eefeab/invoice.png", question="What is the invoice number?") + [DocumentQuestionAnsweringOutputElement(answer='us-001', end=16, score=0.9999666213989258, start=16)] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="document-question-answering", model=model_id) + inputs: dict[str, Any] = {"question": question, "image": _b64_encode(image)} + request_parameters = provider_helper.prepare_request( + inputs=inputs, + parameters={ + "doc_stride": doc_stride, + "handle_impossible_answer": handle_impossible_answer, + "lang": lang, + "max_answer_len": max_answer_len, + "max_question_len": max_question_len, + "max_seq_len": max_seq_len, + "top_k": top_k, + "word_boxes": word_boxes, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return DocumentQuestionAnsweringOutputElement.parse_obj_as_list(response) + + def feature_extraction( + self, + text: str | list[str], + *, + normalize: bool | None = None, + prompt_name: str | None = None, + truncate: bool | None = None, + truncation_direction: Literal["left", "right"] | None = None, + dimensions: int | None = None, + encoding_format: Literal["float", "base64"] | None = None, + model: str | None = None, + ) -> "np.ndarray": + """ + Generate embeddings for a given text or batch of texts. + + Args: + text (`str` or `list[str]`): + The text or list of texts to embed. + model (`str`, *optional*): + The model to use for the feature extraction task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended feature extraction model will be used. + Defaults to None. + normalize (`bool`, *optional*): + Whether to normalize the embeddings or not. + Only available on server powered by Text-Embedding-Inference. + prompt_name (`str`, *optional*): + The name of the prompt that should be used by for encoding. If not set, no prompt will be applied. + Must be a key in the `Sentence Transformers` configuration `prompts` dictionary. + For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ",...}, + then the sentence "What is the capital of France?" will be encoded as "query: What is the capital of France?" + because the prompt text will be prepended before any text to encode. + truncate (`bool`, *optional*): + Whether to truncate the embeddings or not. + Only available on server powered by Text-Embedding-Inference. + truncation_direction (`Literal["left", "right"]`, *optional*): + Which side of the input should be truncated when `truncate=True` is passed. + dimensions (`int`, *optional*): + The number of dimensions the resulting output embeddings should have. + Only available on OpenAI-compatible embedding endpoints. + encoding_format (`Literal["float", "base64"]`, *optional*): + The format of the output embeddings. Either "float" or "base64". + Only available on OpenAI-compatible embedding endpoints. + + Returns: + `np.ndarray`: The embedding representing the input text(s) as a float32 numpy array. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.feature_extraction("Hi, who are you?") + array([[ 2.424802 , 2.93384 , 1.1750331 , ..., 1.240499, -0.13776633, -0.7889173 ], + [-0.42943227, -0.6364878 , -1.693462 , ..., 0.41978157, -2.4336355 , 0.6162071 ], + ..., + [ 0.28552425, -0.928395 , -1.2077185 , ..., 0.76810825, -2.1069427 , 0.6236161 ]], dtype=float32) + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="feature-extraction", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "normalize": normalize, + "prompt_name": prompt_name, + "truncate": truncate, + "truncation_direction": truncation_direction, + "dimensions": dimensions, + "encoding_format": encoding_format, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + np = _import_numpy() + return np.array(provider_helper.get_response(response), dtype="float32") + + def fill_mask( + self, + text: str, + *, + model: str | None = None, + targets: list[str] | None = None, + top_k: int | None = None, + ) -> list[FillMaskOutputElement]: + """ + Fill in a hole with a missing word (token to be precise). + + Args: + text (`str`): + a string to be filled from, must contain the [MASK] token (check model card for exact name of the mask). + model (`str`, *optional*): + The model to use for the fill mask task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended fill mask model will be used. + targets (`list[str`, *optional*): + When passed, the model will limit the scores to the passed targets instead of looking up in the whole + vocabulary. If the provided targets are not in the model vocab, they will be tokenized and the first + resulting token will be used (with a warning, and that might be slower). + top_k (`int`, *optional*): + When passed, overrides the number of predictions to return. + Returns: + `list[FillMaskOutputElement]`: a list of [`FillMaskOutputElement`] items containing the predicted label, associated + probability, token reference, and completed text. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.fill_mask("The goal of life is .") + [ + FillMaskOutputElement(score=0.06897063553333282, token=11098, token_str=' happiness', sequence='The goal of life is happiness.'), + FillMaskOutputElement(score=0.06554922461509705, token=45075, token_str=' immortality', sequence='The goal of life is immortality.') + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="fill-mask", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={"targets": targets, "top_k": top_k}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return FillMaskOutputElement.parse_obj_as_list(response) + + def image_classification( + self, + image: ContentT, + *, + model: str | None = None, + function_to_apply: Optional["ImageClassificationOutputTransform"] = None, + top_k: int | None = None, + ) -> list[ImageClassificationOutputElement]: + """ + Perform image classification on the given image using the specified model. + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The image to classify. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + model (`str`, *optional*): + The model to use for image classification. Can be a model ID hosted on the Hugging Face Hub or a URL to a + deployed Inference Endpoint. If not provided, the default recommended model for image classification will be used. + function_to_apply (`"ImageClassificationOutputTransform"`, *optional*): + The function to apply to the model outputs in order to retrieve the scores. + top_k (`int`, *optional*): + When specified, limits the output to the top K most probable classes. + Returns: + `list[ImageClassificationOutputElement]`: a list of [`ImageClassificationOutputElement`] items containing the predicted label and associated probability. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.image_classification("https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg") + [ImageClassificationOutputElement(label='Blenheim spaniel', score=0.9779096841812134), ...] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="image-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={"function_to_apply": function_to_apply, "top_k": top_k}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return ImageClassificationOutputElement.parse_obj_as_list(response) + + def image_segmentation( + self, + image: ContentT, + *, + model: str | None = None, + mask_threshold: float | None = None, + overlap_mask_area_threshold: float | None = None, + subtask: Optional["ImageSegmentationSubtask"] = None, + threshold: float | None = None, + ) -> list[ImageSegmentationOutputElement]: + """ + Perform image segmentation on the given image using the specified model. + + > [!WARNING] + > You must have `PIL` installed if you want to work with images (`pip install Pillow`). + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The image to segment. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + model (`str`, *optional*): + The model to use for image segmentation. Can be a model ID hosted on the Hugging Face Hub or a URL to a + deployed Inference Endpoint. If not provided, the default recommended model for image segmentation will be used. + mask_threshold (`float`, *optional*): + Threshold to use when turning the predicted masks into binary values. + overlap_mask_area_threshold (`float`, *optional*): + Mask overlap threshold to eliminate small, disconnected segments. + subtask (`"ImageSegmentationSubtask"`, *optional*): + Segmentation task to be performed, depending on model capabilities. + threshold (`float`, *optional*): + Probability threshold to filter out predicted masks. + Returns: + `list[ImageSegmentationOutputElement]`: A list of [`ImageSegmentationOutputElement`] items containing the segmented masks and associated attributes. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.image_segmentation("cat.jpg") + [ImageSegmentationOutputElement(score=0.989008, label='LABEL_184', mask=), ...] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="image-segmentation", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={ + "mask_threshold": mask_threshold, + "overlap_mask_area_threshold": overlap_mask_area_threshold, + "subtask": subtask, + "threshold": threshold, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_parameters) + output = ImageSegmentationOutputElement.parse_obj_as_list(response) + for item in output: + item.mask = _b64_to_image(item.mask) # type: ignore + return output + + def image_to_image( + self, + image: ContentT, + prompt: str | None = None, + *, + negative_prompt: str | None = None, + num_inference_steps: int | None = None, + guidance_scale: float | None = None, + model: str | None = None, + target_size: ImageToImageTargetSize | None = None, + **kwargs, + ) -> "Image": + """ + Perform image-to-image translation using a specified model. + + > [!WARNING] + > You must have `PIL` installed if you want to work with images (`pip install Pillow`). + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The input image for translation. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + prompt (`str`, *optional*): + The text prompt to guide the image generation. + negative_prompt (`str`, *optional*): + One prompt to guide what NOT to include in image generation. + num_inference_steps (`int`, *optional*): + For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher + quality image at the expense of slower inference. + guidance_scale (`float`, *optional*): + For diffusion models. A higher guidance scale value encourages the model to generate images closely + linked to the text prompt at the expense of lower image quality. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + target_size (`ImageToImageTargetSize`, *optional*): + The size in pixels of the output image. This parameter is only supported by some providers and for + specific models. It will be ignored when unsupported. + + Returns: + `Image`: The translated image. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> image = client.image_to_image("cat.jpg", prompt="turn the cat into a tiger") + >>> image.save("tiger.jpg") + ``` + + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="image-to-image", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={ + "prompt": prompt, + "negative_prompt": negative_prompt, + "target_size": target_size, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + **kwargs, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_parameters) + return _bytes_to_image(response) + + def image_to_video( + self, + image: ContentT, + *, + model: str | None = None, + prompt: str | None = None, + negative_prompt: str | None = None, + num_frames: float | None = None, + num_inference_steps: int | None = None, + guidance_scale: float | None = None, + seed: int | None = None, + target_size: ImageToVideoTargetSize | None = None, + **kwargs, + ) -> bytes: + """ + Generate a video from an input image. + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The input image to generate a video from. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + prompt (`str`, *optional*): + The text prompt to guide the video generation. + negative_prompt (`str`, *optional*): + One prompt to guide what NOT to include in video generation. + num_frames (`float`, *optional*): + The num_frames parameter determines how many video frames are generated. + num_inference_steps (`int`, *optional*): + For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher + quality image at the expense of slower inference. + guidance_scale (`float`, *optional*): + For diffusion models. A higher guidance scale value encourages the model to generate videos closely + linked to the text prompt at the expense of lower image quality. + seed (`int`, *optional*): + The seed to use for the video generation. + target_size (`ImageToVideoTargetSize`, *optional*): + The size in pixel of the output video frames. + num_inference_steps (`int`, *optional*): + The number of denoising steps. More denoising steps usually lead to a higher quality video at the + expense of slower inference. + seed (`int`, *optional*): + Seed for the random number generator. + + Returns: + `bytes`: The generated video. + + Examples: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> video = client.image_to_video("cat.jpg", model="Wan-AI/Wan2.2-I2V-A14B", prompt="turn the cat into a tiger") + >>> with open("tiger.mp4", "wb") as f: + ... f.write(video) + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="image-to-video", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={ + "prompt": prompt, + "negative_prompt": negative_prompt, + "num_frames": num_frames, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + "seed": seed, + "target_size": target_size, + **kwargs, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_parameters) + return response + + def image_to_text(self, image: ContentT, *, model: str | None = None) -> ImageToTextOutput: + """ + Takes an input image and return text. + + Models can have very different outputs depending on your use case (image captioning, optical character recognition + (OCR), Pix2Struct, etc.). Please have a look to the model card to learn more about a model's specificities. + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The input image to caption. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + + Returns: + [`ImageToTextOutput`]: The generated text. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.image_to_text("cat.jpg") + 'a cat standing in a grassy field ' + >>> client.image_to_text("https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg") + 'a dog laying on the grass next to a flower pot ' + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="image-to-text", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + output_list: list[ImageToTextOutput] = ImageToTextOutput.parse_obj_as_list(response) + return output_list[0] + + def object_detection( + self, image: ContentT, *, model: str | None = None, threshold: float | None = None + ) -> list[ObjectDetectionOutputElement]: + """ + Perform object detection on the given image using the specified model. + + > [!WARNING] + > You must have `PIL` installed if you want to work with images (`pip install Pillow`). + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The image to detect objects on. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + model (`str`, *optional*): + The model to use for object detection. Can be a model ID hosted on the Hugging Face Hub or a URL to a + deployed Inference Endpoint. If not provided, the default recommended model for object detection (DETR) will be used. + threshold (`float`, *optional*): + The probability necessary to make a prediction. + Returns: + `list[ObjectDetectionOutputElement]`: A list of [`ObjectDetectionOutputElement`] items containing the bounding boxes and associated attributes. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + `ValueError`: + If the request output is not a List. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.object_detection("people.jpg") + [ObjectDetectionOutputElement(score=0.9486683011054993, label='person', box=ObjectDetectionBoundingBox(xmin=59, ymin=39, xmax=420, ymax=510)), ...] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="object-detection", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={"threshold": threshold}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return ObjectDetectionOutputElement.parse_obj_as_list(response) + + def question_answering( + self, + question: str, + context: str, + *, + model: str | None = None, + align_to_words: bool | None = None, + doc_stride: int | None = None, + handle_impossible_answer: bool | None = None, + max_answer_len: int | None = None, + max_question_len: int | None = None, + max_seq_len: int | None = None, + top_k: int | None = None, + ) -> QuestionAnsweringOutputElement | list[QuestionAnsweringOutputElement]: + """ + Retrieve the answer to a question from a given text. + + Args: + question (`str`): + Question to be answered. + context (`str`): + The context of the question. + model (`str`): + The model to use for the question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. + align_to_words (`bool`, *optional*): + Attempts to align the answer to real words. Improves quality on space separated languages. Might hurt + on non-space-separated languages (like Japanese or Chinese) + doc_stride (`int`, *optional*): + If the context is too long to fit with the question for the model, it will be split in several chunks + with some overlap. This argument controls the size of that overlap. + handle_impossible_answer (`bool`, *optional*): + Whether to accept impossible as an answer. + max_answer_len (`int`, *optional*): + The maximum length of predicted answers (e.g., only answers with a shorter length are considered). + max_question_len (`int`, *optional*): + The maximum length of the question after tokenization. It will be truncated if needed. + max_seq_len (`int`, *optional*): + The maximum length of the total sentence (context + question) in tokens of each chunk passed to the + model. The context will be split in several chunks (using docStride as overlap) if needed. + top_k (`int`, *optional*): + The number of answers to return (will be chosen by order of likelihood). Note that we return less than + topk answers if there are not enough options available within the context. + + Returns: + Union[`QuestionAnsweringOutputElement`, list[`QuestionAnsweringOutputElement`]]: + When top_k is 1 or not provided, it returns a single `QuestionAnsweringOutputElement`. + When top_k is greater than 1, it returns a list of `QuestionAnsweringOutputElement`. + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.question_answering(question="What's my name?", context="My name is Clara and I live in Berkeley.") + QuestionAnsweringOutputElement(answer='Clara', end=16, score=0.9326565265655518, start=11) + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="question-answering", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs={"question": question, "context": context}, + parameters={ + "align_to_words": align_to_words, + "doc_stride": doc_stride, + "handle_impossible_answer": handle_impossible_answer, + "max_answer_len": max_answer_len, + "max_question_len": max_question_len, + "max_seq_len": max_seq_len, + "top_k": top_k, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + # Parse the response as a single `QuestionAnsweringOutputElement` when top_k is 1 or not provided, or a list of `QuestionAnsweringOutputElement` to ensure backward compatibility. + output = QuestionAnsweringOutputElement.parse_obj(response) + return output + + def sentence_similarity( + self, sentence: str, other_sentences: list[str], *, model: str | None = None + ) -> list[float]: + """ + Compute the semantic similarity between a sentence and a list of other sentences by comparing their embeddings. + + Args: + sentence (`str`): + The main sentence to compare to others. + other_sentences (`list[str]`): + The list of sentences to compare to. + model (`str`, *optional*): + The model to use for the sentence similarity task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended sentence similarity model will be used. + Defaults to None. + + Returns: + `list[float]`: The embedding representing the input text. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.sentence_similarity( + ... "Machine learning is so easy.", + ... other_sentences=[ + ... "Deep learning is so straightforward.", + ... "This is so difficult, like rocket science.", + ... "I can't believe how much I struggled with this.", + ... ], + ... ) + [0.7785726189613342, 0.45876261591911316, 0.2906220555305481] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="sentence-similarity", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs={"source_sentence": sentence, "sentences": other_sentences}, + parameters={}, + extra_payload={}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return _bytes_to_list(response) + + def summarization( + self, + text: str, + *, + model: str | None = None, + clean_up_tokenization_spaces: bool | None = None, + generate_parameters: dict[str, Any] | None = None, + truncation: Optional["SummarizationTruncationStrategy"] = None, + ) -> SummarizationOutput: + """ + Generate a summary of a given text using a specified model. + + Args: + text (`str`): + The input text to summarize. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended model for summarization will be used. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether to clean up the potential extra spaces in the text output. + generate_parameters (`dict[str, Any]`, *optional*): + Additional parametrization of the text generation algorithm. + truncation (`"SummarizationTruncationStrategy"`, *optional*): + The truncation strategy to use. + Returns: + [`SummarizationOutput`]: The generated summary text. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.summarization("The Eiffel tower...") + SummarizationOutput(generated_text="The Eiffel tower is one of the most famous landmarks in the world....") + ``` + """ + parameters = { + "clean_up_tokenization_spaces": clean_up_tokenization_spaces, + "generate_parameters": generate_parameters, + "truncation": truncation, + } + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="summarization", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters=parameters, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return SummarizationOutput.parse_obj_as_list(response)[0] + + def table_question_answering( + self, + table: dict[str, Any], + query: str, + *, + model: str | None = None, + padding: Optional["Padding"] = None, + sequential: bool | None = None, + truncation: bool | None = None, + ) -> TableQuestionAnsweringOutputElement: + """ + Retrieve the answer to a question from information given in a table. + + Args: + table (`str`): + A table of data represented as a dict of lists where entries are headers and the lists are all the + values, all lists must have the same size. + query (`str`): + The query in plain text that you want to ask the table. + model (`str`): + The model to use for the table-question-answering task. Can be a model ID hosted on the Hugging Face + Hub or a URL to a deployed Inference Endpoint. + padding (`"Padding"`, *optional*): + Activates and controls padding. + sequential (`bool`, *optional*): + Whether to do inference sequentially or as a batch. Batching is faster, but models like SQA require the + inference to be done sequentially to extract relations within sequences, given their conversational + nature. + truncation (`bool`, *optional*): + Activates and controls truncation. + + Returns: + [`TableQuestionAnsweringOutputElement`]: a table question answering output containing the answer, coordinates, cells and the aggregator used. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> query = "How many stars does the transformers repository have?" + >>> table = {"Repository": ["Transformers", "Datasets", "Tokenizers"], "Stars": ["36542", "4512", "3934"]} + >>> client.table_question_answering(table, query, model="google/tapas-base-finetuned-wtq") + TableQuestionAnsweringOutputElement(answer='36542', coordinates=[[0, 1]], cells=['36542'], aggregator='AVERAGE') + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="table-question-answering", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs={"query": query, "table": table}, + parameters={"model": model, "padding": padding, "sequential": sequential, "truncation": truncation}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return TableQuestionAnsweringOutputElement.parse_obj_as_instance(response) + + def tabular_classification(self, table: dict[str, Any], *, model: str | None = None) -> list[str]: + """ + Classifying a target category (a group) based on a set of attributes. + + Args: + table (`dict[str, Any]`): + Set of attributes to classify. + model (`str`, *optional*): + The model to use for the tabular classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended tabular classification model will be used. + Defaults to None. + + Returns: + `List`: a list of labels, one per row in the initial table. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> table = { + ... "fixed_acidity": ["7.4", "7.8", "10.3"], + ... "volatile_acidity": ["0.7", "0.88", "0.32"], + ... "citric_acid": ["0", "0", "0.45"], + ... "residual_sugar": ["1.9", "2.6", "6.4"], + ... "chlorides": ["0.076", "0.098", "0.073"], + ... "free_sulfur_dioxide": ["11", "25", "5"], + ... "total_sulfur_dioxide": ["34", "67", "13"], + ... "density": ["0.9978", "0.9968", "0.9976"], + ... "pH": ["3.51", "3.2", "3.23"], + ... "sulphates": ["0.56", "0.68", "0.82"], + ... "alcohol": ["9.4", "9.8", "12.6"], + ... } + >>> client.tabular_classification(table=table, model="julien-c/wine-quality") + ["5", "5", "5"] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="tabular-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=None, + extra_payload={"table": table}, + parameters={}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return _bytes_to_list(response) + + def tabular_regression(self, table: dict[str, Any], *, model: str | None = None) -> list[float]: + """ + Predicting a numerical target value given a set of attributes/features in a table. + + Args: + table (`dict[str, Any]`): + Set of attributes stored in a table. The attributes used to predict the target can be both numerical and categorical. + model (`str`, *optional*): + The model to use for the tabular regression task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended tabular regression model will be used. + Defaults to None. + + Returns: + `List`: a list of predicted numerical target values. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> table = { + ... "Height": ["11.52", "12.48", "12.3778"], + ... "Length1": ["23.2", "24", "23.9"], + ... "Length2": ["25.4", "26.3", "26.5"], + ... "Length3": ["30", "31.2", "31.1"], + ... "Species": ["Bream", "Bream", "Bream"], + ... "Width": ["4.02", "4.3056", "4.6961"], + ... } + >>> client.tabular_regression(table, model="scikit-learn/Fish-Weight") + [110, 120, 130] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="tabular-regression", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=None, + parameters={}, + extra_payload={"table": table}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return _bytes_to_list(response) + + def text_classification( + self, + text: str, + *, + model: str | None = None, + top_k: int | None = None, + function_to_apply: Optional["TextClassificationOutputTransform"] = None, + ) -> list[TextClassificationOutputElement]: + """ + Perform text classification (e.g. sentiment-analysis) on the given text. + + Args: + text (`str`): + A string to be classified. + model (`str`, *optional*): + The model to use for the text classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended text classification model will be used. + Defaults to None. + top_k (`int`, *optional*): + When specified, limits the output to the top K most probable classes. + function_to_apply (`"TextClassificationOutputTransform"`, *optional*): + The function to apply to the model outputs in order to retrieve the scores. + + Returns: + `list[TextClassificationOutputElement]`: a list of [`TextClassificationOutputElement`] items containing the predicted label and associated probability. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.text_classification("I like you") + [ + TextClassificationOutputElement(label='POSITIVE', score=0.9998695850372314), + TextClassificationOutputElement(label='NEGATIVE', score=0.0001304351753788069), + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="text-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "function_to_apply": function_to_apply, + "top_k": top_k, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return TextClassificationOutputElement.parse_obj_as_list(response)[0] # type: ignore + + @overload + def text_generation( + self, + prompt: str, + *, + details: Literal[True], + stream: Literal[True], + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> Iterable[TextGenerationStreamOutput]: ... + + @overload + def text_generation( + self, + prompt: str, + *, + details: Literal[True], + stream: Literal[False] | None = None, + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> TextGenerationOutput: ... + + @overload + def text_generation( + self, + prompt: str, + *, + details: Literal[False] | None = None, + stream: Literal[True], + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, # Manual default value + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> Iterable[str]: ... + + @overload + def text_generation( + self, + prompt: str, + *, + details: Literal[False] | None = None, + stream: Literal[False] | None = None, + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> str: ... + + @overload + def text_generation( + self, + prompt: str, + *, + details: bool | None = None, + stream: bool | None = None, + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> str | TextGenerationOutput | Iterable[str] | Iterable[TextGenerationStreamOutput]: ... + + def text_generation( + self, + prompt: str, + *, + details: bool | None = None, + stream: bool | None = None, + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> str | TextGenerationOutput | Iterable[str] | Iterable[TextGenerationStreamOutput]: + """ + Given a prompt, generate the following text. + + > [!TIP] + > If you want to generate a response from chat messages, you should use the [`InferenceClient.chat_completion`] method. + > It accepts a list of messages instead of a single text prompt and handles the chat templating for you. + + Args: + prompt (`str`): + Input text. + details (`bool`, *optional*): + By default, text_generation returns a string. Pass `details=True` if you want a detailed output (tokens, + probabilities, seed, finish reason, etc.). Only available for models running on with the + `text-generation-inference` backend. + stream (`bool`, *optional*): + By default, text_generation returns the full generated text. Pass `stream=True` if you want a stream of + tokens to be returned. Only available for models running on with the `text-generation-inference` + backend. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + adapter_id (`str`, *optional*): + Lora adapter id. + best_of (`int`, *optional*): + Generate best_of sequences and return the one if the highest token logprobs. + decoder_input_details (`bool`, *optional*): + Return the decoder input token logprobs and ids. You must set `details=True` as well for it to be taken + into account. Defaults to `False`. + do_sample (`bool`, *optional*): + Activate logits sampling + frequency_penalty (`float`, *optional*): + Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in + the text so far, decreasing the model's likelihood to repeat the same line verbatim. + grammar ([`TextGenerationInputGrammarType`], *optional*): + Grammar constraints. Can be either a JSONSchema or a regex. + max_new_tokens (`int`, *optional*): + Maximum number of generated tokens. Defaults to 100. + repetition_penalty (`float`, *optional*): + The parameter for repetition penalty. 1.0 means no penalty. See [this + paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + return_full_text (`bool`, *optional*): + Whether to prepend the prompt to the generated text + seed (`int`, *optional*): + Random sampling seed + stop (`list[str]`, *optional*): + Stop generating tokens if a member of `stop` is generated. + stop_sequences (`list[str]`, *optional*): + Deprecated argument. Use `stop` instead. + temperature (`float`, *optional*): + The value used to module the logits distribution. + top_n_tokens (`int`, *optional*): + Return information about the `top_n_tokens` most likely tokens at each generation step, instead of + just the sampled token. + top_k (`int`, *optional`): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + top_p (`float`, *optional`): + If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or + higher are kept for generation. + truncate (`int`, *optional`): + Truncate inputs tokens to the given size. + typical_p (`float`, *optional`): + Typical Decoding mass + See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information + watermark (`bool`, *optional*): + Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) + + Returns: + `Union[str, TextGenerationOutput, Iterable[str], Iterable[TextGenerationStreamOutput]]`: + Generated text returned from the server: + - if `stream=False` and `details=False`, the generated text is returned as a `str` (default) + - if `stream=True` and `details=False`, the generated text is returned token by token as a `Iterable[str]` + - if `stream=False` and `details=True`, the generated text is returned with more details as a [`~huggingface_hub.TextGenerationOutput`] + - if `details=True` and `stream=True`, the generated text is returned token by token as a iterable of [`~huggingface_hub.TextGenerationStreamOutput`] + + Raises: + `ValidationError`: + If input values are not valid. No HTTP call is made to the server. + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + + # Case 1: generate text + >>> client.text_generation("The huggingface_hub library is ", max_new_tokens=12) + '100% open source and built to be easy to use.' + + # Case 2: iterate over the generated tokens. Useful for large generation. + >>> for token in client.text_generation("The huggingface_hub library is ", max_new_tokens=12, stream=True): + ... print(token) + 100 + % + open + source + and + built + to + be + easy + to + use + . + + # Case 3: get more details about the generation process. + >>> client.text_generation("The huggingface_hub library is ", max_new_tokens=12, details=True) + TextGenerationOutput( + generated_text='100% open source and built to be easy to use.', + details=TextGenerationDetails( + finish_reason='length', + generated_tokens=12, + seed=None, + prefill=[ + TextGenerationPrefillOutputToken(id=487, text='The', logprob=None), + TextGenerationPrefillOutputToken(id=53789, text=' hugging', logprob=-13.171875), + (...) + TextGenerationPrefillOutputToken(id=204, text=' ', logprob=-7.0390625) + ], + tokens=[ + TokenElement(id=1425, text='100', logprob=-1.0175781, special=False), + TokenElement(id=16, text='%', logprob=-0.0463562, special=False), + (...) + TokenElement(id=25, text='.', logprob=-0.5703125, special=False) + ], + best_of_sequences=None + ) + ) + + # Case 4: iterate over the generated tokens with more details. + # Last object is more complete, containing the full generated text and the finish reason. + >>> for details in client.text_generation("The huggingface_hub library is ", max_new_tokens=12, details=True, stream=True): + ... print(details) + ... + TextGenerationStreamOutput(token=TokenElement(id=1425, text='100', logprob=-1.0175781, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=16, text='%', logprob=-0.0463562, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=1314, text=' open', logprob=-1.3359375, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=3178, text=' source', logprob=-0.28100586, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=273, text=' and', logprob=-0.5961914, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=3426, text=' built', logprob=-1.9423828, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=271, text=' to', logprob=-1.4121094, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=314, text=' be', logprob=-1.5224609, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=1833, text=' easy', logprob=-2.1132812, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=271, text=' to', logprob=-0.08520508, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=745, text=' use', logprob=-0.39453125, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement( + id=25, + text='.', + logprob=-0.5703125, + special=False), + generated_text='100% open source and built to be easy to use.', + details=TextGenerationStreamOutputStreamDetails(finish_reason='length', generated_tokens=12, seed=None) + ) + + # Case 5: generate constrained output using grammar + >>> response = client.text_generation( + ... prompt="I saw a puppy a cat and a raccoon during my bike ride in the park", + ... model="HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1", + ... max_new_tokens=100, + ... repetition_penalty=1.3, + ... grammar={ + ... "type": "json", + ... "value": { + ... "properties": { + ... "location": {"type": "string"}, + ... "activity": {"type": "string"}, + ... "animals_seen": {"type": "integer", "minimum": 1, "maximum": 5}, + ... "animals": {"type": "array", "items": {"type": "string"}}, + ... }, + ... "required": ["location", "activity", "animals_seen", "animals"], + ... }, + ... }, + ... ) + >>> json.loads(response) + { + "activity": "bike riding", + "animals": ["puppy", "cat", "raccoon"], + "animals_seen": 3, + "location": "park" + } + ``` + """ + if decoder_input_details and not details: + warnings.warn( + "`decoder_input_details=True` has been passed to the server but `details=False` is set meaning that" + " the output from the server will be truncated." + ) + decoder_input_details = False + + if stop_sequences is not None: + warnings.warn( + "`stop_sequences` is a deprecated argument for `text_generation` task" + " and will be removed in version '0.28.0'. Use `stop` instead.", + FutureWarning, + ) + if stop is None: + stop = stop_sequences # use deprecated arg if provided + + # Build payload + parameters = { + "adapter_id": adapter_id, + "best_of": best_of, + "decoder_input_details": decoder_input_details, + "details": details, + "do_sample": do_sample, + "frequency_penalty": frequency_penalty, + "grammar": grammar, + "max_new_tokens": max_new_tokens, + "repetition_penalty": repetition_penalty, + "return_full_text": return_full_text, + "seed": seed, + "stop": stop, + "temperature": temperature, + "top_k": top_k, + "top_n_tokens": top_n_tokens, + "top_p": top_p, + "truncate": truncate, + "typical_p": typical_p, + "watermark": watermark, + } + + # Remove some parameters if not a TGI server + unsupported_kwargs = _get_unsupported_text_generation_kwargs(model) + if len(unsupported_kwargs) > 0: + # The server does not support some parameters + # => means it is not a TGI server + # => remove unsupported parameters and warn the user + + ignored_parameters = [] + for key in unsupported_kwargs: + if parameters.get(key): + ignored_parameters.append(key) + parameters.pop(key, None) + if len(ignored_parameters) > 0: + warnings.warn( + "API endpoint/model for text-generation is not served via TGI. Ignoring following parameters:" + f" {', '.join(ignored_parameters)}.", + UserWarning, + ) + if details: + warnings.warn( + "API endpoint/model for text-generation is not served via TGI. Parameter `details=True` will" + " be ignored meaning only the generated text will be returned.", + UserWarning, + ) + details = False + if stream: + raise ValueError( + "API endpoint/model for text-generation is not served via TGI. Cannot return output as a stream." + " Please pass `stream=False` as input." + ) + + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="text-generation", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=prompt, + parameters=parameters, + extra_payload={"stream": stream}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + + # Handle errors separately for more precise error messages + try: + bytes_output = self._inner_post(request_parameters, stream=stream or False) + except HfHubHTTPError as e: + match = MODEL_KWARGS_NOT_USED_REGEX.search(str(e)) + if isinstance(e, BadRequestError) and match: + unused_params = [kwarg.strip("' ") for kwarg in match.group(1).split(",")] + _set_unsupported_text_generation_kwargs(model, unused_params) + return self.text_generation( # type: ignore + prompt=prompt, + details=details, + stream=stream, + model=model_id, + adapter_id=adapter_id, + best_of=best_of, + decoder_input_details=decoder_input_details, + do_sample=do_sample, + frequency_penalty=frequency_penalty, + grammar=grammar, + max_new_tokens=max_new_tokens, + repetition_penalty=repetition_penalty, + return_full_text=return_full_text, + seed=seed, + stop=stop, + temperature=temperature, + top_k=top_k, + top_n_tokens=top_n_tokens, + top_p=top_p, + truncate=truncate, + typical_p=typical_p, + watermark=watermark, + ) + raise_text_generation_error(e) + + # Parse output + if stream: + return _stream_text_generation_response(bytes_output, details) # type: ignore + + data = _bytes_to_dict(bytes_output) # type: ignore + + # Data can be a single element (dict) or an iterable of dicts where we select the first element of. + if isinstance(data, list): + data = data[0] + response = provider_helper.get_response(data, request_parameters) + return TextGenerationOutput.parse_obj_as_instance(response) if details else response["generated_text"] + + def text_to_image( + self, + prompt: str, + *, + negative_prompt: str | None = None, + height: int | None = None, + width: int | None = None, + num_inference_steps: int | None = None, + guidance_scale: float | None = None, + model: str | None = None, + scheduler: str | None = None, + seed: int | None = None, + extra_body: dict[str, Any] | None = None, + ) -> "Image": + """ + Generate an image based on a given text using a specified model. + + > [!WARNING] + > You must have `PIL` installed if you want to work with images (`pip install Pillow`). + + > [!TIP] + > You can pass provider-specific parameters to the model by using the `extra_body` argument. + + Args: + prompt (`str`): + The prompt to generate an image from. + negative_prompt (`str`, *optional*): + One prompt to guide what NOT to include in image generation. + height (`int`, *optional*): + The height in pixels of the output image + width (`int`, *optional*): + The width in pixels of the output image + num_inference_steps (`int`, *optional*): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*): + A higher guidance scale value encourages the model to generate images closely linked to the text + prompt, but values too high may cause saturation and other artifacts. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended text-to-image model will be used. + Defaults to None. + scheduler (`str`, *optional*): + Override the scheduler with a compatible one. + seed (`int`, *optional*): + Seed for the random number generator. + extra_body (`dict[str, Any]`, *optional*): + Additional provider-specific parameters to pass to the model. Refer to the provider's documentation + for supported parameters. + + Returns: + `Image`: The generated image. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + + >>> image = client.text_to_image("An astronaut riding a horse on the moon.") + >>> image.save("astronaut.png") + + >>> image = client.text_to_image( + ... "An astronaut riding a horse on the moon.", + ... negative_prompt="low resolution, blurry", + ... model="stabilityai/stable-diffusion-2-1", + ... ) + >>> image.save("better_astronaut.png") + ``` + Example using a third-party provider directly. Usage will be billed on your fal.ai account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="fal-ai", # Use fal.ai provider + ... api_key="fal-ai-api-key", # Pass your fal.ai API key + ... ) + >>> image = client.text_to_image( + ... "A majestic lion in a fantasy forest", + ... model="black-forest-labs/FLUX.1-schnell", + ... ) + >>> image.save("lion.png") + ``` + + Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", # Use replicate provider + ... api_key="hf_...", # Pass your HF token + ... ) + >>> image = client.text_to_image( + ... "An astronaut riding a horse on the moon.", + ... model="black-forest-labs/FLUX.1-dev", + ... ) + >>> image.save("astronaut.png") + ``` + + Example using Replicate provider with extra parameters + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", # Use replicate provider + ... api_key="hf_...", # Pass your HF token + ... ) + >>> image = client.text_to_image( + ... "An astronaut riding a horse on the moon.", + ... model="black-forest-labs/FLUX.1-schnell", + ... extra_body={"output_quality": 100}, + ... ) + >>> image.save("astronaut.png") + ``` + + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="text-to-image", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=prompt, + parameters={ + "negative_prompt": negative_prompt, + "height": height, + "width": width, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + "scheduler": scheduler, + "seed": seed, + **(extra_body or {}), + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_parameters) + return _bytes_to_image(response) + + def text_to_video( + self, + prompt: str, + *, + model: str | None = None, + guidance_scale: float | None = None, + negative_prompt: list[str] | None = None, + num_frames: float | None = None, + num_inference_steps: int | None = None, + seed: int | None = None, + extra_body: dict[str, Any] | None = None, + ) -> bytes: + """ + Generate a video based on a given text. + + > [!TIP] + > You can pass provider-specific parameters to the model by using the `extra_body` argument. + + Args: + prompt (`str`): + The prompt to generate a video from. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended text-to-video model will be used. + Defaults to None. + guidance_scale (`float`, *optional*): + A higher guidance scale value encourages the model to generate videos closely linked to the text + prompt, but values too high may cause saturation and other artifacts. + negative_prompt (`list[str]`, *optional*): + One or several prompt to guide what NOT to include in video generation. + num_frames (`float`, *optional*): + The num_frames parameter determines how many video frames are generated. + num_inference_steps (`int`, *optional*): + The number of denoising steps. More denoising steps usually lead to a higher quality video at the + expense of slower inference. + seed (`int`, *optional*): + Seed for the random number generator. + extra_body (`dict[str, Any]`, *optional*): + Additional provider-specific parameters to pass to the model. Refer to the provider's documentation + for supported parameters. + + Returns: + `bytes`: The generated video. + + Example: + + Example using a third-party provider directly. Usage will be billed on your fal.ai account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="fal-ai", # Using fal.ai provider + ... api_key="fal-ai-api-key", # Pass your fal.ai API key + ... ) + >>> video = client.text_to_video( + ... "A majestic lion running in a fantasy forest", + ... model="tencent/HunyuanVideo", + ... ) + >>> with open("lion.mp4", "wb") as file: + ... file.write(video) + ``` + + Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", # Using replicate provider + ... api_key="hf_...", # Pass your HF token + ... ) + >>> video = client.text_to_video( + ... "A cat running in a park", + ... model="genmo/mochi-1-preview", + ... ) + >>> with open("cat.mp4", "wb") as file: + ... file.write(video) + ``` + + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="text-to-video", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=prompt, + parameters={ + "guidance_scale": guidance_scale, + "negative_prompt": negative_prompt, + "num_frames": num_frames, + "num_inference_steps": num_inference_steps, + "seed": seed, + **(extra_body or {}), + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_parameters) + return response + + def text_to_speech( + self, + text: str, + *, + model: str | None = None, + do_sample: bool | None = None, + early_stopping: Union[bool, "TextToSpeechEarlyStoppingEnum"] | None = None, + epsilon_cutoff: float | None = None, + eta_cutoff: float | None = None, + max_length: int | None = None, + max_new_tokens: int | None = None, + min_length: int | None = None, + min_new_tokens: int | None = None, + num_beam_groups: int | None = None, + num_beams: int | None = None, + penalty_alpha: float | None = None, + temperature: float | None = None, + top_k: int | None = None, + top_p: float | None = None, + typical_p: float | None = None, + use_cache: bool | None = None, + extra_body: dict[str, Any] | None = None, + ) -> bytes: + """ + Synthesize an audio of a voice pronouncing a given text. + + > [!TIP] + > You can pass provider-specific parameters to the model by using the `extra_body` argument. + + Args: + text (`str`): + The text to synthesize. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended text-to-speech model will be used. + Defaults to None. + do_sample (`bool`, *optional*): + Whether to use sampling instead of greedy decoding when generating new tokens. + early_stopping (`Union[bool, "TextToSpeechEarlyStoppingEnum"]`, *optional*): + Controls the stopping condition for beam-based methods. + epsilon_cutoff (`float`, *optional*): + If set to float strictly between 0 and 1, only tokens with a conditional probability greater than + epsilon_cutoff will be sampled. In the paper, suggested values range from 3e-4 to 9e-4, depending on + the size of the model. See [Truncation Sampling as Language Model + Desmoothing](https://hf.co/papers/2210.15191) for more details. + eta_cutoff (`float`, *optional*): + Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to float strictly + between 0 and 1, a token is only considered if it is greater than either eta_cutoff or sqrt(eta_cutoff) + * exp(-entropy(softmax(next_token_logits))). The latter term is intuitively the expected next token + probability, scaled by sqrt(eta_cutoff). In the paper, suggested values range from 3e-4 to 2e-3, + depending on the size of the model. See [Truncation Sampling as Language Model + Desmoothing](https://hf.co/papers/2210.15191) for more details. + max_length (`int`, *optional*): + The maximum length (in tokens) of the generated text, including the input. + max_new_tokens (`int`, *optional*): + The maximum number of tokens to generate. Takes precedence over max_length. + min_length (`int`, *optional*): + The minimum length (in tokens) of the generated text, including the input. + min_new_tokens (`int`, *optional*): + The minimum number of tokens to generate. Takes precedence over min_length. + num_beam_groups (`int`, *optional*): + Number of groups to divide num_beams into in order to ensure diversity among different groups of beams. + See [this paper](https://hf.co/papers/1610.02424) for more details. + num_beams (`int`, *optional*): + Number of beams to use for beam search. + penalty_alpha (`float`, *optional*): + The value balances the model confidence and the degeneration penalty in contrastive search decoding. + temperature (`float`, *optional*): + The value used to modulate the next token probabilities. + top_k (`int`, *optional*): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + top_p (`float`, *optional*): + If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to + top_p or higher are kept for generation. + typical_p (`float`, *optional*): + Local typicality measures how similar the conditional probability of predicting a target token next is + to the expected conditional probability of predicting a random token next, given the partial text + already generated. If set to float < 1, the smallest set of the most locally typical tokens with + probabilities that add up to typical_p or higher are kept for generation. See [this + paper](https://hf.co/papers/2202.00666) for more details. + use_cache (`bool`, *optional*): + Whether the model should use the past last key/values attentions to speed up decoding + extra_body (`dict[str, Any]`, *optional*): + Additional provider-specific parameters to pass to the model. Refer to the provider's documentation + for supported parameters. + Returns: + `bytes`: The generated audio. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from pathlib import Path + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + + >>> audio = client.text_to_speech("Hello world") + >>> Path("hello_world.flac").write_bytes(audio) + ``` + + Example using a third-party provider directly. Usage will be billed on your Replicate account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", + ... api_key="your-replicate-api-key", # Pass your Replicate API key directly + ... ) + >>> audio = client.text_to_speech( + ... text="Hello world", + ... model="OuteAI/OuteTTS-0.3-500M", + ... ) + >>> Path("hello_world.flac").write_bytes(audio) + ``` + + Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", + ... api_key="hf_...", # Pass your HF token + ... ) + >>> audio =client.text_to_speech( + ... text="Hello world", + ... model="OuteAI/OuteTTS-0.3-500M", + ... ) + >>> Path("hello_world.flac").write_bytes(audio) + ``` + Example using Replicate provider with extra parameters + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", # Use replicate provider + ... api_key="hf_...", # Pass your HF token + ... ) + >>> audio = client.text_to_speech( + ... "Hello, my name is Kororo, an awesome text-to-speech model.", + ... model="hexgrad/Kokoro-82M", + ... extra_body={"voice": "af_nicole"}, + ... ) + >>> Path("hello.flac").write_bytes(audio) + ``` + + Example music-gen using "YuE-s1-7B-anneal-en-cot" on fal.ai + ```py + >>> from huggingface_hub import InferenceClient + >>> lyrics = ''' + ... [verse] + ... In the town where I was born + ... Lived a man who sailed to sea + ... And he told us of his life + ... In the land of submarines + ... So we sailed on to the sun + ... 'Til we found a sea of green + ... And we lived beneath the waves + ... In our yellow submarine + + ... [chorus] + ... We all live in a yellow submarine + ... Yellow submarine, yellow submarine + ... We all live in a yellow submarine + ... Yellow submarine, yellow submarine + ... ''' + >>> genres = "pavarotti-style tenor voice" + >>> client = InferenceClient( + ... provider="fal-ai", + ... model="m-a-p/YuE-s1-7B-anneal-en-cot", + ... api_key=..., + ... ) + >>> audio = client.text_to_speech(lyrics, extra_body={"genres": genres}) + >>> with open("output.mp3", "wb") as f: + ... f.write(audio) + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="text-to-speech", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "do_sample": do_sample, + "early_stopping": early_stopping, + "epsilon_cutoff": epsilon_cutoff, + "eta_cutoff": eta_cutoff, + "max_length": max_length, + "max_new_tokens": max_new_tokens, + "min_length": min_length, + "min_new_tokens": min_new_tokens, + "num_beam_groups": num_beam_groups, + "num_beams": num_beams, + "penalty_alpha": penalty_alpha, + "temperature": temperature, + "top_k": top_k, + "top_p": top_p, + "typical_p": typical_p, + "use_cache": use_cache, + **(extra_body or {}), + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + response = provider_helper.get_response(response) + return response + + def token_classification( + self, + text: str, + *, + model: str | None = None, + aggregation_strategy: Optional["TokenClassificationAggregationStrategy"] = None, + ignore_labels: list[str] | None = None, + stride: int | None = None, + ) -> list[TokenClassificationOutputElement]: + """ + Perform token classification on the given text. + Usually used for sentence parsing, either grammatical, or Named Entity Recognition (NER) to understand keywords contained within text. + + Args: + text (`str`): + A string to be classified. + model (`str`, *optional*): + The model to use for the token classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended token classification model will be used. + Defaults to None. + aggregation_strategy (`"TokenClassificationAggregationStrategy"`, *optional*): + The strategy used to fuse tokens based on model predictions + ignore_labels (`list[str`, *optional*): + A list of labels to ignore + stride (`int`, *optional*): + The number of overlapping tokens between chunks when splitting the input text. + + Returns: + `list[TokenClassificationOutputElement]`: List of [`TokenClassificationOutputElement`] items containing the entity group, confidence score, word, start and end index. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.token_classification("My name is Sarah Jessica Parker but you can call me Jessica") + [ + TokenClassificationOutputElement( + entity_group='PER', + score=0.9971321225166321, + word='Sarah Jessica Parker', + start=11, + end=31, + ), + TokenClassificationOutputElement( + entity_group='PER', + score=0.9773476123809814, + word='Jessica', + start=52, + end=59, + ) + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="token-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "aggregation_strategy": aggregation_strategy, + "ignore_labels": ignore_labels, + "stride": stride, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return TokenClassificationOutputElement.parse_obj_as_list(response) + + def translation( + self, + text: str, + *, + model: str | None = None, + src_lang: str | None = None, + tgt_lang: str | None = None, + clean_up_tokenization_spaces: bool | None = None, + truncation: Optional["TranslationTruncationStrategy"] = None, + generate_parameters: dict[str, Any] | None = None, + ) -> TranslationOutput: + """ + Convert text from one language to another. + + Check out https://huggingface.co/tasks/translation for more information on how to choose the best model for + your specific use case. Source and target languages usually depend on the model. + However, it is possible to specify source and target languages for certain models. If you are working with one of these models, + you can use `src_lang` and `tgt_lang` arguments to pass the relevant information. + + Args: + text (`str`): + A string to be translated. + model (`str`, *optional*): + The model to use for the translation task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended translation model will be used. + Defaults to None. + src_lang (`str`, *optional*): + The source language of the text. Required for models that can translate from multiple languages. + tgt_lang (`str`, *optional*): + Target language to translate to. Required for models that can translate to multiple languages. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether to clean up the potential extra spaces in the text output. + truncation (`"TranslationTruncationStrategy"`, *optional*): + The truncation strategy to use. + generate_parameters (`dict[str, Any]`, *optional*): + Additional parametrization of the text generation algorithm. + + Returns: + [`TranslationOutput`]: The generated translated text. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + `ValueError`: + If only one of the `src_lang` and `tgt_lang` arguments are provided. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.translation("My name is Wolfgang and I live in Berlin") + 'Mein Name ist Wolfgang und ich lebe in Berlin.' + >>> client.translation("My name is Wolfgang and I live in Berlin", model="Helsinki-NLP/opus-mt-en-fr") + TranslationOutput(translation_text='Je m'appelle Wolfgang et je vis à Berlin.') + ``` + + Specifying languages: + ```py + >>> client.translation("My name is Sarah Jessica Parker but you can call me Jessica", model="facebook/mbart-large-50-many-to-many-mmt", src_lang="en_XX", tgt_lang="fr_XX") + "Mon nom est Sarah Jessica Parker mais vous pouvez m'appeler Jessica" + ``` + """ + # Throw error if only one of `src_lang` and `tgt_lang` was given + if src_lang is not None and tgt_lang is None: + raise ValueError("You cannot specify `src_lang` without specifying `tgt_lang`.") + + if src_lang is None and tgt_lang is not None: + raise ValueError("You cannot specify `tgt_lang` without specifying `src_lang`.") + + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="translation", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "src_lang": src_lang, + "tgt_lang": tgt_lang, + "clean_up_tokenization_spaces": clean_up_tokenization_spaces, + "truncation": truncation, + "generate_parameters": generate_parameters, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return TranslationOutput.parse_obj_as_list(response)[0] + + def visual_question_answering( + self, + image: ContentT, + question: str, + *, + model: str | None = None, + top_k: int | None = None, + ) -> list[VisualQuestionAnsweringOutputElement]: + """ + Answering open-ended questions based on an image. + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The input image for the context. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + question (`str`): + Question to be answered. + model (`str`, *optional*): + The model to use for the visual question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended visual question answering model will be used. + Defaults to None. + top_k (`int`, *optional*): + The number of answers to return (will be chosen by order of likelihood). Note that we return less than + topk answers if there are not enough options available within the context. + Returns: + `list[VisualQuestionAnsweringOutputElement]`: a list of [`VisualQuestionAnsweringOutputElement`] items containing the predicted label and associated probability. + + Raises: + `InferenceTimeoutError`: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.visual_question_answering( + ... image="https://huggingface.co/datasets/mishig/sample_images/resolve/main/tiger.jpg", + ... question="What is the animal doing?" + ... ) + [ + VisualQuestionAnsweringOutputElement(score=0.778609573841095, answer='laying down'), + VisualQuestionAnsweringOutputElement(score=0.6957435607910156, answer='sitting'), + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="visual-question-answering", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={"top_k": top_k}, + headers=self.headers, + model=model_id, + api_key=self.token, + extra_payload={"question": question, "image": _b64_encode(image)}, + ) + response = self._inner_post(request_parameters) + return VisualQuestionAnsweringOutputElement.parse_obj_as_list(response) + + def zero_shot_classification( + self, + text: str, + candidate_labels: list[str], + *, + multi_label: bool | None = False, + hypothesis_template: str | None = None, + model: str | None = None, + ) -> list[ZeroShotClassificationOutputElement]: + """ + Provide as input a text and a set of candidate labels to classify the input text. + + Args: + text (`str`): + The input text to classify. + candidate_labels (`list[str]`): + The set of possible class labels to classify the text into. + labels (`list[str]`, *optional*): + (deprecated) List of strings. Each string is the verbalization of a possible label for the input text. + multi_label (`bool`, *optional*): + Whether multiple candidate labels can be true. If false, the scores are normalized such that the sum of + the label likelihoods for each sequence is 1. If true, the labels are considered independent and + probabilities are normalized for each candidate. + hypothesis_template (`str`, *optional*): + The sentence used in conjunction with `candidate_labels` to attempt the text classification by + replacing the placeholder with the candidate labels. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. If not provided, the default recommended zero-shot classification model will be used. + + + Returns: + `list[ZeroShotClassificationOutputElement]`: List of [`ZeroShotClassificationOutputElement`] items containing the predicted labels and their confidence. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example with `multi_label=False`: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> text = ( + ... "A new model offers an explanation for how the Galilean satellites formed around the solar system's" + ... "largest world. Konstantin Batygin did not set out to solve one of the solar system's most puzzling" + ... " mysteries when he went for a run up a hill in Nice, France." + ... ) + >>> labels = ["space & cosmos", "scientific discovery", "microbiology", "robots", "archeology"] + >>> client.zero_shot_classification(text, labels) + [ + ZeroShotClassificationOutputElement(label='scientific discovery', score=0.7961668968200684), + ZeroShotClassificationOutputElement(label='space & cosmos', score=0.18570658564567566), + ZeroShotClassificationOutputElement(label='microbiology', score=0.00730885099619627), + ZeroShotClassificationOutputElement(label='archeology', score=0.006258360575884581), + ZeroShotClassificationOutputElement(label='robots', score=0.004559356719255447), + ] + >>> client.zero_shot_classification(text, labels, multi_label=True) + [ + ZeroShotClassificationOutputElement(label='scientific discovery', score=0.9829297661781311), + ZeroShotClassificationOutputElement(label='space & cosmos', score=0.755190908908844), + ZeroShotClassificationOutputElement(label='microbiology', score=0.0005462635890580714), + ZeroShotClassificationOutputElement(label='archeology', score=0.00047131875180639327), + ZeroShotClassificationOutputElement(label='robots', score=0.00030448526376858354), + ] + ``` + + Example with `multi_label=True` and a custom `hypothesis_template`: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + >>> client.zero_shot_classification( + ... text="I really like our dinner and I'm very happy. I don't like the weather though.", + ... labels=["positive", "negative", "pessimistic", "optimistic"], + ... multi_label=True, + ... hypothesis_template="This text is {} towards the weather" + ... ) + [ + ZeroShotClassificationOutputElement(label='negative', score=0.9231801629066467), + ZeroShotClassificationOutputElement(label='pessimistic', score=0.8760990500450134), + ZeroShotClassificationOutputElement(label='optimistic', score=0.0008674879791215062), + ZeroShotClassificationOutputElement(label='positive', score=0.0005250611575320363) + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="zero-shot-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "candidate_labels": candidate_labels, + "multi_label": multi_label, + "hypothesis_template": hypothesis_template, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + output = _bytes_to_dict(response) + return ZeroShotClassificationOutputElement.parse_obj_as_list(output) + + def zero_shot_image_classification( + self, + image: ContentT, + candidate_labels: list[str], + *, + model: str | None = None, + hypothesis_template: str | None = None, + # deprecated argument + labels: list[str] = None, # type: ignore + ) -> list[ZeroShotImageClassificationOutputElement]: + """ + Provide input image and text labels to predict text labels for the image. + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The input image to caption. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + candidate_labels (`list[str]`): + The candidate labels for this image + labels (`list[str]`, *optional*): + (deprecated) List of string possible labels. There must be at least 2 labels. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. If not provided, the default recommended zero-shot image classification model will be used. + hypothesis_template (`str`, *optional*): + The sentence used in conjunction with `candidate_labels` to attempt the image classification by + replacing the placeholder with the candidate labels. + + Returns: + `list[ZeroShotImageClassificationOutputElement]`: List of [`ZeroShotImageClassificationOutputElement`] items containing the predicted labels and their confidence. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient() + + >>> client.zero_shot_image_classification( + ... "https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg", + ... labels=["dog", "cat", "horse"], + ... ) + [ZeroShotImageClassificationOutputElement(label='dog', score=0.956),...] + ``` + """ + # Raise ValueError if input is less than 2 labels + if len(candidate_labels) < 2: + raise ValueError("You must specify at least 2 classes to compare.") + + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="zero-shot-image-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={ + "candidate_labels": candidate_labels, + "hypothesis_template": hypothesis_template, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = self._inner_post(request_parameters) + return ZeroShotImageClassificationOutputElement.parse_obj_as_list(response) + + def get_endpoint_info(self, *, model: str | None = None) -> dict[str, Any]: + """ + Get information about the deployed endpoint. + + This endpoint is only available on endpoints powered by Text-Generation-Inference (TGI) or Text-Embedding-Inference (TEI). + Endpoints powered by `transformers` return an empty payload. + + Args: + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + + Returns: + `dict[str, Any]`: Information about the endpoint. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") + >>> client.get_endpoint_info() + { + 'model_id': 'meta-llama/Meta-Llama-3-70B-Instruct', + 'model_sha': None, + 'model_dtype': 'torch.float16', + 'model_device_type': 'cuda', + 'model_pipeline_tag': None, + 'max_concurrent_requests': 128, + 'max_best_of': 2, + 'max_stop_sequences': 4, + 'max_input_length': 8191, + 'max_total_tokens': 8192, + 'waiting_served_ratio': 0.3, + 'max_batch_total_tokens': 1259392, + 'max_waiting_tokens': 20, + 'max_batch_size': None, + 'validation_workers': 32, + 'max_client_batch_size': 4, + 'version': '2.0.2', + 'sha': 'dccab72549635c7eb5ddb17f43f0b7cdff07c214', + 'docker_label': 'sha-dccab72' + } + ``` + """ + if self.provider != "hf-inference": + raise ValueError(f"Getting endpoint info is not supported on '{self.provider}'.") + + model = model or self.model + if model is None: + raise ValueError("Model id not provided.") + if model.startswith(("http://", "https://")): + url = model.rstrip("/") + "/info" + else: + url = f"{constants.INFERENCE_ENDPOINT}/models/{model}/info" + + response = get_session().get(url, headers=build_hf_headers(token=self.token)) + hf_raise_for_status(response) + return response.json() + + def health_check(self, model: str | None = None) -> bool: + """ + Check the health of the deployed endpoint. + + Health check is only available with Inference Endpoints powered by Text-Generation-Inference (TGI) or Text-Embedding-Inference (TEI). + + Args: + model (`str`, *optional*): + URL of the Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + + Returns: + `bool`: True if everything is working fine. + + Example: + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient("https://jzgu0buei5.us-east-1.aws.endpoints.huggingface.cloud") + >>> client.health_check() + True + ``` + """ + if self.provider != "hf-inference": + raise ValueError(f"Health check is not supported on '{self.provider}'.") + + model = model or self.model + if model is None: + raise ValueError("Model id not provided.") + if not model.startswith(("http://", "https://")): + raise ValueError("Model must be an Inference Endpoint URL.") + url = model.rstrip("/") + "/health" + + response = get_session().get(url, headers=build_hf_headers(token=self.token)) + return response.status_code == 200 + + @property + def chat(self) -> "ProxyClientChat": + return ProxyClientChat(self) + + +class _ProxyClient: + """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" + + def __init__(self, client: InferenceClient): + self._client = client + + +class ProxyClientChat(_ProxyClient): + """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" + + @property + def completions(self) -> "ProxyClientChatCompletions": + return ProxyClientChatCompletions(self._client) + + +class ProxyClientChatCompletions(_ProxyClient): + """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" + + @property + def create(self): + return self._client.chat_completion diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_common.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_common.py new file mode 100644 index 0000000000000000000000000000000000000000..1c3bdc60610ba80cf0db1140ab88b83e6b20253f --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_common.py @@ -0,0 +1,432 @@ +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities used by both the sync and async inference clients.""" + +import base64 +import io +import json +import logging +import mimetypes +from collections.abc import AsyncIterable, Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, BinaryIO, Literal, NoReturn, Union, overload + +import httpx + +from huggingface_hub.errors import ( + GenerationError, + HfHubHTTPError, + IncompleteGenerationError, + OverloadedError, + TextGenerationError, + UnknownError, + ValidationError, +) + +from ..utils import get_session, is_numpy_available, is_pillow_available +from ._generated.types import ChatCompletionStreamOutput, TextGenerationStreamOutput + + +if TYPE_CHECKING: + from PIL.Image import Image + +# TYPES +UrlT = str +PathT = Union[str, Path] +ContentT = Union[bytes, BinaryIO, PathT, UrlT, "Image", bytearray, memoryview] + +# Use to set an Accept: image/png header +TASKS_EXPECTING_IMAGES = {"text-to-image", "image-to-image"} + +logger = logging.getLogger(__name__) + + +@dataclass +class RequestParameters: + url: str + task: str + model: str | None + json: str | dict | list | None + data: bytes | None + headers: dict[str, Any] + + +class MimeBytes(bytes): + """ + A bytes object with a mime type. + To be returned by `_prepare_payload_open_as_mime_bytes` in subclasses. + + Example: + ```python + >>> b = MimeBytes(b"hello", "text/plain") + >>> isinstance(b, bytes) + True + >>> b.mime_type + 'text/plain' + ``` + """ + + mime_type: str | None + + def __new__(cls, data: bytes, mime_type: str | None = None): + obj = super().__new__(cls, data) + obj.mime_type = mime_type + if isinstance(data, MimeBytes) and mime_type is None: + obj.mime_type = data.mime_type + return obj + + +## IMPORT UTILS + + +def _import_numpy(): + """Make sure `numpy` is installed on the machine.""" + if not is_numpy_available(): + raise ImportError("Please install numpy to use deal with embeddings (`pip install numpy`).") + import numpy + + return numpy + + +def _import_pil_image(): + """Make sure `PIL` is installed on the machine.""" + if not is_pillow_available(): + raise ImportError( + "Please install Pillow to use deal with images (`pip install Pillow`). If you don't want the image to be" + " post-processed, use `client.post(...)` and get the raw response from the server." + ) + from PIL import Image + + return Image + + +## ENCODING / DECODING UTILS + + +@overload +def _open_as_mime_bytes(content: ContentT) -> MimeBytes: ... # means "if input is not None, output is not None" + + +@overload +def _open_as_mime_bytes(content: Literal[None]) -> Literal[None]: ... # means "if input is None, output is None" + + +def _open_as_mime_bytes(content: ContentT | None) -> MimeBytes | None: + """Open `content` as a binary file, either from a URL, a local path, raw bytes, or a PIL Image. + + Do nothing if `content` is None. + """ + # If content is None, yield None + if content is None: + return None + + # If content is bytes, return it + if isinstance(content, bytes): + return MimeBytes(content) + + # If content is raw binary data (bytearray, memoryview) + if isinstance(content, (bytearray, memoryview)): + return MimeBytes(bytes(content)) + + # If content is a binary file-like object + if hasattr(content, "read"): # duck-typing instead of isinstance(content, BinaryIO) + logger.debug("Reading content from BinaryIO") + data = content.read() + mime_type = mimetypes.guess_type(str(content.name))[0] if hasattr(content, "name") else None + if isinstance(data, str): + raise TypeError("Expected binary stream (bytes), but got text stream") + return MimeBytes(data, mime_type=mime_type) + + # If content is a string => must be either a URL or a path + if isinstance(content, str): + if content.startswith("https://") or content.startswith("http://"): + logger.debug(f"Downloading content from {content}") + response = get_session().get(content) + mime_type = response.headers.get("Content-Type") + if mime_type is None: + mime_type = mimetypes.guess_type(content)[0] + return MimeBytes(response.content, mime_type=mime_type) + + content = Path(content) + if not content.exists(): + raise FileNotFoundError( + f"File not found at {content}. If `data` is a string, it must either be a URL or a path to a local" + " file. To pass raw content, please encode it as bytes first." + ) + + # If content is a Path => open it + if isinstance(content, Path): + logger.debug(f"Opening content from {content}") + return MimeBytes(content.read_bytes(), mime_type=mimetypes.guess_type(content)[0]) + + # If content is a PIL Image => convert to bytes + if is_pillow_available(): + from PIL import Image + + if isinstance(content, Image.Image): + logger.debug("Converting PIL Image to bytes") + buffer = io.BytesIO() + format = content.format or "PNG" + content.save(buffer, format=format) + return MimeBytes(buffer.getvalue(), mime_type=f"image/{format.lower()}") + + # If nothing matched, raise error + raise TypeError( + f"Unsupported content type: {type(content)}. " + "Expected one of: bytes, bytearray, BinaryIO, memoryview, Path, str (URL or file path), or PIL.Image.Image." + ) + + +def _b64_encode(content: ContentT) -> str: + """Encode a raw file (image, audio) into base64. Can be bytes, an opened file, a path or a URL.""" + raw_bytes = _open_as_mime_bytes(content) + return base64.b64encode(raw_bytes).decode() + + +def _as_url(content: ContentT, default_mime_type: str) -> str: + if isinstance(content, str) and content.startswith(("http://", "https://", "data:")): + return content + + # Convert content to bytes + raw_bytes = _open_as_mime_bytes(content) + + # Get MIME type + mime_type = raw_bytes.mime_type or default_mime_type + + # Encode content to base64 + encoded_data = base64.b64encode(raw_bytes).decode() + + # Build data URL + return f"data:{mime_type};base64,{encoded_data}" + + +def _b64_to_image(encoded_image: str) -> "Image": + """Parse a base64-encoded string into a PIL Image.""" + Image = _import_pil_image() + return Image.open(io.BytesIO(base64.b64decode(encoded_image))) + + +def _bytes_to_list(content: bytes) -> list: + """Parse bytes from a Response object into a Python list. + + Expects the response body to be JSON-encoded data. + + NOTE: This is exactly the same implementation as `_bytes_to_dict` and will not complain if the returned data is a + dictionary. The only advantage of having both is to help the user (and mypy) understand what kind of data to expect. + """ + return json.loads(content.decode()) + + +def _bytes_to_dict(content: bytes) -> dict: + """Parse bytes from a Response object into a Python dictionary. + + Expects the response body to be JSON-encoded data. + + NOTE: This is exactly the same implementation as `_bytes_to_list` and will not complain if the returned data is a + list. The only advantage of having both is to help the user (and mypy) understand what kind of data to expect. + """ + return json.loads(content.decode()) + + +def _bytes_to_image(content: bytes) -> "Image": + """Parse bytes from a Response object into a PIL Image. + + Expects the response body to be raw bytes. To deal with b64 encoded images, use `_b64_to_image` instead. + """ + Image = _import_pil_image() + return Image.open(io.BytesIO(content)) + + +def _as_dict(response: bytes | dict) -> dict: + return json.loads(response) if isinstance(response, bytes) else response + + +## STREAMING UTILS + + +def _stream_text_generation_response( + output_lines: Iterable[str], details: bool +) -> Iterable[str] | Iterable[TextGenerationStreamOutput]: + """Used in `InferenceClient.text_generation`.""" + # Parse ServerSentEvents + for line in output_lines: + try: + output = _format_text_generation_stream_output(line, details) + except StopIteration: + break + if output is not None: + yield output + + +async def _async_stream_text_generation_response( + output_lines: AsyncIterable[str], details: bool +) -> AsyncIterable[str] | AsyncIterable[TextGenerationStreamOutput]: + """Used in `AsyncInferenceClient.text_generation`.""" + # Parse ServerSentEvents + async for line in output_lines: + try: + output = _format_text_generation_stream_output(line, details) + except StopIteration: + break + if output is not None: + yield output + + +def _format_text_generation_stream_output(line: str, details: bool) -> str | TextGenerationStreamOutput | None: + if not line.startswith("data:"): + return None # empty line + + if line.strip() == "data: [DONE]": + raise StopIteration("[DONE] signal received.") + + # Decode payload + payload = line.lstrip("data:").rstrip("/n") + json_payload = json.loads(payload) + + # Either an error as being returned + if json_payload.get("error") is not None: + raise _parse_text_generation_error(json_payload["error"], json_payload.get("error_type")) + + # Or parse token payload + output = TextGenerationStreamOutput.parse_obj_as_instance(json_payload) + return output.token.text if not details else output + + +def _stream_chat_completion_response( + lines: Iterable[str], +) -> Iterable[ChatCompletionStreamOutput]: + """Used in `InferenceClient.chat_completion` if model is served with TGI.""" + for line in lines: + try: + output = _format_chat_completion_stream_output(line) + except StopIteration: + break + if output is not None: + yield output + + +async def _async_stream_chat_completion_response( + lines: AsyncIterable[str], +) -> AsyncIterable[ChatCompletionStreamOutput]: + """Used in `AsyncInferenceClient.chat_completion`.""" + async for line in lines: + try: + output = _format_chat_completion_stream_output(line) + except StopIteration: + break + if output is not None: + yield output + + +def _format_chat_completion_stream_output( + line: str, +) -> ChatCompletionStreamOutput | None: + if not line.startswith("data:"): + return None # empty line + + if line.strip() == "data: [DONE]": + raise StopIteration("[DONE] signal received.") + + # Decode payload + json_payload = json.loads(line.lstrip("data:").strip()) + + # Either an error as being returned + if json_payload.get("error") is not None: + raise _parse_text_generation_error(json_payload["error"], json_payload.get("error_type")) + + # Or parse token payload + return ChatCompletionStreamOutput.parse_obj_as_instance(json_payload) + + +async def _async_yield_from(client: httpx.AsyncClient, response: httpx.Response) -> AsyncIterable[str]: + async for line in response.aiter_lines(): + yield line.strip() + + +# "TGI servers" are servers running with the `text-generation-inference` backend. +# This backend is the go-to solution to run large language models at scale. However, +# for some smaller models (e.g. "gpt2") the default `transformers` + `api-inference` +# solution is still in use. +# +# Both approaches have very similar APIs, but not exactly the same. What we do first in +# the `text_generation` method is to assume the model is served via TGI. If we realize +# it's not the case (i.e. we receive an HTTP 400 Bad Request), we fall back to the +# default API with a warning message. When that's the case, We remember the unsupported +# attributes for this model in the `_UNSUPPORTED_TEXT_GENERATION_KWARGS` global variable. +# +# In addition, TGI servers have a built-in API route for chat-completion, which is not +# available on the default API. We use this route to provide a more consistent behavior +# when available. +# +# For more details, see https://github.com/huggingface/text-generation-inference and +# https://huggingface.co/docs/api-inference/detailed_parameters#text-generation-task. + +_UNSUPPORTED_TEXT_GENERATION_KWARGS: dict[str | None, list[str]] = {} + + +def _set_unsupported_text_generation_kwargs(model: str | None, unsupported_kwargs: list[str]) -> None: + _UNSUPPORTED_TEXT_GENERATION_KWARGS.setdefault(model, []).extend(unsupported_kwargs) + + +def _get_unsupported_text_generation_kwargs(model: str | None) -> list[str]: + return _UNSUPPORTED_TEXT_GENERATION_KWARGS.get(model, []) + + +# TEXT GENERATION ERRORS +# ---------------------- +# Text-generation errors are parsed separately to handle as much as possible the errors returned by the text generation +# inference project (https://github.com/huggingface/text-generation-inference). +# ---------------------- + + +def raise_text_generation_error(http_error: HfHubHTTPError) -> NoReturn: + """ + Try to parse text-generation-inference error message and raise HTTPError in any case. + + Args: + error (`HTTPError`): + The HTTPError that have been raised. + """ + # Try to parse a Text Generation Inference error + if http_error.response is None: + raise http_error + + try: + # Hacky way to retrieve payload in case of aiohttp error + payload = getattr(http_error, "response_error_payload", None) or http_error.response.json() + error = payload.get("error") + error_type = payload.get("error_type") + except Exception: # no payload + raise http_error + + # If error_type => more information than `hf_raise_for_status` + if error_type is not None: + exception = _parse_text_generation_error(error, error_type) + raise exception from http_error + + # Otherwise, fallback to default error + raise http_error + + +def _parse_text_generation_error(error: str | None, error_type: str | None) -> TextGenerationError: + if error_type == "generation": + return GenerationError(error) # type: ignore + if error_type == "incomplete_generation": + return IncompleteGenerationError(error) # type: ignore + if error_type == "overloaded": + return OverloadedError(error) # type: ignore + if error_type == "validation": + return ValidationError(error) # type: ignore + return UnknownError(error) # type: ignore diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__init__.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b1da5fbcbf4fe3555d9fb21970d3e1eca777449 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__pycache__/_async_client.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__pycache__/_async_client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e34019e1c3472a07e6faa6c21c9647c8687aa7ee --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__pycache__/_async_client.cpython-312.pyc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0a795257c19dcf0807dd6283b9cdf61f06f68d45267045b1db89bd7447e357d +size 159294 diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/_async_client.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/_async_client.py new file mode 100644 index 0000000000000000000000000000000000000000..5bdcb6746da371feb4bc189bca13cee8105c0423 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/_async_client.py @@ -0,0 +1,3452 @@ +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# WARNING +# This entire file has been adapted from the sync-client code in `src/huggingface_hub/inference/_client.py`. +# Any change in InferenceClient will be automatically reflected in AsyncInferenceClient. +# To re-generate the code, run `make style` or `python ./utils/generate_async_inference_client.py --update`. +# WARNING +import asyncio +import base64 +import logging +import os +import re +import warnings +from contextlib import AsyncExitStack +from typing import TYPE_CHECKING, Any, AsyncIterable, Literal, Optional, Union, overload + +import httpx + +from huggingface_hub import constants +from huggingface_hub.errors import BadRequestError, HfHubHTTPError, InferenceTimeoutError +from huggingface_hub.inference._common import ( + TASKS_EXPECTING_IMAGES, + ContentT, + RequestParameters, + _async_stream_chat_completion_response, + _async_stream_text_generation_response, + _b64_encode, + _b64_to_image, + _bytes_to_dict, + _bytes_to_image, + _bytes_to_list, + _get_unsupported_text_generation_kwargs, + _import_numpy, + _set_unsupported_text_generation_kwargs, + raise_text_generation_error, +) +from huggingface_hub.inference._generated.types import ( + AudioClassificationOutputElement, + AudioClassificationOutputTransform, + AudioToAudioOutputElement, + AutomaticSpeechRecognitionOutput, + ChatCompletionInputGrammarType, + ChatCompletionInputMessage, + ChatCompletionInputStreamOptions, + ChatCompletionInputTool, + ChatCompletionInputToolChoiceClass, + ChatCompletionInputToolChoiceEnum, + ChatCompletionOutput, + ChatCompletionStreamOutput, + DocumentQuestionAnsweringOutputElement, + FillMaskOutputElement, + ImageClassificationOutputElement, + ImageClassificationOutputTransform, + ImageSegmentationOutputElement, + ImageSegmentationSubtask, + ImageToImageTargetSize, + ImageToTextOutput, + ImageToVideoTargetSize, + ObjectDetectionOutputElement, + Padding, + QuestionAnsweringOutputElement, + SummarizationOutput, + SummarizationTruncationStrategy, + TableQuestionAnsweringOutputElement, + TextClassificationOutputElement, + TextClassificationOutputTransform, + TextGenerationInputGrammarType, + TextGenerationOutput, + TextGenerationStreamOutput, + TextToSpeechEarlyStoppingEnum, + TokenClassificationAggregationStrategy, + TokenClassificationOutputElement, + TranslationOutput, + TranslationTruncationStrategy, + VisualQuestionAnsweringOutputElement, + ZeroShotClassificationOutputElement, + ZeroShotImageClassificationOutputElement, +) +from huggingface_hub.inference._providers import PROVIDER_OR_POLICY_T, get_provider_helper +from huggingface_hub.utils import ( + build_hf_headers, + get_async_session, + hf_raise_for_status, + validate_hf_hub_args, +) +from huggingface_hub.utils._auth import get_token + +from .._common import _async_yield_from + + +if TYPE_CHECKING: + import numpy as np + from PIL.Image import Image + +logger = logging.getLogger(__name__) + + +MODEL_KWARGS_NOT_USED_REGEX = re.compile(r"The following `model_kwargs` are not used by the model: \[(.*?)\]") + + +class AsyncInferenceClient: + """ + Initialize a new Inference Client. + + [`InferenceClient`] aims to provide a unified experience to perform inference. The client can be used + seamlessly with either the (free) Inference API, self-hosted Inference Endpoints, or third-party Inference Providers. + + Args: + model (`str`, `optional`): + The model to run inference with. Can be a model id hosted on the Hugging Face Hub, e.g. `meta-llama/Meta-Llama-3-8B-Instruct` + or a URL to a deployed Inference Endpoint. Defaults to None, in which case a recommended model is + automatically selected for the task. + Note: for better compatibility with OpenAI's client, `model` has been aliased as `base_url`. Those 2 + arguments are mutually exclusive. If a URL is passed as `model` or `base_url` for chat completion, the `(/v1)/chat/completions` suffix path will be appended to the URL. + provider (`str`, *optional*): + Name of the provider to use for inference. Can be `"cerebras"`, `"cohere"`, `"deepinfra"`, `"fal-ai"`, `"featherless-ai"`, `"fireworks-ai"`, `"groq"`, `"hf-inference"`, `"novita"`, `"nscale"`, `"openai"`, `"ovhcloud"`, `"publicai"`, `"replicate"`, `"scaleway"`, `"together"`, `"wavespeed"` or `"zai-org"`. + Defaults to "auto": automatic routing, which defaults to "fastest" provider; you can + switch to "cheapest" or "preferred" provider order at https://hf.co/settings/inference-providers. + If model is a URL or `base_url` is passed, then `provider` is not used. + token (`str`, *optional*): + Hugging Face token. Will default to the locally saved token if not provided. + Note: for better compatibility with OpenAI's client, `token` has been aliased as `api_key`. Those 2 + arguments are mutually exclusive and have the exact same behavior. + timeout (`float`, `optional`): + The maximum number of seconds to wait for a response from the server. Defaults to None, meaning it will loop until the server is available. + headers (`dict[str, str]`, `optional`): + Additional headers to send to the server. By default only the authorization and user-agent headers are sent. + Values in this dictionary will override the default values. + bill_to (`str`, `optional`): + The billing account to use for the requests. By default the requests are billed on the user's account. + Requests can only be billed to an organization the user is a member of, and which has subscribed to Enterprise Hub. + cookies (`dict[str, str]`, `optional`): + Additional cookies to send to the server. + base_url (`str`, `optional`): + Base URL to run inference. This is a duplicated argument from `model` to make [`InferenceClient`] + follow the same pattern as `openai.OpenAI` client. Cannot be used if `model` is set. Defaults to None. + api_key (`str`, `optional`): + Token to use for authentication. This is a duplicated argument from `token` to make [`InferenceClient`] + follow the same pattern as `openai.OpenAI` client. Cannot be used if `token` is set. Defaults to None. + """ + + provider: PROVIDER_OR_POLICY_T | None + + @validate_hf_hub_args + def __init__( + self, + model: str | None = None, + *, + provider: PROVIDER_OR_POLICY_T | None = None, + token: str | None = None, + timeout: float | None = None, + headers: dict[str, str] | None = None, + cookies: dict[str, str] | None = None, + bill_to: str | None = None, + # OpenAI compatibility + base_url: str | None = None, + api_key: str | None = None, + ) -> None: + if model is not None and base_url is not None: + raise ValueError( + "Received both `model` and `base_url` arguments. Please provide only one of them." + " `base_url` is an alias for `model` to make the API compatible with OpenAI's client." + " If using `base_url` for chat completion, the `/chat/completions` suffix path will be appended to the base url." + " When passing a URL as `model`, the client will not append any suffix path to it." + ) + if token is not None and api_key is not None: + raise ValueError( + "Received both `token` and `api_key` arguments. Please provide only one of them." + " `api_key` is an alias for `token` to make the API compatible with OpenAI's client." + " It has the exact same behavior as `token`." + ) + token = token if token is not None else api_key + if isinstance(token, bool): + # Legacy behavior: previously it was possible to pass `token=False` to disable authentication. This is not + # supported anymore as authentication is required. Better to explicitly raise here rather than risking + # sending the locally saved token without the user knowing about it. + if token is False: + raise ValueError( + "Cannot use `token=False` to disable authentication as authentication is required to run Inference." + ) + warnings.warn( + "Using `token=True` to automatically use the locally saved token is deprecated and will be removed in a future release. " + "Please use `token=None` instead (default).", + DeprecationWarning, + ) + token = get_token() + + self.model: str | None = base_url or model + self.token: str | None = token + + self.headers = {**headers} if headers is not None else {} + if bill_to is not None: + if ( + constants.HUGGINGFACE_HEADER_X_BILL_TO in self.headers + and self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO] != bill_to + ): + warnings.warn( + f"Overriding existing '{self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO]}' value in headers with '{bill_to}'.", + UserWarning, + ) + self.headers[constants.HUGGINGFACE_HEADER_X_BILL_TO] = bill_to + + if token is not None and not token.startswith("hf_"): + warnings.warn( + "You've provided an external provider's API key, so requests will be billed directly by the provider. " + "The `bill_to` parameter is only applicable for Hugging Face billing and will be ignored.", + UserWarning, + ) + + # Configure provider + self.provider = provider # type: ignore[assignment] + + self.cookies = cookies + self.timeout = timeout + + self.exit_stack = AsyncExitStack() + self._async_client: Optional[httpx.AsyncClient] = None + + def __repr__(self): + return f"" + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + await self.close() + + async def close(self): + """Close the client. + + This method is automatically called when using the client as a context manager. + """ + await self.exit_stack.aclose() + + async def _get_async_client(self): + """Get a unique async client for this AsyncInferenceClient instance. + + Returns the same client instance on subsequent calls, ensuring proper + connection reuse and resource management through the exit stack. + """ + if self._async_client is None: + self._async_client = await self.exit_stack.enter_async_context(get_async_session()) + return self._async_client + + @overload + async def _inner_post( # type: ignore[misc] + self, request_parameters: RequestParameters, *, stream: Literal[False] = ... + ) -> bytes: ... + + @overload + async def _inner_post( # type: ignore[misc] + self, request_parameters: RequestParameters, *, stream: Literal[True] = ... + ) -> AsyncIterable[str]: ... + + @overload + async def _inner_post( + self, request_parameters: RequestParameters, *, stream: bool = False + ) -> bytes | AsyncIterable[str]: ... + + async def _inner_post( + self, request_parameters: RequestParameters, *, stream: bool = False + ) -> bytes | AsyncIterable[str]: + """Make a request to the inference server.""" + + # TODO: this should be handled in provider helpers directly + if request_parameters.task in TASKS_EXPECTING_IMAGES and "Accept" not in request_parameters.headers: + request_parameters.headers["Accept"] = "image/png" + + try: + client = await self._get_async_client() + if stream: + response = await self.exit_stack.enter_async_context( + client.stream( + "POST", + request_parameters.url, + json=request_parameters.json, + data=request_parameters.data, + headers=request_parameters.headers, + cookies=self.cookies, + timeout=self.timeout, + ) + ) + hf_raise_for_status(response) + return _async_yield_from(client, response) + else: + response = await client.post( + request_parameters.url, + json=request_parameters.json, + data=request_parameters.data, + headers=request_parameters.headers, + cookies=self.cookies, + timeout=self.timeout, + ) + hf_raise_for_status(response) + return response.content + except asyncio.TimeoutError as error: + # Convert any `TimeoutError` to a `InferenceTimeoutError` + raise InferenceTimeoutError(f"Inference call timed out: {request_parameters.url}") from error # type: ignore + except HfHubHTTPError as error: + if error.response.status_code == 422 and request_parameters.task != "unknown": + msg = str(error.args[0]) + if len(error.response.text) > 0: + msg += f"{os.linesep}{error.response.text}{os.linesep}" + error.args = (msg,) + error.args[1:] + raise + + async def audio_classification( + self, + audio: ContentT, + *, + model: str | None = None, + top_k: int | None = None, + function_to_apply: Optional["AudioClassificationOutputTransform"] = None, + ) -> list[AudioClassificationOutputElement]: + """ + Perform audio classification on the provided audio content. + + Args: + audio (Union[str, Path, bytes, BinaryIO]): + The audio content to classify. It can be raw audio bytes, a local audio file, or a URL pointing to an + audio file. + model (`str`, *optional*): + The model to use for audio classification. Can be a model ID hosted on the Hugging Face Hub + or a URL to a deployed Inference Endpoint. If not provided, the default recommended model for + audio classification will be used. + top_k (`int`, *optional*): + When specified, limits the output to the top K most probable classes. + function_to_apply (`"AudioClassificationOutputTransform"`, *optional*): + The function to apply to the model outputs in order to retrieve the scores. + + Returns: + `list[AudioClassificationOutputElement]`: List of [`AudioClassificationOutputElement`] items containing the predicted labels and their confidence. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.audio_classification("audio.flac") + [ + AudioClassificationOutputElement(score=0.4976358711719513, label='hap'), + AudioClassificationOutputElement(score=0.3677836060523987, label='neu'), + ... + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="audio-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=audio, + parameters={"function_to_apply": function_to_apply, "top_k": top_k}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return AudioClassificationOutputElement.parse_obj_as_list(response) + + async def audio_to_audio( + self, + audio: ContentT, + *, + model: str | None = None, + ) -> list[AudioToAudioOutputElement]: + """ + Performs multiple tasks related to audio-to-audio depending on the model (eg: speech enhancement, source separation). + + Args: + audio (Union[str, Path, bytes, BinaryIO]): + The audio content for the model. It can be raw audio bytes, a local audio file, or a URL pointing to an + audio file. + model (`str`, *optional*): + The model can be any model which takes an audio file and returns another audio file. Can be a model ID hosted on the Hugging Face Hub + or a URL to a deployed Inference Endpoint. If not provided, the default recommended model for + audio_to_audio will be used. + + Returns: + `list[AudioToAudioOutputElement]`: A list of [`AudioToAudioOutputElement`] items containing audios label, content-type, and audio content in blob. + + Raises: + `InferenceTimeoutError`: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> audio_output = await client.audio_to_audio("audio.flac") + >>> async for i, item in enumerate(audio_output): + >>> with open(f"output_{i}.flac", "wb") as f: + f.write(item.blob) + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="audio-to-audio", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=audio, + parameters={}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + audio_output = AudioToAudioOutputElement.parse_obj_as_list(response) + for item in audio_output: + item.blob = base64.b64decode(item.blob) + return audio_output + + async def automatic_speech_recognition( + self, + audio: ContentT, + *, + model: str | None = None, + extra_body: dict | None = None, + ) -> AutomaticSpeechRecognitionOutput: + """ + Perform automatic speech recognition (ASR or audio-to-text) on the given audio content. + + Args: + audio (Union[str, Path, bytes, BinaryIO]): + The content to transcribe. It can be raw audio bytes, local audio file, or a URL to an audio file. + model (`str`, *optional*): + The model to use for ASR. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended model for ASR will be used. + extra_body (`dict`, *optional*): + Additional provider-specific parameters to pass to the model. Refer to the provider's documentation + for supported parameters. + Returns: + [`AutomaticSpeechRecognitionOutput`]: An item containing the transcribed text and optionally the timestamp chunks. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.automatic_speech_recognition("hello_world.flac").text + "hello world" + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="automatic-speech-recognition", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=audio, + parameters={**(extra_body or {})}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_params=request_parameters) + return AutomaticSpeechRecognitionOutput.parse_obj_as_instance(response) + + @overload + async def chat_completion( # type: ignore + self, + messages: list[dict | ChatCompletionInputMessage], + *, + model: str | None = None, + stream: Literal[False] = False, + frequency_penalty: float | None = None, + logit_bias: list[float] | None = None, + logprobs: bool | None = None, + max_tokens: int | None = None, + n: int | None = None, + presence_penalty: float | None = None, + response_format: ChatCompletionInputGrammarType | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stream_options: ChatCompletionInputStreamOptions | None = None, + temperature: float | None = None, + tool_choice: Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"] | None = None, + tool_prompt: str | None = None, + tools: list[ChatCompletionInputTool] | None = None, + top_logprobs: int | None = None, + top_p: float | None = None, + extra_body: dict | None = None, + ) -> ChatCompletionOutput: ... + + @overload + async def chat_completion( # type: ignore + self, + messages: list[dict | ChatCompletionInputMessage], + *, + model: str | None = None, + stream: Literal[True] = True, + frequency_penalty: float | None = None, + logit_bias: list[float] | None = None, + logprobs: bool | None = None, + max_tokens: int | None = None, + n: int | None = None, + presence_penalty: float | None = None, + response_format: ChatCompletionInputGrammarType | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stream_options: ChatCompletionInputStreamOptions | None = None, + temperature: float | None = None, + tool_choice: Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"] | None = None, + tool_prompt: str | None = None, + tools: list[ChatCompletionInputTool] | None = None, + top_logprobs: int | None = None, + top_p: float | None = None, + extra_body: dict | None = None, + ) -> AsyncIterable[ChatCompletionStreamOutput]: ... + + @overload + async def chat_completion( + self, + messages: list[dict | ChatCompletionInputMessage], + *, + model: str | None = None, + stream: bool = False, + frequency_penalty: float | None = None, + logit_bias: list[float] | None = None, + logprobs: bool | None = None, + max_tokens: int | None = None, + n: int | None = None, + presence_penalty: float | None = None, + response_format: ChatCompletionInputGrammarType | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stream_options: ChatCompletionInputStreamOptions | None = None, + temperature: float | None = None, + tool_choice: Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"] | None = None, + tool_prompt: str | None = None, + tools: list[ChatCompletionInputTool] | None = None, + top_logprobs: int | None = None, + top_p: float | None = None, + extra_body: dict | None = None, + ) -> ChatCompletionOutput | AsyncIterable[ChatCompletionStreamOutput]: ... + + async def chat_completion( + self, + messages: list[dict | ChatCompletionInputMessage], + *, + model: str | None = None, + stream: bool = False, + # Parameters from ChatCompletionInput (handled manually) + frequency_penalty: float | None = None, + logit_bias: list[float] | None = None, + logprobs: bool | None = None, + max_tokens: int | None = None, + n: int | None = None, + presence_penalty: float | None = None, + response_format: ChatCompletionInputGrammarType | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stream_options: ChatCompletionInputStreamOptions | None = None, + temperature: float | None = None, + tool_choice: Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"] | None = None, + tool_prompt: str | None = None, + tools: list[ChatCompletionInputTool] | None = None, + top_logprobs: int | None = None, + top_p: float | None = None, + extra_body: dict | None = None, + ) -> ChatCompletionOutput | AsyncIterable[ChatCompletionStreamOutput]: + """ + A method for completing conversations using a specified language model. + + > [!TIP] + > The `client.chat_completion` method is aliased as `client.chat.completions.create` for compatibility with OpenAI's client. + > Inputs and outputs are strictly the same and using either syntax will yield the same results. + > Check out the [Inference guide](https://huggingface.co/docs/huggingface_hub/guides/inference#openai-compatibility) + > for more details about OpenAI's compatibility. + + > [!TIP] + > You can pass provider-specific parameters to the model by using the `extra_body` argument. + + Args: + messages (List of [`ChatCompletionInputMessage`]): + Conversation history consisting of roles and content pairs. + model (`str`, *optional*): + The model to use for chat-completion. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended model for chat-based text-generation will be used. + See https://huggingface.co/tasks/text-generation for more details. + If `model` is a model ID, it is passed to the server as the `model` parameter. If you want to define a + custom URL while setting `model` in the request payload, you must set `base_url` when initializing [`InferenceClient`]. + frequency_penalty (`float`, *optional*): + Penalizes new tokens based on their existing frequency + in the text so far. Range: [-2.0, 2.0]. Defaults to 0.0. + logit_bias (`list[float]`, *optional*): + Adjusts the likelihood of specific tokens appearing in the generated output. + logprobs (`bool`, *optional*): + Whether to return log probabilities of the output tokens or not. If true, returns the log + probabilities of each output token returned in the content of message. + max_tokens (`int`, *optional*): + Maximum number of tokens allowed in the response. Defaults to 100. + n (`int`, *optional*): + The number of completions to generate for each prompt. + presence_penalty (`float`, *optional*): + Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the + text so far, increasing the model's likelihood to talk about new topics. + response_format ([`ChatCompletionInputGrammarType`], *optional*): + Grammar constraints. Can be either a JSONSchema or a regex. + seed (Optional[`int`], *optional*): + Seed for reproducible control flow. Defaults to None. + stop (`list[str]`, *optional*): + Up to four strings which trigger the end of the response. + Defaults to None. + stream (`bool`, *optional*): + Enable realtime streaming of responses. Defaults to False. + stream_options ([`ChatCompletionInputStreamOptions`], *optional*): + Options for streaming completions. + temperature (`float`, *optional*): + Controls randomness of the generations. Lower values ensure + less random completions. Range: [0, 2]. Defaults to 1.0. + top_logprobs (`int`, *optional*): + An integer between 0 and 5 specifying the number of most likely tokens to return at each token + position, each with an associated log probability. logprobs must be set to true if this parameter is + used. + top_p (`float`, *optional*): + Fraction of the most likely next words to sample from. + Must be between 0 and 1. Defaults to 1.0. + tool_choice ([`ChatCompletionInputToolChoiceClass`] or [`ChatCompletionInputToolChoiceEnum`], *optional*): + The tool to use for the completion. Defaults to "auto". + tool_prompt (`str`, *optional*): + A prompt to be appended before the tools. + tools (List of [`ChatCompletionInputTool`], *optional*): + A list of tools the model may call. Currently, only functions are supported as a tool. Use this to + provide a list of functions the model may generate JSON inputs for. + extra_body (`dict`, *optional*): + Additional provider-specific parameters to pass to the model. Refer to the provider's documentation + for supported parameters. + Returns: + [`ChatCompletionOutput`] or Iterable of [`ChatCompletionStreamOutput`]: + Generated text returned from the server: + - if `stream=False`, the generated text is returned as a [`ChatCompletionOutput`] (default). + - if `stream=True`, the generated text is returned token by token as a sequence of [`ChatCompletionStreamOutput`]. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> messages = [{"role": "user", "content": "What is the capital of France?"}] + >>> client = AsyncInferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") + >>> await client.chat_completion(messages, max_tokens=100) + ChatCompletionOutput( + choices=[ + ChatCompletionOutputComplete( + finish_reason='eos_token', + index=0, + message=ChatCompletionOutputMessage( + role='assistant', + content='The capital of France is Paris.', + name=None, + tool_calls=None + ), + logprobs=None + ) + ], + created=1719907176, + id='', + model='meta-llama/Meta-Llama-3-8B-Instruct', + object='text_completion', + system_fingerprint='2.0.4-sha-f426a33', + usage=ChatCompletionOutputUsage( + completion_tokens=8, + prompt_tokens=17, + total_tokens=25 + ) + ) + ``` + + Example using streaming: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> messages = [{"role": "user", "content": "What is the capital of France?"}] + >>> client = AsyncInferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") + >>> async for token in await client.chat_completion(messages, max_tokens=10, stream=True): + ... print(token) + ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content='The', role='assistant'), index=0, finish_reason=None)], created=1710498504) + ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content=' capital', role='assistant'), index=0, finish_reason=None)], created=1710498504) + (...) + ChatCompletionStreamOutput(choices=[ChatCompletionStreamOutputChoice(delta=ChatCompletionStreamOutputDelta(content=' may', role='assistant'), index=0, finish_reason=None)], created=1710498504) + ``` + + Example using OpenAI's syntax: + ```py + # Must be run in an async context + # instead of `from openai import OpenAI` + from huggingface_hub import AsyncInferenceClient + + # instead of `client = OpenAI(...)` + client = AsyncInferenceClient( + base_url=..., + api_key=..., + ) + + output = await client.chat.completions.create( + model="meta-llama/Meta-Llama-3-8B-Instruct", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Count to 10"}, + ], + stream=True, + max_tokens=1024, + ) + + for chunk in output: + print(chunk.choices[0].delta.content) + ``` + + Example using a third-party provider directly with extra (provider-specific) parameters. Usage will be billed on your Together AI account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="together", # Use Together AI provider + ... api_key="", # Pass your Together API key directly + ... ) + >>> client.chat_completion( + ... model="meta-llama/Meta-Llama-3-8B-Instruct", + ... messages=[{"role": "user", "content": "What is the capital of France?"}], + ... extra_body={"safety_model": "Meta-Llama/Llama-Guard-7b"}, + ... ) + ``` + + Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="novita", # Use Novita provider + ... api_key="hf_...", # Pass your HF token + ... ) + >>> client.chat_completion( + ... model="meta-llama/Meta-Llama-3-8B-Instruct", + ... messages=[{"role": "user", "content": "What is the capital of France?"}], + ... ) + ``` + + Example using Image + Text as input: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + + # provide a remote URL + >>> image_url ="https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" + # or a base64-encoded image + >>> image_path = "/path/to/image.jpeg" + >>> with open(image_path, "rb") as f: + ... base64_image = base64.b64encode(f.read()).decode("utf-8") + >>> image_url = f"data:image/jpeg;base64,{base64_image}" + + >>> client = AsyncInferenceClient("meta-llama/Llama-3.2-11B-Vision-Instruct") + >>> output = await client.chat.completions.create( + ... messages=[ + ... { + ... "role": "user", + ... "content": [ + ... { + ... "type": "image_url", + ... "image_url": {"url": image_url}, + ... }, + ... { + ... "type": "text", + ... "text": "Describe this image in one sentence.", + ... }, + ... ], + ... }, + ... ], + ... ) + >>> output + The image depicts the iconic Statue of Liberty situated in New York Harbor, New York, on a clear day. + ``` + + Example using tools: + ```py + # Must be run in an async context + >>> client = AsyncInferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") + >>> messages = [ + ... { + ... "role": "system", + ... "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous.", + ... }, + ... { + ... "role": "user", + ... "content": "What's the weather like the next 3 days in San Francisco, CA?", + ... }, + ... ] + >>> tools = [ + ... { + ... "type": "function", + ... "function": { + ... "name": "get_current_weather", + ... "description": "Get the current weather", + ... "parameters": { + ... "type": "object", + ... "properties": { + ... "location": { + ... "type": "string", + ... "description": "The city and state, e.g. San Francisco, CA", + ... }, + ... "format": { + ... "type": "string", + ... "enum": ["celsius", "fahrenheit"], + ... "description": "The temperature unit to use. Infer this from the users location.", + ... }, + ... }, + ... "required": ["location", "format"], + ... }, + ... }, + ... }, + ... { + ... "type": "function", + ... "function": { + ... "name": "get_n_day_weather_forecast", + ... "description": "Get an N-day weather forecast", + ... "parameters": { + ... "type": "object", + ... "properties": { + ... "location": { + ... "type": "string", + ... "description": "The city and state, e.g. San Francisco, CA", + ... }, + ... "format": { + ... "type": "string", + ... "enum": ["celsius", "fahrenheit"], + ... "description": "The temperature unit to use. Infer this from the users location.", + ... }, + ... "num_days": { + ... "type": "integer", + ... "description": "The number of days to forecast", + ... }, + ... }, + ... "required": ["location", "format", "num_days"], + ... }, + ... }, + ... }, + ... ] + + >>> response = await client.chat_completion( + ... model="meta-llama/Meta-Llama-3-70B-Instruct", + ... messages=messages, + ... tools=tools, + ... tool_choice="auto", + ... max_tokens=500, + ... ) + >>> response.choices[0].message.tool_calls[0].function + ChatCompletionOutputFunctionDefinition( + arguments={ + 'location': 'San Francisco, CA', + 'format': 'fahrenheit', + 'num_days': 3 + }, + name='get_n_day_weather_forecast', + description=None + ) + ``` + + Example using response_format: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") + >>> messages = [ + ... { + ... "role": "user", + ... "content": "I saw a puppy a cat and a raccoon during my bike ride in the park. What did I see and when?", + ... }, + ... ] + >>> response_format = { + ... "type": "json", + ... "value": { + ... "properties": { + ... "location": {"type": "string"}, + ... "activity": {"type": "string"}, + ... "animals_seen": {"type": "integer", "minimum": 1, "maximum": 5}, + ... "animals": {"type": "array", "items": {"type": "string"}}, + ... }, + ... "required": ["location", "activity", "animals_seen", "animals"], + ... }, + ... } + >>> response = await client.chat_completion( + ... messages=messages, + ... response_format=response_format, + ... max_tokens=500, + ... ) + >>> response.choices[0].message.content + '{\n\n"activity": "bike ride",\n"animals": ["puppy", "cat", "raccoon"],\n"animals_seen": 3,\n"location": "park"}' + ``` + """ + # Since `chat_completion(..., model=xxx)` is also a payload parameter for the server, we need to handle 'model' differently. + # `self.model` takes precedence over 'model' argument for building URL. + # `model` takes precedence for payload value. + model_id_or_url = self.model or model + payload_model = model or self.model + + # Get the provider helper + provider_helper = get_provider_helper( + self.provider, + task="conversational", + model=model_id_or_url + if model_id_or_url is not None and model_id_or_url.startswith(("http://", "https://")) + else payload_model, + ) + + # Prepare the payload + parameters = { + "model": payload_model, + "frequency_penalty": frequency_penalty, + "logit_bias": logit_bias, + "logprobs": logprobs, + "max_tokens": max_tokens, + "n": n, + "presence_penalty": presence_penalty, + "response_format": response_format, + "seed": seed, + "stop": stop, + "temperature": temperature, + "tool_choice": tool_choice, + "tool_prompt": tool_prompt, + "tools": tools, + "top_logprobs": top_logprobs, + "top_p": top_p, + "stream": stream, + "stream_options": stream_options, + **(extra_body or {}), + } + request_parameters = provider_helper.prepare_request( + inputs=messages, + parameters=parameters, + headers=self.headers, + model=model_id_or_url, + api_key=self.token, + ) + data = await self._inner_post(request_parameters, stream=stream) + + if stream: + return _async_stream_chat_completion_response(data) # type: ignore + + return ChatCompletionOutput.parse_obj_as_instance(data) # type: ignore + + async def document_question_answering( + self, + image: ContentT, + question: str, + *, + model: str | None = None, + doc_stride: int | None = None, + handle_impossible_answer: bool | None = None, + lang: str | None = None, + max_answer_len: int | None = None, + max_question_len: int | None = None, + max_seq_len: int | None = None, + top_k: int | None = None, + word_boxes: list[list[float] | str] | None = None, + ) -> list[DocumentQuestionAnsweringOutputElement]: + """ + Answer questions on document images. + + Args: + image (`Union[str, Path, bytes, BinaryIO]`): + The input image for the context. It can be raw bytes, an image file, or a URL to an online image. + question (`str`): + Question to be answered. + model (`str`, *optional*): + The model to use for the document question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended document question answering model will be used. + Defaults to None. + doc_stride (`int`, *optional*): + If the words in the document are too long to fit with the question for the model, it will be split in + several chunks with some overlap. This argument controls the size of that overlap. + handle_impossible_answer (`bool`, *optional*): + Whether to accept impossible as an answer + lang (`str`, *optional*): + Language to use while running OCR. Defaults to english. + max_answer_len (`int`, *optional*): + The maximum length of predicted answers (e.g., only answers with a shorter length are considered). + max_question_len (`int`, *optional*): + The maximum length of the question after tokenization. It will be truncated if needed. + max_seq_len (`int`, *optional*): + The maximum length of the total sentence (context + question) in tokens of each chunk passed to the + model. The context will be split in several chunks (using doc_stride as overlap) if needed. + top_k (`int`, *optional*): + The number of answers to return (will be chosen by order of likelihood). Can return less than top_k + answers if there are not enough options available within the context. + word_boxes (`list[Union[list[float], str`, *optional*): + A list of words and bounding boxes (normalized 0->1000). If provided, the inference will skip the OCR + step and use the provided bounding boxes instead. + Returns: + `list[DocumentQuestionAnsweringOutputElement]`: a list of [`DocumentQuestionAnsweringOutputElement`] items containing the predicted label, associated probability, word ids, and page number. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.document_question_answering(image="https://huggingface.co/spaces/impira/docquery/resolve/2359223c1837a7587402bda0f2643382a6eefeab/invoice.png", question="What is the invoice number?") + [DocumentQuestionAnsweringOutputElement(answer='us-001', end=16, score=0.9999666213989258, start=16)] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="document-question-answering", model=model_id) + inputs: dict[str, Any] = {"question": question, "image": _b64_encode(image)} + request_parameters = provider_helper.prepare_request( + inputs=inputs, + parameters={ + "doc_stride": doc_stride, + "handle_impossible_answer": handle_impossible_answer, + "lang": lang, + "max_answer_len": max_answer_len, + "max_question_len": max_question_len, + "max_seq_len": max_seq_len, + "top_k": top_k, + "word_boxes": word_boxes, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return DocumentQuestionAnsweringOutputElement.parse_obj_as_list(response) + + async def feature_extraction( + self, + text: str | list[str], + *, + normalize: bool | None = None, + prompt_name: str | None = None, + truncate: bool | None = None, + truncation_direction: Literal["left", "right"] | None = None, + dimensions: int | None = None, + encoding_format: Literal["float", "base64"] | None = None, + model: str | None = None, + ) -> "np.ndarray": + """ + Generate embeddings for a given text or batch of texts. + + Args: + text (`str` or `list[str]`): + The text or list of texts to embed. + model (`str`, *optional*): + The model to use for the feature extraction task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended feature extraction model will be used. + Defaults to None. + normalize (`bool`, *optional*): + Whether to normalize the embeddings or not. + Only available on server powered by Text-Embedding-Inference. + prompt_name (`str`, *optional*): + The name of the prompt that should be used by for encoding. If not set, no prompt will be applied. + Must be a key in the `Sentence Transformers` configuration `prompts` dictionary. + For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ",...}, + then the sentence "What is the capital of France?" will be encoded as "query: What is the capital of France?" + because the prompt text will be prepended before any text to encode. + truncate (`bool`, *optional*): + Whether to truncate the embeddings or not. + Only available on server powered by Text-Embedding-Inference. + truncation_direction (`Literal["left", "right"]`, *optional*): + Which side of the input should be truncated when `truncate=True` is passed. + dimensions (`int`, *optional*): + The number of dimensions the resulting output embeddings should have. + Only available on OpenAI-compatible embedding endpoints. + encoding_format (`Literal["float", "base64"]`, *optional*): + The format of the output embeddings. Either "float" or "base64". + Only available on OpenAI-compatible embedding endpoints. + + Returns: + `np.ndarray`: The embedding representing the input text(s) as a float32 numpy array. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.feature_extraction("Hi, who are you?") + array([[ 2.424802 , 2.93384 , 1.1750331 , ..., 1.240499, -0.13776633, -0.7889173 ], + [-0.42943227, -0.6364878 , -1.693462 , ..., 0.41978157, -2.4336355 , 0.6162071 ], + ..., + [ 0.28552425, -0.928395 , -1.2077185 , ..., 0.76810825, -2.1069427 , 0.6236161 ]], dtype=float32) + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="feature-extraction", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "normalize": normalize, + "prompt_name": prompt_name, + "truncate": truncate, + "truncation_direction": truncation_direction, + "dimensions": dimensions, + "encoding_format": encoding_format, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + np = _import_numpy() + return np.array(provider_helper.get_response(response), dtype="float32") + + async def fill_mask( + self, + text: str, + *, + model: str | None = None, + targets: list[str] | None = None, + top_k: int | None = None, + ) -> list[FillMaskOutputElement]: + """ + Fill in a hole with a missing word (token to be precise). + + Args: + text (`str`): + a string to be filled from, must contain the [MASK] token (check model card for exact name of the mask). + model (`str`, *optional*): + The model to use for the fill mask task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended fill mask model will be used. + targets (`list[str`, *optional*): + When passed, the model will limit the scores to the passed targets instead of looking up in the whole + vocabulary. If the provided targets are not in the model vocab, they will be tokenized and the first + resulting token will be used (with a warning, and that might be slower). + top_k (`int`, *optional*): + When passed, overrides the number of predictions to return. + Returns: + `list[FillMaskOutputElement]`: a list of [`FillMaskOutputElement`] items containing the predicted label, associated + probability, token reference, and completed text. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.fill_mask("The goal of life is .") + [ + FillMaskOutputElement(score=0.06897063553333282, token=11098, token_str=' happiness', sequence='The goal of life is happiness.'), + FillMaskOutputElement(score=0.06554922461509705, token=45075, token_str=' immortality', sequence='The goal of life is immortality.') + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="fill-mask", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={"targets": targets, "top_k": top_k}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return FillMaskOutputElement.parse_obj_as_list(response) + + async def image_classification( + self, + image: ContentT, + *, + model: str | None = None, + function_to_apply: Optional["ImageClassificationOutputTransform"] = None, + top_k: int | None = None, + ) -> list[ImageClassificationOutputElement]: + """ + Perform image classification on the given image using the specified model. + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The image to classify. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + model (`str`, *optional*): + The model to use for image classification. Can be a model ID hosted on the Hugging Face Hub or a URL to a + deployed Inference Endpoint. If not provided, the default recommended model for image classification will be used. + function_to_apply (`"ImageClassificationOutputTransform"`, *optional*): + The function to apply to the model outputs in order to retrieve the scores. + top_k (`int`, *optional*): + When specified, limits the output to the top K most probable classes. + Returns: + `list[ImageClassificationOutputElement]`: a list of [`ImageClassificationOutputElement`] items containing the predicted label and associated probability. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.image_classification("https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg") + [ImageClassificationOutputElement(label='Blenheim spaniel', score=0.9779096841812134), ...] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="image-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={"function_to_apply": function_to_apply, "top_k": top_k}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return ImageClassificationOutputElement.parse_obj_as_list(response) + + async def image_segmentation( + self, + image: ContentT, + *, + model: str | None = None, + mask_threshold: float | None = None, + overlap_mask_area_threshold: float | None = None, + subtask: Optional["ImageSegmentationSubtask"] = None, + threshold: float | None = None, + ) -> list[ImageSegmentationOutputElement]: + """ + Perform image segmentation on the given image using the specified model. + + > [!WARNING] + > You must have `PIL` installed if you want to work with images (`pip install Pillow`). + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The image to segment. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + model (`str`, *optional*): + The model to use for image segmentation. Can be a model ID hosted on the Hugging Face Hub or a URL to a + deployed Inference Endpoint. If not provided, the default recommended model for image segmentation will be used. + mask_threshold (`float`, *optional*): + Threshold to use when turning the predicted masks into binary values. + overlap_mask_area_threshold (`float`, *optional*): + Mask overlap threshold to eliminate small, disconnected segments. + subtask (`"ImageSegmentationSubtask"`, *optional*): + Segmentation task to be performed, depending on model capabilities. + threshold (`float`, *optional*): + Probability threshold to filter out predicted masks. + Returns: + `list[ImageSegmentationOutputElement]`: A list of [`ImageSegmentationOutputElement`] items containing the segmented masks and associated attributes. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.image_segmentation("cat.jpg") + [ImageSegmentationOutputElement(score=0.989008, label='LABEL_184', mask=), ...] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="image-segmentation", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={ + "mask_threshold": mask_threshold, + "overlap_mask_area_threshold": overlap_mask_area_threshold, + "subtask": subtask, + "threshold": threshold, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_parameters) + output = ImageSegmentationOutputElement.parse_obj_as_list(response) + for item in output: + item.mask = _b64_to_image(item.mask) # type: ignore + return output + + async def image_to_image( + self, + image: ContentT, + prompt: str | None = None, + *, + negative_prompt: str | None = None, + num_inference_steps: int | None = None, + guidance_scale: float | None = None, + model: str | None = None, + target_size: ImageToImageTargetSize | None = None, + **kwargs, + ) -> "Image": + """ + Perform image-to-image translation using a specified model. + + > [!WARNING] + > You must have `PIL` installed if you want to work with images (`pip install Pillow`). + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The input image for translation. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + prompt (`str`, *optional*): + The text prompt to guide the image generation. + negative_prompt (`str`, *optional*): + One prompt to guide what NOT to include in image generation. + num_inference_steps (`int`, *optional*): + For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher + quality image at the expense of slower inference. + guidance_scale (`float`, *optional*): + For diffusion models. A higher guidance scale value encourages the model to generate images closely + linked to the text prompt at the expense of lower image quality. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + target_size (`ImageToImageTargetSize`, *optional*): + The size in pixels of the output image. This parameter is only supported by some providers and for + specific models. It will be ignored when unsupported. + + Returns: + `Image`: The translated image. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> image = await client.image_to_image("cat.jpg", prompt="turn the cat into a tiger") + >>> image.save("tiger.jpg") + ``` + + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="image-to-image", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={ + "prompt": prompt, + "negative_prompt": negative_prompt, + "target_size": target_size, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + **kwargs, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_parameters) + return _bytes_to_image(response) + + async def image_to_video( + self, + image: ContentT, + *, + model: str | None = None, + prompt: str | None = None, + negative_prompt: str | None = None, + num_frames: float | None = None, + num_inference_steps: int | None = None, + guidance_scale: float | None = None, + seed: int | None = None, + target_size: ImageToVideoTargetSize | None = None, + **kwargs, + ) -> bytes: + """ + Generate a video from an input image. + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The input image to generate a video from. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + prompt (`str`, *optional*): + The text prompt to guide the video generation. + negative_prompt (`str`, *optional*): + One prompt to guide what NOT to include in video generation. + num_frames (`float`, *optional*): + The num_frames parameter determines how many video frames are generated. + num_inference_steps (`int`, *optional*): + For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher + quality image at the expense of slower inference. + guidance_scale (`float`, *optional*): + For diffusion models. A higher guidance scale value encourages the model to generate videos closely + linked to the text prompt at the expense of lower image quality. + seed (`int`, *optional*): + The seed to use for the video generation. + target_size (`ImageToVideoTargetSize`, *optional*): + The size in pixel of the output video frames. + num_inference_steps (`int`, *optional*): + The number of denoising steps. More denoising steps usually lead to a higher quality video at the + expense of slower inference. + seed (`int`, *optional*): + Seed for the random number generator. + + Returns: + `bytes`: The generated video. + + Examples: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> video = await client.image_to_video("cat.jpg", model="Wan-AI/Wan2.2-I2V-A14B", prompt="turn the cat into a tiger") + >>> with open("tiger.mp4", "wb") as f: + ... f.write(video) + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="image-to-video", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={ + "prompt": prompt, + "negative_prompt": negative_prompt, + "num_frames": num_frames, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + "seed": seed, + "target_size": target_size, + **kwargs, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_parameters) + return response + + async def image_to_text(self, image: ContentT, *, model: str | None = None) -> ImageToTextOutput: + """ + Takes an input image and return text. + + Models can have very different outputs depending on your use case (image captioning, optical character recognition + (OCR), Pix2Struct, etc.). Please have a look to the model card to learn more about a model's specificities. + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The input image to caption. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + + Returns: + [`ImageToTextOutput`]: The generated text. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.image_to_text("cat.jpg") + 'a cat standing in a grassy field ' + >>> await client.image_to_text("https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg") + 'a dog laying on the grass next to a flower pot ' + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="image-to-text", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + output_list: list[ImageToTextOutput] = ImageToTextOutput.parse_obj_as_list(response) + return output_list[0] + + async def object_detection( + self, image: ContentT, *, model: str | None = None, threshold: float | None = None + ) -> list[ObjectDetectionOutputElement]: + """ + Perform object detection on the given image using the specified model. + + > [!WARNING] + > You must have `PIL` installed if you want to work with images (`pip install Pillow`). + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The image to detect objects on. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + model (`str`, *optional*): + The model to use for object detection. Can be a model ID hosted on the Hugging Face Hub or a URL to a + deployed Inference Endpoint. If not provided, the default recommended model for object detection (DETR) will be used. + threshold (`float`, *optional*): + The probability necessary to make a prediction. + Returns: + `list[ObjectDetectionOutputElement]`: A list of [`ObjectDetectionOutputElement`] items containing the bounding boxes and associated attributes. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + `ValueError`: + If the request output is not a List. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.object_detection("people.jpg") + [ObjectDetectionOutputElement(score=0.9486683011054993, label='person', box=ObjectDetectionBoundingBox(xmin=59, ymin=39, xmax=420, ymax=510)), ...] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="object-detection", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={"threshold": threshold}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return ObjectDetectionOutputElement.parse_obj_as_list(response) + + async def question_answering( + self, + question: str, + context: str, + *, + model: str | None = None, + align_to_words: bool | None = None, + doc_stride: int | None = None, + handle_impossible_answer: bool | None = None, + max_answer_len: int | None = None, + max_question_len: int | None = None, + max_seq_len: int | None = None, + top_k: int | None = None, + ) -> QuestionAnsweringOutputElement | list[QuestionAnsweringOutputElement]: + """ + Retrieve the answer to a question from a given text. + + Args: + question (`str`): + Question to be answered. + context (`str`): + The context of the question. + model (`str`): + The model to use for the question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. + align_to_words (`bool`, *optional*): + Attempts to align the answer to real words. Improves quality on space separated languages. Might hurt + on non-space-separated languages (like Japanese or Chinese) + doc_stride (`int`, *optional*): + If the context is too long to fit with the question for the model, it will be split in several chunks + with some overlap. This argument controls the size of that overlap. + handle_impossible_answer (`bool`, *optional*): + Whether to accept impossible as an answer. + max_answer_len (`int`, *optional*): + The maximum length of predicted answers (e.g., only answers with a shorter length are considered). + max_question_len (`int`, *optional*): + The maximum length of the question after tokenization. It will be truncated if needed. + max_seq_len (`int`, *optional*): + The maximum length of the total sentence (context + question) in tokens of each chunk passed to the + model. The context will be split in several chunks (using docStride as overlap) if needed. + top_k (`int`, *optional*): + The number of answers to return (will be chosen by order of likelihood). Note that we return less than + topk answers if there are not enough options available within the context. + + Returns: + Union[`QuestionAnsweringOutputElement`, list[`QuestionAnsweringOutputElement`]]: + When top_k is 1 or not provided, it returns a single `QuestionAnsweringOutputElement`. + When top_k is greater than 1, it returns a list of `QuestionAnsweringOutputElement`. + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.question_answering(question="What's my name?", context="My name is Clara and I live in Berkeley.") + QuestionAnsweringOutputElement(answer='Clara', end=16, score=0.9326565265655518, start=11) + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="question-answering", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs={"question": question, "context": context}, + parameters={ + "align_to_words": align_to_words, + "doc_stride": doc_stride, + "handle_impossible_answer": handle_impossible_answer, + "max_answer_len": max_answer_len, + "max_question_len": max_question_len, + "max_seq_len": max_seq_len, + "top_k": top_k, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + # Parse the response as a single `QuestionAnsweringOutputElement` when top_k is 1 or not provided, or a list of `QuestionAnsweringOutputElement` to ensure backward compatibility. + output = QuestionAnsweringOutputElement.parse_obj(response) + return output + + async def sentence_similarity( + self, sentence: str, other_sentences: list[str], *, model: str | None = None + ) -> list[float]: + """ + Compute the semantic similarity between a sentence and a list of other sentences by comparing their embeddings. + + Args: + sentence (`str`): + The main sentence to compare to others. + other_sentences (`list[str]`): + The list of sentences to compare to. + model (`str`, *optional*): + The model to use for the sentence similarity task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended sentence similarity model will be used. + Defaults to None. + + Returns: + `list[float]`: The embedding representing the input text. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.sentence_similarity( + ... "Machine learning is so easy.", + ... other_sentences=[ + ... "Deep learning is so straightforward.", + ... "This is so difficult, like rocket science.", + ... "I can't believe how much I struggled with this.", + ... ], + ... ) + [0.7785726189613342, 0.45876261591911316, 0.2906220555305481] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="sentence-similarity", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs={"source_sentence": sentence, "sentences": other_sentences}, + parameters={}, + extra_payload={}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return _bytes_to_list(response) + + async def summarization( + self, + text: str, + *, + model: str | None = None, + clean_up_tokenization_spaces: bool | None = None, + generate_parameters: dict[str, Any] | None = None, + truncation: Optional["SummarizationTruncationStrategy"] = None, + ) -> SummarizationOutput: + """ + Generate a summary of a given text using a specified model. + + Args: + text (`str`): + The input text to summarize. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended model for summarization will be used. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether to clean up the potential extra spaces in the text output. + generate_parameters (`dict[str, Any]`, *optional*): + Additional parametrization of the text generation algorithm. + truncation (`"SummarizationTruncationStrategy"`, *optional*): + The truncation strategy to use. + Returns: + [`SummarizationOutput`]: The generated summary text. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.summarization("The Eiffel tower...") + SummarizationOutput(generated_text="The Eiffel tower is one of the most famous landmarks in the world....") + ``` + """ + parameters = { + "clean_up_tokenization_spaces": clean_up_tokenization_spaces, + "generate_parameters": generate_parameters, + "truncation": truncation, + } + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="summarization", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters=parameters, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return SummarizationOutput.parse_obj_as_list(response)[0] + + async def table_question_answering( + self, + table: dict[str, Any], + query: str, + *, + model: str | None = None, + padding: Optional["Padding"] = None, + sequential: bool | None = None, + truncation: bool | None = None, + ) -> TableQuestionAnsweringOutputElement: + """ + Retrieve the answer to a question from information given in a table. + + Args: + table (`str`): + A table of data represented as a dict of lists where entries are headers and the lists are all the + values, all lists must have the same size. + query (`str`): + The query in plain text that you want to ask the table. + model (`str`): + The model to use for the table-question-answering task. Can be a model ID hosted on the Hugging Face + Hub or a URL to a deployed Inference Endpoint. + padding (`"Padding"`, *optional*): + Activates and controls padding. + sequential (`bool`, *optional*): + Whether to do inference sequentially or as a batch. Batching is faster, but models like SQA require the + inference to be done sequentially to extract relations within sequences, given their conversational + nature. + truncation (`bool`, *optional*): + Activates and controls truncation. + + Returns: + [`TableQuestionAnsweringOutputElement`]: a table question answering output containing the answer, coordinates, cells and the aggregator used. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> query = "How many stars does the transformers repository have?" + >>> table = {"Repository": ["Transformers", "Datasets", "Tokenizers"], "Stars": ["36542", "4512", "3934"]} + >>> await client.table_question_answering(table, query, model="google/tapas-base-finetuned-wtq") + TableQuestionAnsweringOutputElement(answer='36542', coordinates=[[0, 1]], cells=['36542'], aggregator='AVERAGE') + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="table-question-answering", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs={"query": query, "table": table}, + parameters={"model": model, "padding": padding, "sequential": sequential, "truncation": truncation}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return TableQuestionAnsweringOutputElement.parse_obj_as_instance(response) + + async def tabular_classification(self, table: dict[str, Any], *, model: str | None = None) -> list[str]: + """ + Classifying a target category (a group) based on a set of attributes. + + Args: + table (`dict[str, Any]`): + Set of attributes to classify. + model (`str`, *optional*): + The model to use for the tabular classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended tabular classification model will be used. + Defaults to None. + + Returns: + `List`: a list of labels, one per row in the initial table. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> table = { + ... "fixed_acidity": ["7.4", "7.8", "10.3"], + ... "volatile_acidity": ["0.7", "0.88", "0.32"], + ... "citric_acid": ["0", "0", "0.45"], + ... "residual_sugar": ["1.9", "2.6", "6.4"], + ... "chlorides": ["0.076", "0.098", "0.073"], + ... "free_sulfur_dioxide": ["11", "25", "5"], + ... "total_sulfur_dioxide": ["34", "67", "13"], + ... "density": ["0.9978", "0.9968", "0.9976"], + ... "pH": ["3.51", "3.2", "3.23"], + ... "sulphates": ["0.56", "0.68", "0.82"], + ... "alcohol": ["9.4", "9.8", "12.6"], + ... } + >>> await client.tabular_classification(table=table, model="julien-c/wine-quality") + ["5", "5", "5"] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="tabular-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=None, + extra_payload={"table": table}, + parameters={}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return _bytes_to_list(response) + + async def tabular_regression(self, table: dict[str, Any], *, model: str | None = None) -> list[float]: + """ + Predicting a numerical target value given a set of attributes/features in a table. + + Args: + table (`dict[str, Any]`): + Set of attributes stored in a table. The attributes used to predict the target can be both numerical and categorical. + model (`str`, *optional*): + The model to use for the tabular regression task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended tabular regression model will be used. + Defaults to None. + + Returns: + `List`: a list of predicted numerical target values. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> table = { + ... "Height": ["11.52", "12.48", "12.3778"], + ... "Length1": ["23.2", "24", "23.9"], + ... "Length2": ["25.4", "26.3", "26.5"], + ... "Length3": ["30", "31.2", "31.1"], + ... "Species": ["Bream", "Bream", "Bream"], + ... "Width": ["4.02", "4.3056", "4.6961"], + ... } + >>> await client.tabular_regression(table, model="scikit-learn/Fish-Weight") + [110, 120, 130] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="tabular-regression", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=None, + parameters={}, + extra_payload={"table": table}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return _bytes_to_list(response) + + async def text_classification( + self, + text: str, + *, + model: str | None = None, + top_k: int | None = None, + function_to_apply: Optional["TextClassificationOutputTransform"] = None, + ) -> list[TextClassificationOutputElement]: + """ + Perform text classification (e.g. sentiment-analysis) on the given text. + + Args: + text (`str`): + A string to be classified. + model (`str`, *optional*): + The model to use for the text classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended text classification model will be used. + Defaults to None. + top_k (`int`, *optional*): + When specified, limits the output to the top K most probable classes. + function_to_apply (`"TextClassificationOutputTransform"`, *optional*): + The function to apply to the model outputs in order to retrieve the scores. + + Returns: + `list[TextClassificationOutputElement]`: a list of [`TextClassificationOutputElement`] items containing the predicted label and associated probability. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.text_classification("I like you") + [ + TextClassificationOutputElement(label='POSITIVE', score=0.9998695850372314), + TextClassificationOutputElement(label='NEGATIVE', score=0.0001304351753788069), + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="text-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "function_to_apply": function_to_apply, + "top_k": top_k, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return TextClassificationOutputElement.parse_obj_as_list(response)[0] # type: ignore + + @overload + async def text_generation( + self, + prompt: str, + *, + details: Literal[True], + stream: Literal[True], + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> AsyncIterable[TextGenerationStreamOutput]: ... + + @overload + async def text_generation( + self, + prompt: str, + *, + details: Literal[True], + stream: Literal[False] | None = None, + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> TextGenerationOutput: ... + + @overload + async def text_generation( + self, + prompt: str, + *, + details: Literal[False] | None = None, + stream: Literal[True], + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, # Manual default value + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> AsyncIterable[str]: ... + + @overload + async def text_generation( + self, + prompt: str, + *, + details: Literal[False] | None = None, + stream: Literal[False] | None = None, + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> str: ... + + @overload + async def text_generation( + self, + prompt: str, + *, + details: bool | None = None, + stream: bool | None = None, + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> str | TextGenerationOutput | AsyncIterable[str] | AsyncIterable[TextGenerationStreamOutput]: ... + + async def text_generation( + self, + prompt: str, + *, + details: bool | None = None, + stream: bool | None = None, + model: str | None = None, + # Parameters from `TextGenerationInputGenerateParameters` (maintained manually) + adapter_id: str | None = None, + best_of: int | None = None, + decoder_input_details: bool | None = None, + do_sample: bool | None = None, + frequency_penalty: float | None = None, + grammar: TextGenerationInputGrammarType | None = None, + max_new_tokens: int | None = None, + repetition_penalty: float | None = None, + return_full_text: bool | None = None, + seed: int | None = None, + stop: list[str] | None = None, + stop_sequences: list[str] | None = None, # Deprecated, use `stop` instead + temperature: float | None = None, + top_k: int | None = None, + top_n_tokens: int | None = None, + top_p: float | None = None, + truncate: int | None = None, + typical_p: float | None = None, + watermark: bool | None = None, + ) -> str | TextGenerationOutput | AsyncIterable[str] | AsyncIterable[TextGenerationStreamOutput]: + """ + Given a prompt, generate the following text. + + > [!TIP] + > If you want to generate a response from chat messages, you should use the [`InferenceClient.chat_completion`] method. + > It accepts a list of messages instead of a single text prompt and handles the chat templating for you. + + Args: + prompt (`str`): + Input text. + details (`bool`, *optional*): + By default, text_generation returns a string. Pass `details=True` if you want a detailed output (tokens, + probabilities, seed, finish reason, etc.). Only available for models running on with the + `text-generation-inference` backend. + stream (`bool`, *optional*): + By default, text_generation returns the full generated text. Pass `stream=True` if you want a stream of + tokens to be returned. Only available for models running on with the `text-generation-inference` + backend. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + adapter_id (`str`, *optional*): + Lora adapter id. + best_of (`int`, *optional*): + Generate best_of sequences and return the one if the highest token logprobs. + decoder_input_details (`bool`, *optional*): + Return the decoder input token logprobs and ids. You must set `details=True` as well for it to be taken + into account. Defaults to `False`. + do_sample (`bool`, *optional*): + Activate logits sampling + frequency_penalty (`float`, *optional*): + Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in + the text so far, decreasing the model's likelihood to repeat the same line verbatim. + grammar ([`TextGenerationInputGrammarType`], *optional*): + Grammar constraints. Can be either a JSONSchema or a regex. + max_new_tokens (`int`, *optional*): + Maximum number of generated tokens. Defaults to 100. + repetition_penalty (`float`, *optional*): + The parameter for repetition penalty. 1.0 means no penalty. See [this + paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + return_full_text (`bool`, *optional*): + Whether to prepend the prompt to the generated text + seed (`int`, *optional*): + Random sampling seed + stop (`list[str]`, *optional*): + Stop generating tokens if a member of `stop` is generated. + stop_sequences (`list[str]`, *optional*): + Deprecated argument. Use `stop` instead. + temperature (`float`, *optional*): + The value used to module the logits distribution. + top_n_tokens (`int`, *optional*): + Return information about the `top_n_tokens` most likely tokens at each generation step, instead of + just the sampled token. + top_k (`int`, *optional`): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + top_p (`float`, *optional`): + If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or + higher are kept for generation. + truncate (`int`, *optional`): + Truncate inputs tokens to the given size. + typical_p (`float`, *optional`): + Typical Decoding mass + See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information + watermark (`bool`, *optional*): + Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) + + Returns: + `Union[str, TextGenerationOutput, AsyncIterable[str], AsyncIterable[TextGenerationStreamOutput]]`: + Generated text returned from the server: + - if `stream=False` and `details=False`, the generated text is returned as a `str` (default) + - if `stream=True` and `details=False`, the generated text is returned token by token as a `AsyncIterable[str]` + - if `stream=False` and `details=True`, the generated text is returned with more details as a [`~huggingface_hub.TextGenerationOutput`] + - if `details=True` and `stream=True`, the generated text is returned token by token as a iterable of [`~huggingface_hub.TextGenerationStreamOutput`] + + Raises: + `ValidationError`: + If input values are not valid. No HTTP call is made to the server. + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + + # Case 1: generate text + >>> await client.text_generation("The huggingface_hub library is ", max_new_tokens=12) + '100% open source and built to be easy to use.' + + # Case 2: iterate over the generated tokens. Useful for large generation. + >>> async for token in await client.text_generation("The huggingface_hub library is ", max_new_tokens=12, stream=True): + ... print(token) + 100 + % + open + source + and + built + to + be + easy + to + use + . + + # Case 3: get more details about the generation process. + >>> await client.text_generation("The huggingface_hub library is ", max_new_tokens=12, details=True) + TextGenerationOutput( + generated_text='100% open source and built to be easy to use.', + details=TextGenerationDetails( + finish_reason='length', + generated_tokens=12, + seed=None, + prefill=[ + TextGenerationPrefillOutputToken(id=487, text='The', logprob=None), + TextGenerationPrefillOutputToken(id=53789, text=' hugging', logprob=-13.171875), + (...) + TextGenerationPrefillOutputToken(id=204, text=' ', logprob=-7.0390625) + ], + tokens=[ + TokenElement(id=1425, text='100', logprob=-1.0175781, special=False), + TokenElement(id=16, text='%', logprob=-0.0463562, special=False), + (...) + TokenElement(id=25, text='.', logprob=-0.5703125, special=False) + ], + best_of_sequences=None + ) + ) + + # Case 4: iterate over the generated tokens with more details. + # Last object is more complete, containing the full generated text and the finish reason. + >>> async for details in await client.text_generation("The huggingface_hub library is ", max_new_tokens=12, details=True, stream=True): + ... print(details) + ... + TextGenerationStreamOutput(token=TokenElement(id=1425, text='100', logprob=-1.0175781, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=16, text='%', logprob=-0.0463562, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=1314, text=' open', logprob=-1.3359375, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=3178, text=' source', logprob=-0.28100586, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=273, text=' and', logprob=-0.5961914, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=3426, text=' built', logprob=-1.9423828, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=271, text=' to', logprob=-1.4121094, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=314, text=' be', logprob=-1.5224609, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=1833, text=' easy', logprob=-2.1132812, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=271, text=' to', logprob=-0.08520508, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement(id=745, text=' use', logprob=-0.39453125, special=False), generated_text=None, details=None) + TextGenerationStreamOutput(token=TokenElement( + id=25, + text='.', + logprob=-0.5703125, + special=False), + generated_text='100% open source and built to be easy to use.', + details=TextGenerationStreamOutputStreamDetails(finish_reason='length', generated_tokens=12, seed=None) + ) + + # Case 5: generate constrained output using grammar + >>> response = await client.text_generation( + ... prompt="I saw a puppy a cat and a raccoon during my bike ride in the park", + ... model="HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1", + ... max_new_tokens=100, + ... repetition_penalty=1.3, + ... grammar={ + ... "type": "json", + ... "value": { + ... "properties": { + ... "location": {"type": "string"}, + ... "activity": {"type": "string"}, + ... "animals_seen": {"type": "integer", "minimum": 1, "maximum": 5}, + ... "animals": {"type": "array", "items": {"type": "string"}}, + ... }, + ... "required": ["location", "activity", "animals_seen", "animals"], + ... }, + ... }, + ... ) + >>> json.loads(response) + { + "activity": "bike riding", + "animals": ["puppy", "cat", "raccoon"], + "animals_seen": 3, + "location": "park" + } + ``` + """ + if decoder_input_details and not details: + warnings.warn( + "`decoder_input_details=True` has been passed to the server but `details=False` is set meaning that" + " the output from the server will be truncated." + ) + decoder_input_details = False + + if stop_sequences is not None: + warnings.warn( + "`stop_sequences` is a deprecated argument for `text_generation` task" + " and will be removed in version '0.28.0'. Use `stop` instead.", + FutureWarning, + ) + if stop is None: + stop = stop_sequences # use deprecated arg if provided + + # Build payload + parameters = { + "adapter_id": adapter_id, + "best_of": best_of, + "decoder_input_details": decoder_input_details, + "details": details, + "do_sample": do_sample, + "frequency_penalty": frequency_penalty, + "grammar": grammar, + "max_new_tokens": max_new_tokens, + "repetition_penalty": repetition_penalty, + "return_full_text": return_full_text, + "seed": seed, + "stop": stop, + "temperature": temperature, + "top_k": top_k, + "top_n_tokens": top_n_tokens, + "top_p": top_p, + "truncate": truncate, + "typical_p": typical_p, + "watermark": watermark, + } + + # Remove some parameters if not a TGI server + unsupported_kwargs = _get_unsupported_text_generation_kwargs(model) + if len(unsupported_kwargs) > 0: + # The server does not support some parameters + # => means it is not a TGI server + # => remove unsupported parameters and warn the user + + ignored_parameters = [] + for key in unsupported_kwargs: + if parameters.get(key): + ignored_parameters.append(key) + parameters.pop(key, None) + if len(ignored_parameters) > 0: + warnings.warn( + "API endpoint/model for text-generation is not served via TGI. Ignoring following parameters:" + f" {', '.join(ignored_parameters)}.", + UserWarning, + ) + if details: + warnings.warn( + "API endpoint/model for text-generation is not served via TGI. Parameter `details=True` will" + " be ignored meaning only the generated text will be returned.", + UserWarning, + ) + details = False + if stream: + raise ValueError( + "API endpoint/model for text-generation is not served via TGI. Cannot return output as a stream." + " Please pass `stream=False` as input." + ) + + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="text-generation", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=prompt, + parameters=parameters, + extra_payload={"stream": stream}, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + + # Handle errors separately for more precise error messages + try: + bytes_output = await self._inner_post(request_parameters, stream=stream or False) + except HfHubHTTPError as e: + match = MODEL_KWARGS_NOT_USED_REGEX.search(str(e)) + if isinstance(e, BadRequestError) and match: + unused_params = [kwarg.strip("' ") for kwarg in match.group(1).split(",")] + _set_unsupported_text_generation_kwargs(model, unused_params) + return await self.text_generation( # type: ignore + prompt=prompt, + details=details, + stream=stream, + model=model_id, + adapter_id=adapter_id, + best_of=best_of, + decoder_input_details=decoder_input_details, + do_sample=do_sample, + frequency_penalty=frequency_penalty, + grammar=grammar, + max_new_tokens=max_new_tokens, + repetition_penalty=repetition_penalty, + return_full_text=return_full_text, + seed=seed, + stop=stop, + temperature=temperature, + top_k=top_k, + top_n_tokens=top_n_tokens, + top_p=top_p, + truncate=truncate, + typical_p=typical_p, + watermark=watermark, + ) + raise_text_generation_error(e) + + # Parse output + if stream: + return _async_stream_text_generation_response(bytes_output, details) # type: ignore + + data = _bytes_to_dict(bytes_output) # type: ignore + + # Data can be a single element (dict) or an iterable of dicts where we select the first element of. + if isinstance(data, list): + data = data[0] + response = provider_helper.get_response(data, request_parameters) + return TextGenerationOutput.parse_obj_as_instance(response) if details else response["generated_text"] + + async def text_to_image( + self, + prompt: str, + *, + negative_prompt: str | None = None, + height: int | None = None, + width: int | None = None, + num_inference_steps: int | None = None, + guidance_scale: float | None = None, + model: str | None = None, + scheduler: str | None = None, + seed: int | None = None, + extra_body: dict[str, Any] | None = None, + ) -> "Image": + """ + Generate an image based on a given text using a specified model. + + > [!WARNING] + > You must have `PIL` installed if you want to work with images (`pip install Pillow`). + + > [!TIP] + > You can pass provider-specific parameters to the model by using the `extra_body` argument. + + Args: + prompt (`str`): + The prompt to generate an image from. + negative_prompt (`str`, *optional*): + One prompt to guide what NOT to include in image generation. + height (`int`, *optional*): + The height in pixels of the output image + width (`int`, *optional*): + The width in pixels of the output image + num_inference_steps (`int`, *optional*): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*): + A higher guidance scale value encourages the model to generate images closely linked to the text + prompt, but values too high may cause saturation and other artifacts. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended text-to-image model will be used. + Defaults to None. + scheduler (`str`, *optional*): + Override the scheduler with a compatible one. + seed (`int`, *optional*): + Seed for the random number generator. + extra_body (`dict[str, Any]`, *optional*): + Additional provider-specific parameters to pass to the model. Refer to the provider's documentation + for supported parameters. + + Returns: + `Image`: The generated image. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + + >>> image = await client.text_to_image("An astronaut riding a horse on the moon.") + >>> image.save("astronaut.png") + + >>> image = await client.text_to_image( + ... "An astronaut riding a horse on the moon.", + ... negative_prompt="low resolution, blurry", + ... model="stabilityai/stable-diffusion-2-1", + ... ) + >>> image.save("better_astronaut.png") + ``` + Example using a third-party provider directly. Usage will be billed on your fal.ai account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="fal-ai", # Use fal.ai provider + ... api_key="fal-ai-api-key", # Pass your fal.ai API key + ... ) + >>> image = client.text_to_image( + ... "A majestic lion in a fantasy forest", + ... model="black-forest-labs/FLUX.1-schnell", + ... ) + >>> image.save("lion.png") + ``` + + Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", # Use replicate provider + ... api_key="hf_...", # Pass your HF token + ... ) + >>> image = client.text_to_image( + ... "An astronaut riding a horse on the moon.", + ... model="black-forest-labs/FLUX.1-dev", + ... ) + >>> image.save("astronaut.png") + ``` + + Example using Replicate provider with extra parameters + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", # Use replicate provider + ... api_key="hf_...", # Pass your HF token + ... ) + >>> image = client.text_to_image( + ... "An astronaut riding a horse on the moon.", + ... model="black-forest-labs/FLUX.1-schnell", + ... extra_body={"output_quality": 100}, + ... ) + >>> image.save("astronaut.png") + ``` + + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="text-to-image", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=prompt, + parameters={ + "negative_prompt": negative_prompt, + "height": height, + "width": width, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + "scheduler": scheduler, + "seed": seed, + **(extra_body or {}), + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_parameters) + return _bytes_to_image(response) + + async def text_to_video( + self, + prompt: str, + *, + model: str | None = None, + guidance_scale: float | None = None, + negative_prompt: list[str] | None = None, + num_frames: float | None = None, + num_inference_steps: int | None = None, + seed: int | None = None, + extra_body: dict[str, Any] | None = None, + ) -> bytes: + """ + Generate a video based on a given text. + + > [!TIP] + > You can pass provider-specific parameters to the model by using the `extra_body` argument. + + Args: + prompt (`str`): + The prompt to generate a video from. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended text-to-video model will be used. + Defaults to None. + guidance_scale (`float`, *optional*): + A higher guidance scale value encourages the model to generate videos closely linked to the text + prompt, but values too high may cause saturation and other artifacts. + negative_prompt (`list[str]`, *optional*): + One or several prompt to guide what NOT to include in video generation. + num_frames (`float`, *optional*): + The num_frames parameter determines how many video frames are generated. + num_inference_steps (`int`, *optional*): + The number of denoising steps. More denoising steps usually lead to a higher quality video at the + expense of slower inference. + seed (`int`, *optional*): + Seed for the random number generator. + extra_body (`dict[str, Any]`, *optional*): + Additional provider-specific parameters to pass to the model. Refer to the provider's documentation + for supported parameters. + + Returns: + `bytes`: The generated video. + + Example: + + Example using a third-party provider directly. Usage will be billed on your fal.ai account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="fal-ai", # Using fal.ai provider + ... api_key="fal-ai-api-key", # Pass your fal.ai API key + ... ) + >>> video = client.text_to_video( + ... "A majestic lion running in a fantasy forest", + ... model="tencent/HunyuanVideo", + ... ) + >>> with open("lion.mp4", "wb") as file: + ... file.write(video) + ``` + + Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", # Using replicate provider + ... api_key="hf_...", # Pass your HF token + ... ) + >>> video = client.text_to_video( + ... "A cat running in a park", + ... model="genmo/mochi-1-preview", + ... ) + >>> with open("cat.mp4", "wb") as file: + ... file.write(video) + ``` + + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="text-to-video", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=prompt, + parameters={ + "guidance_scale": guidance_scale, + "negative_prompt": negative_prompt, + "num_frames": num_frames, + "num_inference_steps": num_inference_steps, + "seed": seed, + **(extra_body or {}), + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + response = provider_helper.get_response(response, request_parameters) + return response + + async def text_to_speech( + self, + text: str, + *, + model: str | None = None, + do_sample: bool | None = None, + early_stopping: Union[bool, "TextToSpeechEarlyStoppingEnum"] | None = None, + epsilon_cutoff: float | None = None, + eta_cutoff: float | None = None, + max_length: int | None = None, + max_new_tokens: int | None = None, + min_length: int | None = None, + min_new_tokens: int | None = None, + num_beam_groups: int | None = None, + num_beams: int | None = None, + penalty_alpha: float | None = None, + temperature: float | None = None, + top_k: int | None = None, + top_p: float | None = None, + typical_p: float | None = None, + use_cache: bool | None = None, + extra_body: dict[str, Any] | None = None, + ) -> bytes: + """ + Synthesize an audio of a voice pronouncing a given text. + + > [!TIP] + > You can pass provider-specific parameters to the model by using the `extra_body` argument. + + Args: + text (`str`): + The text to synthesize. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. If not provided, the default recommended text-to-speech model will be used. + Defaults to None. + do_sample (`bool`, *optional*): + Whether to use sampling instead of greedy decoding when generating new tokens. + early_stopping (`Union[bool, "TextToSpeechEarlyStoppingEnum"]`, *optional*): + Controls the stopping condition for beam-based methods. + epsilon_cutoff (`float`, *optional*): + If set to float strictly between 0 and 1, only tokens with a conditional probability greater than + epsilon_cutoff will be sampled. In the paper, suggested values range from 3e-4 to 9e-4, depending on + the size of the model. See [Truncation Sampling as Language Model + Desmoothing](https://hf.co/papers/2210.15191) for more details. + eta_cutoff (`float`, *optional*): + Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to float strictly + between 0 and 1, a token is only considered if it is greater than either eta_cutoff or sqrt(eta_cutoff) + * exp(-entropy(softmax(next_token_logits))). The latter term is intuitively the expected next token + probability, scaled by sqrt(eta_cutoff). In the paper, suggested values range from 3e-4 to 2e-3, + depending on the size of the model. See [Truncation Sampling as Language Model + Desmoothing](https://hf.co/papers/2210.15191) for more details. + max_length (`int`, *optional*): + The maximum length (in tokens) of the generated text, including the input. + max_new_tokens (`int`, *optional*): + The maximum number of tokens to generate. Takes precedence over max_length. + min_length (`int`, *optional*): + The minimum length (in tokens) of the generated text, including the input. + min_new_tokens (`int`, *optional*): + The minimum number of tokens to generate. Takes precedence over min_length. + num_beam_groups (`int`, *optional*): + Number of groups to divide num_beams into in order to ensure diversity among different groups of beams. + See [this paper](https://hf.co/papers/1610.02424) for more details. + num_beams (`int`, *optional*): + Number of beams to use for beam search. + penalty_alpha (`float`, *optional*): + The value balances the model confidence and the degeneration penalty in contrastive search decoding. + temperature (`float`, *optional*): + The value used to modulate the next token probabilities. + top_k (`int`, *optional*): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + top_p (`float`, *optional*): + If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to + top_p or higher are kept for generation. + typical_p (`float`, *optional*): + Local typicality measures how similar the conditional probability of predicting a target token next is + to the expected conditional probability of predicting a random token next, given the partial text + already generated. If set to float < 1, the smallest set of the most locally typical tokens with + probabilities that add up to typical_p or higher are kept for generation. See [this + paper](https://hf.co/papers/2202.00666) for more details. + use_cache (`bool`, *optional*): + Whether the model should use the past last key/values attentions to speed up decoding + extra_body (`dict[str, Any]`, *optional*): + Additional provider-specific parameters to pass to the model. Refer to the provider's documentation + for supported parameters. + Returns: + `bytes`: The generated audio. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from pathlib import Path + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + + >>> audio = await client.text_to_speech("Hello world") + >>> Path("hello_world.flac").write_bytes(audio) + ``` + + Example using a third-party provider directly. Usage will be billed on your Replicate account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", + ... api_key="your-replicate-api-key", # Pass your Replicate API key directly + ... ) + >>> audio = client.text_to_speech( + ... text="Hello world", + ... model="OuteAI/OuteTTS-0.3-500M", + ... ) + >>> Path("hello_world.flac").write_bytes(audio) + ``` + + Example using a third-party provider through Hugging Face Routing. Usage will be billed on your Hugging Face account. + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", + ... api_key="hf_...", # Pass your HF token + ... ) + >>> audio =client.text_to_speech( + ... text="Hello world", + ... model="OuteAI/OuteTTS-0.3-500M", + ... ) + >>> Path("hello_world.flac").write_bytes(audio) + ``` + Example using Replicate provider with extra parameters + ```py + >>> from huggingface_hub import InferenceClient + >>> client = InferenceClient( + ... provider="replicate", # Use replicate provider + ... api_key="hf_...", # Pass your HF token + ... ) + >>> audio = client.text_to_speech( + ... "Hello, my name is Kororo, an awesome text-to-speech model.", + ... model="hexgrad/Kokoro-82M", + ... extra_body={"voice": "af_nicole"}, + ... ) + >>> Path("hello.flac").write_bytes(audio) + ``` + + Example music-gen using "YuE-s1-7B-anneal-en-cot" on fal.ai + ```py + >>> from huggingface_hub import InferenceClient + >>> lyrics = ''' + ... [verse] + ... In the town where I was born + ... Lived a man who sailed to sea + ... And he told us of his life + ... In the land of submarines + ... So we sailed on to the sun + ... 'Til we found a sea of green + ... And we lived beneath the waves + ... In our yellow submarine + + ... [chorus] + ... We all live in a yellow submarine + ... Yellow submarine, yellow submarine + ... We all live in a yellow submarine + ... Yellow submarine, yellow submarine + ... ''' + >>> genres = "pavarotti-style tenor voice" + >>> client = InferenceClient( + ... provider="fal-ai", + ... model="m-a-p/YuE-s1-7B-anneal-en-cot", + ... api_key=..., + ... ) + >>> audio = client.text_to_speech(lyrics, extra_body={"genres": genres}) + >>> with open("output.mp3", "wb") as f: + ... f.write(audio) + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="text-to-speech", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "do_sample": do_sample, + "early_stopping": early_stopping, + "epsilon_cutoff": epsilon_cutoff, + "eta_cutoff": eta_cutoff, + "max_length": max_length, + "max_new_tokens": max_new_tokens, + "min_length": min_length, + "min_new_tokens": min_new_tokens, + "num_beam_groups": num_beam_groups, + "num_beams": num_beams, + "penalty_alpha": penalty_alpha, + "temperature": temperature, + "top_k": top_k, + "top_p": top_p, + "typical_p": typical_p, + "use_cache": use_cache, + **(extra_body or {}), + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + response = provider_helper.get_response(response) + return response + + async def token_classification( + self, + text: str, + *, + model: str | None = None, + aggregation_strategy: Optional["TokenClassificationAggregationStrategy"] = None, + ignore_labels: list[str] | None = None, + stride: int | None = None, + ) -> list[TokenClassificationOutputElement]: + """ + Perform token classification on the given text. + Usually used for sentence parsing, either grammatical, or Named Entity Recognition (NER) to understand keywords contained within text. + + Args: + text (`str`): + A string to be classified. + model (`str`, *optional*): + The model to use for the token classification task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended token classification model will be used. + Defaults to None. + aggregation_strategy (`"TokenClassificationAggregationStrategy"`, *optional*): + The strategy used to fuse tokens based on model predictions + ignore_labels (`list[str`, *optional*): + A list of labels to ignore + stride (`int`, *optional*): + The number of overlapping tokens between chunks when splitting the input text. + + Returns: + `list[TokenClassificationOutputElement]`: List of [`TokenClassificationOutputElement`] items containing the entity group, confidence score, word, start and end index. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.token_classification("My name is Sarah Jessica Parker but you can call me Jessica") + [ + TokenClassificationOutputElement( + entity_group='PER', + score=0.9971321225166321, + word='Sarah Jessica Parker', + start=11, + end=31, + ), + TokenClassificationOutputElement( + entity_group='PER', + score=0.9773476123809814, + word='Jessica', + start=52, + end=59, + ) + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="token-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "aggregation_strategy": aggregation_strategy, + "ignore_labels": ignore_labels, + "stride": stride, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return TokenClassificationOutputElement.parse_obj_as_list(response) + + async def translation( + self, + text: str, + *, + model: str | None = None, + src_lang: str | None = None, + tgt_lang: str | None = None, + clean_up_tokenization_spaces: bool | None = None, + truncation: Optional["TranslationTruncationStrategy"] = None, + generate_parameters: dict[str, Any] | None = None, + ) -> TranslationOutput: + """ + Convert text from one language to another. + + Check out https://huggingface.co/tasks/translation for more information on how to choose the best model for + your specific use case. Source and target languages usually depend on the model. + However, it is possible to specify source and target languages for certain models. If you are working with one of these models, + you can use `src_lang` and `tgt_lang` arguments to pass the relevant information. + + Args: + text (`str`): + A string to be translated. + model (`str`, *optional*): + The model to use for the translation task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended translation model will be used. + Defaults to None. + src_lang (`str`, *optional*): + The source language of the text. Required for models that can translate from multiple languages. + tgt_lang (`str`, *optional*): + Target language to translate to. Required for models that can translate to multiple languages. + clean_up_tokenization_spaces (`bool`, *optional*): + Whether to clean up the potential extra spaces in the text output. + truncation (`"TranslationTruncationStrategy"`, *optional*): + The truncation strategy to use. + generate_parameters (`dict[str, Any]`, *optional*): + Additional parametrization of the text generation algorithm. + + Returns: + [`TranslationOutput`]: The generated translated text. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + `ValueError`: + If only one of the `src_lang` and `tgt_lang` arguments are provided. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.translation("My name is Wolfgang and I live in Berlin") + 'Mein Name ist Wolfgang und ich lebe in Berlin.' + >>> await client.translation("My name is Wolfgang and I live in Berlin", model="Helsinki-NLP/opus-mt-en-fr") + TranslationOutput(translation_text='Je m'appelle Wolfgang et je vis à Berlin.') + ``` + + Specifying languages: + ```py + >>> client.translation("My name is Sarah Jessica Parker but you can call me Jessica", model="facebook/mbart-large-50-many-to-many-mmt", src_lang="en_XX", tgt_lang="fr_XX") + "Mon nom est Sarah Jessica Parker mais vous pouvez m'appeler Jessica" + ``` + """ + # Throw error if only one of `src_lang` and `tgt_lang` was given + if src_lang is not None and tgt_lang is None: + raise ValueError("You cannot specify `src_lang` without specifying `tgt_lang`.") + + if src_lang is None and tgt_lang is not None: + raise ValueError("You cannot specify `tgt_lang` without specifying `src_lang`.") + + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="translation", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "src_lang": src_lang, + "tgt_lang": tgt_lang, + "clean_up_tokenization_spaces": clean_up_tokenization_spaces, + "truncation": truncation, + "generate_parameters": generate_parameters, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return TranslationOutput.parse_obj_as_list(response)[0] + + async def visual_question_answering( + self, + image: ContentT, + question: str, + *, + model: str | None = None, + top_k: int | None = None, + ) -> list[VisualQuestionAnsweringOutputElement]: + """ + Answering open-ended questions based on an image. + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The input image for the context. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + question (`str`): + Question to be answered. + model (`str`, *optional*): + The model to use for the visual question answering task. Can be a model ID hosted on the Hugging Face Hub or a URL to + a deployed Inference Endpoint. If not provided, the default recommended visual question answering model will be used. + Defaults to None. + top_k (`int`, *optional*): + The number of answers to return (will be chosen by order of likelihood). Note that we return less than + topk answers if there are not enough options available within the context. + Returns: + `list[VisualQuestionAnsweringOutputElement]`: a list of [`VisualQuestionAnsweringOutputElement`] items containing the predicted label and associated probability. + + Raises: + `InferenceTimeoutError`: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.visual_question_answering( + ... image="https://huggingface.co/datasets/mishig/sample_images/resolve/main/tiger.jpg", + ... question="What is the animal doing?" + ... ) + [ + VisualQuestionAnsweringOutputElement(score=0.778609573841095, answer='laying down'), + VisualQuestionAnsweringOutputElement(score=0.6957435607910156, answer='sitting'), + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="visual-question-answering", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={"top_k": top_k}, + headers=self.headers, + model=model_id, + api_key=self.token, + extra_payload={"question": question, "image": _b64_encode(image)}, + ) + response = await self._inner_post(request_parameters) + return VisualQuestionAnsweringOutputElement.parse_obj_as_list(response) + + async def zero_shot_classification( + self, + text: str, + candidate_labels: list[str], + *, + multi_label: bool | None = False, + hypothesis_template: str | None = None, + model: str | None = None, + ) -> list[ZeroShotClassificationOutputElement]: + """ + Provide as input a text and a set of candidate labels to classify the input text. + + Args: + text (`str`): + The input text to classify. + candidate_labels (`list[str]`): + The set of possible class labels to classify the text into. + labels (`list[str]`, *optional*): + (deprecated) List of strings. Each string is the verbalization of a possible label for the input text. + multi_label (`bool`, *optional*): + Whether multiple candidate labels can be true. If false, the scores are normalized such that the sum of + the label likelihoods for each sequence is 1. If true, the labels are considered independent and + probabilities are normalized for each candidate. + hypothesis_template (`str`, *optional*): + The sentence used in conjunction with `candidate_labels` to attempt the text classification by + replacing the placeholder with the candidate labels. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. If not provided, the default recommended zero-shot classification model will be used. + + + Returns: + `list[ZeroShotClassificationOutputElement]`: List of [`ZeroShotClassificationOutputElement`] items containing the predicted labels and their confidence. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example with `multi_label=False`: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> text = ( + ... "A new model offers an explanation for how the Galilean satellites formed around the solar system's" + ... "largest world. Konstantin Batygin did not set out to solve one of the solar system's most puzzling" + ... " mysteries when he went for a run up a hill in Nice, France." + ... ) + >>> labels = ["space & cosmos", "scientific discovery", "microbiology", "robots", "archeology"] + >>> await client.zero_shot_classification(text, labels) + [ + ZeroShotClassificationOutputElement(label='scientific discovery', score=0.7961668968200684), + ZeroShotClassificationOutputElement(label='space & cosmos', score=0.18570658564567566), + ZeroShotClassificationOutputElement(label='microbiology', score=0.00730885099619627), + ZeroShotClassificationOutputElement(label='archeology', score=0.006258360575884581), + ZeroShotClassificationOutputElement(label='robots', score=0.004559356719255447), + ] + >>> await client.zero_shot_classification(text, labels, multi_label=True) + [ + ZeroShotClassificationOutputElement(label='scientific discovery', score=0.9829297661781311), + ZeroShotClassificationOutputElement(label='space & cosmos', score=0.755190908908844), + ZeroShotClassificationOutputElement(label='microbiology', score=0.0005462635890580714), + ZeroShotClassificationOutputElement(label='archeology', score=0.00047131875180639327), + ZeroShotClassificationOutputElement(label='robots', score=0.00030448526376858354), + ] + ``` + + Example with `multi_label=True` and a custom `hypothesis_template`: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + >>> await client.zero_shot_classification( + ... text="I really like our dinner and I'm very happy. I don't like the weather though.", + ... labels=["positive", "negative", "pessimistic", "optimistic"], + ... multi_label=True, + ... hypothesis_template="This text is {} towards the weather" + ... ) + [ + ZeroShotClassificationOutputElement(label='negative', score=0.9231801629066467), + ZeroShotClassificationOutputElement(label='pessimistic', score=0.8760990500450134), + ZeroShotClassificationOutputElement(label='optimistic', score=0.0008674879791215062), + ZeroShotClassificationOutputElement(label='positive', score=0.0005250611575320363) + ] + ``` + """ + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="zero-shot-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=text, + parameters={ + "candidate_labels": candidate_labels, + "multi_label": multi_label, + "hypothesis_template": hypothesis_template, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + output = _bytes_to_dict(response) + return ZeroShotClassificationOutputElement.parse_obj_as_list(output) + + async def zero_shot_image_classification( + self, + image: ContentT, + candidate_labels: list[str], + *, + model: str | None = None, + hypothesis_template: str | None = None, + # deprecated argument + labels: list[str] = None, # type: ignore + ) -> list[ZeroShotImageClassificationOutputElement]: + """ + Provide input image and text labels to predict text labels for the image. + + Args: + image (`Union[str, Path, bytes, BinaryIO, PIL.Image.Image]`): + The input image to caption. It can be raw bytes, an image file, a URL to an online image, or a PIL Image. + candidate_labels (`list[str]`): + The candidate labels for this image + labels (`list[str]`, *optional*): + (deprecated) List of string possible labels. There must be at least 2 labels. + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. If not provided, the default recommended zero-shot image classification model will be used. + hypothesis_template (`str`, *optional*): + The sentence used in conjunction with `candidate_labels` to attempt the image classification by + replacing the placeholder with the candidate labels. + + Returns: + `list[ZeroShotImageClassificationOutputElement]`: List of [`ZeroShotImageClassificationOutputElement`] items containing the predicted labels and their confidence. + + Raises: + [`InferenceTimeoutError`]: + If the model is unavailable or the request times out. + [`HfHubHTTPError`]: + If the request fails with an HTTP error status code other than HTTP 503. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient() + + >>> await client.zero_shot_image_classification( + ... "https://upload.wikimedia.org/wikipedia/commons/thumb/4/43/Cute_dog.jpg/320px-Cute_dog.jpg", + ... labels=["dog", "cat", "horse"], + ... ) + [ZeroShotImageClassificationOutputElement(label='dog', score=0.956),...] + ``` + """ + # Raise ValueError if input is less than 2 labels + if len(candidate_labels) < 2: + raise ValueError("You must specify at least 2 classes to compare.") + + model_id = model or self.model + provider_helper = get_provider_helper(self.provider, task="zero-shot-image-classification", model=model_id) + request_parameters = provider_helper.prepare_request( + inputs=image, + parameters={ + "candidate_labels": candidate_labels, + "hypothesis_template": hypothesis_template, + }, + headers=self.headers, + model=model_id, + api_key=self.token, + ) + response = await self._inner_post(request_parameters) + return ZeroShotImageClassificationOutputElement.parse_obj_as_list(response) + + async def get_endpoint_info(self, *, model: str | None = None) -> dict[str, Any]: + """ + Get information about the deployed endpoint. + + This endpoint is only available on endpoints powered by Text-Generation-Inference (TGI) or Text-Embedding-Inference (TEI). + Endpoints powered by `transformers` return an empty payload. + + Args: + model (`str`, *optional*): + The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed + Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + + Returns: + `dict[str, Any]`: Information about the endpoint. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient("meta-llama/Meta-Llama-3-70B-Instruct") + >>> await client.get_endpoint_info() + { + 'model_id': 'meta-llama/Meta-Llama-3-70B-Instruct', + 'model_sha': None, + 'model_dtype': 'torch.float16', + 'model_device_type': 'cuda', + 'model_pipeline_tag': None, + 'max_concurrent_requests': 128, + 'max_best_of': 2, + 'max_stop_sequences': 4, + 'max_input_length': 8191, + 'max_total_tokens': 8192, + 'waiting_served_ratio': 0.3, + 'max_batch_total_tokens': 1259392, + 'max_waiting_tokens': 20, + 'max_batch_size': None, + 'validation_workers': 32, + 'max_client_batch_size': 4, + 'version': '2.0.2', + 'sha': 'dccab72549635c7eb5ddb17f43f0b7cdff07c214', + 'docker_label': 'sha-dccab72' + } + ``` + """ + if self.provider != "hf-inference": + raise ValueError(f"Getting endpoint info is not supported on '{self.provider}'.") + + model = model or self.model + if model is None: + raise ValueError("Model id not provided.") + if model.startswith(("http://", "https://")): + url = model.rstrip("/") + "/info" + else: + url = f"{constants.INFERENCE_ENDPOINT}/models/{model}/info" + + client = await self._get_async_client() + response = await client.get(url, headers=build_hf_headers(token=self.token)) + hf_raise_for_status(response) + return response.json() + + async def health_check(self, model: str | None = None) -> bool: + """ + Check the health of the deployed endpoint. + + Health check is only available with Inference Endpoints powered by Text-Generation-Inference (TGI) or Text-Embedding-Inference (TEI). + + Args: + model (`str`, *optional*): + URL of the Inference Endpoint. This parameter overrides the model defined at the instance level. Defaults to None. + + Returns: + `bool`: True if everything is working fine. + + Example: + ```py + # Must be run in an async context + >>> from huggingface_hub import AsyncInferenceClient + >>> client = AsyncInferenceClient("https://jzgu0buei5.us-east-1.aws.endpoints.huggingface.cloud") + >>> await client.health_check() + True + ``` + """ + if self.provider != "hf-inference": + raise ValueError(f"Health check is not supported on '{self.provider}'.") + + model = model or self.model + if model is None: + raise ValueError("Model id not provided.") + if not model.startswith(("http://", "https://")): + raise ValueError("Model must be an Inference Endpoint URL.") + url = model.rstrip("/") + "/health" + + client = await self._get_async_client() + response = await client.get(url, headers=build_hf_headers(token=self.token)) + return response.status_code == 200 + + @property + def chat(self) -> "ProxyClientChat": + return ProxyClientChat(self) + + +class _ProxyClient: + """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" + + def __init__(self, client: AsyncInferenceClient): + self._client = client + + +class ProxyClientChat(_ProxyClient): + """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" + + @property + def completions(self) -> "ProxyClientChatCompletions": + return ProxyClientChatCompletions(self._client) + + +class ProxyClientChatCompletions(_ProxyClient): + """Proxy class to be able to call `client.chat.completion.create(...)` as OpenAI client.""" + + @property + def create(self): + return self._client.chat_completion diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__init__.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9f95dca555d80774ec863bf26d147dd43f15aeb6 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__init__.py @@ -0,0 +1,204 @@ +# This file is auto-generated by `utils/generate_inference_types.py`. +# Do not modify it manually. +# +# ruff: noqa: F401 + +from .audio_classification import ( + AudioClassificationInput, + AudioClassificationOutputElement, + AudioClassificationOutputTransform, + AudioClassificationParameters, +) +from .audio_to_audio import AudioToAudioInput, AudioToAudioOutputElement +from .automatic_speech_recognition import ( + AutomaticSpeechRecognitionEarlyStoppingEnum, + AutomaticSpeechRecognitionGenerationParameters, + AutomaticSpeechRecognitionInput, + AutomaticSpeechRecognitionOutput, + AutomaticSpeechRecognitionOutputChunk, + AutomaticSpeechRecognitionParameters, +) +from .base import BaseInferenceType +from .chat_completion import ( + ChatCompletionInput, + ChatCompletionInputFunctionDefinition, + ChatCompletionInputFunctionName, + ChatCompletionInputGrammarType, + ChatCompletionInputJSONSchema, + ChatCompletionInputMessage, + ChatCompletionInputMessageChunk, + ChatCompletionInputMessageChunkType, + ChatCompletionInputResponseFormatJSONObject, + ChatCompletionInputResponseFormatJSONSchema, + ChatCompletionInputResponseFormatText, + ChatCompletionInputStreamOptions, + ChatCompletionInputTool, + ChatCompletionInputToolCall, + ChatCompletionInputToolChoiceClass, + ChatCompletionInputToolChoiceEnum, + ChatCompletionInputURL, + ChatCompletionOutput, + ChatCompletionOutputComplete, + ChatCompletionOutputFunctionDefinition, + ChatCompletionOutputLogprob, + ChatCompletionOutputLogprobs, + ChatCompletionOutputMessage, + ChatCompletionOutputToolCall, + ChatCompletionOutputTopLogprob, + ChatCompletionOutputUsage, + ChatCompletionStreamOutput, + ChatCompletionStreamOutputChoice, + ChatCompletionStreamOutputDelta, + ChatCompletionStreamOutputDeltaToolCall, + ChatCompletionStreamOutputFunction, + ChatCompletionStreamOutputLogprob, + ChatCompletionStreamOutputLogprobs, + ChatCompletionStreamOutputTopLogprob, + ChatCompletionStreamOutputUsage, +) +from .depth_estimation import DepthEstimationInput, DepthEstimationOutput +from .document_question_answering import ( + DocumentQuestionAnsweringInput, + DocumentQuestionAnsweringInputData, + DocumentQuestionAnsweringOutputElement, + DocumentQuestionAnsweringParameters, +) +from .feature_extraction import FeatureExtractionInput, FeatureExtractionInputTruncationDirection +from .fill_mask import FillMaskInput, FillMaskOutputElement, FillMaskParameters +from .image_classification import ( + ImageClassificationInput, + ImageClassificationOutputElement, + ImageClassificationOutputTransform, + ImageClassificationParameters, +) +from .image_segmentation import ( + ImageSegmentationInput, + ImageSegmentationOutputElement, + ImageSegmentationParameters, + ImageSegmentationSubtask, +) +from .image_text_to_image import ( + ImageTextToImageInput, + ImageTextToImageOutput, + ImageTextToImageParameters, + ImageTextToImageTargetSize, +) +from .image_text_to_video import ( + ImageTextToVideoInput, + ImageTextToVideoOutput, + ImageTextToVideoParameters, + ImageTextToVideoTargetSize, +) +from .image_to_image import ImageToImageInput, ImageToImageOutput, ImageToImageParameters, ImageToImageTargetSize +from .image_to_text import ( + ImageToTextEarlyStoppingEnum, + ImageToTextGenerationParameters, + ImageToTextInput, + ImageToTextOutput, + ImageToTextParameters, +) +from .image_to_video import ImageToVideoInput, ImageToVideoOutput, ImageToVideoParameters, ImageToVideoTargetSize +from .object_detection import ( + ObjectDetectionBoundingBox, + ObjectDetectionInput, + ObjectDetectionOutputElement, + ObjectDetectionParameters, +) +from .question_answering import ( + QuestionAnsweringInput, + QuestionAnsweringInputData, + QuestionAnsweringOutputElement, + QuestionAnsweringParameters, +) +from .sentence_similarity import SentenceSimilarityInput, SentenceSimilarityInputData +from .summarization import ( + SummarizationInput, + SummarizationOutput, + SummarizationParameters, + SummarizationTruncationStrategy, +) +from .table_question_answering import ( + Padding, + TableQuestionAnsweringInput, + TableQuestionAnsweringInputData, + TableQuestionAnsweringOutputElement, + TableQuestionAnsweringParameters, +) +from .text2text_generation import ( + Text2TextGenerationInput, + Text2TextGenerationOutput, + Text2TextGenerationParameters, + Text2TextGenerationTruncationStrategy, +) +from .text_classification import ( + TextClassificationInput, + TextClassificationOutputElement, + TextClassificationOutputTransform, + TextClassificationParameters, +) +from .text_generation import ( + TextGenerationInput, + TextGenerationInputGenerateParameters, + TextGenerationInputGrammarType, + TextGenerationOutput, + TextGenerationOutputBestOfSequence, + TextGenerationOutputDetails, + TextGenerationOutputFinishReason, + TextGenerationOutputPrefillToken, + TextGenerationOutputToken, + TextGenerationStreamOutput, + TextGenerationStreamOutputStreamDetails, + TextGenerationStreamOutputToken, + TypeEnum, +) +from .text_to_audio import ( + TextToAudioEarlyStoppingEnum, + TextToAudioGenerationParameters, + TextToAudioInput, + TextToAudioOutput, + TextToAudioParameters, +) +from .text_to_image import TextToImageInput, TextToImageOutput, TextToImageParameters +from .text_to_speech import ( + TextToSpeechEarlyStoppingEnum, + TextToSpeechGenerationParameters, + TextToSpeechInput, + TextToSpeechOutput, + TextToSpeechParameters, +) +from .text_to_video import TextToVideoInput, TextToVideoOutput, TextToVideoParameters +from .token_classification import ( + TokenClassificationAggregationStrategy, + TokenClassificationInput, + TokenClassificationOutputElement, + TokenClassificationParameters, +) +from .translation import TranslationInput, TranslationOutput, TranslationParameters, TranslationTruncationStrategy +from .video_classification import ( + VideoClassificationInput, + VideoClassificationOutputElement, + VideoClassificationOutputTransform, + VideoClassificationParameters, +) +from .visual_question_answering import ( + VisualQuestionAnsweringInput, + VisualQuestionAnsweringInputData, + VisualQuestionAnsweringOutputElement, + VisualQuestionAnsweringParameters, +) +from .zero_shot_classification import ( + ZeroShotClassificationInput, + ZeroShotClassificationOutputElement, + ZeroShotClassificationParameters, +) +from .zero_shot_image_classification import ( + ZeroShotImageClassificationInput, + ZeroShotImageClassificationOutputElement, + ZeroShotImageClassificationParameters, +) +from .zero_shot_object_detection import ( + ZeroShotObjectDetectionBoundingBox, + ZeroShotObjectDetectionInput, + ZeroShotObjectDetectionOutputElement, + ZeroShotObjectDetectionParameters, +) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9e480a36a827ad62a1b50b8680ec7a3ab40f795f Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/audio_classification.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/audio_classification.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..27adad18e7b4dbfc737f76f908c0e4dc400c67fb Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/audio_classification.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/audio_to_audio.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/audio_to_audio.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77008bc8ed49dfcf1a83b24aea60198bc736b926 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/audio_to_audio.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/automatic_speech_recognition.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/automatic_speech_recognition.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f72f159b7e1ab7be9c3134ce911235ecd077145 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/automatic_speech_recognition.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/base.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/base.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3569f9514021505542b0e9121b09d1ad6aebef9f Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/base.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/chat_completion.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/chat_completion.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3cbb4390a4f9944641ad42c2ac03267109e96e54 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/chat_completion.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/depth_estimation.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/depth_estimation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..149512adb53f226ca40fbb41872487e084d2f339 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/depth_estimation.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/document_question_answering.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/document_question_answering.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..193f6ec1ad153ee92499c03199c9dda85b08a24f Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/document_question_answering.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/feature_extraction.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/feature_extraction.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e484cb640de0ab09696aeb21ec0fabe52e3f90ae Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/feature_extraction.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/fill_mask.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/fill_mask.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7ce31ad9e7ee3bd38c0edab5a7c682a882b810f2 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/fill_mask.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_classification.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_classification.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..07a10c061e66f149f708ff8363c991bc1e0d4ee0 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_classification.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_segmentation.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_segmentation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..776d2720b3dfc8d183636533f510b60e04449c02 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_segmentation.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_text_to_image.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_text_to_image.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4e1bef28c68f01b11193e19ee109018380c67ae5 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_text_to_image.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_text_to_video.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_text_to_video.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be25ab74dbc9ef38dcd1b92a3a21c78a709e59c1 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_text_to_video.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_to_image.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_to_image.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a8f8818e436d2abf5ce3e9c13358c793615780b Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_to_image.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_to_text.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_to_text.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..276509013934eefe8eb02a7e43e4943e448228d7 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_to_text.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_to_video.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_to_video.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0fa21ad44fd42d8e00eeef47ef7502dca8e52964 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/image_to_video.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/object_detection.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/object_detection.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d7baed119edb646ffd2f49cd57d6a47717833978 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/object_detection.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/question_answering.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/question_answering.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c3a70291702d17c646f495fa295a9c2519994d18 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/question_answering.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/sentence_similarity.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/sentence_similarity.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7cc62fc86fcf5b67316ac7f05511ef9ec1af287b Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/sentence_similarity.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/summarization.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/summarization.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..746b26668b048dcb4d16f78246c4787b564437ed Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/summarization.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/table_question_answering.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/table_question_answering.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b8f0968d50d9fad8c94ba9a67b415db39514bae Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/table_question_answering.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text2text_generation.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text2text_generation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6407a1cf5cca75c4b8f58382833a56040c29922 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text2text_generation.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_classification.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_classification.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..574916d8d4dffa2fc605d5533f06ba9abe1e1c70 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_classification.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_generation.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_generation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9f27178f998f62bca530af5e955933850282e6d Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_generation.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_audio.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_audio.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7ea144b70fcba6f11776dfee3268fe3096ccbbdd Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_audio.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_image.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_image.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24d2d4aef926ba5531dbc8253bd0f9b183adcff9 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_image.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_speech.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_speech.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e62b5f106abe64dc4cad87a157075c1e6fe99df5 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_speech.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_video.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_video.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b6af2a14573113edb41ac9995b8533c90793845 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/text_to_video.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/token_classification.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/token_classification.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..09778bae4eac4dc929c982fe9e9addbb3285aa20 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/token_classification.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/translation.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/translation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d3f466d5257785c8559d11120015ed647e351da7 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/translation.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/video_classification.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/video_classification.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d8cab85c664de81bdfc3ba28d8b548a1919a05e Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/video_classification.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/visual_question_answering.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/visual_question_answering.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61a19a6eb848fbba48995f8110c70de072492ef0 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/visual_question_answering.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/zero_shot_classification.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/zero_shot_classification.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c26f0e6a8e04def56c140eae302eef1bb305ac23 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/zero_shot_classification.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/zero_shot_image_classification.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/zero_shot_image_classification.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d6ea9cd9e1c1e31eee15ff51e735af965727863 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/zero_shot_image_classification.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/zero_shot_object_detection.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/zero_shot_object_detection.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5f11038dbb4bcee555a620e2ef0194217450faf7 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__pycache__/zero_shot_object_detection.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_classification.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..b99507c4a7caaeaa306a5b20348c36eb2099b2e2 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_classification.py @@ -0,0 +1,43 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +AudioClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] + + +@dataclass_with_extra +class AudioClassificationParameters(BaseInferenceType): + """Additional inference parameters for Audio Classification""" + + function_to_apply: Optional["AudioClassificationOutputTransform"] = None + """The function to apply to the model outputs in order to retrieve the scores.""" + top_k: int | None = None + """When specified, limits the output to the top K most probable classes.""" + + +@dataclass_with_extra +class AudioClassificationInput(BaseInferenceType): + """Inputs for Audio Classification inference""" + + inputs: str + """The input audio data as a base64-encoded string. If no `parameters` are provided, you can + also provide the audio data as a raw bytes payload. + """ + parameters: AudioClassificationParameters | None = None + """Additional inference parameters for Audio Classification""" + + +@dataclass_with_extra +class AudioClassificationOutputElement(BaseInferenceType): + """Outputs for Audio Classification inference""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_to_audio.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_to_audio.py new file mode 100644 index 0000000000000000000000000000000000000000..43f376b5345fab6b854b028d1c17416c020d7bc1 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_to_audio.py @@ -0,0 +1,30 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class AudioToAudioInput(BaseInferenceType): + """Inputs for Audio to Audio inference""" + + inputs: Any + """The input audio data""" + + +@dataclass_with_extra +class AudioToAudioOutputElement(BaseInferenceType): + """Outputs of inference for the Audio To Audio task + A generated audio file with its label. + """ + + blob: Any + """The generated audio file.""" + content_type: str + """The content type of audio file.""" + label: str + """The label of the audio file.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/automatic_speech_recognition.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/automatic_speech_recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..9d728bfdb83377c6726fce804013a33fdb2fbdbd --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/automatic_speech_recognition.py @@ -0,0 +1,113 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +AutomaticSpeechRecognitionEarlyStoppingEnum = Literal["never"] + + +@dataclass_with_extra +class AutomaticSpeechRecognitionGenerationParameters(BaseInferenceType): + """Parametrization of the text generation process""" + + do_sample: bool | None = None + """Whether to use sampling instead of greedy decoding when generating new tokens.""" + early_stopping: Union[bool, "AutomaticSpeechRecognitionEarlyStoppingEnum"] | None = None + """Controls the stopping condition for beam-based methods.""" + epsilon_cutoff: float | None = None + """If set to float strictly between 0 and 1, only tokens with a conditional probability + greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + """ + eta_cutoff: float | None = None + """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + float strictly between 0 and 1, a token is only considered if it is greater than either + eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + for more details. + """ + max_length: int | None = None + """The maximum length (in tokens) of the generated text, including the input.""" + max_new_tokens: int | None = None + """The maximum number of tokens to generate. Takes precedence over max_length.""" + min_length: int | None = None + """The minimum length (in tokens) of the generated text, including the input.""" + min_new_tokens: int | None = None + """The minimum number of tokens to generate. Takes precedence over min_length.""" + num_beam_groups: int | None = None + """Number of groups to divide num_beams into in order to ensure diversity among different + groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + """ + num_beams: int | None = None + """Number of beams to use for beam search.""" + penalty_alpha: float | None = None + """The value balances the model confidence and the degeneration penalty in contrastive + search decoding. + """ + temperature: float | None = None + """The value used to modulate the next token probabilities.""" + top_k: int | None = None + """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" + top_p: float | None = None + """If set to float < 1, only the smallest set of most probable tokens with probabilities + that add up to top_p or higher are kept for generation. + """ + typical_p: float | None = None + """Local typicality measures how similar the conditional probability of predicting a target + token next is to the expected conditional probability of predicting a random token next, + given the partial text already generated. If set to float < 1, the smallest set of the + most locally typical tokens with probabilities that add up to typical_p or higher are + kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + """ + use_cache: bool | None = None + """Whether the model should use the past last key/values attentions to speed up decoding""" + + +@dataclass_with_extra +class AutomaticSpeechRecognitionParameters(BaseInferenceType): + """Additional inference parameters for Automatic Speech Recognition""" + + generation_parameters: AutomaticSpeechRecognitionGenerationParameters | None = None + """Parametrization of the text generation process""" + return_timestamps: bool | None = None + """Whether to output corresponding timestamps with the generated text""" + + +@dataclass_with_extra +class AutomaticSpeechRecognitionInput(BaseInferenceType): + """Inputs for Automatic Speech Recognition inference""" + + inputs: str + """The input audio data as a base64-encoded string. If no `parameters` are provided, you can + also provide the audio data as a raw bytes payload. + """ + parameters: AutomaticSpeechRecognitionParameters | None = None + """Additional inference parameters for Automatic Speech Recognition""" + + +@dataclass_with_extra +class AutomaticSpeechRecognitionOutputChunk(BaseInferenceType): + text: str + """A chunk of text identified by the model""" + timestamp: list[float] + """The start and end timestamps corresponding with the text""" + + +@dataclass_with_extra +class AutomaticSpeechRecognitionOutput(BaseInferenceType): + """Outputs of inference for the Automatic Speech Recognition task""" + + text: str + """The recognized text.""" + chunks: list[AutomaticSpeechRecognitionOutputChunk] | None = None + """When returnTimestamps is enabled, chunks contains a list of audio chunks identified by + the model. + """ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/base.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/base.py new file mode 100644 index 0000000000000000000000000000000000000000..ad4016f2d57d11e13dad44c161d3bce3ef741d50 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/base.py @@ -0,0 +1,167 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains a base class for all inference types.""" + +import inspect +import json +import types +from dataclasses import asdict, dataclass +from typing import Any, TypeVar, get_args + +from typing_extensions import dataclass_transform + + +T = TypeVar("T", bound="BaseInferenceType") + + +def _repr_with_extra(self): + fields = list(self.__dataclass_fields__.keys()) + other_fields = list(k for k in self.__dict__ if k not in fields) + return f"{self.__class__.__name__}({', '.join(f'{k}={self.__dict__[k]!r}' for k in fields + other_fields)})" + + +@dataclass_transform() +def dataclass_with_extra(cls: type[T]) -> type[T]: + """Decorator to add a custom __repr__ method to a dataclass, showing all fields, including extra ones. + + This decorator only works with dataclasses that inherit from `BaseInferenceType`. + """ + cls = dataclass(cls) + cls.__repr__ = _repr_with_extra # type: ignore[method-assign] + return cls + + +@dataclass +class BaseInferenceType(dict): + """Base class for all inference types. + + Object is a dataclass and a dict for backward compatibility but plan is to remove the dict part in the future. + + Handle parsing from dict, list and json strings in a permissive way to ensure future-compatibility (e.g. all fields + are made optional, and non-expected fields are added as dict attributes). + """ + + @classmethod + def parse_obj_as_list(cls: type[T], data: bytes | str | list | dict) -> list[T]: + """Alias to parse server response and return a single instance. + + See `parse_obj` for more details. + """ + output = cls.parse_obj(data) + if not isinstance(output, list): + raise ValueError(f"Invalid input data for {cls}. Expected a list, but got {type(output)}.") + return output + + @classmethod + def parse_obj_as_instance(cls: type[T], data: bytes | str | list | dict) -> T: + """Alias to parse server response and return a single instance. + + See `parse_obj` for more details. + """ + output = cls.parse_obj(data) + if isinstance(output, list): + raise ValueError(f"Invalid input data for {cls}. Expected a single instance, but got a list.") + return output + + @classmethod + def parse_obj(cls: type[T], data: bytes | str | list | dict) -> list[T] | T: + """Parse server response as a dataclass or list of dataclasses. + + To enable future-compatibility, we want to handle cases where the server return more fields than expected. + In such cases, we don't want to raise an error but still create the dataclass object. Remaining fields are + added as dict attributes. + """ + # Parse server response (from bytes) + if isinstance(data, bytes): + data = data.decode() + if isinstance(data, str): + data = json.loads(data) + + # If a list, parse each item individually + if isinstance(data, list): + return [cls.parse_obj(d) for d in data] # type: ignore + + # At this point, we expect a dict + if not isinstance(data, dict): + raise ValueError(f"Invalid data type: {type(data)}") + + init_values = {} + other_values = {} + for key, value in data.items(): + key = normalize_key(key) + if key in cls.__dataclass_fields__ and cls.__dataclass_fields__[key].init: + if isinstance(value, dict) or isinstance(value, list): + field_type = cls.__dataclass_fields__[key].type + + # if `field_type` is a `BaseInferenceType`, parse it + if inspect.isclass(field_type) and issubclass(field_type, BaseInferenceType): + value = field_type.parse_obj(value) + + # otherwise, recursively parse nested dataclasses (if possible) + # `get_args` returns handle Union and Optional for us + else: + expected_types = get_args(field_type) + for expected_type in expected_types: + if ( + isinstance(expected_type, types.GenericAlias) and expected_type.__origin__ is list + ) or getattr(expected_type, "_name", None) == "List": + expected_type = get_args(expected_type)[ + 0 + ] # assume same type for all items in the list + if inspect.isclass(expected_type) and issubclass(expected_type, BaseInferenceType): + value = expected_type.parse_obj(value) + break + init_values[key] = value + else: + other_values[key] = value + + # Make all missing fields default to None + # => ensure that dataclass initialization will never fail even if the server does not return all fields. + for key in cls.__dataclass_fields__: + if key not in init_values: + init_values[key] = None + + # Initialize dataclass with expected values + item = cls(**init_values) + + # Add remaining fields as dict attributes + item.update(other_values) + + # Add remaining fields as extra dataclass fields. + # They won't be part of the dataclass fields but will be accessible as attributes. + # Use @dataclass_with_extra to show them in __repr__. + item.__dict__.update(other_values) + return item + + def __post_init__(self): + self.update(asdict(self)) + + def __setitem__(self, __key: Any, __value: Any) -> None: + # Hacky way to keep dataclass values in sync when dict is updated + super().__setitem__(__key, __value) + if __key in self.__dataclass_fields__ and getattr(self, __key, None) != __value: + self.__setattr__(__key, __value) + return + + def __setattr__(self, __name: str, __value: Any) -> None: + # Hacky way to keep dict values is sync when dataclass is updated + super().__setattr__(__name, __value) + if self.get(__name) != __value: + self[__name] = __value + return + + +def normalize_key(key: str) -> str: + # e.g "content-type" -> "content_type", "Accept" -> "accept" + return key.replace("-", "_").replace(" ", "_").lower() diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/chat_completion.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/chat_completion.py new file mode 100644 index 0000000000000000000000000000000000000000..3a5d69ef70ed25d0ed0731c6a6b536e3130cdd73 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/chat_completion.py @@ -0,0 +1,347 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ChatCompletionInputURL(BaseInferenceType): + url: str + + +ChatCompletionInputMessageChunkType = Literal["text", "image_url"] + + +@dataclass_with_extra +class ChatCompletionInputMessageChunk(BaseInferenceType): + type: "ChatCompletionInputMessageChunkType" + image_url: ChatCompletionInputURL | None = None + text: str | None = None + + +@dataclass_with_extra +class ChatCompletionInputFunctionDefinition(BaseInferenceType): + name: str + parameters: Any + description: str | None = None + + +@dataclass_with_extra +class ChatCompletionInputToolCall(BaseInferenceType): + function: ChatCompletionInputFunctionDefinition + id: str + type: str + + +@dataclass_with_extra +class ChatCompletionInputMessage(BaseInferenceType): + role: str + content: list[ChatCompletionInputMessageChunk] | str | None = None + name: str | None = None + tool_calls: list[ChatCompletionInputToolCall] | None = None + + +@dataclass_with_extra +class ChatCompletionInputJSONSchema(BaseInferenceType): + name: str + """ + The name of the response format. + """ + description: str | None = None + """ + A description of what the response format is for, used by the model to determine + how to respond in the format. + """ + schema: dict[str, object] | None = None + """ + The schema for the response format, described as a JSON Schema object. Learn how + to build JSON schemas [here](https://json-schema.org/). + """ + strict: bool | None = None + """ + Whether to enable strict schema adherence when generating the output. If set to + true, the model will always follow the exact schema defined in the `schema` + field. + """ + + +@dataclass_with_extra +class ChatCompletionInputResponseFormatText(BaseInferenceType): + type: Literal["text"] + + +@dataclass_with_extra +class ChatCompletionInputResponseFormatJSONSchema(BaseInferenceType): + type: Literal["json_schema"] + json_schema: ChatCompletionInputJSONSchema + + +@dataclass_with_extra +class ChatCompletionInputResponseFormatJSONObject(BaseInferenceType): + type: Literal["json_object"] + + +ChatCompletionInputGrammarType = Union[ + ChatCompletionInputResponseFormatText, + ChatCompletionInputResponseFormatJSONSchema, + ChatCompletionInputResponseFormatJSONObject, +] + + +@dataclass_with_extra +class ChatCompletionInputStreamOptions(BaseInferenceType): + include_usage: bool | None = None + """If set, an additional chunk will be streamed before the data: [DONE] message. The usage + field on this chunk shows the token usage statistics for the entire request, and the + choices field will always be an empty array. All other chunks will also include a usage + field, but with a null value. + """ + + +@dataclass_with_extra +class ChatCompletionInputFunctionName(BaseInferenceType): + name: str + + +@dataclass_with_extra +class ChatCompletionInputToolChoiceClass(BaseInferenceType): + function: ChatCompletionInputFunctionName + + +ChatCompletionInputToolChoiceEnum = Literal["auto", "none", "required"] + + +@dataclass_with_extra +class ChatCompletionInputTool(BaseInferenceType): + function: ChatCompletionInputFunctionDefinition + type: str + + +@dataclass_with_extra +class ChatCompletionInput(BaseInferenceType): + """Chat Completion Input. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + messages: list[ChatCompletionInputMessage] + """A list of messages comprising the conversation so far.""" + frequency_penalty: float | None = None + """Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing + frequency in the text so far, + decreasing the model's likelihood to repeat the same line verbatim. + """ + logit_bias: list[float] | None = None + """UNUSED + Modify the likelihood of specified tokens appearing in the completion. Accepts a JSON + object that maps tokens + (specified by their token ID in the tokenizer) to an associated bias value from -100 to + 100. Mathematically, + the bias is added to the logits generated by the model prior to sampling. The exact + effect will vary per model, + but values between -1 and 1 should decrease or increase likelihood of selection; values + like -100 or 100 should + result in a ban or exclusive selection of the relevant token. + """ + logprobs: bool | None = None + """Whether to return log probabilities of the output tokens or not. If true, returns the log + probabilities of each + output token returned in the content of message. + """ + max_tokens: int | None = None + """The maximum number of tokens that can be generated in the chat completion.""" + model: str | None = None + """[UNUSED] ID of the model to use. See the model endpoint compatibility table for details + on which models work with the Chat API. + """ + n: int | None = None + """UNUSED + How many chat completion choices to generate for each input message. Note that you will + be charged based on the + number of generated tokens across all of the choices. Keep n as 1 to minimize costs. + """ + presence_penalty: float | None = None + """Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they + appear in the text so far, + increasing the model's likelihood to talk about new topics + """ + response_format: ChatCompletionInputGrammarType | None = None + seed: int | None = None + stop: list[str] | None = None + """Up to 4 sequences where the API will stop generating further tokens.""" + stream: bool | None = None + stream_options: ChatCompletionInputStreamOptions | None = None + temperature: float | None = None + """What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the + output more random, while + lower values like 0.2 will make it more focused and deterministic. + We generally recommend altering this or `top_p` but not both. + """ + tool_choice: Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"] | None = None + tool_prompt: str | None = None + """A prompt to be appended before the tools""" + tools: list[ChatCompletionInputTool] | None = None + """A list of tools the model may call. Currently, only functions are supported as a tool. + Use this to provide a list of + functions the model may generate JSON inputs for. + """ + top_logprobs: int | None = None + """An integer between 0 and 5 specifying the number of most likely tokens to return at each + token position, each with + an associated log probability. logprobs must be set to true if this parameter is used. + """ + top_p: float | None = None + """An alternative to sampling with temperature, called nucleus sampling, where the model + considers the results of the + tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% + probability mass are considered. + """ + + +@dataclass_with_extra +class ChatCompletionOutputTopLogprob(BaseInferenceType): + logprob: float + token: str + + +@dataclass_with_extra +class ChatCompletionOutputLogprob(BaseInferenceType): + logprob: float + token: str + top_logprobs: list[ChatCompletionOutputTopLogprob] + + +@dataclass_with_extra +class ChatCompletionOutputLogprobs(BaseInferenceType): + content: list[ChatCompletionOutputLogprob] + + +@dataclass_with_extra +class ChatCompletionOutputFunctionDefinition(BaseInferenceType): + arguments: str + name: str + description: str | None = None + + +@dataclass_with_extra +class ChatCompletionOutputToolCall(BaseInferenceType): + function: ChatCompletionOutputFunctionDefinition + id: str + type: str + + +@dataclass_with_extra +class ChatCompletionOutputMessage(BaseInferenceType): + role: str + content: str | None = None + reasoning: str | None = None + tool_call_id: str | None = None + tool_calls: list[ChatCompletionOutputToolCall] | None = None + + +@dataclass_with_extra +class ChatCompletionOutputComplete(BaseInferenceType): + finish_reason: str + index: int + message: ChatCompletionOutputMessage + logprobs: ChatCompletionOutputLogprobs | None = None + + +@dataclass_with_extra +class ChatCompletionOutputUsage(BaseInferenceType): + completion_tokens: int + prompt_tokens: int + total_tokens: int + + +@dataclass_with_extra +class ChatCompletionOutput(BaseInferenceType): + """Chat Completion Output. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + choices: list[ChatCompletionOutputComplete] + created: int + id: str + model: str + system_fingerprint: str + usage: ChatCompletionOutputUsage + + +@dataclass_with_extra +class ChatCompletionStreamOutputFunction(BaseInferenceType): + arguments: str + name: str | None = None + + +@dataclass_with_extra +class ChatCompletionStreamOutputDeltaToolCall(BaseInferenceType): + function: ChatCompletionStreamOutputFunction + id: str + index: int + type: str + + +@dataclass_with_extra +class ChatCompletionStreamOutputDelta(BaseInferenceType): + role: str + content: str | None = None + reasoning: str | None = None + tool_call_id: str | None = None + tool_calls: list[ChatCompletionStreamOutputDeltaToolCall] | None = None + + +@dataclass_with_extra +class ChatCompletionStreamOutputTopLogprob(BaseInferenceType): + logprob: float + token: str + + +@dataclass_with_extra +class ChatCompletionStreamOutputLogprob(BaseInferenceType): + logprob: float + token: str + top_logprobs: list[ChatCompletionStreamOutputTopLogprob] + + +@dataclass_with_extra +class ChatCompletionStreamOutputLogprobs(BaseInferenceType): + content: list[ChatCompletionStreamOutputLogprob] + + +@dataclass_with_extra +class ChatCompletionStreamOutputChoice(BaseInferenceType): + delta: ChatCompletionStreamOutputDelta + index: int + finish_reason: str | None = None + logprobs: ChatCompletionStreamOutputLogprobs | None = None + + +@dataclass_with_extra +class ChatCompletionStreamOutputUsage(BaseInferenceType): + completion_tokens: int + prompt_tokens: int + total_tokens: int + + +@dataclass_with_extra +class ChatCompletionStreamOutput(BaseInferenceType): + """Chat Completion Stream Output. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + choices: list[ChatCompletionStreamOutputChoice] + created: int + id: str + model: str + system_fingerprint: str + usage: ChatCompletionStreamOutputUsage | None = None diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/depth_estimation.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/depth_estimation.py new file mode 100644 index 0000000000000000000000000000000000000000..cf26998ed518acd9e0f16d2f0f0de264a941840c --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/depth_estimation.py @@ -0,0 +1,28 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class DepthEstimationInput(BaseInferenceType): + """Inputs for Depth Estimation inference""" + + inputs: Any + """The input image data""" + parameters: dict[str, Any] | None = None + """Additional inference parameters for Depth Estimation""" + + +@dataclass_with_extra +class DepthEstimationOutput(BaseInferenceType): + """Outputs of inference for the Depth Estimation task""" + + depth: Any + """The predicted depth as an image""" + predicted_depth: Any + """The predicted depth as a tensor""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/document_question_answering.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/document_question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..0ec9c29e306671a389321e332b9bc197e77e3c65 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/document_question_answering.py @@ -0,0 +1,80 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class DocumentQuestionAnsweringInputData(BaseInferenceType): + """One (document, question) pair to answer""" + + image: Any + """The image on which the question is asked""" + question: str + """A question to ask of the document""" + + +@dataclass_with_extra +class DocumentQuestionAnsweringParameters(BaseInferenceType): + """Additional inference parameters for Document Question Answering""" + + doc_stride: int | None = None + """If the words in the document are too long to fit with the question for the model, it will + be split in several chunks with some overlap. This argument controls the size of that + overlap. + """ + handle_impossible_answer: bool | None = None + """Whether to accept impossible as an answer""" + lang: str | None = None + """Language to use while running OCR. Defaults to english.""" + max_answer_len: int | None = None + """The maximum length of predicted answers (e.g., only answers with a shorter length are + considered). + """ + max_question_len: int | None = None + """The maximum length of the question after tokenization. It will be truncated if needed.""" + max_seq_len: int | None = None + """The maximum length of the total sentence (context + question) in tokens of each chunk + passed to the model. The context will be split in several chunks (using doc_stride as + overlap) if needed. + """ + top_k: int | None = None + """The number of answers to return (will be chosen by order of likelihood). Can return less + than top_k answers if there are not enough options available within the context. + """ + word_boxes: list[list[float] | str] | None = None + """A list of words and bounding boxes (normalized 0->1000). If provided, the inference will + skip the OCR step and use the provided bounding boxes instead. + """ + + +@dataclass_with_extra +class DocumentQuestionAnsweringInput(BaseInferenceType): + """Inputs for Document Question Answering inference""" + + inputs: DocumentQuestionAnsweringInputData + """One (document, question) pair to answer""" + parameters: DocumentQuestionAnsweringParameters | None = None + """Additional inference parameters for Document Question Answering""" + + +@dataclass_with_extra +class DocumentQuestionAnsweringOutputElement(BaseInferenceType): + """Outputs of inference for the Document Question Answering task""" + + answer: str + """The answer to the question.""" + end: int + """The end word index of the answer (in the OCR’d version of the input or provided word + boxes). + """ + score: float + """The probability associated to the answer.""" + start: int + """The start word index of the answer (in the OCR’d version of the input or provided word + boxes). + """ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/feature_extraction.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/feature_extraction.py new file mode 100644 index 0000000000000000000000000000000000000000..e2868432b8dc937582b467ba14bcb12c3148eaf7 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/feature_extraction.py @@ -0,0 +1,36 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +FeatureExtractionInputTruncationDirection = Literal["left", "right"] + + +@dataclass_with_extra +class FeatureExtractionInput(BaseInferenceType): + """Feature Extraction Input. + Auto-generated from TEI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts. + """ + + inputs: list[str] | str + """The text or list of texts to embed.""" + normalize: bool | None = None + prompt_name: str | None = None + """The name of the prompt that should be used by for encoding. If not set, no prompt + will be applied. + Must be a key in the `sentence-transformers` configuration `prompts` dictionary. + For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ", + ...}, + then the sentence "What is the capital of France?" will be encoded as + "query: What is the capital of France?" because the prompt text will be prepended before + any text to encode. + """ + truncate: bool | None = None + truncation_direction: Optional["FeatureExtractionInputTruncationDirection"] = None diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/fill_mask.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/fill_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..84fcac730ee43763150068ddeca63a1ed127d59c --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/fill_mask.py @@ -0,0 +1,47 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class FillMaskParameters(BaseInferenceType): + """Additional inference parameters for Fill Mask""" + + targets: list[str] | None = None + """When passed, the model will limit the scores to the passed targets instead of looking up + in the whole vocabulary. If the provided targets are not in the model vocab, they will be + tokenized and the first resulting token will be used (with a warning, and that might be + slower). + """ + top_k: int | None = None + """When passed, overrides the number of predictions to return.""" + + +@dataclass_with_extra +class FillMaskInput(BaseInferenceType): + """Inputs for Fill Mask inference""" + + inputs: str + """The text with masked tokens""" + parameters: FillMaskParameters | None = None + """Additional inference parameters for Fill Mask""" + + +@dataclass_with_extra +class FillMaskOutputElement(BaseInferenceType): + """Outputs of inference for the Fill Mask task""" + + score: float + """The corresponding probability""" + sequence: str + """The corresponding input with the mask token prediction.""" + token: int + """The predicted token id (to replace the masked one).""" + token_str: Any + fill_mask_output_token_str: str | None = None + """The predicted token (to replace the masked one).""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_classification.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..a0d2d564b7d573cea0847308bbb723d519a5cede --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_classification.py @@ -0,0 +1,43 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +ImageClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] + + +@dataclass_with_extra +class ImageClassificationParameters(BaseInferenceType): + """Additional inference parameters for Image Classification""" + + function_to_apply: Optional["ImageClassificationOutputTransform"] = None + """The function to apply to the model outputs in order to retrieve the scores.""" + top_k: int | None = None + """When specified, limits the output to the top K most probable classes.""" + + +@dataclass_with_extra +class ImageClassificationInput(BaseInferenceType): + """Inputs for Image Classification inference""" + + inputs: str + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. + """ + parameters: ImageClassificationParameters | None = None + """Additional inference parameters for Image Classification""" + + +@dataclass_with_extra +class ImageClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Image Classification task""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_segmentation.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..d2938d89cab79329d1aeac3b16d85c8b76e49cdc --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_segmentation.py @@ -0,0 +1,51 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +ImageSegmentationSubtask = Literal["instance", "panoptic", "semantic"] + + +@dataclass_with_extra +class ImageSegmentationParameters(BaseInferenceType): + """Additional inference parameters for Image Segmentation""" + + mask_threshold: float | None = None + """Threshold to use when turning the predicted masks into binary values.""" + overlap_mask_area_threshold: float | None = None + """Mask overlap threshold to eliminate small, disconnected segments.""" + subtask: Optional["ImageSegmentationSubtask"] = None + """Segmentation task to be performed, depending on model capabilities.""" + threshold: float | None = None + """Probability threshold to filter out predicted masks.""" + + +@dataclass_with_extra +class ImageSegmentationInput(BaseInferenceType): + """Inputs for Image Segmentation inference""" + + inputs: str + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. + """ + parameters: ImageSegmentationParameters | None = None + """Additional inference parameters for Image Segmentation""" + + +@dataclass_with_extra +class ImageSegmentationOutputElement(BaseInferenceType): + """Outputs of inference for the Image Segmentation task + A predicted mask / segment + """ + + label: str + """The label of the predicted segment.""" + mask: str + """The corresponding mask as a black-and-white image (base64-encoded).""" + score: float | None = None + """The score or confidence degree the model has.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_text_to_image.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_text_to_image.py new file mode 100644 index 0000000000000000000000000000000000000000..d711f40ca021e5dfa3c072d19b5730d615dec6eb --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_text_to_image.py @@ -0,0 +1,67 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ImageTextToImageTargetSize(BaseInferenceType): + """The size in pixels of the output image. This parameter is only supported by some + providers and for specific models. It will be ignored when unsupported. + """ + + height: int + width: int + + +@dataclass_with_extra +class ImageTextToImageParameters(BaseInferenceType): + """Additional inference parameters for Image Text To Image""" + + guidance_scale: float | None = None + """For diffusion models. A higher guidance scale value encourages the model to generate + images closely linked to the text prompt at the expense of lower image quality. + """ + negative_prompt: str | None = None + """One prompt to guide what NOT to include in image generation.""" + num_inference_steps: int | None = None + """For diffusion models. The number of denoising steps. More denoising steps usually lead to + a higher quality image at the expense of slower inference. + """ + prompt: str | None = None + """The text prompt to guide the image generation. Either this or inputs (image) must be + provided. + """ + seed: int | None = None + """Seed for the random number generator.""" + target_size: ImageTextToImageTargetSize | None = None + """The size in pixels of the output image. This parameter is only supported by some + providers and for specific models. It will be ignored when unsupported. + """ + + +@dataclass_with_extra +class ImageTextToImageInput(BaseInferenceType): + """Inputs for Image Text To Image inference. Either inputs (image) or prompt (in parameters) + must be provided, or both. + """ + + inputs: str | None = None + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. Either this or prompt must be + provided. + """ + parameters: ImageTextToImageParameters | None = None + """Additional inference parameters for Image Text To Image""" + + +@dataclass_with_extra +class ImageTextToImageOutput(BaseInferenceType): + """Outputs of inference for the Image Text To Image task""" + + image: Any + """The generated image returned as raw bytes in the payload.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_text_to_video.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_text_to_video.py new file mode 100644 index 0000000000000000000000000000000000000000..870bb16c04add8cfb7909da74cef902b98486f85 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_text_to_video.py @@ -0,0 +1,65 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ImageTextToVideoTargetSize(BaseInferenceType): + """The size in pixel of the output video frames.""" + + height: int + width: int + + +@dataclass_with_extra +class ImageTextToVideoParameters(BaseInferenceType): + """Additional inference parameters for Image Text To Video""" + + guidance_scale: float | None = None + """For diffusion models. A higher guidance scale value encourages the model to generate + videos closely linked to the text prompt at the expense of lower image quality. + """ + negative_prompt: str | None = None + """One prompt to guide what NOT to include in video generation.""" + num_frames: float | None = None + """The num_frames parameter determines how many video frames are generated.""" + num_inference_steps: int | None = None + """The number of denoising steps. More denoising steps usually lead to a higher quality + video at the expense of slower inference. + """ + prompt: str | None = None + """The text prompt to guide the video generation. Either this or inputs (image) must be + provided. + """ + seed: int | None = None + """Seed for the random number generator.""" + target_size: ImageTextToVideoTargetSize | None = None + """The size in pixel of the output video frames.""" + + +@dataclass_with_extra +class ImageTextToVideoInput(BaseInferenceType): + """Inputs for Image Text To Video inference. Either inputs (image) or prompt (in parameters) + must be provided, or both. + """ + + inputs: str | None = None + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. Either this or prompt must be + provided. + """ + parameters: ImageTextToVideoParameters | None = None + """Additional inference parameters for Image Text To Video""" + + +@dataclass_with_extra +class ImageTextToVideoOutput(BaseInferenceType): + """Outputs of inference for the Image Text To Video task""" + + video: Any + """The generated video returned as raw bytes in the payload.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_image.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_image.py new file mode 100644 index 0000000000000000000000000000000000000000..6e943d73915803c05211e0d8b1a0e64fc4fdba3a --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_image.py @@ -0,0 +1,60 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ImageToImageTargetSize(BaseInferenceType): + """The size in pixels of the output image. This parameter is only supported by some + providers and for specific models. It will be ignored when unsupported. + """ + + height: int + width: int + + +@dataclass_with_extra +class ImageToImageParameters(BaseInferenceType): + """Additional inference parameters for Image To Image""" + + guidance_scale: float | None = None + """For diffusion models. A higher guidance scale value encourages the model to generate + images closely linked to the text prompt at the expense of lower image quality. + """ + negative_prompt: str | None = None + """One prompt to guide what NOT to include in image generation.""" + num_inference_steps: int | None = None + """For diffusion models. The number of denoising steps. More denoising steps usually lead to + a higher quality image at the expense of slower inference. + """ + prompt: str | None = None + """The text prompt to guide the image generation.""" + target_size: ImageToImageTargetSize | None = None + """The size in pixels of the output image. This parameter is only supported by some + providers and for specific models. It will be ignored when unsupported. + """ + + +@dataclass_with_extra +class ImageToImageInput(BaseInferenceType): + """Inputs for Image To Image inference""" + + inputs: str + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. + """ + parameters: ImageToImageParameters | None = None + """Additional inference parameters for Image To Image""" + + +@dataclass_with_extra +class ImageToImageOutput(BaseInferenceType): + """Outputs of inference for the Image To Image task""" + + image: Any + """The output image returned as raw bytes in the payload.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_text.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_text.py new file mode 100644 index 0000000000000000000000000000000000000000..3924a6612c8b246069215e2ad581dc058f54ac41 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_text.py @@ -0,0 +1,100 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +ImageToTextEarlyStoppingEnum = Literal["never"] + + +@dataclass_with_extra +class ImageToTextGenerationParameters(BaseInferenceType): + """Parametrization of the text generation process""" + + do_sample: bool | None = None + """Whether to use sampling instead of greedy decoding when generating new tokens.""" + early_stopping: Union[bool, "ImageToTextEarlyStoppingEnum"] | None = None + """Controls the stopping condition for beam-based methods.""" + epsilon_cutoff: float | None = None + """If set to float strictly between 0 and 1, only tokens with a conditional probability + greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + """ + eta_cutoff: float | None = None + """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + float strictly between 0 and 1, a token is only considered if it is greater than either + eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + for more details. + """ + max_length: int | None = None + """The maximum length (in tokens) of the generated text, including the input.""" + max_new_tokens: int | None = None + """The maximum number of tokens to generate. Takes precedence over max_length.""" + min_length: int | None = None + """The minimum length (in tokens) of the generated text, including the input.""" + min_new_tokens: int | None = None + """The minimum number of tokens to generate. Takes precedence over min_length.""" + num_beam_groups: int | None = None + """Number of groups to divide num_beams into in order to ensure diversity among different + groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + """ + num_beams: int | None = None + """Number of beams to use for beam search.""" + penalty_alpha: float | None = None + """The value balances the model confidence and the degeneration penalty in contrastive + search decoding. + """ + temperature: float | None = None + """The value used to modulate the next token probabilities.""" + top_k: int | None = None + """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" + top_p: float | None = None + """If set to float < 1, only the smallest set of most probable tokens with probabilities + that add up to top_p or higher are kept for generation. + """ + typical_p: float | None = None + """Local typicality measures how similar the conditional probability of predicting a target + token next is to the expected conditional probability of predicting a random token next, + given the partial text already generated. If set to float < 1, the smallest set of the + most locally typical tokens with probabilities that add up to typical_p or higher are + kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + """ + use_cache: bool | None = None + """Whether the model should use the past last key/values attentions to speed up decoding""" + + +@dataclass_with_extra +class ImageToTextParameters(BaseInferenceType): + """Additional inference parameters for Image To Text""" + + generation_parameters: ImageToTextGenerationParameters | None = None + """Parametrization of the text generation process""" + max_new_tokens: int | None = None + """The amount of maximum tokens to generate.""" + + +@dataclass_with_extra +class ImageToTextInput(BaseInferenceType): + """Inputs for Image To Text inference""" + + inputs: Any + """The input image data""" + parameters: ImageToTextParameters | None = None + """Additional inference parameters for Image To Text""" + + +@dataclass_with_extra +class ImageToTextOutput(BaseInferenceType): + """Outputs of inference for the Image To Text task""" + + generated_text: Any + image_to_text_output_generated_text: str | None = None + """The generated text.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_video.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_video.py new file mode 100644 index 0000000000000000000000000000000000000000..b14883f044c13c935f7b3d41f23a50f0b7c29850 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_video.py @@ -0,0 +1,60 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ImageToVideoTargetSize(BaseInferenceType): + """The size in pixel of the output video frames.""" + + height: int + width: int + + +@dataclass_with_extra +class ImageToVideoParameters(BaseInferenceType): + """Additional inference parameters for Image To Video""" + + guidance_scale: float | None = None + """For diffusion models. A higher guidance scale value encourages the model to generate + videos closely linked to the text prompt at the expense of lower image quality. + """ + negative_prompt: str | None = None + """One prompt to guide what NOT to include in video generation.""" + num_frames: float | None = None + """The num_frames parameter determines how many video frames are generated.""" + num_inference_steps: int | None = None + """The number of denoising steps. More denoising steps usually lead to a higher quality + video at the expense of slower inference. + """ + prompt: str | None = None + """The text prompt to guide the video generation.""" + seed: int | None = None + """Seed for the random number generator.""" + target_size: ImageToVideoTargetSize | None = None + """The size in pixel of the output video frames.""" + + +@dataclass_with_extra +class ImageToVideoInput(BaseInferenceType): + """Inputs for Image To Video inference""" + + inputs: str + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. + """ + parameters: ImageToVideoParameters | None = None + """Additional inference parameters for Image To Video""" + + +@dataclass_with_extra +class ImageToVideoOutput(BaseInferenceType): + """Outputs of inference for the Image To Video task""" + + video: Any + """The generated video returned as raw bytes in the payload.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/object_detection.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/object_detection.py new file mode 100644 index 0000000000000000000000000000000000000000..1c7ef7843424c75e0af2d7363a8561bec57b81f3 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/object_detection.py @@ -0,0 +1,56 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ObjectDetectionParameters(BaseInferenceType): + """Additional inference parameters for Object Detection""" + + threshold: float | None = None + """The probability necessary to make a prediction.""" + + +@dataclass_with_extra +class ObjectDetectionInput(BaseInferenceType): + """Inputs for Object Detection inference""" + + inputs: str + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. + """ + parameters: ObjectDetectionParameters | None = None + """Additional inference parameters for Object Detection""" + + +@dataclass_with_extra +class ObjectDetectionBoundingBox(BaseInferenceType): + """The predicted bounding box. Coordinates are relative to the top left corner of the input + image. + """ + + xmax: int + """The x-coordinate of the bottom-right corner of the bounding box.""" + xmin: int + """The x-coordinate of the top-left corner of the bounding box.""" + ymax: int + """The y-coordinate of the bottom-right corner of the bounding box.""" + ymin: int + """The y-coordinate of the top-left corner of the bounding box.""" + + +@dataclass_with_extra +class ObjectDetectionOutputElement(BaseInferenceType): + """Outputs of inference for the Object Detection task""" + + box: ObjectDetectionBoundingBox + """The predicted bounding box. Coordinates are relative to the top left corner of the input + image. + """ + label: str + """The predicted label for the bounding box.""" + score: float + """The associated score / probability.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/question_answering.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..ee97c638d68c3bedf4fe8e8f9a8f996d4955e30a --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/question_answering.py @@ -0,0 +1,72 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class QuestionAnsweringInputData(BaseInferenceType): + """One (context, question) pair to answer""" + + context: str + """The context to be used for answering the question""" + question: str + """The question to be answered""" + + +@dataclass_with_extra +class QuestionAnsweringParameters(BaseInferenceType): + """Additional inference parameters for Question Answering""" + + align_to_words: bool | None = None + """Attempts to align the answer to real words. Improves quality on space separated + languages. Might hurt on non-space-separated languages (like Japanese or Chinese) + """ + doc_stride: int | None = None + """If the context is too long to fit with the question for the model, it will be split in + several chunks with some overlap. This argument controls the size of that overlap. + """ + handle_impossible_answer: bool | None = None + """Whether to accept impossible as an answer.""" + max_answer_len: int | None = None + """The maximum length of predicted answers (e.g., only answers with a shorter length are + considered). + """ + max_question_len: int | None = None + """The maximum length of the question after tokenization. It will be truncated if needed.""" + max_seq_len: int | None = None + """The maximum length of the total sentence (context + question) in tokens of each chunk + passed to the model. The context will be split in several chunks (using docStride as + overlap) if needed. + """ + top_k: int | None = None + """The number of answers to return (will be chosen by order of likelihood). Note that we + return less than topk answers if there are not enough options available within the + context. + """ + + +@dataclass_with_extra +class QuestionAnsweringInput(BaseInferenceType): + """Inputs for Question Answering inference""" + + inputs: QuestionAnsweringInputData + """One (context, question) pair to answer""" + parameters: QuestionAnsweringParameters | None = None + """Additional inference parameters for Question Answering""" + + +@dataclass_with_extra +class QuestionAnsweringOutputElement(BaseInferenceType): + """Outputs of inference for the Question Answering task""" + + answer: str + """The answer to the question.""" + end: int + """The character position in the input where the answer ends.""" + score: float + """The probability associated to the answer.""" + start: int + """The character position in the input where the answer begins.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/sentence_similarity.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/sentence_similarity.py new file mode 100644 index 0000000000000000000000000000000000000000..a06c32d395fce557784f95ca77c0a62f04b255db --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/sentence_similarity.py @@ -0,0 +1,27 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class SentenceSimilarityInputData(BaseInferenceType): + sentences: list[str] + """A list of strings which will be compared against the source_sentence.""" + source_sentence: str + """The string that you wish to compare the other strings with. This can be a phrase, + sentence, or longer passage, depending on the model being used. + """ + + +@dataclass_with_extra +class SentenceSimilarityInput(BaseInferenceType): + """Inputs for Sentence similarity inference""" + + inputs: SentenceSimilarityInputData + parameters: dict[str, Any] | None = None + """Additional inference parameters for Sentence Similarity""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/summarization.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/summarization.py new file mode 100644 index 0000000000000000000000000000000000000000..35f2a86f308f9d4839f46ac6496097961a2df723 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/summarization.py @@ -0,0 +1,41 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +SummarizationTruncationStrategy = Literal["do_not_truncate", "longest_first", "only_first", "only_second"] + + +@dataclass_with_extra +class SummarizationParameters(BaseInferenceType): + """Additional inference parameters for summarization.""" + + clean_up_tokenization_spaces: bool | None = None + """Whether to clean up the potential extra spaces in the text output.""" + generate_parameters: dict[str, Any] | None = None + """Additional parametrization of the text generation algorithm.""" + truncation: Optional["SummarizationTruncationStrategy"] = None + """The truncation strategy to use.""" + + +@dataclass_with_extra +class SummarizationInput(BaseInferenceType): + """Inputs for Summarization inference""" + + inputs: str + """The input text to summarize.""" + parameters: SummarizationParameters | None = None + """Additional inference parameters for summarization.""" + + +@dataclass_with_extra +class SummarizationOutput(BaseInferenceType): + """Outputs of inference for the Summarization task""" + + summary_text: str + """The summarized text.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/table_question_answering.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/table_question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..1909f0d2b77551ff8b7b4cf0df199a46e5c72197 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/table_question_answering.py @@ -0,0 +1,62 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class TableQuestionAnsweringInputData(BaseInferenceType): + """One (table, question) pair to answer""" + + question: str + """The question to be answered about the table""" + table: dict[str, list[str]] + """The table to serve as context for the questions""" + + +Padding = Literal["do_not_pad", "longest", "max_length"] + + +@dataclass_with_extra +class TableQuestionAnsweringParameters(BaseInferenceType): + """Additional inference parameters for Table Question Answering""" + + padding: Optional["Padding"] = None + """Activates and controls padding.""" + sequential: bool | None = None + """Whether to do inference sequentially or as a batch. Batching is faster, but models like + SQA require the inference to be done sequentially to extract relations within sequences, + given their conversational nature. + """ + truncation: bool | None = None + """Activates and controls truncation.""" + + +@dataclass_with_extra +class TableQuestionAnsweringInput(BaseInferenceType): + """Inputs for Table Question Answering inference""" + + inputs: TableQuestionAnsweringInputData + """One (table, question) pair to answer""" + parameters: TableQuestionAnsweringParameters | None = None + """Additional inference parameters for Table Question Answering""" + + +@dataclass_with_extra +class TableQuestionAnsweringOutputElement(BaseInferenceType): + """Outputs of inference for the Table Question Answering task""" + + answer: str + """The answer of the question given the table. If there is an aggregator, the answer will be + preceded by `AGGREGATOR >`. + """ + cells: list[str] + """list of strings made up of the answer cell values.""" + coordinates: list[list[int]] + """Coordinates of the cells of the answers.""" + aggregator: str | None = None + """If the model has an aggregator, this returns the aggregator.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text2text_generation.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text2text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..b4508823749a52d3238fc288e62e479e126e1827 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text2text_generation.py @@ -0,0 +1,42 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +Text2TextGenerationTruncationStrategy = Literal["do_not_truncate", "longest_first", "only_first", "only_second"] + + +@dataclass_with_extra +class Text2TextGenerationParameters(BaseInferenceType): + """Additional inference parameters for Text2text Generation""" + + clean_up_tokenization_spaces: bool | None = None + """Whether to clean up the potential extra spaces in the text output.""" + generate_parameters: dict[str, Any] | None = None + """Additional parametrization of the text generation algorithm""" + truncation: Optional["Text2TextGenerationTruncationStrategy"] = None + """The truncation strategy to use""" + + +@dataclass_with_extra +class Text2TextGenerationInput(BaseInferenceType): + """Inputs for Text2text Generation inference""" + + inputs: str + """The input text data""" + parameters: Text2TextGenerationParameters | None = None + """Additional inference parameters for Text2text Generation""" + + +@dataclass_with_extra +class Text2TextGenerationOutput(BaseInferenceType): + """Outputs of inference for the Text2text Generation task""" + + generated_text: Any + text2_text_generation_output_generated_text: str | None = None + """The generated text.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_classification.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..9df576b4de724a73be32ff5ecbf81dd5ea576f91 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_classification.py @@ -0,0 +1,41 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +TextClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] + + +@dataclass_with_extra +class TextClassificationParameters(BaseInferenceType): + """Additional inference parameters for Text Classification""" + + function_to_apply: Optional["TextClassificationOutputTransform"] = None + """The function to apply to the model outputs in order to retrieve the scores.""" + top_k: int | None = None + """When specified, limits the output to the top K most probable classes.""" + + +@dataclass_with_extra +class TextClassificationInput(BaseInferenceType): + """Inputs for Text Classification inference""" + + inputs: str + """The text to classify""" + parameters: TextClassificationParameters | None = None + """Additional inference parameters for Text Classification""" + + +@dataclass_with_extra +class TextClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Text Classification task""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_generation.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..1b2269955bcaef7f23972b00ca8991b6854f8181 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_generation.py @@ -0,0 +1,168 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal + +from .base import BaseInferenceType, dataclass_with_extra + + +TypeEnum = Literal["json", "regex", "json_schema"] + + +@dataclass_with_extra +class TextGenerationInputGrammarType(BaseInferenceType): + type: "TypeEnum" + value: Any + """A string that represents a [JSON Schema](https://json-schema.org/). + JSON Schema is a declarative language that allows to annotate JSON documents + with types and descriptions. + """ + + +@dataclass_with_extra +class TextGenerationInputGenerateParameters(BaseInferenceType): + adapter_id: str | None = None + """Lora adapter id""" + best_of: int | None = None + """Generate best_of sequences and return the one if the highest token logprobs.""" + decoder_input_details: bool | None = None + """Whether to return decoder input token logprobs and ids.""" + details: bool | None = None + """Whether to return generation details.""" + do_sample: bool | None = None + """Activate logits sampling.""" + frequency_penalty: float | None = None + """The parameter for frequency penalty. 1.0 means no penalty + Penalize new tokens based on their existing frequency in the text so far, + decreasing the model's likelihood to repeat the same line verbatim. + """ + grammar: TextGenerationInputGrammarType | None = None + max_new_tokens: int | None = None + """Maximum number of tokens to generate.""" + repetition_penalty: float | None = None + """The parameter for repetition penalty. 1.0 means no penalty. + See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + """ + return_full_text: bool | None = None + """Whether to prepend the prompt to the generated text""" + seed: int | None = None + """Random sampling seed.""" + stop: list[str] | None = None + """Stop generating tokens if a member of `stop` is generated.""" + temperature: float | None = None + """The value used to module the logits distribution.""" + top_k: int | None = None + """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" + top_n_tokens: int | None = None + """The number of highest probability vocabulary tokens to keep for top-n-filtering.""" + top_p: float | None = None + """Top-p value for nucleus sampling.""" + truncate: int | None = None + """Truncate inputs tokens to the given size.""" + typical_p: float | None = None + """Typical Decoding mass + See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) + for more information. + """ + watermark: bool | None = None + """Watermarking with [A Watermark for Large Language + Models](https://arxiv.org/abs/2301.10226). + """ + + +@dataclass_with_extra +class TextGenerationInput(BaseInferenceType): + """Text Generation Input. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + inputs: str + parameters: TextGenerationInputGenerateParameters | None = None + stream: bool | None = None + + +TextGenerationOutputFinishReason = Literal["length", "eos_token", "stop_sequence"] + + +@dataclass_with_extra +class TextGenerationOutputPrefillToken(BaseInferenceType): + id: int + logprob: float + text: str + + +@dataclass_with_extra +class TextGenerationOutputToken(BaseInferenceType): + id: int + logprob: float + special: bool + text: str + + +@dataclass_with_extra +class TextGenerationOutputBestOfSequence(BaseInferenceType): + finish_reason: "TextGenerationOutputFinishReason" + generated_text: str + generated_tokens: int + prefill: list[TextGenerationOutputPrefillToken] + tokens: list[TextGenerationOutputToken] + seed: int | None = None + top_tokens: list[list[TextGenerationOutputToken]] | None = None + + +@dataclass_with_extra +class TextGenerationOutputDetails(BaseInferenceType): + finish_reason: "TextGenerationOutputFinishReason" + generated_tokens: int + prefill: list[TextGenerationOutputPrefillToken] + tokens: list[TextGenerationOutputToken] + best_of_sequences: list[TextGenerationOutputBestOfSequence] | None = None + seed: int | None = None + top_tokens: list[list[TextGenerationOutputToken]] | None = None + + +@dataclass_with_extra +class TextGenerationOutput(BaseInferenceType): + """Text Generation Output. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + generated_text: str + details: TextGenerationOutputDetails | None = None + + +@dataclass_with_extra +class TextGenerationStreamOutputStreamDetails(BaseInferenceType): + finish_reason: "TextGenerationOutputFinishReason" + generated_tokens: int + input_length: int + seed: int | None = None + + +@dataclass_with_extra +class TextGenerationStreamOutputToken(BaseInferenceType): + id: int + logprob: float + special: bool + text: str + + +@dataclass_with_extra +class TextGenerationStreamOutput(BaseInferenceType): + """Text Generation Stream Output. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + index: int + token: TextGenerationStreamOutputToken + details: TextGenerationStreamOutputStreamDetails | None = None + generated_text: str | None = None + top_tokens: list[TextGenerationStreamOutputToken] | None = None diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_audio.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_audio.py new file mode 100644 index 0000000000000000000000000000000000000000..35033f5129cdfb3d3e7512727bdfc248c882d35e --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_audio.py @@ -0,0 +1,99 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +TextToAudioEarlyStoppingEnum = Literal["never"] + + +@dataclass_with_extra +class TextToAudioGenerationParameters(BaseInferenceType): + """Parametrization of the text generation process""" + + do_sample: bool | None = None + """Whether to use sampling instead of greedy decoding when generating new tokens.""" + early_stopping: Union[bool, "TextToAudioEarlyStoppingEnum"] | None = None + """Controls the stopping condition for beam-based methods.""" + epsilon_cutoff: float | None = None + """If set to float strictly between 0 and 1, only tokens with a conditional probability + greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + """ + eta_cutoff: float | None = None + """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + float strictly between 0 and 1, a token is only considered if it is greater than either + eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + for more details. + """ + max_length: int | None = None + """The maximum length (in tokens) of the generated text, including the input.""" + max_new_tokens: int | None = None + """The maximum number of tokens to generate. Takes precedence over max_length.""" + min_length: int | None = None + """The minimum length (in tokens) of the generated text, including the input.""" + min_new_tokens: int | None = None + """The minimum number of tokens to generate. Takes precedence over min_length.""" + num_beam_groups: int | None = None + """Number of groups to divide num_beams into in order to ensure diversity among different + groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + """ + num_beams: int | None = None + """Number of beams to use for beam search.""" + penalty_alpha: float | None = None + """The value balances the model confidence and the degeneration penalty in contrastive + search decoding. + """ + temperature: float | None = None + """The value used to modulate the next token probabilities.""" + top_k: int | None = None + """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" + top_p: float | None = None + """If set to float < 1, only the smallest set of most probable tokens with probabilities + that add up to top_p or higher are kept for generation. + """ + typical_p: float | None = None + """Local typicality measures how similar the conditional probability of predicting a target + token next is to the expected conditional probability of predicting a random token next, + given the partial text already generated. If set to float < 1, the smallest set of the + most locally typical tokens with probabilities that add up to typical_p or higher are + kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + """ + use_cache: bool | None = None + """Whether the model should use the past last key/values attentions to speed up decoding""" + + +@dataclass_with_extra +class TextToAudioParameters(BaseInferenceType): + """Additional inference parameters for Text To Audio""" + + generation_parameters: TextToAudioGenerationParameters | None = None + """Parametrization of the text generation process""" + + +@dataclass_with_extra +class TextToAudioInput(BaseInferenceType): + """Inputs for Text To Audio inference""" + + inputs: str + """The input text data""" + parameters: TextToAudioParameters | None = None + """Additional inference parameters for Text To Audio""" + + +@dataclass_with_extra +class TextToAudioOutput(BaseInferenceType): + """Outputs of inference for the Text To Audio task""" + + audio: Any + """The generated audio waveform.""" + sampling_rate: float + """The sampling rate of the generated audio waveform.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_image.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_image.py new file mode 100644 index 0000000000000000000000000000000000000000..716f240385fba6116b8b19b88e177536a3d18ec0 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_image.py @@ -0,0 +1,50 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class TextToImageParameters(BaseInferenceType): + """Additional inference parameters for Text To Image""" + + guidance_scale: float | None = None + """A higher guidance scale value encourages the model to generate images closely linked to + the text prompt, but values too high may cause saturation and other artifacts. + """ + height: int | None = None + """The height in pixels of the output image""" + negative_prompt: str | None = None + """One prompt to guide what NOT to include in image generation.""" + num_inference_steps: int | None = None + """The number of denoising steps. More denoising steps usually lead to a higher quality + image at the expense of slower inference. + """ + scheduler: str | None = None + """Override the scheduler with a compatible one.""" + seed: int | None = None + """Seed for the random number generator.""" + width: int | None = None + """The width in pixels of the output image""" + + +@dataclass_with_extra +class TextToImageInput(BaseInferenceType): + """Inputs for Text To Image inference""" + + inputs: str + """The input text data (sometimes called "prompt")""" + parameters: TextToImageParameters | None = None + """Additional inference parameters for Text To Image""" + + +@dataclass_with_extra +class TextToImageOutput(BaseInferenceType): + """Outputs of inference for the Text To Image task""" + + image: Any + """The generated image returned as raw bytes in the payload.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_speech.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_speech.py new file mode 100644 index 0000000000000000000000000000000000000000..588e0d1a566bde3bf66a3a07271638c6f42ecb95 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_speech.py @@ -0,0 +1,99 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +TextToSpeechEarlyStoppingEnum = Literal["never"] + + +@dataclass_with_extra +class TextToSpeechGenerationParameters(BaseInferenceType): + """Parametrization of the text generation process""" + + do_sample: bool | None = None + """Whether to use sampling instead of greedy decoding when generating new tokens.""" + early_stopping: Union[bool, "TextToSpeechEarlyStoppingEnum"] | None = None + """Controls the stopping condition for beam-based methods.""" + epsilon_cutoff: float | None = None + """If set to float strictly between 0 and 1, only tokens with a conditional probability + greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + """ + eta_cutoff: float | None = None + """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + float strictly between 0 and 1, a token is only considered if it is greater than either + eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + for more details. + """ + max_length: int | None = None + """The maximum length (in tokens) of the generated text, including the input.""" + max_new_tokens: int | None = None + """The maximum number of tokens to generate. Takes precedence over max_length.""" + min_length: int | None = None + """The minimum length (in tokens) of the generated text, including the input.""" + min_new_tokens: int | None = None + """The minimum number of tokens to generate. Takes precedence over min_length.""" + num_beam_groups: int | None = None + """Number of groups to divide num_beams into in order to ensure diversity among different + groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + """ + num_beams: int | None = None + """Number of beams to use for beam search.""" + penalty_alpha: float | None = None + """The value balances the model confidence and the degeneration penalty in contrastive + search decoding. + """ + temperature: float | None = None + """The value used to modulate the next token probabilities.""" + top_k: int | None = None + """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" + top_p: float | None = None + """If set to float < 1, only the smallest set of most probable tokens with probabilities + that add up to top_p or higher are kept for generation. + """ + typical_p: float | None = None + """Local typicality measures how similar the conditional probability of predicting a target + token next is to the expected conditional probability of predicting a random token next, + given the partial text already generated. If set to float < 1, the smallest set of the + most locally typical tokens with probabilities that add up to typical_p or higher are + kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + """ + use_cache: bool | None = None + """Whether the model should use the past last key/values attentions to speed up decoding""" + + +@dataclass_with_extra +class TextToSpeechParameters(BaseInferenceType): + """Additional inference parameters for Text To Speech""" + + generation_parameters: TextToSpeechGenerationParameters | None = None + """Parametrization of the text generation process""" + + +@dataclass_with_extra +class TextToSpeechInput(BaseInferenceType): + """Inputs for Text To Speech inference""" + + inputs: str + """The input text data""" + parameters: TextToSpeechParameters | None = None + """Additional inference parameters for Text To Speech""" + + +@dataclass_with_extra +class TextToSpeechOutput(BaseInferenceType): + """Outputs of inference for the Text To Speech task""" + + audio: Any + """The generated audio""" + sampling_rate: float | None = None + """The sampling rate of the generated audio waveform.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_video.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_video.py new file mode 100644 index 0000000000000000000000000000000000000000..6e357113cecccce1653c8db977ecd2ca392a79b1 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_video.py @@ -0,0 +1,46 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class TextToVideoParameters(BaseInferenceType): + """Additional inference parameters for Text To Video""" + + guidance_scale: float | None = None + """A higher guidance scale value encourages the model to generate videos closely linked to + the text prompt, but values too high may cause saturation and other artifacts. + """ + negative_prompt: list[str] | None = None + """One or several prompt to guide what NOT to include in video generation.""" + num_frames: float | None = None + """The num_frames parameter determines how many video frames are generated.""" + num_inference_steps: int | None = None + """The number of denoising steps. More denoising steps usually lead to a higher quality + video at the expense of slower inference. + """ + seed: int | None = None + """Seed for the random number generator.""" + + +@dataclass_with_extra +class TextToVideoInput(BaseInferenceType): + """Inputs for Text To Video inference""" + + inputs: str + """The input text data (sometimes called "prompt")""" + parameters: TextToVideoParameters | None = None + """Additional inference parameters for Text To Video""" + + +@dataclass_with_extra +class TextToVideoOutput(BaseInferenceType): + """Outputs of inference for the Text To Video task""" + + video: Any + """The generated video returned as raw bytes in the payload.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/token_classification.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/token_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..abf21d19a60b4b90b87309dba367e2ea578f1fea --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/token_classification.py @@ -0,0 +1,51 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +TokenClassificationAggregationStrategy = Literal["none", "simple", "first", "average", "max"] + + +@dataclass_with_extra +class TokenClassificationParameters(BaseInferenceType): + """Additional inference parameters for Token Classification""" + + aggregation_strategy: Optional["TokenClassificationAggregationStrategy"] = None + """The strategy used to fuse tokens based on model predictions""" + ignore_labels: list[str] | None = None + """A list of labels to ignore""" + stride: int | None = None + """The number of overlapping tokens between chunks when splitting the input text.""" + + +@dataclass_with_extra +class TokenClassificationInput(BaseInferenceType): + """Inputs for Token Classification inference""" + + inputs: str + """The input text data""" + parameters: TokenClassificationParameters | None = None + """Additional inference parameters for Token Classification""" + + +@dataclass_with_extra +class TokenClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Token Classification task""" + + end: int + """The character position in the input where this group ends.""" + score: float + """The associated score / probability""" + start: int + """The character position in the input where this group begins.""" + word: str + """The corresponding text""" + entity: str | None = None + """The predicted label for a single token""" + entity_group: str | None = None + """The predicted label for a group of one or more tokens""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/translation.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/translation.py new file mode 100644 index 0000000000000000000000000000000000000000..58e0b9de2921843846ac0d3cc65faf7d7089e59f --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/translation.py @@ -0,0 +1,49 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +TranslationTruncationStrategy = Literal["do_not_truncate", "longest_first", "only_first", "only_second"] + + +@dataclass_with_extra +class TranslationParameters(BaseInferenceType): + """Additional inference parameters for Translation""" + + clean_up_tokenization_spaces: bool | None = None + """Whether to clean up the potential extra spaces in the text output.""" + generate_parameters: dict[str, Any] | None = None + """Additional parametrization of the text generation algorithm.""" + src_lang: str | None = None + """The source language of the text. Required for models that can translate from multiple + languages. + """ + tgt_lang: str | None = None + """Target language to translate to. Required for models that can translate to multiple + languages. + """ + truncation: Optional["TranslationTruncationStrategy"] = None + """The truncation strategy to use.""" + + +@dataclass_with_extra +class TranslationInput(BaseInferenceType): + """Inputs for Translation inference""" + + inputs: str + """The text to translate.""" + parameters: TranslationParameters | None = None + """Additional inference parameters for Translation""" + + +@dataclass_with_extra +class TranslationOutput(BaseInferenceType): + """Outputs of inference for the Translation task""" + + translation_text: str + """The translated text.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/video_classification.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/video_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..489a602c171e0b86fd5404d805c5773adea3b0ee --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/video_classification.py @@ -0,0 +1,45 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +VideoClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] + + +@dataclass_with_extra +class VideoClassificationParameters(BaseInferenceType): + """Additional inference parameters for Video Classification""" + + frame_sampling_rate: int | None = None + """The sampling rate used to select frames from the video.""" + function_to_apply: Optional["VideoClassificationOutputTransform"] = None + """The function to apply to the model outputs in order to retrieve the scores.""" + num_frames: int | None = None + """The number of sampled frames to consider for classification.""" + top_k: int | None = None + """When specified, limits the output to the top K most probable classes.""" + + +@dataclass_with_extra +class VideoClassificationInput(BaseInferenceType): + """Inputs for Video Classification inference""" + + inputs: Any + """The input video data""" + parameters: VideoClassificationParameters | None = None + """Additional inference parameters for Video Classification""" + + +@dataclass_with_extra +class VideoClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Video Classification task""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/visual_question_answering.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/visual_question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..73f532aa06981778515a82fa5e151a1cff6b687d --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/visual_question_answering.py @@ -0,0 +1,49 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class VisualQuestionAnsweringInputData(BaseInferenceType): + """One (image, question) pair to answer""" + + image: Any + """The image.""" + question: str + """The question to answer based on the image.""" + + +@dataclass_with_extra +class VisualQuestionAnsweringParameters(BaseInferenceType): + """Additional inference parameters for Visual Question Answering""" + + top_k: int | None = None + """The number of answers to return (will be chosen by order of likelihood). Note that we + return less than topk answers if there are not enough options available within the + context. + """ + + +@dataclass_with_extra +class VisualQuestionAnsweringInput(BaseInferenceType): + """Inputs for Visual Question Answering inference""" + + inputs: VisualQuestionAnsweringInputData + """One (image, question) pair to answer""" + parameters: VisualQuestionAnsweringParameters | None = None + """Additional inference parameters for Visual Question Answering""" + + +@dataclass_with_extra +class VisualQuestionAnsweringOutputElement(BaseInferenceType): + """Outputs of inference for the Visual Question Answering task""" + + score: float + """The associated score / probability""" + answer: str | None = None + """The answer to the question""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_classification.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..a04f1a59353f97b9840d994e9eaed636479189a7 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_classification.py @@ -0,0 +1,43 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ZeroShotClassificationParameters(BaseInferenceType): + """Additional inference parameters for Zero Shot Classification""" + + candidate_labels: list[str] + """The set of possible class labels to classify the text into.""" + hypothesis_template: str | None = None + """The sentence used in conjunction with `candidate_labels` to attempt the text + classification by replacing the placeholder with the candidate labels. + """ + multi_label: bool | None = None + """Whether multiple candidate labels can be true. If false, the scores are normalized such + that the sum of the label likelihoods for each sequence is 1. If true, the labels are + considered independent and probabilities are normalized for each candidate. + """ + + +@dataclass_with_extra +class ZeroShotClassificationInput(BaseInferenceType): + """Inputs for Zero Shot Classification inference""" + + inputs: str + """The text to classify""" + parameters: ZeroShotClassificationParameters + """Additional inference parameters for Zero Shot Classification""" + + +@dataclass_with_extra +class ZeroShotClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Zero Shot Classification task""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_image_classification.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_image_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..65c5cd2530665787e1e7d1fc49a6826646b141c8 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_image_classification.py @@ -0,0 +1,38 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ZeroShotImageClassificationParameters(BaseInferenceType): + """Additional inference parameters for Zero Shot Image Classification""" + + candidate_labels: list[str] + """The candidate labels for this image""" + hypothesis_template: str | None = None + """The sentence used in conjunction with `candidate_labels` to attempt the image + classification by replacing the placeholder with the candidate labels. + """ + + +@dataclass_with_extra +class ZeroShotImageClassificationInput(BaseInferenceType): + """Inputs for Zero Shot Image Classification inference""" + + inputs: str + """The input image data to classify as a base64-encoded string.""" + parameters: ZeroShotImageClassificationParameters + """Additional inference parameters for Zero Shot Image Classification""" + + +@dataclass_with_extra +class ZeroShotImageClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Zero Shot Image Classification task""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_object_detection.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_object_detection.py new file mode 100644 index 0000000000000000000000000000000000000000..e981463b253f61aa0f4c71636813e5fb65d48717 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_object_detection.py @@ -0,0 +1,50 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ZeroShotObjectDetectionParameters(BaseInferenceType): + """Additional inference parameters for Zero Shot Object Detection""" + + candidate_labels: list[str] + """The candidate labels for this image""" + + +@dataclass_with_extra +class ZeroShotObjectDetectionInput(BaseInferenceType): + """Inputs for Zero Shot Object Detection inference""" + + inputs: str + """The input image data as a base64-encoded string.""" + parameters: ZeroShotObjectDetectionParameters + """Additional inference parameters for Zero Shot Object Detection""" + + +@dataclass_with_extra +class ZeroShotObjectDetectionBoundingBox(BaseInferenceType): + """The predicted bounding box. Coordinates are relative to the top left corner of the input + image. + """ + + xmax: int + xmin: int + ymax: int + ymin: int + + +@dataclass_with_extra +class ZeroShotObjectDetectionOutputElement(BaseInferenceType): + """Outputs of inference for the Zero Shot Object Detection task""" + + box: ZeroShotObjectDetectionBoundingBox + """The predicted bounding box. Coordinates are relative to the top left corner of the input + image. + """ + label: str + """A candidate label""" + score: float + """The associated score / probability""" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__init__.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5f768d564ff427f927e0e643fe592d177958e0dc Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/_cli_hacks.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/_cli_hacks.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b577013887d9e81e884ace4ac7fc8031012318d9 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/_cli_hacks.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/agent.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/agent.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..537e09eb82ff6c9a4a93be919f1290d555a73477 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/agent.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/cli.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/cli.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e46d1edc610ff9a233d6ee59392087255b1e168f Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/cli.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/constants.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/constants.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d1bd13ba34f95c54e3d91f941715546d023853f6 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/constants.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/mcp_client.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/mcp_client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c0b77aa8827b7472cfc51f8c3400339c33395577 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/mcp_client.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/types.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a226d42d1fc5a171a7d839c88195f90252f233db Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/types.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/utils.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f020605d1cd387035ed551a8b1748098e65bf2c5 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/utils.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/_cli_hacks.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/_cli_hacks.py new file mode 100644 index 0000000000000000000000000000000000000000..abf3c41dd687cdaab2969cc5a28455c804105856 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/_cli_hacks.py @@ -0,0 +1,88 @@ +import asyncio +import sys +from functools import partial + +import click + + +def _patch_anyio_open_process(): + """ + Patch anyio.open_process to allow detached processes on Windows and Unix-like systems. + + This is necessary to prevent the MCP client from being interrupted by Ctrl+C when running in the CLI. + """ + import subprocess + + import anyio + + if getattr(anyio, "_tiny_agents_patched", False): + return + anyio._tiny_agents_patched = True # ty: ignore[invalid-assignment] + + original_open_process = anyio.open_process + + if sys.platform == "win32": + # On Windows, we need to set the creation flags to create a new process group + + async def open_process_in_new_group(*args, **kwargs): + """ + Wrapper for open_process to handle Windows-specific process creation flags. + """ + # Ensure we pass the creation flags for Windows + kwargs.setdefault("creationflags", subprocess.CREATE_NEW_PROCESS_GROUP) + return await original_open_process(*args, **kwargs) + + anyio.open_process = open_process_in_new_group # ty: ignore[invalid-assignment] + else: + # For Unix-like systems, we can use setsid to create a new session + async def open_process_in_new_group(*args, **kwargs): + """ + Wrapper for open_process to handle Unix-like systems with start_new_session=True. + """ + kwargs.setdefault("start_new_session", True) + return await original_open_process(*args, **kwargs) + + anyio.open_process = open_process_in_new_group # ty: ignore[invalid-assignment] + + +async def _async_prompt(exit_event: asyncio.Event, prompt: str = "» ") -> str: + """ + Asynchronous prompt function that reads input from stdin without blocking. + + This function is designed to work in an asynchronous context, allowing the event loop to gracefully stop it (e.g. on Ctrl+C). + + Alternatively, we could use https://github.com/vxgmichel/aioconsole but that would be an additional dependency. + """ + loop = asyncio.get_event_loop() + + if sys.platform == "win32": + # Windows: Use run_in_executor to avoid blocking the event loop + # Degraded solution: this is not ideal as user will have to CTRL+C once more to stop the prompt (and it'll not be graceful) + return await loop.run_in_executor(None, partial(click.prompt, prompt, prompt_suffix=" ")) + else: + # UNIX-like: Use loop.add_reader for non-blocking stdin read + future = loop.create_future() + + def on_input(): + line = sys.stdin.readline() + loop.remove_reader(sys.stdin) + future.set_result(line) + + print(prompt, end=" ", flush=True) + loop.add_reader(sys.stdin, on_input) # not supported on Windows + + # Wait for user input or exit event + # Wait until either the user hits enter or exit_event is set + exit_task = asyncio.create_task(exit_event.wait()) + await asyncio.wait( + [future, exit_task], + return_when=asyncio.FIRST_COMPLETED, + ) + + # Check which one has been triggered + if exit_event.is_set(): + future.cancel() + return "" + + line = await future + return line.strip() diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/agent.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/agent.py new file mode 100644 index 0000000000000000000000000000000000000000..1d867032fa53f721928ef182329d0aa694b83885 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/agent.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import asyncio +from typing import AsyncGenerator, Iterable, Optional, Union + +from huggingface_hub import ChatCompletionInputMessage, ChatCompletionStreamOutput, MCPClient + +from .._providers import PROVIDER_OR_POLICY_T +from .constants import DEFAULT_SYSTEM_PROMPT, EXIT_LOOP_TOOLS, MAX_NUM_TURNS +from .types import ServerConfig + + +class Agent(MCPClient): + """ + Implementation of a Simple Agent, which is a simple while loop built right on top of an [`MCPClient`]. + + > [!WARNING] + > This class is experimental and might be subject to breaking changes in the future without prior notice. + + Args: + model (`str`, *optional*): + The model to run inference with. Can be a model id hosted on the Hugging Face Hub, e.g. `meta-llama/Meta-Llama-3-8B-Instruct` + or a URL to a deployed Inference Endpoint or other local or remote endpoint. + servers (`Iterable[dict]`): + MCP servers to connect to. Each server is a dictionary containing a `type` key and a `config` key. The `type` key can be `"stdio"` or `"sse"`, and the `config` key is a dictionary of arguments for the server. + provider (`str`, *optional*): + Name of the provider to use for inference. Defaults to "auto" i.e. the first of the providers available for the model, sorted by the user's order in https://hf.co/settings/inference-providers. + If model is a URL or `base_url` is passed, then `provider` is not used. + base_url (`str`, *optional*): + The base URL to run inference. Defaults to None. + api_key (`str`, *optional*): + Token to use for authentication. Will default to the locally Hugging Face saved token if not provided. You can also use your own provider API key to interact directly with the provider's service. + prompt (`str`, *optional*): + The system prompt to use for the agent. Defaults to the default system prompt in `constants.py`. + """ + + def __init__( + self, + *, + model: Optional[str] = None, + servers: Iterable[ServerConfig], + provider: Optional[PROVIDER_OR_POLICY_T] = None, + base_url: Optional[str] = None, + api_key: Optional[str] = None, + prompt: Optional[str] = None, + ): + super().__init__(model=model, provider=provider, base_url=base_url, api_key=api_key) + self._servers_cfg = list(servers) + self.messages: list[Union[dict, ChatCompletionInputMessage]] = [ + {"role": "system", "content": prompt or DEFAULT_SYSTEM_PROMPT} + ] + + async def load_tools(self) -> None: + for cfg in self._servers_cfg: + await self.add_mcp_server(**cfg) + + async def run( + self, + user_input: str, + *, + abort_event: Optional[asyncio.Event] = None, + ) -> AsyncGenerator[Union[ChatCompletionStreamOutput, ChatCompletionInputMessage], None]: + """ + Run the agent with the given user input. + + Args: + user_input (`str`): + The user input to run the agent with. + abort_event (`asyncio.Event`, *optional*): + An event that can be used to abort the agent. If the event is set, the agent will stop running. + """ + self.messages.append({"role": "user", "content": user_input}) + + num_turns: int = 0 + next_turn_should_call_tools = True + + while True: + if abort_event and abort_event.is_set(): + return + + async for item in self.process_single_turn_with_tools( + self.messages, + exit_loop_tools=EXIT_LOOP_TOOLS, + exit_if_first_chunk_no_tool=(num_turns > 0 and next_turn_should_call_tools), + ): + yield item + + num_turns += 1 + last = self.messages[-1] + + if last.get("role") == "tool" and last.get("name") in {t.function.name for t in EXIT_LOOP_TOOLS}: + return + + if last.get("role") != "tool" and num_turns > MAX_NUM_TURNS: + return + + if last.get("role") != "tool" and next_turn_should_call_tools: + return + + next_turn_should_call_tools = last.get("role") != "tool" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/cli.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..2d1d14d478838893e03a953e96771f4048a6b713 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/cli.py @@ -0,0 +1,245 @@ +import asyncio +import os +import signal +import traceback +from typing import Optional + +import click + +from ...utils import ANSI +from ._cli_hacks import _async_prompt, _patch_anyio_open_process +from .agent import Agent +from .utils import _load_agent_config + + +@click.group( + help="A squad of lightweight composable AI applications built on Hugging Face's Inference Client and MCP stack." +) +def app() -> None: + pass + + +async def run_agent( + agent_path: Optional[str], +) -> None: + """ + Tiny Agent loop. + + Args: + agent_path (`str`, *optional*): + Path to a local folder containing an `agent.json` and optionally a custom `PROMPT.md` or `AGENTS.md` file or a built-in agent stored in a Hugging Face dataset. + + """ + _patch_anyio_open_process() # Hacky way to prevent stdio connections to be stopped by Ctrl+C + + config, prompt = _load_agent_config(agent_path) + + inputs = config.get("inputs", []) + servers = config.get("servers", []) + + abort_event = asyncio.Event() + exit_event = asyncio.Event() + first_sigint = True + + loop = asyncio.get_running_loop() + original_sigint_handler = signal.getsignal(signal.SIGINT) + + def _sigint_handler() -> None: + nonlocal first_sigint + if first_sigint: + first_sigint = False + abort_event.set() + print(ANSI.red("\nInterrupted. Press Ctrl+C again to quit."), flush=True) + return + + print(ANSI.red("\nExiting..."), flush=True) + exit_event.set() + + try: + sigint_registered_in_loop = False + try: + loop.add_signal_handler(signal.SIGINT, _sigint_handler) + sigint_registered_in_loop = True + except (AttributeError, NotImplementedError): + # Windows (or any loop that doesn't support it) : fall back to sync + signal.signal(signal.SIGINT, lambda *_: _sigint_handler()) + + # Handle inputs (i.e. env variables injection) + resolved_inputs: dict[str, str] = {} + + if len(inputs) > 0: + print( + ANSI.bold( + ANSI.blue( + "Some initial inputs are required by the agent. " + "Please provide a value or leave empty to load from env." + ) + ) + ) + for input_item in inputs: + input_id = input_item["id"] + description = input_item["description"] + env_special_value = f"${{input:{input_id}}}" + + # Check if the input is used by any server or as an apiKey + input_usages = set() + for server in servers: + # Check stdio's "env" and http/sse's "headers" mappings + env_or_headers = server.get("env", {}) if server["type"] == "stdio" else server.get("headers", {}) + for key, value in env_or_headers.items(): + if env_special_value in value: + input_usages.add(key) + + raw_api_key = config.get("apiKey") + if isinstance(raw_api_key, str) and env_special_value in raw_api_key: + input_usages.add("apiKey") + + if not input_usages: + print( + ANSI.yellow( + f"Input '{input_id}' defined in config but not used by any server or as an API key." + " Skipping." + ) + ) + continue + + # Prompt user for input + env_variable_key = input_id.replace("-", "_").upper() + print( + ANSI.blue(f" • {input_id}") + f": {description}. (default: load from {env_variable_key}).", + end=" ", + ) + user_input = (await _async_prompt(exit_event=exit_event)).strip() + if exit_event.is_set(): + return + + # Fallback to environment variable when user left blank + final_value = user_input + if not final_value: + final_value = os.getenv(env_variable_key, "") + if final_value: + print(ANSI.green(f"Value successfully loaded from '{env_variable_key}'")) + else: + print( + ANSI.yellow( + f"No value found for '{env_variable_key}' in environment variables. Continuing." + ) + ) + resolved_inputs[input_id] = final_value + + # Inject resolved value (can be empty) into stdio's env or http/sse's headers + for server in servers: + env_or_headers = server.get("env", {}) if server["type"] == "stdio" else server.get("headers", {}) + for key, value in env_or_headers.items(): + if env_special_value in value: + env_or_headers[key] = env_or_headers[key].replace(env_special_value, final_value) + + print() + + raw_api_key = config.get("apiKey") + if isinstance(raw_api_key, str): + substituted_api_key = raw_api_key + for input_id, val in resolved_inputs.items(): + substituted_api_key = substituted_api_key.replace(f"${{input:{input_id}}}", val) + config["apiKey"] = substituted_api_key + # Main agent loop + async with Agent( + provider=config.get("provider"), # type: ignore + model=config.get("model"), + base_url=config.get("endpointUrl"), # type: ignore[arg-type] + api_key=config.get("apiKey"), + servers=servers, # type: ignore[arg-type] + prompt=prompt, + ) as agent: + await agent.load_tools() + print(ANSI.bold(ANSI.blue("Agent loaded with {} tools:".format(len(agent.available_tools))))) + for t in agent.available_tools: + print(ANSI.blue(f" • {t.function.name}")) + + while True: + abort_event.clear() + + # Check if we should exit + if exit_event.is_set(): + return + + try: + user_input = await _async_prompt(exit_event=exit_event) + first_sigint = True + except EOFError: + print(ANSI.red("\nEOF received, exiting."), flush=True) + break + except KeyboardInterrupt: + if not first_sigint and abort_event.is_set(): + continue + else: + print(ANSI.red("\nKeyboard interrupt during input processing."), flush=True) + break + + try: + async for chunk in agent.run(user_input, abort_event=abort_event): + if abort_event.is_set() and not first_sigint: + break + if exit_event.is_set(): + return + + if hasattr(chunk, "choices"): + delta = chunk.choices[0].delta + if delta.content: + print(delta.content, end="", flush=True) + if delta.tool_calls: + for call in delta.tool_calls: + if call.id: + print(f"", end="") + if call.function.name: + print(f"{call.function.name}", end=" ") + if call.function.arguments: + print(f"{call.function.arguments}", end="") + else: + print( + ANSI.green(f"\n\nTool[{chunk.name}] {chunk.tool_call_id}\n{chunk.content}\n"), + flush=True, + ) + + print() + + except Exception as e: + tb_str = traceback.format_exc() + print(ANSI.red(f"\nError during agent run: {e}\n{tb_str}"), flush=True) + first_sigint = True # Allow graceful interrupt for the next command + + except Exception as e: + tb_str = traceback.format_exc() + print(ANSI.red(f"\nAn unexpected error occurred: {e}\n{tb_str}"), flush=True) + raise e + + finally: + if sigint_registered_in_loop: + try: + loop.remove_signal_handler(signal.SIGINT) + except (AttributeError, NotImplementedError): + pass + else: + signal.signal(signal.SIGINT, original_sigint_handler) + + +@app.command("run", help="Run the Agent in the CLI") +@click.argument("path", required=False) +def run(path: Optional[str]) -> None: + """ + Run the agent from PATH: a local folder containing an agent.json file or a built-in agent + stored in the 'tiny-agents/tiny-agents' Hugging Face dataset + (https://huggingface.co/datasets/tiny-agents/tiny-agents). + """ + try: + asyncio.run(run_agent(path)) + except KeyboardInterrupt: + print(ANSI.red("\nApplication terminated by KeyboardInterrupt."), flush=True) + raise click.exceptions.Exit(code=130) + except Exception as e: + print(ANSI.red(f"\nAn unexpected error occurred: {e}"), flush=True) + raise e + + +if __name__ == "__main__": + app() diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/constants.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..7fb7b562c4ff2df3c0fadb72dfd64cfe01144f36 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/constants.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +from huggingface_hub import ChatCompletionInputTool + + +FILENAME_CONFIG = "agent.json" +PROMPT_FILENAMES = ("PROMPT.md", "AGENTS.md") + +DEFAULT_AGENT = { + "model": "Qwen/Qwen2.5-72B-Instruct", + "provider": "novita", + "servers": [ + { + "type": "stdio", + "command": "npx", + "args": [ + "-y", + "@modelcontextprotocol/server-filesystem", + str(Path.home() / ("Desktop" if sys.platform == "darwin" else "")), + ], + }, + { + "type": "stdio", + "command": "npx", + "args": ["@playwright/mcp@latest"], + }, + ], +} + + +DEFAULT_SYSTEM_PROMPT = """ +You are an agent - please keep going until the user’s query is completely +resolved, before ending your turn and yielding back to the user. Only terminate +your turn when you are sure that the problem is solved, or if you need more +info from the user to solve the problem. +If you are not sure about anything pertaining to the user’s request, use your +tools to read files and gather the relevant information: do NOT guess or make +up an answer. +You MUST plan extensively before each function call, and reflect extensively +on the outcomes of the previous function calls. DO NOT do this entire process +by making function calls only, as this can impair your ability to solve the +problem and think insightfully. +""".strip() + +MAX_NUM_TURNS = 10 + +TASK_COMPLETE_TOOL: ChatCompletionInputTool = ChatCompletionInputTool.parse_obj( # type: ignore + { + "type": "function", + "function": { + "name": "task_complete", + "description": "Call this tool when the task given by the user is complete", + "parameters": { + "type": "object", + "properties": {}, + }, + }, + } +) + +ASK_QUESTION_TOOL: ChatCompletionInputTool = ChatCompletionInputTool.parse_obj( # type: ignore + { + "type": "function", + "function": { + "name": "ask_question", + "description": "Ask the user for more info required to solve or clarify their problem.", + "parameters": { + "type": "object", + "properties": {}, + }, + }, + } +) + +EXIT_LOOP_TOOLS: list[ChatCompletionInputTool] = [TASK_COMPLETE_TOOL, ASK_QUESTION_TOOL] + + +DEFAULT_REPO_ID = "tiny-agents/tiny-agents" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/mcp_client.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/mcp_client.py new file mode 100644 index 0000000000000000000000000000000000000000..7331d13fa619f513c78ac8f1bd795a26f0695201 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/mcp_client.py @@ -0,0 +1,395 @@ +import json +import logging +from contextlib import AsyncExitStack +from datetime import timedelta +from pathlib import Path +from typing import TYPE_CHECKING, Any, AsyncIterable, Literal, Optional, TypedDict, Union, overload + +from typing_extensions import NotRequired, TypeAlias, Unpack + +from ...utils._runtime import get_hf_hub_version +from .._generated._async_client import AsyncInferenceClient +from .._generated.types import ( + ChatCompletionInputMessage, + ChatCompletionInputTool, + ChatCompletionStreamOutput, + ChatCompletionStreamOutputDeltaToolCall, +) +from .._providers import PROVIDER_OR_POLICY_T +from .utils import format_result + + +if TYPE_CHECKING: + from mcp import ClientSession + +logger = logging.getLogger(__name__) + +# Type alias for tool names +ToolName: TypeAlias = str + +ServerType: TypeAlias = Literal["stdio", "sse", "http"] + + +class StdioServerParameters_T(TypedDict): + command: str + args: NotRequired[list[str]] + env: NotRequired[dict[str, str]] + cwd: NotRequired[Union[str, Path, None]] + + +class SSEServerParameters_T(TypedDict): + url: str + headers: NotRequired[dict[str, Any]] + timeout: NotRequired[float] + sse_read_timeout: NotRequired[float] + + +class StreamableHTTPParameters_T(TypedDict): + url: str + headers: NotRequired[dict[str, Any]] + timeout: NotRequired[timedelta] + sse_read_timeout: NotRequired[timedelta] + terminate_on_close: NotRequired[bool] + + +class MCPClient: + """ + Client for connecting to one or more MCP servers and processing chat completions with tools. + + > [!WARNING] + > This class is experimental and might be subject to breaking changes in the future without prior notice. + + Args: + model (`str`, `optional`): + The model to run inference with. Can be a model id hosted on the Hugging Face Hub, e.g. `meta-llama/Meta-Llama-3-8B-Instruct` + or a URL to a deployed Inference Endpoint or other local or remote endpoint. + provider (`str`, *optional*): + Name of the provider to use for inference. Defaults to "auto" i.e. the first of the providers available for the model, sorted by the user's order in https://hf.co/settings/inference-providers. + If model is a URL or `base_url` is passed, then `provider` is not used. + base_url (`str`, *optional*): + The base URL to run inference. Defaults to None. + api_key (`str`, `optional`): + Token to use for authentication. Will default to the locally Hugging Face saved token if not provided. You can also use your own provider API key to interact directly with the provider's service. + """ + + def __init__( + self, + *, + model: Optional[str] = None, + provider: Optional[PROVIDER_OR_POLICY_T] = None, + base_url: Optional[str] = None, + api_key: Optional[str] = None, + ): + # Initialize MCP sessions as a dictionary of ClientSession objects + self.sessions: dict[ToolName, "ClientSession"] = {} + self.exit_stack = AsyncExitStack() + self.available_tools: list[ChatCompletionInputTool] = [] + # To be able to send the model in the payload if `base_url` is provided + if model is None and base_url is None: + raise ValueError("At least one of `model` or `base_url` should be set in `MCPClient`.") + self.payload_model = model + self.client = AsyncInferenceClient( + model=None if base_url is not None else model, + provider=provider, + api_key=api_key, + base_url=base_url, + ) + + async def __aenter__(self): + """Enter the context manager""" + await self.client.__aenter__() + await self.exit_stack.__aenter__() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Exit the context manager""" + await self.client.__aexit__(exc_type, exc_val, exc_tb) + await self.cleanup() + + async def cleanup(self): + """Clean up resources""" + await self.client.close() + await self.exit_stack.aclose() + + @overload + async def add_mcp_server(self, type: Literal["stdio"], **params: Unpack[StdioServerParameters_T]): ... + + @overload + async def add_mcp_server(self, type: Literal["sse"], **params: Unpack[SSEServerParameters_T]): ... + + @overload + async def add_mcp_server(self, type: Literal["http"], **params: Unpack[StreamableHTTPParameters_T]): ... + + async def add_mcp_server(self, type: ServerType, **params: Any): + """Connect to an MCP server + + Args: + type (`str`): + Type of the server to connect to. Can be one of: + - "stdio": Standard input/output server (local) + - "sse": Server-sent events (SSE) server + - "http": StreamableHTTP server + **params (`dict[str, Any]`): + Server parameters that can be either: + - For stdio servers: + - command (str): The command to run the MCP server + - args (list[str], optional): Arguments for the command + - env (dict[str, str], optional): Environment variables for the command + - cwd (Union[str, Path, None], optional): Working directory for the command + - allowed_tools (list[str], optional): List of tool names to allow from this server + - For SSE servers: + - url (str): The URL of the SSE server + - headers (dict[str, Any], optional): Headers for the SSE connection + - timeout (float, optional): Connection timeout + - sse_read_timeout (float, optional): SSE read timeout + - allowed_tools (list[str], optional): List of tool names to allow from this server + - For StreamableHTTP servers: + - url (str): The URL of the StreamableHTTP server + - headers (dict[str, Any], optional): Headers for the StreamableHTTP connection + - timeout (timedelta, optional): Connection timeout + - sse_read_timeout (timedelta, optional): SSE read timeout + - terminate_on_close (bool, optional): Whether to terminate on close + - allowed_tools (list[str], optional): List of tool names to allow from this server + """ + from mcp import ClientSession, StdioServerParameters + from mcp import types as mcp_types + + # Extract allowed_tools configuration if provided + allowed_tools = params.pop("allowed_tools", None) + + # Determine server type and create appropriate parameters + if type == "stdio": + # Handle stdio server + from mcp.client.stdio import stdio_client + + logger.info(f"Connecting to stdio MCP server with command: {params['command']} {params.get('args', [])}") + + client_kwargs = {"command": params["command"]} + for key in ["args", "env", "cwd"]: + if params.get(key) is not None: + client_kwargs[key] = params[key] + server_params = StdioServerParameters(**client_kwargs) + read, write = await self.exit_stack.enter_async_context(stdio_client(server_params)) + elif type == "sse": + # Handle SSE server + from mcp.client.sse import sse_client + + logger.info(f"Connecting to SSE MCP server at: {params['url']}") + + client_kwargs = {"url": params["url"]} + for key in ["headers", "timeout", "sse_read_timeout"]: + if params.get(key) is not None: + client_kwargs[key] = params[key] + read, write = await self.exit_stack.enter_async_context(sse_client(**client_kwargs)) + elif type == "http": + # Handle StreamableHTTP server + from mcp.client.streamable_http import streamablehttp_client + + logger.info(f"Connecting to StreamableHTTP MCP server at: {params['url']}") + + client_kwargs = {"url": params["url"]} + for key in ["headers", "timeout", "sse_read_timeout", "terminate_on_close"]: + if params.get(key) is not None: + client_kwargs[key] = params[key] + read, write, _ = await self.exit_stack.enter_async_context(streamablehttp_client(**client_kwargs)) + # ^ TODO: should be handle `get_session_id_callback`? (function to retrieve the current session ID) + else: + raise ValueError(f"Unsupported server type: {type}") + + session = await self.exit_stack.enter_async_context( + ClientSession( + read_stream=read, + write_stream=write, + client_info=mcp_types.Implementation( + name="huggingface_hub.MCPClient", + version=get_hf_hub_version(), + ), + ) + ) + + logger.debug("Initializing session...") + await session.initialize() + + # List available tools + response = await session.list_tools() + logger.debug("Connected to server with tools:", [tool.name for tool in response.tools]) + + # Filter tools based on allowed_tools configuration + filtered_tools = response.tools + + if allowed_tools is not None: + filtered_tools = [tool for tool in response.tools if tool.name in allowed_tools] + logger.debug( + f"Tool filtering applied. Using {len(filtered_tools)} of {len(response.tools)} available tools: {[tool.name for tool in filtered_tools]}" + ) + + for tool in filtered_tools: + if tool.name in self.sessions: + logger.warning(f"Tool '{tool.name}' already defined by another server. Skipping.") + continue + + # Map tool names to their server for later lookup + self.sessions[tool.name] = session + + # Add tool to the list of available tools (for use in chat completions) + self.available_tools.append( + ChatCompletionInputTool.parse_obj_as_instance( + { + "type": "function", + "function": { + "name": tool.name, + "description": tool.description, + "parameters": tool.inputSchema, + }, + } + ) + ) + + async def process_single_turn_with_tools( + self, + messages: list[Union[dict, ChatCompletionInputMessage]], + exit_loop_tools: Optional[list[ChatCompletionInputTool]] = None, + exit_if_first_chunk_no_tool: bool = False, + ) -> AsyncIterable[Union[ChatCompletionStreamOutput, ChatCompletionInputMessage]]: + """Process a query using `self.model` and available tools, yielding chunks and tool outputs. + + Args: + messages (`list[dict]`): + List of message objects representing the conversation history + exit_loop_tools (`list[ChatCompletionInputTool]`, *optional*): + List of tools that should exit the generator when called + exit_if_first_chunk_no_tool (`bool`, *optional*): + Exit if no tool is present in the first chunks. Default to False. + + Yields: + [`ChatCompletionStreamOutput`] chunks or [`ChatCompletionInputMessage`] objects + """ + # Prepare tools list based on options + tools = self.available_tools + if exit_loop_tools is not None: + tools = [*exit_loop_tools, *self.available_tools] + + # Create the streaming request + response = await self.client.chat.completions.create( + model=self.payload_model, + messages=messages, + tools=tools, + tool_choice="auto", + stream=True, + ) + + message: dict[str, Any] = {"role": "unknown", "content": ""} + final_tool_calls: dict[int, ChatCompletionStreamOutputDeltaToolCall] = {} + num_of_chunks = 0 + + # Read from stream + async for chunk in response: + num_of_chunks += 1 + delta = chunk.choices[0].delta if chunk.choices and len(chunk.choices) > 0 else None + if not delta: + continue + + # Process message + if delta.role: + message["role"] = delta.role + if delta.content: + message["content"] += delta.content + + # Process tool calls + if delta.tool_calls: + for tool_call in delta.tool_calls: + idx = tool_call.index + # first chunk for this tool call + if idx not in final_tool_calls: + final_tool_calls[idx] = tool_call + if final_tool_calls[idx].function.arguments is None: + final_tool_calls[idx].function.arguments = "" + continue + # safety before concatenating text to .function.arguments + if final_tool_calls[idx].function.arguments is None: + final_tool_calls[idx].function.arguments = "" + + if tool_call.function.arguments: + final_tool_calls[idx].function.arguments += tool_call.function.arguments + + # Optionally exit early if no tools in first chunks + if exit_if_first_chunk_no_tool and num_of_chunks <= 2 and len(final_tool_calls) == 0: + return + + # Yield each chunk to caller + yield chunk + + # Add the assistant message with tool calls (if any) to messages + if message["content"] or final_tool_calls: + # if the role is unknown, set it to assistant + if message.get("role") == "unknown": + message["role"] = "assistant" + # Convert final_tool_calls to the format expected by OpenAI + if final_tool_calls: + tool_calls_list: list[dict[str, Any]] = [] + for tc in final_tool_calls.values(): + tool_calls_list.append( + { + "id": tc.id, + "type": "function", + "function": { + "name": tc.function.name, + "arguments": tc.function.arguments or "{}", + }, + } + ) + message["tool_calls"] = tool_calls_list + messages.append(message) + + # Process tool calls one by one + for tool_call in final_tool_calls.values(): + function_name = tool_call.function.name + if function_name is None: + message = ChatCompletionInputMessage.parse_obj_as_instance( + { + "role": "tool", + "tool_call_id": tool_call.id, + "content": "Invalid tool call with no function name.", + } + ) + messages.append(message) + yield message + continue # move to next tool call + try: + function_args = json.loads(tool_call.function.arguments or "{}") + except json.JSONDecodeError as err: + tool_message = { + "role": "tool", + "tool_call_id": tool_call.id, + "name": function_name, + "content": f"Invalid JSON generated by the model: {err}", + } + tool_message_as_obj = ChatCompletionInputMessage.parse_obj_as_instance(tool_message) + messages.append(tool_message_as_obj) + yield tool_message_as_obj + continue # move to next tool call + + tool_message = {"role": "tool", "tool_call_id": tool_call.id, "content": "", "name": function_name} + + # Check if this is an exit loop tool + if exit_loop_tools and function_name in [t.function.name for t in exit_loop_tools]: + tool_message_as_obj = ChatCompletionInputMessage.parse_obj_as_instance(tool_message) + messages.append(tool_message_as_obj) + yield tool_message_as_obj + return + + # Execute tool call with the appropriate session + session = self.sessions.get(function_name) + if session is not None: + try: + result = await session.call_tool(function_name, function_args) + tool_message["content"] = format_result(result) + except Exception as err: + tool_message["content"] = f"Error: MCP tool call failed with error message: {err}" + else: + tool_message["content"] = f"Error: No session found for tool: {function_name}" + + # Yield tool message + tool_message_as_obj = ChatCompletionInputMessage.parse_obj_as_instance(tool_message) + messages.append(tool_message_as_obj) + yield tool_message_as_obj diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/types.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/types.py new file mode 100644 index 0000000000000000000000000000000000000000..a531929a8e574e52075b59cec88e24d93cb1dbd7 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/types.py @@ -0,0 +1,45 @@ +from typing import Literal, TypedDict, Union + +from typing_extensions import NotRequired + + +class InputConfig(TypedDict, total=False): + id: str + description: str + type: str + password: bool + + +class StdioServerConfig(TypedDict): + type: Literal["stdio"] + command: str + args: list[str] + env: dict[str, str] + cwd: str + allowed_tools: NotRequired[list[str]] + + +class HTTPServerConfig(TypedDict): + type: Literal["http"] + url: str + headers: dict[str, str] + allowed_tools: NotRequired[list[str]] + + +class SSEServerConfig(TypedDict): + type: Literal["sse"] + url: str + headers: dict[str, str] + allowed_tools: NotRequired[list[str]] + + +ServerConfig = Union[StdioServerConfig, HTTPServerConfig, SSEServerConfig] + + +# AgentConfig root object +class AgentConfig(TypedDict): + model: str + provider: str + apiKey: NotRequired[str] + inputs: list[InputConfig] + servers: list[ServerConfig] diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/utils.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5df9d5183df75f11ad902f514b9f2b6cf5041d6f --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/utils.py @@ -0,0 +1,130 @@ +""" +Utility functions for MCPClient and Tiny Agents. + +Formatting utilities taken from the JS SDK: https://github.com/huggingface/huggingface.js/blob/main/packages/mcp-client/src/ResultFormatter.ts. +""" + +import json +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +from huggingface_hub import snapshot_download +from huggingface_hub.errors import EntryNotFoundError + +from .constants import DEFAULT_AGENT, DEFAULT_REPO_ID, FILENAME_CONFIG, PROMPT_FILENAMES +from .types import AgentConfig + + +if TYPE_CHECKING: + from mcp import types as mcp_types + + +def format_result(result: "mcp_types.CallToolResult") -> str: + """ + Formats a mcp.types.CallToolResult content into a human-readable string. + + Args: + result (CallToolResult) + Object returned by mcp.ClientSession.call_tool. + + Returns: + str + A formatted string representing the content of the result. + """ + content = result.content + + if len(content) == 0: + return "[No content]" + + formatted_parts: list[str] = [] + + for item in content: + match item.type: + case "text": + formatted_parts.append(item.text) + + case "image": + formatted_parts.append( + f"[Binary Content: Image {item.mimeType}, {_get_base64_size(item.data)} bytes]\n" + f"The task is complete and the content accessible to the User" + ) + + case "audio": + formatted_parts.append( + f"[Binary Content: Audio {item.mimeType}, {_get_base64_size(item.data)} bytes]\n" + f"The task is complete and the content accessible to the User" + ) + + case "resource": + resource = item.resource + + if hasattr(resource, "text") and isinstance(resource.text, str): + formatted_parts.append(resource.text) + + elif hasattr(resource, "blob") and isinstance(resource.blob, str): + formatted_parts.append( + f"[Binary Content ({resource.uri}): {resource.mimeType}," + f" {_get_base64_size(resource.blob)} bytes]\n" + f"The task is complete and the content accessible to the User" + ) + + return "\n".join(formatted_parts) + + +def _get_base64_size(base64_str: str) -> int: + """Estimate the byte size of a base64-encoded string.""" + # Remove any prefix like "data:image/png;base64," + if "," in base64_str: + base64_str = base64_str.split(",")[1] + + padding = 0 + if base64_str.endswith("=="): + padding = 2 + elif base64_str.endswith("="): + padding = 1 + + return (len(base64_str) * 3) // 4 - padding + + +def _load_agent_config(agent_path: Optional[str]) -> tuple[AgentConfig, Optional[str]]: + """Load server config and prompt.""" + + def _read_dir(directory: Path) -> tuple[AgentConfig, Optional[str]]: + cfg_file = directory / FILENAME_CONFIG + if not cfg_file.exists(): + raise FileNotFoundError(f" Config file not found in {directory}! Please make sure it exists locally") + + config: AgentConfig = json.loads(cfg_file.read_text(encoding="utf-8")) + prompt: Optional[str] = None + for filename in PROMPT_FILENAMES: + prompt_file = directory / filename + if prompt_file.exists(): + prompt = prompt_file.read_text(encoding="utf-8") + break + return config, prompt + + if agent_path is None: + return DEFAULT_AGENT, None # type: ignore + + path = Path(agent_path).expanduser() + + if path.is_file(): + return json.loads(path.read_text(encoding="utf-8")), None + + if path.is_dir(): + return _read_dir(path) + + # fetch from the Hub + try: + repo_dir = Path( + snapshot_download( + repo_id=DEFAULT_REPO_ID, + allow_patterns=f"{agent_path}/*", + repo_type="dataset", + ) + ) + return _read_dir(repo_dir / agent_path) + except Exception as err: + raise EntryNotFoundError( + f" Agent {agent_path} not found in tiny-agents/tiny-agents! Please make sure it exists in https://huggingface.co/datasets/tiny-agents/tiny-agents." + ) from err diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__init__.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fec1b5040ec7bb259d5444e9a714beb9fb4f8e87 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__init__.py @@ -0,0 +1,254 @@ +from typing import Literal, Union + +from huggingface_hub.inference._providers.featherless_ai import ( + FeatherlessConversationalTask, + FeatherlessTextGenerationTask, +) +from huggingface_hub.utils import logging + +from ._common import AutoRouterConversationalTask, TaskProviderHelper, _fetch_inference_provider_mapping +from .cerebras import CerebrasConversationalTask +from .cohere import CohereConversationalTask +from .deepinfra import ( + DeepInfraAutomaticSpeechRecognitionTask, + DeepInfraConversationalTask, + DeepInfraTextGenerationTask, +) +from .fal_ai import ( + FalAIAutomaticSpeechRecognitionTask, + FalAIImageSegmentationTask, + FalAIImageToImageTask, + FalAIImageToVideoTask, + FalAITextToImageTask, + FalAITextToSpeechTask, + FalAITextToVideoTask, +) +from .fireworks_ai import FireworksAIConversationalTask +from .groq import GroqConversationalTask +from .hf_inference import ( + HFInferenceBinaryInputTask, + HFInferenceConversational, + HFInferenceFeatureExtractionTask, + HFInferenceTask, +) +from .novita import NovitaConversationalTask, NovitaTextGenerationTask, NovitaTextToVideoTask +from .nscale import NscaleConversationalTask, NscaleTextToImageTask +from .openai import OpenAIConversationalTask +from .ovhcloud import OVHcloudConversationalTask +from .publicai import PublicAIConversationalTask +from .replicate import ( + ReplicateAutomaticSpeechRecognitionTask, + ReplicateImageToImageTask, + ReplicateTask, + ReplicateTextToImageTask, + ReplicateTextToSpeechTask, +) +from .scaleway import ScalewayConversationalTask, ScalewayFeatureExtractionTask +from .together import ( + TogetherConversationalTask, + TogetherFeatureExtractionTask, + TogetherImageToImageTask, + TogetherImageToVideoTask, + TogetherTextGenerationTask, + TogetherTextToImageTask, + TogetherTextToSpeechTask, + TogetherTextToVideoTask, +) +from .wavespeed import ( + WavespeedAIImageToImageTask, + WavespeedAIImageToVideoTask, + WavespeedAITextToImageTask, + WavespeedAITextToVideoTask, +) +from .zai_org import ZaiConversationalTask, ZaiTextToImageTask + + +logger = logging.get_logger(__name__) + + +PROVIDER_T = Literal[ + "cerebras", + "cohere", + "deepinfra", + "fal-ai", + "featherless-ai", + "fireworks-ai", + "groq", + "hf-inference", + "novita", + "nscale", + "openai", + "ovhcloud", + "publicai", + "replicate", + "scaleway", + "together", + "wavespeed", + "zai-org", +] + +PROVIDER_OR_POLICY_T = Union[PROVIDER_T, Literal["auto"]] + +CONVERSATIONAL_AUTO_ROUTER = AutoRouterConversationalTask() + +PROVIDERS: dict[PROVIDER_T, dict[str, TaskProviderHelper]] = { + "cerebras": { + "conversational": CerebrasConversationalTask(), + }, + "cohere": { + "conversational": CohereConversationalTask(), + }, + "deepinfra": { + "automatic-speech-recognition": DeepInfraAutomaticSpeechRecognitionTask(), + "conversational": DeepInfraConversationalTask(), + "text-generation": DeepInfraTextGenerationTask(), + }, + "fal-ai": { + "automatic-speech-recognition": FalAIAutomaticSpeechRecognitionTask(), + "text-to-image": FalAITextToImageTask(), + "text-to-speech": FalAITextToSpeechTask(), + "text-to-video": FalAITextToVideoTask(), + "image-to-video": FalAIImageToVideoTask(), + "image-to-image": FalAIImageToImageTask(), + "image-segmentation": FalAIImageSegmentationTask(), + }, + "featherless-ai": { + "conversational": FeatherlessConversationalTask(), + "text-generation": FeatherlessTextGenerationTask(), + }, + "fireworks-ai": { + "conversational": FireworksAIConversationalTask(), + }, + "groq": { + "conversational": GroqConversationalTask(), + }, + "hf-inference": { + "text-to-image": HFInferenceTask("text-to-image"), + "conversational": HFInferenceConversational(), + "text-generation": HFInferenceTask("text-generation"), + "text-classification": HFInferenceTask("text-classification"), + "question-answering": HFInferenceTask("question-answering"), + "audio-classification": HFInferenceBinaryInputTask("audio-classification"), + "automatic-speech-recognition": HFInferenceBinaryInputTask("automatic-speech-recognition"), + "fill-mask": HFInferenceTask("fill-mask"), + "feature-extraction": HFInferenceFeatureExtractionTask(), + "image-classification": HFInferenceBinaryInputTask("image-classification"), + "image-segmentation": HFInferenceBinaryInputTask("image-segmentation"), + "document-question-answering": HFInferenceTask("document-question-answering"), + "image-to-text": HFInferenceBinaryInputTask("image-to-text"), + "object-detection": HFInferenceBinaryInputTask("object-detection"), + "audio-to-audio": HFInferenceBinaryInputTask("audio-to-audio"), + "zero-shot-image-classification": HFInferenceBinaryInputTask("zero-shot-image-classification"), + "zero-shot-classification": HFInferenceTask("zero-shot-classification"), + "image-to-image": HFInferenceBinaryInputTask("image-to-image"), + "sentence-similarity": HFInferenceTask("sentence-similarity"), + "table-question-answering": HFInferenceTask("table-question-answering"), + "tabular-classification": HFInferenceTask("tabular-classification"), + "text-to-speech": HFInferenceTask("text-to-speech"), + "token-classification": HFInferenceTask("token-classification"), + "translation": HFInferenceTask("translation"), + "summarization": HFInferenceTask("summarization"), + "visual-question-answering": HFInferenceBinaryInputTask("visual-question-answering"), + }, + "novita": { + "text-generation": NovitaTextGenerationTask(), + "conversational": NovitaConversationalTask(), + "text-to-video": NovitaTextToVideoTask(), + }, + "nscale": { + "conversational": NscaleConversationalTask(), + "text-to-image": NscaleTextToImageTask(), + }, + "openai": { + "conversational": OpenAIConversationalTask(), + }, + "ovhcloud": { + "conversational": OVHcloudConversationalTask(), + }, + "publicai": { + "conversational": PublicAIConversationalTask(), + }, + "replicate": { + "automatic-speech-recognition": ReplicateAutomaticSpeechRecognitionTask(), + "image-to-image": ReplicateImageToImageTask(), + "text-to-image": ReplicateTextToImageTask(), + "text-to-speech": ReplicateTextToSpeechTask(), + "text-to-video": ReplicateTask("text-to-video"), + }, + "scaleway": { + "conversational": ScalewayConversationalTask(), + "feature-extraction": ScalewayFeatureExtractionTask(), + }, + "together": { + "conversational": TogetherConversationalTask(), + "feature-extraction": TogetherFeatureExtractionTask(), + "image-to-image": TogetherImageToImageTask(), + "image-to-video": TogetherImageToVideoTask(), + "text-generation": TogetherTextGenerationTask(), + "text-to-image": TogetherTextToImageTask(), + "text-to-speech": TogetherTextToSpeechTask(), + "text-to-video": TogetherTextToVideoTask(), + }, + "wavespeed": { + "text-to-image": WavespeedAITextToImageTask(), + "text-to-video": WavespeedAITextToVideoTask(), + "image-to-image": WavespeedAIImageToImageTask(), + "image-to-video": WavespeedAIImageToVideoTask(), + }, + "zai-org": { + "conversational": ZaiConversationalTask(), + "text-to-image": ZaiTextToImageTask(), + }, +} + + +def get_provider_helper(provider: PROVIDER_OR_POLICY_T | None, task: str, model: str | None) -> TaskProviderHelper: + """Get provider helper instance by name and task. + + Args: + provider (`str`, *optional*): name of the provider, or "auto" to automatically select the provider for the model. + task (`str`): Name of the task + model (`str`, *optional*): Name of the model + Returns: + TaskProviderHelper: Helper instance for the specified provider and task + + Raises: + ValueError: If provider or task is not supported + """ + + if (model is None and provider in (None, "auto")) or ( + model is not None and model.startswith(("http://", "https://")) + ): + provider = "hf-inference" + + if provider is None: + logger.info( + "No provider specified for task `conversational`. Defaulting to server-side auto routing." + if task == "conversational" + else "Defaulting to 'auto' which will select the first provider available for the model, sorted by the user's order in https://hf.co/settings/inference-providers." + ) + provider = "auto" + + if provider == "auto": + if model is None: + raise ValueError("Specifying a model is required when provider is 'auto'") + if task == "conversational": + # Special case: we have a dedicated auto-router for conversational models. No need to fetch provider mapping. + return CONVERSATIONAL_AUTO_ROUTER + + provider_mapping = _fetch_inference_provider_mapping(model) + provider = next(iter(provider_mapping)).provider + + provider_tasks = PROVIDERS.get(provider) # type: ignore + if provider_tasks is None: + raise ValueError( + f"Provider '{provider}' not supported. Available values: 'auto' or any provider from {list(PROVIDERS.keys())}." + "Passing 'auto' (default value) will automatically select the first provider available for the model, sorted " + "by the user's order in https://hf.co/settings/inference-providers." + ) + + if task not in provider_tasks: + raise ValueError( + f"Task '{task}' not supported for provider '{provider}'. Available tasks: {list(provider_tasks.keys())}" + ) + return provider_tasks[task] diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..265ed4672bd362c485711194bedaa58397863311 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/_common.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/_common.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b47a854175fc20787350b53ba0bce8137d27ac0 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/_common.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cerebras.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cerebras.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2655668ed5afb89fa19d8500addb661bc321e42b Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cerebras.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cohere.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cohere.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..74557c01a80627643659fb9e05cc44708b1efff9 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cohere.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/deepinfra.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/deepinfra.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d684e6f64197929ae920ee73143274c9b96a25ef Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/deepinfra.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fal_ai.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fal_ai.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2087bbca35160939e37404586590a7bbb4d88c1d Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fal_ai.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/featherless_ai.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/featherless_ai.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..31cd3327f07b18c7b3b6028bd406fad3df98eb04 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/featherless_ai.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fireworks_ai.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fireworks_ai.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d92efc651fa7953921f093a0aa6f4beb4d2c78d Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fireworks_ai.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/groq.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/groq.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b2fa9dcef26bb51e63545d7bb8297b9893b0e53 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/groq.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/hf_inference.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/hf_inference.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66ff56c7ce9a7b1837636f72ab088b7e91529a35 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/hf_inference.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/novita.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/novita.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6e7645788ea22a5b63a666a307c8513755457f94 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/novita.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/nscale.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/nscale.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4421648d72a4298e0fb4e8eb4df967dd6a5f2576 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/nscale.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/openai.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/openai.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d0b4deb493ac84cebeb8b94941d146461ea895c7 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/openai.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/ovhcloud.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/ovhcloud.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8ba71ed1bc86276c81f74b7687a3efa8c376871 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/ovhcloud.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/publicai.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/publicai.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cd945401fd497a4261e933356e4b2f7ace7ce1c9 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/publicai.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/replicate.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/replicate.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b86bedc10a5ba0fb7f56e3494238329dd53ac0b Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/replicate.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/scaleway.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/scaleway.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff90654dc5b849968c084fa2af540a65dba522be Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/scaleway.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/together.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/together.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..815c14810d73cc9f623f05966ab4b2adb2b0d0ab Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/together.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/wavespeed.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/wavespeed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dca41b0b3f38ebbb45ba9f218d64d7e716f50684 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/wavespeed.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/zai_org.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/zai_org.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2852f9d41cf841c1a7ea9164583cf605b26f2727 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/zai_org.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/_common.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/_common.py new file mode 100644 index 0000000000000000000000000000000000000000..25330e70de8aab6f9cd47ab4d8fca183a6bef434 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/_common.py @@ -0,0 +1,360 @@ +from functools import lru_cache +from typing import Any, overload + +from huggingface_hub import constants +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import MimeBytes, RequestParameters +from huggingface_hub.inference._generated.types.chat_completion import ChatCompletionInputMessage +from huggingface_hub.utils import build_hf_headers, get_token, logging + + +logger = logging.get_logger(__name__) + + +# Dev purposes only. +# If you want to try to run inference for a new model locally before it's registered on huggingface.co +# for a given Inference Provider, you can add it to the following dictionary. +HARDCODED_MODEL_INFERENCE_MAPPING: dict[str, dict[str, InferenceProviderMapping]] = { + # "HF model ID" => InferenceProviderMapping object initialized with "Model ID on Inference Provider's side" + # + # Example: + # "Qwen/Qwen2.5-Coder-32B-Instruct": InferenceProviderMapping(hf_model_id="Qwen/Qwen2.5-Coder-32B-Instruct", + # provider_id="Qwen2.5-Coder-32B-Instruct", + # task="conversational", + # status="live") + "cerebras": {}, + "cohere": {}, + "deepinfra": {}, + "fal-ai": {}, + "fireworks-ai": {}, + "groq": {}, + "hf-inference": {}, + "nscale": {}, + "ovhcloud": {}, + "replicate": {}, + "scaleway": {}, + "together": {}, + "wavespeed": {}, + "zai-org": {}, +} + + +@overload +def filter_none(obj: dict[str, Any]) -> dict[str, Any]: ... +@overload +def filter_none(obj: list[Any]) -> list[Any]: ... + + +def filter_none(obj: dict[str, Any] | list[Any]) -> dict[str, Any] | list[Any]: + if isinstance(obj, dict): + cleaned: dict[str, Any] = {} + for k, v in obj.items(): + if v is None: + continue + if isinstance(v, (dict, list)): + v = filter_none(v) + cleaned[k] = v + return cleaned + + if isinstance(obj, list): + return [filter_none(v) if isinstance(v, (dict, list)) else v for v in obj] + + raise ValueError(f"Expected dict or list, got {type(obj)}") + + +class TaskProviderHelper: + """Base class for task-specific provider helpers.""" + + def __init__(self, provider: str, base_url: str, task: str) -> None: + self.provider = provider + self.task = task + self.base_url = base_url + + def prepare_request( + self, + *, + inputs: Any, + parameters: dict[str, Any], + headers: dict, + model: str | None, + api_key: str | None, + extra_payload: dict[str, Any] | None = None, + ) -> RequestParameters: + """ + Prepare the request to be sent to the provider. + + Each step (api_key, model, headers, url, payload) can be customized in subclasses. + """ + # api_key from user, or local token, or raise error + api_key = self._prepare_api_key(api_key) + + # mapped model from HF model ID + provider_mapping_info = self._prepare_mapping_info(model) + + # default HF headers + user headers (to customize in subclasses) + headers = self._prepare_headers(headers, api_key) + + # routed URL if HF token, or direct URL (to customize in '_prepare_route' in subclasses) + url = self._prepare_url(api_key, provider_mapping_info.provider_id) + + # prepare payload (to customize in subclasses) + payload = self._prepare_payload_as_dict(inputs, parameters, provider_mapping_info=provider_mapping_info) + if payload is not None: + payload = recursive_merge(payload, filter_none(extra_payload or {})) + + # body data (to customize in subclasses) + data = self._prepare_payload_as_bytes(inputs, parameters, provider_mapping_info, extra_payload) + + # check if both payload and data are set and return + if payload is not None and data is not None: + raise ValueError("Both payload and data cannot be set in the same request.") + if payload is None and data is None: + raise ValueError("Either payload or data must be set in the request.") + + # normalize headers to lowercase and add content-type if not present + normalized_headers = self._normalize_headers(headers, payload, data) + + return RequestParameters( + url=url, + task=self.task, + model=provider_mapping_info.provider_id, + json=payload, + data=data, + headers=normalized_headers, + ) + + def get_response( + self, + response: bytes | dict, + request_params: RequestParameters | None = None, + ) -> Any: + """ + Return the response in the expected format. + + Override this method in subclasses for customized response handling.""" + return response + + def _prepare_api_key(self, api_key: str | None) -> str: + """Return the API key to use for the request. + + Usually not overwritten in subclasses.""" + if api_key is None: + api_key = get_token() + if api_key is None: + raise ValueError( + f"You must provide an api_key to work with {self.provider} API or log in with `hf auth login`." + ) + return api_key + + def _prepare_mapping_info(self, model: str | None) -> InferenceProviderMapping: + """Return the mapped model ID to use for the request. + + Usually not overwritten in subclasses.""" + if model is None: + raise ValueError(f"Please provide an HF model ID supported by {self.provider}.") + + # hardcoded mapping for local testing + if HARDCODED_MODEL_INFERENCE_MAPPING.get(self.provider, {}).get(model): + return HARDCODED_MODEL_INFERENCE_MAPPING[self.provider][model] + + provider_mapping = None + for mapping in _fetch_inference_provider_mapping(model): + if mapping.provider == self.provider: + provider_mapping = mapping + break + + if provider_mapping is None: + raise ValueError(f"Model {model} is not supported by provider {self.provider}.") + + if provider_mapping.task != self.task: + raise ValueError( + f"Model {model} is not supported for task {self.task} and provider {self.provider}. " + f"Supported task: {provider_mapping.task}." + ) + + if provider_mapping.status == "staging": + logger.warning( + f"Model {model} is in staging mode for provider {self.provider}. Meant for test purposes only." + ) + if provider_mapping.status == "error": + logger.warning( + f"Our latest automated health check on model '{model}' for provider '{self.provider}' did not complete successfully. " + "Inference call might fail." + ) + return provider_mapping + + def _normalize_headers( + self, headers: dict[str, Any], payload: dict[str, Any] | None, data: MimeBytes | None + ) -> dict[str, Any]: + """Normalize the headers to use for the request. + + Override this method in subclasses for customized headers. + """ + normalized_headers = {key.lower(): value for key, value in headers.items() if value is not None} + if normalized_headers.get("content-type") is None: + if data is not None and data.mime_type is not None: + normalized_headers["content-type"] = data.mime_type + elif payload is not None: + normalized_headers["content-type"] = "application/json" + return normalized_headers + + def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]: + """Return the headers to use for the request. + + Override this method in subclasses for customized headers. + """ + return {**build_hf_headers(token=api_key), **headers} + + def _prepare_url(self, api_key: str, mapped_model: str) -> str: + """Return the URL to use for the request. + + Usually not overwritten in subclasses.""" + base_url = self._prepare_base_url(api_key) + route = self._prepare_route(mapped_model, api_key) + return f"{base_url.rstrip('/')}/{route.lstrip('/')}" + + def _prepare_base_url(self, api_key: str) -> str: + """Return the base URL to use for the request. + + Usually not overwritten in subclasses.""" + # Route to the proxy if the api_key is a HF TOKEN + if api_key.startswith("hf_"): + logger.info(f"Calling '{self.provider}' provider through Hugging Face router.") + return constants.INFERENCE_PROXY_TEMPLATE.format(provider=self.provider) + else: + logger.info(f"Calling '{self.provider}' provider directly.") + return self.base_url + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + """Return the route to use for the request. + + Override this method in subclasses for customized routes. + """ + return "" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + """Return the payload to use for the request, as a dict. + + Override this method in subclasses for customized payloads. + Only one of `_prepare_payload_as_dict` and `_prepare_payload_as_bytes` should return a value. + """ + return None + + def _prepare_payload_as_bytes( + self, + inputs: Any, + parameters: dict, + provider_mapping_info: InferenceProviderMapping, + extra_payload: dict | None, + ) -> MimeBytes | None: + """Return the body to use for the request, as bytes. + + Override this method in subclasses for customized body data. + Only one of `_prepare_payload_as_dict` and `_prepare_payload_as_bytes` should return a value. + """ + return None + + +class BaseConversationalTask(TaskProviderHelper): + """ + Base class for conversational (chat completion) tasks. + The schema follows the OpenAI API format defined here: https://platform.openai.com/docs/api-reference/chat + """ + + def __init__(self, provider: str, base_url: str): + super().__init__(provider=provider, base_url=base_url, task="conversational") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/chat/completions" + + def _prepare_payload_as_dict( + self, + inputs: list[dict | ChatCompletionInputMessage], + parameters: dict, + provider_mapping_info: InferenceProviderMapping, + ) -> dict | None: + return filter_none({"messages": inputs, **parameters, "model": provider_mapping_info.provider_id}) + + +class AutoRouterConversationalTask(BaseConversationalTask): + """ + Auto-router for conversational tasks. + + We let the Hugging Face router select the best provider for the model, based on availability and user preferences. + This is a special case since the selection is done server-side (avoid 1 API call to fetch provider mapping). + """ + + def __init__(self): + super().__init__(provider="auto", base_url="https://router.huggingface.co") + + def _prepare_base_url(self, api_key: str) -> str: + """Return the base URL to use for the request. + + Usually not overwritten in subclasses.""" + # Route to the proxy if the api_key is a HF TOKEN + if not api_key.startswith("hf_"): + raise ValueError("Cannot select auto-router when using non-Hugging Face API key.") + else: + return self.base_url # No `/auto` suffix in the URL + + def _prepare_mapping_info(self, model: str | None) -> InferenceProviderMapping: + """ + In auto-router, we don't need to fetch provider mapping info. + We just return a dummy mapping info with provider_id set to the HF model ID. + """ + if model is None: + raise ValueError("Please provide an HF model ID.") + + return InferenceProviderMapping( + provider="auto", + hf_model_id=model, + providerId=model, + status="live", + task="conversational", + ) + + +class BaseTextGenerationTask(TaskProviderHelper): + """ + Base class for text-generation (completion) tasks. + The schema follows the OpenAI API format defined here: https://platform.openai.com/docs/api-reference/completions + """ + + def __init__(self, provider: str, base_url: str): + super().__init__(provider=provider, base_url=base_url, task="text-generation") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/completions" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + return filter_none({"prompt": inputs, **parameters, "model": provider_mapping_info.provider_id}) + + +@lru_cache(maxsize=None) +def _fetch_inference_provider_mapping(model: str) -> list["InferenceProviderMapping"]: + """ + Fetch provider mappings for a model from the Hub. + """ + from huggingface_hub.hf_api import HfApi + + info = HfApi().model_info(model, expand=["inferenceProviderMapping"]) + provider_mapping = info.inference_provider_mapping + if provider_mapping is None: + raise ValueError(f"No provider mapping found for model {model}") + return provider_mapping + + +def recursive_merge(dict1: dict, dict2: dict) -> dict: + return { + **dict1, + **{ + key: recursive_merge(dict1[key], value) + if (key in dict1 and isinstance(dict1[key], dict) and isinstance(value, dict)) + else value + for key, value in dict2.items() + }, + } diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cerebras.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cerebras.py new file mode 100644 index 0000000000000000000000000000000000000000..a9b9c3aacb3e134a8e755297c15ece198ffe633d --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cerebras.py @@ -0,0 +1,6 @@ +from ._common import BaseConversationalTask + + +class CerebrasConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="cerebras", base_url="https://api.cerebras.ai") diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cohere.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cohere.py new file mode 100644 index 0000000000000000000000000000000000000000..57ddfc8246bcd2f3d63c74f78ded130dcca53992 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cohere.py @@ -0,0 +1,32 @@ +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping + +from ._common import BaseConversationalTask + + +_PROVIDER = "cohere" +_BASE_URL = "https://api.cohere.com" + + +class CohereConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/compatibility/v1/chat/completions" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) + response_format = parameters.get("response_format") + if isinstance(response_format, dict) and response_format.get("type") == "json_schema": + json_schema_details = response_format.get("json_schema") + if isinstance(json_schema_details, dict) and "schema" in json_schema_details: + payload["response_format"] = { # type: ignore + "type": "json_object", + "schema": json_schema_details["schema"], + } + + return payload diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/deepinfra.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/deepinfra.py new file mode 100644 index 0000000000000000000000000000000000000000..19431258b4b083e61b94166bbbe12f676e382d0c --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/deepinfra.py @@ -0,0 +1,120 @@ +import json +import mimetypes +import uuid +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import MimeBytes, RequestParameters, _as_dict, _open_as_mime_bytes + +from ._common import BaseConversationalTask, BaseTextGenerationTask, TaskProviderHelper, filter_none + + +_PROVIDER = "deepinfra" +_BASE_URL = "https://api.deepinfra.com" + + +def _form_field_value(value: Any) -> str: + if isinstance(value, str): + return value + if isinstance(value, bool): # bool before int: bool is an int subclass + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + return json.dumps(value) + + +def _encode_multipart(audio: MimeBytes, fields: dict[str, Any]) -> tuple[bytes, str]: + boundary = uuid.uuid4().hex + # Fall back to .wav when the MIME type is unknown: transcription servers sniff the format from the filename. + filename = "audio" + (mimetypes.guess_extension(audio.mime_type or "") or ".wav") + lines: list[bytes] = [ + f"--{boundary}".encode(), + f'Content-Disposition: form-data; name="file"; filename="{filename}"'.encode(), + f"Content-Type: {audio.mime_type or 'application/octet-stream'}".encode(), + b"", + bytes(audio), + ] + for key, value in fields.items(): + lines += [ + f"--{boundary}".encode(), + f'Content-Disposition: form-data; name="{key}"'.encode(), + b"", + _form_field_value(value).encode(), + ] + lines += [f"--{boundary}--".encode(), b""] + return b"\r\n".join(lines), f"multipart/form-data; boundary={boundary}" + + +class DeepInfraTextGenerationTask(BaseTextGenerationTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/openai/completions" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + params = filter_none(parameters.copy()) + params["max_tokens"] = params.pop("max_new_tokens", None) + + return {"prompt": inputs, **params, "model": provider_mapping_info.provider_id} + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + output = _as_dict(response)["choices"][0] + return { + "generated_text": output["text"], + "details": { + "finish_reason": output.get("finish_reason"), + "seed": output.get("seed"), + }, + } + + +class DeepInfraConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/openai/chat/completions" + + +class DeepInfraAutomaticSpeechRecognitionTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task="automatic-speech-recognition") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/openai/audio/transcriptions" + + def _prepare_payload_as_bytes( + self, + inputs: Any, + parameters: dict, + provider_mapping_info: InferenceProviderMapping, + extra_payload: dict | None, + ) -> MimeBytes | None: + # OpenAI-compatible transcription endpoint expects a multipart/form-data body, not JSON. + audio = _open_as_mime_bytes(inputs) + # `model` is applied last so parameters cannot override the mapped provider model. + fields: dict[str, Any] = { + **filter_none(parameters), + **filter_none(extra_payload or {}), + "model": provider_mapping_info.provider_id, + } + body, content_type = _encode_multipart(audio, fields) + return MimeBytes(body, mime_type=content_type) + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + output = _as_dict(response) + text = output["text"] + if not isinstance(text, str): + raise ValueError(f"Unexpected output format from DeepInfra API. Expected string, got {type(text)}.") + result: dict[str, Any] = {"text": text} + segments = output.get("segments") + if isinstance(segments, list): + result["chunks"] = [ + {"text": segment.get("text"), "timestamp": [segment.get("start"), segment.get("end")]} + for segment in segments + if isinstance(segment, dict) + ] + return result diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fal_ai.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fal_ai.py new file mode 100644 index 0000000000000000000000000000000000000000..41b761d5cebeb319918648c62ac3d82481a621d3 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fal_ai.py @@ -0,0 +1,300 @@ +import base64 +import time +from abc import ABC +from typing import Any +from urllib.parse import urlparse + +from huggingface_hub import constants +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict, _as_url +from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none +from huggingface_hub.utils import get_session, hf_raise_for_status +from huggingface_hub.utils.logging import get_logger + + +logger = get_logger(__name__) + +# Arbitrary polling interval +_POLLING_INTERVAL = 0.5 + + +class FalAITask(TaskProviderHelper, ABC): + def __init__(self, task: str): + super().__init__(provider="fal-ai", base_url="https://fal.run", task=task) + + def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + if not api_key.startswith("hf_"): + headers["authorization"] = f"Key {api_key}" + return headers + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return f"/{mapped_model}" + + +class FalAIQueueTask(TaskProviderHelper, ABC): + def __init__(self, task: str): + super().__init__(provider="fal-ai", base_url="https://queue.fal.run", task=task) + + def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + if not api_key.startswith("hf_"): + headers["authorization"] = f"Key {api_key}" + return headers + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + if api_key.startswith("hf_"): + # Use the queue subdomain for HF routing + return f"/{mapped_model}?_subdomain=queue" + return f"/{mapped_model}" + + def get_response( + self, + response: bytes | dict, + request_params: RequestParameters | None = None, + ) -> Any: + response_dict = _as_dict(response) + + request_id = response_dict.get("request_id") + if not request_id: + raise ValueError("No request ID found in the response") + if request_params is None: + raise ValueError( + f"A `RequestParameters` object should be provided to get {self.task} responses with Fal AI." + ) + + # extract the base url and query params + parsed_url = urlparse(request_params.url) + # a bit hacky way to concatenate the provider name without parsing `parsed_url.path` + base_url = f"{parsed_url.scheme}://{parsed_url.netloc}{'/fal-ai' if parsed_url.netloc == 'router.huggingface.co' else ''}" + query_param = f"?{parsed_url.query}" if parsed_url.query else "" + + # extracting the provider model id for status and result urls + # from the response as it might be different from the mapped model in `request_params.url` + model_id = urlparse(response_dict.get("response_url")).path + status_url = f"{base_url}{str(model_id)}/status{query_param}" + result_url = f"{base_url}{str(model_id)}{query_param}" + + status = response_dict.get("status") + logger.info("Generating the output.. this can take several minutes.") + while status != "COMPLETED": + time.sleep(_POLLING_INTERVAL) + status_response = get_session().get(status_url, headers=request_params.headers) + hf_raise_for_status(status_response) + status = status_response.json().get("status") + + return get_session().get(result_url, headers=request_params.headers).json() + + +class FalAIAutomaticSpeechRecognitionTask(FalAITask): + def __init__(self): + super().__init__("automatic-speech-recognition") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + if isinstance(inputs, str) and inputs.startswith(("http://", "https://")): + # If input is a URL, pass it directly + audio_url = inputs + else: + # If input is a file path, read it first + if isinstance(inputs, str): + with open(inputs, "rb") as f: + inputs = f.read() + + audio_b64 = base64.b64encode(inputs).decode() + content_type = "audio/mpeg" + audio_url = f"data:{content_type};base64,{audio_b64}" + + return {"audio_url": audio_url, **filter_none(parameters)} + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + text = _as_dict(response)["text"] + if not isinstance(text, str): + raise ValueError(f"Unexpected output format from FalAI API. Expected string, got {type(text)}.") + return {"text": text} + + +class FalAITextToImageTask(FalAITask): + def __init__(self): + super().__init__("text-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + payload: dict[str, Any] = { + "prompt": inputs, + **filter_none(parameters), + } + if "width" in payload and "height" in payload: + payload["image_size"] = { + "width": payload.pop("width"), + "height": payload.pop("height"), + } + if provider_mapping_info.adapter_weights_path is not None: + lora_path = constants.HUGGINGFACE_CO_URL_TEMPLATE.format( + repo_id=provider_mapping_info.hf_model_id, + revision="main", + filename=provider_mapping_info.adapter_weights_path, + ) + payload["loras"] = [{"path": lora_path, "scale": 1}] + if provider_mapping_info.provider_id == "fal-ai/lora": + # little hack: fal requires the base model for stable-diffusion-based loras but not for flux-based + # See payloads in https://fal.ai/models/fal-ai/lora/api vs https://fal.ai/models/fal-ai/flux-lora/api + payload["model_name"] = "stabilityai/stable-diffusion-xl-base-1.0" + + return payload + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + url = _as_dict(response)["images"][0]["url"] + return get_session().get(url).content + + +class FalAITextToSpeechTask(FalAITask): + def __init__(self): + super().__init__("text-to-speech") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + return {"text": inputs, **filter_none(parameters)} + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + url = _as_dict(response)["audio"]["url"] + return get_session().get(url).content + + +class FalAITextToVideoTask(FalAIQueueTask): + def __init__(self): + super().__init__("text-to-video") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + return {"prompt": inputs, **filter_none(parameters)} + + def get_response( + self, + response: bytes | dict, + request_params: RequestParameters | None = None, + ) -> Any: + output = super().get_response(response, request_params) + url = _as_dict(output)["video"]["url"] + return get_session().get(url).content + + +class FalAIImageToImageTask(FalAIQueueTask): + def __init__(self): + super().__init__("image-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + image_url = _as_url(inputs, default_mime_type="image/jpeg") + if "target_size" in parameters: + parameters["image_size"] = parameters.pop("target_size") + payload: dict[str, Any] = { + "image_url": image_url, + "image_urls": [image_url], + **filter_none(parameters), + } + if provider_mapping_info.adapter_weights_path is not None: + lora_path = constants.HUGGINGFACE_CO_URL_TEMPLATE.format( + repo_id=provider_mapping_info.hf_model_id, + revision="main", + filename=provider_mapping_info.adapter_weights_path, + ) + payload["loras"] = [{"path": lora_path, "scale": 1}] + + return payload + + def get_response( + self, + response: bytes | dict, + request_params: RequestParameters | None = None, + ) -> Any: + output = super().get_response(response, request_params) + url = _as_dict(output)["images"][0]["url"] + return get_session().get(url).content + + +class FalAIImageToVideoTask(FalAIQueueTask): + def __init__(self): + super().__init__("image-to-video") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + image_url = _as_url(inputs, default_mime_type="image/jpeg") + payload: dict[str, Any] = { + "image_url": image_url, + **filter_none(parameters), + } + if provider_mapping_info.adapter_weights_path is not None: + lora_path = constants.HUGGINGFACE_CO_URL_TEMPLATE.format( + repo_id=provider_mapping_info.hf_model_id, + revision="main", + filename=provider_mapping_info.adapter_weights_path, + ) + payload["loras"] = [{"path": lora_path, "scale": 1}] + return payload + + def get_response( + self, + response: bytes | dict, + request_params: RequestParameters | None = None, + ) -> Any: + output = super().get_response(response, request_params) + url = _as_dict(output)["video"]["url"] + return get_session().get(url).content + + +class FalAIImageSegmentationTask(FalAIQueueTask): + def __init__(self): + super().__init__("image-segmentation") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + image_url = _as_url(inputs, default_mime_type="image/png") + payload: dict[str, Any] = { + "image_url": image_url, + **filter_none(parameters), + "sync_mode": True, + } + return payload + + def get_response( + self, + response: bytes | dict, + request_params: RequestParameters | None = None, + ) -> Any: + result = super().get_response(response, request_params) + result_dict = _as_dict(result) + + if "image" not in result_dict: + raise ValueError(f"Response from fal ai image-segmentation API does not contain an image: {result_dict}") + + image_data = result_dict["image"] + if "url" not in image_data: + raise ValueError(f"Image data from fal ai image-segmentation API does not contain a URL: {image_data}") + + image_url = image_data["url"] + + if isinstance(image_url, str) and image_url.startswith("data:"): + if "," in image_url: + mask_base64 = image_url.split(",", 1)[1] + else: + raise ValueError(f"Invalid data URL format: {image_url}") + else: + # or it's a regular URL, fetch it + mask_response = get_session().get(image_url) + hf_raise_for_status(mask_response) + mask_base64 = base64.b64encode(mask_response.content).decode() + + return [ + { + "label": "mask", + "mask": mask_base64, + } + ] diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/featherless_ai.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/featherless_ai.py new file mode 100644 index 0000000000000000000000000000000000000000..1a90b332a5050340fb4e472040e05beb24e747b6 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/featherless_ai.py @@ -0,0 +1,38 @@ +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict + +from ._common import BaseConversationalTask, BaseTextGenerationTask, filter_none + + +_PROVIDER = "featherless-ai" +_BASE_URL = "https://api.featherless.ai" + + +class FeatherlessTextGenerationTask(BaseTextGenerationTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + params = filter_none(parameters.copy()) + params["max_tokens"] = params.pop("max_new_tokens", None) + + return {"prompt": inputs, **params, "model": provider_mapping_info.provider_id} + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + output = _as_dict(response)["choices"][0] + return { + "generated_text": output["text"], + "details": { + "finish_reason": output.get("finish_reason"), + "seed": output.get("seed"), + }, + } + + +class FeatherlessConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fireworks_ai.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fireworks_ai.py new file mode 100644 index 0000000000000000000000000000000000000000..4ae57662cf888c258c9b310156c8b986b8643923 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fireworks_ai.py @@ -0,0 +1,27 @@ +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping + +from ._common import BaseConversationalTask + + +class FireworksAIConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="fireworks-ai", base_url="https://api.fireworks.ai") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/inference/v1/chat/completions" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) + response_format = parameters.get("response_format") + if isinstance(response_format, dict) and response_format.get("type") == "json_schema": + json_schema_details = response_format.get("json_schema") + if isinstance(json_schema_details, dict) and "schema" in json_schema_details: + payload["response_format"] = { # type: ignore + "type": "json_object", + "schema": json_schema_details["schema"], + } + return payload diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/groq.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/groq.py new file mode 100644 index 0000000000000000000000000000000000000000..11e677504e89bc02b966e7d37d9e11f1b94b297f --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/groq.py @@ -0,0 +1,9 @@ +from ._common import BaseConversationalTask + + +class GroqConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="groq", base_url="https://api.groq.com") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/openai/v1/chat/completions" diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/hf_inference.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/hf_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..dce871ca1d643be49ed32caf788eb04571fb9e06 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/hf_inference.py @@ -0,0 +1,228 @@ +import json +from functools import lru_cache +from pathlib import Path +from typing import Any +from urllib.parse import urlparse, urlunparse + +from huggingface_hub import constants +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import ( + MimeBytes, + RequestParameters, + _b64_encode, + _bytes_to_dict, + _open_as_mime_bytes, +) +from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none +from huggingface_hub.utils import build_hf_headers, get_session, get_token, hf_raise_for_status + + +class HFInferenceTask(TaskProviderHelper): + """Base class for HF Inference API tasks.""" + + def __init__(self, task: str): + super().__init__( + provider="hf-inference", + base_url=constants.INFERENCE_PROXY_TEMPLATE.format(provider="hf-inference"), + task=task, + ) + + def _prepare_api_key(self, api_key: str | None) -> str: + # special case: for HF Inference we allow not providing an API key + return api_key or get_token() # type: ignore + + def _prepare_mapping_info(self, model: str | None) -> InferenceProviderMapping: + if model is not None and model.startswith(("http://", "https://")): + return InferenceProviderMapping( + provider="hf-inference", providerId=model, hf_model_id=model, task=self.task, status="live" + ) + model_id = model if model is not None else _fetch_recommended_models().get(self.task) + if model_id is None: + raise ValueError( + f"Task {self.task} has no recommended model for HF Inference. Please specify a model" + " explicitly. Visit https://huggingface.co/tasks for more info." + ) + _check_supported_task(model_id, self.task) + return InferenceProviderMapping( + provider="hf-inference", providerId=model_id, hf_model_id=model_id, task=self.task, status="live" + ) + + def _prepare_url(self, api_key: str, mapped_model: str) -> str: + # hf-inference provider can handle URLs (e.g. Inference Endpoints or TGI deployment) + if mapped_model.startswith(("http://", "https://")): + return mapped_model + return ( + # Feature-extraction and sentence-similarity are the only cases where we handle models with several tasks. + f"{self.base_url}/models/{mapped_model}/pipeline/{self.task}" + if self.task in ("feature-extraction", "sentence-similarity") + # Otherwise, we use the default endpoint + else f"{self.base_url}/models/{mapped_model}" + ) + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + if isinstance(inputs, bytes): + raise ValueError(f"Unexpected binary input for task {self.task}.") + if isinstance(inputs, Path): + raise ValueError(f"Unexpected path input for task {self.task} (got {inputs})") + return filter_none({"inputs": inputs, "parameters": parameters}) + + +class HFInferenceBinaryInputTask(HFInferenceTask): + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + return None + + def _prepare_payload_as_bytes( + self, + inputs: Any, + parameters: dict, + provider_mapping_info: InferenceProviderMapping, + extra_payload: dict | None, + ) -> MimeBytes | None: + parameters = filter_none(parameters) + extra_payload = extra_payload or {} + has_parameters = len(parameters) > 0 or len(extra_payload) > 0 + + # Raise if not a binary object or a local path or a URL. + if not isinstance(inputs, (bytes, Path)) and not isinstance(inputs, str): + raise ValueError(f"Expected binary inputs or a local path or a URL. Got {inputs}") + + # Send inputs as raw content when no parameters are provided + if not has_parameters: + return _open_as_mime_bytes(inputs) + + # Otherwise encode as b64 + return MimeBytes( + json.dumps({"inputs": _b64_encode(inputs), "parameters": parameters, **extra_payload}).encode("utf-8"), + mime_type="application/json", + ) + + +class HFInferenceConversational(HFInferenceTask): + def __init__(self): + super().__init__("conversational") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + payload = filter_none(parameters) + mapped_model = provider_mapping_info.provider_id + payload_model = parameters.get("model") or mapped_model + + if payload_model is None or payload_model.startswith(("http://", "https://")): + payload_model = "dummy" + + response_format = parameters.get("response_format") + if isinstance(response_format, dict) and response_format.get("type") == "json_schema": + payload["response_format"] = { + "type": "json_object", + "value": response_format["json_schema"]["schema"], + } + return {**payload, "model": payload_model, "messages": inputs} + + def _prepare_url(self, api_key: str, mapped_model: str) -> str: + base_url = ( + mapped_model + if mapped_model.startswith(("http://", "https://")) + else f"{constants.INFERENCE_PROXY_TEMPLATE.format(provider='hf-inference')}/models/{mapped_model}" + ) + return _build_chat_completion_url(base_url) + + +def _build_chat_completion_url(model_url: str) -> str: + parsed = urlparse(model_url) + path = parsed.path.rstrip("/") + + # If the path already ends with /chat/completions, we're done! + if path.endswith("/chat/completions"): + return model_url + + # Append /chat/completions if not already present + if path.endswith("/v1"): + new_path = path + "/chat/completions" + # If path was empty or just "/", set the full path + elif not path: + new_path = "/v1/chat/completions" + # Append /v1/chat/completions if not already present + else: + new_path = path + "/v1/chat/completions" + + # Reconstruct the URL with the new path and original query parameters. + new_parsed = parsed._replace(path=new_path) + return str(urlunparse(new_parsed)) + + +@lru_cache(maxsize=1) +def _fetch_recommended_models() -> dict[str, str | None]: + response = get_session().get(f"{constants.ENDPOINT}/api/tasks", headers=build_hf_headers()) + hf_raise_for_status(response) + return {task: next(iter(details["widgetModels"]), None) for task, details in response.json().items()} + + +@lru_cache(maxsize=None) +def _check_supported_task(model: str, task: str) -> None: + from huggingface_hub.hf_api import HfApi + + model_info = HfApi().model_info(model) + pipeline_tag = model_info.pipeline_tag + tags = model_info.tags or [] + is_conversational = "conversational" in tags + if task in ("text-generation", "conversational"): + if pipeline_tag == "text-generation": + # text-generation + conversational tag -> both tasks allowed + if is_conversational: + return + # text-generation without conversational tag -> only text-generation allowed + if task == "text-generation": + return + raise ValueError(f"Model '{model}' doesn't support task '{task}'.") + + if pipeline_tag == "text2text-generation": + if task == "text-generation": + return + raise ValueError(f"Model '{model}' doesn't support task '{task}'.") + + if pipeline_tag == "image-text-to-text": + if is_conversational and task == "conversational": + return # Only conversational allowed if tagged as conversational + raise ValueError("Non-conversational image-text-to-text task is not supported.") + + if ( + task in ("feature-extraction", "sentence-similarity") + and pipeline_tag in ("feature-extraction", "sentence-similarity") + and task in tags + ): + # feature-extraction and sentence-similarity are interchangeable for HF Inference + return + + # For all other tasks, just check pipeline tag + if pipeline_tag != task: + raise ValueError( + f"Model '{model}' doesn't support task '{task}'. Supported tasks: '{pipeline_tag}', got: '{task}'" + ) + return + + +class HFInferenceFeatureExtractionTask(HFInferenceTask): + def __init__(self): + super().__init__("feature-extraction") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + if isinstance(inputs, bytes): + raise ValueError(f"Unexpected binary input for task {self.task}.") + if isinstance(inputs, Path): + raise ValueError(f"Unexpected path input for task {self.task} (got {inputs})") + + # Parameters are sent at root-level for feature-extraction task + # See specs: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/tasks/feature-extraction/spec/input.json + return {"inputs": inputs, **filter_none(parameters)} + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + if isinstance(response, bytes): + return _bytes_to_dict(response) + return response diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/novita.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/novita.py new file mode 100644 index 0000000000000000000000000000000000000000..af29e72032a7eb82f1ec20bc3df529612ab8730c --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/novita.py @@ -0,0 +1,69 @@ +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import ( + BaseConversationalTask, + BaseTextGenerationTask, + TaskProviderHelper, + filter_none, +) +from huggingface_hub.utils import get_session + + +_PROVIDER = "novita" +_BASE_URL = "https://api.novita.ai" + + +class NovitaTextGenerationTask(BaseTextGenerationTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + # there is no v1/ route for novita + return "/v3/openai/completions" + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + output = _as_dict(response)["choices"][0] + return { + "generated_text": output["text"], + "details": { + "finish_reason": output.get("finish_reason"), + "seed": output.get("seed"), + }, + } + + +class NovitaConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + # there is no v1/ route for novita + return "/v3/openai/chat/completions" + + +class NovitaTextToVideoTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task="text-to-video") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return f"/v3/hf/{mapped_model}" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + return {"prompt": inputs, **filter_none(parameters)} + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + if not ( + isinstance(response_dict, dict) + and "video" in response_dict + and isinstance(response_dict["video"], dict) + and "video_url" in response_dict["video"] + ): + raise ValueError("Expected response format: { 'video': { 'video_url': string } }") + + video_url = response_dict["video"]["video_url"] + return get_session().get(video_url).content diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/nscale.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/nscale.py new file mode 100644 index 0000000000000000000000000000000000000000..6a5a4c38e9f0f8ede459d36c584813cf5ef7dbac --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/nscale.py @@ -0,0 +1,44 @@ +import base64 +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict + +from ._common import BaseConversationalTask, TaskProviderHelper, filter_none + + +class NscaleConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="nscale", base_url="https://inference.api.nscale.com") + + +class NscaleTextToImageTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider="nscale", base_url="https://inference.api.nscale.com", task="text-to-image") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/images/generations" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + mapped_model = provider_mapping_info.provider_id + # Combine all parameters except inputs and parameters + parameters = filter_none(parameters) + if "width" in parameters and "height" in parameters: + parameters["size"] = f"{parameters.pop('width')}x{parameters.pop('height')}" + if "num_inference_steps" in parameters: + parameters.pop("num_inference_steps") + if "cfg_scale" in parameters: + parameters.pop("cfg_scale") + payload = { + "response_format": "b64_json", + "prompt": inputs, + "model": mapped_model, + **parameters, + } + return payload + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + return base64.b64decode(response_dict["data"][0]["b64_json"]) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/openai.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/openai.py new file mode 100644 index 0000000000000000000000000000000000000000..9f0f56002a6493c852836a1e5539900204fc0bb3 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/openai.py @@ -0,0 +1,23 @@ +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._providers._common import BaseConversationalTask + + +class OpenAIConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="openai", base_url="https://api.openai.com") + + def _prepare_api_key(self, api_key: str | None) -> str: + if api_key is None: + raise ValueError("You must provide an api_key to work with OpenAI API.") + if api_key.startswith("hf_"): + raise ValueError( + "OpenAI provider is not available through Hugging Face routing, please use your own OpenAI API key." + ) + return api_key + + def _prepare_mapping_info(self, model: str | None) -> InferenceProviderMapping: + if model is None: + raise ValueError("Please provide an OpenAI model ID, e.g. `gpt-4o` or `o1`.") + return InferenceProviderMapping( + provider="openai", providerId=model, task="conversational", status="live", hf_model_id=model + ) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/ovhcloud.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/ovhcloud.py new file mode 100644 index 0000000000000000000000000000000000000000..79be8d55089975b1ad9a0282d503cd2fc012127c --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/ovhcloud.py @@ -0,0 +1,10 @@ +from huggingface_hub.inference._providers._common import BaseConversationalTask + + +_PROVIDER = "ovhcloud" +_BASE_URL = "https://oai.endpoints.kepler.ai.cloud.ovh.net" + + +class OVHcloudConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/publicai.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/publicai.py new file mode 100644 index 0000000000000000000000000000000000000000..4c88528e4f1e2eefaf6be9315c490db19ff5ca1e --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/publicai.py @@ -0,0 +1,6 @@ +from ._common import BaseConversationalTask + + +class PublicAIConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="publicai", base_url="https://api.publicai.co") diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/replicate.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/replicate.py new file mode 100644 index 0000000000000000000000000000000000000000..d1aff75ec56460791b35b3acde7208d8c4a03bdb --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/replicate.py @@ -0,0 +1,160 @@ +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict, _as_url +from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none +from huggingface_hub.utils import get_session + + +_PROVIDER = "replicate" +_BASE_URL = "https://api.replicate.com" + + +class ReplicateTask(TaskProviderHelper): + def __init__(self, task: str): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task) + + def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + headers["Prefer"] = "wait" + return headers + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + if ":" in mapped_model: + return "/v1/predictions" + return f"/v1/models/{mapped_model}/predictions" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + mapped_model = provider_mapping_info.provider_id + payload: dict[str, Any] = {"input": {"prompt": inputs, **filter_none(parameters)}} + if ":" in mapped_model: + version = mapped_model.split(":", 1)[1] + payload["version"] = version + return payload + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + if response_dict.get("output") is None: + raise TimeoutError( + f"Inference request timed out after 60 seconds. No output generated for model {response_dict.get('model')}" + "The model might be in cold state or starting up. Please try again later." + ) + output_url = ( + response_dict["output"] if isinstance(response_dict["output"], str) else response_dict["output"][0] + ) + return get_session().get(output_url).content + + +class ReplicateTextToImageTask(ReplicateTask): + def __init__(self): + super().__init__("text-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + payload: dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore + if provider_mapping_info.adapter_weights_path is not None: + payload["input"]["lora_weights"] = f"https://huggingface.co/{provider_mapping_info.hf_model_id}" + return payload + + +class ReplicateTextToSpeechTask(ReplicateTask): + def __init__(self): + super().__init__("text-to-speech") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + payload: dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore + payload["input"]["text"] = payload["input"].pop("prompt") # rename "prompt" to "text" for TTS + return payload + + +class ReplicateAutomaticSpeechRecognitionTask(ReplicateTask): + def __init__(self) -> None: + super().__init__("automatic-speech-recognition") + + def _prepare_payload_as_dict( + self, + inputs: Any, + parameters: dict, + provider_mapping_info: InferenceProviderMapping, + ) -> dict | None: + mapped_model = provider_mapping_info.provider_id + audio_url = _as_url(inputs, default_mime_type="audio/wav") + + payload: dict[str, Any] = { + "input": { + **{"audio": audio_url}, + **filter_none(parameters), + } + } + + if ":" in mapped_model: + payload["version"] = mapped_model.split(":", 1)[1] + + return payload + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + output = response_dict.get("output") + + if isinstance(output, str): + return {"text": output} + + if isinstance(output, list) and output: + first_item = output[0] + if isinstance(first_item, str): + return {"text": first_item} + if isinstance(first_item, dict): + output = first_item + + text: str | None = None + if isinstance(output, dict): + transcription = output.get("transcription") + if isinstance(transcription, str): + text = transcription + + translation = output.get("translation") + if isinstance(translation, str): + text = translation + + txt_file = output.get("txt_file") + if isinstance(txt_file, str): + text_response = get_session().get(txt_file) + text_response.raise_for_status() + text = text_response.text + + if text is not None: + return {"text": text} + + raise ValueError("Received malformed response from Replicate automatic-speech-recognition API") + + +class ReplicateImageToImageTask(ReplicateTask): + def __init__(self): + super().__init__("image-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + image_url = _as_url(inputs, default_mime_type="image/jpeg") + + # Different Replicate models expect the image in different keys + payload: dict[str, Any] = { + "input": { + "image": image_url, + "images": [image_url], + "input_image": image_url, + "input_images": [image_url], + **filter_none(parameters), + } + } + + mapped_model = provider_mapping_info.provider_id + if ":" in mapped_model: + version = mapped_model.split(":", 1)[1] + payload["version"] = version + return payload diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/scaleway.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/scaleway.py new file mode 100644 index 0000000000000000000000000000000000000000..a64462075a091f4a9d9a664e09be229e0fb7bdf1 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/scaleway.py @@ -0,0 +1,28 @@ +from typing import Any + +from huggingface_hub.inference._common import RequestParameters, _as_dict + +from ._common import BaseConversationalTask, InferenceProviderMapping, TaskProviderHelper, filter_none + + +class ScalewayConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="scaleway", base_url="https://api.scaleway.ai") + + +class ScalewayFeatureExtractionTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider="scaleway", base_url="https://api.scaleway.ai", task="feature-extraction") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/embeddings" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + parameters = filter_none(parameters) + return {"input": inputs, "model": provider_mapping_info.provider_id, **parameters} + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + embeddings = _as_dict(response)["data"] + return [embedding["embedding"] for embedding in embeddings] diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/together.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/together.py new file mode 100644 index 0000000000000000000000000000000000000000..1f26595af3f919f675f14b5c4d10baa130e647d8 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/together.py @@ -0,0 +1,291 @@ +import base64 +import time +from abc import ABC +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import ( + RequestParameters, + _as_dict, + _as_url, +) +from huggingface_hub.inference._providers._common import ( + BaseConversationalTask, + BaseTextGenerationTask, + TaskProviderHelper, + filter_none, +) +from huggingface_hub.utils import get_session, hf_raise_for_status, logging + + +logger = logging.get_logger(__name__) + +_PROVIDER = "together" +_BASE_URL = "https://api.together.xyz" + +# Polling interval for async video generation (in seconds). +_VIDEO_POLLING_INTERVAL = 2.0 + +# Upper bound on status polls (initial response may already be terminal; each further poll is one attempt). +_VIDEO_MAX_POLL_ATTEMPTS = 150 # ~5 minutes at _VIDEO_POLLING_INTERVAL + +# Job statuses that mean "keep polling". Together returns "queued" before transitioning to +# "in_progress", so we must treat both as pending. +_VIDEO_PENDING_STATUSES = {"queued", "in_progress"} + + +class TogetherTask(TaskProviderHelper, ABC): + """Base class for Together API tasks.""" + + def __init__(self, task: str): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + match self.task: + case "text-to-image" | "image-to-image": + return "/v1/images/generations" + case "text-to-speech": + return "/v1/audio/speech" + case "feature-extraction": + return "/v1/embeddings" + case "text-to-video" | "image-to-video": + # Video creation lives under /v2 (see https://docs.together.ai/reference/create-videos). + return "/v2/videos" + raise ValueError(f"Unsupported task '{self.task}' for Together API.") + + +class TogetherTextGenerationTask(BaseTextGenerationTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + output = _as_dict(response)["choices"][0] + return { + "generated_text": output["text"], + "details": { + "finish_reason": output.get("finish_reason"), + "seed": output.get("seed"), + }, + } + + +class TogetherConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) + if payload is None: + return None + # Together accepts response_format `{type: "json_schema", schema: }` (flattened), + # so unwrap the OpenAI-style `{type: "json_schema", json_schema: {schema}}` envelope. + response_format = payload.get("response_format") + if ( + isinstance(response_format, dict) + and response_format.get("type") == "json_schema" + and isinstance(response_format.get("json_schema"), dict) + and "schema" in response_format["json_schema"] + ): + payload["response_format"] = { + "type": "json_schema", + "schema": response_format["json_schema"]["schema"], + } + return payload + + +class TogetherTextToImageTask(TogetherTask): + def __init__(self): + super().__init__("text-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + mapped_model = provider_mapping_info.provider_id + parameters = filter_none(parameters) + if "num_inference_steps" in parameters: + parameters["steps"] = parameters.pop("num_inference_steps") + + return {"prompt": inputs, "response_format": "base64", **parameters, "model": mapped_model} + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + return base64.b64decode(response_dict["data"][0]["b64_json"]) + + +class TogetherImageToImageTask(TogetherTask): + def __init__(self): + super().__init__("image-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + mapped_model = provider_mapping_info.provider_id + image_url = _as_url(inputs, default_mime_type="image/jpeg") + + # Filter `None` values first: the client always passes `"prompt": None` when the user + # omits the argument, so popping before filtering would yield `None` instead of the + # `""` default and send `"prompt": null` to Together (rejected by Flux Kontext). + parameters = filter_none(parameters) + prompt = parameters.pop("prompt", "") + if "num_inference_steps" in parameters: + parameters["steps"] = parameters.pop("num_inference_steps") + + # Together exposes two mutually-exclusive image inputs (see + # https://docs.together.ai/docs/image-to-image): FLUX.1 Kontext only accepts + # `image_url`; FLUX.2 [dev] and Google models (Gemini 3 Pro Image, Flash Image + # 2.5) only accept `reference_images`. FLUX.2 [pro]/[flex] accept either but + # `reference_images` is the documented default. Use `image_url` only for + # FLUX.1 Kontext models and `reference_images` for everything else. + lowered = mapped_model.lower() + use_image_url = "kontext" in lowered and "flux.1" in lowered + image_field: dict[str, Any] = {"image_url": image_url} if use_image_url else {"reference_images": [image_url]} + return { + "prompt": prompt, + **image_field, + "response_format": "base64", + **parameters, + "model": mapped_model, + } + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + response_dict = _as_dict(response) + return base64.b64decode(response_dict["data"][0]["b64_json"]) + + +class TogetherFeatureExtractionTask(TogetherTask): + def __init__(self): + super().__init__("feature-extraction") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + return { + "input": inputs, + "model": provider_mapping_info.provider_id, + **filter_none(parameters), + } + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + return [item["embedding"] for item in _as_dict(response)["data"]] + + +class TogetherTextToSpeechTask(TogetherTask): + def __init__(self): + super().__init__("text-to-speech") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + # `voice` is required by the Together API and is model-specific + # (see https://docs.together.ai/docs/text-to-speech#supported-voices), + # so we don't set a default and let the API surface a clear error if missing. + return { + "input": inputs, + "model": provider_mapping_info.provider_id, + **filter_none(parameters), + } + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + if isinstance(response, bytes): + return response + raise ValueError(f"Expected raw audio bytes for text-to-speech, got {type(response).__name__}.") + + +def _normalize_video_parameters(parameters: dict) -> dict: + """Map HF inference-client conventions onto Together's video API parameter names.""" + parameters = filter_none(parameters) + if "num_inference_steps" in parameters: + parameters["steps"] = parameters.pop("num_inference_steps") + if "target_size" in parameters: + target_size = parameters.pop("target_size") + if "width" in target_size: + parameters["width"] = target_size["width"] + if "height" in target_size: + parameters["height"] = target_size["height"] + return parameters + + +class TogetherVideoTask(TogetherTask, ABC): + """Base class for Together's asynchronous video generation tasks.""" + + def get_response(self, response: bytes | dict, request_params: RequestParameters | None = None) -> Any: + if request_params is None: + raise ValueError("A `RequestParameters` object is required to poll Together video jobs.") + + job = _as_dict(response) + job_id = job.get("id") + if not job_id: + raise ValueError("No job ID found in Together video generation response.") + + # Status polling lives at the same /v2/videos URL with the job ID appended. + status_url = f"{request_params.url}/{job_id}" + + logger.info("Generating video, polling for completion...") + # Together usually returns `status: "queued"` on the initial POST, but the field is + # optional per the spec — treat a missing status as "still pending" and poll, rather + # than falling through to the "unexpected status" error below. + status = job.get("status") + for _ in range(_VIDEO_MAX_POLL_ATTEMPTS): + if status is not None and status not in _VIDEO_PENDING_STATUSES: + break + time.sleep(_VIDEO_POLLING_INTERVAL) + status_response = get_session().get(status_url, headers=request_params.headers) + hf_raise_for_status(status_response) + job = status_response.json() + status = job.get("status") + if status is not None and status not in _VIDEO_PENDING_STATUSES: + break + else: + raise ValueError( + "Timed out while waiting for Together video generation " + f"— aborting after {_VIDEO_MAX_POLL_ATTEMPTS} status polls" + ) + + if status == "failed": + error = job.get("error") or {} + raise RuntimeError(f"Together video generation failed: {error.get('message') or 'Unknown error'}") + if status != "completed": + raise RuntimeError(f"Unexpected Together video job status: {status!r}") + + video_url = (job.get("outputs") or {}).get("video_url") + if not video_url: + raise ValueError("No video URL found in completed Together video job.") + + video_response = get_session().get(video_url) + hf_raise_for_status(video_response) + return video_response.content + + +class TogetherTextToVideoTask(TogetherVideoTask): + def __init__(self): + super().__init__("text-to-video") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + return { + "prompt": inputs, + "model": provider_mapping_info.provider_id, + **_normalize_video_parameters(parameters), + } + + +class TogetherImageToVideoTask(TogetherVideoTask): + def __init__(self): + super().__init__("image-to-video") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + # Together expects each keyframe as `{input_image, frame: "first" | "last"}` + # for i2v models. See https://docs.together.ai/docs/inference/videos/reference-and-keyframes. + # Note: `input_image` accepts a data URL or an HTTP(S) URL but the field is capped + # at ~60KB — users with larger inputs should host the image and pass `frame_images` + # directly via `extra_body`. + return { + "model": provider_mapping_info.provider_id, + "frame_images": [{"input_image": _as_url(inputs, default_mime_type="image/png"), "frame": "first"}], + **_normalize_video_parameters(parameters), + } diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/wavespeed.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/wavespeed.py new file mode 100644 index 0000000000000000000000000000000000000000..aa38eb9da2d47c491e746af6495f669147fe181e --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/wavespeed.py @@ -0,0 +1,138 @@ +import base64 +import time +from abc import ABC +from typing import Any +from urllib.parse import urlparse + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none +from huggingface_hub.utils import get_session, hf_raise_for_status +from huggingface_hub.utils.logging import get_logger + + +logger = get_logger(__name__) + +# Polling interval (in seconds) +_POLLING_INTERVAL = 0.5 + + +class WavespeedAITask(TaskProviderHelper, ABC): + def __init__(self, task: str): + super().__init__(provider="wavespeed", base_url="https://api.wavespeed.ai", task=task) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return f"/api/v3/{mapped_model}" + + def get_response( + self, + response: bytes | dict, + request_params: RequestParameters | None = None, + ) -> Any: + response_dict = _as_dict(response) + data = response_dict.get("data", {}) + result_path = data.get("urls", {}).get("get") + + if not result_path: + raise ValueError("No result URL found in the response") + if request_params is None: + raise ValueError("A `RequestParameters` object should be provided to get responses with WaveSpeed AI.") + + # Parse the request URL to determine base URL + parsed_url = urlparse(request_params.url) + # Add /wavespeed to base URL if going through HF router + if parsed_url.netloc == "router.huggingface.co": + base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/wavespeed" + else: + base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" + + # Extract path from result_path URL + if isinstance(result_path, str): + result_url_path = urlparse(result_path).path + else: + result_url_path = result_path + + result_url = f"{base_url}{result_url_path}" + + logger.info("Processing request, polling for results...") + + # Poll until task is completed + while True: + time.sleep(_POLLING_INTERVAL) + result_response = get_session().get(result_url, headers=request_params.headers) + hf_raise_for_status(result_response) + + result = result_response.json() + task_result = result.get("data", {}) + status = task_result.get("status") + + if status == "completed": + # Get content from the first output URL + if not task_result.get("outputs") or len(task_result["outputs"]) == 0: + raise ValueError("No output URL in completed response") + + output_url = task_result["outputs"][0] + return get_session().get(output_url).content + elif status == "failed": + error_msg = task_result.get("error", "Task failed with no specific error message") + raise ValueError(f"WaveSpeed AI task failed: {error_msg}") + elif status in ["processing", "created"]: + continue + else: + raise ValueError(f"Unknown status: {status}") + + +class WavespeedAITextToImageTask(WavespeedAITask): + def __init__(self): + super().__init__("text-to-image") + + def _prepare_payload_as_dict( + self, + inputs: Any, + parameters: dict, + provider_mapping_info: InferenceProviderMapping, + ) -> dict | None: + return {"prompt": inputs, **filter_none(parameters)} + + +class WavespeedAITextToVideoTask(WavespeedAITextToImageTask): + def __init__(self): + WavespeedAITask.__init__(self, "text-to-video") + + +class WavespeedAIImageToImageTask(WavespeedAITask): + def __init__(self): + super().__init__("image-to-image") + + def _prepare_payload_as_dict( + self, + inputs: Any, + parameters: dict, + provider_mapping_info: InferenceProviderMapping, + ) -> dict | None: + # Convert inputs to image (URL or base64) + if isinstance(inputs, str) and inputs.startswith(("http://", "https://")): + image = inputs + elif isinstance(inputs, str): + # If input is a file path, read it first + with open(inputs, "rb") as f: + file_content = f.read() + image_b64 = base64.b64encode(file_content).decode("utf-8") + image = f"data:image/jpeg;base64,{image_b64}" + else: + # If input is binary data + image_b64 = base64.b64encode(inputs).decode("utf-8") + image = f"data:image/jpeg;base64,{image_b64}" + + # Extract prompt from parameters if present + prompt = parameters.pop("prompt", None) + payload = {"image": image, **filter_none(parameters)} + if prompt is not None: + payload["prompt"] = prompt + + return payload + + +class WavespeedAIImageToVideoTask(WavespeedAIImageToImageTask): + def __init__(self): + WavespeedAITask.__init__(self, "image-to-video") diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/zai_org.py b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/zai_org.py new file mode 100644 index 0000000000000000000000000000000000000000..97225435084ab61860ef838747b5e0e7517f7bd1 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/zai_org.py @@ -0,0 +1,127 @@ +import time +from abc import ABC +from typing import Any + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import BaseConversationalTask, TaskProviderHelper, filter_none +from huggingface_hub.utils import get_session + + +_PROVIDER = "zai-org" +_BASE_URL = "https://api.z.ai" +_POLLING_INTERVAL = 5 # seconds +_MAX_POLL_ATTEMPTS = 60 + + +class ZaiTask(TaskProviderHelper, ABC): + def __init__(self, task: str): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task) + + def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + headers["Accept-Language"] = "en-US,en" + headers["x-source-channel"] = "hugging_face" + return headers + + +class ZaiConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_headers(self, headers: dict, api_key: str) -> dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + headers["Accept-Language"] = "en-US,en" + headers["x-source-channel"] = "hugging_face" + return headers + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/api/paas/v4/chat/completions" + + +class ZaiTextToImageTask(ZaiTask): + """Text-to-image task for ZAI provider using async API.""" + + def __init__(self): + super().__init__("text-to-image") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/api/paas/v4/async/images/generations" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: dict, provider_mapping_info: InferenceProviderMapping + ) -> dict | None: + width = parameters.pop("width", None) + height = parameters.pop("height", None) + size = None + if width is not None and height is not None: + size = f"{width}x{height}" + + payload: dict[str, Any] = { + "model": provider_mapping_info.provider_id, + "prompt": inputs, + } + if size is not None: + payload["size"] = size + + payload.update(filter_none(parameters)) + return payload + + def get_response( + self, + response: bytes | dict, + request_params: RequestParameters | None = None, + ) -> Any: + """Handle async response by polling for results.""" + response_dict = _as_dict(response) + + task_id = response_dict.get("id") + if task_id is None: + raise ValueError("No task_id in response from ZAI API") + + task_status = response_dict.get("task_status") + if task_status == "FAIL": + raise ValueError(f"ZAI image generation failed for request {task_id}") + + if task_status == "PROCESSING" and request_params is not None: + return self._poll_for_result(task_id, request_params) + + return self._extract_image(response_dict) + + def _poll_for_result(self, task_id: str, request_params: RequestParameters) -> bytes: + """Poll the async-result endpoint until completion.""" + session = get_session() + base_url = request_params.url.rsplit("/api/paas/v4/async/images/generations", 1)[0] + poll_url = f"{base_url}/api/paas/v4/async-result/{task_id}" + + for _ in range(_MAX_POLL_ATTEMPTS): + poll_response = session.get(poll_url, headers=request_params.headers) + poll_response.raise_for_status() + result = poll_response.json() + + task_status = result.get("task_status") + if task_status == "SUCCESS": + return self._extract_image(result) + elif task_status == "FAIL": + raise ValueError(f"Zai text-to-image generation failed for request {task_id}") + + time.sleep(_POLLING_INTERVAL) + + raise ValueError( + f"Timed out while waiting for the result from Zai API - aborting after {_MAX_POLL_ATTEMPTS} attempts" + ) + + def _extract_image(self, result: dict) -> bytes: + """Extract and download the image from the result.""" + image_result = result.get("image_result") + if not image_result or not isinstance(image_result, list) or len(image_result) == 0: + raise ValueError("No image_result in response from ZAI API") + + image_url = image_result[0].get("url") + if not image_url: + raise ValueError("No image URL in response from ZAI API") + + session = get_session() + image_response = session.get(image_url) + image_response.raise_for_status() + return image_response.content diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/lfs.py b/venv/lib/python3.12/site-packages/huggingface_hub/lfs.py new file mode 100644 index 0000000000000000000000000000000000000000..9958a1c332f0ab938cb9769a6384e178fb5e7a96 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/lfs.py @@ -0,0 +1,442 @@ +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Git LFS related type definitions and utilities""" + +import io +import re +from collections.abc import Iterable +from math import ceil +from os.path import getsize +from typing import TYPE_CHECKING, BinaryIO, TypedDict +from urllib.parse import unquote + +from huggingface_hub import constants + +from .utils import ( + build_hf_headers, + fix_hf_endpoint_in_url, + hf_raise_for_status, + http_backoff, + logging, + validate_hf_hub_args, +) +from .utils._lfs import SliceFileObj +from .utils.sha import sha256, sha_fileobj + + +if TYPE_CHECKING: + from ._commit_api import CommitOperationAdd + +logger = logging.get_logger(__name__) + +OID_REGEX = re.compile(r"^[0-9a-f]{40}$") + +LFS_MULTIPART_UPLOAD_COMMAND = "lfs-multipart-upload" + +LFS_HEADERS = { + "Accept": "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json", +} + + +class UploadInfo: + """ + Data structure holding required information to determine whether a blob + should be uploaded to the hub using the LFS protocol or the regular protocol. + + The SHA256 of the blob is computed lazily: creating an `UploadInfo` from a local path only reads + the first 512 bytes of the file. The full file is read (and hashed) only if `sha256` is accessed + before it has been set. When a file is uploaded through the Xet protocol, the SHA256 is computed + during upload (single read pass) and set afterwards. + + Args: + size (`int`): + Size in bytes of the blob + sample (`bytes`): + First 512 bytes of the blob + sha256 (`bytes`, *optional*): + SHA256 hash of the blob, if already known. Otherwise computed lazily from `source_path`. + source_path (`str`, *optional*): + Path to the local file the blob comes from. Required to lazily compute `sha256` if not provided. + """ + + def __init__( + self, + size: int, + sample: bytes, + sha256: bytes | None = None, + source_path: str | None = None, + ): + if sha256 is None and source_path is None: + raise ValueError("Either `sha256` or `source_path` must be provided.") + self.size = size + self.sample = sample + self._sha256 = sha256 + self._source_path = source_path + + @property + def sha256(self) -> bytes: + """SHA256 of the blob. If not set yet, reads the whole file from `source_path` to compute it.""" + if self._sha256 is None: + assert self._source_path is not None # guaranteed by __init__ + with open(self._source_path, "rb") as file: + self._sha256 = sha_fileobj(file) + return self._sha256 + + @sha256.setter + def sha256(self, value: bytes) -> None: + self._sha256 = value + + @property + def is_hashed(self) -> bool: + """Whether the SHA256 is already known (accessing `sha256` will not trigger a file read).""" + return self._sha256 is not None + + def __repr__(self) -> str: + sha = self._sha256.hex() if self._sha256 is not None else "" + return f"UploadInfo(size={self.size}, sha256={sha})" + + @classmethod + def from_path(cls, path: str): + size = getsize(path) + with open(path, "rb") as file: + sample = file.peek(512)[:512] + return cls(size=size, sample=sample, source_path=path) + + @classmethod + def from_bytes(cls, data: bytes): + sha = sha256(data).digest() + return cls(size=len(data), sample=data[:512], sha256=sha) + + @classmethod + def from_fileobj(cls, fileobj: BinaryIO): + sample = fileobj.read(512) + fileobj.seek(0, io.SEEK_SET) + sha = sha_fileobj(fileobj) + size = fileobj.tell() + fileobj.seek(0, io.SEEK_SET) + return cls(size=size, sha256=sha, sample=sample) + + +@validate_hf_hub_args +def post_lfs_batch_info( + upload_infos: Iterable[UploadInfo], + token: str | None, + repo_type: str, + repo_id: str, + revision: str | None = None, + endpoint: str | None = None, + headers: dict[str, str] | None = None, + transfers: list[str] | None = None, +) -> tuple[list[dict], list[dict], str | None]: + """ + Requests the LFS batch endpoint to retrieve upload instructions + + Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md + + Args: + upload_infos (`Iterable` of `UploadInfo`): + `UploadInfo` for the files that are being uploaded, typically obtained + from `CommitOperationAdd.upload_info` + token (`str` or `None`): + An authentication token (see https://huggingface.co/settings/token). + Pass `None` to fall back to the local cached token (or no token if unauthenticated). + repo_type (`str`): + Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. + repo_id (`str`): + A namespace (user or an organization) and a repo name separated + by a `/`. + revision (`str`, *optional*): + The git revision to upload to. + endpoint (`str`, *optional*): + The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`. + headers (`dict`, *optional*): + Additional headers to include in the request + transfers (`list`, *optional*): + List of transfer methods to use. Defaults to ["basic", "multipart"]. + + Returns: + `LfsBatchInfo`: 3-tuple: + - First element is the list of upload instructions from the server + - Second element is a list of errors, if any + - Third element is the chosen transfer adapter if provided by the server (e.g. "basic", "multipart", "xet") + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If an argument is invalid or the server response is malformed. + [`HfHubHTTPError`] + If the server returned an error. + """ + endpoint = endpoint if endpoint is not None else constants.ENDPOINT + url_prefix = "" + if repo_type in constants.REPO_TYPES_URL_PREFIXES: + url_prefix = constants.REPO_TYPES_URL_PREFIXES[repo_type] + batch_url = f"{endpoint}/{url_prefix}{repo_id}.git/info/lfs/objects/batch" + payload: dict = { + "operation": "upload", + "transfers": transfers if transfers is not None else ["basic", "multipart"], + "objects": [ + { + "oid": upload.sha256.hex(), + "size": upload.size, + } + for upload in upload_infos + ], + "hash_algo": "sha256", + } + if revision is not None: + payload["ref"] = {"name": unquote(revision)} # revision has been previously 'quoted' + + headers = { + **LFS_HEADERS, + **build_hf_headers(token=token), + **(headers or {}), + } + resp = http_backoff("POST", batch_url, headers=headers, json=payload) + hf_raise_for_status(resp) + batch_info = resp.json() + + objects = batch_info.get("objects", None) + if not isinstance(objects, list): + raise ValueError("Malformed response from server") + + chosen_transfer = batch_info.get("transfer") + chosen_transfer = chosen_transfer if isinstance(chosen_transfer, str) else None + + return ( + [_validate_batch_actions(obj) for obj in objects if "error" not in obj], + [_validate_batch_error(obj) for obj in objects if "error" in obj], + chosen_transfer, + ) + + +class PayloadPartT(TypedDict): + partNumber: int + etag: str + + +class CompletionPayloadT(TypedDict): + """Payload that will be sent to the Hub when uploading multi-part.""" + + oid: str + parts: list[PayloadPartT] + + +def lfs_upload( + operation: "CommitOperationAdd", + lfs_batch_action: dict, + token: str | None = None, + headers: dict[str, str] | None = None, + endpoint: str | None = None, +) -> None: + """ + Handles uploading a given object to the Hub with the LFS protocol. + + Can be a No-op if the content of the file is already present on the hub large file storage. + + Args: + operation (`CommitOperationAdd`): + The add operation triggering this upload. + lfs_batch_action (`dict`): + Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for + more details. + token (`str`, *optional*): + An authentication token (see https://huggingface.co/settings/token). Used to call the + optional LFS verify step at the end of the upload. If `None`, falls back to the local + cached token. + headers (`dict`, *optional*): + Headers to include in the request, including authentication and user agent headers. + endpoint (`str`, *optional*): + The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`. + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If `lfs_batch_action` is improperly formatted + [`HfHubHTTPError`] + If the upload resulted in an error + """ + # 0. If LFS file is already present, skip upload + _validate_batch_actions(lfs_batch_action) + actions = lfs_batch_action.get("actions") + if actions is None: + # The file was already uploaded + logger.debug(f"Content of file {operation.path_in_repo} is already present upstream - skipping upload") + return + + # 1. Validate server response (check required keys in dict) + upload_action = lfs_batch_action["actions"]["upload"] + _validate_lfs_action(upload_action) + verify_action = lfs_batch_action["actions"].get("verify") + if verify_action is not None: + _validate_lfs_action(verify_action) + + # 2. Upload file (either single part or multi-part) + header = upload_action.get("header", {}) + chunk_size = header.get("chunk_size") + upload_url = fix_hf_endpoint_in_url(upload_action["href"], endpoint=endpoint) + if chunk_size is not None: + try: + chunk_size = int(chunk_size) + except (ValueError, TypeError): + raise ValueError( + f"Malformed response from LFS batch endpoint: `chunk_size` should be an integer. Got '{chunk_size}'." + ) + _upload_multi_part(operation=operation, header=header, chunk_size=chunk_size, upload_url=upload_url) + else: + _upload_single_part(operation=operation, upload_url=upload_url) + + # 3. Verify upload went well + if verify_action is not None: + _validate_lfs_action(verify_action) + verify_url = fix_hf_endpoint_in_url(verify_action["href"], endpoint) + verify_resp = http_backoff( + "POST", + verify_url, + headers=build_hf_headers(token=token, headers=headers), + json={"oid": operation.upload_info.sha256.hex(), "size": operation.upload_info.size}, + ) + hf_raise_for_status(verify_resp) + logger.debug(f"{operation.path_in_repo}: Upload successful") + + +def _validate_lfs_action(lfs_action: dict): + """validates response from the LFS batch endpoint""" + if not ( + isinstance(lfs_action.get("href"), str) + and (lfs_action.get("header") is None or isinstance(lfs_action.get("header"), dict)) + ): + raise ValueError("lfs_action is improperly formatted") + return lfs_action + + +def _validate_batch_actions(lfs_batch_actions: dict): + """validates response from the LFS batch endpoint""" + if not (isinstance(lfs_batch_actions.get("oid"), str) and isinstance(lfs_batch_actions.get("size"), int)): + raise ValueError("lfs_batch_actions is improperly formatted") + + upload_action = lfs_batch_actions.get("actions", {}).get("upload") + verify_action = lfs_batch_actions.get("actions", {}).get("verify") + if upload_action is not None: + _validate_lfs_action(upload_action) + if verify_action is not None: + _validate_lfs_action(verify_action) + return lfs_batch_actions + + +def _validate_batch_error(lfs_batch_error: dict): + """validates response from the LFS batch endpoint""" + if not (isinstance(lfs_batch_error.get("oid"), str) and isinstance(lfs_batch_error.get("size"), int)): + raise ValueError("lfs_batch_error is improperly formatted") + error_info = lfs_batch_error.get("error") + if not ( + isinstance(error_info, dict) + and isinstance(error_info.get("message"), str) + and isinstance(error_info.get("code"), int) + ): + raise ValueError("lfs_batch_error is improperly formatted") + return lfs_batch_error + + +def _upload_single_part(operation: "CommitOperationAdd", upload_url: str) -> None: + """ + Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol) + + Args: + upload_url (`str`): + The URL to PUT the file to. + fileobj: + The file-like object holding the data to upload. + + Raises: + [`HfHubHTTPError`] + If the upload resulted in an error. + """ + with operation.as_file(with_tqdm=True) as fileobj: + # S3 might raise a transient 500 error -> let's retry if that happens + response = http_backoff("PUT", upload_url, data=fileobj) + hf_raise_for_status(response) + + +def _upload_multi_part(operation: "CommitOperationAdd", header: dict, chunk_size: int, upload_url: str) -> None: + """ + Uploads file using HF multipart LFS transfer protocol. + """ + # 1. Get upload URLs for each part + sorted_parts_urls = _get_sorted_parts_urls(header=header, upload_info=operation.upload_info, chunk_size=chunk_size) + + # 2. Upload parts (pure Python) + response_headers = _upload_parts_iteratively( + operation=operation, sorted_parts_urls=sorted_parts_urls, chunk_size=chunk_size + ) + + # 3. Send completion request + # NOTE: `upload_url` is the Hub completion endpoint (not the S3 upload URLs). + completion_res = http_backoff( + "POST", + upload_url, + json=_get_completion_payload(response_headers, operation.upload_info.sha256.hex()), + headers=LFS_HEADERS, + ) + hf_raise_for_status(completion_res) + + +def _get_sorted_parts_urls(header: dict, upload_info: UploadInfo, chunk_size: int) -> list[str]: + sorted_part_upload_urls = [ + upload_url + for _, upload_url in sorted( + [ + (int(part_num, 10), upload_url) + for part_num, upload_url in header.items() + if part_num.isdigit() and len(part_num) > 0 + ], + key=lambda t: t[0], + ) + ] + num_parts = len(sorted_part_upload_urls) + if num_parts != ceil(upload_info.size / chunk_size): + raise ValueError("Invalid server response to upload large LFS file") + return sorted_part_upload_urls + + +def _get_completion_payload(response_headers: list[dict], oid: str) -> CompletionPayloadT: + parts: list[PayloadPartT] = [] + for part_number, header in enumerate(response_headers): + etag = header.get("etag") + if etag is None or etag == "": + raise ValueError(f"Invalid etag (`{etag}`) returned for part {part_number + 1}") + parts.append( + { + "partNumber": part_number + 1, + "etag": etag, + } + ) + return {"oid": oid, "parts": parts} + + +def _upload_parts_iteratively( + operation: "CommitOperationAdd", sorted_parts_urls: list[str], chunk_size: int +) -> list[dict]: + headers = [] + with operation.as_file(with_tqdm=True) as fileobj: + for part_idx, part_upload_url in enumerate(sorted_parts_urls): + with SliceFileObj( + fileobj, + seek_from=chunk_size * part_idx, + read_limit=chunk_size, + ) as fileobj_slice: + # S3 might raise a transient 500 error -> let's retry if that happens + part_upload_res = http_backoff("PUT", part_upload_url, data=fileobj_slice) + hf_raise_for_status(part_upload_res) + headers.append(part_upload_res.headers) + return headers # type: ignore diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/py.typed b/venv/lib/python3.12/site-packages/huggingface_hub/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/repocard.py b/venv/lib/python3.12/site-packages/huggingface_hub/repocard.py new file mode 100644 index 0000000000000000000000000000000000000000..1bb03da924520ab87ad202b6032b2e4d85bae195 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/repocard.py @@ -0,0 +1,835 @@ +import os +import re +from pathlib import Path +from typing import Any, Literal + +import yaml + +from huggingface_hub.file_download import hf_hub_download +from huggingface_hub.hf_api import upload_file +from huggingface_hub.repocard_data import ( + CardData, + DatasetCardData, + EvalResult, + ModelCardData, + SpaceCardData, + eval_results_to_model_index, + model_index_to_eval_results, +) +from huggingface_hub.utils import HfHubHTTPError, get_session, hf_raise_for_status, is_jinja_available, yaml_dump + +from . import constants +from .errors import EntryNotFoundError +from .utils import SoftTemporaryDirectory, logging, validate_hf_hub_args + + +logger = logging.get_logger(__name__) + + +TEMPLATE_MODELCARD_PATH = Path(__file__).parent / "templates" / "modelcard_template.md" +TEMPLATE_DATASETCARD_PATH = Path(__file__).parent / "templates" / "datasetcard_template.md" + +# exact same regex as in the Hub server. Please keep in sync. +# See https://github.com/huggingface/moon-landing/blob/main/server/lib/ViewMarkdown.ts#L18 +REGEX_YAML_BLOCK = re.compile(r"^(\s*---(?:\r\n|\r|\n))([\S\s]*?)((?:\r\n|\r|\n)---[ \t]*(\r\n|\n|$))") + + +class RepoCard: + card_data_class = CardData + default_template_path = TEMPLATE_MODELCARD_PATH + repo_type = "model" + + def __init__(self, content: str, ignore_metadata_errors: bool = False): + """Initialize a RepoCard from string content. The content should be a + Markdown file with a YAML block at the beginning and a Markdown body. + + Args: + content (`str`): The content of the Markdown file. + + Example: + ```python + >>> from huggingface_hub.repocard import RepoCard + >>> text = ''' + ... --- + ... language: en + ... license: mit + ... --- + ... + ... # My repo + ... ''' + >>> card = RepoCard(text) + >>> card.data.to_dict() + {'language': 'en', 'license': 'mit'} + >>> card.text + '\\n# My repo\\n' + + ``` + > [!TIP] + > Raises the following error: + > + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > when the content of the repo card metadata is not a dictionary. + """ + + # Set the content of the RepoCard, as well as underlying .data and .text attributes. + # See the `content` property setter for more details. + self.ignore_metadata_errors = ignore_metadata_errors + self.content = content + + @property + def content(self): + """The content of the RepoCard, including the YAML block and the Markdown body.""" + line_break = _detect_line_ending(self._content) or "\n" + return f"---{line_break}{self.data.to_yaml(line_break=line_break, original_order=self._original_order)}{line_break}---{line_break}{self.text}" + + @content.setter + def content(self, content: str): + """Set the content of the RepoCard.""" + self._content = content + + match = REGEX_YAML_BLOCK.search(content) + if match: + # Metadata found in the YAML block + yaml_block = match.group(2) + self.text = content[match.end() :] + data_dict = yaml.safe_load(yaml_block) + + if data_dict is None: + data_dict = {} + + # The YAML block's data should be a dictionary + if not isinstance(data_dict, dict): + raise ValueError("repo card metadata block should be a dict") + else: + # Model card without metadata... create empty metadata + logger.warning("Repo card metadata block was not found. Setting CardData to empty.") + data_dict = {} + self.text = content + + self.data = self.card_data_class(**data_dict, ignore_metadata_errors=self.ignore_metadata_errors) + self._original_order = list(data_dict.keys()) + + def __str__(self): + return self.content + + def save(self, filepath: Path | str): + r"""Save a RepoCard to a file. + + Args: + filepath (`Union[Path, str]`): Filepath to the markdown file to save. + + Example: + ```python + >>> from huggingface_hub.repocard import RepoCard + >>> card = RepoCard("---\nlanguage: en\n---\n# This is a test repo card") + >>> card.save("/tmp/test.md") + + ``` + """ + filepath = Path(filepath) + filepath.parent.mkdir(parents=True, exist_ok=True) + # Preserve newlines as in the existing file. + with open(filepath, mode="w", newline="", encoding="utf-8") as f: + f.write(str(self)) + + @classmethod + def load( + cls, + repo_id_or_path: str | Path, + repo_type: str | None = None, + token: str | None = None, + ignore_metadata_errors: bool = False, + ): + """Initialize a RepoCard from a Hugging Face Hub repo's README.md or a local filepath. + + Args: + repo_id_or_path (`Union[str, Path]`): + The repo ID associated with a Hugging Face Hub repo or a local filepath. + repo_type (`str`, *optional*): + The type of Hugging Face repo to push to. Defaults to None, which will use "model". Other options + are "dataset" and "space". Not used when loading from a local filepath. If this is called from a child + class, the default value will be the child class's `repo_type`. + token (`str`, *optional*): + Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to the stored token. + ignore_metadata_errors (`str`): + If True, errors while parsing the metadata section will be ignored. Some information might be lost during + the process. Use it at your own risk. + + Returns: + [`huggingface_hub.repocard.RepoCard`]: The RepoCard (or subclass) initialized from the repo's + README.md file or filepath. + + Example: + ```python + >>> from huggingface_hub.repocard import RepoCard + >>> card = RepoCard.load("nateraw/food") + >>> assert card.data.tags == ["generated_from_trainer", "image-classification", "pytorch"] + + ``` + """ + + if Path(repo_id_or_path).is_file(): + card_path = Path(repo_id_or_path) + elif isinstance(repo_id_or_path, str): + card_path = Path( + hf_hub_download( + repo_id_or_path, + constants.REPOCARD_NAME, + repo_type=repo_type or cls.repo_type, + token=token, + ) + ) + else: + raise ValueError(f"Cannot load RepoCard: path not found on disk ({repo_id_or_path}).") + + # Preserve newlines in the existing file. + with card_path.open(mode="r", newline="", encoding="utf-8") as f: + return cls(f.read(), ignore_metadata_errors=ignore_metadata_errors) + + def validate(self, repo_type: str | None = None): + """Validates card against Hugging Face Hub's card validation logic. + Using this function requires access to the internet, so it is only called + internally by [`huggingface_hub.repocard.RepoCard.push_to_hub`]. + + Args: + repo_type (`str`, *optional*, defaults to "model"): + The type of Hugging Face repo to push to. Options are "model", "dataset", and "space". + If this function is called from a child class, the default will be the child class's `repo_type`. + + > [!TIP] + > Raises the following errors: + > + > - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > if the card fails validation checks. + > - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) + > if the request to the Hub API fails for any other reason. + """ + + # If repo type is provided, otherwise, use the repo type of the card. + repo_type = repo_type or self.repo_type + + body = { + "repoType": repo_type, + "content": str(self), + } + headers = {"Accept": "text/plain"} + + try: + response = get_session().post("https://huggingface.co/api/validate-yaml", json=body, headers=headers) + hf_raise_for_status(response) + except HfHubHTTPError as exc: + if response.status_code == 400: + raise ValueError(response.text) + else: + raise exc + + def push_to_hub( + self, + repo_id: str, + token: str | None = None, + repo_type: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + revision: str | None = None, + create_pr: bool | None = None, + parent_commit: str | None = None, + ): + """Push a RepoCard to a Hugging Face Hub repo. + + Args: + repo_id (`str`): + The repo ID of the Hugging Face Hub repo to push to. Example: "nateraw/food". + token (`str`, *optional*): + Authentication token, obtained with `huggingface_hub.HfApi.login` method. Will default to + the stored token. + repo_type (`str`, *optional*, defaults to "model"): + The type of Hugging Face repo to push to. Options are "model", "dataset", and "space". If this + function is called by a child class, it will default to the child class's `repo_type`. + commit_message (`str`, *optional*): + The summary / title / first line of the generated commit. + commit_description (`str`, *optional*) + The description of the generated commit. + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the `"main"` branch. + create_pr (`bool`, *optional*): + Whether or not to create a Pull Request with this commit. Defaults to `False`. + parent_commit (`str`, *optional*): + The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. + If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. + If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. + Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be + especially useful if the repo is updated / committed too concurrently. + Returns: + `str`: URL of the commit which updated the card metadata. + """ + + # If repo type is provided, otherwise, use the repo type of the card. + repo_type = repo_type or self.repo_type + + # Validate card before pushing to hub + self.validate(repo_type=repo_type) + + with SoftTemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) / constants.REPOCARD_NAME + tmp_path.write_text(str(self), encoding="utf-8") + url = upload_file( + path_or_fileobj=str(tmp_path), + path_in_repo=constants.REPOCARD_NAME, + repo_id=repo_id, + token=token, + repo_type=repo_type, + commit_message=commit_message, + commit_description=commit_description, + create_pr=create_pr, + revision=revision, + parent_commit=parent_commit, + ) + return url + + @classmethod + def from_template( + cls, + card_data: CardData, + template_path: str | None = None, + template_str: str | None = None, + **template_kwargs, + ): + """Initialize a RepoCard from a template. By default, it uses the default template. + + Templates are Jinja2 templates that can be customized by passing keyword arguments. + + Args: + card_data (`huggingface_hub.CardData`): + A huggingface_hub.CardData instance containing the metadata you want to include in the YAML + header of the repo card on the Hugging Face Hub. + template_path (`str`, *optional*): + A path to a markdown file with optional Jinja template variables that can be filled + in with `template_kwargs`. Defaults to the default template. + template_str (`str`, *optional*): + A raw Jinja template string with optional variables. Used when neither `template_path` + nor the default template is appropriate. Ignored if `template_path` is also provided. + + Returns: + [`huggingface_hub.repocard.RepoCard`]: A RepoCard instance with the specified card data and content from the + template. + """ + if is_jinja_available(): + import jinja2 + else: + raise ImportError( + "Using RepoCard.from_template requires Jinja2 to be installed. Please" + " install it with `pip install Jinja2`." + ) + + kwargs = card_data.to_dict().copy() + kwargs.update(template_kwargs) # Template_kwargs have priority + + if template_path is not None: + template_str = Path(template_path).read_text() + if template_str is None: + template_str = Path(cls.default_template_path).read_text() + template = jinja2.Template(template_str) + content = template.render(card_data=card_data.to_yaml(), **kwargs) + return cls(content) + + +class ModelCard(RepoCard): + card_data_class = ModelCardData # type: ignore[assignment] + default_template_path = TEMPLATE_MODELCARD_PATH + repo_type = "model" + + @classmethod + def from_template( # type: ignore # violates Liskov property but easier to use + cls, + card_data: ModelCardData, + template_path: str | None = None, + template_str: str | None = None, + **template_kwargs, + ): + """Initialize a ModelCard from a template. By default, it uses the default template, which can be found here: + https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/modelcard_template.md + + Templates are Jinja2 templates that can be customized by passing keyword arguments. + + Args: + card_data (`huggingface_hub.ModelCardData`): + A huggingface_hub.ModelCardData instance containing the metadata you want to include in the YAML + header of the model card on the Hugging Face Hub. + template_path (`str`, *optional*): + A path to a markdown file with optional Jinja template variables that can be filled + in with `template_kwargs`. Defaults to the default template. + template_str (`str`, *optional*): + A raw Jinja template string with optional variables. Used when neither `template_path` + nor the default template is appropriate. Ignored if `template_path` is also provided. + + Returns: + [`huggingface_hub.ModelCard`]: A ModelCard instance with the specified card data and content from the + template. + + Example: + ```python + >>> from huggingface_hub import ModelCard, ModelCardData, EvalResult + + >>> # Using the Default Template + >>> card_data = ModelCardData( + ... language='en', + ... license='mit', + ... library_name='timm', + ... tags=['image-classification', 'resnet'], + ... datasets=['beans'], + ... metrics=['accuracy'], + ... ) + >>> card = ModelCard.from_template( + ... card_data, + ... model_description='This model does x + y...' + ... ) + + >>> # Including Evaluation Results + >>> card_data = ModelCardData( + ... language='en', + ... tags=['image-classification', 'resnet'], + ... eval_results=[ + ... EvalResult( + ... task_type='image-classification', + ... dataset_type='beans', + ... dataset_name='Beans', + ... metric_type='accuracy', + ... metric_value=0.9, + ... ), + ... ], + ... model_name='my-cool-model', + ... ) + >>> card = ModelCard.from_template(card_data) + + >>> # Using a Custom Template + >>> card_data = ModelCardData( + ... language='en', + ... tags=['image-classification', 'resnet'] + ... ) + >>> card = ModelCard.from_template( + ... card_data=card_data, + ... template_path='./src/huggingface_hub/templates/modelcard_template.md', + ... custom_template_var='custom value', # will be replaced in template if it exists + ... ) + + ``` + """ + return super().from_template(card_data, template_path, template_str, **template_kwargs) + + +class DatasetCard(RepoCard): + card_data_class = DatasetCardData # type: ignore[assignment] + default_template_path = TEMPLATE_DATASETCARD_PATH + repo_type = "dataset" + + @classmethod + def from_template( # type: ignore # violates Liskov property but easier to use + cls, + card_data: DatasetCardData, + template_path: str | None = None, + template_str: str | None = None, + **template_kwargs, + ): + """Initialize a DatasetCard from a template. By default, it uses the default template, which can be found here: + https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/datasetcard_template.md + + Templates are Jinja2 templates that can be customized by passing keyword arguments. + + Args: + card_data (`huggingface_hub.DatasetCardData`): + A huggingface_hub.DatasetCardData instance containing the metadata you want to include in the YAML + header of the dataset card on the Hugging Face Hub. + template_path (`str`, *optional*): + A path to a markdown file with optional Jinja template variables that can be filled + in with `template_kwargs`. Defaults to the default template. + template_str (`str`, *optional*): + A raw Jinja template string with optional variables. Used when neither `template_path` + nor the default template is appropriate. Ignored if `template_path` is also provided. + + Returns: + [`huggingface_hub.DatasetCard`]: A DatasetCard instance with the specified card data and content from the + template. + + Example: + ```python + >>> from huggingface_hub import DatasetCard, DatasetCardData + + >>> # Using the Default Template + >>> card_data = DatasetCardData( + ... language='en', + ... license='mit', + ... annotations_creators='crowdsourced', + ... task_categories=['text-classification'], + ... task_ids=['sentiment-classification', 'text-scoring'], + ... multilinguality='monolingual', + ... pretty_name='My Text Classification Dataset', + ... ) + >>> card = DatasetCard.from_template( + ... card_data, + ... pretty_name=card_data.pretty_name, + ... ) + + >>> # Using a Custom Template + >>> card_data = DatasetCardData( + ... language='en', + ... license='mit', + ... ) + >>> card = DatasetCard.from_template( + ... card_data=card_data, + ... template_path='./src/huggingface_hub/templates/datasetcard_template.md', + ... custom_template_var='custom value', # will be replaced in template if it exists + ... ) + + ``` + """ + return super().from_template(card_data, template_path, template_str, **template_kwargs) + + +class SpaceCard(RepoCard): + card_data_class = SpaceCardData # type: ignore[assignment] + default_template_path = TEMPLATE_MODELCARD_PATH + repo_type = "space" + + +def _detect_line_ending(content: str) -> Literal["\r", "\n", "\r\n", None]: # noqa: F722 + """Detect the line ending of a string. Used by RepoCard to avoid making huge diff on newlines. + + Uses same implementation as in Hub server, keep it in sync. + + Returns: + str: The detected line ending of the string. + """ + cr = content.count("\r") + lf = content.count("\n") + crlf = content.count("\r\n") + if cr + lf == 0: + return None + if crlf == cr and crlf == lf: + return "\r\n" + if cr > lf: + return "\r" + else: + return "\n" + + +def metadata_load(local_path: str | Path) -> dict | None: + content = Path(local_path).read_text() + match = REGEX_YAML_BLOCK.search(content) + if match: + yaml_block = match.group(2) + data = yaml.safe_load(yaml_block) + if data is None or isinstance(data, dict): + return data + raise ValueError("repo card metadata block should be a dict") + else: + return None + + +def metadata_save(local_path: str | Path, data: dict) -> None: + """ + Save the metadata dict in the upper YAML part Trying to preserve newlines as + in the existing file. Docs about open() with newline="" parameter: + https://docs.python.org/3/library/functions.html?highlight=open#open Does + not work with "^M" linebreaks, which are replaced by \n + """ + line_break = "\n" + content = "" + # try to detect existing newline character + if os.path.exists(local_path): + with open(local_path, newline="", encoding="utf8") as readme: + content = readme.read() + if isinstance(readme.newlines, tuple): + line_break = readme.newlines[0] + elif isinstance(readme.newlines, str): + line_break = readme.newlines + + # creates a new file if it not + with open(local_path, "w", newline="", encoding="utf8") as readme: + data_yaml = yaml_dump(data, sort_keys=False, line_break=line_break) + # sort_keys: keep dict order + match = REGEX_YAML_BLOCK.search(content) + if match: + output = content[: match.start()] + f"---{line_break}{data_yaml}---{line_break}" + content[match.end() :] + else: + output = f"---{line_break}{data_yaml}---{line_break}{content}" + + readme.write(output) + readme.close() + + +def metadata_eval_result( + *, + model_pretty_name: str, + task_pretty_name: str, + task_id: str, + metrics_pretty_name: str, + metrics_id: str, + metrics_value: Any, + dataset_pretty_name: str, + dataset_id: str, + metrics_config: str | None = None, + metrics_verified: bool = False, + dataset_config: str | None = None, + dataset_split: str | None = None, + dataset_revision: str | None = None, + metrics_verification_token: str | None = None, +) -> dict: + """ + Creates a metadata dict with the result from a model evaluated on a dataset. + + Args: + model_pretty_name (`str`): + The name of the model in natural language. + task_pretty_name (`str`): + The name of a task in natural language. + task_id (`str`): + Example: automatic-speech-recognition. A task id. + metrics_pretty_name (`str`): + A name for the metric in natural language. Example: Test WER. + metrics_id (`str`): + Example: wer. A metric id from https://hf.co/metrics. + metrics_value (`Any`): + The value from the metric. Example: 20.0 or "20.0 ± 1.2". + dataset_pretty_name (`str`): + The name of the dataset in natural language. + dataset_id (`str`): + Example: common_voice. A dataset id from https://hf.co/datasets. + metrics_config (`str`, *optional*): + The name of the metric configuration used in `load_metric()`. + Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`. + metrics_verified (`bool`, *optional*, defaults to `False`): + Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set. + dataset_config (`str`, *optional*): + Example: fr. The name of the dataset configuration used in `load_dataset()`. + dataset_split (`str`, *optional*): + Example: test. The name of the dataset split used in `load_dataset()`. + dataset_revision (`str`, *optional*): + Example: 5503434ddd753f426f4b38109466949a1217c2bb. The name of the dataset dataset revision + used in `load_dataset()`. + metrics_verification_token (`bool`, *optional*): + A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. + + Returns: + `dict`: a metadata dict with the result from a model evaluated on a dataset. + + Example: + ```python + >>> from huggingface_hub import metadata_eval_result + >>> results = metadata_eval_result( + ... model_pretty_name="RoBERTa fine-tuned on ReactionGIF", + ... task_pretty_name="Text Classification", + ... task_id="text-classification", + ... metrics_pretty_name="Accuracy", + ... metrics_id="accuracy", + ... metrics_value=0.2662102282047272, + ... dataset_pretty_name="ReactionJPEG", + ... dataset_id="julien-c/reactionjpeg", + ... dataset_config="default", + ... dataset_split="test", + ... ) + >>> results == { + ... 'model-index': [ + ... { + ... 'name': 'RoBERTa fine-tuned on ReactionGIF', + ... 'results': [ + ... { + ... 'task': { + ... 'type': 'text-classification', + ... 'name': 'Text Classification' + ... }, + ... 'dataset': { + ... 'name': 'ReactionJPEG', + ... 'type': 'julien-c/reactionjpeg', + ... 'config': 'default', + ... 'split': 'test' + ... }, + ... 'metrics': [ + ... { + ... 'type': 'accuracy', + ... 'value': 0.2662102282047272, + ... 'name': 'Accuracy', + ... 'verified': False + ... } + ... ] + ... } + ... ] + ... } + ... ] + ... } + True + + ``` + """ + + return { + "model-index": eval_results_to_model_index( + model_name=model_pretty_name, + eval_results=[ + EvalResult( + task_name=task_pretty_name, + task_type=task_id, + metric_name=metrics_pretty_name, + metric_type=metrics_id, + metric_value=metrics_value, + dataset_name=dataset_pretty_name, + dataset_type=dataset_id, + metric_config=metrics_config, + verified=metrics_verified, + verify_token=metrics_verification_token, + dataset_config=dataset_config, + dataset_split=dataset_split, + dataset_revision=dataset_revision, + ) + ], + ) + } + + +@validate_hf_hub_args +def metadata_update( + repo_id: str, + metadata: dict, + *, + repo_type: str | None = None, + overwrite: bool = False, + token: str | None = None, + commit_message: str | None = None, + commit_description: str | None = None, + revision: str | None = None, + create_pr: bool = False, + parent_commit: str | None = None, +) -> str: + """ + Updates the metadata in the README.md of a repository on the Hugging Face Hub. + If the README.md file doesn't exist yet, a new one is created with metadata and + the default ModelCard or DatasetCard template. For `space` repo, an error is thrown + as a Space cannot exist without a `README.md` file. + + Args: + repo_id (`str`): + The name of the repository. + metadata (`dict`): + A dictionary containing the metadata to be updated. + repo_type (`str`, *optional*): + Set to `"dataset"` or `"space"` if updating to a dataset or space, + `None` or `"model"` if updating to a model. Default is `None`. + overwrite (`bool`, *optional*, defaults to `False`): + If set to `True` an existing field can be overwritten, otherwise + attempting to overwrite an existing field will cause an error. + token (`str`, *optional*): + The Hugging Face authentication token. + commit_message (`str`, *optional*): + The summary / title / first line of the generated commit. Defaults to + `f"Update metadata with huggingface_hub"` + commit_description (`str` *optional*) + The description of the generated commit + revision (`str`, *optional*): + The git revision to commit from. Defaults to the head of the + `"main"` branch. + create_pr (`boolean`, *optional*): + Whether or not to create a Pull Request from `revision` with that commit. + Defaults to `False`. + parent_commit (`str`, *optional*): + The OID / SHA of the parent commit, as a hexadecimal string. Shorthands (7 first characters) are also supported. + If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`. + If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`. + Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be + especially useful if the repo is updated / committed too concurrently. + Returns: + `str`: URL of the commit which updated the card metadata. + + Example: + ```python + >>> from huggingface_hub import metadata_update + >>> metadata = {'model-index': [{'name': 'RoBERTa fine-tuned on ReactionGIF', + ... 'results': [{'dataset': {'name': 'ReactionGIF', + ... 'type': 'julien-c/reactiongif'}, + ... 'metrics': [{'name': 'Recall', + ... 'type': 'recall', + ... 'value': 0.7762102282047272}], + ... 'task': {'name': 'Text Classification', + ... 'type': 'text-classification'}}]}]} + >>> url = metadata_update("hf-internal-testing/reactiongif-roberta-card", metadata) + + ``` + """ + commit_message = commit_message if commit_message is not None else "Update metadata with huggingface_hub" + + # Card class given repo_type + card_class: type[RepoCard] + if repo_type is None or repo_type == "model": + card_class = ModelCard + elif repo_type == "dataset": + card_class = DatasetCard + elif repo_type == "space": + card_class = RepoCard + else: + raise ValueError(f"Unknown repo_type: {repo_type}") + + # Either load repo_card from the Hub or create an empty one. + # NOTE: Will not create the repo if it doesn't exist. + try: + card = card_class.load(repo_id, token=token, repo_type=repo_type) + except EntryNotFoundError: + if repo_type == "space": + raise ValueError("Cannot update metadata on a Space that doesn't contain a `README.md` file.") + + # Initialize a ModelCard or DatasetCard from default template and no data. + # Cast to the concrete expected card type to satisfy type checkers. + card = card_class.from_template(CardData()) # type: ignore + + for key, value in metadata.items(): + if key == "model-index": + # if the new metadata doesn't include a name, either use existing one or repo name + if "name" not in value[0]: + value[0]["name"] = getattr(card, "model_name", repo_id) + model_name, new_results = model_index_to_eval_results(value) + if card.data.eval_results is None: + card.data.eval_results = new_results + card.data.model_name = model_name + else: + existing_results = card.data.eval_results + + # Iterate over new results + # Iterate over existing results + # If both results describe the same metric but value is different: + # If overwrite=True: overwrite the metric value + # Else: raise ValueError + # Else: append new result to existing ones. + for new_result in new_results: + result_found = False + for existing_result in existing_results: + if new_result.is_equal_except_value(existing_result): + if new_result != existing_result and not overwrite: + raise ValueError( + "You passed a new value for the existing metric" + f" 'name: {new_result.metric_name}, type: " + f"{new_result.metric_type}'. Set `overwrite=True`" + " to overwrite existing metrics." + ) + result_found = True + existing_result.metric_value = new_result.metric_value + if existing_result.verified is True: + existing_result.verify_token = new_result.verify_token + if not result_found: + card.data.eval_results.append(new_result) + else: + # Any metadata that is not a result metric + if card.data.get(key) is not None and not overwrite and card.data.get(key) != value: + raise ValueError( + f"You passed a new value for the existing meta data field '{key}'." + " Set `overwrite=True` to overwrite existing metadata." + ) + else: + card.data[key] = value + + return card.push_to_hub( + repo_id, + token=token, + repo_type=repo_type, + commit_message=commit_message, + commit_description=commit_description, + create_pr=create_pr, + revision=revision, + parent_commit=parent_commit, + ) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/repocard_data.py b/venv/lib/python3.12/site-packages/huggingface_hub/repocard_data.py new file mode 100644 index 0000000000000000000000000000000000000000..69539f809b7f38b54be76fe689153b3c3ec4ff7a --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/repocard_data.py @@ -0,0 +1,776 @@ +import copy +from collections import defaultdict +from dataclasses import dataclass +from typing import Any + +from huggingface_hub.utils import logging, yaml_dump + + +logger = logging.get_logger(__name__) + + +@dataclass +class EvalResult: + """ + Flattened representation of individual evaluation results found in model-index of Model Cards. + + For more information on the model-index spec, see https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1. + + Args: + task_type (`str`): + The task identifier. Example: "image-classification". + dataset_type (`str`): + The dataset identifier. Example: "common_voice". Use dataset id from https://hf.co/datasets. + dataset_name (`str`): + A pretty name for the dataset. Example: "Common Voice (French)". + metric_type (`str`): + The metric identifier. Example: "wer". Use metric id from https://hf.co/metrics. + metric_value (`Any`): + The metric value. Example: 0.9 or "20.0 ± 1.2". + task_name (`str`, *optional*): + A pretty name for the task. Example: "Speech Recognition". + dataset_config (`str`, *optional*): + The name of the dataset configuration used in `load_dataset()`. + Example: fr in `load_dataset("common_voice", "fr")`. See the `datasets` docs for more info: + https://hf.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name + dataset_split (`str`, *optional*): + The split used in `load_dataset()`. Example: "test". + dataset_revision (`str`, *optional*): + The revision (AKA Git Sha) of the dataset used in `load_dataset()`. + Example: 5503434ddd753f426f4b38109466949a1217c2bb + dataset_args (`dict[str, Any]`, *optional*): + The arguments passed during `Metric.compute()`. Example for `bleu`: `{"max_order": 4}` + metric_name (`str`, *optional*): + A pretty name for the metric. Example: "Test WER". + metric_config (`str`, *optional*): + The name of the metric configuration used in `load_metric()`. + Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`. + See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations + metric_args (`dict[str, Any]`, *optional*): + The arguments passed during `Metric.compute()`. Example for `bleu`: max_order: 4 + verified (`bool`, *optional*): + Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set. + verify_token (`str`, *optional*): + A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. + source_name (`str`, *optional*): + The name of the source of the evaluation result. Example: "Open LLM Leaderboard". + source_url (`str`, *optional*): + The URL of the source of the evaluation result. Example: "https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard". + """ + + # Required + + # The task identifier + # Example: automatic-speech-recognition + task_type: str + + # The dataset identifier + # Example: common_voice. Use dataset id from https://hf.co/datasets + dataset_type: str + + # A pretty name for the dataset. + # Example: Common Voice (French) + dataset_name: str + + # The metric identifier + # Example: wer. Use metric id from https://hf.co/metrics + metric_type: str + + # Value of the metric. + # Example: 20.0 or "20.0 ± 1.2" + metric_value: Any + + # Optional + + # A pretty name for the task. + # Example: Speech Recognition + task_name: str | None = None + + # The name of the dataset configuration used in `load_dataset()`. + # Example: fr in `load_dataset("common_voice", "fr")`. + # See the `datasets` docs for more info: + # https://huggingface.co/docs/datasets/package_reference/loading_methods#datasets.load_dataset.name + dataset_config: str | None = None + + # The split used in `load_dataset()`. + # Example: test + dataset_split: str | None = None + + # The revision (AKA Git Sha) of the dataset used in `load_dataset()`. + # Example: 5503434ddd753f426f4b38109466949a1217c2bb + dataset_revision: str | None = None + + # The arguments passed during `Metric.compute()`. + # Example for `bleu`: max_order: 4 + dataset_args: dict[str, Any] | None = None + + # A pretty name for the metric. + # Example: Test WER + metric_name: str | None = None + + # The name of the metric configuration used in `load_metric()`. + # Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`. + # See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations + metric_config: str | None = None + + # The arguments passed during `Metric.compute()`. + # Example for `bleu`: max_order: 4 + metric_args: dict[str, Any] | None = None + + # Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set. + verified: bool | None = None + + # A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. + verify_token: str | None = None + + # The name of the source of the evaluation result. + # Example: Open LLM Leaderboard + source_name: str | None = None + + # The URL of the source of the evaluation result. + # Example: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard + source_url: str | None = None + + @property + def unique_identifier(self) -> tuple: + """Returns a tuple that uniquely identifies this evaluation.""" + return ( + self.task_type, + self.dataset_type, + self.dataset_config, + self.dataset_split, + self.dataset_revision, + ) + + def is_equal_except_value(self, other: "EvalResult") -> bool: + """ + Return True if `self` and `other` describe exactly the same metric but with a + different value. + """ + for key, _ in self.__dict__.items(): + if key == "metric_value": + continue + # For metrics computed by Hugging Face's evaluation service, `verify_token` is derived from `metric_value`, + # so we exclude it here in the comparison. + if key != "verify_token" and getattr(self, key) != getattr(other, key): + return False + return True + + def __post_init__(self) -> None: + if self.source_name is not None and self.source_url is None: + raise ValueError("If `source_name` is provided, `source_url` must also be provided.") + + +@dataclass +class CardData: + """Structure containing metadata from a RepoCard. + + [`CardData`] is the parent class of [`ModelCardData`] and [`DatasetCardData`]. + + Metadata can be exported as a dictionary or YAML. Export can be customized to alter the representation of the data + (example: flatten evaluation results). `CardData` behaves as a dictionary (can get, pop, set values) but do not + inherit from `dict` to allow this export step. + """ + + def __init__(self, ignore_metadata_errors: bool = False, **kwargs): + self.__dict__.update(kwargs) + + def to_dict(self): + """Converts CardData to a dict. + + Returns: + `dict`: CardData represented as a dictionary ready to be dumped to a YAML + block for inclusion in a README.md file. + """ + + data_dict = copy.deepcopy(self.__dict__) + self._to_dict(data_dict) + return {key: value for key, value in data_dict.items() if value is not None} + + def _to_dict(self, data_dict): + """Use this method in child classes to alter the dict representation of the data. Alter the dict in-place. + + Args: + data_dict (`dict`): The raw dict representation of the card data. + """ + pass + + def to_yaml(self, line_break=None, original_order: list[str] | None = None) -> str: + """Dumps CardData to a YAML block for inclusion in a README.md file. + + Args: + line_break (str, *optional*): + The line break to use when dumping to yaml. + original_order (`list[str]`, *optional*): + If provided, reorder the metadata fields to match this list before dumping. + Any keys not in `original_order` are appended after the listed keys, preserving + their existing relative order. Useful for round-tripping a YAML block without + shuffling its keys. + + Returns: + `str`: CardData represented as a YAML block. + """ + if original_order: + original_order_set = set(original_order) + self.__dict__ = { + k: self.__dict__[k] + for k in original_order + [k for k in self.__dict__ if k not in original_order_set] + if k in self.__dict__ + } + return yaml_dump(self.to_dict(), sort_keys=False, line_break=line_break).strip() + + def __repr__(self): + return repr(self.__dict__) + + def __str__(self): + return self.to_yaml() + + def get(self, key: str, default: Any = None) -> Any: + """Get value for a given metadata key.""" + value = self.__dict__.get(key) + return default if value is None else value + + def pop(self, key: str, default: Any = None) -> Any: + """Pop value for a given metadata key.""" + return self.__dict__.pop(key, default) + + def __getitem__(self, key: str) -> Any: + """Get value for a given metadata key.""" + return self.__dict__[key] + + def __setitem__(self, key: str, value: Any) -> None: + """Set value for a given metadata key.""" + self.__dict__[key] = value + + def __contains__(self, key: str) -> bool: + """Check if a given metadata key is set.""" + return key in self.__dict__ + + def __len__(self) -> int: + """Return the number of metadata keys set.""" + return len(self.__dict__) + + +def _validate_eval_results( + eval_results: EvalResult | list[EvalResult] | None, + model_name: str | None, +) -> list[EvalResult]: + if eval_results is None: + return [] + if isinstance(eval_results, EvalResult): + eval_results = [eval_results] + if not isinstance(eval_results, list) or not all(isinstance(r, EvalResult) for r in eval_results): + raise ValueError( + f"`eval_results` should be of type `EvalResult` or a list of `EvalResult`, got {type(eval_results)}." + ) + if model_name is None: + raise ValueError("Passing `eval_results` requires `model_name` to be set.") + return eval_results + + +class ModelCardData(CardData): + """Model Card Metadata that is used by Hugging Face Hub when included at the top of your README.md + + Args: + base_model (`str` or `list[str]`, *optional*): + The identifier of the base model from which the model derives. This is applicable for example if your model is a + fine-tune or adapter of an existing model. The value must be the ID of a model on the Hub (or a list of IDs + if your model derives from multiple models). Defaults to None. + datasets (`Union[str, list[str]]`, *optional*): + Dataset or list of datasets that were used to train this model. Should be a dataset ID + found on https://hf.co/datasets. Defaults to None. + eval_results (`Union[list[EvalResult], EvalResult]`, *optional*): + List of `huggingface_hub.EvalResult` that define evaluation results of the model. If provided, + `model_name` is used to as a name on PapersWithCode's leaderboards. Defaults to `None`. + language (`Union[str, list[str]]`, *optional*): + Language of model's training data or metadata. It must be an ISO 639-1, 639-2 or + 639-3 code (two/three letters), or a special value like "code", "multilingual". Defaults to `None`. + library_name (`str`, *optional*): + Name of library used by this model. Example: keras or any library from + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/model-libraries.ts. + Defaults to None. + license (`str`, *optional*): + License of this model. Example: apache-2.0 or any license from + https://huggingface.co/docs/hub/repositories-licenses. Defaults to None. + license_name (`str`, *optional*): + Name of the license of this model. Defaults to None. To be used in conjunction with `license_link`. + Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a name. In that case, use `license` instead. + license_link (`str`, *optional*): + Link to the license of this model. Defaults to None. To be used in conjunction with `license_name`. + Common licenses (Apache-2.0, MIT, CC-BY-SA-4.0) do not need a link. In that case, use `license` instead. + metrics (`list[str]`, *optional*): + List of metrics used to evaluate this model. Should be a metric name that can be found + at https://hf.co/metrics. Example: 'accuracy'. Defaults to None. + model_name (`str`, *optional*): + A name for this model. It is used along with + `eval_results` to construct the `model-index` within the card's metadata. The name + you supply here is what will be used on PapersWithCode's leaderboards. If None is provided + then the repo name is used as a default. Defaults to None. + pipeline_tag (`str`, *optional*): + The pipeline tag associated with the model. Example: "text-classification". + tags (`list[str]`, *optional*): + List of tags to add to your model that can be used when filtering on the Hugging + Face Hub. Defaults to None. + ignore_metadata_errors (`str`): + If True, errors while parsing the metadata section will be ignored. Some information might be lost during + the process. Use it at your own risk. + kwargs (`dict`, *optional*): + Additional metadata that will be added to the model card. Defaults to None. + + Example: + ```python + >>> from huggingface_hub import ModelCardData + >>> card_data = ModelCardData( + ... language="en", + ... license="mit", + ... library_name="timm", + ... tags=['image-classification', 'resnet'], + ... ) + >>> card_data.to_dict() + {'language': 'en', 'license': 'mit', 'library_name': 'timm', 'tags': ['image-classification', 'resnet']} + + ``` + """ + + def __init__( + self, + *, + base_model: str | list[str] | None = None, + datasets: str | list[str] | None = None, + eval_results: list[EvalResult] | None = None, + language: str | list[str] | None = None, + library_name: str | None = None, + license: str | None = None, + license_name: str | None = None, + license_link: str | None = None, + metrics: list[str] | None = None, + model_name: str | None = None, + pipeline_tag: str | None = None, + tags: list[str] | None = None, + ignore_metadata_errors: bool = False, + **kwargs, + ): + self.base_model = base_model + self.datasets = datasets + self.eval_results = eval_results + self.language = language + self.library_name = library_name + self.license = license + self.license_name = license_name + self.license_link = license_link + self.metrics = metrics + self.model_name = model_name + self.pipeline_tag = pipeline_tag + self.tags = _to_unique_list(tags) + + model_index = kwargs.pop("model-index", None) + if model_index: + try: + model_name, eval_results = model_index_to_eval_results(model_index) + self.model_name = model_name + self.eval_results = eval_results + except (KeyError, TypeError) as error: + if ignore_metadata_errors: + logger.warning("Invalid model-index. Not loading eval results into CardData.") + else: + raise ValueError( + f"Invalid `model_index` in metadata cannot be parsed: {error.__class__} {error}. Pass" + " `ignore_metadata_errors=True` to ignore this error while loading a Model Card. Warning:" + " some information will be lost. Use it at your own risk." + ) + + super().__init__(**kwargs) + + if self.eval_results: + try: + self.eval_results = _validate_eval_results(self.eval_results, self.model_name) + except Exception as e: + if ignore_metadata_errors: + logger.warning(f"Failed to validate eval_results: {e}. Not loading eval results into CardData.") + else: + raise ValueError(f"Failed to validate eval_results: {e}") from e + + def _to_dict(self, data_dict): + """Format the internal data dict. In this case, we convert eval results to a valid model index""" + if self.eval_results is not None: + data_dict["model-index"] = eval_results_to_model_index(self.model_name, self.eval_results) # type: ignore + del data_dict["eval_results"], data_dict["model_name"] + + +class DatasetCardData(CardData): + """Dataset Card Metadata that is used by Hugging Face Hub when included at the top of your README.md + + Args: + language (`list[str]`, *optional*): + Language of dataset's data or metadata. It must be an ISO 639-1, 639-2 or + 639-3 code (two/three letters), or a special value like "code", "multilingual". + license (`Union[str, list[str]]`, *optional*): + License(s) of this dataset. Example: apache-2.0 or any license from + https://huggingface.co/docs/hub/repositories-licenses. + annotations_creators (`Union[str, list[str]]`, *optional*): + How the annotations for the dataset were created. + Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'no-annotation', 'other'. + language_creators (`Union[str, list[str]]`, *optional*): + How the text-based data in the dataset was created. + Options are: 'found', 'crowdsourced', 'expert-generated', 'machine-generated', 'other' + multilinguality (`Union[str, list[str]]`, *optional*): + Whether the dataset is multilingual. + Options are: 'monolingual', 'multilingual', 'translation', 'other'. + size_categories (`Union[str, list[str]]`, *optional*): + The number of examples in the dataset. Options are: 'n<1K', '1K1T', and 'other'. + source_datasets (`list[str]]`, *optional*): + Indicates whether the dataset is an original dataset or extended from another existing dataset. + Options are: 'original' and 'extended'. + task_categories (`Union[str, list[str]]`, *optional*): + What categories of task does the dataset support? + task_ids (`Union[str, list[str]]`, *optional*): + What specific tasks does the dataset support? + paperswithcode_id (`str`, *optional*): + ID of the dataset on PapersWithCode. + pretty_name (`str`, *optional*): + A more human-readable name for the dataset. (ex. "Cats vs. Dogs") + train_eval_index (`dict`, *optional*): + A dictionary that describes the necessary spec for doing evaluation on the Hub. + If not provided, it will be gathered from the 'train-eval-index' key of the kwargs. + config_names (`Union[str, list[str]]`, *optional*): + A list of the available dataset configs for the dataset. + """ + + def __init__( + self, + *, + language: str | list[str] | None = None, + license: str | list[str] | None = None, + annotations_creators: str | list[str] | None = None, + language_creators: str | list[str] | None = None, + multilinguality: str | list[str] | None = None, + size_categories: str | list[str] | None = None, + source_datasets: list[str] | None = None, + task_categories: str | list[str] | None = None, + task_ids: str | list[str] | None = None, + paperswithcode_id: str | None = None, + pretty_name: str | None = None, + train_eval_index: dict | None = None, + config_names: str | list[str] | None = None, + ignore_metadata_errors: bool = False, + **kwargs, + ): + self.annotations_creators = annotations_creators + self.language_creators = language_creators + self.language = language + self.license = license + self.multilinguality = multilinguality + self.size_categories = size_categories + self.source_datasets = source_datasets + self.task_categories = task_categories + self.task_ids = task_ids + self.paperswithcode_id = paperswithcode_id + self.pretty_name = pretty_name + self.config_names = config_names + + # TODO - maybe handle this similarly to EvalResult? + self.train_eval_index = train_eval_index or kwargs.pop("train-eval-index", None) + super().__init__(**kwargs) + + def _to_dict(self, data_dict): + data_dict["train-eval-index"] = data_dict.pop("train_eval_index") + + +class SpaceCardData(CardData): + """Space Card Metadata that is used by Hugging Face Hub when included at the top of your README.md + + To get an exhaustive reference of Spaces configuration, please visit https://huggingface.co/docs/hub/spaces-config-reference#spaces-configuration-reference. + + Args: + title (`str`, *optional*) + Title of the Space. + sdk (`str`, *optional*) + SDK of the Space (one of `gradio`, `streamlit`, `docker`, or `static`). + sdk_version (`str`, *optional*) + Version of the used SDK (if Gradio/Streamlit sdk). + python_version (`str`, *optional*) + Python version used in the Space (if Gradio/Streamlit sdk). + app_file (`str`, *optional*) + Path to your main application file (which contains either gradio or streamlit Python code, or static html code). + Path is relative to the root of the repository. + app_port (`str`, *optional*) + Port on which your application is running. Used only if sdk is `docker`. + license (`str`, *optional*) + License of this model. Example: apache-2.0 or any license from + https://huggingface.co/docs/hub/repositories-licenses. + duplicated_from (`str`, *optional*) + ID of the original Space if this is a duplicated Space. + models (list[`str`], *optional*) + List of models related to this Space. Should be a dataset ID found on https://hf.co/models. + datasets (`list[str]`, *optional*) + List of datasets related to this Space. Should be a dataset ID found on https://hf.co/datasets. + tags (`list[str]`, *optional*) + List of tags to add to your Space that can be used when filtering on the Hub. + ignore_metadata_errors (`str`): + If True, errors while parsing the metadata section will be ignored. Some information might be lost during + the process. Use it at your own risk. + kwargs (`dict`, *optional*): + Additional metadata that will be added to the space card. + + Example: + ```python + >>> from huggingface_hub import SpaceCardData + >>> card_data = SpaceCardData( + ... title="Dreambooth Training", + ... license="mit", + ... sdk="gradio", + ... duplicated_from="multimodalart/dreambooth-training" + ... ) + >>> card_data.to_dict() + {'title': 'Dreambooth Training', 'sdk': 'gradio', 'license': 'mit', 'duplicated_from': 'multimodalart/dreambooth-training'} + ``` + """ + + def __init__( + self, + *, + title: str | None = None, + sdk: str | None = None, + sdk_version: str | None = None, + python_version: str | None = None, + app_file: str | None = None, + app_port: int | None = None, + license: str | None = None, + duplicated_from: str | None = None, + models: list[str] | None = None, + datasets: list[str] | None = None, + tags: list[str] | None = None, + ignore_metadata_errors: bool = False, + **kwargs, + ): + self.title = title + self.sdk = sdk + self.sdk_version = sdk_version + self.python_version = python_version + self.app_file = app_file + self.app_port = app_port + self.license = license + self.duplicated_from = duplicated_from + self.models = models + self.datasets = datasets + self.tags = _to_unique_list(tags) + super().__init__(**kwargs) + + +def model_index_to_eval_results(model_index: list[dict[str, Any]]) -> tuple[str, list[EvalResult]]: + """Takes in a model index and returns the model name and a list of `huggingface_hub.EvalResult` objects. + + A detailed spec of the model index can be found here: + https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 + + Args: + model_index (`list[dict[str, Any]]`): + A model index data structure, likely coming from a README.md file on the + Hugging Face Hub. + + Returns: + model_name (`str`): + The name of the model as found in the model index. This is used as the + identifier for the model on leaderboards like PapersWithCode. + eval_results (`list[EvalResult]`): + A list of `huggingface_hub.EvalResult` objects containing the metrics + reported in the provided model_index. + + Example: + ```python + >>> from huggingface_hub.repocard_data import model_index_to_eval_results + >>> # Define a minimal model index + >>> model_index = [ + ... { + ... "name": "my-cool-model", + ... "results": [ + ... { + ... "task": { + ... "type": "image-classification" + ... }, + ... "dataset": { + ... "type": "beans", + ... "name": "Beans" + ... }, + ... "metrics": [ + ... { + ... "type": "accuracy", + ... "value": 0.9 + ... } + ... ] + ... } + ... ] + ... } + ... ] + >>> model_name, eval_results = model_index_to_eval_results(model_index) + >>> model_name + 'my-cool-model' + >>> eval_results[0].task_type + 'image-classification' + >>> eval_results[0].metric_type + 'accuracy' + + ``` + """ + + eval_results = [] + for elem in model_index: + name = elem["name"] + results = elem["results"] + for result in results: + task_type = result["task"]["type"] + task_name = result["task"].get("name") + dataset_type = result["dataset"]["type"] + dataset_name = result["dataset"]["name"] + dataset_config = result["dataset"].get("config") + dataset_split = result["dataset"].get("split") + dataset_revision = result["dataset"].get("revision") + dataset_args = result["dataset"].get("args") + source_name = result.get("source", {}).get("name") + source_url = result.get("source", {}).get("url") + + for metric in result["metrics"]: + metric_type = metric["type"] + metric_value = metric["value"] + metric_name = metric.get("name") + metric_args = metric.get("args") + metric_config = metric.get("config") + verified = metric.get("verified") + verify_token = metric.get("verifyToken") + + eval_result = EvalResult( + task_type=task_type, # Required + dataset_type=dataset_type, # Required + dataset_name=dataset_name, # Required + metric_type=metric_type, # Required + metric_value=metric_value, # Required + task_name=task_name, + dataset_config=dataset_config, + dataset_split=dataset_split, + dataset_revision=dataset_revision, + dataset_args=dataset_args, + metric_name=metric_name, + metric_args=metric_args, + metric_config=metric_config, + verified=verified, + verify_token=verify_token, + source_name=source_name, + source_url=source_url, + ) + eval_results.append(eval_result) + return name, eval_results + + +def _remove_none(obj): + """ + Recursively remove `None` values from a dict. Borrowed from: https://stackoverflow.com/a/20558778 + """ + if isinstance(obj, (list, tuple, set)): + return type(obj)(_remove_none(x) for x in obj if x is not None) + elif isinstance(obj, dict): + return type(obj)((_remove_none(k), _remove_none(v)) for k, v in obj.items() if k is not None and v is not None) + else: + return obj + + +def eval_results_to_model_index(model_name: str, eval_results: list[EvalResult]) -> list[dict[str, Any]]: + """Takes in given model name and list of `huggingface_hub.EvalResult` and returns a + valid model-index that will be compatible with the format expected by the + Hugging Face Hub. + + Args: + model_name (`str`): + Name of the model (ex. "my-cool-model"). This is used as the identifier + for the model on leaderboards like PapersWithCode. + eval_results (`list[EvalResult]`): + List of `huggingface_hub.EvalResult` objects containing the metrics to be + reported in the model-index. + + Returns: + model_index (`list[dict[str, Any]]`): The eval_results converted to a model-index. + + Example: + ```python + >>> from huggingface_hub.repocard_data import eval_results_to_model_index, EvalResult + >>> # Define minimal eval_results + >>> eval_results = [ + ... EvalResult( + ... task_type="image-classification", # Required + ... dataset_type="beans", # Required + ... dataset_name="Beans", # Required + ... metric_type="accuracy", # Required + ... metric_value=0.9, # Required + ... ) + ... ] + >>> eval_results_to_model_index("my-cool-model", eval_results) + [{'name': 'my-cool-model', 'results': [{'task': {'type': 'image-classification'}, 'dataset': {'name': 'Beans', 'type': 'beans'}, 'metrics': [{'type': 'accuracy', 'value': 0.9}]}]}] + + ``` + """ + + # Metrics are reported on a unique task-and-dataset basis. + # Here, we make a map of those pairs and the associated EvalResults. + task_and_ds_types_map: dict[Any, list[EvalResult]] = defaultdict(list) + for eval_result in eval_results: + task_and_ds_types_map[eval_result.unique_identifier].append(eval_result) + + # Use the map from above to generate the model index data. + model_index_data: list[dict[str, Any]] = [] + for results in task_and_ds_types_map.values(): + # All items from `results` share same metadata + sample_result = results[0] + data: dict[str, Any] = { + "task": { + "type": sample_result.task_type, + "name": sample_result.task_name, + }, + "dataset": { + "name": sample_result.dataset_name, + "type": sample_result.dataset_type, + "config": sample_result.dataset_config, + "split": sample_result.dataset_split, + "revision": sample_result.dataset_revision, + "args": sample_result.dataset_args, + }, + "metrics": [ + { + "type": result.metric_type, + "value": result.metric_value, + "name": result.metric_name, + "config": result.metric_config, + "args": result.metric_args, + "verified": result.verified, + "verifyToken": result.verify_token, + } + for result in results + ], + } + if sample_result.source_url is not None: + source: dict[str, str] = { + "url": sample_result.source_url, + } + if sample_result.source_name is not None: + source["name"] = sample_result.source_name + data["source"] = source + model_index_data.append(data) + + # TODO - Check if there cases where this list is longer than one? + # Finally, the model index itself is list of dicts. + model_index = [ + { + "name": model_name, + "results": model_index_data, + } + ] + return _remove_none(model_index) + + +def _to_unique_list(tags: list[str] | None) -> list[str] | None: + if tags is None: + return tags + unique_tags = [] # make tags unique + keep order explicitly + for tag in tags: + if tag not in unique_tags: + unique_tags.append(tag) + return unique_tags diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__init__.py b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6e624a754176079eb63540513297099025a0685c --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ruff: noqa: F401 +"""Contains helpers to serialize tensors.""" + +from ._base import StateDictSplit, split_state_dict_into_shards_factory +from ._torch import ( + get_torch_storage_id, + get_torch_storage_size, + load_state_dict_from_file, + load_torch_model, + save_torch_model, + save_torch_state_dict, + split_torch_state_dict_into_shards, +) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..86f322ceb3020e775bc16cbd816907aa5354a38d Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/_base.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/_base.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..22cd7c9c0f64dbe2cd5960f8e68bd04c11fcf52a Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/_base.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/_dduf.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/_dduf.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..22751e4564cc0a357fe9081dafb0a50c2957ceff Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/_dduf.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/_torch.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/_torch.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..072507b496d2e4415f2524344043bd3ffe598654 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/__pycache__/_torch.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/serialization/_base.py b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/_base.py new file mode 100644 index 0000000000000000000000000000000000000000..f58bcb1f4ed57706be7d62bd39729d50dc98a5c1 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/_base.py @@ -0,0 +1,212 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains helpers to split tensors into shards.""" + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any, TypeVar + +from .. import logging + + +TensorT = TypeVar("TensorT") +TensorSizeFn_T = Callable[[TensorT], int] +StorageIDFn_T = Callable[[TensorT], Any | None] + +MAX_SHARD_SIZE = "5GB" +SIZE_UNITS = { + "TB": 10**12, + "GB": 10**9, + "MB": 10**6, + "KB": 10**3, +} + + +logger = logging.get_logger(__file__) + + +@dataclass +class StateDictSplit: + is_sharded: bool = field(init=False) + metadata: dict[str, Any] + filename_to_tensors: dict[str, list[str]] + tensor_to_filename: dict[str, str] + + def __post_init__(self): + self.is_sharded = len(self.filename_to_tensors) > 1 + + +def split_state_dict_into_shards_factory( + state_dict: dict[str, TensorT], + *, + get_storage_size: TensorSizeFn_T, + filename_pattern: str, + get_storage_id: StorageIDFn_T = lambda tensor: None, + max_shard_size: int | str = MAX_SHARD_SIZE, +) -> StateDictSplit: + """ + Split a model state dictionary in shards so that each shard is smaller than a given size. + + The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization + made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we + have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not + [6+2+2GB], [6+2GB], [6GB]. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + Args: + state_dict (`dict[str, Tensor]`): + The state dictionary to save. + get_storage_size (`Callable[[Tensor], int]`): + A function that returns the size of a tensor when saved on disk in bytes. + get_storage_id (`Callable[[Tensor], Optional[Any]]`, *optional*): + A function that returns a unique identifier to a tensor storage. Multiple different tensors can share the + same underlying storage. This identifier is guaranteed to be unique and constant for this tensor's storage + during its lifetime. Two tensor storages with non-overlapping lifetimes may have the same id. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + + Returns: + [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them. + """ + storage_id_to_tensors: dict[Any, list[str]] = {} + + shard_list: list[dict[str, TensorT]] = [] + current_shard: dict[str, TensorT] = {} + current_shard_size = 0 + total_size = 0 + + if isinstance(max_shard_size, str): + max_shard_size = parse_size_to_int(max_shard_size) + + for key, tensor in state_dict.items(): + # when bnb serialization is used the weights in the state dict can be strings + # check: https://github.com/huggingface/transformers/pull/24416 for more details + if isinstance(tensor, str): + logger.info("Skipping tensor %s as it is a string (bnb serialization)", key) + continue + + # If a `tensor` shares the same underlying storage as another tensor, we put `tensor` in the same `block` + storage_id = get_storage_id(tensor) # type: ignore[invalid-argument-type] + if storage_id is not None: + if storage_id in storage_id_to_tensors: + # We skip this tensor for now and will reassign to correct shard later + storage_id_to_tensors[storage_id].append(key) + continue + else: + # This is the first tensor with this storage_id, we create a new entry + # in the storage_id_to_tensors dict => we will assign the shard id later + storage_id_to_tensors[storage_id] = [key] + + # Compute tensor size + tensor_size = get_storage_size(tensor) # type: ignore[invalid-argument-type] + + # If this tensor is bigger than the maximal size, we put it in its own shard + if tensor_size > max_shard_size: + total_size += tensor_size + shard_list.append({key: tensor}) + continue + + # If this tensor is going to tip up over the maximal size, we split. + # Current shard already has some tensors, we add it to the list of shards and create a new one. + if current_shard_size + tensor_size > max_shard_size: + shard_list.append(current_shard) + current_shard = {} + current_shard_size = 0 + + # Add the tensor to the current shard + current_shard[key] = tensor + current_shard_size += tensor_size + total_size += tensor_size + + # Add the last shard + if len(current_shard) > 0: + shard_list.append(current_shard) + nb_shards = len(shard_list) + + # Loop over the tensors that share the same storage and assign them together + for storage_id, keys in storage_id_to_tensors.items(): + # Let's try to find the shard where the first tensor of this storage is and put all tensors in the same shard + for shard in shard_list: + if keys[0] in shard: + for key in keys: + shard[key] = state_dict[key] + break + + # If we only have one shard, we return it => no need to build the index + if nb_shards == 1: + filename = filename_pattern.format(suffix="") + # Use the keys from the shard itself rather than `state_dict` directly, so that tensors + # skipped above (e.g. string tensors from bnb serialization) are excluded from the index, + # consistently with the multi-shard path below. + keys = list(shard_list[0].keys()) + return StateDictSplit( + metadata={"total_size": total_size}, + filename_to_tensors={filename: keys}, + tensor_to_filename={key: filename for key in keys}, + ) + + # Now that each tensor is assigned to a shard, let's assign a filename to each shard + tensor_name_to_filename = {} + filename_to_tensors = {} + for idx, shard in enumerate(shard_list): + filename = filename_pattern.format(suffix=f"-{idx + 1:05d}-of-{nb_shards:05d}") + for key in shard: + tensor_name_to_filename[key] = filename + filename_to_tensors[filename] = list(shard.keys()) + + # Build the index and return + return StateDictSplit( + metadata={"total_size": total_size}, + filename_to_tensors=filename_to_tensors, + tensor_to_filename=tensor_name_to_filename, + ) + + +def parse_size_to_int(size_as_str: str) -> int: + """ + Parse a size expressed as a string with digits and unit (like `"5MB"`) to an integer (in bytes). + + Supported units are "TB", "GB", "MB", "KB". + + Args: + size_as_str (`str`): The size to convert. Will be directly returned if an `int`. + + Example: + + ```py + >>> parse_size_to_int("5MB") + 5000000 + ``` + """ + size_as_str = size_as_str.strip() + + # Parse unit + unit = size_as_str[-2:].upper() + if unit not in SIZE_UNITS: + raise ValueError(f"Unit '{unit}' not supported. Supported units are TB, GB, MB, KB. Got '{size_as_str}'.") + multiplier = SIZE_UNITS[unit] + + # Parse value + try: + value = float(size_as_str[:-2].strip()) + except ValueError as e: + raise ValueError(f"Could not parse the size value from '{size_as_str}': {e}") from e + + return int(value * multiplier) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/serialization/_dduf.py b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/_dduf.py new file mode 100644 index 0000000000000000000000000000000000000000..37e916d419d5a5934525bcb169fd5dae48d6389d --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/_dduf.py @@ -0,0 +1,386 @@ +import json +import logging +import mmap +import os +import shutil +import zipfile +from collections.abc import Generator, Iterable +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from ..errors import DDUFCorruptedFileError, DDUFExportError, DDUFInvalidEntryNameError + + +logger = logging.getLogger(__name__) + +DDUF_ALLOWED_ENTRIES = { + # Allowed file extensions in a DDUF file + ".json", + ".model", + ".safetensors", + ".txt", +} + +DDUF_FOLDER_REQUIRED_ENTRIES = { + # Each folder must contain at least one of these entries + "config.json", + "tokenizer_config.json", + "preprocessor_config.json", + "scheduler_config.json", +} + + +@dataclass +class DDUFEntry: + """Object representing a file entry in a DDUF file. + + See [`read_dduf_file`] for how to read a DDUF file. + + Attributes: + filename (str): + The name of the file in the DDUF archive. + offset (int): + The offset of the file in the DDUF archive. + length (int): + The length of the file in the DDUF archive. + dduf_path (str): + The path to the DDUF archive (for internal use). + """ + + filename: str + length: int + offset: int + + dduf_path: Path = field(repr=False) + + @contextmanager + def as_mmap(self) -> Generator[bytes, None, None]: + """Open the file as a memory-mapped file. + + Useful to load safetensors directly from the file. + + Example: + ```py + >>> import safetensors.torch + >>> with entry.as_mmap() as mm: + ... tensors = safetensors.torch.load(mm) + ``` + """ + with self.dduf_path.open("rb") as f: + with mmap.mmap(f.fileno(), length=0, access=mmap.ACCESS_READ) as mm: + yield mm[self.offset : self.offset + self.length] + + def read_text(self, encoding: str = "utf-8") -> str: + """Read the file as text. + + Useful for '.txt' and '.json' entries. + + Example: + ```py + >>> import json + >>> index = json.loads(entry.read_text()) + ``` + """ + with self.dduf_path.open("rb") as f: + f.seek(self.offset) + return f.read(self.length).decode(encoding=encoding) + + +def read_dduf_file(dduf_path: os.PathLike | str) -> dict[str, DDUFEntry]: + """ + Read a DDUF file and return a dictionary of entries. + + Only the metadata is read, the data is not loaded in memory. + + Args: + dduf_path (`str` or `os.PathLike`): + The path to the DDUF file to read. + + Returns: + `dict[str, DDUFEntry]`: + A dictionary of [`DDUFEntry`] indexed by filename. + + Raises: + - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format). + + Example: + ```python + >>> import json + >>> import safetensors.torch + >>> from huggingface_hub import read_dduf_file + + # Read DDUF metadata + >>> dduf_entries = read_dduf_file("FLUX.1-dev.dduf") + + # Returns a mapping filename <> DDUFEntry + >>> dduf_entries["model_index.json"] + DDUFEntry(filename='model_index.json', offset=66, length=587) + + # Load model index as JSON + >>> json.loads(dduf_entries["model_index.json"].read_text()) + {'_class_name': 'FluxPipeline', '_diffusers_version': '0.32.0.dev0', '_name_or_path': 'black-forest-labs/FLUX.1-dev', ... + + # Load VAE weights using safetensors + >>> with dduf_entries["vae/diffusion_pytorch_model.safetensors"].as_mmap() as mm: + ... state_dict = safetensors.torch.load(mm) + ``` + """ + entries = {} + dduf_path = Path(dduf_path) + logger.info(f"Reading DDUF file {dduf_path}") + with zipfile.ZipFile(str(dduf_path), "r") as zf: + for info in zf.infolist(): + logger.debug(f"Reading entry {info.filename}") + if info.compress_type != zipfile.ZIP_STORED: + raise DDUFCorruptedFileError("Data must not be compressed in DDUF file.") + + try: + _validate_dduf_entry_name(info.filename) + except DDUFInvalidEntryNameError as e: + raise DDUFCorruptedFileError(f"Invalid entry name in DDUF file: {info.filename}") from e + + offset = _get_data_offset(zf, info) + + entries[info.filename] = DDUFEntry( + filename=info.filename, offset=offset, length=info.file_size, dduf_path=dduf_path + ) + + # Consistency checks on the DDUF file + if "model_index.json" not in entries: + raise DDUFCorruptedFileError("Missing required 'model_index.json' entry in DDUF file.") + index = json.loads(entries["model_index.json"].read_text()) + _validate_dduf_structure(index, entries.keys()) + + logger.info(f"Done reading DDUF file {dduf_path}. Found {len(entries)} entries") + return entries + + +def export_entries_as_dduf(dduf_path: str | os.PathLike, entries: Iterable[tuple[str, str | Path | bytes]]) -> None: + """Write a DDUF file from an iterable of entries. + + This is a lower-level helper than [`export_folder_as_dduf`] that allows more flexibility when serializing data. + In particular, you don't need to save the data on disk before exporting it in the DDUF file. + + Args: + dduf_path (`str` or `os.PathLike`): + The path to the DDUF file to write. + entries (`Iterable[tuple[str, Union[str, Path, bytes]]]`): + An iterable of entries to write in the DDUF file. Each entry is a tuple with the filename and the content. + The filename should be the path to the file in the DDUF archive. + The content can be a string or a pathlib.Path representing a path to a file on the local disk or directly the content as bytes. + + Raises: + - [`DDUFExportError`]: If anything goes wrong during the export (e.g. invalid entry name, missing 'model_index.json', etc.). + + Example: + ```python + # Export specific files from the local disk. + >>> from huggingface_hub import export_entries_as_dduf + >>> export_entries_as_dduf( + ... dduf_path="stable-diffusion-v1-4-FP16.dduf", + ... entries=[ # List entries to add to the DDUF file (here, only FP16 weights) + ... ("model_index.json", "path/to/model_index.json"), + ... ("vae/config.json", "path/to/vae/config.json"), + ... ("vae/diffusion_pytorch_model.fp16.safetensors", "path/to/vae/diffusion_pytorch_model.fp16.safetensors"), + ... ("text_encoder/config.json", "path/to/text_encoder/config.json"), + ... ("text_encoder/model.fp16.safetensors", "path/to/text_encoder/model.fp16.safetensors"), + ... # ... add more entries here + ... ] + ... ) + ``` + + ```python + # Export state_dicts one by one from a loaded pipeline + >>> from diffusers import DiffusionPipeline + >>> from typing import Generator, Tuple + >>> import safetensors.torch + >>> from huggingface_hub import export_entries_as_dduf + >>> pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + ... # ... do some work with the pipeline + + >>> def as_entries(pipe: DiffusionPipeline) -> Generator[tuple[str, bytes], None, None]: + ... # Build a generator that yields the entries to add to the DDUF file. + ... # The first element of the tuple is the filename in the DDUF archive (must use UNIX separator!). The second element is the content of the file. + ... # Entries will be evaluated lazily when the DDUF file is created (only 1 entry is loaded in memory at a time) + ... yield "vae/config.json", pipe.vae.to_json_string().encode() + ... yield "vae/diffusion_pytorch_model.safetensors", safetensors.torch.save(pipe.vae.state_dict()) + ... yield "text_encoder/config.json", pipe.text_encoder.config.to_json_string().encode() + ... yield "text_encoder/model.safetensors", safetensors.torch.save(pipe.text_encoder.state_dict()) + ... # ... add more entries here + + >>> export_entries_as_dduf(dduf_path="stable-diffusion-v1-4.dduf", entries=as_entries(pipe)) + ``` + """ + logger.info(f"Exporting DDUF file '{dduf_path}'") + filenames = set() + index = None + with zipfile.ZipFile(str(dduf_path), "w", zipfile.ZIP_STORED) as archive: + for filename, content in entries: + if filename in filenames: + raise DDUFExportError(f"Can't add duplicate entry: {filename}") + filenames.add(filename) + + if filename == "model_index.json": + try: + index = json.loads(_load_content(content).decode()) + except json.JSONDecodeError as e: + raise DDUFExportError("Failed to parse 'model_index.json'.") from e + + try: + filename = _validate_dduf_entry_name(filename) + except DDUFInvalidEntryNameError as e: + raise DDUFExportError(f"Invalid entry name: {filename}") from e + logger.debug(f"Adding entry '{filename}' to DDUF file") + _dump_content_in_archive(archive, filename, content) + + # Consistency checks on the DDUF file + if index is None: + raise DDUFExportError("Missing required 'model_index.json' entry in DDUF file.") + try: + _validate_dduf_structure(index, filenames) + except DDUFCorruptedFileError as e: + raise DDUFExportError("Invalid DDUF file structure.") from e + + logger.info(f"Done writing DDUF file {dduf_path}") + + +def export_folder_as_dduf(dduf_path: str | os.PathLike, folder_path: str | os.PathLike) -> None: + """ + Export a folder as a DDUF file. + + AUses [`export_entries_as_dduf`] under the hood. + + Args: + dduf_path (`str` or `os.PathLike`): + The path to the DDUF file to write. + folder_path (`str` or `os.PathLike`): + The path to the folder containing the diffusion model. + + Example: + ```python + >>> from huggingface_hub import export_folder_as_dduf + >>> export_folder_as_dduf(dduf_path="FLUX.1-dev.dduf", folder_path="path/to/FLUX.1-dev") + ``` + """ + folder_path = Path(folder_path) + + def _iterate_over_folder() -> Iterable[tuple[str, Path]]: + for path in Path(folder_path).glob("**/*"): + if not path.is_file(): + continue + if path.suffix not in DDUF_ALLOWED_ENTRIES: + logger.debug(f"Skipping file '{path}' (file type not allowed)") + continue + path_in_archive = path.relative_to(folder_path) + if len(path_in_archive.parts) >= 3: + logger.debug(f"Skipping file '{path}' (nested directories not allowed)") + continue + yield path_in_archive.as_posix(), path + + export_entries_as_dduf(dduf_path, _iterate_over_folder()) + + +def _dump_content_in_archive(archive: zipfile.ZipFile, filename: str, content: str | os.PathLike | bytes) -> None: + with archive.open(filename, "w", force_zip64=True) as archive_fh: + if isinstance(content, (str, Path)): + content_path = Path(content) + with content_path.open("rb") as content_fh: + shutil.copyfileobj(content_fh, archive_fh, 1024 * 1024 * 8) # type: ignore[misc] + elif isinstance(content, bytes): + archive_fh.write(content) + else: + raise DDUFExportError(f"Invalid content type for {filename}. Must be str, Path or bytes.") + + +def _load_content(content: str | Path | bytes) -> bytes: + """Load the content of an entry as bytes. + + Used only for small checks (not to dump content into archive). + """ + if isinstance(content, (str, Path)): + return Path(content).read_bytes() + elif isinstance(content, bytes): + return content + else: + raise DDUFExportError(f"Invalid content type. Must be str, Path or bytes. Got {type(content)}.") + + +def _validate_dduf_entry_name(entry_name: str) -> str: + if "." + entry_name.split(".")[-1] not in DDUF_ALLOWED_ENTRIES: + raise DDUFInvalidEntryNameError(f"File type not allowed: {entry_name}") + if "\\" in entry_name: + raise DDUFInvalidEntryNameError(f"Entry names must use UNIX separators ('/'). Got {entry_name}.") + entry_name = entry_name.strip("/") + if entry_name.count("/") > 1: + raise DDUFInvalidEntryNameError(f"DDUF only supports 1 level of directory. Got {entry_name}.") + return entry_name + + +def _validate_dduf_structure(index: Any, entry_names: Iterable[str]) -> None: + """ + Consistency checks on the DDUF file structure. + + Rules: + - The 'model_index.json' entry is required and must contain a dictionary. + - Each folder name must correspond to an entry in 'model_index.json'. + - Each folder must contain at least a config file ('config.json', 'tokenizer_config.json', 'preprocessor_config.json', 'scheduler_config.json'). + + Args: + index (Any): + The content of the 'model_index.json' entry. + entry_names (Iterable[str]): + The list of entry names in the DDUF file. + + Raises: + - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format). + """ + if not isinstance(index, dict): + raise DDUFCorruptedFileError(f"Invalid 'model_index.json' content. Must be a dictionary. Got {type(index)}.") + + dduf_folders = {entry.split("/")[0] for entry in entry_names if "/" in entry} + for folder in dduf_folders: + if folder not in index: + raise DDUFCorruptedFileError(f"Missing required entry '{folder}' in 'model_index.json'.") + if not any(f"{folder}/{required_entry}" in entry_names for required_entry in DDUF_FOLDER_REQUIRED_ENTRIES): + raise DDUFCorruptedFileError( + f"Missing required file in folder '{folder}'. Must contains at least one of {DDUF_FOLDER_REQUIRED_ENTRIES}." + ) + + +def _get_data_offset(zf: zipfile.ZipFile, info: zipfile.ZipInfo) -> int: + """ + Calculate the data offset for a file in a ZIP archive. + + Args: + zf (`zipfile.ZipFile`): + The opened ZIP file. Must be opened in read mode. + info (`zipfile.ZipInfo`): + The file info. + + Returns: + int: The offset of the file data in the ZIP archive. + """ + if zf.fp is None: + raise DDUFCorruptedFileError("ZipFile object must be opened in read mode.") + + # Step 1: Get the local file header offset + header_offset = info.header_offset + + # Step 2: Read the local file header + zf.fp.seek(header_offset) + local_file_header = zf.fp.read(30) # Fixed-size part of the local header + + if len(local_file_header) < 30: + raise DDUFCorruptedFileError("Incomplete local file header.") + + # Step 3: Parse the header fields to calculate the start of file data + # Local file header: https://en.wikipedia.org/wiki/ZIP_(file_format)#File_headers + filename_len = int.from_bytes(local_file_header[26:28], "little") + extra_field_len = int.from_bytes(local_file_header[28:30], "little") + + # Data offset is after the fixed header, filename, and extra fields + data_offset = header_offset + 30 + filename_len + extra_field_len + + return data_offset diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/serialization/_torch.py b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/_torch.py new file mode 100644 index 0000000000000000000000000000000000000000..19516aa7feea7c206391dd3b9deecf6157b7e455 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/serialization/_torch.py @@ -0,0 +1,1046 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains pytorch-specific helpers.""" + +import importlib +import importlib.util +import json +import os +import re +from collections import defaultdict, namedtuple +from collections.abc import Iterable +from functools import lru_cache +from pathlib import Path, PureWindowsPath +from typing import TYPE_CHECKING, Any, NamedTuple, Union + +from packaging import version + +from .. import constants, logging +from ._base import MAX_SHARD_SIZE, StateDictSplit, split_state_dict_into_shards_factory + + +logger = logging.get_logger(__file__) + +if TYPE_CHECKING: + import torch + +# SAVING + + +def save_torch_model( + model: "torch.nn.Module", + save_directory: str | Path, + *, + filename_pattern: str | None = None, + force_contiguous: bool = True, + max_shard_size: int | str = MAX_SHARD_SIZE, + metadata: dict[str, str] | None = None, + safe_serialization: bool = True, + is_main_process: bool = True, + shared_tensors_to_discard: list[str] | None = None, +): + """ + Saves a given torch model to disk, handling sharding and shared tensors issues. + + See also [`save_torch_state_dict`] to save a state dict with more flexibility. + + For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors). + + The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are + saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard, + an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses + [`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as + safetensors (the default). Otherwise, the shards are saved as pickle. + + Before saving the model, the `save_directory` is cleaned from any previous shard files. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + > [!WARNING] + > If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving. + + Args: + model (`torch.nn.Module`): + The model to save on disk. + save_directory (`str` or `Path`): + The directory in which the model will be saved. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization` + parameter. + force_contiguous (`boolean`, *optional*): + Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the + model, but it could potentially change performance if the layout of the tensor was chosen specifically for + that reason. Defaults to `True`. + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + metadata (`dict[str, str]`, *optional*): + Extra information to save along with the model. Some metadata will be added for each dropped tensors. + This information will not be enough to recover the entire shared structure but might help understanding + things. + safe_serialization (`bool`, *optional*): + Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle. + Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed + in a future version. + is_main_process (`bool`, *optional*): + Whether the process calling this is the main process or not. Useful when in distributed training like + TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on + the main process to avoid race conditions. Defaults to True. + shared_tensors_to_discard (`list[str]`, *optional*): + List of tensor names to drop when saving shared tensors. If not provided and shared tensors are + detected, it will drop the first name alphabetically. + + Example: + + ```py + >>> from huggingface_hub import save_torch_model + >>> model = ... # A PyTorch model + + # Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors. + >>> save_torch_model(model, "path/to/folder") + + # Load model back + >>> from huggingface_hub import load_torch_model # TODO + >>> load_torch_model(model, "path/to/folder") + >>> + ``` + """ + save_torch_state_dict( + state_dict=model.state_dict(), + filename_pattern=filename_pattern, + force_contiguous=force_contiguous, + max_shard_size=max_shard_size, + metadata=metadata, + safe_serialization=safe_serialization, + save_directory=save_directory, + is_main_process=is_main_process, + shared_tensors_to_discard=shared_tensors_to_discard, + ) + + +def save_torch_state_dict( + state_dict: dict[str, "torch.Tensor"], + save_directory: str | Path, + *, + filename_pattern: str | None = None, + force_contiguous: bool = True, + max_shard_size: int | str = MAX_SHARD_SIZE, + metadata: dict[str, str] | None = None, + safe_serialization: bool = True, + is_main_process: bool = True, + shared_tensors_to_discard: list[str] | None = None, +) -> None: + """ + Save a model state dictionary to the disk, handling sharding and shared tensors issues. + + See also [`save_torch_model`] to directly save a PyTorch model. + + For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors). + + The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are + saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard, + an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses + [`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as + safetensors (the default). Otherwise, the shards are saved as pickle. + + Before saving the model, the `save_directory` is cleaned from any previous shard files. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + > [!WARNING] + > If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving. + + Args: + state_dict (`dict[str, torch.Tensor]`): + The state dictionary to save. + save_directory (`str` or `Path`): + The directory in which the model will be saved. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization` + parameter. + force_contiguous (`boolean`, *optional*): + Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the + model, but it could potentially change performance if the layout of the tensor was chosen specifically for + that reason. Defaults to `True`. + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + metadata (`dict[str, str]`, *optional*): + Extra information to save along with the model. Some metadata will be added for each dropped tensors. + This information will not be enough to recover the entire shared structure but might help understanding + things. + safe_serialization (`bool`, *optional*): + Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle. + Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed + in a future version. + is_main_process (`bool`, *optional*): + Whether the process calling this is the main process or not. Useful when in distributed training like + TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on + the main process to avoid race conditions. Defaults to True. + shared_tensors_to_discard (`list[str]`, *optional*): + List of tensor names to drop when saving shared tensors. If not provided and shared tensors are + detected, it will drop the first name alphabetically. + + Example: + + ```py + >>> from huggingface_hub import save_torch_state_dict + >>> model = ... # A PyTorch model + + # Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors. + >>> state_dict = model_to_save.state_dict() + >>> save_torch_state_dict(state_dict, "path/to/folder") + ``` + """ + save_directory = str(save_directory) + + if filename_pattern is None: + filename_pattern = ( + constants.SAFETENSORS_WEIGHTS_FILE_PATTERN + if safe_serialization + else constants.PYTORCH_WEIGHTS_FILE_PATTERN + ) + + if metadata is None: + metadata = {} + if safe_serialization: + try: + from safetensors.torch import save_file as save_file_fn + except ImportError as e: + raise ImportError( + "Please install `safetensors` to use safe serialization. " + "You can install it with `pip install safetensors`." + ) from e + # Clean state dict for safetensors + state_dict = _clean_state_dict_for_safetensors( + state_dict, + metadata, + force_contiguous=force_contiguous, + shared_tensors_to_discard=shared_tensors_to_discard, + ) + else: + from torch import save as save_file_fn # type: ignore[assignment, no-redef] + + logger.warning( + "You are using unsafe serialization. Due to security reasons, it is recommended not to load " + "pickled models from untrusted sources. If you intend to share your model, we strongly recommend " + "using safe serialization by installing `safetensors` with `pip install safetensors`." + ) + # Split dict + state_dict_split = split_torch_state_dict_into_shards( + state_dict, filename_pattern=filename_pattern, max_shard_size=max_shard_size + ) + + # Only main process should clean up existing files to avoid race conditions in distributed environment + if is_main_process: + existing_files_regex = re.compile(filename_pattern.format(suffix=r"(-\d{5}-of-\d{5})?") + r"(\.index\.json)?") + for filename in os.listdir(save_directory): + if existing_files_regex.match(filename): + try: + logger.debug(f"Removing existing file '{filename}' from folder.") + os.remove(os.path.join(save_directory, filename)) + except Exception as e: + logger.warning( + f"Error when trying to remove existing '{filename}' from folder: {e}. Continuing..." + ) + + # Save each shard + per_file_metadata = {"format": "pt"} + if not state_dict_split.is_sharded: + per_file_metadata.update(metadata) + safe_file_kwargs = {"metadata": per_file_metadata} if safe_serialization else {} + for filename, tensors in state_dict_split.filename_to_tensors.items(): + shard = {tensor: state_dict[tensor] for tensor in tensors} + save_file_fn(shard, os.path.join(save_directory, filename), **safe_file_kwargs) # ty: ignore[invalid-argument-type] + logger.debug(f"Shard saved to {filename}") + + # Save the index (if any) + if state_dict_split.is_sharded: + index_path = filename_pattern.format(suffix="") + ".index.json" + index = { + "metadata": {**state_dict_split.metadata, **metadata}, + "weight_map": state_dict_split.tensor_to_filename, + } + with open(os.path.join(save_directory, index_path), "w") as f: + json.dump(index, f, indent=2) + logger.info( + f"The model is bigger than the maximum size per checkpoint ({max_shard_size}). " + f"Model weighs have been saved in {len(state_dict_split.filename_to_tensors)} checkpoint shards. " + f"You can find where each parameters has been saved in the index located at {index_path}." + ) + + logger.info(f"Model weights successfully saved to {save_directory}!") + + +def split_torch_state_dict_into_shards( + state_dict: dict[str, "torch.Tensor"], + *, + filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN, + max_shard_size: int | str = MAX_SHARD_SIZE, +) -> StateDictSplit: + """ + Split a model state dictionary in shards so that each shard is smaller than a given size. + + The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization + made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we + have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not + [6+2+2GB], [6+2GB], [6GB]. + + + > [!TIP] + > To save a model state dictionary to the disk, see [`save_torch_state_dict`]. This helper uses + > `split_torch_state_dict_into_shards` under the hood. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + Args: + state_dict (`dict[str, torch.Tensor]`): + The state dictionary to save. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"`. + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + + Returns: + [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them. + + Example: + ```py + >>> import json + >>> import os + >>> from safetensors.torch import save_file as safe_save_file + >>> from huggingface_hub import split_torch_state_dict_into_shards + + >>> def save_state_dict(state_dict: dict[str, torch.Tensor], save_directory: str): + ... state_dict_split = split_torch_state_dict_into_shards(state_dict) + ... for filename, tensors in state_dict_split.filename_to_tensors.items(): + ... shard = {tensor: state_dict[tensor] for tensor in tensors} + ... safe_save_file( + ... shard, + ... os.path.join(save_directory, filename), + ... metadata={"format": "pt"}, + ... ) + ... if state_dict_split.is_sharded: + ... index = { + ... "metadata": state_dict_split.metadata, + ... "weight_map": state_dict_split.tensor_to_filename, + ... } + ... with open(os.path.join(save_directory, "model.safetensors.index.json"), "w") as f: + ... f.write(json.dumps(index, indent=2)) + ``` + """ + return split_state_dict_into_shards_factory( + state_dict, + max_shard_size=max_shard_size, + filename_pattern=filename_pattern, + get_storage_size=get_torch_storage_size, + get_storage_id=get_torch_storage_id, + ) + + +# LOADING + + +def load_torch_model( + model: "torch.nn.Module", + checkpoint_path: str | os.PathLike, + *, + strict: bool = False, + safe: bool = True, + weights_only: bool = False, + map_location: Union[str, "torch.device"] | None = None, + mmap: bool = False, + filename_pattern: str | None = None, +) -> NamedTuple: + """ + Load a checkpoint into a model, handling both sharded and non-sharded checkpoints. + + Args: + model (`torch.nn.Module`): + The model in which to load the checkpoint. + checkpoint_path (`str` or `os.PathLike`): + Path to either the checkpoint file or directory containing the checkpoint(s). + strict (`bool`, *optional*, defaults to `False`): + Whether to strictly enforce that the keys in the model state dict match the keys in the checkpoint. + safe (`bool`, *optional*, defaults to `True`): + If `safe` is True, the safetensors files will be loaded. If `safe` is False, the function + will first attempt to load safetensors files if they are available, otherwise it will fall back to loading + pickle files. `filename_pattern` parameter takes precedence over `safe` parameter. + weights_only (`bool`, *optional*, defaults to `False`): + If True, only loads the model weights without optimizer states and other metadata. + Only supported in PyTorch >= 1.13. + map_location (`str` or `torch.device`, *optional*): + A `torch.device` object, string or a dict specifying how to remap storage locations. It + indicates the location where all tensors should be loaded. + mmap (`bool`, *optional*, defaults to `False`): + Whether to use memory-mapped file loading. Memory mapping can improve loading performance + for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints. + filename_pattern (`str`, *optional*): + The pattern to look for the index file. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"`. + Returns: + `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields. + - `missing_keys` is a list of str containing the missing keys, i.e. keys that are in the model but not in the checkpoint. + - `unexpected_keys` is a list of str containing the unexpected keys, i.e. keys that are in the checkpoint but not in the model. + + Raises: + [`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError) + If the checkpoint file or directory does not exist. + [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) + If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the checkpoint path is invalid or if the checkpoint format cannot be determined. + + Example: + ```python + >>> from huggingface_hub import load_torch_model + >>> model = ... # A PyTorch model + >>> load_torch_model(model, "path/to/checkpoint") + ``` + """ + checkpoint_path = Path(checkpoint_path) + + if not checkpoint_path.exists(): + raise ValueError(f"Checkpoint path {checkpoint_path} does not exist") + # 1. Check if checkpoint is a single file + if checkpoint_path.is_file(): + state_dict = load_state_dict_from_file( + checkpoint_file=checkpoint_path, + map_location=map_location, + weights_only=weights_only, + ) + return model.load_state_dict(state_dict, strict=strict) + + # 2. If not, checkpoint_path is a directory + if filename_pattern is None: + filename_pattern = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN + index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json") + # Only fallback to pickle format if safetensors index is not found and safe is False. + if not index_path.is_file() and not safe: + filename_pattern = constants.PYTORCH_WEIGHTS_FILE_PATTERN + + index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json") + + if index_path.is_file(): + return _load_sharded_checkpoint( + model=model, + save_directory=checkpoint_path, + strict=strict, + weights_only=weights_only, + filename_pattern=filename_pattern, + ) + + # Look for single model file + model_files = list(checkpoint_path.glob("*.safetensors" if safe else "*.bin")) + if len(model_files) == 1: + state_dict = load_state_dict_from_file( + checkpoint_file=model_files[0], + map_location=map_location, + weights_only=weights_only, + mmap=mmap, + ) + return model.load_state_dict(state_dict, strict=strict) + + raise ValueError( + f"Directory '{checkpoint_path}' does not contain a valid checkpoint. " + "Expected either a sharded checkpoint with an index file, or a single model file." + ) + + +def _load_sharded_checkpoint( + model: "torch.nn.Module", + save_directory: os.PathLike, + *, + strict: bool = False, + weights_only: bool = False, + filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN, +) -> NamedTuple: + """ + Loads a sharded checkpoint into a model. This is the same as + [`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict) + but for a sharded checkpoint. Each shard is loaded one by one and removed from memory after being loaded into the model. + + Args: + model (`torch.nn.Module`): + The model in which to load the checkpoint. + save_directory (`str` or `os.PathLike`): + A path to a folder containing the sharded checkpoint. + strict (`bool`, *optional*, defaults to `False`): + Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint. + weights_only (`bool`, *optional*, defaults to `False`): + If True, only loads the model weights without optimizer states and other metadata. + Only supported in PyTorch >= 1.13. + filename_pattern (`str`, *optional*, defaults to `"model{suffix}.safetensors"`): + The pattern to look for the index file. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"`. + + Returns: + `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields, + - `missing_keys` is a list of str containing the missing keys + - `unexpected_keys` is a list of str containing the unexpected keys + """ + + # 1. Load and validate index file + # The index file contains mapping of parameter names to shard files + index_path = filename_pattern.format(suffix="") + ".index.json" + index_file = os.path.join(save_directory, index_path) + with open(index_file, encoding="utf-8") as f: + index = json.load(f) + + # 2. Validate shard filenames from the index + # This prevents path traversal attacks and extension confusion attacks + # (e.g. a safetensors index referencing .bin pickle files) + expected_extension = Path(filename_pattern.format(suffix="")).suffix # e.g. ".safetensors" + shard_files = list(set(index["weight_map"].values())) + for shard_file in shard_files: + # Reject anything that could escape `save_directory` on any host OS: + # POSIX absolute ("/tmp/x"), Windows drive ("C:x", "C:\\x"), UNC + # ("\\\\server\\share\\x"), rooted-without-drive ("\\x", "/x"), or + # ".." traversal — including "..\\x" which `os.path.isabs` never caught on POSIX. + # + # We parse with `PureWindowsPath` *regardless of host OS*: it treats both "/" and + # "\\" as separators and exposes `drive` / `root`, so a single check rejects a + # malicious index file on Linux too (e.g. if it's later opened on Windows). The + # only over-strict case is a POSIX filename like "a:foo" which would be parsed as + # drive "a:" — such names are never produced for safetensors shards and would + # break on Windows anyway, so rejecting them is fine. + win_path = PureWindowsPath(shard_file) + if win_path.drive or win_path.root or ".." in win_path.parts: + raise ValueError( + f"Invalid shard filename '{shard_file}' in index file '{index_file}'. " + "Shard filenames must be relative paths without '..' components." + ) + # Reject extension mismatch (e.g. .bin shard in a .safetensors index) + if not shard_file.endswith(expected_extension): + raise ValueError( + f"Invalid shard filename '{shard_file}' in index file '{index_file}'. " + f"Expected '{expected_extension}' extension to match the index format." + ) + + # 3. Validate keys if in strict mode + # This is done before loading any shards to fail fast + if strict: + _validate_keys_for_strict_loading(model, index["weight_map"].keys()) + + # 4. Load each shard using `load_state_dict` + # Get unique shard files (multiple parameters can be in same shard) + for shard_file in shard_files: + # Load shard into memory + shard_path = os.path.join(save_directory, shard_file) + state_dict = load_state_dict_from_file( + shard_path, + map_location="cpu", + weights_only=weights_only, + ) + # Update model with parameters from this shard + model.load_state_dict(state_dict, strict=strict) + # Explicitly remove the state dict from memory + del state_dict + + # 5. Return compatibility info + loaded_keys = set(index["weight_map"].keys()) + model_keys = set(model.state_dict().keys()) + return _IncompatibleKeys( + missing_keys=list(model_keys - loaded_keys), unexpected_keys=list(loaded_keys - model_keys) + ) + + +def load_state_dict_from_file( + checkpoint_file: str | os.PathLike, + map_location: Union[str, "torch.device"] | None = None, + weights_only: bool = False, + mmap: bool = False, +) -> dict[str, "torch.Tensor"] | Any: + """ + Loads a checkpoint file, handling both safetensors and pickle checkpoint formats. + + Args: + checkpoint_file (`str` or `os.PathLike`): + Path to the checkpoint file to load. Can be either a safetensors or pickle (`.bin`) checkpoint. + map_location (`str` or `torch.device`, *optional*): + A `torch.device` object, string or a dict specifying how to remap storage locations. It + indicates the location where all tensors should be loaded. + weights_only (`bool`, *optional*, defaults to `False`): + If True, only loads the model weights without optimizer states and other metadata. + Only supported for pickle (`.bin`) checkpoints with PyTorch >= 1.13. Has no effect when + loading safetensors files. + mmap (`bool`, *optional*, defaults to `False`): + Whether to use memory-mapped file loading. Memory mapping can improve loading performance + for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints. Has no effect when + loading safetensors files, as the `safetensors` library uses memory mapping by default. + + Returns: + `Union[dict[str, "torch.Tensor"], Any]`: The loaded checkpoint. + - For safetensors files: always returns a dictionary mapping parameter names to tensors. + - For pickle files: returns any Python object that was pickled (commonly a state dict, but could be + an entire model, optimizer state, or any other Python object). + + Raises: + [`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError) + If the checkpoint file does not exist. + [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) + If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively. + [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) + If the checkpoint file format is invalid or if git-lfs files are not properly downloaded. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the checkpoint file path is empty or invalid. + + Example: + ```python + >>> from huggingface_hub import load_state_dict_from_file + + # Load a PyTorch checkpoint + >>> state_dict = load_state_dict_from_file("path/to/model.bin", map_location="cpu") + >>> model.load_state_dict(state_dict) + + # Load a safetensors checkpoint + >>> state_dict = load_state_dict_from_file("path/to/model.safetensors") + >>> model.load_state_dict(state_dict) + ``` + """ + checkpoint_path = Path(checkpoint_file) + + # Check if file exists and is a regular file (not a directory) + if not checkpoint_path.is_file(): + raise FileNotFoundError( + f"No checkpoint file found at '{checkpoint_path}'. Please verify the path is correct and " + "the file has been properly downloaded." + ) + + # Load safetensors checkpoint + if checkpoint_path.suffix == ".safetensors": + try: + from safetensors import safe_open + from safetensors.torch import load_file + except ImportError as e: + raise ImportError( + "Please install `safetensors` to load safetensors checkpoint. " + "You can install it with `pip install safetensors`." + ) from e + + # Check format of the archive + with safe_open(checkpoint_file, framework="pt") as f: # type: ignore[attr-defined] + metadata = f.metadata() + # see comment: https://github.com/huggingface/transformers/blob/3d213b57fe74302e5902d68ed9478c3ad1aaa713/src/transformers/modeling_utils.py#L3966 + if metadata is not None and metadata.get("format") not in ["pt", "mlx"]: + raise OSError( + f"The safetensors archive passed at {checkpoint_file} does not contain the valid metadata. Make sure " + "you save your model with the `save_torch_model` method." + ) + device = str(map_location.type) if map_location is not None and hasattr(map_location, "type") else map_location + # meta device is not supported with safetensors, falling back to CPU + if device == "meta": + logger.warning("Meta device is not supported with safetensors. Falling back to CPU device.") + device = "cpu" + return load_file(checkpoint_file, device=device) # type: ignore[arg-type] + # Otherwise, load from pickle + try: + import torch + from torch import load + except ImportError as e: + raise ImportError( + "Please install `torch` to load torch tensors. You can install it with `pip install torch`." + ) from e + # Add additional kwargs, mmap is only supported in torch >= 2.1.0 + additional_kwargs = {} + if version.parse(torch.__version__) >= version.parse("2.1.0"): + additional_kwargs["mmap"] = mmap + + # weights_only is only supported in torch >= 1.13.0 + if version.parse(torch.__version__) >= version.parse("1.13.0"): + additional_kwargs["weights_only"] = weights_only + + return load( + checkpoint_file, + map_location=map_location, + **additional_kwargs, + ) + + +# HELPERS + + +def _validate_keys_for_strict_loading( + model: "torch.nn.Module", + loaded_keys: Iterable[str], +) -> None: + """ + Validate that model keys match loaded keys when strict loading is enabled. + + Args: + model: The PyTorch model being loaded + loaded_keys: The keys present in the checkpoint + + Raises: + RuntimeError: If there are missing or unexpected keys in strict mode + """ + loaded_keys_set = set(loaded_keys) + model_keys = set(model.state_dict().keys()) + missing_keys = model_keys - loaded_keys_set # Keys in model but not in checkpoint + unexpected_keys = loaded_keys_set - model_keys # Keys in checkpoint but not in model + + if missing_keys or unexpected_keys: + error_message = f"Error(s) in loading state_dict for {model.__class__.__name__}" + if missing_keys: + str_missing_keys = ",".join([f'"{k}"' for k in sorted(missing_keys)]) + error_message += f"\nMissing key(s): {str_missing_keys}." + if unexpected_keys: + str_unexpected_keys = ",".join([f'"{k}"' for k in sorted(unexpected_keys)]) + error_message += f"\nUnexpected key(s): {str_unexpected_keys}." + raise RuntimeError(error_message) + + +def _get_unique_id(tensor: "torch.Tensor") -> int | tuple[Any, ...]: + """Returns a unique id for plain tensor + or a (potentially nested) Tuple of unique id for the flattened Tensor + if the input is a wrapper tensor subclass Tensor + """ + + try: + from torch.distributed.tensor import DTensor + + if isinstance(tensor, DTensor): + local_tensor = tensor.to_local() + return local_tensor.storage().data_ptr() + except ImportError: + pass + + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] + return tuple(_get_unique_id(getattr(tensor, attr)) for attr in attrs) + + except ImportError: + # for torch version less than 2.1, we can fall back to original implementation + pass + + if tensor.device.type == "xla" and is_torch_tpu_available(): + # NOTE: xla tensors don't have storage + # use some other unique id to distinguish. + # this is a XLA tensor, it must be created using torch_xla's + # device. So the following import is safe: + import torch_xla # type: ignore[import] + + unique_id = torch_xla._XLAC._xla_get_tensor_id(tensor) + else: + unique_id = storage_ptr(tensor) + + return unique_id + + +def get_torch_storage_id(tensor: "torch.Tensor") -> tuple["torch.device", int | tuple[Any, ...], int] | None: + """ + Return unique identifier to a tensor storage. + + Multiple different tensors can share the same underlying storage. This identifier is + guaranteed to be unique and constant for this tensor's storage during its lifetime. Two tensor storages with + non-overlapping lifetimes may have the same id. + In the case of meta tensors, we return None since we can't tell if they share the same storage. + + Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/pytorch_utils.py#L278. + """ + if tensor.device.type == "meta": + return None + else: + return tensor.device, _get_unique_id(tensor), get_torch_storage_size(tensor) + + +def get_torch_storage_size(tensor: "torch.Tensor") -> int: + """ + Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L31C1-L41C59 + """ + try: + from torch.distributed.tensor import DTensor + + if isinstance(tensor, DTensor): + # this returns the size of the FULL tensor in bytes + return tensor.nbytes + except ImportError: + pass + + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] + return sum(get_torch_storage_size(getattr(tensor, attr)) for attr in attrs) + except ImportError: + # for torch version less than 2.1, we can fall back to original implementation + pass + + try: + return tensor.untyped_storage().nbytes() + except AttributeError: + # Fallback for torch==1.10 + try: + return tensor.storage().size() * _get_dtype_size(tensor.dtype) + except NotImplementedError: + # Fallback for meta storage + # On torch >=2.0 this is the tensor size + return tensor.nelement() * _get_dtype_size(tensor.dtype) + + +@lru_cache +def is_torch_tpu_available(check_device=True): + """ + Checks if `torch_xla` is installed and potentially if a TPU is in the environment + + Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/utils/import_utils.py#L463. + """ + if importlib.util.find_spec("torch_xla") is not None: + if check_device: + # We need to check if `xla_device` can be found, will raise a RuntimeError if not + try: + import torch_xla.core.xla_model as xm # type: ignore[import] + + _ = xm.xla_device() + return True + except RuntimeError: + return False + return True + return False + + +def storage_ptr(tensor: "torch.Tensor") -> int | tuple[Any, ...]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L11. + """ + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + return _get_unique_id(tensor) # type: ignore + except ImportError: + # for torch version less than 2.1, we can fall back to original implementation + pass + + try: + return tensor.untyped_storage().data_ptr() + except Exception: + # Fallback for torch==1.10 + try: + return tensor.storage().data_ptr() + except NotImplementedError: + # Fallback for meta storage + return 0 + + +def _clean_state_dict_for_safetensors( + state_dict: dict[str, "torch.Tensor"], + metadata: dict[str, str], + force_contiguous: bool = True, + shared_tensors_to_discard: list[str] | None = None, +): + """Remove shared tensors from state_dict and update metadata accordingly (for reloading). + + Warning: `state_dict` and `metadata` are mutated in-place! + + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L155. + """ + to_removes = _remove_duplicate_names(state_dict, discard_names=shared_tensors_to_discard) + for kept_name, to_remove_group in to_removes.items(): + for to_remove in to_remove_group: + if metadata is None: + metadata = {} + + if to_remove not in metadata: + # Do not override user data + metadata[to_remove] = kept_name + del state_dict[to_remove] + if force_contiguous: + state_dict = {k: v.contiguous() for k, v in state_dict.items()} + return state_dict + + +def _end_ptr(tensor: "torch.Tensor") -> int: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L23. + """ + if tensor.nelement(): + stop = tensor.view(-1)[-1].data_ptr() + _get_dtype_size(tensor.dtype) + else: + stop = tensor.data_ptr() + return stop + + +def _filter_shared_not_shared(tensors: list[set[str]], state_dict: dict[str, "torch.Tensor"]) -> list[set[str]]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L44 + """ + filtered_tensors = [] + for shared in tensors: + if len(shared) < 2: + filtered_tensors.append(shared) + continue + + areas = [] + for name in shared: + tensor = state_dict[name] + areas.append((tensor.data_ptr(), _end_ptr(tensor), name)) + areas.sort() + + _, last_stop, last_name = areas[0] + filtered_tensors.append({last_name}) + for start, stop, name in areas[1:]: + if start >= last_stop: + filtered_tensors.append({name}) + else: + filtered_tensors[-1].add(name) + last_stop = stop + + return filtered_tensors + + +def _find_shared_tensors(state_dict: dict[str, "torch.Tensor"]) -> list[set[str]]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L69. + """ + import torch + + tensors_dict = defaultdict(set) + for k, v in state_dict.items(): + if v.device != torch.device("meta") and storage_ptr(v) != 0 and get_torch_storage_size(v) != 0: + # Need to add device as key because of multiple GPU. + tensors_dict[(v.device, storage_ptr(v), get_torch_storage_size(v))].add(k) + tensors = list(sorted(tensors_dict.values())) + tensors = _filter_shared_not_shared(tensors, state_dict) + return tensors + + +def _is_complete(tensor: "torch.Tensor") -> bool: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80 + """ + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] + return all(_is_complete(getattr(tensor, attr)) for attr in attrs) + except ImportError: + # for torch version less than 2.1, we can fall back to original implementation + pass + + return tensor.data_ptr() == storage_ptr(tensor) and tensor.nelement() * _get_dtype_size( + tensor.dtype + ) == get_torch_storage_size(tensor) + + +def _remove_duplicate_names( + state_dict: dict[str, "torch.Tensor"], + *, + preferred_names: list[str] | None = None, + discard_names: list[str] | None = None, +) -> dict[str, list[str]]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80 + """ + if preferred_names is None: + preferred_names = [] + unique_preferred_names = set(preferred_names) + if discard_names is None: + discard_names = [] + unique_discard_names = set(discard_names) + + shareds = _find_shared_tensors(state_dict) + to_remove = defaultdict(list) + for shared in shareds: + complete_names = {name for name in shared if _is_complete(state_dict[name])} + if not complete_names: + raise RuntimeError( + "Error while trying to find names to remove to save state dict, but found no suitable name to keep" + f" for saving amongst: {shared}. None is covering the entire storage. Refusing to save/load the model" + " since you could be storing much more memory than needed. Please refer to" + " https://huggingface.co/docs/safetensors/torch_shared_tensors for more information. Or open an" + " issue." + ) + + keep_name = sorted(list(complete_names))[0] + + # Mechanism to preferentially select keys to keep + # coming from the on-disk file to allow + # loading models saved with a different choice + # of keep_name + preferred = complete_names.difference(unique_discard_names) + if preferred: + keep_name = sorted(list(preferred))[0] + + if unique_preferred_names: + preferred = unique_preferred_names.intersection(complete_names) + if preferred: + keep_name = sorted(list(preferred))[0] + for name in sorted(shared): + if name != keep_name: + to_remove[keep_name].append(name) + return to_remove + + +@lru_cache +def _get_dtype_size(dtype: "torch.dtype") -> int: + """ + Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L344 + """ + import torch + + # torch.float8 formats require 2.1; we do not support these dtypes on earlier versions + _float8_e4m3fn = getattr(torch, "float8_e4m3fn", None) + _float8_e5m2 = getattr(torch, "float8_e5m2", None) + _SIZE = { + torch.int64: 8, + torch.float32: 4, + torch.int32: 4, + torch.bfloat16: 2, + torch.float16: 2, + torch.int16: 2, + torch.uint8: 1, + torch.int8: 1, + torch.bool: 1, + torch.float64: 8, + _float8_e4m3fn: 1, + _float8_e5m2: 1, + } + return _SIZE[dtype] + + +class _IncompatibleKeys(namedtuple("IncompatibleKeys", ["missing_keys", "unexpected_keys"])): + """ + This is used to report missing and unexpected keys in the state dict. + Taken from https://github.com/pytorch/pytorch/blob/main/torch/nn/modules/module.py#L52. + + """ + + def __repr__(self) -> str: + if not self.missing_keys and not self.unexpected_keys: + return "" + return super().__repr__() + + __str__ = __repr__ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/templates/datasetcard_template.md b/venv/lib/python3.12/site-packages/huggingface_hub/templates/datasetcard_template.md new file mode 100644 index 0000000000000000000000000000000000000000..9af29ebbed93653ec74a8952e314e7554323ef15 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/templates/datasetcard_template.md @@ -0,0 +1,143 @@ +--- +# For reference on dataset card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/datasetcard.md?plain=1 +# Doc / guide: https://huggingface.co/docs/hub/datasets-cards +{{ card_data }} +--- + +# Dataset Card for {{ pretty_name | default("Dataset Name", true) }} + + + +{{ dataset_summary | default("", true) }} + +## Dataset Details + +### Dataset Description + + + +{{ dataset_description | default("", true) }} + +- **Curated by:** {{ curators | default("[More Information Needed]", true)}} +- **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}} +- **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}} +- **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}} +- **License:** {{ license | default("[More Information Needed]", true)}} + +### Dataset Sources [optional] + + + +- **Repository:** {{ repo | default("[More Information Needed]", true)}} +- **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}} +- **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}} + +## Uses + + + +### Direct Use + + + +{{ direct_use | default("[More Information Needed]", true)}} + +### Out-of-Scope Use + + + +{{ out_of_scope_use | default("[More Information Needed]", true)}} + +## Dataset Structure + + + +{{ dataset_structure | default("[More Information Needed]", true)}} + +## Dataset Creation + +### Curation Rationale + + + +{{ curation_rationale_section | default("[More Information Needed]", true)}} + +### Source Data + + + +#### Data Collection and Processing + + + +{{ data_collection_and_processing_section | default("[More Information Needed]", true)}} + +#### Who are the source data producers? + + + +{{ source_data_producers_section | default("[More Information Needed]", true)}} + +### Annotations [optional] + + + +#### Annotation process + + + +{{ annotation_process_section | default("[More Information Needed]", true)}} + +#### Who are the annotators? + + + +{{ who_are_annotators_section | default("[More Information Needed]", true)}} + +#### Personal and Sensitive Information + + + +{{ personal_and_sensitive_information | default("[More Information Needed]", true)}} + +## Bias, Risks, and Limitations + + + +{{ bias_risks_limitations | default("[More Information Needed]", true)}} + +### Recommendations + + + +{{ bias_recommendations | default("Users should be made aware of the risks, biases and limitations of the dataset. More information needed for further recommendations.", true)}} + +## Citation [optional] + + + +**BibTeX:** + +{{ citation_bibtex | default("[More Information Needed]", true)}} + +**APA:** + +{{ citation_apa | default("[More Information Needed]", true)}} + +## Glossary [optional] + + + +{{ glossary | default("[More Information Needed]", true)}} + +## More Information [optional] + +{{ more_information | default("[More Information Needed]", true)}} + +## Dataset Card Authors [optional] + +{{ dataset_card_authors | default("[More Information Needed]", true)}} + +## Dataset Card Contact + +{{ dataset_card_contact | default("[More Information Needed]", true)}} diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/templates/modelcard_template.md b/venv/lib/python3.12/site-packages/huggingface_hub/templates/modelcard_template.md new file mode 100644 index 0000000000000000000000000000000000000000..79ca15e4547debac763b390ef8e4b715e6f6403f --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/templates/modelcard_template.md @@ -0,0 +1,200 @@ +--- +# For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 +# Doc / guide: https://huggingface.co/docs/hub/model-cards +{{ card_data }} +--- + +# Model Card for {{ model_id | default("Model ID", true) }} + + + +{{ model_summary | default("", true) }} + +## Model Details + +### Model Description + + + +{{ model_description | default("", true) }} + +- **Developed by:** {{ developers | default("[More Information Needed]", true)}} +- **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}} +- **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}} +- **Model type:** {{ model_type | default("[More Information Needed]", true)}} +- **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}} +- **License:** {{ license | default("[More Information Needed]", true)}} +- **Finetuned from model [optional]:** {{ base_model | default("[More Information Needed]", true)}} + +### Model Sources [optional] + + + +- **Repository:** {{ repo | default("[More Information Needed]", true)}} +- **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}} +- **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}} + +## Uses + + + +### Direct Use + + + +{{ direct_use | default("[More Information Needed]", true)}} + +### Downstream Use [optional] + + + +{{ downstream_use | default("[More Information Needed]", true)}} + +### Out-of-Scope Use + + + +{{ out_of_scope_use | default("[More Information Needed]", true)}} + +## Bias, Risks, and Limitations + + + +{{ bias_risks_limitations | default("[More Information Needed]", true)}} + +### Recommendations + + + +{{ bias_recommendations | default("Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.", true)}} + +## How to Get Started with the Model + +Use the code below to get started with the model. + +{{ get_started_code | default("[More Information Needed]", true)}} + +## Training Details + +### Training Data + + + +{{ training_data | default("[More Information Needed]", true)}} + +### Training Procedure + + + +#### Preprocessing [optional] + +{{ preprocessing | default("[More Information Needed]", true)}} + + +#### Training Hyperparameters + +- **Training regime:** {{ training_regime | default("[More Information Needed]", true)}} + +#### Speeds, Sizes, Times [optional] + + + +{{ speeds_sizes_times | default("[More Information Needed]", true)}} + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +{{ testing_data | default("[More Information Needed]", true)}} + +#### Factors + + + +{{ testing_factors | default("[More Information Needed]", true)}} + +#### Metrics + + + +{{ testing_metrics | default("[More Information Needed]", true)}} + +### Results + +{{ results | default("[More Information Needed]", true)}} + +#### Summary + +{{ results_summary | default("", true) }} + +## Model Examination [optional] + + + +{{ model_examination | default("[More Information Needed]", true)}} + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** {{ hardware_type | default("[More Information Needed]", true)}} +- **Hours used:** {{ hours_used | default("[More Information Needed]", true)}} +- **Cloud Provider:** {{ cloud_provider | default("[More Information Needed]", true)}} +- **Compute Region:** {{ cloud_region | default("[More Information Needed]", true)}} +- **Carbon Emitted:** {{ co2_emitted | default("[More Information Needed]", true)}} + +## Technical Specifications [optional] + +### Model Architecture and Objective + +{{ model_specs | default("[More Information Needed]", true)}} + +### Compute Infrastructure + +{{ compute_infrastructure | default("[More Information Needed]", true)}} + +#### Hardware + +{{ hardware_requirements | default("[More Information Needed]", true)}} + +#### Software + +{{ software | default("[More Information Needed]", true)}} + +## Citation [optional] + + + +**BibTeX:** + +{{ citation_bibtex | default("[More Information Needed]", true)}} + +**APA:** + +{{ citation_apa | default("[More Information Needed]", true)}} + +## Glossary [optional] + + + +{{ glossary | default("[More Information Needed]", true)}} + +## More Information [optional] + +{{ more_information | default("[More Information Needed]", true)}} + +## Model Card Authors [optional] + +{{ model_card_authors | default("[More Information Needed]", true)}} + +## Model Card Contact + +{{ model_card_contact | default("[More Information Needed]", true)}} diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/utils/__init__.py b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d9b96202748ec537c0a7d7a3edf5cbca11ade591 --- /dev/null +++ b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__init__.py @@ -0,0 +1,134 @@ +# Copyright 2021 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# ruff: noqa: F401 +from huggingface_hub.errors import ( + BadRequestError, + BucketNotFoundError, + CacheNotFound, + CorruptedCacheException, + DisabledRepoError, + EntryNotFoundError, + FileMetadataError, + GatedRepoError, + HfHubHTTPError, + HFValidationError, + JobNotFoundError, + LocalEntryNotFoundError, + LocalTokenNotFoundError, + NotASafetensorsRepoError, + OfflineModeIsEnabled, + RepositoryNotFoundError, + RevisionNotFoundError, + SafetensorsParsingError, +) + +from . import tqdm as _tqdm # _tqdm is the module +from ._auth import get_stored_tokens, get_token +from ._cache_assets import cached_assets_path +from ._cache_manager import ( + CachedFileInfo, + CachedIncompleteFileInfo, + CachedRepoInfo, + CachedRevisionInfo, + DeleteCacheStrategy, + HFCacheInfo, + _format_size, + scan_cache_dir, +) +from ._chunk_utils import chunk_iterable +from ._datetime import parse_datetime +from ._detect_agent import detect_agent, is_agent +from ._experimental import experimental +from ._fixes import SoftTemporaryDirectory, WeakFileLock, yaml_dump +from ._git_credential import list_credential_helpers, set_git_credential, unset_git_credential +from ._headers import build_hf_headers, get_token_to_send +from ._hf_uris import HfMount, HfUri, is_hf_uri, parse_hf_mount, parse_hf_uri +from ._http import ( + ASYNC_CLIENT_FACTORY_T, + CLIENT_FACTORY_T, + RateLimitInfo, + close_session, + fix_hf_endpoint_in_url, + get_async_session, + get_session, + hf_raise_for_status, + http_backoff, + http_stream_backoff, + parse_ratelimit_headers, + set_async_client_factory, + set_client_factory, +) +from ._pagination import paginate +from ._paths import DEFAULT_IGNORE_PATTERNS, FORBIDDEN_FOLDERS, filter_repo_objects +from ._runtime import ( + dump_environment_info, + get_aiohttp_version, + get_fastai_version, + get_fastapi_version, + get_fastcore_version, + get_gradio_version, + get_graphviz_version, + get_hf_hub_version, + get_jinja_version, + get_numpy_version, + get_pillow_version, + get_pydantic_version, + get_pydot_version, + get_python_version, + get_tensorboard_version, + get_tf_version, + get_torch_version, + installation_method, + is_aiohttp_available, + is_colab_enterprise, + is_fastai_available, + is_fastapi_available, + is_fastcore_available, + is_google_colab, + is_gradio_available, + is_graphviz_available, + is_jinja_available, + is_notebook, + is_numpy_available, + is_package_available, + is_pillow_available, + is_pydantic_available, + is_pydot_available, + is_safetensors_available, + is_tensorboard_available, + is_tf_available, + is_torch_available, +) +from ._safetensors import SafetensorsFileMetadata, SafetensorsRepoMetadata, TensorInfo +from ._subprocess import capture_output, run_interactive_subprocess, run_subprocess +from ._telemetry import send_telemetry +from ._terminal import ANSI, StatusLine, select_choice, tabulate +from ._typing import is_jsonable, is_simple_optional_type, unwrap_simple_optional_type +from ._validators import validate_hf_hub_args, validate_repo_id +from ._xet import ( + XetFileData, + XetTokenType, + parse_xet_file_data_from_response, +) +from .tqdm import ( + are_progress_bars_disabled, + disable_progress_bars, + enable_progress_bars, + hf_thread_map, + is_tqdm_disabled, + silent_tqdm, + tqdm, + tqdm_stream_file, +) diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/__init__.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b45faa985f6932051a1208c8a285b137d1905306 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_auth.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_auth.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c2fd7306ff3e153ff7f892d11499511370467c9b Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_auth.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_cache_assets.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_cache_assets.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6886baede5475ef2e64a07c41f89b9de4ff29458 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_cache_assets.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_cache_manager.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_cache_manager.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0634a34975e6d589fcaedd2f351bf3df383f5fca Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_cache_manager.cpython-312.pyc differ diff --git a/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_chunk_utils.cpython-312.pyc b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_chunk_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f8143643a434d9fc21e9c2f3ecc34d060623c25 Binary files /dev/null and b/venv/lib/python3.12/site-packages/huggingface_hub/utils/__pycache__/_chunk_utils.cpython-312.pyc differ