Buckets:

download
raw
19.6 kB
"""The REST API client."""
from __future__ import annotations
import base64
import json
import logging
import os
import socket
import sys
import threading
import time
import webbrowser
from functools import partial
from typing import Literal
from urllib.parse import urlparse
import requests
from requests.exceptions import ConnectionError as RequestsConnectionError
from requests.exceptions import HTTPError, Timeout
from . import config
logger = logging.getLogger(__name__)
# A dictionary of tokens keyed by base URL
_tokens = {}
# Single lock guarding all token-refresh operations to prevent thundering herds
# (e.g., many concurrent fsspec threads all attempting to refresh at once).
_refresh_lock = threading.Lock()
# Seconds before JWT expiry at which we proactively refresh.
_REFRESH_BUFFER_SECONDS = 60
# Serializes interactive device-login attempts so that, when several
# concurrent requests all hit an auth failure, only one of them actually
# prompts the user; the rest wait on the lock and then reuse the freshly
# minted token. The device-flow's own polling calls use ``auth=False`` so
# they don't recurse into the auth-retry paths in ``_request``.
_device_login_lock = threading.Lock()
class DeviceLoginError(RuntimeError):
"""Raised when the OAuth device login flow cannot complete."""
def run_device_flow() -> str:
"""Run the OAuth device login flow and return the new access token.
On success the token is written to the config file and the in-memory
token cache. Raises :class:`DeviceLoginError` on any failure (server
error, expired/missing device code, timeout, etc.).
"""
# Serialize concurrent attempts (e.g. multiple fsspec threads all hitting
# 403 at the same time): the first thread runs the flow, the rest wait
# and then reuse the freshly minted token.
base_url = get_base_url()
token_before = _tokens.get(base_url)
with _device_login_lock:
token_after = _tokens.get(base_url)
# If the cached token changed while we were waiting on the lock,
# another thread just completed a device flow — reuse its result.
if token_after is not None and token_after != token_before:
return token_after
try:
hostname = socket.gethostname()
except Exception:
hostname = None
print("Initiating device login flow", flush=True)
try:
resp = post(
"/login/device",
json={"hostname": hostname},
auth=False,
)
device_code = resp["device_code"]
verification_uri = resp["verification_uri"]
expires_in = int(resp["expires_in"])
interval = int(resp["interval"])
except Exception as e:
raise DeviceLoginError(
f"Failed to initiate device login flow: {e}"
) from e
print("Authorize this device by opening this URL:", flush=True)
print(verification_uri, flush=True)
print("Waiting for authorization", flush=True)
try:
webbrowser.open(verification_uri)
except Exception:
pass
deadline = time.monotonic() + expires_in
while time.monotonic() < deadline:
try:
token_resp = post(
"/login/device/token",
json={"device_code": device_code},
auth=False,
)
except Exception as e:
txt = str(e)
if "Device code has expired" in txt:
raise DeviceLoginError(
"Device code has expired; Run 'calkit hub login' again"
) from e
if "Device code not found" in txt:
raise DeviceLoginError(
"Device code not found; Run 'calkit hub login' again"
) from e
raise DeviceLoginError(
f"Error while polling for device authorization: {e}"
) from e
access_token = token_resp.get("access_token")
if access_token:
refresh_token = token_resp.get("refresh_token")
try:
cfg = config.read()
cfg.access_token = access_token
if refresh_token:
cfg.refresh_token = refresh_token
# A stored PAT (``token``) takes priority over
# ``access_token`` in get_token(), so a stale PAT would
# keep being used by future processes and re-trigger
# the device flow every time. The user just
# re-authenticated, so device-flow credentials win.
if getattr(cfg, "token", None) is not None:
cfg.token = None
# A stored DVC token may have been revoked alongside
# the access token that just failed; clear it so the
# next remote-auth check mints a fresh one.
if getattr(cfg, "dvc_token", None) is not None:
cfg.dvc_token = None
cfg.write()
_tokens[base_url] = access_token
except Exception as e:
raise DeviceLoginError(
f"Failed to save token in config: {e}"
) from e
print("Logged in successfully ✅", flush=True)
return access_token
sleep_seconds = min(
interval, max(0.0, deadline - time.monotonic())
)
if sleep_seconds > 0:
time.sleep(sleep_seconds)
raise DeviceLoginError(
"Timed out waiting for device authorization; "
"Run 'calkit hub login' again"
)
# Web app base URLs for the built-in instances. Note these are not derivable
# from the API URLs by any convention (production is a prefix, local is not),
# so both sets are declared explicitly.
HUB_URLS = {
"local": "http://localhost:5173",
"staging": "https://staging.calkit.io",
"production": "https://calkit.io",
"test": "http://localhost:5173",
}
DEFAULT_HUB_URL = HUB_URLS["production"]
def api_url_from_hub_url(hub_url: str) -> str:
"""Derive a hub's API base URL from its web URL.
A hub serves its API from the ``api`` subdomain of the host serving its
web app, so a hub URL is all that's needed to find its API. Every
self-hosted instance is expected to follow this, which is why they need
no configuration beyond their URL. The built-in instances are still
declared explicitly, since the local development stack predates the rule
and doesn't follow it.
"""
hub_url = config.normalize_hub_url(hub_url)
parsed = urlparse(hub_url)
host = parsed.hostname
if not host:
raise ValueError(f"Cannot determine the API URL for hub '{hub_url}'")
# A hub URL that already names the API host is taken as-is, so a
# mistakenly-doubled prefix (api.api.example.edu) can't happen
netloc = host if host.startswith("api.") else f"api.{host}"
if parsed.port is not None:
netloc = f"{netloc}:{parsed.port}"
return f"{parsed.scheme}://{netloc}"
def get_base_url() -> str:
"""Get the API base URL."""
# CALKIT_CLOUD_BASE_URL is the old name, still honored so existing
# setups keep working
override = os.environ.get("CALKIT_HUB_API_BASE_URL") or os.environ.get(
"CALKIT_CLOUD_BASE_URL"
)
if override:
return override
urls = {
"local": "http://api.localhost",
"staging": "https://api.staging.calkit.io",
"production": "https://api.calkit.io",
"test": "http://api.localhost",
}
hub = config.get_hub()
if hub in urls:
return urls[hub]
return api_url_from_hub_url(hub)
def get_hub_url() -> str:
"""Get the web app base URL for the currently active hub."""
hub = config.get_hub()
if hub in HUB_URLS:
return HUB_URLS[hub]
# Arbitrary hubs are identified by their web URL
return hub
def env_for_hub(hub_url: str) -> str | None:
"""Return the built-in environment name serving ``hub_url``, if any.
The URL may omit its scheme. Arbitrary (e.g., self-hosted) hub URLs
return ``None``; their API URLs come from the ``api`` subdomain rule
in ``api_url_from_hub_url`` rather than from a built-in environment.
"""
hub_url = config.normalize_hub_url(hub_url)
for env in ["production", "staging", "local"]:
if hub_url == HUB_URLS[env]:
return env
return None
def _jwt_exp(token: str) -> float | None:
"""Return the ``exp`` claim of a JWT as a UTC timestamp, or ``None``.
Does not verify the signature — only used for proactive expiry checks.
"""
try:
# JWTs are <header>.<payload>.<sig>, all base64url-encoded
payload_b64 = token.split(".")[1]
# Pad to a multiple of 4 for standard base64 decoding
padding = 4 - len(payload_b64) % 4
if padding != 4:
payload_b64 += "=" * padding
payload = json.loads(base64.urlsafe_b64decode(payload_b64))
return float(payload["exp"])
except Exception:
return None
def _token_is_expiring(token: str) -> bool:
"""Return True if ``token`` is a JWT that expires within the refresh
buffer.
"""
exp = _jwt_exp(token)
if exp is None:
return False # Not a JWT (e.g. a PAT) — never proactively refresh
return time.time() >= exp - _REFRESH_BUFFER_SECONDS
def get_token() -> str:
"""Return a valid bearer token.
Priority: in-memory cache (refreshed proactively if expiring), then PAT
(``token`` / ``CALKIT_TOKEN`` env var), then short-lived ``access_token``
from device login.
For access tokens, expiry is read from the JWT ``exp`` claim and a refresh
is attempted before the token expires. Only one thread performs the
refresh at a time; all others wait behind ``_refresh_lock`` and then use
the new token that was stored in the cache.
"""
base_url = get_base_url()
cached = _tokens.get(base_url)
if cached is not None and not _token_is_expiring(cached):
return cached
# Token missing or expiring — acquire the lock so only one thread refreshes
with _refresh_lock:
# Re-check after acquiring lock; another thread may have refreshed
cached = _tokens.get(base_url)
if cached is not None and not _token_is_expiring(cached):
return cached
# If we have a refresh token, try to get a new access token now
if cached is not None and _token_is_expiring(cached):
new_token = _do_refresh()
if new_token is not None:
return new_token
# No usable cached token — load from config
cfg = config.read()
# Prefer long-lived PAT if present (env var CALKIT_TOKEN or config)
if cfg.token is not None:
_tokens[base_url] = cfg.token
return cfg.token
# Fall back to short-lived access token from device login
if cfg.access_token is not None:
if not _token_is_expiring(cfg.access_token):
_tokens[base_url] = cfg.access_token
return cfg.access_token
# Config token is also expiring — attempt refresh
new_token = _do_refresh()
if new_token is not None:
return new_token
raise ValueError(
"No token found; Run 'calkit hub login' to authenticate"
)
def _do_refresh() -> str | None:
"""Perform a token refresh request.
Must be called with ``_refresh_lock`` already held (or from within
``_try_refresh`` which acquires it). Returns the new access token on
success or ``None`` on failure.
"""
cfg = config.read()
if cfg.refresh_token is None:
return None
base_url = get_base_url()
try:
resp = requests.post(
base_url + "/login/refresh",
json={"refresh_token": cfg.refresh_token},
)
resp.raise_for_status()
data = resp.json()
new_access = data["access_token"]
new_refresh = data.get("refresh_token", cfg.refresh_token)
cfg.access_token = new_access
cfg.refresh_token = new_refresh
cfg.write()
_tokens[base_url] = new_access
return new_access
except Exception as exc:
logger.debug("Token refresh failed: %s", exc)
return None
def _try_refresh() -> str | None:
"""Attempt a token refresh from outside the normal request cycle.
Acquires ``_refresh_lock`` to prevent concurrent refresh storms.
Returns the new access token on success, or ``None``.
"""
with _refresh_lock:
return _do_refresh()
def get_headers(headers: dict | None = None, auth: bool = True) -> dict:
if auth:
base_headers = {"Authorization": f"Bearer {get_token()}"}
else:
base_headers = {}
if headers is not None:
return base_headers | headers
else:
return base_headers
def _request(
kind: Literal["get", "post", "put", "patch", "delete"],
path: str,
params: dict | None = None,
json: dict | None = None,
data: dict | None = None,
headers: dict | None = None,
as_json=True,
auth: bool = True,
base_url: str | None = None,
max_retries: int = 10,
allow_login: bool = True,
**kwargs,
):
"""Make a request to the hub.
``max_retries`` and ``allow_login`` exist for calls that are worth
making but not worth waiting on: reporting something in the background
should not sit through two minutes of backoff, nor stop to ask an
unauthenticated user to log in.
"""
base_delay_seconds = 0.25
max_delay_seconds = 30
# Bound how long a single attempt can hang so stalled connections become
# retryable timeouts rather than blocking forever. Callers can override.
kwargs.setdefault("timeout", (10, 120))
func = getattr(requests, kind)
if base_url is None:
base_url = get_base_url()
refresh_attempted = False
device_login_attempted = False
# We may prompt the user via the device login flow only when stdin and
# stdout are TTYs (i.e. not in CI, daemons, or piped subprocesses) and
# the user hasn't opted out via the env var.
can_prompt = (
auth
and allow_login
and not os.environ.get("CALKIT_NO_INTERACTIVE_LOGIN")
)
if can_prompt:
try:
can_prompt = sys.stdin.isatty() and sys.stdout.isatty()
except Exception:
can_prompt = False
for retry_num in range(max_retries + 1):
try:
req_headers = get_headers(headers, auth=auth)
except ValueError:
# No token in config at all. If we can prompt the user, run
# the device flow and try again; otherwise re-raise so callers
# see the original "no token" error. ``run_device_flow``
# serializes concurrent callers via ``_device_login_lock`` and
# reuses the freshly minted token, so threads that arrive
# mid-flow simply wait their turn rather than re-prompting.
if can_prompt and not device_login_attempted:
device_login_attempted = True
run_device_flow()
continue
raise
try:
resp = func(
base_url + path,
params=params,
json=json,
data=data,
headers=req_headers,
**kwargs,
)
except (Timeout, RequestsConnectionError):
# Transient network failure (read/connect timeout, connection
# reset/refused). Retry with the same backoff as transient 5xx
# responses; re-raise once retries are exhausted.
if retry_num >= max_retries:
raise
wait = min(base_delay_seconds * (2**retry_num), max_delay_seconds)
time.sleep(wait)
continue
# Retry transient server-side errors (bad gateway, overloaded,
# unavailable, gateway timeout) with exponential backoff. These are
# typically infrastructure hiccups that resolve on a retry.
is_transient_5xx = resp.status_code in (500, 502, 503, 504)
if is_transient_5xx and retry_num < max_retries:
wait = min(base_delay_seconds * (2**retry_num), max_delay_seconds)
time.sleep(wait)
continue
# Decide whether this response is a credential rejection (vs. a
# permission/authorization failure that happens to also be 403).
# 401 is always a credential rejection. For 403, only treat it as
# one if the API's ``detail`` identifies it as such — otherwise
# we'd hijack legitimate "you can't do that" errors and pop a
# browser-based login flow at the user.
looks_like_auth_failure = auth and resp.status_code == 401
if auth and resp.status_code == 403:
try:
detail = resp.json().get("detail")
except Exception:
detail = None
looks_like_auth_failure = isinstance(detail, str) and any(
s in detail
for s in (
"Could not validate credentials",
"Not authenticated",
"Invalid token",
"Token has expired",
)
)
# On credential rejection, attempt a token refresh once.
# ``_try_refresh`` returns ``None`` when no refresh token is stored
# (e.g. PAT-only sessions). ``get_headers()`` re-calls
# ``get_token()`` on the next iteration, so it will automatically
# pick up the new token stored in ``_tokens`` by ``_try_refresh``.
if looks_like_auth_failure and not refresh_attempted:
refresh_attempted = True
new_token = _try_refresh()
if new_token is not None:
continue
# If refresh didn't help, fall back to the interactive device login
# flow once. This covers both HTTP remotes (where /user/tokens
# 401/403s) and ck:// remotes (which go through this same
# ``_request`` path). ``run_device_flow`` serializes via
# ``_device_login_lock`` and reuses the new token, so concurrent
# callers wait rather than each starting their own flow.
if (
looks_like_auth_failure
and can_prompt
and not device_login_attempted
):
device_login_attempted = True
try:
run_device_flow()
except DeviceLoginError as e:
logger.warning("Device login failed: %s", e)
else:
continue
try:
resp.raise_for_status()
except HTTPError as e:
try:
detail = resp.json()["detail"]
except Exception:
raise e
raise HTTPError(f"{resp.status_code}: {detail}")
break
if as_json:
return resp.json()
else:
return resp
get = partial(_request, "get")
post = partial(_request, "post")
patch = partial(_request, "patch")
put = partial(_request, "put")
delete = partial(_request, "delete")
def get_current_user() -> dict:
return get("/user")

Xet Storage Details

Size:
19.6 kB
·
Xet hash:
80e1c00627fc6c86830080c674bc86a20f08f36b9c24a27c05c18b0ca6a5aa34

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.