Package sizes computed on the Space (package-download background job: pages the local index, one HF tree listing per (repo, revision), cached; nginx /package_sizes.json -> 8020, complete:false while pending). Restart loads media adds 13-25.

docker/hf-space/nginx.conf.template

@@ -155,6 +155,21 @@ server {
         proxy_read_timeout 300s;
     }
 
+    # Every indexed asset's download size (the Explorer's Size column): package-download keeps the table current
+    # (pipeline/package_download/sizes.py), so the dist's static package_sizes.json is never served. While
+    # package-download is not up, or fails, a 200 with complete:false: the Explorer asks again in 60 s, where
+    # a 5xx would end its polling for the session. A 404 (sizes switched off) passes through: no sizes.
+    location = /package_sizes.json {
+        if ($request_method = OPTIONS) { add_header Content-Length 0 always; add_header Content-Type "text/plain" always; return 204; }
+        proxy_pass http://127.0.0.1:8020;
+        proxy_intercept_errors on;
+        error_page 500 502 503 504 = @package_sizes_pending;
+    }
+    location @package_sizes_pending {
+        default_type application/json;
+        return 200 '{"index":"usdsearch-dev","complete":false,"generated_at":null,"basis":"package sizes are not available yet","total":0,"sizes":{}}';
+    }
+
     location /images {
         if ($request_method = OPTIONS) { add_header Content-Length 0 always; add_header Content-Type "text/plain" always; return 204; }
         rewrite ^/images(.*)$ /v3/images$1 break;

pipeline/package_download/__init__.py

@@ -10,10 +10,14 @@ laid out exactly as on Hugging Face, contents at the archive root.
 
 Bytes come through the in-container hf-storage-api gRPC server (FileFolderService.ListStat + FileObjectService.
 ReadFromAddress), the same server deepsearch-api's own download already reads through: one HF token, one cache,
-one access boundary (only catalogued folders resolve). This process holds no HF token.
+one access boundary (only catalogued folders resolve). Downloads use no HF token of this process.
 
 ``scope=dependencies``, and any URL that is not inside a catalogued package, is forwarded unchanged to
 deepsearch-api's ``/v3/download/asset`` — the old asset + dependencies bundle.
 
+``GET /package_sizes.json`` is every indexed asset's download size for the Explorer's Size column: a background
+task pages this Space's index and sums each package folder from one HF tree listing per (repo, revision), with the
+read-only corpus token (``sizes.py``).
+
 Design and decision log: docs/plans/hf-space-package-download.md.
 """

pipeline/package_download/__main__.py

@@ -10,6 +10,7 @@ import uvicorn
 from .app import create_app
 from .catalog import Catalog
 from .settings import Settings
+from .sizes import PackageSizes, SizesSettings
 
 
 def main() -> int:
@@ -27,7 +28,12 @@ def main() -> int:
             "no hf-storage-api catalog at %s: forwarding every download to deepsearch-api", settings.catalog_path
         )
         catalog = Catalog({})
-    uvicorn.run(create_app(settings, catalog=catalog), host=settings.host, port=settings.port, log_config=None)
+    # /package_sizes.json: every indexed asset's download size, kept current in the background (sizes.py).
+    sizes_settings = SizesSettings.from_env()
+    sizes = PackageSizes(sizes_settings) if sizes_settings.enabled else None
+    uvicorn.run(
+        create_app(settings, catalog=catalog, sizes=sizes), host=settings.host, port=settings.port, log_config=None
+    )
     return 0
 
 

pipeline/package_download/app.py

@@ -1,4 +1,5 @@
-"""HTTP surface: ``GET /download/asset`` (also ``/v3/download/asset``), ``/health``, ``/metrics``.
+"""HTTP surface: ``GET /download/asset`` (also ``/v3/download/asset``), ``/package_sizes.json``, ``/health``,
+``/metrics``.
 
 Drop-in for usdsearch's download route (see the package docstring); nginx proxies ``/download`` here.
 """
@@ -24,6 +25,7 @@ from starlette.background import BackgroundTask
 from . import bundle, telemetry
 from .catalog import Catalog
 from .settings import Settings
+from .sizes import PackageSizes
 
 logger = logging.getLogger("package_download")
 
@@ -181,6 +183,7 @@ def create_app(
     source: Any = None,
     url_mapping: Optional[UrlMapping] = None,
     fallback_transport: Optional[httpx.AsyncBaseTransport] = None,
+    sizes: Optional[PackageSizes] = None,
 ) -> FastAPI:
     settings = settings or Settings.from_env()
     to_internal, to_public, to_public_url = url_mapping or _default_url_mapping()
@@ -215,9 +218,13 @@ def create_app(
             "package-download: %d catalogued package(s), storage API %s, fallback %s",
             len(catalog), settings.storage_api_endpoint, settings.fallback_url,
         )
+        if sizes is not None:
+            sizes.start()
         try:
             yield
         finally:
+            if sizes is not None:
+                await sizes.stop()
             await app.state.fallback.aclose()
             if owned_source:
                 await source.close()
@@ -248,6 +255,15 @@ def create_app(
     async def prometheus():
         return metrics.render()
 
+    if sizes is not None:
+        @app.get("/package_sizes.json")
+        async def package_sizes(request: Request):
+            body, encoding = sizes.body(request.headers.get("accept-encoding", ""))
+            headers = {"Cache-Control": "no-cache", "Vary": "Accept-Encoding"}
+            if encoding:
+                headers["Content-Encoding"] = encoding
+            return Response(body, media_type="application/json", headers=headers)
+
     async def forward(request: Request) -> Response:
         """Hand the request to deepsearch-api's own /v3/download/asset, unchanged."""
         params = [(k, v) for k, v in request.query_params.multi_items() if k != "scope"]

pipeline/package_download/sizes.py

@@ -0,0 +1,458 @@
+"""``GET /package_sizes.json``: every indexed asset's download size, for the Explorer's Size column.
+
+The Explorer (SimReady Central's ``utils/packageSizes.js``) reads ONE table and matches each hit's
+``source.base_key`` exactly (``https://huggingface.co/datasets/<repo>/blob/<revision>/<path>``). An asset's size is
+its package folder at its pinned revision: every file under the asset's top-level folder (the whole repo for a
+file at the repo root), summed from one recursive HF tree listing per (repo, revision) — the definition of the
+one-off table of 2026-10-16 (hf-space-work/package-sizes-1016/full_sizes.py), reproduced byte for byte.
+
+A background task keeps the table current. At start, and every ``interval`` seconds, it pages this Space's own
+index (deepsearch-api's query-less search, ``page_size`` hits a page, by cursor) for every base_key, lists each
+(repo, revision) it has no sums for, and caches those sums on disk: a new asset or a re-pin costs one listing for
+its new revision only. A pinned revision (a sha) never changes; a branch name is listed again after
+``relist_after``. HF requests are paced (at most one per ``hf_interval``) and back off on 429 / 5xx. A listing
+that still fails keeps the last table: its group stays pending (``complete: false``, which the Explorer polls
+again every 60 s) until a later pass lists it. A repo HF refuses outright (401/403/404: gone, gated, private)
+settles unsized and is asked again after ``relist_after``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import gzip
+import hashlib
+import json
+import logging
+import os
+import time
+from collections import Counter, defaultdict
+from dataclasses import dataclass, fields
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+from pathlib import Path
+from typing import Awaitable, Callable, Dict, Mapping, Optional, Tuple
+from urllib.parse import quote, unquote
+
+import httpx
+
+from .. import hf_addressing
+
+logger = logging.getLogger("package_download.sizes")
+
+PREFIX = "PACKAGE_SIZES_"
+BASIS = (
+    "each asset's package folder (every file under its top-level folder) at its pinned revision, "
+    "from one HF tree listing per (repo, revision)"
+)
+
+
+@dataclass(frozen=True)
+class SizesSettings:
+    enabled: bool = True
+    # The Explorer's index key: /config.js's only server ("usdsearch-dev", nginx.conf.template).
+    index: str = "usdsearch-dev"
+    # deepsearch-api's v3 search (nginx's /search_hybrid), loopback.
+    search_url: str = "http://127.0.0.1:8000/v3/deepsearch/search"
+    page_size: int = 500
+    # The service-account Basic tag of the Explorer's own background probes (SimReady Central's utils/svcAuth.js):
+    # not access control, it keeps these walks out of the usage telemetry's user searches. Empty user: no tag.
+    search_user: str = "simready-central-svc-user"
+    search_password: str = "simready-central-svc-pass"
+    # Seconds between passes; after a pass that left groups pending; after a failed index walk.
+    interval: float = 900.0
+    pending_interval: float = 300.0
+    retry_interval: float = 30.0
+    hf_endpoint: str = "https://huggingface.co"
+    # At most one HF request per this many seconds, across every listing.
+    hf_interval: float = 1.0
+    # Tries per HF request (429, 5xx, a dropped connection); waits follow Retry-After, else 30 s doubling,
+    # at most hf_max_backoff. A request that still fails ends the pass's listings.
+    hf_attempts: int = 4
+    hf_max_backoff: float = 600.0
+    # Empty: HF_STORAGE_API_TOKEN (the Space's read-only corpus token), else HF_TOKEN, else anonymous.
+    hf_token: str = ""
+    timeout: float = 60.0
+    relist_after: float = 6 * 3600.0
+    # Outside .build/restore/, which the index restore empties at every start.
+    cache_dir: Path = Path("/app/hf-space-pipeline/.build/package_sizes")
+
+    @classmethod
+    def from_env(cls, env: Optional[Mapping[str, str]] = None) -> "SizesSettings":
+        env = os.environ if env is None else env
+        values = {}
+        for field in fields(cls):
+            raw = env.get(PREFIX + field.name.upper())
+            if raw is None or raw == "":
+                continue
+            default = field.default
+            if isinstance(default, bool):
+                values[field.name] = raw.strip().lower() in ("1", "true", "yes", "on")
+            elif isinstance(default, int):
+                values[field.name] = int(raw)
+            elif isinstance(default, float):
+                values[field.name] = float(raw)
+            elif isinstance(default, Path):
+                values[field.name] = Path(raw)
+            else:
+                values[field.name] = raw
+        if not values.get("hf_token"):
+            values["hf_token"] = (env.get("HF_STORAGE_API_TOKEN") or env.get("HF_TOKEN") or "").strip()
+        return cls(**values)
+
+
+@dataclass(frozen=True, order=True)
+class Group:
+    """One (repo, revision): everything one recursive tree listing answers."""
+
+    kind: str  # the API's path segment: datasets | models | spaces
+    repo_id: str
+    revision: str
+
+    def __str__(self) -> str:
+        return f"{self.kind}/{self.repo_id}@{self.revision[:8]}"
+
+    @property
+    def cache_name(self) -> str:
+        return hashlib.sha256(f"{self.kind}/{self.repo_id}@{self.revision}".encode()).hexdigest()[:32] + ".json"
+
+
+@dataclass
+class Sums:
+    whole: int
+    folders: Dict[str, int]
+    files: int
+    listed_at: float
+
+    def size_of(self, path: str) -> int:
+        # full_sizes.py: the top-level folder's files, or the whole repo for a file at the root.
+        return self.folders.get(path.split("/", 1)[0], 0) if "/" in path else self.whole
+
+
+def parse_key(key: str, endpoint: str = "https://huggingface.co") -> Optional[Tuple[Group, str]]:
+    """``<endpoint>/[datasets/|spaces/]<org>/<name>/blob/<revision>/<path>`` → (group, repo path), else None."""
+    prefix = endpoint.rstrip("/") + "/"
+    if not isinstance(key, str) or not key.startswith(prefix):
+        return None
+    parts = key[len(prefix):].split("?", 1)[0].split("/")
+    kind = parts.pop(0) if parts and parts[0] in ("datasets", "spaces") else "models"
+    if len(parts) < 5 or parts[2] != "blob" or not all(parts[:2]):
+        return None
+    revision, path = unquote(parts[3]), unquote("/".join(parts[4:]))
+    if not revision or not path:
+        return None
+    return Group(kind, f"{parts[0]}/{parts[1]}", revision), path
+
+
+class IndexUnavailable(Exception):
+    pass
+
+
+class HFRefused(Exception):
+    """HF answered 401/403/404: the repo or revision is gone, gated or private to this token."""
+
+
+class HFUnavailable(Exception):
+    """HF kept answering 429 / 5xx (or the connection kept failing) after every retry."""
+
+
+def _retry_after(response: httpx.Response) -> Optional[float]:
+    raw = response.headers.get("retry-after")
+    if not raw:
+        return None
+    try:
+        return max(0.0, float(raw))
+    except ValueError:
+        pass
+    try:
+        return max(0.0, (parsedate_to_datetime(raw) - datetime.now(timezone.utc)).total_seconds())
+    except (TypeError, ValueError):
+        return None
+
+
+class PackageSizes:
+    """The table and the task that keeps it current. ``body()`` is what the route sends."""
+
+    def __init__(
+        self,
+        settings: Optional[SizesSettings] = None,
+        *,
+        search_transport: Optional[httpx.AsyncBaseTransport] = None,
+        hf_transport: Optional[httpx.AsyncBaseTransport] = None,
+        clock: Callable[[], float] = time.monotonic,
+        wall: Callable[[], float] = time.time,
+        sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+    ):
+        self.settings = settings or SizesSettings.from_env()
+        self._search_transport, self._hf_transport = search_transport, hf_transport
+        self._clock, self._wall, self._sleep = clock, wall, sleep
+        self._keys: Dict[str, Optional[Tuple[Group, str]]] = {}
+        self._walked = False
+        self._sums: Dict[Group, Sums] = {}
+        self._refused: Dict[Group, float] = {}  # group -> wall time it may be asked again
+        self._next_hf = 0.0
+        self._task: Optional[asyncio.Task] = None
+        self._search: Optional[httpx.AsyncClient] = None
+        self._hf: Optional[httpx.AsyncClient] = None
+        self._body = self._gz = b""
+        self.sized = 0
+        self._rebuild()
+
+    # --- serving ---------------------------------------------------------------------------------------------
+
+    def body(self, accept_encoding: str = "") -> Tuple[bytes, Optional[str]]:
+        """The table's JSON, gzipped for a client that takes gzip (nginx doesn't compress proxied responses)."""
+        if "gzip" in (accept_encoding or "").lower():
+            return self._gz, "gzip"
+        return self._body, None
+
+    @property
+    def complete(self) -> bool:
+        return self._walked and not self._pending()
+
+    def _pending(self) -> set:
+        return {v[0] for v in self._keys.values() if v and v[0] not in self._sums and v[0] not in self._refused}
+
+    def _rebuild(self) -> None:
+        sizes = {}
+        for key, parsed in self._keys.items():
+            if parsed and parsed[0] in self._sums:
+                n = self._sums[parsed[0]].size_of(parsed[1])
+                if n > 0:
+                    sizes[key] = n
+        doc = {
+            "index": self.settings.index,
+            "complete": self.complete,
+            "generated_at": datetime.fromtimestamp(self._wall(), timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "basis": BASIS,
+            "total": len(self._keys),
+            "sizes": dict(sorted(sizes.items())),
+        }
+        self.sized = len(sizes)
+        self._body = json.dumps(doc, separators=(",", ":")).encode()
+        self._gz = gzip.compress(self._body, compresslevel=6, mtime=0)
+
+    # --- the task --------------------------------------------------------------------------------------------
+
+    def start(self) -> None:
+        self._task = asyncio.get_running_loop().create_task(self.run(), name="package-sizes")
+
+    async def stop(self) -> None:
+        if self._task is not None:
+            self._task.cancel()
+            await asyncio.gather(self._task, return_exceptions=True)
+            self._task = None
+        await self.close()
+
+    async def close(self) -> None:
+        for client in (self._search, self._hf):
+            if client is not None:
+                await client.aclose()
+        self._search = self._hf = None
+
+    async def run(self) -> None:
+        self.load_cache()
+        while True:
+            try:
+                delay = await self.refresh()
+            except asyncio.CancelledError:
+                raise
+            except Exception:  # noqa: BLE001 — the table stays as it was; try again later
+                logger.exception("package sizes: pass failed")
+                delay = self.settings.retry_interval
+            await self._sleep(delay)
+
+    def _clients(self) -> Tuple[httpx.AsyncClient, httpx.AsyncClient]:
+        s = self.settings
+        if self._search is None:
+            auth = (s.search_user, s.search_password) if s.search_user else None
+            self._search = httpx.AsyncClient(
+                timeout=s.timeout, transport=self._search_transport, trust_env=False, auth=auth
+            )
+        if self._hf is None:
+            headers = {"Authorization": f"Bearer {s.hf_token}"} if s.hf_token else {}
+            self._hf = httpx.AsyncClient(
+                timeout=s.timeout, transport=self._hf_transport, headers=headers, follow_redirects=True
+            )
+        return self._search, self._hf
+
+    async def refresh(self) -> float:
+        """One pass: walk the index, list what is missing (biggest group first). Returns the delay to the next."""
+        s = self.settings
+        try:
+            keys = await self.walk_index()
+        except IndexUnavailable as exc:
+            logger.info("package sizes: index not readable yet (%s); next try in %.0f s", exc, s.retry_interval)
+            return s.retry_interval
+        self._keys, self._walked = keys, True
+        now = self._wall()
+        self._refused = {g: t for g, t in self._refused.items() if t > now}
+        counts = Counter(v[0] for v in keys.values() if v)
+        stale = {
+            g for g, sums in self._sums.items()
+            if not hf_addressing.looks_like_sha(g.revision) and now - sums.listed_at >= s.relist_after
+        }
+        todo = sorted(
+            (g for g in counts if (g not in self._sums or g in stale) and g not in self._refused),
+            key=lambda g: (-counts[g], g),
+        )
+        self._rebuild()
+        for group in todo:
+            try:
+                sums, pages = await self.list_group(group)
+            except HFRefused as exc:
+                logger.warning("package sizes: %s refused (%s); %d asset(s) unsized", group, exc, counts[group])
+                self._refused[group] = now + s.relist_after
+            except HFUnavailable as exc:
+                logger.warning("package sizes: listing %s failed (%s); the rest waits for the next pass", group, exc)
+                break
+            else:
+                self._sums[group] = sums
+                self._write_cache(group, sums)
+                logger.info(
+                    "package sizes: listed %s: %d asset(s), %d file(s) in %d page(s)", group, counts[group], sums.files,
+                    pages,
+                )
+            self._rebuild()
+        self._rebuild()
+        pending = self._pending()
+        logger.info(
+            "package sizes: %d asset(s), %d sized, %d group(s), %d pending%s", len(keys), self.sized, len(counts),
+            len(pending), "" if pending else " (complete)",
+        )
+        return s.pending_interval if pending else s.interval
+
+    async def walk_index(self) -> Dict[str, Optional[Tuple[Group, str]]]:
+        """Every base_key in the index (deepsearch-api's query-less search, by cursor)."""
+        search, _ = self._clients()
+        s = self.settings
+        keys: Dict[str, Optional[Tuple[Group, str]]] = {}
+        cursor = None
+        for _ in range(10_000):
+            body = {"limit": s.page_size}
+            if cursor:
+                body["cursor"] = cursor
+            try:
+                r = await search.post(s.search_url, json=body)
+                r.raise_for_status()
+                doc = r.json()
+            except (httpx.HTTPError, ValueError) as exc:
+                raise IndexUnavailable(f"{type(exc).__name__}: {exc}") from exc
+            hits = doc.get("hits") or []
+            for hit in hits:
+                key = (hit.get("source") or {}).get("base_key")
+                if isinstance(key, str) and key:
+                    keys[key] = parse_key(key, s.hf_endpoint)
+            if not doc.get("has_more"):
+                return keys
+            cursor = doc.get("next_cursor")
+            if not hits or not cursor:  # more to come but no way to it: never a complete table from a part
+                raise IndexUnavailable("has_more without hits or a cursor")
+        raise IndexUnavailable("the walk did not end")
+
+    async def list_group(self, group: Group) -> Tuple[Sums, int]:
+        """One recursive tree listing (every page): the whole repo's bytes and each top-level folder's."""
+        url = (
+            f"{self.settings.hf_endpoint.rstrip('/')}/api/{group.kind}/{group.repo_id}/tree/"
+            f"{quote(group.revision, safe='')}"
+        )
+        params: Optional[dict] = {"recursive": "true", "expand": "false"}
+        whole, files, pages = 0, 0, 0
+        folders: Dict[str, int] = defaultdict(int)
+        while url:
+            r = await self._hf_get(url, params)
+            pages += 1
+            try:
+                items = r.json()
+            except ValueError as exc:
+                raise HFUnavailable(f"page {pages} is not JSON") from exc
+            if not isinstance(items, list):
+                raise HFUnavailable(f"page {pages} is not a list")
+            for item in items:
+                if not isinstance(item, dict) or item.get("type") != "file":
+                    continue
+                size, path = item.get("size"), item.get("path")
+                if isinstance(size, bool) or not isinstance(size, int) or not isinstance(path, str):
+                    raise HFUnavailable(f"page {pages} has a file without a size or path")
+                whole += size
+                files += 1
+                if "/" in path:
+                    folders[path.split("/", 1)[0]] += size
+            url = r.links.get("next", {}).get("url")
+            params = None  # the next page's URL carries its own query
+        return Sums(whole, dict(folders), files, self._wall()), pages
+
+    async def _hf_get(self, url: str, params: Optional[dict]) -> httpx.Response:
+        _, hf = self._clients()
+        s = self.settings
+        problem = ""
+        for attempt in range(max(1, s.hf_attempts)):
+            await self._pace()
+            wait = None
+            try:
+                r = await hf.get(url, params=params)
+            except httpx.HTTPError as exc:
+                problem = type(exc).__name__
+            else:
+                if r.status_code == 200:
+                    return r
+                if r.status_code in (401, 403, 404):
+                    raise HFRefused(f"HTTP {r.status_code}")
+                problem = f"HTTP {r.status_code}"
+                if r.status_code != 429 and r.status_code < 500:
+                    raise HFUnavailable(problem)
+                wait = _retry_after(r)
+            if attempt + 1 >= s.hf_attempts:
+                break
+            if wait is None:
+                wait = 30.0 * 2 ** attempt
+            if wait > s.hf_max_backoff:
+                raise HFUnavailable(f"{problem}, Retry-After {wait:.0f} s")
+            logger.info("package sizes: HF %s; waiting %.0f s", problem, wait)
+            # The pace holds every later request back as well.
+            self._next_hf = max(self._next_hf, self._clock() + wait)
+        raise HFUnavailable(problem or "no answer")
+
+    async def _pace(self) -> None:
+        wait = self._next_hf - self._clock()
+        if wait > 0:
+            await self._sleep(wait)
+        self._next_hf = max(self._next_hf, self._clock()) + self.settings.hf_interval
+
+    # --- the on-disk cache: one file per group -----------------------------------------------------------------
+
+    def _groups_dir(self) -> Path:
+        return Path(self.settings.cache_dir) / "groups"
+
+    def load_cache(self) -> int:
+        loaded = 0
+        folder = self._groups_dir()
+        if not folder.is_dir():
+            return 0
+        for path in sorted(folder.glob("*.json")):
+            try:
+                doc = json.loads(path.read_text())
+                group = Group(doc["kind"], doc["repo_id"], doc["revision"])
+                if path.name != group.cache_name:
+                    continue
+                self._sums[group] = Sums(
+                    int(doc["whole"]), {str(k): int(v) for k, v in doc["folders"].items()}, int(doc["files"]),
+                    float(doc["listed_at"]),
+                )
+                loaded += 1
+            except (OSError, ValueError, KeyError, TypeError, AttributeError) as exc:
+                logger.warning("package sizes: ignoring cache file %s (%s)", path.name, exc)
+        if loaded:
+            logger.info("package sizes: %d group(s) from %s", loaded, folder)
+        return loaded
+
+    def _write_cache(self, group: Group, sums: Sums) -> None:
+        folder = self._groups_dir()
+        try:
+            folder.mkdir(parents=True, exist_ok=True)
+            doc = {"kind": group.kind, "repo_id": group.repo_id, "revision": group.revision, "whole": sums.whole,
+                   "folders": sums.folders, "files": sums.files, "listed_at": sums.listed_at}
+            tmp = folder / f".{group.cache_name}.tmp"
+            tmp.write_text(json.dumps(doc, separators=(",", ":"), sort_keys=True))
+            os.replace(tmp, folder / group.cache_name)
+        except OSError as exc:  # the table still has it; only a restart lists it again
+            logger.warning("package sizes: could not cache %s (%s)", group, exc)