UI main.f3a49a85.js (Central main 14ff836d), HD library media loader, restore skips library-media/

licenses/extras-usd-search-explorer-ui-bundle-javascript-dependencies.txt

@@ -1,4 +1,4 @@
-Third-party notices — SimReady Central Explorer slim-30948535
+Third-party notices — SimReady Central Explorer slim-8c93f7a5
 
 The software in this web application's browser bundle includes the packages below, each under its own
 license. Each entry gives the package, its version, its license and the license / notice text it ships.

pipeline/artifact_store.py

@@ -179,6 +179,22 @@ def read_text(cfg: Config, uri: str) -> Optional[str]:
     return with_retry(attempt, what=f"read of {uri}")
 
 
+def open_stream(cfg: Config, uri: str):
+    """The object's body as a stream (read once, in order), or None if it doesn't exist. Opening is retried like
+    read_text; a stream that breaks mid-read is the caller's to start again."""
+    bucket, key = split_uri(uri)
+
+    def attempt():
+        try:
+            return s3_client(cfg).get_object(Bucket=bucket, Key=key)["Body"]
+        except ClientError as exc:
+            if _is_missing(exc):
+                return None
+            raise
+
+    return with_retry(attempt, what=f"stream of {uri}")
+
+
 def download_file(cfg: Config, uri: str, dest: Path) -> bool:
     """Download one object to `dest`. False if it doesn't exist."""
     bucket, key = split_uri(uri)
@@ -216,16 +232,17 @@ def list_subprefixes(cfg: Config, uri: str) -> List[str]:
     return names
 
 
-def download_prefix(cfg: Config, uri: str, dest: Path) -> int:
+def download_prefix(cfg: Config, uri: str, dest: Path, skip: tuple = ()) -> int:
     """Download everything under `uri/` into `dest`, keeping relative paths —
     the `aws s3 sync <uri>/ <dest>` replacement. Files already present with
     the same size are skipped, so a retried or resumed download doesn't
-    start over. Returns the number of objects under the prefix."""
+    start over; so is every object whose relative path starts with one of
+    `skip`. Returns the number of objects under the prefix."""
     bucket, key = split_uri(uri)
     prefix = _prefix(key)
 
     def attempt() -> int:
-        objects = list_objects(cfg, uri)
+        objects = [(rel, size) for rel, size in list_objects(cfg, uri) if not rel.startswith(skip)]
         todo = [rel for rel, size in objects if not ((dest / rel).is_file() and (dest / rel).stat().st_size == size)]
         with _transfer_manager(s3_client(cfg)) as manager:
             futures = []

pipeline/library_media.py

@@ -23,6 +23,13 @@ serves it at /library_media/ (docker/hf-space/nginx.conf.template), where the Ex
 Nothing here is needed to search. Any failure — no archive, no bucket credentials, a network error, an unsafe or
 malformed archive — leaves the previous tree (none at boot): nginx answers 404 and the Explorer shows no media.
 LIBRARY_MEDIA=off disables the fetch.
+
+The HD tiers (2026-10-09: tier-2 model_hd.glb, a spin's HD frames, an RTX clip's HD file) are too big for that
+boot-time download (about 16 GB of tier-2 models alone, against about 7 GB for everything else), so they travel in
+a sibling archive of the same stamp, ``media-<stamp>.hd.tar``, which ARCHIVE never matches. Once the tree above is
+installed and served, it is STREAMED in (no copy of the archive on disk): each member the manifest lists lands
+under ``<id>/<file>`` through a temporary file, anything else is skipped. Until a file lands the Explorer reads a
+404 and stays on tier 1, quietly; a failure leaves what landed.
 """
 
 from __future__ import annotations
@@ -45,6 +52,14 @@ ARCHIVE = re.compile(r"^media-\d{8}T\d{4}Z\.tar$")   # media-20261005T1010Z.tar:
 INDEX_KEY = "usdsearch-dev"
 MEDIA_ROOT = BUILD_DIR / "library_media"
 DISABLED = {"off", "none", "false", "0"}
+# The HD tiers (tier-2 model_hd.glb, a spin's HD frames, an RTX clip's HD file) travel in a sibling archive of the
+# same stamp, media-<stamp>.hd.tar (ARCHIVE never matches it), streamed into the installed tree AFTER it is served:
+# the Explorer reads them only after the tier-1 picture or model, and quietly stays on tier 1 until a file lands.
+HD_SUFFIX = ".hd.tar"
+SAFE_ID = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
+SAFE_NAME = re.compile(r"^[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}$")
+HD_TRIES = 2
+HD_DISK_MARGIN = 8 * 10**9   # free disk the HD stream always leaves (the index, Neo4j and logs keep writing)
 
 
 def run_prefix(cfg: Config, store, corpus: str) -> str:
@@ -106,6 +121,106 @@ def install(src: Path, dest: Path) -> None:
     shutil.rmtree(old, ignore_errors=True)
 
 
+def hd_files(manifest: dict) -> dict[str, set[str]]:
+    """{entry id: file names} of the HD tiers the manifest lists: an entry's ``hd`` files (model_hd.glb), its spin's
+    HD frames (``spin.hd``: ``spin_hd_{k:03d}.webp`` for every k below ``spin.frames``) and its RTX clip's HD files
+    (``rtxclip.hd``). The only members an HD archive may write."""
+    out: dict[str, set[str]] = {}
+    for e in (manifest.get("assets") or {}).values():
+        if not isinstance(e, dict) or not isinstance(e.get("id"), str) or not SAFE_ID.match(e["id"]):
+            continue
+        names: set[str] = set()
+        hd = e.get("hd") if isinstance(e.get("hd"), dict) else {}
+        names |= {n for n in (hd.get("files") or {}).values() if isinstance(n, str)}
+        spin = e.get("spin") if isinstance(e.get("spin"), dict) else {}
+        spin_hd = spin.get("hd") if isinstance(spin.get("hd"), dict) else {}
+        frames = spin.get("frames")
+        if (spin_hd.get("files") or {}).get("pattern") == "spin_hd_{k:03d}.webp" and isinstance(frames, int):
+            names |= {f"spin_hd_{k:03d}.webp" for k in range(min(frames, 360))}
+        clip = e.get("rtxclip") or e.get("rtx_clip") or (e.get("clip") or {}).get("rtx") or {}
+        clip_hd = clip.get("hd") if isinstance(clip, dict) and isinstance(clip.get("hd"), dict) else {}
+        names |= {n for n in (clip_hd.get("files") or {}).values() if isinstance(n, str)}
+        names = {n for n in names if SAFE_NAME.match(n)}
+        if names:
+            out[e["id"]] = names
+    return out
+
+
+def stream_hd(body, tree: Path, allowed: dict[str, set[str]]) -> tuple[int, int, int]:
+    """Write the HD archive ``body`` (a stream, read once in order) into ``tree``: each member the manifest lists
+    (``allowed``) lands under ``<id>/<file>`` through a temporary file, so nginx never serves half of one; a file
+    already there at the member's size is kept (a retried stream). Anything else is skipped. (written, bytes,
+    skipped)."""
+    written = size = skipped = 0
+    with tarfile.open(fileobj=body, mode="r|") as tf:
+        for m in tf:
+            if m.isdir():
+                continue
+            parts = m.name.removeprefix("./").split("/")
+            if not m.isfile() or len(parts) != 2 or parts[1] not in allowed.get(parts[0], ()):
+                skipped += 1
+                continue
+            target = tree / parts[0] / parts[1]
+            if target.is_file() and target.stat().st_size == m.size:
+                continue
+            target.parent.mkdir(parents=True, exist_ok=True)
+            part = target.with_name(f".{target.name}.part")
+            try:
+                with tf.extractfile(m) as src, open(part, "wb") as out:
+                    shutil.copyfileobj(src, out, 1 << 20)
+                if part.stat().st_size != m.size:
+                    raise ValueError(f"{m.name}: {part.stat().st_size} of {m.size} bytes")
+            except BaseException:
+                part.unlink(missing_ok=True)
+                raise
+            os.replace(part, target)
+            written += 1
+            size += m.size
+    return written, size, skipped
+
+
+def fetch_hd(cfg: Config, store, uri: str, tree: Path) -> int:
+    """Stream ``uri``'s HD sibling (media-<stamp>.hd.tar) into the installed ``tree``; returns the files written.
+    No sibling, or a manifest that lists no HD tier: nothing. A broken stream is started again once (files already
+    complete are kept); a failure leaves what landed — the Explorer stays on tier 1 for the rest."""
+    hd_uri = uri[: -len(".tar")] + HD_SUFFIX
+    if cfg.get("LIBRARY_MEDIA_HD", "").strip().lower() in DISABLED:
+        print("==> library media HD: off (LIBRARY_MEDIA_HD)")
+        return 0
+    allowed = hd_files(json.loads((tree / "manifest.json").read_text()))
+    if not allowed:
+        return 0
+    folder, name = hd_uri.rsplit("/", 1)
+    size = next((n for rel, n in store.list_objects(cfg, folder) if rel == name), None)
+    if size is None:
+        print(f"==> library media HD: none at {hd_uri}")
+        return 0
+    free = shutil.disk_usage(tree).free
+    if size + HD_DISK_MARGIN > free:
+        # never fill the disk OpenSearch and Neo4j write to: a smaller Space keeps tier 1
+        print(f"==> library media HD: {size / 1e9:.1f} GB would leave under {HD_DISK_MARGIN / 1e9:.0f} GB of "
+              f"{free / 1e9:.1f} GB free — not fetched")
+        return 0
+    t0 = time.time()
+    total = [0, 0, 0]
+    for attempt in range(HD_TRIES):
+        body = store.open_stream(cfg, hd_uri)
+        if body is None:
+            print(f"==> library media HD: none at {hd_uri}")
+            return 0
+        try:
+            written, size, skipped = stream_hd(body, tree, allowed)
+            total = [total[0] + written, total[1] + size, skipped]
+            break
+        except (OSError, ValueError, tarfile.TarError) as exc:
+            if attempt == HD_TRIES - 1:
+                raise
+            print(f"==> library media HD: stream broke ({type(exc).__name__}: {str(exc)[:200]}), starting again")
+    print(f"==> library media HD: {total[0]} files ({total[1] / 1e6:.0f} MB) from {hd_uri} in {time.time() - t0:.0f} s"
+          + (f"; {total[2]} members not in the manifest skipped" if total[2] else ""))
+    return total[0]
+
+
 def fetch(env: dict[str, str], root: Path = MEDIA_ROOT, store=None) -> int:
     """Fetch this corpus's media into ``root/INDEX_KEY``; returns the number of assets installed (0: none).
     ``store``: artifact_store's read_text / list_subprefixes / list_objects / download_file / bucket_base."""
@@ -143,6 +258,10 @@ def fetch(env: dict[str, str], root: Path = MEDIA_ROOT, store=None) -> int:
         shutil.rmtree(tmp, ignore_errors=True)
     size = sum(p.stat().st_size for p in (root / INDEX_KEY).rglob("*") if p.is_file())
     print(f"==> library media: {count} assets from {uri} ({size / 1e6:.0f} MB) in {time.time() - t0:.0f} s")
+    try:
+        fetch_hd(cfg, store, uri, root / INDEX_KEY)
+    except Exception as exc:  # noqa: BLE001 — the tier-1 tree is served already; HD is a bonus
+        print(f"==> library media HD: stopped ({type(exc).__name__}: {str(exc)[:300]}) — tier 1 stays", file=sys.stderr)
     return count
 
 

pipeline/restore.py

@@ -63,6 +63,11 @@ def resolve_baseline_prefix(cfg: Config) -> Optional[str]:
     return json.loads(pointer)["prefix"]
 
 
+# What a restore never downloads: the run's library media archives (2026-10-09: ~48 GB in all, and every start
+# downloaded them, twice: the OpenSearch and the Neo4j restores). pipeline/library_media.py fetches the one it
+# installs itself; nothing reads them from a restore dir.
+RESTORE_SKIP = ("library-media/",)
+
 def download(cfg: Config, prefix: str, dest: Path = RESTORE_DIR) -> None:
     # dest is overridable so two independent restores (e.g. the HF Space's
     # OpenSearch and Neo4j restores, run by separate supervisord programs —
@@ -72,8 +77,8 @@ def download(cfg: Config, prefix: str, dest: Path = RESTORE_DIR) -> None:
     if dest.exists():
         shutil.rmtree(dest)
     dest.mkdir(parents=True)
-    print(f"  downloading {prefix}/ -> {dest}")
-    download_prefix(cfg, prefix, dest)
+    print(f"  downloading {prefix}/ -> {dest} (without {', '.join(RESTORE_SKIP)})")
+    download_prefix(cfg, prefix, dest, skip=RESTORE_SKIP)
 
 
 def bulk_load(opensearch_url: str, index: str, data_path: Path, chunk_docs: int) -> int:

services/explorer/dist/build-info.json

@@ -1,7 +1,7 @@
 {
   "product": "simready-central-slim-explorer",
-  "version": "slim-30948535",
-  "bundle_id": "main.955ad1da.js",
-  "built_at": "2026-10-09T16:59:49Z",
+  "version": "slim-8c93f7a5",
+  "bundle_id": "main.f3a49a85.js",
+  "built_at": "2026-10-09T20:04:00Z",
   "flags": {"public_mode": true, "email_auth": false, "create_ui": false, "hierarchy_editor": false, "create_search": false, "auth_ui": false, "credentials_ui": false, "index_management": false, "inference_backend": "hub", "vlm_asset_refs": false, "validation_image_max_px": 448, "validation_max_retries": 0, "validation_max_concurrent": 32, "validation_tail_concurrent": 8, "validation_eager_count": 50, "validation_batch_stream": true, "library_idle_prefetch": false, "validation_score_scale": "percent", "parse_gate_fields": true, "parse_fast_path": true, "dependency_views": true, "model_viewer": false, "library_viewer": true, "library_media": "runtime", "spell_correction": false, "downloads": "runtime", "submit_content": "runtime", "third_party_notices": "runtime"}
 }

services/explorer/dist/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/ds.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="SimReady Central — SimReady asset search"/><meta name="ui-version" content="slim-30948535"><script src="/config.js"></script><title>SimReady Central</title><script defer="defer" src="/static/js/main.955ad1da.js"></script><link href="/static/css/main.1d1f5271.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/ds.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="SimReady Central — SimReady asset search"/><meta name="ui-version" content="slim-8c93f7a5"><script src="/config.js"></script><title>SimReady Central</title><script defer="defer" src="/static/js/main.f3a49a85.js"></script><link href="/static/css/main.1d1f5271.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

services/explorer/dist/third-party-notices.txt

@@ -1,4 +1,4 @@
-Third-party notices — SimReady Central Explorer slim-30948535
+Third-party notices — SimReady Central Explorer slim-8c93f7a5
 
 The software in this web application's browser bundle includes the packages below, each under its own
 license. Each entry gives the package, its version, its license and the license / notice text it ships.