UI slim-ui-f0276569 (query normalization + 5k verdict memory) + USD add-1 loader (#78)

Dist = indexing #79 (slim-ui-license f0276569 = 8174ebc0 + caching a5e567dc). pipeline/library_media.py = indexing #78 @5ea312f (reads media-<stamp>.usd-add-<n>.tar: the 8 OEM props + 9 loop props with joints; archives in S3, read back). One restart.

pipeline/library_media.py

@@ -49,6 +49,16 @@ skipped or failed USD HD archive leaves tier 1; a skipped or failed USD archive
 either is fetched unless the Space's ``VIEWER_FORMAT`` (the Explorer's /config.js ``features.viewerFormat``, rendered
 by supervisord.conf) is ``usd`` or ``auto``; LIBRARY_MEDIA_USD=off / LIBRARY_MEDIA_USD_HD=off skip one or both. nginx
 serves a ``.usdc`` as its ``.br`` to a client that accepts brotli.
+
+Web USD additions (2026-10-15): the pair above is write-once, so blocks made later (new assets, or a better
+conversion of some) travel in supplementary pairs of the same stamp, ``media-<stamp>.usd-add-<n>.tar`` (same layout
+as the ``.usd.tar``: its own ``usd-manifest.json`` first) and ``media-<stamp>.usdhd-add-<n>.tar``, n = 1, 2, ...
+They are applied after the pair, in order of n, each with its own disk check, under the same switches. An add's
+block replaces whatever block that entry had: before the add writes anything, its entries' ``usd`` blocks leave the
+served manifest and their ``<id>/usd/`` folders are removed (no file of an older block stays beside the new one);
+then each block is served at tier 1 once its files landed, and at full once its ``.usdhd-add-<n>.tar`` files did.
+A failed add leaves its entries on the GLB and never stops a later add. Revert: delete the add's two objects and
+restart.
 """
 
 from __future__ import annotations
@@ -87,6 +97,9 @@ USD_OVERLAY = "usd-manifest.json"     # the USD archive's first member: the bloc
 USD_OVERLAY_MAX = 64 * 2**20
 USD_PROFILE = "simready-web-usd/1"
 USD_ASSET = "usd/asset.usdc"
+# Web USD additions (see the module doc): media-<stamp>.usd-add-<n>.tar + .usdhd-add-<n>.tar, n without a leading 0
+USD_ADD = r"\.usd-add-([1-9][0-9]{0,2})\.tar"
+USD_HD_ADD = ".usdhd-add-{n}.tar"
 VIEWER_FORMATS = ("glb", "usd", "auto")   # the Space's VIEWER_FORMAT; anything else is "glb" (supervisord.conf too)
 
 
@@ -296,10 +309,12 @@ def usd_overlay(doc, manifest: dict) -> tuple[dict[str, tuple[str, dict]], int]:
     return out, left
 
 
-def stream_usd(body, tree: Path, manifest: dict) -> tuple[dict[str, tuple[str, dict]], int, int, int, int]:
+def stream_usd(body, tree: Path, manifest: dict,
+               before=None) -> tuple[dict[str, tuple[str, dict]], int, int, int, int]:
     """Write the USD archive ``body`` into ``tree``: its FIRST member must be ``usd-manifest.json`` (else ValueError,
     nothing written), whose usable blocks (``usd_overlay``) name the only members written — each block's
-    ``usd_tier1_files``. (blocks, left out, written, bytes, skipped)."""
+    ``usd_tier1_files``. ``before(blocks)``, when given, runs once the blocks are read and before any member is
+    written (an add clears its entries there). (blocks, left out, written, bytes, skipped)."""
     blocks, left, total = None, 0, [0, 0, 0]
     with tarfile.open(fileobj=body, mode="r|") as tf:
         for m in tf:
@@ -308,6 +323,8 @@ def stream_usd(body, tree: Path, manifest: dict) -> tuple[dict[str, tuple[str, d
                     raise ValueError(f"the first member is {m.name!r}, not {USD_OVERLAY}")
                 with tf.extractfile(m) as src:
                     blocks, left = usd_overlay(json.loads(src.read()), manifest)
+                if before is not None:
+                    before(blocks)
                 allowed: dict[str, set[str]] = {}
                 for i, block in blocks.values():
                     allowed.setdefault(i, set()).update(usd_tier1_files(block))
@@ -460,33 +477,114 @@ def fetch_usd(cfg: Config, store, uri: str, tree: Path) -> int:
     return shown
 
 
-def fetch_usd_hd(cfg: Config, store, uri: str, tree: Path, blocks: dict[str, tuple[str, dict]]) -> set[str]:
-    """Stream the ``.usdhd.tar`` ``uri`` into ``tree`` (only ``blocks``' ``usd_hd_files``); the Space keys whose
-    full-level files are all there. Off (LIBRARY_MEDIA_USD_HD), absent or without room: none."""
+def fetch_usd_hd(cfg: Config, store, uri: str, tree: Path, blocks: dict[str, tuple[str, dict]],
+                 what: str = "USD HD") -> set[str]:
+    """Stream the ``.usdhd.tar`` ``uri`` (or an add's ``.usdhd-add-<n>.tar``; ``what`` names it in the log) into
+    ``tree`` (only ``blocks``' ``usd_hd_files``); the Space keys whose full-level files are all there. Off
+    (LIBRARY_MEDIA_USD_HD), absent or without room: none."""
     if cfg.get("LIBRARY_MEDIA_USD_HD", "").strip().lower() in DISABLED:
-        print("==> library media USD HD: off (LIBRARY_MEDIA_USD_HD) — USD tier 1 only")
+        print(f"==> library media {what}: off (LIBRARY_MEDIA_USD_HD) — USD tier 1 only")
         return set()
     size = listed_size(cfg, store, uri)
     if size is None:
-        print(f"==> library media USD HD: none at {uri} — USD tier 1 only")
+        print(f"==> library media {what}: none at {uri} — USD tier 1 only")
         return set()
-    if not room_for(tree, size, "USD HD"):
+    if not room_for(tree, size, what):
         return set()
     allowed: dict[str, set[str]] = {}
     for i, block in blocks.values():
         allowed.setdefault(i, set()).update(usd_hd_files(block))
     t0 = time.time()
-    got = stream_with_retry(cfg, store, uri, "USD HD", lambda body: stream_hd(body, tree, allowed))
+    got = stream_with_retry(cfg, store, uri, what, lambda body: stream_hd(body, tree, allowed))
     if got is None:
         return set()
     written, nbytes, skipped = got
     keys = {k for k, (i, b) in blocks.items() if landed(tree, i, usd_hd_files(b))}
-    print(f"==> library media USD HD: {len(keys)} of {len(blocks)} blocks with their full level ({written} files, "
+    print(f"==> library media {what}: {len(keys)} of {len(blocks)} blocks with their full level ({written} files, "
           f"{nbytes / 1e6:.0f} MB) from {uri} in {time.time() - t0:.0f} s"
           + (f"; {skipped} members not listed skipped" if skipped else ""))
     return keys
 
 
+def served_usd(tree: Path) -> dict[str, dict]:
+    """{Space key: usd block} of the served manifest's entries that carry one."""
+    assets = json.loads((tree / "manifest.json").read_text()).get("assets") or {}
+    return {k: e["usd"] for k, e in assets.items() if isinstance(e, dict) and isinstance(e.get("usd"), dict)}
+
+
+def usd_adds(cfg: Config, store, uri: str) -> list[tuple[int, str, int]]:
+    """The supplementary web USD archives of ``uri``'s stamp (media-<stamp>.usd-add-<n>.tar): [(n, uri, size)] by n."""
+    folder, name = uri.rsplit("/", 1)
+    pattern = re.compile(re.escape(name[: -len(".tar")]) + USD_ADD)
+    found = [(int(m.group(1)), f"{folder}/{rel}", size) for rel, size in store.list_objects(cfg, folder)
+             if (m := pattern.fullmatch(rel))]
+    return sorted(found)
+
+
+def fetch_usd_add(cfg: Config, store, n: int, uri: str, size: int, tree: Path) -> int:
+    """Apply one add (see the module doc): clear its entries' blocks and ``<id>/usd/`` before it writes, stream its
+    tier-1 files and serve the blocks that landed, then its ``.usdhd-add-<n>.tar`` and give back their full level.
+    Returns the number of the add's blocks served."""
+    what = f"USD add {n}"
+    if not room_for(tree, size, what):
+        return 0
+    manifest = json.loads((tree / "manifest.json").read_text())
+    cleared: set[str] = set()
+    replaced: list[int] = [0]
+
+    def clear(blocks: dict[str, tuple[str, dict]]) -> None:
+        if cleared:                                   # a restarted stream: cleared already, keep what landed since
+            return
+        before = served_usd(tree)
+        replaced[0] = sum(1 for k in blocks if k in before)
+        publish_usd(tree, {k: b for k, b in before.items() if k not in blocks})
+        for i, _ in blocks.values():
+            shutil.rmtree(tree / i / "usd", ignore_errors=True)
+            cleared.add(i)
+
+    t0 = time.time()
+    got = stream_with_retry(cfg, store, uri, what, lambda body: stream_usd(body, tree, manifest, before=clear))
+    del manifest
+    if got is None:
+        return 0
+    blocks, left, written, nbytes, skipped = got
+    tier1 = {k: usd_without_hd(b) for k, (i, b) in blocks.items() if landed(tree, i, usd_tier1_files(b))}
+    rest = {k: b for k, b in served_usd(tree).items() if k not in blocks}
+    publish_usd(tree, {**rest, **tier1})
+    print(f"==> library media {what}: {len(tier1)} of {len(blocks)} blocks served at tier 1 ({replaced[0]} replacing "
+          f"an earlier block; {written} files, {nbytes / 1e6:.0f} MB) from {uri} in {time.time() - t0:.0f} s"
+          + (f"; {left} blocks not in this manifest (or malformed) left out" if left else "")
+          + (f"; {skipped} members not listed skipped" if skipped else ""))
+    full = {k: (i, b) for k, (i, b) in blocks.items() if k in tier1 and usd_hd_files(b)}
+    if not full:
+        return len(tier1)
+    hd_uri = uri[: -len(f".usd-add-{n}.tar")] + USD_HD_ADD.format(n=n)
+    try:
+        landed_hd = fetch_usd_hd(cfg, store, hd_uri, tree, full, what=f"USD HD add {n}")
+    except Exception as exc:  # noqa: BLE001 — the add's tier 1 is served already; its full level is a bonus
+        print(f"==> library media USD HD add {n}: stopped ({type(exc).__name__}: {str(exc)[:300]}) — "
+              f"its tier 1 stays", file=sys.stderr)
+        return len(tier1)
+    if landed_hd:
+        publish_usd(tree, {**served_usd(tree), **{k: blocks[k][1] for k in landed_hd}})
+    return len(tier1)
+
+
+def fetch_usd_adds(cfg: Config, store, uri: str, tree: Path) -> int:
+    """Every add of ``uri``'s stamp, in order of n, under the switches ``fetch_usd`` reads (it already said when they
+    are off). A failed add leaves its entries on the GLB; the next one is applied anyway. Blocks served by the adds."""
+    if viewer_format(cfg) == "glb" or cfg.get("LIBRARY_MEDIA_USD", "").strip().lower() in DISABLED:
+        return 0
+    total = 0
+    for n, add_uri, size in usd_adds(cfg, store, uri):
+        try:
+            total += fetch_usd_add(cfg, store, n, add_uri, size, tree)
+        except Exception as exc:  # noqa: BLE001 — one add never stops the next, nor the GLB
+            print(f"==> library media USD add {n}: stopped ({type(exc).__name__}: {str(exc)[:300]}) — its entries "
+                  f"stay on the GLB", file=sys.stderr)
+    return total
+
+
 def fetch(env: dict[str, str], root: Path = MEDIA_ROOT, store=None) -> int:
     """Fetch this corpus's media into ``root/INDEX_KEY``; returns the number of assets installed (0: none).
     ``store``: artifact_store's read_text / list_subprefixes / list_objects / download_file / bucket_base."""
@@ -536,6 +634,10 @@ def fetch(env: dict[str, str], root: Path = MEDIA_ROOT, store=None) -> int:
     except Exception as exc:  # noqa: BLE001 — the GLB tiers are served already; web USD is a bonus
         print(f"==> library media USD: stopped ({type(exc).__name__}: {str(exc)[:300]}) — the GLB stays",
               file=sys.stderr)
+    try:
+        fetch_usd_adds(cfg, store, uri, root / INDEX_KEY)
+    except Exception as exc:  # noqa: BLE001 — as above: the adds are a bonus on top of whatever is served
+        print(f"==> library media USD adds: stopped ({type(exc).__name__}: {str(exc)[:300]})", file=sys.stderr)
     return count
 
 

services/explorer/dist/build-info.json

@@ -1,7 +1,7 @@
 {
   "product": "simready-central-slim-explorer",
-  "version": "slim-ui-8174ebc0",
-  "bundle_id": "main.ca1c1a68.js",
-  "built_at": "2026-10-15T14:40:01Z",
+  "version": "slim-ui-f0276569",
+  "bundle_id": "main.ca52152d.js",
+  "built_at": "2026-10-15T15:28:11Z",
   "flags": {"public_mode": true, "email_auth": false, "create_ui": false, "hierarchy_editor": false, "create_search": false, "auth_ui": false, "credentials_ui": false, "index_management": false, "inference_backend": "hub", "vlm_asset_refs": false, "validation_image_max_px": 448, "validation_max_retries": 0, "validation_max_concurrent": 32, "validation_tail_concurrent": 8, "validation_eager_count": 50, "validation_batch_stream": true, "library_idle_prefetch": false, "validation_score_scale": "percent", "parse_gate_fields": true, "parse_fast_path": true, "featured_landing": true, "dependency_views": true, "model_viewer": false, "library_viewer": true, "library_media": "runtime", "spell_correction": false, "downloads": "runtime", "submit_content": "runtime", "third_party_notices": "runtime"}
 }

services/explorer/dist/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/ds.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="SimReady Central — SimReady asset search"/><meta name="ui-version" content="slim-ui-8174ebc0"><script src="/config.js"></script><title>SimReady Central</title><script defer="defer" src="/static/js/main.ca1c1a68.js"></script><link href="/static/css/main.01671ca7.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/ds.png"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="SimReady Central — SimReady asset search"/><meta name="ui-version" content="slim-ui-f0276569"><script src="/config.js"></script><title>SimReady Central</title><script defer="defer" src="/static/js/main.ca52152d.js"></script><link href="/static/css/main.01671ca7.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

services/explorer/dist/static/js/main.ca52152d.js.LICENSE.txt

@@ -0,0 +1,39 @@
+/**
+ * @license React
+ * react-dom.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * @license React
+ * react-jsx-runtime.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * @license React
+ * react.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * @license React
+ * scheduler.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */

services/explorer/dist/third-party-notices.txt

@@ -1,4 +1,4 @@
-Third-party notices — SimReady Central Explorer slim-ui-8174ebc0
+Third-party notices — SimReady Central Explorer slim-ui-f0276569
 
 The software in this web application's browser bundle includes the packages below, each under its own
 license. Each entry gives the package, its version, its license and the license / notice text it ships.